xref: /freebsd/contrib/llvm-project/lld/MachO/Arch/ARM64.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
1 //===- ARM64.cpp ----------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Arch/ARM64Common.h"
10 #include "InputFiles.h"
11 #include "Symbols.h"
12 #include "SyntheticSections.h"
13 #include "Target.h"
14 
15 #include "lld/Common/ErrorHandler.h"
16 #include "mach-o/compact_unwind_encoding.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/BinaryFormat/MachO.h"
20 #include "llvm/Support/Endian.h"
21 #include "llvm/Support/LEB128.h"
22 #include "llvm/Support/MathExtras.h"
23 
24 using namespace llvm;
25 using namespace llvm::MachO;
26 using namespace llvm::support::endian;
27 using namespace lld;
28 using namespace lld::macho;
29 
30 namespace {
31 
32 struct ARM64 : ARM64Common {
33   ARM64();
34   void writeStub(uint8_t *buf, const Symbol &, uint64_t) const override;
35   void writeStubHelperHeader(uint8_t *buf) const override;
36   void writeStubHelperEntry(uint8_t *buf, const Symbol &,
37                             uint64_t entryAddr) const override;
38 
39   void writeObjCMsgSendStub(uint8_t *buf, Symbol *sym, uint64_t stubsAddr,
40                             uint64_t &stubOffset, uint64_t selrefVA,
41                             Symbol *objcMsgSend) const override;
42   void populateThunk(InputSection *thunk, Symbol *funcSym) override;
43   void applyOptimizationHints(uint8_t *, const ObjFile &) const override;
44 };
45 
46 } // namespace
47 
48 // Random notes on reloc types:
49 // ADDEND always pairs with BRANCH26, PAGE21, or PAGEOFF12
50 // POINTER_TO_GOT: ld64 supports a 4-byte pc-relative form as well as an 8-byte
51 // absolute version of this relocation. The semantics of the absolute relocation
52 // are weird -- it results in the value of the GOT slot being written, instead
53 // of the address. Let's not support it unless we find a real-world use case.
54 static constexpr std::array<RelocAttrs, 11> relocAttrsArray{{
55 #define B(x) RelocAttrBits::x
56     {"UNSIGNED",
57      B(UNSIGNED) | B(ABSOLUTE) | B(EXTERN) | B(LOCAL) | B(BYTE4) | B(BYTE8)},
58     {"SUBTRACTOR", B(SUBTRAHEND) | B(EXTERN) | B(BYTE4) | B(BYTE8)},
59     {"BRANCH26", B(PCREL) | B(EXTERN) | B(BRANCH) | B(BYTE4)},
60     {"PAGE21", B(PCREL) | B(EXTERN) | B(BYTE4)},
61     {"PAGEOFF12", B(ABSOLUTE) | B(EXTERN) | B(BYTE4)},
62     {"GOT_LOAD_PAGE21", B(PCREL) | B(EXTERN) | B(GOT) | B(BYTE4)},
63     {"GOT_LOAD_PAGEOFF12",
64      B(ABSOLUTE) | B(EXTERN) | B(GOT) | B(LOAD) | B(BYTE4)},
65     {"POINTER_TO_GOT", B(PCREL) | B(EXTERN) | B(GOT) | B(POINTER) | B(BYTE4)},
66     {"TLVP_LOAD_PAGE21", B(PCREL) | B(EXTERN) | B(TLV) | B(BYTE4)},
67     {"TLVP_LOAD_PAGEOFF12",
68      B(ABSOLUTE) | B(EXTERN) | B(TLV) | B(LOAD) | B(BYTE4)},
69     {"ADDEND", B(ADDEND)},
70 #undef B
71 }};
72 
73 static constexpr uint32_t stubCode[] = {
74     0x90000010, // 00: adrp  x16, __la_symbol_ptr@page
75     0xf9400210, // 04: ldr   x16, [x16, __la_symbol_ptr@pageoff]
76     0xd61f0200, // 08: br    x16
77 };
78 
writeStub(uint8_t * buf8,const Symbol & sym,uint64_t pointerVA) const79 void ARM64::writeStub(uint8_t *buf8, const Symbol &sym,
80                       uint64_t pointerVA) const {
81   ::writeStub(buf8, stubCode, sym, pointerVA);
82 }
83 
84 static constexpr uint32_t stubHelperHeaderCode[] = {
85     0x90000011, // 00: adrp  x17, _dyld_private@page
86     0x91000231, // 04: add   x17, x17, _dyld_private@pageoff
87     0xa9bf47f0, // 08: stp   x16/x17, [sp, #-16]!
88     0x90000010, // 0c: adrp  x16, dyld_stub_binder@page
89     0xf9400210, // 10: ldr   x16, [x16, dyld_stub_binder@pageoff]
90     0xd61f0200, // 14: br    x16
91 };
92 
writeStubHelperHeader(uint8_t * buf8) const93 void ARM64::writeStubHelperHeader(uint8_t *buf8) const {
94   ::writeStubHelperHeader<LP64>(buf8, stubHelperHeaderCode);
95 }
96 
97 static constexpr uint32_t stubHelperEntryCode[] = {
98     0x18000050, // 00: ldr  w16, l0
99     0x14000000, // 04: b    stubHelperHeader
100     0x00000000, // 08: l0: .long 0
101 };
102 
writeStubHelperEntry(uint8_t * buf8,const Symbol & sym,uint64_t entryVA) const103 void ARM64::writeStubHelperEntry(uint8_t *buf8, const Symbol &sym,
104                                  uint64_t entryVA) const {
105   ::writeStubHelperEntry(buf8, stubHelperEntryCode, sym, entryVA);
106 }
107 
108 static constexpr uint32_t objcStubsFastCode[] = {
109     0x90000001, // adrp  x1, __objc_selrefs@page
110     0xf9400021, // ldr   x1, [x1, @selector("foo")@pageoff]
111     0x90000010, // adrp  x16, _got@page
112     0xf9400210, // ldr   x16, [x16, _objc_msgSend@pageoff]
113     0xd61f0200, // br    x16
114     0xd4200020, // brk   #0x1
115     0xd4200020, // brk   #0x1
116     0xd4200020, // brk   #0x1
117 };
118 
119 static constexpr uint32_t objcStubsSmallCode[] = {
120     0x90000001, // adrp  x1, __objc_selrefs@page
121     0xf9400021, // ldr   x1, [x1, @selector("foo")@pageoff]
122     0x14000000, // b     _objc_msgSend
123 };
124 
writeObjCMsgSendStub(uint8_t * buf,Symbol * sym,uint64_t stubsAddr,uint64_t & stubOffset,uint64_t selrefVA,Symbol * objcMsgSend) const125 void ARM64::writeObjCMsgSendStub(uint8_t *buf, Symbol *sym, uint64_t stubsAddr,
126                                  uint64_t &stubOffset, uint64_t selrefVA,
127                                  Symbol *objcMsgSend) const {
128   uint64_t objcMsgSendAddr;
129   uint64_t objcStubSize;
130   uint64_t objcMsgSendIndex;
131 
132   if (config->objcStubsMode == ObjCStubsMode::fast) {
133     objcStubSize = target->objcStubsFastSize;
134     objcMsgSendAddr = in.got->addr;
135     objcMsgSendIndex = objcMsgSend->gotIndex;
136     ::writeObjCMsgSendFastStub<LP64>(buf, objcStubsFastCode, sym, stubsAddr,
137                                      stubOffset, selrefVA, objcMsgSendAddr,
138                                      objcMsgSendIndex);
139   } else {
140     assert(config->objcStubsMode == ObjCStubsMode::small);
141     objcStubSize = target->objcStubsSmallSize;
142     if (auto *d = dyn_cast<Defined>(objcMsgSend)) {
143       objcMsgSendAddr = d->getVA();
144       objcMsgSendIndex = 0;
145     } else {
146       objcMsgSendAddr = in.stubs->addr;
147       objcMsgSendIndex = objcMsgSend->stubsIndex;
148     }
149     ::writeObjCMsgSendSmallStub<LP64>(buf, objcStubsSmallCode, sym, stubsAddr,
150                                       stubOffset, selrefVA, objcMsgSendAddr,
151                                       objcMsgSendIndex);
152   }
153   stubOffset += objcStubSize;
154 }
155 
156 // A thunk is the relaxed variation of stubCode. We don't need the
157 // extra indirection through a lazy pointer because the target address
158 // is known at link time.
159 static constexpr uint32_t thunkCode[] = {
160     0x90000010, // 00: adrp  x16, <thunk.ptr>@page
161     0x91000210, // 04: add   x16, [x16,<thunk.ptr>@pageoff]
162     0xd61f0200, // 08: br    x16
163 };
164 
populateThunk(InputSection * thunk,Symbol * funcSym)165 void ARM64::populateThunk(InputSection *thunk, Symbol *funcSym) {
166   thunk->align = 4;
167   thunk->data = {reinterpret_cast<const uint8_t *>(thunkCode),
168                  sizeof(thunkCode)};
169   thunk->relocs.emplace_back(/*type=*/ARM64_RELOC_PAGEOFF12,
170                              /*pcrel=*/false, /*length=*/2,
171                              /*offset=*/4, /*addend=*/0,
172                              /*referent=*/funcSym);
173   thunk->relocs.emplace_back(/*type=*/ARM64_RELOC_PAGE21,
174                              /*pcrel=*/true, /*length=*/2,
175                              /*offset=*/0, /*addend=*/0,
176                              /*referent=*/funcSym);
177 }
178 
ARM64()179 ARM64::ARM64() : ARM64Common(LP64()) {
180   cpuType = CPU_TYPE_ARM64;
181   cpuSubtype = CPU_SUBTYPE_ARM64_ALL;
182 
183   stubSize = sizeof(stubCode);
184   thunkSize = sizeof(thunkCode);
185 
186   objcStubsFastSize = sizeof(objcStubsFastCode);
187   objcStubsFastAlignment = 32;
188   objcStubsSmallSize = sizeof(objcStubsSmallCode);
189   objcStubsSmallAlignment = 4;
190 
191   // Branch immediate is two's complement 26 bits, which is implicitly
192   // multiplied by 4 (since all functions are 4-aligned: The branch range
193   // is -4*(2**(26-1))..4*(2**(26-1) - 1).
194   backwardBranchRange = 128 * 1024 * 1024;
195   forwardBranchRange = backwardBranchRange - 4;
196 
197   modeDwarfEncoding = UNWIND_ARM64_MODE_DWARF;
198   subtractorRelocType = ARM64_RELOC_SUBTRACTOR;
199   unsignedRelocType = ARM64_RELOC_UNSIGNED;
200 
201   stubHelperHeaderSize = sizeof(stubHelperHeaderCode);
202   stubHelperEntrySize = sizeof(stubHelperEntryCode);
203 
204   relocAttrs = {relocAttrsArray.data(), relocAttrsArray.size()};
205 }
206 
207 namespace {
208 struct Adrp {
209   uint32_t destRegister;
210   int64_t addend;
211 };
212 
213 struct Add {
214   uint8_t destRegister;
215   uint8_t srcRegister;
216   uint32_t addend;
217 };
218 
219 enum ExtendType { ZeroExtend = 1, Sign64 = 2, Sign32 = 3 };
220 
221 struct Ldr {
222   uint8_t destRegister;
223   uint8_t baseRegister;
224   uint8_t p2Size;
225   bool isFloat;
226   ExtendType extendType;
227   int64_t offset;
228 };
229 } // namespace
230 
parseAdrp(uint32_t insn,Adrp & adrp)231 static bool parseAdrp(uint32_t insn, Adrp &adrp) {
232   if ((insn & 0x9f000000) != 0x90000000)
233     return false;
234   adrp.destRegister = insn & 0x1f;
235   uint64_t immHi = (insn >> 5) & 0x7ffff;
236   uint64_t immLo = (insn >> 29) & 0x3;
237   adrp.addend = SignExtend64<21>(immLo | (immHi << 2)) * 4096;
238   return true;
239 }
240 
parseAdd(uint32_t insn,Add & add)241 static bool parseAdd(uint32_t insn, Add &add) {
242   if ((insn & 0xffc00000) != 0x91000000)
243     return false;
244   add.destRegister = insn & 0x1f;
245   add.srcRegister = (insn >> 5) & 0x1f;
246   add.addend = (insn >> 10) & 0xfff;
247   return true;
248 }
249 
parseLdr(uint32_t insn,Ldr & ldr)250 static bool parseLdr(uint32_t insn, Ldr &ldr) {
251   ldr.destRegister = insn & 0x1f;
252   ldr.baseRegister = (insn >> 5) & 0x1f;
253   uint8_t size = insn >> 30;
254   uint8_t opc = (insn >> 22) & 3;
255 
256   if ((insn & 0x3fc00000) == 0x39400000) {
257     // LDR (immediate), LDRB (immediate), LDRH (immediate)
258     ldr.p2Size = size;
259     ldr.extendType = ZeroExtend;
260     ldr.isFloat = false;
261   } else if ((insn & 0x3f800000) == 0x39800000) {
262     // LDRSB (immediate), LDRSH (immediate), LDRSW (immediate)
263     ldr.p2Size = size;
264     ldr.extendType = static_cast<ExtendType>(opc);
265     ldr.isFloat = false;
266   } else if ((insn & 0x3f400000) == 0x3d400000) {
267     // LDR (immediate, SIMD&FP)
268     ldr.extendType = ZeroExtend;
269     ldr.isFloat = true;
270     if (opc == 1)
271       ldr.p2Size = size;
272     else if (size == 0 && opc == 3)
273       ldr.p2Size = 4;
274     else
275       return false;
276   } else {
277     return false;
278   }
279   ldr.offset = ((insn >> 10) & 0xfff) << ldr.p2Size;
280   return true;
281 }
282 
isValidAdrOffset(int32_t delta)283 static bool isValidAdrOffset(int32_t delta) { return isInt<21>(delta); }
284 
writeAdr(void * loc,uint32_t dest,int32_t delta)285 static void writeAdr(void *loc, uint32_t dest, int32_t delta) {
286   assert(isValidAdrOffset(delta));
287   uint32_t opcode = 0x10000000;
288   uint32_t immHi = (delta & 0x001ffffc) << 3;
289   uint32_t immLo = (delta & 0x00000003) << 29;
290   write32le(loc, opcode | immHi | immLo | dest);
291 }
292 
writeNop(void * loc)293 static void writeNop(void *loc) { write32le(loc, 0xd503201f); }
294 
isLiteralLdrEligible(const Ldr & ldr)295 static bool isLiteralLdrEligible(const Ldr &ldr) {
296   return ldr.p2Size > 1 && isShiftedInt<19, 2>(ldr.offset);
297 }
298 
writeLiteralLdr(void * loc,const Ldr & ldr)299 static void writeLiteralLdr(void *loc, const Ldr &ldr) {
300   assert(isLiteralLdrEligible(ldr));
301   uint32_t imm19 = (ldr.offset / 4 & maskTrailingOnes<uint32_t>(19)) << 5;
302   uint32_t opcode;
303   switch (ldr.p2Size) {
304   case 2:
305     if (ldr.isFloat)
306       opcode = 0x1c000000;
307     else
308       opcode = ldr.extendType == Sign64 ? 0x98000000 : 0x18000000;
309     break;
310   case 3:
311     opcode = ldr.isFloat ? 0x5c000000 : 0x58000000;
312     break;
313   case 4:
314     opcode = 0x9c000000;
315     break;
316   default:
317     llvm_unreachable("Invalid literal ldr size");
318   }
319   write32le(loc, opcode | imm19 | ldr.destRegister);
320 }
321 
isImmediateLdrEligible(const Ldr & ldr)322 static bool isImmediateLdrEligible(const Ldr &ldr) {
323   // Note: We deviate from ld64's behavior, which converts to immediate loads
324   // only if ldr.offset < 4096, even though the offset is divided by the load's
325   // size in the 12-bit immediate operand. Only the unsigned offset variant is
326   // supported.
327 
328   uint32_t size = 1 << ldr.p2Size;
329   return ldr.offset >= 0 && (ldr.offset % size) == 0 &&
330          isUInt<12>(ldr.offset >> ldr.p2Size);
331 }
332 
writeImmediateLdr(void * loc,const Ldr & ldr)333 static void writeImmediateLdr(void *loc, const Ldr &ldr) {
334   assert(isImmediateLdrEligible(ldr));
335   uint32_t opcode = 0x39000000;
336   if (ldr.isFloat) {
337     opcode |= 0x04000000;
338     assert(ldr.extendType == ZeroExtend);
339   }
340   opcode |= ldr.destRegister;
341   opcode |= ldr.baseRegister << 5;
342   uint8_t size, opc;
343   if (ldr.p2Size == 4) {
344     size = 0;
345     opc = 3;
346   } else {
347     opc = ldr.extendType;
348     size = ldr.p2Size;
349   }
350   uint32_t immBits = ldr.offset >> ldr.p2Size;
351   write32le(loc, opcode | (immBits << 10) | (opc << 22) | (size << 30));
352 }
353 
354 // Transforms a pair of adrp+add instructions into an adr instruction if the
355 // target is within the +/- 1 MiB range allowed by the adr's 21 bit signed
356 // immediate offset.
357 //
358 //   adrp xN, _foo@PAGE
359 //   add  xM, xN, _foo@PAGEOFF
360 // ->
361 //   adr  xM, _foo
362 //   nop
applyAdrpAdd(uint8_t * buf,const ConcatInputSection * isec,uint64_t offset1,uint64_t offset2)363 static void applyAdrpAdd(uint8_t *buf, const ConcatInputSection *isec,
364                          uint64_t offset1, uint64_t offset2) {
365   uint32_t ins1 = read32le(buf + offset1);
366   uint32_t ins2 = read32le(buf + offset2);
367   Adrp adrp;
368   Add add;
369   if (!parseAdrp(ins1, adrp) || !parseAdd(ins2, add))
370     return;
371   if (adrp.destRegister != add.srcRegister)
372     return;
373 
374   uint64_t addr1 = isec->getVA() + offset1;
375   uint64_t referent = pageBits(addr1) + adrp.addend + add.addend;
376   int64_t delta = referent - addr1;
377   if (!isValidAdrOffset(delta))
378     return;
379 
380   writeAdr(buf + offset1, add.destRegister, delta);
381   writeNop(buf + offset2);
382 }
383 
384 // Transforms two adrp instructions into a single adrp if their referent
385 // addresses are located on the same 4096 byte page.
386 //
387 //   adrp xN, _foo@PAGE
388 //   adrp xN, _bar@PAGE
389 // ->
390 //   adrp xN, _foo@PAGE
391 //   nop
applyAdrpAdrp(uint8_t * buf,const ConcatInputSection * isec,uint64_t offset1,uint64_t offset2)392 static void applyAdrpAdrp(uint8_t *buf, const ConcatInputSection *isec,
393                           uint64_t offset1, uint64_t offset2) {
394   uint32_t ins1 = read32le(buf + offset1);
395   uint32_t ins2 = read32le(buf + offset2);
396   Adrp adrp1, adrp2;
397   if (!parseAdrp(ins1, adrp1) || !parseAdrp(ins2, adrp2))
398     return;
399   if (adrp1.destRegister != adrp2.destRegister)
400     return;
401 
402   uint64_t page1 = pageBits(offset1 + isec->getVA()) + adrp1.addend;
403   uint64_t page2 = pageBits(offset2 + isec->getVA()) + adrp2.addend;
404   if (page1 != page2)
405     return;
406 
407   writeNop(buf + offset2);
408 }
409 
410 // Transforms a pair of adrp+ldr (immediate) instructions into an ldr (literal)
411 // load from a PC-relative address if it is 4-byte aligned and within +/- 1 MiB,
412 // as ldr can encode a signed 19-bit offset that gets multiplied by 4.
413 //
414 //   adrp xN, _foo@PAGE
415 //   ldr  xM, [xN, _foo@PAGEOFF]
416 // ->
417 //   nop
418 //   ldr  xM, _foo
applyAdrpLdr(uint8_t * buf,const ConcatInputSection * isec,uint64_t offset1,uint64_t offset2)419 static void applyAdrpLdr(uint8_t *buf, const ConcatInputSection *isec,
420                          uint64_t offset1, uint64_t offset2) {
421   uint32_t ins1 = read32le(buf + offset1);
422   uint32_t ins2 = read32le(buf + offset2);
423   Adrp adrp;
424   Ldr ldr;
425   if (!parseAdrp(ins1, adrp) || !parseLdr(ins2, ldr))
426     return;
427   if (adrp.destRegister != ldr.baseRegister)
428     return;
429 
430   uint64_t addr1 = isec->getVA() + offset1;
431   uint64_t addr2 = isec->getVA() + offset2;
432   uint64_t referent = pageBits(addr1) + adrp.addend + ldr.offset;
433   ldr.offset = referent - addr2;
434   if (!isLiteralLdrEligible(ldr))
435     return;
436 
437   writeNop(buf + offset1);
438   writeLiteralLdr(buf + offset2, ldr);
439 }
440 
441 // GOT loads are emitted by the compiler as a pair of adrp and ldr instructions,
442 // but they may be changed to adrp+add by relaxGotLoad(). This hint performs
443 // the AdrpLdr or AdrpAdd transformation depending on whether it was relaxed.
applyAdrpLdrGot(uint8_t * buf,const ConcatInputSection * isec,uint64_t offset1,uint64_t offset2)444 static void applyAdrpLdrGot(uint8_t *buf, const ConcatInputSection *isec,
445                             uint64_t offset1, uint64_t offset2) {
446   uint32_t ins2 = read32le(buf + offset2);
447   Add add;
448   Ldr ldr;
449   if (parseAdd(ins2, add))
450     applyAdrpAdd(buf, isec, offset1, offset2);
451   else if (parseLdr(ins2, ldr))
452     applyAdrpLdr(buf, isec, offset1, offset2);
453 }
454 
455 // Optimizes an adrp+add+ldr sequence used for loading from a local symbol's
456 // address by loading directly if it's close enough, or to an adrp(p)+ldr
457 // sequence if it's not.
458 //
459 //   adrp x0, _foo@PAGE
460 //   add  x1, x0, _foo@PAGEOFF
461 //   ldr  x2, [x1, #off]
applyAdrpAddLdr(uint8_t * buf,const ConcatInputSection * isec,uint64_t offset1,uint64_t offset2,uint64_t offset3)462 static void applyAdrpAddLdr(uint8_t *buf, const ConcatInputSection *isec,
463                             uint64_t offset1, uint64_t offset2,
464                             uint64_t offset3) {
465   uint32_t ins1 = read32le(buf + offset1);
466   Adrp adrp;
467   if (!parseAdrp(ins1, adrp))
468     return;
469   uint32_t ins2 = read32le(buf + offset2);
470   Add add;
471   if (!parseAdd(ins2, add))
472     return;
473   uint32_t ins3 = read32le(buf + offset3);
474   Ldr ldr;
475   if (!parseLdr(ins3, ldr))
476     return;
477   if (adrp.destRegister != add.srcRegister)
478     return;
479   if (add.destRegister != ldr.baseRegister)
480     return;
481 
482   // Load from the target address directly.
483   //   nop
484   //   nop
485   //   ldr x2, [_foo + #off]
486   uint64_t addr1 = isec->getVA() + offset1;
487   uint64_t addr3 = isec->getVA() + offset3;
488   uint64_t referent = pageBits(addr1) + adrp.addend + add.addend;
489   Ldr literalLdr = ldr;
490   literalLdr.offset += referent - addr3;
491   if (isLiteralLdrEligible(literalLdr)) {
492     writeNop(buf + offset1);
493     writeNop(buf + offset2);
494     writeLiteralLdr(buf + offset3, literalLdr);
495     return;
496   }
497 
498   // Load the target address into a register and load from there indirectly.
499   //   adr x1, _foo
500   //   nop
501   //   ldr x2, [x1, #off]
502   int64_t adrOffset = referent - addr1;
503   if (isValidAdrOffset(adrOffset)) {
504     writeAdr(buf + offset1, ldr.baseRegister, adrOffset);
505     // Note: ld64 moves the offset into the adr instruction for AdrpAddLdr, but
506     // not for AdrpLdrGotLdr. Its effect is the same either way.
507     writeNop(buf + offset2);
508     return;
509   }
510 
511   // Move the target's page offset into the ldr's immediate offset.
512   //   adrp x0, _foo@PAGE
513   //   nop
514   //   ldr x2, [x0, _foo@PAGEOFF + #off]
515   Ldr immediateLdr = ldr;
516   immediateLdr.baseRegister = adrp.destRegister;
517   immediateLdr.offset += add.addend;
518   if (isImmediateLdrEligible(immediateLdr)) {
519     writeNop(buf + offset2);
520     writeImmediateLdr(buf + offset3, immediateLdr);
521     return;
522   }
523 }
524 
525 // Relaxes a GOT-indirect load.
526 // If the referenced symbol is external and its GOT entry is within +/- 1 MiB,
527 // the GOT entry can be loaded with a single literal ldr instruction.
528 // If the referenced symbol is local and thus has been relaxed to adrp+add+ldr,
529 // we perform the AdrpAddLdr transformation.
applyAdrpLdrGotLdr(uint8_t * buf,const ConcatInputSection * isec,uint64_t offset1,uint64_t offset2,uint64_t offset3)530 static void applyAdrpLdrGotLdr(uint8_t *buf, const ConcatInputSection *isec,
531                                uint64_t offset1, uint64_t offset2,
532                                uint64_t offset3) {
533   uint32_t ins2 = read32le(buf + offset2);
534   Add add;
535   Ldr ldr2;
536 
537   if (parseAdd(ins2, add)) {
538     applyAdrpAddLdr(buf, isec, offset1, offset2, offset3);
539   } else if (parseLdr(ins2, ldr2)) {
540     // adrp x1, _foo@GOTPAGE
541     // ldr  x2, [x1, _foo@GOTPAGEOFF]
542     // ldr  x3, [x2, #off]
543 
544     uint32_t ins1 = read32le(buf + offset1);
545     Adrp adrp;
546     if (!parseAdrp(ins1, adrp))
547       return;
548     uint32_t ins3 = read32le(buf + offset3);
549     Ldr ldr3;
550     if (!parseLdr(ins3, ldr3))
551       return;
552 
553     if (ldr2.baseRegister != adrp.destRegister)
554       return;
555     if (ldr3.baseRegister != ldr2.destRegister)
556       return;
557     // Loads from the GOT must be pointer sized.
558     if (ldr2.p2Size != 3 || ldr2.isFloat)
559       return;
560 
561     uint64_t addr1 = isec->getVA() + offset1;
562     uint64_t addr2 = isec->getVA() + offset2;
563     uint64_t referent = pageBits(addr1) + adrp.addend + ldr2.offset;
564     // Load the GOT entry's address directly.
565     //   nop
566     //   ldr x2, _foo@GOTPAGE + _foo@GOTPAGEOFF
567     //   ldr x3, [x2, #off]
568     Ldr literalLdr = ldr2;
569     literalLdr.offset = referent - addr2;
570     if (isLiteralLdrEligible(literalLdr)) {
571       writeNop(buf + offset1);
572       writeLiteralLdr(buf + offset2, literalLdr);
573     }
574   }
575 }
576 
readValue(const uint8_t * & ptr,const uint8_t * end)577 static uint64_t readValue(const uint8_t *&ptr, const uint8_t *end) {
578   unsigned int n = 0;
579   uint64_t value = decodeULEB128(ptr, &n, end);
580   ptr += n;
581   return value;
582 }
583 
584 template <typename Callback>
forEachHint(ArrayRef<uint8_t> data,Callback callback)585 static void forEachHint(ArrayRef<uint8_t> data, Callback callback) {
586   std::array<uint64_t, 3> args;
587 
588   for (const uint8_t *p = data.begin(), *end = data.end(); p < end;) {
589     uint64_t type = readValue(p, end);
590     if (type == 0)
591       break;
592 
593     uint64_t argCount = readValue(p, end);
594     // All known LOH types as of 2022-09 have 3 or fewer arguments; skip others.
595     if (argCount > 3) {
596       for (unsigned i = 0; i < argCount; ++i)
597         readValue(p, end);
598       continue;
599     }
600 
601     for (unsigned i = 0; i < argCount; ++i)
602       args[i] = readValue(p, end);
603     callback(type, ArrayRef<uint64_t>(args.data(), argCount));
604   }
605 }
606 
607 // On RISC architectures like arm64, materializing a memory address generally
608 // takes multiple instructions. If the referenced symbol is located close enough
609 // in memory, fewer instructions are needed.
610 //
611 // Linker optimization hints record where addresses are computed. After
612 // addresses have been assigned, if possible, we change them to a shorter
613 // sequence of instructions. The size of the binary is not modified; the
614 // eliminated instructions are replaced with NOPs. This still leads to faster
615 // code as the CPU can skip over NOPs quickly.
616 //
617 // LOHs are specified by the LC_LINKER_OPTIMIZATION_HINTS load command, which
618 // points to a sequence of ULEB128-encoded numbers. Each entry specifies a
619 // transformation kind, and 2 or 3 addresses where the instructions are located.
applyOptimizationHints(uint8_t * outBuf,const ObjFile & obj) const620 void ARM64::applyOptimizationHints(uint8_t *outBuf, const ObjFile &obj) const {
621   ArrayRef<uint8_t> data = obj.getOptimizationHints();
622   if (data.empty())
623     return;
624 
625   const ConcatInputSection *section = nullptr;
626   uint64_t sectionAddr = 0;
627   uint8_t *buf = nullptr;
628 
629   auto findSection = [&](uint64_t addr) {
630     if (section && addr >= sectionAddr &&
631         addr < sectionAddr + section->getSize())
632       return true;
633 
634     if (obj.sections.empty())
635       return false;
636     auto secIt = std::prev(llvm::upper_bound(
637         obj.sections, addr,
638         [](uint64_t off, const Section *sec) { return off < sec->addr; }));
639     const Section *sec = *secIt;
640 
641     if (sec->subsections.empty())
642       return false;
643     auto subsecIt = std::prev(llvm::upper_bound(
644         sec->subsections, addr - sec->addr,
645         [](uint64_t off, Subsection subsec) { return off < subsec.offset; }));
646     const Subsection &subsec = *subsecIt;
647     const ConcatInputSection *isec =
648         dyn_cast_or_null<ConcatInputSection>(subsec.isec);
649     if (!isec || isec->shouldOmitFromOutput())
650       return false;
651 
652     section = isec;
653     sectionAddr = subsec.offset + sec->addr;
654     buf = outBuf + section->outSecOff + section->parent->fileOff;
655     return true;
656   };
657 
658   auto isValidOffset = [&](uint64_t offset) {
659     if (offset < sectionAddr || offset >= sectionAddr + section->getSize()) {
660       error(toString(&obj) +
661             ": linker optimization hint spans multiple sections");
662       return false;
663     }
664     return true;
665   };
666 
667   bool hasAdrpAdrp = false;
668   forEachHint(data, [&](uint64_t kind, ArrayRef<uint64_t> args) {
669     if (kind == LOH_ARM64_ADRP_ADRP) {
670       hasAdrpAdrp = true;
671       return;
672     }
673 
674     if (!findSection(args[0]))
675       return;
676     switch (kind) {
677     case LOH_ARM64_ADRP_ADD:
678       if (isValidOffset(args[1]))
679         applyAdrpAdd(buf, section, args[0] - sectionAddr,
680                      args[1] - sectionAddr);
681       break;
682     case LOH_ARM64_ADRP_LDR:
683       if (isValidOffset(args[1]))
684         applyAdrpLdr(buf, section, args[0] - sectionAddr,
685                      args[1] - sectionAddr);
686       break;
687     case LOH_ARM64_ADRP_LDR_GOT:
688       if (isValidOffset(args[1]))
689         applyAdrpLdrGot(buf, section, args[0] - sectionAddr,
690                         args[1] - sectionAddr);
691       break;
692     case LOH_ARM64_ADRP_ADD_LDR:
693       if (isValidOffset(args[1]) && isValidOffset(args[2]))
694         applyAdrpAddLdr(buf, section, args[0] - sectionAddr,
695                         args[1] - sectionAddr, args[2] - sectionAddr);
696       break;
697     case LOH_ARM64_ADRP_LDR_GOT_LDR:
698       if (isValidOffset(args[1]) && isValidOffset(args[2]))
699         applyAdrpLdrGotLdr(buf, section, args[0] - sectionAddr,
700                            args[1] - sectionAddr, args[2] - sectionAddr);
701       break;
702     case LOH_ARM64_ADRP_ADD_STR:
703     case LOH_ARM64_ADRP_LDR_GOT_STR:
704       // TODO: Implement these
705       break;
706     }
707   });
708 
709   if (!hasAdrpAdrp)
710     return;
711 
712   // AdrpAdrp optimization hints are performed in a second pass because they
713   // might interfere with other transformations. For instance, consider the
714   // following input:
715   //
716   //   adrp x0, _foo@PAGE
717   //   add  x1, x0, _foo@PAGEOFF
718   //   adrp x0, _bar@PAGE
719   //   add  x2, x0, _bar@PAGEOFF
720   //
721   // If we perform the AdrpAdrp relaxation first, we get:
722   //
723   //   adrp x0, _foo@PAGE
724   //   add  x1, x0, _foo@PAGEOFF
725   //   nop
726   //   add x2, x0, _bar@PAGEOFF
727   //
728   // If we then apply AdrpAdd to the first two instructions, the add will have a
729   // garbage value in x0:
730   //
731   //   adr  x1, _foo
732   //   nop
733   //   nop
734   //   add  x2, x0, _bar@PAGEOFF
735   forEachHint(data, [&](uint64_t kind, ArrayRef<uint64_t> args) {
736     if (kind != LOH_ARM64_ADRP_ADRP)
737       return;
738     if (!findSection(args[0]))
739       return;
740     if (isValidOffset(args[1]))
741       applyAdrpAdrp(buf, section, args[0] - sectionAddr, args[1] - sectionAddr);
742   });
743 }
744 
createARM64TargetInfo()745 TargetInfo *macho::createARM64TargetInfo() {
746   static ARM64 t;
747   return &t;
748 }
749