1 //===---- aarch64.cpp - Generic JITLink aarch64 edge kinds, utilities -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Generic utilities for graphs representing aarch64 objects.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
14
15 #include "llvm/Support/BinaryStreamWriter.h"
16
17 #define DEBUG_TYPE "jitlink"
18
19 namespace llvm {
20 namespace jitlink {
21 namespace aarch64 {
22
23 const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00,
24 0x00, 0x00, 0x00, 0x00};
25
26 const char PointerJumpStubContent[12] = {
27 0x10, 0x00, 0x00, (char)0x90u, // ADRP x16, <imm>@page21
28 0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12]
29 0x00, 0x02, 0x1f, (char)0xd6u // BR x16
30 };
31
32 const char ReentryTrampolineContent[8] = {
33 (char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8]
34 0x00, 0x00, 0x00, (char)0x94 // BL
35 };
36
getEdgeKindName(Edge::Kind R)37 const char *getEdgeKindName(Edge::Kind R) {
38 switch (R) {
39 case Pointer64:
40 return "Pointer64";
41 case Pointer64Authenticated:
42 return "Pointer64Authenticated";
43 case Pointer32:
44 return "Pointer32";
45 case Delta64:
46 return "Delta64";
47 case Delta32:
48 return "Delta32";
49 case NegDelta64:
50 return "NegDelta64";
51 case NegDelta32:
52 return "NegDelta32";
53 case Branch26PCRel:
54 return "Branch26PCRel";
55 case MoveWide16:
56 return "MoveWide16";
57 case LDRLiteral19:
58 return "LDRLiteral19";
59 case TestAndBranch14PCRel:
60 return "TestAndBranch14PCRel";
61 case CondBranch19PCRel:
62 return "CondBranch19PCRel";
63 case ADRLiteral21:
64 return "ADRLiteral21";
65 case Page21:
66 return "Page21";
67 case PageOffset12:
68 return "PageOffset12";
69 case GotPageOffset15:
70 return "GotPageOffset15";
71 case RequestGOTAndTransformToPage21:
72 return "RequestGOTAndTransformToPage21";
73 case RequestGOTAndTransformToPageOffset12:
74 return "RequestGOTAndTransformToPageOffset12";
75 case RequestGOTAndTransformToPageOffset15:
76 return "RequestGOTAndTransformToPageOffset15";
77 case RequestGOTAndTransformToDelta32:
78 return "RequestGOTAndTransformToDelta32";
79 case RequestTLVPAndTransformToPage21:
80 return "RequestTLVPAndTransformToPage21";
81 case RequestTLVPAndTransformToPageOffset12:
82 return "RequestTLVPAndTransformToPageOffset12";
83 case RequestTLSDescEntryAndTransformToPage21:
84 return "RequestTLSDescEntryAndTransformToPage21";
85 case RequestTLSDescEntryAndTransformToPageOffset12:
86 return "RequestTLSDescEntryAndTransformToPageOffset12";
87 default:
88 return getGenericEdgeKindName(R);
89 }
90 }
91
92 // Write a 64-bit GPR -> GPR move.
93 template <typename AppendFtor>
writeMovRegRegSeq(AppendFtor & Append,uint64_t DstReg,uint64_t SrcReg)94 static Error writeMovRegRegSeq(AppendFtor &Append, uint64_t DstReg,
95 uint64_t SrcReg) {
96 assert(DstReg < 32 && "Dst reg out of range");
97 assert(SrcReg < 32 && "Src reg out of range");
98
99 if (DstReg == SrcReg)
100 return Error::success();
101
102 constexpr uint32_t MOVGPR64Template = 0xaa0003e0;
103 constexpr uint32_t DstRegIndex = 0;
104 constexpr uint32_t SrcRegIndex = 16;
105 uint32_t Instr = MOVGPR64Template;
106 Instr |= DstReg << DstRegIndex;
107 Instr |= SrcReg << SrcRegIndex;
108 return Append(Instr);
109 }
110
111 // Generate a sequence of imm writes to assign the given value.
112 template <typename AppendFtor>
writeMovRegImm64Seq(AppendFtor & Append,uint64_t Reg,uint64_t Imm)113 static Error writeMovRegImm64Seq(AppendFtor &Append, uint64_t Reg,
114 uint64_t Imm) {
115 assert(Reg < 32 && "Invalid register number");
116
117 constexpr uint32_t MovRegImm64Template = 0xd2800000;
118 constexpr unsigned PreserveBitIndex = 29;
119 constexpr unsigned ShiftBitsIndex = 21;
120 constexpr unsigned ImmBitsIndex = 5;
121
122 bool PreserveRegValue = false;
123 for (unsigned I = 0; I != 4; ++I) {
124 uint32_t ImmBits = Imm & 0xffff;
125 Imm >>= 16;
126
127 // Skip any all-zero immediates after the first one.
128 if (PreserveRegValue && !ImmBits)
129 continue;
130
131 uint32_t Instr = MovRegImm64Template;
132 Instr |= PreserveRegValue << PreserveBitIndex;
133 Instr |= (I << ShiftBitsIndex);
134 Instr |= ImmBits << ImmBitsIndex;
135 Instr |= Reg;
136 if (auto Err = Append(Instr))
137 return Err;
138 PreserveRegValue = true;
139 }
140
141 return Error::success();
142 }
143
144 template <typename AppendFtor>
145 static Error
writePACSignSeq(AppendFtor & Append,unsigned DstReg,orc::ExecutorAddr RawAddr,unsigned RawAddrReg,unsigned DiscriminatorReg,unsigned Key,uint64_t EncodedDiscriminator,bool AddressDiversify)146 writePACSignSeq(AppendFtor &Append, unsigned DstReg, orc::ExecutorAddr RawAddr,
147 unsigned RawAddrReg, unsigned DiscriminatorReg, unsigned Key,
148 uint64_t EncodedDiscriminator, bool AddressDiversify) {
149 assert(DstReg < 32 && "DstReg out of range");
150 assert(RawAddrReg < 32 && "AddrReg out of range");
151 assert(DiscriminatorReg < 32 && "DiscriminatorReg out of range");
152 assert(EncodedDiscriminator < 0x10000 && "EncodedDiscriminator out of range");
153
154 if (AddressDiversify) {
155 // Move the address into the discriminator register.
156 if (auto Err = writeMovRegRegSeq(Append, DiscriminatorReg, RawAddrReg))
157 return Err;
158 // Blend encoded discriminator if there is one.
159 if (EncodedDiscriminator) {
160 constexpr uint32_t MOVKTemplate = 0xf2e00000;
161 constexpr unsigned ImmIndex = 5;
162 uint32_t BlendInstr = MOVKTemplate;
163 BlendInstr |= EncodedDiscriminator << ImmIndex;
164 BlendInstr |= DiscriminatorReg;
165 if (auto Err = Append(BlendInstr))
166 return Err;
167 }
168 } else if (EncodedDiscriminator) {
169 // Move the encoded discriminator into the discriminator register.
170 if (auto Err =
171 writeMovRegImm64Seq(Append, DiscriminatorReg, EncodedDiscriminator))
172 return Err;
173 } else
174 DiscriminatorReg = 31; // WZR
175
176 constexpr uint32_t PACTemplate = 0xdac10000;
177 constexpr unsigned ZBitIndex = 13;
178 constexpr unsigned KeyIndex = 10;
179 constexpr unsigned DiscriminatorRegIndex = 5;
180
181 uint32_t Instr = PACTemplate;
182 Instr |= (DiscriminatorReg == 31) << ZBitIndex;
183 Instr |= Key << KeyIndex;
184 Instr |= DiscriminatorReg << DiscriminatorRegIndex;
185 Instr |= DstReg;
186
187 return Append(Instr);
188 }
189
190 template <typename AppendFtor>
writeStoreRegSeq(AppendFtor & Append,unsigned DstLocReg,unsigned SrcReg)191 static Error writeStoreRegSeq(AppendFtor &Append, unsigned DstLocReg,
192 unsigned SrcReg) {
193 assert(DstLocReg < 32 && "DstLocReg out of range");
194 assert(SrcReg < 32 && "SrcReg out of range");
195
196 constexpr uint32_t STRTemplate = 0xf9000000;
197 constexpr unsigned DstLocRegIndex = 5;
198 constexpr unsigned SrcRegIndex = 0;
199
200 uint32_t Instr = STRTemplate;
201 Instr |= DstLocReg << DstLocRegIndex;
202 Instr |= SrcReg << SrcRegIndex;
203
204 return Append(Instr);
205 }
206
registerExistingEntries()207 void GOTTableManager::registerExistingEntries() {
208 for (auto *EntrySym : GOTSection->symbols()) {
209 assert(EntrySym->getBlock().edges_size() == 1 &&
210 "GOT block edge count != 1");
211 registerPreExistingEntry(EntrySym->getBlock().edges().begin()->getTarget(),
212 *EntrySym);
213 }
214 }
215
registerExistingEntries()216 void PLTTableManager::registerExistingEntries() {
217 for (auto *EntrySym : StubsSection->symbols()) {
218 assert(EntrySym->getBlock().edges_size() == 2 &&
219 "PLT block edge count != 2");
220 auto &GOTSym = EntrySym->getBlock().edges().begin()->getTarget();
221 assert(GOTSym.getBlock().edges_size() == 1 && "GOT block edge count != 1");
222 registerPreExistingEntry(GOTSym.getBlock().edges().begin()->getTarget(),
223 *EntrySym);
224 }
225 }
226
getPointerSigningFunctionSectionName()227 const char *getPointerSigningFunctionSectionName() { return "$__ptrauth_sign"; }
228
229 /// Creates a pointer signing function section, block, and symbol to reserve
230 /// space for a signing function for this LinkGraph. Clients should insert this
231 /// pass in the post-prune phase, and add the paired
232 /// lowerPointer64AuthEdgesToSigningFunction pass to the pre-fixup phase.
createEmptyPointerSigningFunction(LinkGraph & G)233 Error createEmptyPointerSigningFunction(LinkGraph &G) {
234 LLVM_DEBUG({
235 dbgs() << "Creating empty pointer signing function for " << G.getName()
236 << "\n";
237 });
238
239 // FIXME: We could put a tighter bound on this if we inspected the ptrauth
240 // info encoded in the addend -- the only actually unknown quantity is the
241 // fixup location, and we can probably put constraints even on that.
242 size_t NumPtrAuthFixupLocations = 0;
243 for (auto &Sec : G.sections()) {
244
245 // No-alloc sections can't have ptrauth edges. We don't need to error out
246 // here: applyFixup will catch these edges if any make it to the fixup
247 // stage.
248 if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)
249 continue;
250
251 for (auto *B : Sec.blocks()) {
252 for (auto &E : B->edges())
253 NumPtrAuthFixupLocations +=
254 E.getKind() == aarch64::Pointer64Authenticated;
255 }
256 }
257
258 constexpr size_t MaxPtrSignSeqLength =
259 4 + // To materialize the value to sign.
260 4 + // To materialize the fixup location.
261 3 + // To copy, blend discriminator, and sign
262 1; // To store the result.
263
264 // The maximum number of signing instructions required is the maximum per
265 // location, times the number of locations, plus three instructions to
266 // materialize the return value and return.
267 size_t NumSigningInstrs = NumPtrAuthFixupLocations * MaxPtrSignSeqLength + 3;
268
269 // Create signing function section.
270 auto &SigningSection =
271 G.createSection(getPointerSigningFunctionSectionName(),
272 orc::MemProt::Read | orc::MemProt::Exec);
273 SigningSection.setMemLifetime(orc::MemLifetime::Finalize);
274
275 size_t SigningFunctionSize = NumSigningInstrs * 4;
276 auto &SigningFunctionBlock = G.createMutableContentBlock(
277 SigningSection, G.allocateBuffer(SigningFunctionSize),
278 orc::ExecutorAddr(), 4, 0);
279 G.addAnonymousSymbol(SigningFunctionBlock, 0, SigningFunctionBlock.getSize(),
280 true, true);
281
282 LLVM_DEBUG({
283 dbgs() << " " << NumPtrAuthFixupLocations << " location(s) to sign, up to "
284 << NumSigningInstrs << " instructions required ("
285 << formatv("{0:x}", SigningFunctionBlock.getSize()) << " bytes)\n";
286 });
287
288 return Error::success();
289 }
290
291 /// Given a LinkGraph containing Pointer64Auth edges, transform those edges to
292 /// Pointer64 and add code to sign the pointers in the executor.
293 ///
294 /// This function will add a $__ptrauth_sign section with finalization-lifetime
295 /// containing an anonymous function that will sign all pointers in the graph.
296 /// An allocation action will be added to run this function during finalization.
lowerPointer64AuthEdgesToSigningFunction(LinkGraph & G)297 Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G) {
298 LLVM_DEBUG({
299 dbgs() << "Writing pointer signing function for " << G.getName() << "\n";
300 });
301
302 constexpr unsigned Reg1 = 8; // Holds pointer value to sign.
303 constexpr unsigned Reg2 = 9; // Holds fixup address.
304 constexpr unsigned Reg3 = 10; // Temporary for discriminator value if needed.
305
306 // Find the signing function.
307 auto *SigningSection =
308 G.findSectionByName(getPointerSigningFunctionSectionName());
309 assert(SigningSection && "Siging section missing");
310 assert(SigningSection->blocks_size() == 1 &&
311 "Unexpected number of blocks in signing section");
312 assert(SigningSection->symbols_size() == 1 &&
313 "Unexpected number of symbols in signing section");
314
315 auto &SigningFunctionSym = **SigningSection->symbols().begin();
316 auto &SigningFunctionBlock = SigningFunctionSym.getBlock();
317 auto SigningFunctionBuf = SigningFunctionBlock.getAlreadyMutableContent();
318
319 // Write the instructions to the block content.
320 BinaryStreamWriter InstrWriter(
321 {reinterpret_cast<uint8_t *>(SigningFunctionBuf.data()),
322 SigningFunctionBuf.size()},
323 G.getEndianness());
324
325 auto AppendInstr = [&](uint32_t Instr) {
326 return InstrWriter.writeInteger(Instr);
327 };
328
329 for (auto &Sec : G.sections()) {
330
331 if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)
332 continue;
333
334 for (auto *B : Sec.blocks()) {
335 for (auto &E : B->edges()) {
336 // We're only concerned with Pointer64Authenticated edges here.
337 if (E.getKind() != aarch64::Pointer64Authenticated)
338 continue;
339
340 uint64_t EncodedInfo = E.getAddend();
341 int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff);
342 auto ValueToSign = E.getTarget().getAddress() + RealAddend;
343 if (!ValueToSign) {
344 LLVM_DEBUG(dbgs() << " " << B->getFixupAddress(E) << " <- null\n");
345 E.setAddend(RealAddend);
346 E.setKind(aarch64::Pointer64);
347 continue;
348 }
349
350 uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff;
351 bool AddressDiversify = (EncodedInfo >> 48) & 0x1;
352 uint32_t Key = (EncodedInfo >> 49) & 0x3;
353 uint32_t HighBits = EncodedInfo >> 51;
354
355 if (HighBits != 0x1000)
356 return make_error<JITLinkError>(
357 "Pointer64Auth edge at " +
358 formatv("{0:x}", B->getFixupAddress(E).getValue()) +
359 " has invalid encoded addend " + formatv("{0:x}", EncodedInfo));
360
361 LLVM_DEBUG({
362 const char *const KeyNames[] = {"IA", "IB", "DA", "DB"};
363 dbgs() << " " << B->getFixupAddress(E) << " <- " << ValueToSign
364 << " : key = " << KeyNames[Key] << ", discriminator = "
365 << formatv("{0:x4}", InitialDiscriminator)
366 << ", address diversified = "
367 << (AddressDiversify ? "yes" : "no") << "\n";
368 });
369
370 // Materialize pointer value.
371 cantFail(
372 writeMovRegImm64Seq(AppendInstr, Reg1, ValueToSign.getValue()));
373
374 // Materialize fixup pointer.
375 cantFail(writeMovRegImm64Seq(AppendInstr, Reg2,
376 B->getFixupAddress(E).getValue()));
377
378 // Write signing instruction(s).
379 cantFail(writePACSignSeq(AppendInstr, Reg1, ValueToSign, Reg2, Reg3,
380 Key, InitialDiscriminator, AddressDiversify));
381
382 // Store signed pointer.
383 cantFail(writeStoreRegSeq(AppendInstr, Reg2, Reg1));
384
385 // Replace edge with a keep-alive to preserve dependence info.
386 E.setKind(Edge::KeepAlive);
387 }
388 }
389 }
390
391 // Write epilogue. x0 = 0, x1 = 1 is an SPS serialized Error::success value.
392 constexpr uint32_t RETInstr = 0xd65f03c0;
393 cantFail(writeMovRegImm64Seq(AppendInstr, 0, 0)); // mov x0, #0
394 cantFail(writeMovRegImm64Seq(AppendInstr, 1, 1)); // mov x1, #1
395 cantFail(AppendInstr(RETInstr)); // ret
396
397 // Add an allocation action to call the signing function.
398 using namespace orc::shared;
399 G.allocActions().push_back(
400 {cantFail(WrapperFunctionCall::Create<SPSArgList<>>(
401 SigningFunctionSym.getAddress())),
402 {}});
403
404 return Error::success();
405 }
406
407 } // namespace aarch64
408 } // namespace jitlink
409 } // namespace llvm
410