1 //===---- aarch64.cpp - Generic JITLink aarch64 edge kinds, utilities -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Generic utilities for graphs representing aarch64 objects. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/aarch64.h" 14 15 #include "llvm/Support/BinaryStreamWriter.h" 16 17 #define DEBUG_TYPE "jitlink" 18 19 namespace llvm { 20 namespace jitlink { 21 namespace aarch64 { 22 23 const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00, 24 0x00, 0x00, 0x00, 0x00}; 25 26 const char PointerJumpStubContent[12] = { 27 0x10, 0x00, 0x00, (char)0x90u, // ADRP x16, <imm>@page21 28 0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12] 29 0x00, 0x02, 0x1f, (char)0xd6u // BR x16 30 }; 31 32 const char ReentryTrampolineContent[8] = { 33 (char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8] 34 0x00, 0x00, 0x00, (char)0x94 // BL 35 }; 36 37 const char *getEdgeKindName(Edge::Kind R) { 38 switch (R) { 39 case Pointer64: 40 return "Pointer64"; 41 case Pointer64Authenticated: 42 return "Pointer64Authenticated"; 43 case Pointer32: 44 return "Pointer32"; 45 case Delta64: 46 return "Delta64"; 47 case Delta32: 48 return "Delta32"; 49 case NegDelta64: 50 return "NegDelta64"; 51 case NegDelta32: 52 return "NegDelta32"; 53 case Branch26PCRel: 54 return "Branch26PCRel"; 55 case MoveWide16: 56 return "MoveWide16"; 57 case LDRLiteral19: 58 return "LDRLiteral19"; 59 case TestAndBranch14PCRel: 60 return "TestAndBranch14PCRel"; 61 case CondBranch19PCRel: 62 return "CondBranch19PCRel"; 63 case ADRLiteral21: 64 return "ADRLiteral21"; 65 case Page21: 66 return "Page21"; 67 case PageOffset12: 68 return "PageOffset12"; 69 case GotPageOffset15: 70 return "GotPageOffset15"; 71 case RequestGOTAndTransformToPage21: 72 return "RequestGOTAndTransformToPage21"; 73 case RequestGOTAndTransformToPageOffset12: 74 return "RequestGOTAndTransformToPageOffset12"; 75 case RequestGOTAndTransformToPageOffset15: 76 return "RequestGOTAndTransformToPageOffset15"; 77 case RequestGOTAndTransformToDelta32: 78 return "RequestGOTAndTransformToDelta32"; 79 case RequestTLVPAndTransformToPage21: 80 return "RequestTLVPAndTransformToPage21"; 81 case RequestTLVPAndTransformToPageOffset12: 82 return "RequestTLVPAndTransformToPageOffset12"; 83 case RequestTLSDescEntryAndTransformToPage21: 84 return "RequestTLSDescEntryAndTransformToPage21"; 85 case RequestTLSDescEntryAndTransformToPageOffset12: 86 return "RequestTLSDescEntryAndTransformToPageOffset12"; 87 default: 88 return getGenericEdgeKindName(R); 89 } 90 } 91 92 // Write a 64-bit GPR -> GPR move. 93 template <typename AppendFtor> 94 static Error writeMovRegRegSeq(AppendFtor &Append, uint64_t DstReg, 95 uint64_t SrcReg) { 96 assert(DstReg < 32 && "Dst reg out of range"); 97 assert(SrcReg < 32 && "Src reg out of range"); 98 99 if (DstReg == SrcReg) 100 return Error::success(); 101 102 constexpr uint32_t MOVGPR64Template = 0xaa0003e0; 103 constexpr uint32_t DstRegIndex = 0; 104 constexpr uint32_t SrcRegIndex = 16; 105 uint32_t Instr = MOVGPR64Template; 106 Instr |= DstReg << DstRegIndex; 107 Instr |= SrcReg << SrcRegIndex; 108 return Append(Instr); 109 } 110 111 // Generate a sequence of imm writes to assign the given value. 112 template <typename AppendFtor> 113 static Error writeMovRegImm64Seq(AppendFtor &Append, uint64_t Reg, 114 uint64_t Imm) { 115 assert(Reg < 32 && "Invalid register number"); 116 117 constexpr uint32_t MovRegImm64Template = 0xd2800000; 118 constexpr unsigned PreserveBitIndex = 29; 119 constexpr unsigned ShiftBitsIndex = 21; 120 constexpr unsigned ImmBitsIndex = 5; 121 122 bool PreserveRegValue = false; 123 for (unsigned I = 0; I != 4; ++I) { 124 uint32_t ImmBits = Imm & 0xffff; 125 Imm >>= 16; 126 127 // Skip any all-zero immediates after the first one. 128 if (PreserveRegValue && !ImmBits) 129 continue; 130 131 uint32_t Instr = MovRegImm64Template; 132 Instr |= PreserveRegValue << PreserveBitIndex; 133 Instr |= (I << ShiftBitsIndex); 134 Instr |= ImmBits << ImmBitsIndex; 135 Instr |= Reg; 136 if (auto Err = Append(Instr)) 137 return Err; 138 PreserveRegValue = true; 139 } 140 141 return Error::success(); 142 } 143 144 template <typename AppendFtor> 145 static Error 146 writePACSignSeq(AppendFtor &Append, unsigned DstReg, orc::ExecutorAddr RawAddr, 147 unsigned RawAddrReg, unsigned DiscriminatorReg, unsigned Key, 148 uint64_t EncodedDiscriminator, bool AddressDiversify) { 149 assert(DstReg < 32 && "DstReg out of range"); 150 assert(RawAddrReg < 32 && "AddrReg out of range"); 151 assert(DiscriminatorReg < 32 && "DiscriminatorReg out of range"); 152 assert(EncodedDiscriminator < 0x10000 && "EncodedDiscriminator out of range"); 153 154 if (AddressDiversify) { 155 // Move the address into the discriminator register. 156 if (auto Err = writeMovRegRegSeq(Append, DiscriminatorReg, RawAddrReg)) 157 return Err; 158 // Blend encoded discriminator if there is one. 159 if (EncodedDiscriminator) { 160 constexpr uint32_t MOVKTemplate = 0xf2e00000; 161 constexpr unsigned ImmIndex = 5; 162 uint32_t BlendInstr = MOVKTemplate; 163 BlendInstr |= EncodedDiscriminator << ImmIndex; 164 BlendInstr |= DiscriminatorReg; 165 if (auto Err = Append(BlendInstr)) 166 return Err; 167 } 168 } else if (EncodedDiscriminator) { 169 // Move the encoded discriminator into the discriminator register. 170 if (auto Err = 171 writeMovRegImm64Seq(Append, DiscriminatorReg, EncodedDiscriminator)) 172 return Err; 173 } else 174 DiscriminatorReg = 31; // WZR 175 176 constexpr uint32_t PACTemplate = 0xdac10000; 177 constexpr unsigned ZBitIndex = 13; 178 constexpr unsigned KeyIndex = 10; 179 constexpr unsigned DiscriminatorRegIndex = 5; 180 181 uint32_t Instr = PACTemplate; 182 Instr |= (DiscriminatorReg == 31) << ZBitIndex; 183 Instr |= Key << KeyIndex; 184 Instr |= DiscriminatorReg << DiscriminatorRegIndex; 185 Instr |= DstReg; 186 187 return Append(Instr); 188 } 189 190 template <typename AppendFtor> 191 static Error writeStoreRegSeq(AppendFtor &Append, unsigned DstLocReg, 192 unsigned SrcReg) { 193 assert(DstLocReg < 32 && "DstLocReg out of range"); 194 assert(SrcReg < 32 && "SrcReg out of range"); 195 196 constexpr uint32_t STRTemplate = 0xf9000000; 197 constexpr unsigned DstLocRegIndex = 5; 198 constexpr unsigned SrcRegIndex = 0; 199 200 uint32_t Instr = STRTemplate; 201 Instr |= DstLocReg << DstLocRegIndex; 202 Instr |= SrcReg << SrcRegIndex; 203 204 return Append(Instr); 205 } 206 207 void GOTTableManager::registerExistingEntries() { 208 for (auto *EntrySym : GOTSection->symbols()) { 209 assert(EntrySym->getBlock().edges_size() == 1 && 210 "GOT block edge count != 1"); 211 registerPreExistingEntry(EntrySym->getBlock().edges().begin()->getTarget(), 212 *EntrySym); 213 } 214 } 215 216 void PLTTableManager::registerExistingEntries() { 217 for (auto *EntrySym : StubsSection->symbols()) { 218 assert(EntrySym->getBlock().edges_size() == 2 && 219 "PLT block edge count != 2"); 220 auto &GOTSym = EntrySym->getBlock().edges().begin()->getTarget(); 221 assert(GOTSym.getBlock().edges_size() == 1 && "GOT block edge count != 1"); 222 registerPreExistingEntry(GOTSym.getBlock().edges().begin()->getTarget(), 223 *EntrySym); 224 } 225 } 226 227 const char *getPointerSigningFunctionSectionName() { return "$__ptrauth_sign"; } 228 229 /// Creates a pointer signing function section, block, and symbol to reserve 230 /// space for a signing function for this LinkGraph. Clients should insert this 231 /// pass in the post-prune phase, and add the paired 232 /// lowerPointer64AuthEdgesToSigningFunction pass to the pre-fixup phase. 233 Error createEmptyPointerSigningFunction(LinkGraph &G) { 234 LLVM_DEBUG({ 235 dbgs() << "Creating empty pointer signing function for " << G.getName() 236 << "\n"; 237 }); 238 239 // FIXME: We could put a tighter bound on this if we inspected the ptrauth 240 // info encoded in the addend -- the only actually unknown quantity is the 241 // fixup location, and we can probably put constraints even on that. 242 size_t NumPtrAuthFixupLocations = 0; 243 for (auto &Sec : G.sections()) { 244 245 // No-alloc sections can't have ptrauth edges. We don't need to error out 246 // here: applyFixup will catch these edges if any make it to the fixup 247 // stage. 248 if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc) 249 continue; 250 251 for (auto *B : Sec.blocks()) { 252 for (auto &E : B->edges()) 253 NumPtrAuthFixupLocations += 254 E.getKind() == aarch64::Pointer64Authenticated; 255 } 256 } 257 258 constexpr size_t MaxPtrSignSeqLength = 259 4 + // To materialize the value to sign. 260 4 + // To materialize the fixup location. 261 3 + // To copy, blend discriminator, and sign 262 1; // To store the result. 263 264 // The maximum number of signing instructions required is the maximum per 265 // location, times the number of locations, plus three instructions to 266 // materialize the return value and return. 267 size_t NumSigningInstrs = NumPtrAuthFixupLocations * MaxPtrSignSeqLength + 3; 268 269 // Create signing function section. 270 auto &SigningSection = 271 G.createSection(getPointerSigningFunctionSectionName(), 272 orc::MemProt::Read | orc::MemProt::Exec); 273 SigningSection.setMemLifetime(orc::MemLifetime::Finalize); 274 275 size_t SigningFunctionSize = NumSigningInstrs * 4; 276 auto &SigningFunctionBlock = G.createMutableContentBlock( 277 SigningSection, G.allocateBuffer(SigningFunctionSize), 278 orc::ExecutorAddr(), 4, 0); 279 G.addAnonymousSymbol(SigningFunctionBlock, 0, SigningFunctionBlock.getSize(), 280 true, true); 281 282 LLVM_DEBUG({ 283 dbgs() << " " << NumPtrAuthFixupLocations << " location(s) to sign, up to " 284 << NumSigningInstrs << " instructions required (" 285 << formatv("{0:x}", SigningFunctionBlock.getSize()) << " bytes)\n"; 286 }); 287 288 return Error::success(); 289 } 290 291 /// Given a LinkGraph containing Pointer64Auth edges, transform those edges to 292 /// Pointer64 and add code to sign the pointers in the executor. 293 /// 294 /// This function will add a $__ptrauth_sign section with finalization-lifetime 295 /// containing an anonymous function that will sign all pointers in the graph. 296 /// An allocation action will be added to run this function during finalization. 297 Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G) { 298 LLVM_DEBUG({ 299 dbgs() << "Writing pointer signing function for " << G.getName() << "\n"; 300 }); 301 302 constexpr unsigned Reg1 = 8; // Holds pointer value to sign. 303 constexpr unsigned Reg2 = 9; // Holds fixup address. 304 constexpr unsigned Reg3 = 10; // Temporary for discriminator value if needed. 305 306 // Find the signing function. 307 auto *SigningSection = 308 G.findSectionByName(getPointerSigningFunctionSectionName()); 309 assert(SigningSection && "Siging section missing"); 310 assert(SigningSection->blocks_size() == 1 && 311 "Unexpected number of blocks in signing section"); 312 assert(SigningSection->symbols_size() == 1 && 313 "Unexpected number of symbols in signing section"); 314 315 auto &SigningFunctionSym = **SigningSection->symbols().begin(); 316 auto &SigningFunctionBlock = SigningFunctionSym.getBlock(); 317 auto SigningFunctionBuf = SigningFunctionBlock.getAlreadyMutableContent(); 318 319 // Write the instructions to the block content. 320 BinaryStreamWriter InstrWriter( 321 {reinterpret_cast<uint8_t *>(SigningFunctionBuf.data()), 322 SigningFunctionBuf.size()}, 323 G.getEndianness()); 324 325 auto AppendInstr = [&](uint32_t Instr) { 326 return InstrWriter.writeInteger(Instr); 327 }; 328 329 for (auto &Sec : G.sections()) { 330 331 if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc) 332 continue; 333 334 for (auto *B : Sec.blocks()) { 335 for (auto &E : B->edges()) { 336 // We're only concerned with Pointer64Authenticated edges here. 337 if (E.getKind() != aarch64::Pointer64Authenticated) 338 continue; 339 340 uint64_t EncodedInfo = E.getAddend(); 341 int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff); 342 auto ValueToSign = E.getTarget().getAddress() + RealAddend; 343 if (!ValueToSign) { 344 LLVM_DEBUG(dbgs() << " " << B->getFixupAddress(E) << " <- null\n"); 345 E.setAddend(RealAddend); 346 E.setKind(aarch64::Pointer64); 347 continue; 348 } 349 350 uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff; 351 bool AddressDiversify = (EncodedInfo >> 48) & 0x1; 352 uint32_t Key = (EncodedInfo >> 49) & 0x3; 353 uint32_t HighBits = EncodedInfo >> 51; 354 355 if (HighBits != 0x1000) 356 return make_error<JITLinkError>( 357 "Pointer64Auth edge at " + 358 formatv("{0:x}", B->getFixupAddress(E).getValue()) + 359 " has invalid encoded addend " + formatv("{0:x}", EncodedInfo)); 360 361 LLVM_DEBUG({ 362 const char *const KeyNames[] = {"IA", "IB", "DA", "DB"}; 363 dbgs() << " " << B->getFixupAddress(E) << " <- " << ValueToSign 364 << " : key = " << KeyNames[Key] << ", discriminator = " 365 << formatv("{0:x4}", InitialDiscriminator) 366 << ", address diversified = " 367 << (AddressDiversify ? "yes" : "no") << "\n"; 368 }); 369 370 // Materialize pointer value. 371 cantFail( 372 writeMovRegImm64Seq(AppendInstr, Reg1, ValueToSign.getValue())); 373 374 // Materialize fixup pointer. 375 cantFail(writeMovRegImm64Seq(AppendInstr, Reg2, 376 B->getFixupAddress(E).getValue())); 377 378 // Write signing instruction(s). 379 cantFail(writePACSignSeq(AppendInstr, Reg1, ValueToSign, Reg2, Reg3, 380 Key, InitialDiscriminator, AddressDiversify)); 381 382 // Store signed pointer. 383 cantFail(writeStoreRegSeq(AppendInstr, Reg2, Reg1)); 384 385 // Replace edge with a keep-alive to preserve dependence info. 386 E.setKind(Edge::KeepAlive); 387 } 388 } 389 } 390 391 // Write epilogue. x0 = 0, x1 = 1 is an SPS serialized Error::success value. 392 constexpr uint32_t RETInstr = 0xd65f03c0; 393 cantFail(writeMovRegImm64Seq(AppendInstr, 0, 0)); // mov x0, #0 394 cantFail(writeMovRegImm64Seq(AppendInstr, 1, 1)); // mov x1, #1 395 cantFail(AppendInstr(RETInstr)); // ret 396 397 // Add an allocation action to call the signing function. 398 using namespace orc::shared; 399 G.allocActions().push_back( 400 {cantFail(WrapperFunctionCall::Create<SPSArgList<>>( 401 SigningFunctionSym.getAddress())), 402 {}}); 403 404 return Error::success(); 405 } 406 407 } // namespace aarch64 408 } // namespace jitlink 409 } // namespace llvm 410