1 //===- AArch64ErrataFix.cpp -----------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // This file implements Section Patching for the purpose of working around 9 // errata in CPUs. The general principle is that an erratum sequence of one or 10 // more instructions is detected in the instruction stream, one of the 11 // instructions in the sequence is replaced with a branch to a patch sequence 12 // of replacement instructions. At the end of the replacement sequence the 13 // patch branches back to the instruction stream. 14 15 // This technique is only suitable for fixing an erratum when: 16 // - There is a set of necessary conditions required to trigger the erratum that 17 // can be detected at static link time. 18 // - There is a set of replacement instructions that can be used to remove at 19 // least one of the necessary conditions that trigger the erratum. 20 // - We can overwrite an instruction in the erratum sequence with a branch to 21 // the replacement sequence. 22 // - We can place the replacement sequence within range of the branch. 23 24 // FIXME: 25 // - The implementation here only supports one patch, the AArch64 Cortex-53 26 // errata 843419 that affects r0p0, r0p1, r0p2 and r0p4 versions of the core. 27 // To keep the initial version simple there is no support for multiple 28 // architectures or selection of different patches. 29 //===----------------------------------------------------------------------===// 30 31 #include "AArch64ErrataFix.h" 32 #include "Config.h" 33 #include "LinkerScript.h" 34 #include "OutputSections.h" 35 #include "Relocations.h" 36 #include "Symbols.h" 37 #include "SyntheticSections.h" 38 #include "Target.h" 39 #include "lld/Common/Memory.h" 40 #include "lld/Common/Strings.h" 41 #include "llvm/Support/Endian.h" 42 #include "llvm/Support/raw_ostream.h" 43 #include <algorithm> 44 45 using namespace llvm; 46 using namespace llvm::ELF; 47 using namespace llvm::object; 48 using namespace llvm::support; 49 using namespace llvm::support::endian; 50 51 using namespace lld; 52 using namespace lld::elf; 53 54 // Helper functions to identify instructions and conditions needed to trigger 55 // the Cortex-A53-843419 erratum. 56 57 // ADRP 58 // | 1 | immlo (2) | 1 | 0 0 0 0 | immhi (19) | Rd (5) | 59 static bool isADRP(uint32_t instr) { 60 return (instr & 0x9f000000) == 0x90000000; 61 } 62 63 // Load and store bit patterns from ARMv8-A ARM ARM. 64 // Instructions appear in order of appearance starting from table in 65 // C4.1.3 Loads and Stores. 66 67 // All loads and stores have 1 (at bit postion 27), (0 at bit position 25). 68 // | op0 x op1 (2) | 1 op2 0 op3 (2) | x | op4 (5) | xxxx | op5 (2) | x (10) | 69 static bool isLoadStoreClass(uint32_t instr) { 70 return (instr & 0x0a000000) == 0x08000000; 71 } 72 73 // LDN/STN multiple no offset 74 // | 0 Q 00 | 1100 | 0 L 00 | 0000 | opcode (4) | size (2) | Rn (5) | Rt (5) | 75 // LDN/STN multiple post-indexed 76 // | 0 Q 00 | 1100 | 1 L 0 | Rm (5)| opcode (4) | size (2) | Rn (5) | Rt (5) | 77 // L == 0 for stores. 78 79 // Utility routine to decode opcode field of LDN/STN multiple structure 80 // instructions to find the ST1 instructions. 81 // opcode == 0010 ST1 4 registers. 82 // opcode == 0110 ST1 3 registers. 83 // opcode == 0111 ST1 1 register. 84 // opcode == 1010 ST1 2 registers. 85 static bool isST1MultipleOpcode(uint32_t instr) { 86 return (instr & 0x0000f000) == 0x00002000 || 87 (instr & 0x0000f000) == 0x00006000 || 88 (instr & 0x0000f000) == 0x00007000 || 89 (instr & 0x0000f000) == 0x0000a000; 90 } 91 92 static bool isST1Multiple(uint32_t instr) { 93 return (instr & 0xbfff0000) == 0x0c000000 && isST1MultipleOpcode(instr); 94 } 95 96 // Writes to Rn (writeback). 97 static bool isST1MultiplePost(uint32_t instr) { 98 return (instr & 0xbfe00000) == 0x0c800000 && isST1MultipleOpcode(instr); 99 } 100 101 // LDN/STN single no offset 102 // | 0 Q 00 | 1101 | 0 L R 0 | 0000 | opc (3) S | size (2) | Rn (5) | Rt (5)| 103 // LDN/STN single post-indexed 104 // | 0 Q 00 | 1101 | 1 L R | Rm (5) | opc (3) S | size (2) | Rn (5) | Rt (5)| 105 // L == 0 for stores 106 107 // Utility routine to decode opcode field of LDN/STN single structure 108 // instructions to find the ST1 instructions. 109 // R == 0 for ST1 and ST3, R == 1 for ST2 and ST4. 110 // opcode == 000 ST1 8-bit. 111 // opcode == 010 ST1 16-bit. 112 // opcode == 100 ST1 32 or 64-bit (Size determines which). 113 static bool isST1SingleOpcode(uint32_t instr) { 114 return (instr & 0x0040e000) == 0x00000000 || 115 (instr & 0x0040e000) == 0x00004000 || 116 (instr & 0x0040e000) == 0x00008000; 117 } 118 119 static bool isST1Single(uint32_t instr) { 120 return (instr & 0xbfff0000) == 0x0d000000 && isST1SingleOpcode(instr); 121 } 122 123 // Writes to Rn (writeback). 124 static bool isST1SinglePost(uint32_t instr) { 125 return (instr & 0xbfe00000) == 0x0d800000 && isST1SingleOpcode(instr); 126 } 127 128 static bool isST1(uint32_t instr) { 129 return isST1Multiple(instr) || isST1MultiplePost(instr) || 130 isST1Single(instr) || isST1SinglePost(instr); 131 } 132 133 // Load/store exclusive 134 // | size (2) 00 | 1000 | o2 L o1 | Rs (5) | o0 | Rt2 (5) | Rn (5) | Rt (5) | 135 // L == 0 for Stores. 136 static bool isLoadStoreExclusive(uint32_t instr) { 137 return (instr & 0x3f000000) == 0x08000000; 138 } 139 140 static bool isLoadExclusive(uint32_t instr) { 141 return (instr & 0x3f400000) == 0x08400000; 142 } 143 144 // Load register literal 145 // | opc (2) 01 | 1 V 00 | imm19 | Rt (5) | 146 static bool isLoadLiteral(uint32_t instr) { 147 return (instr & 0x3b000000) == 0x18000000; 148 } 149 150 // Load/store no-allocate pair 151 // (offset) 152 // | opc (2) 10 | 1 V 00 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | 153 // L == 0 for stores. 154 // Never writes to register 155 static bool isSTNP(uint32_t instr) { 156 return (instr & 0x3bc00000) == 0x28000000; 157 } 158 159 // Load/store register pair 160 // (post-indexed) 161 // | opc (2) 10 | 1 V 00 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | 162 // L == 0 for stores, V == 0 for Scalar, V == 1 for Simd/FP 163 // Writes to Rn. 164 static bool isSTPPost(uint32_t instr) { 165 return (instr & 0x3bc00000) == 0x28800000; 166 } 167 168 // (offset) 169 // | opc (2) 10 | 1 V 01 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | 170 static bool isSTPOffset(uint32_t instr) { 171 return (instr & 0x3bc00000) == 0x29000000; 172 } 173 174 // (pre-index) 175 // | opc (2) 10 | 1 V 01 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | 176 // Writes to Rn. 177 static bool isSTPPre(uint32_t instr) { 178 return (instr & 0x3bc00000) == 0x29800000; 179 } 180 181 static bool isSTP(uint32_t instr) { 182 return isSTPPost(instr) || isSTPOffset(instr) || isSTPPre(instr); 183 } 184 185 // Load/store register (unscaled immediate) 186 // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 00 | Rn (5) | Rt (5) | 187 // V == 0 for Scalar, V == 1 for Simd/FP. 188 static bool isLoadStoreUnscaled(uint32_t instr) { 189 return (instr & 0x3b000c00) == 0x38000000; 190 } 191 192 // Load/store register (immediate post-indexed) 193 // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 01 | Rn (5) | Rt (5) | 194 static bool isLoadStoreImmediatePost(uint32_t instr) { 195 return (instr & 0x3b200c00) == 0x38000400; 196 } 197 198 // Load/store register (unprivileged) 199 // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 10 | Rn (5) | Rt (5) | 200 static bool isLoadStoreUnpriv(uint32_t instr) { 201 return (instr & 0x3b200c00) == 0x38000800; 202 } 203 204 // Load/store register (immediate pre-indexed) 205 // | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 11 | Rn (5) | Rt (5) | 206 static bool isLoadStoreImmediatePre(uint32_t instr) { 207 return (instr & 0x3b200c00) == 0x38000c00; 208 } 209 210 // Load/store register (register offset) 211 // | size (2) 11 | 1 V 00 | opc (2) 1 | Rm (5) | option (3) S | 10 | Rn | Rt | 212 static bool isLoadStoreRegisterOff(uint32_t instr) { 213 return (instr & 0x3b200c00) == 0x38200800; 214 } 215 216 // Load/store register (unsigned immediate) 217 // | size (2) 11 | 1 V 01 | opc (2) | imm12 | Rn (5) | Rt (5) | 218 static bool isLoadStoreRegisterUnsigned(uint32_t instr) { 219 return (instr & 0x3b000000) == 0x39000000; 220 } 221 222 // Rt is always in bit position 0 - 4. 223 static uint32_t getRt(uint32_t instr) { return (instr & 0x1f); } 224 225 // Rn is always in bit position 5 - 9. 226 static uint32_t getRn(uint32_t instr) { return (instr >> 5) & 0x1f; } 227 228 // C4.1.2 Branches, Exception Generating and System instructions 229 // | op0 (3) 1 | 01 op1 (4) | x (22) | 230 // op0 == 010 101 op1 == 0xxx Conditional Branch. 231 // op0 == 110 101 op1 == 1xxx Unconditional Branch Register. 232 // op0 == x00 101 op1 == xxxx Unconditional Branch immediate. 233 // op0 == x01 101 op1 == 0xxx Compare and branch immediate. 234 // op0 == x01 101 op1 == 1xxx Test and branch immediate. 235 static bool isBranch(uint32_t instr) { 236 return ((instr & 0xfe000000) == 0xd6000000) || // Cond branch. 237 ((instr & 0xfe000000) == 0x54000000) || // Uncond branch reg. 238 ((instr & 0x7c000000) == 0x14000000) || // Uncond branch imm. 239 ((instr & 0x7c000000) == 0x34000000); // Compare and test branch. 240 } 241 242 static bool isV8SingleRegisterNonStructureLoadStore(uint32_t instr) { 243 return isLoadStoreUnscaled(instr) || isLoadStoreImmediatePost(instr) || 244 isLoadStoreUnpriv(instr) || isLoadStoreImmediatePre(instr) || 245 isLoadStoreRegisterOff(instr) || isLoadStoreRegisterUnsigned(instr); 246 } 247 248 // Note that this function refers to v8.0 only and does not include the 249 // additional load and store instructions added for in later revisions of 250 // the architecture such as the Atomic memory operations introduced 251 // in v8.1. 252 static bool isV8NonStructureLoad(uint32_t instr) { 253 if (isLoadExclusive(instr)) 254 return true; 255 if (isLoadLiteral(instr)) 256 return true; 257 else if (isV8SingleRegisterNonStructureLoadStore(instr)) { 258 // For Load and Store single register, Loads are derived from a 259 // combination of the Size, V and Opc fields. 260 uint32_t size = (instr >> 30) & 0xff; 261 uint32_t v = (instr >> 26) & 0x1; 262 uint32_t opc = (instr >> 22) & 0x3; 263 // For the load and store instructions that we are decoding. 264 // Opc == 0 are all stores. 265 // Opc == 1 with a couple of exceptions are loads. The exceptions are: 266 // Size == 00 (0), V == 1, Opc == 10 (2) which is a store and 267 // Size == 11 (3), V == 0, Opc == 10 (2) which is a prefetch. 268 return opc != 0 && !(size == 0 && v == 1 && opc == 2) && 269 !(size == 3 && v == 0 && opc == 2); 270 } 271 return false; 272 } 273 274 // The following decode instructions are only complete up to the instructions 275 // needed for errata 843419. 276 277 // Instruction with writeback updates the index register after the load/store. 278 static bool hasWriteback(uint32_t instr) { 279 return isLoadStoreImmediatePre(instr) || isLoadStoreImmediatePost(instr) || 280 isSTPPre(instr) || isSTPPost(instr) || isST1SinglePost(instr) || 281 isST1MultiplePost(instr); 282 } 283 284 // For the load and store class of instructions, a load can write to the 285 // destination register, a load and a store can write to the base register when 286 // the instruction has writeback. 287 static bool doesLoadStoreWriteToReg(uint32_t instr, uint32_t reg) { 288 return (isV8NonStructureLoad(instr) && getRt(instr) == reg) || 289 (hasWriteback(instr) && getRn(instr) == reg); 290 } 291 292 // Scanner for Cortex-A53 errata 843419 293 // Full details are available in the Cortex A53 MPCore revision 0 Software 294 // Developers Errata Notice (ARM-EPM-048406). 295 // 296 // The instruction sequence that triggers the erratum is common in compiled 297 // AArch64 code, however it is sensitive to the offset of the sequence within 298 // a 4k page. This means that by scanning and fixing the patch after we have 299 // assigned addresses we only need to disassemble and fix instances of the 300 // sequence in the range of affected offsets. 301 // 302 // In summary the erratum conditions are a series of 4 instructions: 303 // 1.) An ADRP instruction that writes to register Rn with low 12 bits of 304 // address of instruction either 0xff8 or 0xffc. 305 // 2.) A load or store instruction that can be: 306 // - A single register load or store, of either integer or vector registers. 307 // - An STP or STNP, of either integer or vector registers. 308 // - An Advanced SIMD ST1 store instruction. 309 // - Must not write to Rn, but may optionally read from it. 310 // 3.) An optional instruction that is not a branch and does not write to Rn. 311 // 4.) A load or store from the Load/store register (unsigned immediate) class 312 // that uses Rn as the base address register. 313 // 314 // Note that we do not attempt to scan for Sequence 2 as described in the 315 // Software Developers Errata Notice as this has been assessed to be extremely 316 // unlikely to occur in compiled code. This matches gold and ld.bfd behavior. 317 318 // Return true if the Instruction sequence Adrp, Instr2, and Instr4 match 319 // the erratum sequence. The Adrp, Instr2 and Instr4 correspond to 1.), 2.), 320 // and 4.) in the Scanner for Cortex-A53 errata comment above. 321 static bool is843419ErratumSequence(uint32_t instr1, uint32_t instr2, 322 uint32_t instr4) { 323 if (!isADRP(instr1)) 324 return false; 325 326 uint32_t rn = getRt(instr1); 327 return isLoadStoreClass(instr2) && 328 (isLoadStoreExclusive(instr2) || isLoadLiteral(instr2) || 329 isV8SingleRegisterNonStructureLoadStore(instr2) || isSTP(instr2) || 330 isSTNP(instr2) || isST1(instr2)) && 331 !doesLoadStoreWriteToReg(instr2, rn) && 332 isLoadStoreRegisterUnsigned(instr4) && getRn(instr4) == rn; 333 } 334 335 // Scan the instruction sequence starting at Offset Off from the base of 336 // InputSection IS. We update Off in this function rather than in the caller as 337 // we can skip ahead much further into the section when we know how many 338 // instructions we've scanned. 339 // Return the offset of the load or store instruction in IS that we want to 340 // patch or 0 if no patch required. 341 static uint64_t scanCortexA53Errata843419(InputSection *isec, uint64_t &off, 342 uint64_t limit) { 343 uint64_t isecAddr = isec->getVA(0); 344 345 // Advance Off so that (ISAddr + Off) modulo 0x1000 is at least 0xff8. 346 uint64_t initialPageOff = (isecAddr + off) & 0xfff; 347 if (initialPageOff < 0xff8) 348 off += 0xff8 - initialPageOff; 349 350 bool optionalAllowed = limit - off > 12; 351 if (off >= limit || limit - off < 12) { 352 // Need at least 3 4-byte sized instructions to trigger erratum. 353 off = limit; 354 return 0; 355 } 356 357 uint64_t patchOff = 0; 358 const uint8_t *buf = isec->data().begin(); 359 const ulittle32_t *instBuf = reinterpret_cast<const ulittle32_t *>(buf + off); 360 uint32_t instr1 = *instBuf++; 361 uint32_t instr2 = *instBuf++; 362 uint32_t instr3 = *instBuf++; 363 if (is843419ErratumSequence(instr1, instr2, instr3)) { 364 patchOff = off + 8; 365 } else if (optionalAllowed && !isBranch(instr3)) { 366 uint32_t instr4 = *instBuf++; 367 if (is843419ErratumSequence(instr1, instr2, instr4)) 368 patchOff = off + 12; 369 } 370 if (((isecAddr + off) & 0xfff) == 0xff8) 371 off += 4; 372 else 373 off += 0xffc; 374 return patchOff; 375 } 376 377 class lld::elf::Patch843419Section : public SyntheticSection { 378 public: 379 Patch843419Section(InputSection *p, uint64_t off); 380 381 void writeTo(uint8_t *buf) override; 382 383 size_t getSize() const override { return 8; } 384 385 uint64_t getLDSTAddr() const; 386 387 // The Section we are patching. 388 const InputSection *patchee; 389 // The offset of the instruction in the Patchee section we are patching. 390 uint64_t patcheeOffset; 391 // A label for the start of the Patch that we can use as a relocation target. 392 Symbol *patchSym; 393 }; 394 395 lld::elf::Patch843419Section::Patch843419Section(InputSection *p, uint64_t off) 396 : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 4, 397 ".text.patch"), 398 patchee(p), patcheeOffset(off) { 399 this->parent = p->getParent(); 400 patchSym = addSyntheticLocal( 401 saver.save("__CortexA53843419_" + utohexstr(getLDSTAddr())), STT_FUNC, 0, 402 getSize(), *this); 403 addSyntheticLocal(saver.save("$x"), STT_NOTYPE, 0, 0, *this); 404 } 405 406 uint64_t lld::elf::Patch843419Section::getLDSTAddr() const { 407 return patchee->getVA(patcheeOffset); 408 } 409 410 void lld::elf::Patch843419Section::writeTo(uint8_t *buf) { 411 // Copy the instruction that we will be replacing with a branch in the 412 // Patchee Section. 413 write32le(buf, read32le(patchee->data().begin() + patcheeOffset)); 414 415 // Apply any relocation transferred from the original PatcheeSection. 416 // For a SyntheticSection Buf already has outSecOff added, but relocateAlloc 417 // also adds outSecOff so we need to subtract to avoid double counting. 418 this->relocateAlloc(buf - outSecOff, buf - outSecOff + getSize()); 419 420 // Return address is the next instruction after the one we have just copied. 421 uint64_t s = getLDSTAddr() + 4; 422 uint64_t p = patchSym->getVA() + 4; 423 target->relocateOne(buf + 4, R_AARCH64_JUMP26, s - p); 424 } 425 426 void AArch64Err843419Patcher::init() { 427 // The AArch64 ABI permits data in executable sections. We must avoid scanning 428 // this data as if it were instructions to avoid false matches. We use the 429 // mapping symbols in the InputObjects to identify this data, caching the 430 // results in sectionMap so we don't have to recalculate it each pass. 431 432 // The ABI Section 4.5.4 Mapping symbols; defines local symbols that describe 433 // half open intervals [Symbol Value, Next Symbol Value) of code and data 434 // within sections. If there is no next symbol then the half open interval is 435 // [Symbol Value, End of section). The type, code or data, is determined by 436 // the mapping symbol name, $x for code, $d for data. 437 auto isCodeMapSymbol = [](const Symbol *b) { 438 return b->getName() == "$x" || b->getName().startswith("$x."); 439 }; 440 auto isDataMapSymbol = [](const Symbol *b) { 441 return b->getName() == "$d" || b->getName().startswith("$d."); 442 }; 443 444 // Collect mapping symbols for every executable InputSection. 445 for (InputFile *file : objectFiles) { 446 auto *f = cast<ObjFile<ELF64LE>>(file); 447 for (Symbol *b : f->getLocalSymbols()) { 448 auto *def = dyn_cast<Defined>(b); 449 if (!def) 450 continue; 451 if (!isCodeMapSymbol(def) && !isDataMapSymbol(def)) 452 continue; 453 if (auto *sec = dyn_cast_or_null<InputSection>(def->section)) 454 if (sec->flags & SHF_EXECINSTR) 455 sectionMap[sec].push_back(def); 456 } 457 } 458 // For each InputSection make sure the mapping symbols are in sorted in 459 // ascending order and free from consecutive runs of mapping symbols with 460 // the same type. For example we must remove the redundant $d.1 from $x.0 461 // $d.0 $d.1 $x.1. 462 for (auto &kv : sectionMap) { 463 std::vector<const Defined *> &mapSyms = kv.second; 464 if (mapSyms.size() <= 1) 465 continue; 466 llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) { 467 return a->value < b->value; 468 }); 469 mapSyms.erase( 470 std::unique(mapSyms.begin(), mapSyms.end(), 471 [=](const Defined *a, const Defined *b) { 472 return (isCodeMapSymbol(a) && isCodeMapSymbol(b)) || 473 (isDataMapSymbol(a) && isDataMapSymbol(b)); 474 }), 475 mapSyms.end()); 476 } 477 initialized = true; 478 } 479 480 // Insert the PatchSections we have created back into the 481 // InputSectionDescription. As inserting patches alters the addresses of 482 // InputSections that follow them, we try and place the patches after all the 483 // executable sections, although we may need to insert them earlier if the 484 // InputSectionDescription is larger than the maximum branch range. 485 void AArch64Err843419Patcher::insertPatches( 486 InputSectionDescription &isd, std::vector<Patch843419Section *> &patches) { 487 uint64_t isecLimit; 488 uint64_t prevIsecLimit = isd.sections.front()->outSecOff; 489 uint64_t patchUpperBound = prevIsecLimit + target->getThunkSectionSpacing(); 490 uint64_t outSecAddr = isd.sections.front()->getParent()->addr; 491 492 // Set the outSecOff of patches to the place where we want to insert them. 493 // We use a similar strategy to Thunk placement. Place patches roughly 494 // every multiple of maximum branch range. 495 auto patchIt = patches.begin(); 496 auto patchEnd = patches.end(); 497 for (const InputSection *isec : isd.sections) { 498 isecLimit = isec->outSecOff + isec->getSize(); 499 if (isecLimit > patchUpperBound) { 500 while (patchIt != patchEnd) { 501 if ((*patchIt)->getLDSTAddr() - outSecAddr >= prevIsecLimit) 502 break; 503 (*patchIt)->outSecOff = prevIsecLimit; 504 ++patchIt; 505 } 506 patchUpperBound = prevIsecLimit + target->getThunkSectionSpacing(); 507 } 508 prevIsecLimit = isecLimit; 509 } 510 for (; patchIt != patchEnd; ++patchIt) { 511 (*patchIt)->outSecOff = isecLimit; 512 } 513 514 // merge all patch sections. We use the outSecOff assigned above to 515 // determine the insertion point. This is ok as we only merge into an 516 // InputSectionDescription once per pass, and at the end of the pass 517 // assignAddresses() will recalculate all the outSecOff values. 518 std::vector<InputSection *> tmp; 519 tmp.reserve(isd.sections.size() + patches.size()); 520 auto mergeCmp = [](const InputSection *a, const InputSection *b) { 521 if (a->outSecOff < b->outSecOff) 522 return true; 523 if (a->outSecOff == b->outSecOff && isa<Patch843419Section>(a) && 524 !isa<Patch843419Section>(b)) 525 return true; 526 return false; 527 }; 528 std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(), 529 patches.end(), std::back_inserter(tmp), mergeCmp); 530 isd.sections = std::move(tmp); 531 } 532 533 // Given an erratum sequence that starts at address adrpAddr, with an 534 // instruction that we need to patch at patcheeOffset from the start of 535 // InputSection IS, create a Patch843419 Section and add it to the 536 // Patches that we need to insert. 537 static void implementPatch(uint64_t adrpAddr, uint64_t patcheeOffset, 538 InputSection *isec, 539 std::vector<Patch843419Section *> &patches) { 540 // There may be a relocation at the same offset that we are patching. There 541 // are four cases that we need to consider. 542 // Case 1: R_AARCH64_JUMP26 branch relocation. We have already patched this 543 // instance of the erratum on a previous patch and altered the relocation. We 544 // have nothing more to do. 545 // Case 2: A TLS Relaxation R_RELAX_TLS_IE_TO_LE. In this case the ADRP that 546 // we read will be transformed into a MOVZ later so we actually don't match 547 // the sequence and have nothing more to do. 548 // Case 3: A load/store register (unsigned immediate) class relocation. There 549 // are two of these R_AARCH_LD64_ABS_LO12_NC and R_AARCH_LD64_GOT_LO12_NC and 550 // they are both absolute. We need to add the same relocation to the patch, 551 // and replace the relocation with a R_AARCH_JUMP26 branch relocation. 552 // Case 4: No relocation. We must create a new R_AARCH64_JUMP26 branch 553 // relocation at the offset. 554 auto relIt = llvm::find_if(isec->relocations, [=](const Relocation &r) { 555 return r.offset == patcheeOffset; 556 }); 557 if (relIt != isec->relocations.end() && 558 (relIt->type == R_AARCH64_JUMP26 || relIt->expr == R_RELAX_TLS_IE_TO_LE)) 559 return; 560 561 log("detected cortex-a53-843419 erratum sequence starting at " + 562 utohexstr(adrpAddr) + " in unpatched output."); 563 564 auto *ps = make<Patch843419Section>(isec, patcheeOffset); 565 patches.push_back(ps); 566 567 auto makeRelToPatch = [](uint64_t offset, Symbol *patchSym) { 568 return Relocation{R_PC, R_AARCH64_JUMP26, offset, 0, patchSym}; 569 }; 570 571 if (relIt != isec->relocations.end()) { 572 ps->relocations.push_back( 573 {relIt->expr, relIt->type, 0, relIt->addend, relIt->sym}); 574 *relIt = makeRelToPatch(patcheeOffset, ps->patchSym); 575 } else 576 isec->relocations.push_back(makeRelToPatch(patcheeOffset, ps->patchSym)); 577 } 578 579 // Scan all the instructions in InputSectionDescription, for each instance of 580 // the erratum sequence create a Patch843419Section. We return the list of 581 // Patch843419Sections that need to be applied to ISD. 582 std::vector<Patch843419Section *> 583 AArch64Err843419Patcher::patchInputSectionDescription( 584 InputSectionDescription &isd) { 585 std::vector<Patch843419Section *> patches; 586 for (InputSection *isec : isd.sections) { 587 // LLD doesn't use the erratum sequence in SyntheticSections. 588 if (isa<SyntheticSection>(isec)) 589 continue; 590 // Use sectionMap to make sure we only scan code and not inline data. 591 // We have already sorted MapSyms in ascending order and removed consecutive 592 // mapping symbols of the same type. Our range of executable instructions to 593 // scan is therefore [codeSym->value, dataSym->value) or [codeSym->value, 594 // section size). 595 std::vector<const Defined *> &mapSyms = sectionMap[isec]; 596 597 auto codeSym = llvm::find_if(mapSyms, [&](const Defined *ms) { 598 return ms->getName().startswith("$x"); 599 }); 600 601 while (codeSym != mapSyms.end()) { 602 auto dataSym = std::next(codeSym); 603 uint64_t off = (*codeSym)->value; 604 uint64_t limit = 605 (dataSym == mapSyms.end()) ? isec->data().size() : (*dataSym)->value; 606 607 while (off < limit) { 608 uint64_t startAddr = isec->getVA(off); 609 if (uint64_t patcheeOffset = scanCortexA53Errata843419(isec, off, limit)) 610 implementPatch(startAddr, patcheeOffset, isec, patches); 611 } 612 if (dataSym == mapSyms.end()) 613 break; 614 codeSym = std::next(dataSym); 615 } 616 } 617 return patches; 618 } 619 620 // For each InputSectionDescription make one pass over the executable sections 621 // looking for the erratum sequence; creating a synthetic Patch843419Section 622 // for each instance found. We insert these synthetic patch sections after the 623 // executable code in each InputSectionDescription. 624 // 625 // PreConditions: 626 // The Output and Input Sections have had their final addresses assigned. 627 // 628 // PostConditions: 629 // Returns true if at least one patch was added. The addresses of the 630 // Ouptut and Input Sections may have been changed. 631 // Returns false if no patches were required and no changes were made. 632 bool AArch64Err843419Patcher::createFixes() { 633 if (initialized == false) 634 init(); 635 636 bool addressesChanged = false; 637 for (OutputSection *os : outputSections) { 638 if (!(os->flags & SHF_ALLOC) || !(os->flags & SHF_EXECINSTR)) 639 continue; 640 for (BaseCommand *bc : os->sectionCommands) 641 if (auto *isd = dyn_cast<InputSectionDescription>(bc)) { 642 std::vector<Patch843419Section *> patches = 643 patchInputSectionDescription(*isd); 644 if (!patches.empty()) { 645 insertPatches(*isd, patches); 646 addressesChanged = true; 647 } 648 } 649 } 650 return addressesChanged; 651 } 652