1 //===- AArch64SpeculationHardening.cpp - Harden Against Missspeculation --===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a pass to insert code to mitigate against side channel 10 // vulnerabilities that may happen under control flow miss-speculation. 11 // 12 // The pass implements tracking of control flow miss-speculation into a "taint" 13 // register. That taint register can then be used to mask off registers with 14 // sensitive data when executing under miss-speculation, a.k.a. "transient 15 // execution". 16 // This pass is aimed at mitigating against SpectreV1-style vulnarabilities. 17 // 18 // It also implements speculative load hardening, i.e. using the taint register 19 // to automatically mask off loaded data. 20 // 21 // As a possible follow-on improvement, also an intrinsics-based approach as 22 // explained at https://lwn.net/Articles/759423/ could be implemented on top of 23 // the current design. 24 // 25 // For AArch64, the following implementation choices are made to implement the 26 // tracking of control flow miss-speculation into a taint register: 27 // Some of these are different than the implementation choices made in 28 // the similar pass implemented in X86SpeculativeLoadHardening.cpp, as 29 // the instruction set characteristics result in different trade-offs. 30 // - The speculation hardening is done after register allocation. With a 31 // relative abundance of registers, one register is reserved (X16) to be 32 // the taint register. X16 is expected to not clash with other register 33 // reservation mechanisms with very high probability because: 34 // . The AArch64 ABI doesn't guarantee X16 to be retained across any call. 35 // . The only way to request X16 to be used as a programmer is through 36 // inline assembly. In the rare case a function explicitly demands to 37 // use X16/W16, this pass falls back to hardening against speculation 38 // by inserting a DSB SYS/ISB barrier pair which will prevent control 39 // flow speculation. 40 // - It is easy to insert mask operations at this late stage as we have 41 // mask operations available that don't set flags. 42 // - The taint variable contains all-ones when no miss-speculation is detected, 43 // and contains all-zeros when miss-speculation is detected. Therefore, when 44 // masking, an AND instruction (which only changes the register to be masked, 45 // no other side effects) can easily be inserted anywhere that's needed. 46 // - The tracking of miss-speculation is done by using a data-flow conditional 47 // select instruction (CSEL) to evaluate the flags that were also used to 48 // make conditional branch direction decisions. Speculation of the CSEL 49 // instruction can be limited with a CSDB instruction - so the combination of 50 // CSEL + a later CSDB gives the guarantee that the flags as used in the CSEL 51 // aren't speculated. When conditional branch direction gets miss-speculated, 52 // the semantics of the inserted CSEL instruction is such that the taint 53 // register will contain all zero bits. 54 // One key requirement for this to work is that the conditional branch is 55 // followed by an execution of the CSEL instruction, where the CSEL 56 // instruction needs to use the same flags status as the conditional branch. 57 // This means that the conditional branches must not be implemented as one 58 // of the AArch64 conditional branches that do not use the flags as input 59 // (CB(N)Z and TB(N)Z). This is implemented by ensuring in the instruction 60 // selectors to not produce these instructions when speculation hardening 61 // is enabled. This pass will assert if it does encounter such an instruction. 62 // - On function call boundaries, the miss-speculation state is transferred from 63 // the taint register X16 to be encoded in the SP register as value 0. 64 // 65 // For the aspect of automatically hardening loads, using the taint register, 66 // (a.k.a. speculative load hardening, see 67 // https://llvm.org/docs/SpeculativeLoadHardening.html), the following 68 // implementation choices are made for AArch64: 69 // - Many of the optimizations described at 70 // https://llvm.org/docs/SpeculativeLoadHardening.html to harden fewer 71 // loads haven't been implemented yet - but for some of them there are 72 // FIXMEs in the code. 73 // - loads that load into general purpose (X or W) registers get hardened by 74 // masking the loaded data. For loads that load into other registers, the 75 // address loaded from gets hardened. It is expected that hardening the 76 // loaded data may be more efficient; but masking data in registers other 77 // than X or W is not easy and may result in being slower than just 78 // hardening the X address register loaded from. 79 // - On AArch64, CSDB instructions are inserted between the masking of the 80 // register and its first use, to ensure there's no non-control-flow 81 // speculation that might undermine the hardening mechanism. 82 // 83 // Future extensions/improvements could be: 84 // - Implement this functionality using full speculation barriers, akin to the 85 // x86-slh-lfence option. This may be more useful for the intrinsics-based 86 // approach than for the SLH approach to masking. 87 // Note that this pass already inserts the full speculation barriers if the 88 // function for some niche reason makes use of X16/W16. 89 // - no indirect branch misprediction gets protected/instrumented; but this 90 // could be done for some indirect branches, such as switch jump tables. 91 //===----------------------------------------------------------------------===// 92 93 #include "AArch64InstrInfo.h" 94 #include "AArch64Subtarget.h" 95 #include "Utils/AArch64BaseInfo.h" 96 #include "llvm/ADT/BitVector.h" 97 #include "llvm/ADT/SmallVector.h" 98 #include "llvm/CodeGen/MachineBasicBlock.h" 99 #include "llvm/CodeGen/MachineFunction.h" 100 #include "llvm/CodeGen/MachineFunctionPass.h" 101 #include "llvm/CodeGen/MachineInstr.h" 102 #include "llvm/CodeGen/MachineInstrBuilder.h" 103 #include "llvm/CodeGen/MachineOperand.h" 104 #include "llvm/CodeGen/MachineRegisterInfo.h" 105 #include "llvm/CodeGen/RegisterScavenging.h" 106 #include "llvm/IR/DebugLoc.h" 107 #include "llvm/Pass.h" 108 #include "llvm/Support/CodeGen.h" 109 #include "llvm/Target/TargetMachine.h" 110 #include <cassert> 111 112 using namespace llvm; 113 114 #define DEBUG_TYPE "aarch64-speculation-hardening" 115 116 #define AARCH64_SPECULATION_HARDENING_NAME "AArch64 speculation hardening pass" 117 118 cl::opt<bool> HardenLoads("aarch64-slh-loads", cl::Hidden, 119 cl::desc("Sanitize loads from memory."), 120 cl::init(true)); 121 122 namespace { 123 124 class AArch64SpeculationHardening : public MachineFunctionPass { 125 public: 126 const TargetInstrInfo *TII; 127 const TargetRegisterInfo *TRI; 128 129 static char ID; 130 131 AArch64SpeculationHardening() : MachineFunctionPass(ID) { 132 initializeAArch64SpeculationHardeningPass(*PassRegistry::getPassRegistry()); 133 } 134 135 bool runOnMachineFunction(MachineFunction &Fn) override; 136 137 StringRef getPassName() const override { 138 return AARCH64_SPECULATION_HARDENING_NAME; 139 } 140 141 private: 142 unsigned MisspeculatingTaintReg; 143 unsigned MisspeculatingTaintReg32Bit; 144 bool UseControlFlowSpeculationBarrier; 145 BitVector RegsNeedingCSDBBeforeUse; 146 BitVector RegsAlreadyMasked; 147 148 bool functionUsesHardeningRegister(MachineFunction &MF) const; 149 bool instrumentControlFlow(MachineBasicBlock &MBB, 150 bool &UsesFullSpeculationBarrier); 151 bool endsWithCondControlFlow(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, 152 MachineBasicBlock *&FBB, 153 AArch64CC::CondCode &CondCode) const; 154 void insertTrackingCode(MachineBasicBlock &SplitEdgeBB, 155 AArch64CC::CondCode &CondCode, DebugLoc DL) const; 156 void insertSPToRegTaintPropagation(MachineBasicBlock &MBB, 157 MachineBasicBlock::iterator MBBI) const; 158 void insertRegToSPTaintPropagation(MachineBasicBlock &MBB, 159 MachineBasicBlock::iterator MBBI, 160 unsigned TmpReg) const; 161 void insertFullSpeculationBarrier(MachineBasicBlock &MBB, 162 MachineBasicBlock::iterator MBBI, 163 DebugLoc DL) const; 164 165 bool slhLoads(MachineBasicBlock &MBB); 166 bool makeGPRSpeculationSafe(MachineBasicBlock &MBB, 167 MachineBasicBlock::iterator MBBI, 168 MachineInstr &MI, unsigned Reg); 169 bool lowerSpeculationSafeValuePseudos(MachineBasicBlock &MBB, 170 bool UsesFullSpeculationBarrier); 171 bool expandSpeculationSafeValue(MachineBasicBlock &MBB, 172 MachineBasicBlock::iterator MBBI, 173 bool UsesFullSpeculationBarrier); 174 bool insertCSDB(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 175 DebugLoc DL); 176 }; 177 178 } // end anonymous namespace 179 180 char AArch64SpeculationHardening::ID = 0; 181 182 INITIALIZE_PASS(AArch64SpeculationHardening, "aarch64-speculation-hardening", 183 AARCH64_SPECULATION_HARDENING_NAME, false, false) 184 185 bool AArch64SpeculationHardening::endsWithCondControlFlow( 186 MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, 187 AArch64CC::CondCode &CondCode) const { 188 SmallVector<MachineOperand, 1> analyzeBranchCondCode; 189 if (TII->analyzeBranch(MBB, TBB, FBB, analyzeBranchCondCode, false)) 190 return false; 191 192 // Ignore if the BB ends in an unconditional branch/fall-through. 193 if (analyzeBranchCondCode.empty()) 194 return false; 195 196 // If the BB ends with a single conditional branch, FBB will be set to 197 // nullptr (see API docs for TII->analyzeBranch). For the rest of the 198 // analysis we want the FBB block to be set always. 199 assert(TBB != nullptr); 200 if (FBB == nullptr) 201 FBB = MBB.getFallThrough(); 202 203 // If both the true and the false condition jump to the same basic block, 204 // there isn't need for any protection - whether the branch is speculated 205 // correctly or not, we end up executing the architecturally correct code. 206 if (TBB == FBB) 207 return false; 208 209 assert(MBB.succ_size() == 2); 210 // translate analyzeBranchCondCode to CondCode. 211 assert(analyzeBranchCondCode.size() == 1 && "unknown Cond array format"); 212 CondCode = AArch64CC::CondCode(analyzeBranchCondCode[0].getImm()); 213 return true; 214 } 215 216 void AArch64SpeculationHardening::insertFullSpeculationBarrier( 217 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 218 DebugLoc DL) const { 219 // A full control flow speculation barrier consists of (DSB SYS + ISB) 220 BuildMI(MBB, MBBI, DL, TII->get(AArch64::DSB)).addImm(0xf); 221 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ISB)).addImm(0xf); 222 } 223 224 void AArch64SpeculationHardening::insertTrackingCode( 225 MachineBasicBlock &SplitEdgeBB, AArch64CC::CondCode &CondCode, 226 DebugLoc DL) const { 227 if (UseControlFlowSpeculationBarrier) { 228 insertFullSpeculationBarrier(SplitEdgeBB, SplitEdgeBB.begin(), DL); 229 } else { 230 BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::CSELXr)) 231 .addDef(MisspeculatingTaintReg) 232 .addUse(MisspeculatingTaintReg) 233 .addUse(AArch64::XZR) 234 .addImm(CondCode); 235 SplitEdgeBB.addLiveIn(AArch64::NZCV); 236 } 237 } 238 239 bool AArch64SpeculationHardening::instrumentControlFlow( 240 MachineBasicBlock &MBB, bool &UsesFullSpeculationBarrier) { 241 LLVM_DEBUG(dbgs() << "Instrument control flow tracking on MBB: " << MBB); 242 243 bool Modified = false; 244 MachineBasicBlock *TBB = nullptr; 245 MachineBasicBlock *FBB = nullptr; 246 AArch64CC::CondCode CondCode; 247 248 if (!endsWithCondControlFlow(MBB, TBB, FBB, CondCode)) { 249 LLVM_DEBUG(dbgs() << "... doesn't end with CondControlFlow\n"); 250 } else { 251 // Now insert: 252 // "CSEL MisSpeculatingR, MisSpeculatingR, XZR, cond" on the True edge and 253 // "CSEL MisSpeculatingR, MisSpeculatingR, XZR, Invertcond" on the False 254 // edge. 255 AArch64CC::CondCode InvCondCode = AArch64CC::getInvertedCondCode(CondCode); 256 257 MachineBasicBlock *SplitEdgeTBB = MBB.SplitCriticalEdge(TBB, *this); 258 MachineBasicBlock *SplitEdgeFBB = MBB.SplitCriticalEdge(FBB, *this); 259 260 assert(SplitEdgeTBB != nullptr); 261 assert(SplitEdgeFBB != nullptr); 262 263 DebugLoc DL; 264 if (MBB.instr_end() != MBB.instr_begin()) 265 DL = (--MBB.instr_end())->getDebugLoc(); 266 267 insertTrackingCode(*SplitEdgeTBB, CondCode, DL); 268 insertTrackingCode(*SplitEdgeFBB, InvCondCode, DL); 269 270 LLVM_DEBUG(dbgs() << "SplitEdgeTBB: " << *SplitEdgeTBB << "\n"); 271 LLVM_DEBUG(dbgs() << "SplitEdgeFBB: " << *SplitEdgeFBB << "\n"); 272 Modified = true; 273 } 274 275 // Perform correct code generation around function calls and before returns. 276 // The below variables record the return/terminator instructions and the call 277 // instructions respectively; including which register is available as a 278 // temporary register just before the recorded instructions. 279 SmallVector<std::pair<MachineInstr *, unsigned>, 4> ReturnInstructions; 280 SmallVector<std::pair<MachineInstr *, unsigned>, 4> CallInstructions; 281 // if a temporary register is not available for at least one of the 282 // instructions for which we need to transfer taint to the stack pointer, we 283 // need to insert a full speculation barrier. 284 // TmpRegisterNotAvailableEverywhere tracks that condition. 285 bool TmpRegisterNotAvailableEverywhere = false; 286 287 RegScavenger RS; 288 RS.enterBasicBlock(MBB); 289 290 for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); I++) { 291 MachineInstr &MI = *I; 292 if (!MI.isReturn() && !MI.isCall()) 293 continue; 294 295 // The RegScavenger represents registers available *after* the MI 296 // instruction pointed to by RS.getCurrentPosition(). 297 // We need to have a register that is available *before* the MI is executed. 298 if (I != MBB.begin()) 299 RS.forward(std::prev(I)); 300 // FIXME: The below just finds *a* unused register. Maybe code could be 301 // optimized more if this looks for the register that isn't used for the 302 // longest time around this place, to enable more scheduling freedom. Not 303 // sure if that would actually result in a big performance difference 304 // though. Maybe RegisterScavenger::findSurvivorBackwards has some logic 305 // already to do this - but it's unclear if that could easily be used here. 306 unsigned TmpReg = RS.FindUnusedReg(&AArch64::GPR64commonRegClass); 307 LLVM_DEBUG(dbgs() << "RS finds " 308 << ((TmpReg == 0) ? "no register " : "register "); 309 if (TmpReg != 0) dbgs() << printReg(TmpReg, TRI) << " "; 310 dbgs() << "to be available at MI " << MI); 311 if (TmpReg == 0) 312 TmpRegisterNotAvailableEverywhere = true; 313 if (MI.isReturn()) 314 ReturnInstructions.push_back({&MI, TmpReg}); 315 else if (MI.isCall()) 316 CallInstructions.push_back({&MI, TmpReg}); 317 } 318 319 if (TmpRegisterNotAvailableEverywhere) { 320 // When a temporary register is not available everywhere in this basic 321 // basic block where a propagate-taint-to-sp operation is needed, just 322 // emit a full speculation barrier at the start of this basic block, which 323 // renders the taint/speculation tracking in this basic block unnecessary. 324 insertFullSpeculationBarrier(MBB, MBB.begin(), 325 (MBB.begin())->getDebugLoc()); 326 UsesFullSpeculationBarrier = true; 327 Modified = true; 328 } else { 329 for (auto MI_Reg : ReturnInstructions) { 330 assert(MI_Reg.second != 0); 331 LLVM_DEBUG( 332 dbgs() 333 << " About to insert Reg to SP taint propagation with temp register " 334 << printReg(MI_Reg.second, TRI) 335 << " on instruction: " << *MI_Reg.first); 336 insertRegToSPTaintPropagation(MBB, MI_Reg.first, MI_Reg.second); 337 Modified = true; 338 } 339 340 for (auto MI_Reg : CallInstructions) { 341 assert(MI_Reg.second != 0); 342 LLVM_DEBUG(dbgs() << " About to insert Reg to SP and back taint " 343 "propagation with temp register " 344 << printReg(MI_Reg.second, TRI) 345 << " around instruction: " << *MI_Reg.first); 346 // Just after the call: 347 insertSPToRegTaintPropagation( 348 MBB, std::next((MachineBasicBlock::iterator)MI_Reg.first)); 349 // Just before the call: 350 insertRegToSPTaintPropagation(MBB, MI_Reg.first, MI_Reg.second); 351 Modified = true; 352 } 353 } 354 return Modified; 355 } 356 357 void AArch64SpeculationHardening::insertSPToRegTaintPropagation( 358 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { 359 // If full control flow speculation barriers are used, emit a control flow 360 // barrier to block potential miss-speculation in flight coming in to this 361 // function. 362 if (UseControlFlowSpeculationBarrier) { 363 insertFullSpeculationBarrier(MBB, MBBI, DebugLoc()); 364 return; 365 } 366 367 // CMP SP, #0 === SUBS xzr, SP, #0 368 BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::SUBSXri)) 369 .addDef(AArch64::XZR) 370 .addUse(AArch64::SP) 371 .addImm(0) 372 .addImm(0); // no shift 373 // CSETM x16, NE === CSINV x16, xzr, xzr, EQ 374 BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::CSINVXr)) 375 .addDef(MisspeculatingTaintReg) 376 .addUse(AArch64::XZR) 377 .addUse(AArch64::XZR) 378 .addImm(AArch64CC::EQ); 379 } 380 381 void AArch64SpeculationHardening::insertRegToSPTaintPropagation( 382 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 383 unsigned TmpReg) const { 384 // If full control flow speculation barriers are used, there will not be 385 // miss-speculation when returning from this function, and therefore, also 386 // no need to encode potential miss-speculation into the stack pointer. 387 if (UseControlFlowSpeculationBarrier) 388 return; 389 390 // mov Xtmp, SP === ADD Xtmp, SP, #0 391 BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) 392 .addDef(TmpReg) 393 .addUse(AArch64::SP) 394 .addImm(0) 395 .addImm(0); // no shift 396 // and Xtmp, Xtmp, TaintReg === AND Xtmp, Xtmp, TaintReg, #0 397 BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ANDXrs)) 398 .addDef(TmpReg, RegState::Renamable) 399 .addUse(TmpReg, RegState::Kill | RegState::Renamable) 400 .addUse(MisspeculatingTaintReg, RegState::Kill) 401 .addImm(0); 402 // mov SP, Xtmp === ADD SP, Xtmp, #0 403 BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri)) 404 .addDef(AArch64::SP) 405 .addUse(TmpReg, RegState::Kill) 406 .addImm(0) 407 .addImm(0); // no shift 408 } 409 410 bool AArch64SpeculationHardening::functionUsesHardeningRegister( 411 MachineFunction &MF) const { 412 for (MachineBasicBlock &MBB : MF) { 413 for (MachineInstr &MI : MBB) { 414 // treat function calls specially, as the hardening register does not 415 // need to remain live across function calls. 416 if (MI.isCall()) 417 continue; 418 if (MI.readsRegister(MisspeculatingTaintReg, TRI) || 419 MI.modifiesRegister(MisspeculatingTaintReg, TRI)) 420 return true; 421 } 422 } 423 return false; 424 } 425 426 // Make GPR register Reg speculation-safe by putting it through the 427 // SpeculationSafeValue pseudo instruction, if we can't prove that 428 // the value in the register has already been hardened. 429 bool AArch64SpeculationHardening::makeGPRSpeculationSafe( 430 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineInstr &MI, 431 unsigned Reg) { 432 assert(AArch64::GPR32allRegClass.contains(Reg) || 433 AArch64::GPR64allRegClass.contains(Reg)); 434 435 // Loads cannot directly load a value into the SP (nor WSP). 436 // Therefore, if Reg is SP or WSP, it is because the instruction loads from 437 // the stack through the stack pointer. 438 // 439 // Since the stack pointer is never dynamically controllable, don't harden it. 440 if (Reg == AArch64::SP || Reg == AArch64::WSP) 441 return false; 442 443 // Do not harden the register again if already hardened before. 444 if (RegsAlreadyMasked[Reg]) 445 return false; 446 447 const bool Is64Bit = AArch64::GPR64allRegClass.contains(Reg); 448 LLVM_DEBUG(dbgs() << "About to harden register : " << Reg << "\n"); 449 BuildMI(MBB, MBBI, MI.getDebugLoc(), 450 TII->get(Is64Bit ? AArch64::SpeculationSafeValueX 451 : AArch64::SpeculationSafeValueW)) 452 .addDef(Reg) 453 .addUse(Reg); 454 RegsAlreadyMasked.set(Reg); 455 return true; 456 } 457 458 bool AArch64SpeculationHardening::slhLoads(MachineBasicBlock &MBB) { 459 bool Modified = false; 460 461 LLVM_DEBUG(dbgs() << "slhLoads running on MBB: " << MBB); 462 463 RegsAlreadyMasked.reset(); 464 465 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); 466 MachineBasicBlock::iterator NextMBBI; 467 for (; MBBI != E; MBBI = NextMBBI) { 468 MachineInstr &MI = *MBBI; 469 NextMBBI = std::next(MBBI); 470 // Only harden loaded values or addresses used in loads. 471 if (!MI.mayLoad()) 472 continue; 473 474 LLVM_DEBUG(dbgs() << "About to harden: " << MI); 475 476 // For general purpose register loads, harden the registers loaded into. 477 // For other loads, harden the address loaded from. 478 // Masking the loaded value is expected to result in less performance 479 // overhead, as the load can still execute speculatively in comparison to 480 // when the address loaded from gets masked. However, masking is only 481 // easy to do efficiently on GPR registers, so for loads into non-GPR 482 // registers (e.g. floating point loads), mask the address loaded from. 483 bool AllDefsAreGPR = llvm::all_of(MI.defs(), [&](MachineOperand &Op) { 484 return Op.isReg() && (AArch64::GPR32allRegClass.contains(Op.getReg()) || 485 AArch64::GPR64allRegClass.contains(Op.getReg())); 486 }); 487 // FIXME: it might be a worthwhile optimization to not mask loaded 488 // values if all the registers involved in address calculation are already 489 // hardened, leading to this load not able to execute on a miss-speculated 490 // path. 491 bool HardenLoadedData = AllDefsAreGPR; 492 bool HardenAddressLoadedFrom = !HardenLoadedData; 493 494 // First remove registers from AlreadyMaskedRegisters if their value is 495 // updated by this instruction - it makes them contain a new value that is 496 // not guaranteed to already have been masked. 497 for (MachineOperand Op : MI.defs()) 498 for (MCRegAliasIterator AI(Op.getReg(), TRI, true); AI.isValid(); ++AI) 499 RegsAlreadyMasked.reset(*AI); 500 501 // FIXME: loads from the stack with an immediate offset from the stack 502 // pointer probably shouldn't be hardened, which could result in a 503 // significant optimization. See section "Don’t check loads from 504 // compile-time constant stack offsets", in 505 // https://llvm.org/docs/SpeculativeLoadHardening.html 506 507 if (HardenLoadedData) 508 for (auto Def : MI.defs()) { 509 if (Def.isDead()) 510 // Do not mask a register that is not used further. 511 continue; 512 // FIXME: For pre/post-increment addressing modes, the base register 513 // used in address calculation is also defined by this instruction. 514 // It might be a worthwhile optimization to not harden that 515 // base register increment/decrement when the increment/decrement is 516 // an immediate. 517 Modified |= makeGPRSpeculationSafe(MBB, NextMBBI, MI, Def.getReg()); 518 } 519 520 if (HardenAddressLoadedFrom) 521 for (auto Use : MI.uses()) { 522 if (!Use.isReg()) 523 continue; 524 unsigned Reg = Use.getReg(); 525 // Some loads of floating point data have implicit defs/uses on a 526 // super register of that floating point data. Some examples: 527 // $s0 = LDRSui $sp, 22, implicit-def $q0 528 // $q0 = LD1i64 $q0, 1, renamable $x0 529 // We need to filter out these uses for non-GPR register which occur 530 // because the load partially fills a non-GPR register with the loaded 531 // data. Just skipping all non-GPR registers is safe (for now) as all 532 // AArch64 load instructions only use GPR registers to perform the 533 // address calculation. FIXME: However that might change once we can 534 // produce SVE gather instructions. 535 if (!(AArch64::GPR32allRegClass.contains(Reg) || 536 AArch64::GPR64allRegClass.contains(Reg))) 537 continue; 538 Modified |= makeGPRSpeculationSafe(MBB, MBBI, MI, Reg); 539 } 540 } 541 return Modified; 542 } 543 544 /// \brief If MBBI references a pseudo instruction that should be expanded 545 /// here, do the expansion and return true. Otherwise return false. 546 bool AArch64SpeculationHardening::expandSpeculationSafeValue( 547 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, 548 bool UsesFullSpeculationBarrier) { 549 MachineInstr &MI = *MBBI; 550 unsigned Opcode = MI.getOpcode(); 551 bool Is64Bit = true; 552 553 switch (Opcode) { 554 default: 555 break; 556 case AArch64::SpeculationSafeValueW: 557 Is64Bit = false; 558 LLVM_FALLTHROUGH; 559 case AArch64::SpeculationSafeValueX: 560 // Just remove the SpeculationSafe pseudo's if control flow 561 // miss-speculation isn't happening because we're already inserting barriers 562 // to guarantee that. 563 if (!UseControlFlowSpeculationBarrier && !UsesFullSpeculationBarrier) { 564 unsigned DstReg = MI.getOperand(0).getReg(); 565 unsigned SrcReg = MI.getOperand(1).getReg(); 566 // Mark this register and all its aliasing registers as needing to be 567 // value speculation hardened before its next use, by using a CSDB 568 // barrier instruction. 569 for (MachineOperand Op : MI.defs()) 570 for (MCRegAliasIterator AI(Op.getReg(), TRI, true); AI.isValid(); ++AI) 571 RegsNeedingCSDBBeforeUse.set(*AI); 572 573 // Mask off with taint state. 574 BuildMI(MBB, MBBI, MI.getDebugLoc(), 575 Is64Bit ? TII->get(AArch64::ANDXrs) : TII->get(AArch64::ANDWrs)) 576 .addDef(DstReg) 577 .addUse(SrcReg, RegState::Kill) 578 .addUse(Is64Bit ? MisspeculatingTaintReg 579 : MisspeculatingTaintReg32Bit) 580 .addImm(0); 581 } 582 MI.eraseFromParent(); 583 return true; 584 } 585 return false; 586 } 587 588 bool AArch64SpeculationHardening::insertCSDB(MachineBasicBlock &MBB, 589 MachineBasicBlock::iterator MBBI, 590 DebugLoc DL) { 591 assert(!UseControlFlowSpeculationBarrier && "No need to insert CSDBs when " 592 "control flow miss-speculation " 593 "is already blocked"); 594 // insert data value speculation barrier (CSDB) 595 BuildMI(MBB, MBBI, DL, TII->get(AArch64::HINT)).addImm(0x14); 596 RegsNeedingCSDBBeforeUse.reset(); 597 return true; 598 } 599 600 bool AArch64SpeculationHardening::lowerSpeculationSafeValuePseudos( 601 MachineBasicBlock &MBB, bool UsesFullSpeculationBarrier) { 602 bool Modified = false; 603 604 RegsNeedingCSDBBeforeUse.reset(); 605 606 // The following loop iterates over all instructions in the basic block, 607 // and performs 2 operations: 608 // 1. Insert a CSDB at this location if needed. 609 // 2. Expand the SpeculationSafeValuePseudo if the current instruction is 610 // one. 611 // 612 // The insertion of the CSDB is done as late as possible (i.e. just before 613 // the use of a masked register), in the hope that that will reduce the 614 // total number of CSDBs in a block when there are multiple masked registers 615 // in the block. 616 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); 617 DebugLoc DL; 618 while (MBBI != E) { 619 MachineInstr &MI = *MBBI; 620 DL = MI.getDebugLoc(); 621 MachineBasicBlock::iterator NMBBI = std::next(MBBI); 622 623 // First check if a CSDB needs to be inserted due to earlier registers 624 // that were masked and that are used by the next instruction. 625 // Also emit the barrier on any potential control flow changes. 626 bool NeedToEmitBarrier = false; 627 if (RegsNeedingCSDBBeforeUse.any() && (MI.isCall() || MI.isTerminator())) 628 NeedToEmitBarrier = true; 629 if (!NeedToEmitBarrier) 630 for (MachineOperand Op : MI.uses()) 631 if (Op.isReg() && RegsNeedingCSDBBeforeUse[Op.getReg()]) { 632 NeedToEmitBarrier = true; 633 break; 634 } 635 636 if (NeedToEmitBarrier && !UsesFullSpeculationBarrier) 637 Modified |= insertCSDB(MBB, MBBI, DL); 638 639 Modified |= 640 expandSpeculationSafeValue(MBB, MBBI, UsesFullSpeculationBarrier); 641 642 MBBI = NMBBI; 643 } 644 645 if (RegsNeedingCSDBBeforeUse.any() && !UsesFullSpeculationBarrier) 646 Modified |= insertCSDB(MBB, MBBI, DL); 647 648 return Modified; 649 } 650 651 bool AArch64SpeculationHardening::runOnMachineFunction(MachineFunction &MF) { 652 if (!MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening)) 653 return false; 654 655 MisspeculatingTaintReg = AArch64::X16; 656 MisspeculatingTaintReg32Bit = AArch64::W16; 657 TII = MF.getSubtarget().getInstrInfo(); 658 TRI = MF.getSubtarget().getRegisterInfo(); 659 RegsNeedingCSDBBeforeUse.resize(TRI->getNumRegs()); 660 RegsAlreadyMasked.resize(TRI->getNumRegs()); 661 UseControlFlowSpeculationBarrier = functionUsesHardeningRegister(MF); 662 663 bool Modified = false; 664 665 // Step 1: Enable automatic insertion of SpeculationSafeValue. 666 if (HardenLoads) { 667 LLVM_DEBUG( 668 dbgs() << "***** AArch64SpeculationHardening - automatic insertion of " 669 "SpeculationSafeValue intrinsics *****\n"); 670 for (auto &MBB : MF) 671 Modified |= slhLoads(MBB); 672 } 673 674 // 2. Add instrumentation code to function entry and exits. 675 LLVM_DEBUG( 676 dbgs() 677 << "***** AArch64SpeculationHardening - track control flow *****\n"); 678 679 SmallVector<MachineBasicBlock *, 2> EntryBlocks; 680 EntryBlocks.push_back(&MF.front()); 681 for (const LandingPadInfo &LPI : MF.getLandingPads()) 682 EntryBlocks.push_back(LPI.LandingPadBlock); 683 for (auto Entry : EntryBlocks) 684 insertSPToRegTaintPropagation( 685 *Entry, Entry->SkipPHIsLabelsAndDebug(Entry->begin())); 686 687 // 3. Add instrumentation code to every basic block. 688 for (auto &MBB : MF) { 689 bool UsesFullSpeculationBarrier = false; 690 Modified |= instrumentControlFlow(MBB, UsesFullSpeculationBarrier); 691 Modified |= 692 lowerSpeculationSafeValuePseudos(MBB, UsesFullSpeculationBarrier); 693 } 694 695 return Modified; 696 } 697 698 /// \brief Returns an instance of the pseudo instruction expansion pass. 699 FunctionPass *llvm::createAArch64SpeculationHardeningPass() { 700 return new AArch64SpeculationHardening(); 701 } 702