1 //===- ARMSLSHardening.cpp - Harden Straight Line Missspeculation ---------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains a pass to insert code to mitigate against side channel 10 // vulnerabilities that may happen under straight line miss-speculation. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "ARM.h" 15 #include "ARMInstrInfo.h" 16 #include "ARMSubtarget.h" 17 #include "llvm/CodeGen/IndirectThunks.h" 18 #include "llvm/CodeGen/MachineBasicBlock.h" 19 #include "llvm/CodeGen/MachineFunction.h" 20 #include "llvm/CodeGen/MachineFunctionPass.h" 21 #include "llvm/CodeGen/MachineInstr.h" 22 #include "llvm/CodeGen/MachineInstrBuilder.h" 23 #include "llvm/CodeGen/MachineOperand.h" 24 #include "llvm/IR/DebugLoc.h" 25 #include <cassert> 26 27 using namespace llvm; 28 29 #define DEBUG_TYPE "arm-sls-hardening" 30 31 #define ARM_SLS_HARDENING_NAME "ARM sls hardening pass" 32 33 namespace { 34 35 class ARMSLSHardening : public MachineFunctionPass { 36 public: 37 const TargetInstrInfo *TII; 38 const ARMSubtarget *ST; 39 40 static char ID; 41 42 ARMSLSHardening() : MachineFunctionPass(ID) { 43 initializeARMSLSHardeningPass(*PassRegistry::getPassRegistry()); 44 } 45 46 bool runOnMachineFunction(MachineFunction &Fn) override; 47 48 StringRef getPassName() const override { return ARM_SLS_HARDENING_NAME; } 49 50 void getAnalysisUsage(AnalysisUsage &AU) const override { 51 AU.setPreservesCFG(); 52 MachineFunctionPass::getAnalysisUsage(AU); 53 } 54 55 private: 56 bool hardenReturnsAndBRs(MachineBasicBlock &MBB) const; 57 bool hardenIndirectCalls(MachineBasicBlock &MBB) const; 58 MachineBasicBlock & 59 ConvertIndirectCallToIndirectJump(MachineBasicBlock &MBB, 60 MachineBasicBlock::iterator) const; 61 }; 62 63 } // end anonymous namespace 64 65 char ARMSLSHardening::ID = 0; 66 67 INITIALIZE_PASS(ARMSLSHardening, "arm-sls-hardening", 68 ARM_SLS_HARDENING_NAME, false, false) 69 70 static void insertSpeculationBarrier(const ARMSubtarget *ST, 71 MachineBasicBlock &MBB, 72 MachineBasicBlock::iterator MBBI, 73 DebugLoc DL, 74 bool AlwaysUseISBDSB = false) { 75 assert(MBBI != MBB.begin() && 76 "Must not insert SpeculationBarrierEndBB as only instruction in MBB."); 77 assert(std::prev(MBBI)->isBarrier() && 78 "SpeculationBarrierEndBB must only follow unconditional control flow " 79 "instructions."); 80 assert(std::prev(MBBI)->isTerminator() && 81 "SpeculationBarrierEndBB must only follow terminators."); 82 const TargetInstrInfo *TII = ST->getInstrInfo(); 83 assert(ST->hasDataBarrier() || ST->hasSB()); 84 bool ProduceSB = ST->hasSB() && !AlwaysUseISBDSB; 85 unsigned BarrierOpc = 86 ProduceSB ? (ST->isThumb() ? ARM::t2SpeculationBarrierSBEndBB 87 : ARM::SpeculationBarrierSBEndBB) 88 : (ST->isThumb() ? ARM::t2SpeculationBarrierISBDSBEndBB 89 : ARM::SpeculationBarrierISBDSBEndBB); 90 if (MBBI == MBB.end() || !isSpeculationBarrierEndBBOpcode(MBBI->getOpcode())) 91 BuildMI(MBB, MBBI, DL, TII->get(BarrierOpc)); 92 } 93 94 bool ARMSLSHardening::runOnMachineFunction(MachineFunction &MF) { 95 ST = &MF.getSubtarget<ARMSubtarget>(); 96 TII = MF.getSubtarget().getInstrInfo(); 97 98 bool Modified = false; 99 for (auto &MBB : MF) { 100 Modified |= hardenReturnsAndBRs(MBB); 101 Modified |= hardenIndirectCalls(MBB); 102 } 103 104 return Modified; 105 } 106 107 bool ARMSLSHardening::hardenReturnsAndBRs(MachineBasicBlock &MBB) const { 108 if (!ST->hardenSlsRetBr()) 109 return false; 110 assert(!ST->isThumb1Only()); 111 bool Modified = false; 112 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(), E = MBB.end(); 113 MachineBasicBlock::iterator NextMBBI; 114 for (; MBBI != E; MBBI = NextMBBI) { 115 MachineInstr &MI = *MBBI; 116 NextMBBI = std::next(MBBI); 117 if (isIndirectControlFlowNotComingBack(MI)) { 118 assert(MI.isTerminator()); 119 assert(!TII->isPredicated(MI)); 120 insertSpeculationBarrier(ST, MBB, std::next(MBBI), MI.getDebugLoc()); 121 Modified = true; 122 } 123 } 124 return Modified; 125 } 126 127 static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_"; 128 129 static const struct ThunkNameRegMode { 130 const char* Name; 131 Register Reg; 132 bool isThumb; 133 } SLSBLRThunks[] = { 134 {"__llvm_slsblr_thunk_arm_r0", ARM::R0, false}, 135 {"__llvm_slsblr_thunk_arm_r1", ARM::R1, false}, 136 {"__llvm_slsblr_thunk_arm_r2", ARM::R2, false}, 137 {"__llvm_slsblr_thunk_arm_r3", ARM::R3, false}, 138 {"__llvm_slsblr_thunk_arm_r4", ARM::R4, false}, 139 {"__llvm_slsblr_thunk_arm_r5", ARM::R5, false}, 140 {"__llvm_slsblr_thunk_arm_r6", ARM::R6, false}, 141 {"__llvm_slsblr_thunk_arm_r7", ARM::R7, false}, 142 {"__llvm_slsblr_thunk_arm_r8", ARM::R8, false}, 143 {"__llvm_slsblr_thunk_arm_r9", ARM::R9, false}, 144 {"__llvm_slsblr_thunk_arm_r10", ARM::R10, false}, 145 {"__llvm_slsblr_thunk_arm_r11", ARM::R11, false}, 146 {"__llvm_slsblr_thunk_arm_sp", ARM::SP, false}, 147 {"__llvm_slsblr_thunk_arm_pc", ARM::PC, false}, 148 {"__llvm_slsblr_thunk_thumb_r0", ARM::R0, true}, 149 {"__llvm_slsblr_thunk_thumb_r1", ARM::R1, true}, 150 {"__llvm_slsblr_thunk_thumb_r2", ARM::R2, true}, 151 {"__llvm_slsblr_thunk_thumb_r3", ARM::R3, true}, 152 {"__llvm_slsblr_thunk_thumb_r4", ARM::R4, true}, 153 {"__llvm_slsblr_thunk_thumb_r5", ARM::R5, true}, 154 {"__llvm_slsblr_thunk_thumb_r6", ARM::R6, true}, 155 {"__llvm_slsblr_thunk_thumb_r7", ARM::R7, true}, 156 {"__llvm_slsblr_thunk_thumb_r8", ARM::R8, true}, 157 {"__llvm_slsblr_thunk_thumb_r9", ARM::R9, true}, 158 {"__llvm_slsblr_thunk_thumb_r10", ARM::R10, true}, 159 {"__llvm_slsblr_thunk_thumb_r11", ARM::R11, true}, 160 {"__llvm_slsblr_thunk_thumb_sp", ARM::SP, true}, 161 {"__llvm_slsblr_thunk_thumb_pc", ARM::PC, true}, 162 }; 163 164 // An enum for tracking whether Arm and Thumb thunks have been inserted into the 165 // current module so far. 166 enum ArmInsertedThunks { ArmThunk = 1, ThumbThunk = 2 }; 167 168 inline ArmInsertedThunks &operator|=(ArmInsertedThunks &X, 169 ArmInsertedThunks Y) { 170 return X = static_cast<ArmInsertedThunks>(X | Y); 171 } 172 173 namespace { 174 struct SLSBLRThunkInserter 175 : ThunkInserter<SLSBLRThunkInserter, ArmInsertedThunks> { 176 const char *getThunkPrefix() { return SLSBLRNamePrefix; } 177 bool mayUseThunk(const MachineFunction &MF, 178 ArmInsertedThunks InsertedThunks) { 179 if ((InsertedThunks & ArmThunk && 180 !MF.getSubtarget<ARMSubtarget>().isThumb()) || 181 (InsertedThunks & ThumbThunk && 182 MF.getSubtarget<ARMSubtarget>().isThumb())) 183 return false; 184 ComdatThunks &= !MF.getSubtarget<ARMSubtarget>().hardenSlsNoComdat(); 185 // FIXME: This could also check if there are any indirect calls in the 186 // function to more accurately reflect if a thunk will be needed. 187 return MF.getSubtarget<ARMSubtarget>().hardenSlsBlr(); 188 } 189 ArmInsertedThunks insertThunks(MachineModuleInfo &MMI, MachineFunction &MF); 190 void populateThunk(MachineFunction &MF); 191 192 private: 193 bool ComdatThunks = true; 194 }; 195 } // namespace 196 197 ArmInsertedThunks SLSBLRThunkInserter::insertThunks(MachineModuleInfo &MMI, 198 MachineFunction &MF) { 199 // FIXME: It probably would be possible to filter which thunks to produce 200 // based on which registers are actually used in indirect calls in this 201 // function. But would that be a worthwhile optimization? 202 const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>(); 203 for (auto T : SLSBLRThunks) 204 if (ST->isThumb() == T.isThumb) 205 createThunkFunction(MMI, T.Name, ComdatThunks, 206 T.isThumb ? "+thumb-mode" : ""); 207 return ST->isThumb() ? ThumbThunk : ArmThunk; 208 } 209 210 void SLSBLRThunkInserter::populateThunk(MachineFunction &MF) { 211 // FIXME: How to better communicate Register number, rather than through 212 // name and lookup table? 213 assert(MF.getName().startswith(getThunkPrefix())); 214 auto ThunkIt = llvm::find_if( 215 SLSBLRThunks, [&MF](auto T) { return T.Name == MF.getName(); }); 216 assert(ThunkIt != std::end(SLSBLRThunks)); 217 Register ThunkReg = ThunkIt->Reg; 218 bool isThumb = ThunkIt->isThumb; 219 220 const TargetInstrInfo *TII = MF.getSubtarget<ARMSubtarget>().getInstrInfo(); 221 MachineBasicBlock *Entry = &MF.front(); 222 Entry->clear(); 223 224 // These thunks need to consist of the following instructions: 225 // __llvm_slsblr_thunk_(arm/thumb)_rN: 226 // bx rN 227 // barrierInsts 228 Entry->addLiveIn(ThunkReg); 229 if (isThumb) 230 BuildMI(Entry, DebugLoc(), TII->get(ARM::tBX)) 231 .addReg(ThunkReg) 232 .add(predOps(ARMCC::AL)); 233 else 234 BuildMI(Entry, DebugLoc(), TII->get(ARM::BX)) 235 .addReg(ThunkReg); 236 237 // Make sure the thunks do not make use of the SB extension in case there is 238 // a function somewhere that will call to it that for some reason disabled 239 // the SB extension locally on that function, even though it's enabled for 240 // the module otherwise. Therefore set AlwaysUseISBSDB to true. 241 insertSpeculationBarrier(&MF.getSubtarget<ARMSubtarget>(), *Entry, 242 Entry->end(), DebugLoc(), true /*AlwaysUseISBDSB*/); 243 } 244 245 MachineBasicBlock &ARMSLSHardening::ConvertIndirectCallToIndirectJump( 246 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { 247 // Transform an indirect call to an indirect jump as follows: 248 // Before: 249 // |-----------------------------| 250 // | ... | 251 // | instI | 252 // | BLX rN | 253 // | instJ | 254 // | ... | 255 // |-----------------------------| 256 // 257 // After: 258 // |---------- -------------------------| 259 // | ... | 260 // | instI | 261 // | *call* __llvm_slsblr_thunk_mode_xN | 262 // | instJ | 263 // | ... | 264 // |--------------------------------------| 265 // 266 // __llvm_slsblr_thunk_mode_xN: 267 // |-----------------------------| 268 // | BX rN | 269 // | barrierInsts | 270 // |-----------------------------| 271 // 272 // The __llvm_slsblr_thunk_mode_xN thunks are created by the 273 // SLSBLRThunkInserter. 274 // This function merely needs to transform an indirect call to a direct call 275 // to __llvm_slsblr_thunk_xN. 276 MachineInstr &IndirectCall = *MBBI; 277 assert(isIndirectCall(IndirectCall) && !IndirectCall.isReturn()); 278 int RegOpIdxOnIndirectCall = -1; 279 bool isThumb; 280 switch (IndirectCall.getOpcode()) { 281 case ARM::BLX: // !isThumb2 282 case ARM::BLX_noip: // !isThumb2 283 isThumb = false; 284 RegOpIdxOnIndirectCall = 0; 285 break; 286 case ARM::tBLXr: // isThumb2 287 case ARM::tBLXr_noip: // isThumb2 288 isThumb = true; 289 RegOpIdxOnIndirectCall = 2; 290 break; 291 default: 292 llvm_unreachable("unhandled Indirect Call"); 293 } 294 295 Register Reg = IndirectCall.getOperand(RegOpIdxOnIndirectCall).getReg(); 296 // Since linkers are allowed to clobber R12 on function calls, the above 297 // mitigation only works if the original indirect call instruction was not 298 // using R12. Code generation before must make sure that no indirect call 299 // using R12 was produced if the mitigation is enabled. 300 // Also, the transformation is incorrect if the indirect call uses LR, so 301 // also have to avoid that. 302 assert(Reg != ARM::R12 && Reg != ARM::LR); 303 bool RegIsKilled = IndirectCall.getOperand(RegOpIdxOnIndirectCall).isKill(); 304 305 DebugLoc DL = IndirectCall.getDebugLoc(); 306 307 MachineFunction &MF = *MBBI->getMF(); 308 auto ThunkIt = llvm::find_if(SLSBLRThunks, [Reg, isThumb](auto T) { 309 return T.Reg == Reg && T.isThumb == isThumb; 310 }); 311 assert(ThunkIt != std::end(SLSBLRThunks)); 312 Module *M = MF.getFunction().getParent(); 313 const GlobalValue *GV = cast<GlobalValue>(M->getNamedValue(ThunkIt->Name)); 314 315 MachineInstr *BL = 316 isThumb ? BuildMI(MBB, MBBI, DL, TII->get(ARM::tBL)) 317 .addImm(IndirectCall.getOperand(0).getImm()) 318 .addReg(IndirectCall.getOperand(1).getReg()) 319 .addGlobalAddress(GV) 320 : BuildMI(MBB, MBBI, DL, TII->get(ARM::BL)).addGlobalAddress(GV); 321 322 // Now copy the implicit operands from IndirectCall to BL and copy other 323 // necessary info. 324 // However, both IndirectCall and BL instructions implictly use SP and 325 // implicitly define LR. Blindly copying implicit operands would result in SP 326 // and LR operands to be present multiple times. While this may not be too 327 // much of an issue, let's avoid that for cleanliness, by removing those 328 // implicit operands from the BL created above before we copy over all 329 // implicit operands from the IndirectCall. 330 int ImpLROpIdx = -1; 331 int ImpSPOpIdx = -1; 332 for (unsigned OpIdx = BL->getNumExplicitOperands(); 333 OpIdx < BL->getNumOperands(); OpIdx++) { 334 MachineOperand Op = BL->getOperand(OpIdx); 335 if (!Op.isReg()) 336 continue; 337 if (Op.getReg() == ARM::LR && Op.isDef()) 338 ImpLROpIdx = OpIdx; 339 if (Op.getReg() == ARM::SP && !Op.isDef()) 340 ImpSPOpIdx = OpIdx; 341 } 342 assert(ImpLROpIdx != -1); 343 assert(ImpSPOpIdx != -1); 344 int FirstOpIdxToRemove = std::max(ImpLROpIdx, ImpSPOpIdx); 345 int SecondOpIdxToRemove = std::min(ImpLROpIdx, ImpSPOpIdx); 346 BL->removeOperand(FirstOpIdxToRemove); 347 BL->removeOperand(SecondOpIdxToRemove); 348 // Now copy over the implicit operands from the original IndirectCall 349 BL->copyImplicitOps(MF, IndirectCall); 350 MF.moveCallSiteInfo(&IndirectCall, BL); 351 // Also add the register called in the IndirectCall as being used in the 352 // called thunk. 353 BL->addOperand(MachineOperand::CreateReg(Reg, false /*isDef*/, true /*isImp*/, 354 RegIsKilled /*isKill*/)); 355 // Remove IndirectCallinstruction 356 MBB.erase(MBBI); 357 return MBB; 358 } 359 360 bool ARMSLSHardening::hardenIndirectCalls(MachineBasicBlock &MBB) const { 361 if (!ST->hardenSlsBlr()) 362 return false; 363 bool Modified = false; 364 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); 365 MachineBasicBlock::iterator NextMBBI; 366 for (; MBBI != E; MBBI = NextMBBI) { 367 MachineInstr &MI = *MBBI; 368 NextMBBI = std::next(MBBI); 369 // Tail calls are both indirect calls and "returns". 370 // They are also indirect jumps, so should be handled by sls-harden-retbr, 371 // rather than sls-harden-blr. 372 if (isIndirectCall(MI) && !MI.isReturn()) { 373 ConvertIndirectCallToIndirectJump(MBB, MBBI); 374 Modified = true; 375 } 376 } 377 return Modified; 378 } 379 380 381 382 FunctionPass *llvm::createARMSLSHardeningPass() { 383 return new ARMSLSHardening(); 384 } 385 386 namespace { 387 class ARMIndirectThunks : public MachineFunctionPass { 388 public: 389 static char ID; 390 391 ARMIndirectThunks() : MachineFunctionPass(ID) {} 392 393 StringRef getPassName() const override { return "ARM Indirect Thunks"; } 394 395 bool doInitialization(Module &M) override; 396 bool runOnMachineFunction(MachineFunction &MF) override; 397 398 void getAnalysisUsage(AnalysisUsage &AU) const override { 399 MachineFunctionPass::getAnalysisUsage(AU); 400 AU.addRequired<MachineModuleInfoWrapperPass>(); 401 AU.addPreserved<MachineModuleInfoWrapperPass>(); 402 } 403 404 private: 405 std::tuple<SLSBLRThunkInserter> TIs; 406 407 // FIXME: When LLVM moves to C++17, these can become folds 408 template <typename... ThunkInserterT> 409 static void initTIs(Module &M, 410 std::tuple<ThunkInserterT...> &ThunkInserters) { 411 (void)std::initializer_list<int>{ 412 (std::get<ThunkInserterT>(ThunkInserters).init(M), 0)...}; 413 } 414 template <typename... ThunkInserterT> 415 static bool runTIs(MachineModuleInfo &MMI, MachineFunction &MF, 416 std::tuple<ThunkInserterT...> &ThunkInserters) { 417 bool Modified = false; 418 (void)std::initializer_list<int>{ 419 Modified |= std::get<ThunkInserterT>(ThunkInserters).run(MMI, MF)...}; 420 return Modified; 421 } 422 }; 423 424 } // end anonymous namespace 425 426 char ARMIndirectThunks::ID = 0; 427 428 FunctionPass *llvm::createARMIndirectThunks() { 429 return new ARMIndirectThunks(); 430 } 431 432 bool ARMIndirectThunks::doInitialization(Module &M) { 433 initTIs(M, TIs); 434 return false; 435 } 436 437 bool ARMIndirectThunks::runOnMachineFunction(MachineFunction &MF) { 438 LLVM_DEBUG(dbgs() << getPassName() << '\n'); 439 auto &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI(); 440 return runTIs(MMI, MF, TIs); 441 } 442