1 //==- X86IndirectThunks.cpp - Construct indirect call/jump thunks for x86 --=// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// 10 /// Pass that injects an MI thunk that is used to lower indirect calls in a way 11 /// that prevents speculation on some x86 processors and can be used to mitigate 12 /// security vulnerabilities due to targeted speculative execution and side 13 /// channels such as CVE-2017-5715. 14 /// 15 /// Currently supported thunks include: 16 /// - Retpoline -- A RET-implemented trampoline that lowers indirect calls 17 /// - LVI Thunk -- A CALL/JMP-implemented thunk that forces load serialization 18 /// before making an indirect call/jump 19 /// 20 /// Note that the reason that this is implemented as a MachineFunctionPass and 21 /// not a ModulePass is that ModulePasses at this point in the LLVM X86 pipeline 22 /// serialize all transformations, which can consume lots of memory. 23 /// 24 /// TODO(chandlerc): All of this code could use better comments and 25 /// documentation. 26 /// 27 //===----------------------------------------------------------------------===// 28 29 #include "X86.h" 30 #include "X86InstrBuilder.h" 31 #include "X86Subtarget.h" 32 #include "llvm/CodeGen/IndirectThunks.h" 33 #include "llvm/CodeGen/MachineFunction.h" 34 #include "llvm/CodeGen/MachineFunctionPass.h" 35 #include "llvm/CodeGen/MachineInstrBuilder.h" 36 #include "llvm/CodeGen/MachineModuleInfo.h" 37 #include "llvm/CodeGen/Passes.h" 38 #include "llvm/CodeGen/TargetPassConfig.h" 39 #include "llvm/IR/IRBuilder.h" 40 #include "llvm/IR/Instructions.h" 41 #include "llvm/IR/Module.h" 42 #include "llvm/Support/CommandLine.h" 43 #include "llvm/Support/Debug.h" 44 #include "llvm/Support/raw_ostream.h" 45 #include "llvm/Target/TargetMachine.h" 46 47 using namespace llvm; 48 49 #define DEBUG_TYPE "x86-retpoline-thunks" 50 51 static const char RetpolineNamePrefix[] = "__llvm_retpoline_"; 52 static const char R11RetpolineName[] = "__llvm_retpoline_r11"; 53 static const char EAXRetpolineName[] = "__llvm_retpoline_eax"; 54 static const char ECXRetpolineName[] = "__llvm_retpoline_ecx"; 55 static const char EDXRetpolineName[] = "__llvm_retpoline_edx"; 56 static const char EDIRetpolineName[] = "__llvm_retpoline_edi"; 57 58 static const char LVIThunkNamePrefix[] = "__llvm_lvi_thunk_"; 59 static const char R11LVIThunkName[] = "__llvm_lvi_thunk_r11"; 60 61 namespace { 62 struct RetpolineThunkInserter : ThunkInserter<RetpolineThunkInserter> { 63 const char *getThunkPrefix() { return RetpolineNamePrefix; } 64 bool mayUseThunk(const MachineFunction &MF, bool InsertedThunks) { 65 if (InsertedThunks) 66 return false; 67 const auto &STI = MF.getSubtarget<X86Subtarget>(); 68 return (STI.useRetpolineIndirectCalls() || 69 STI.useRetpolineIndirectBranches()) && 70 !STI.useRetpolineExternalThunk(); 71 } 72 bool insertThunks(MachineModuleInfo &MMI, MachineFunction &MF); 73 void populateThunk(MachineFunction &MF); 74 }; 75 76 struct LVIThunkInserter : ThunkInserter<LVIThunkInserter> { 77 const char *getThunkPrefix() { return LVIThunkNamePrefix; } 78 bool mayUseThunk(const MachineFunction &MF, bool InsertedThunks) { 79 if (InsertedThunks) 80 return false; 81 return MF.getSubtarget<X86Subtarget>().useLVIControlFlowIntegrity(); 82 } 83 bool insertThunks(MachineModuleInfo &MMI, MachineFunction &MF) { 84 createThunkFunction(MMI, R11LVIThunkName); 85 return true; 86 } 87 void populateThunk(MachineFunction &MF) { 88 assert (MF.size() == 1); 89 MachineBasicBlock *Entry = &MF.front(); 90 Entry->clear(); 91 92 // This code mitigates LVI by replacing each indirect call/jump with a 93 // direct call/jump to a thunk that looks like: 94 // ``` 95 // lfence 96 // jmpq *%r11 97 // ``` 98 // This ensures that if the value in register %r11 was loaded from memory, 99 // then the value in %r11 is (architecturally) correct prior to the jump. 100 const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo(); 101 BuildMI(&MF.front(), DebugLoc(), TII->get(X86::LFENCE)); 102 BuildMI(&MF.front(), DebugLoc(), TII->get(X86::JMP64r)).addReg(X86::R11); 103 MF.front().addLiveIn(X86::R11); 104 } 105 }; 106 107 class X86IndirectThunks : public MachineFunctionPass { 108 public: 109 static char ID; 110 111 X86IndirectThunks() : MachineFunctionPass(ID) {} 112 113 StringRef getPassName() const override { return "X86 Indirect Thunks"; } 114 115 bool doInitialization(Module &M) override; 116 bool runOnMachineFunction(MachineFunction &MF) override; 117 118 private: 119 std::tuple<RetpolineThunkInserter, LVIThunkInserter> TIs; 120 121 // FIXME: When LLVM moves to C++17, these can become folds 122 template <typename... ThunkInserterT> 123 static void initTIs(Module &M, 124 std::tuple<ThunkInserterT...> &ThunkInserters) { 125 (void)std::initializer_list<int>{ 126 (std::get<ThunkInserterT>(ThunkInserters).init(M), 0)...}; 127 } 128 template <typename... ThunkInserterT> 129 static bool runTIs(MachineModuleInfo &MMI, MachineFunction &MF, 130 std::tuple<ThunkInserterT...> &ThunkInserters) { 131 bool Modified = false; 132 (void)std::initializer_list<int>{ 133 Modified |= std::get<ThunkInserterT>(ThunkInserters).run(MMI, MF)...}; 134 return Modified; 135 } 136 }; 137 138 } // end anonymous namespace 139 140 bool RetpolineThunkInserter::insertThunks(MachineModuleInfo &MMI, 141 MachineFunction &MF) { 142 if (MMI.getTarget().getTargetTriple().getArch() == Triple::x86_64) 143 createThunkFunction(MMI, R11RetpolineName); 144 else 145 for (StringRef Name : {EAXRetpolineName, ECXRetpolineName, EDXRetpolineName, 146 EDIRetpolineName}) 147 createThunkFunction(MMI, Name); 148 return true; 149 } 150 151 void RetpolineThunkInserter::populateThunk(MachineFunction &MF) { 152 bool Is64Bit = MF.getTarget().getTargetTriple().getArch() == Triple::x86_64; 153 Register ThunkReg; 154 if (Is64Bit) { 155 assert(MF.getName() == "__llvm_retpoline_r11" && 156 "Should only have an r11 thunk on 64-bit targets"); 157 158 // __llvm_retpoline_r11: 159 // callq .Lr11_call_target 160 // .Lr11_capture_spec: 161 // pause 162 // lfence 163 // jmp .Lr11_capture_spec 164 // .align 16 165 // .Lr11_call_target: 166 // movq %r11, (%rsp) 167 // retq 168 ThunkReg = X86::R11; 169 } else { 170 // For 32-bit targets we need to emit a collection of thunks for various 171 // possible scratch registers as well as a fallback that uses EDI, which is 172 // normally callee saved. 173 // __llvm_retpoline_eax: 174 // calll .Leax_call_target 175 // .Leax_capture_spec: 176 // pause 177 // jmp .Leax_capture_spec 178 // .align 16 179 // .Leax_call_target: 180 // movl %eax, (%esp) # Clobber return addr 181 // retl 182 // 183 // __llvm_retpoline_ecx: 184 // ... # Same setup 185 // movl %ecx, (%esp) 186 // retl 187 // 188 // __llvm_retpoline_edx: 189 // ... # Same setup 190 // movl %edx, (%esp) 191 // retl 192 // 193 // __llvm_retpoline_edi: 194 // ... # Same setup 195 // movl %edi, (%esp) 196 // retl 197 if (MF.getName() == EAXRetpolineName) 198 ThunkReg = X86::EAX; 199 else if (MF.getName() == ECXRetpolineName) 200 ThunkReg = X86::ECX; 201 else if (MF.getName() == EDXRetpolineName) 202 ThunkReg = X86::EDX; 203 else if (MF.getName() == EDIRetpolineName) 204 ThunkReg = X86::EDI; 205 else 206 llvm_unreachable("Invalid thunk name on x86-32!"); 207 } 208 209 const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo(); 210 assert (MF.size() == 1); 211 MachineBasicBlock *Entry = &MF.front(); 212 Entry->clear(); 213 214 MachineBasicBlock *CaptureSpec = 215 MF.CreateMachineBasicBlock(Entry->getBasicBlock()); 216 MachineBasicBlock *CallTarget = 217 MF.CreateMachineBasicBlock(Entry->getBasicBlock()); 218 MCSymbol *TargetSym = MF.getContext().createTempSymbol(); 219 MF.push_back(CaptureSpec); 220 MF.push_back(CallTarget); 221 222 const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32; 223 const unsigned RetOpc = Is64Bit ? X86::RET64 : X86::RET32; 224 225 Entry->addLiveIn(ThunkReg); 226 BuildMI(Entry, DebugLoc(), TII->get(CallOpc)).addSym(TargetSym); 227 228 // The MIR verifier thinks that the CALL in the entry block will fall through 229 // to CaptureSpec, so mark it as the successor. Technically, CaptureTarget is 230 // the successor, but the MIR verifier doesn't know how to cope with that. 231 Entry->addSuccessor(CaptureSpec); 232 233 // In the capture loop for speculation, we want to stop the processor from 234 // speculating as fast as possible. On Intel processors, the PAUSE instruction 235 // will block speculation without consuming any execution resources. On AMD 236 // processors, the PAUSE instruction is (essentially) a nop, so we also use an 237 // LFENCE instruction which they have advised will stop speculation as well 238 // with minimal resource utilization. We still end the capture with a jump to 239 // form an infinite loop to fully guarantee that no matter what implementation 240 // of the x86 ISA, speculating this code path never escapes. 241 BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::PAUSE)); 242 BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::LFENCE)); 243 BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::JMP_1)).addMBB(CaptureSpec); 244 CaptureSpec->setMachineBlockAddressTaken(); 245 CaptureSpec->addSuccessor(CaptureSpec); 246 247 CallTarget->addLiveIn(ThunkReg); 248 CallTarget->setMachineBlockAddressTaken(); 249 CallTarget->setAlignment(Align(16)); 250 251 // Insert return address clobber 252 const unsigned MovOpc = Is64Bit ? X86::MOV64mr : X86::MOV32mr; 253 const Register SPReg = Is64Bit ? X86::RSP : X86::ESP; 254 addRegOffset(BuildMI(CallTarget, DebugLoc(), TII->get(MovOpc)), SPReg, false, 255 0) 256 .addReg(ThunkReg); 257 258 CallTarget->back().setPreInstrSymbol(MF, TargetSym); 259 BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc)); 260 } 261 262 FunctionPass *llvm::createX86IndirectThunksPass() { 263 return new X86IndirectThunks(); 264 } 265 266 char X86IndirectThunks::ID = 0; 267 268 bool X86IndirectThunks::doInitialization(Module &M) { 269 initTIs(M, TIs); 270 return false; 271 } 272 273 bool X86IndirectThunks::runOnMachineFunction(MachineFunction &MF) { 274 LLVM_DEBUG(dbgs() << getPassName() << '\n'); 275 auto &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI(); 276 return runTIs(MMI, MF, TIs); 277 } 278