1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_ 4 #define _ASM_X86_NOSPEC_BRANCH_H_ 5 6 #include <linux/static_key.h> 7 8 #include <asm/alternative.h> 9 #include <asm/alternative-asm.h> 10 #include <asm/cpufeatures.h> 11 #include <asm/msr-index.h> 12 13 /* 14 * This should be used immediately before a retpoline alternative. It tells 15 * objtool where the retpolines are so that it can make sense of the control 16 * flow by just reading the original instruction(s) and ignoring the 17 * alternatives. 18 */ 19 #define ANNOTATE_NOSPEC_ALTERNATIVE \ 20 ANNOTATE_IGNORE_ALTERNATIVE 21 22 /* 23 * Fill the CPU return stack buffer. 24 * 25 * Each entry in the RSB, if used for a speculative 'ret', contains an 26 * infinite 'pause; lfence; jmp' loop to capture speculative execution. 27 * 28 * This is required in various cases for retpoline and IBRS-based 29 * mitigations for the Spectre variant 2 vulnerability. Sometimes to 30 * eliminate potentially bogus entries from the RSB, and sometimes 31 * purely to ensure that it doesn't get empty, which on some CPUs would 32 * allow predictions from other (unwanted!) sources to be used. 33 * 34 * We define a CPP macro such that it can be used from both .S files and 35 * inline assembly. It's possible to do a .macro and then include that 36 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. 37 */ 38 39 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ 40 #define RSB_FILL_LOOPS 16 /* To avoid underflow */ 41 42 /* 43 * Google experimented with loop-unrolling and this turned out to be 44 * the optimal version — two calls, each with their own speculation 45 * trap should their return address end up getting used, in a loop. 46 */ 47 #define __FILL_RETURN_BUFFER(reg, nr, sp) \ 48 mov $(nr/2), reg; \ 49 771: \ 50 call 772f; \ 51 773: /* speculation trap */ \ 52 pause; \ 53 lfence; \ 54 jmp 773b; \ 55 772: \ 56 call 774f; \ 57 775: /* speculation trap */ \ 58 pause; \ 59 lfence; \ 60 jmp 775b; \ 61 774: \ 62 dec reg; \ 63 jnz 771b; \ 64 add $(BITS_PER_LONG/8) * nr, sp; 65 66 #ifdef __ASSEMBLY__ 67 68 /* 69 * This should be used immediately before an indirect jump/call. It tells 70 * objtool the subsequent indirect jump/call is vouched safe for retpoline 71 * builds. 72 */ 73 .macro ANNOTATE_RETPOLINE_SAFE 74 .Lannotate_\@: 75 .pushsection .discard.retpoline_safe 76 _ASM_PTR .Lannotate_\@ 77 .popsection 78 .endm 79 80 /* 81 * These are the bare retpoline primitives for indirect jmp and call. 82 * Do not use these directly; they only exist to make the ALTERNATIVE 83 * invocation below less ugly. 84 */ 85 .macro RETPOLINE_JMP reg:req 86 call .Ldo_rop_\@ 87 .Lspec_trap_\@: 88 pause 89 lfence 90 jmp .Lspec_trap_\@ 91 .Ldo_rop_\@: 92 mov \reg, (%_ASM_SP) 93 ret 94 .endm 95 96 /* 97 * This is a wrapper around RETPOLINE_JMP so the called function in reg 98 * returns to the instruction after the macro. 99 */ 100 .macro RETPOLINE_CALL reg:req 101 jmp .Ldo_call_\@ 102 .Ldo_retpoline_jmp_\@: 103 RETPOLINE_JMP \reg 104 .Ldo_call_\@: 105 call .Ldo_retpoline_jmp_\@ 106 .endm 107 108 /* 109 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple 110 * indirect jmp/call which may be susceptible to the Spectre variant 2 111 * attack. 112 */ 113 .macro JMP_NOSPEC reg:req 114 #ifdef CONFIG_RETPOLINE 115 ANNOTATE_NOSPEC_ALTERNATIVE 116 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ 117 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ 118 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD 119 #else 120 jmp *\reg 121 #endif 122 .endm 123 124 .macro CALL_NOSPEC reg:req 125 #ifdef CONFIG_RETPOLINE 126 ANNOTATE_NOSPEC_ALTERNATIVE 127 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ 128 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ 129 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD 130 #else 131 call *\reg 132 #endif 133 .endm 134 135 /* 136 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP 137 * monstrosity above, manually. 138 */ 139 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req 140 #ifdef CONFIG_RETPOLINE 141 ANNOTATE_NOSPEC_ALTERNATIVE 142 ALTERNATIVE "jmp .Lskip_rsb_\@", \ 143 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ 144 \ftr 145 .Lskip_rsb_\@: 146 #endif 147 .endm 148 149 #else /* __ASSEMBLY__ */ 150 151 #define ANNOTATE_RETPOLINE_SAFE \ 152 "999:\n\t" \ 153 ".pushsection .discard.retpoline_safe\n\t" \ 154 _ASM_PTR " 999b\n\t" \ 155 ".popsection\n\t" 156 157 #ifdef CONFIG_RETPOLINE 158 #ifdef CONFIG_X86_64 159 160 /* 161 * Inline asm uses the %V modifier which is only in newer GCC 162 * which is ensured when CONFIG_RETPOLINE is defined. 163 */ 164 # define CALL_NOSPEC \ 165 ANNOTATE_NOSPEC_ALTERNATIVE \ 166 ALTERNATIVE_2( \ 167 ANNOTATE_RETPOLINE_SAFE \ 168 "call *%[thunk_target]\n", \ 169 "call __x86_indirect_thunk_%V[thunk_target]\n", \ 170 X86_FEATURE_RETPOLINE, \ 171 "lfence;\n" \ 172 ANNOTATE_RETPOLINE_SAFE \ 173 "call *%[thunk_target]\n", \ 174 X86_FEATURE_RETPOLINE_AMD) 175 # define THUNK_TARGET(addr) [thunk_target] "r" (addr) 176 177 #else /* CONFIG_X86_32 */ 178 /* 179 * For i386 we use the original ret-equivalent retpoline, because 180 * otherwise we'll run out of registers. We don't care about CET 181 * here, anyway. 182 */ 183 # define CALL_NOSPEC \ 184 ANNOTATE_NOSPEC_ALTERNATIVE \ 185 ALTERNATIVE_2( \ 186 ANNOTATE_RETPOLINE_SAFE \ 187 "call *%[thunk_target]\n", \ 188 " jmp 904f;\n" \ 189 " .align 16\n" \ 190 "901: call 903f;\n" \ 191 "902: pause;\n" \ 192 " lfence;\n" \ 193 " jmp 902b;\n" \ 194 " .align 16\n" \ 195 "903: lea 4(%%esp), %%esp;\n" \ 196 " pushl %[thunk_target];\n" \ 197 " ret;\n" \ 198 " .align 16\n" \ 199 "904: call 901b;\n", \ 200 X86_FEATURE_RETPOLINE, \ 201 "lfence;\n" \ 202 ANNOTATE_RETPOLINE_SAFE \ 203 "call *%[thunk_target]\n", \ 204 X86_FEATURE_RETPOLINE_AMD) 205 206 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 207 #endif 208 #else /* No retpoline for C / inline asm */ 209 # define CALL_NOSPEC "call *%[thunk_target]\n" 210 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 211 #endif 212 213 /* The Spectre V2 mitigation variants */ 214 enum spectre_v2_mitigation { 215 SPECTRE_V2_NONE, 216 SPECTRE_V2_RETPOLINE_GENERIC, 217 SPECTRE_V2_RETPOLINE_AMD, 218 SPECTRE_V2_IBRS_ENHANCED, 219 }; 220 221 /* The indirect branch speculation control variants */ 222 enum spectre_v2_user_mitigation { 223 SPECTRE_V2_USER_NONE, 224 SPECTRE_V2_USER_STRICT, 225 SPECTRE_V2_USER_STRICT_PREFERRED, 226 SPECTRE_V2_USER_PRCTL, 227 SPECTRE_V2_USER_SECCOMP, 228 }; 229 230 /* The Speculative Store Bypass disable variants */ 231 enum ssb_mitigation { 232 SPEC_STORE_BYPASS_NONE, 233 SPEC_STORE_BYPASS_DISABLE, 234 SPEC_STORE_BYPASS_PRCTL, 235 SPEC_STORE_BYPASS_SECCOMP, 236 }; 237 238 extern char __indirect_thunk_start[]; 239 extern char __indirect_thunk_end[]; 240 241 /* 242 * On VMEXIT we must ensure that no RSB predictions learned in the guest 243 * can be followed in the host, by overwriting the RSB completely. Both 244 * retpoline and IBRS mitigations for Spectre v2 need this; only on future 245 * CPUs with IBRS_ALL *might* it be avoided. 246 */ 247 static inline void vmexit_fill_RSB(void) 248 { 249 #ifdef CONFIG_RETPOLINE 250 unsigned long loops; 251 252 asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE 253 ALTERNATIVE("jmp 910f", 254 __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), 255 X86_FEATURE_RETPOLINE) 256 "910:" 257 : "=r" (loops), ASM_CALL_CONSTRAINT 258 : : "memory" ); 259 #endif 260 } 261 262 static __always_inline 263 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) 264 { 265 asm volatile(ALTERNATIVE("", "wrmsr", %c[feature]) 266 : : "c" (msr), 267 "a" ((u32)val), 268 "d" ((u32)(val >> 32)), 269 [feature] "i" (feature) 270 : "memory"); 271 } 272 273 static inline void indirect_branch_prediction_barrier(void) 274 { 275 u64 val = PRED_CMD_IBPB; 276 277 alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB); 278 } 279 280 /* The Intel SPEC CTRL MSR base value cache */ 281 extern u64 x86_spec_ctrl_base; 282 283 /* 284 * With retpoline, we must use IBRS to restrict branch prediction 285 * before calling into firmware. 286 * 287 * (Implemented as CPP macros due to header hell.) 288 */ 289 #define firmware_restrict_branch_speculation_start() \ 290 do { \ 291 u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \ 292 \ 293 preempt_disable(); \ 294 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ 295 X86_FEATURE_USE_IBRS_FW); \ 296 } while (0) 297 298 #define firmware_restrict_branch_speculation_end() \ 299 do { \ 300 u64 val = x86_spec_ctrl_base; \ 301 \ 302 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ 303 X86_FEATURE_USE_IBRS_FW); \ 304 preempt_enable(); \ 305 } while (0) 306 307 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp); 308 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 309 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 310 311 DECLARE_STATIC_KEY_FALSE(mds_user_clear); 312 DECLARE_STATIC_KEY_FALSE(mds_idle_clear); 313 314 #include <asm/segment.h> 315 316 /** 317 * mds_clear_cpu_buffers - Mitigation for MDS vulnerability 318 * 319 * This uses the otherwise unused and obsolete VERW instruction in 320 * combination with microcode which triggers a CPU buffer flush when the 321 * instruction is executed. 322 */ 323 static inline void mds_clear_cpu_buffers(void) 324 { 325 static const u16 ds = __KERNEL_DS; 326 327 /* 328 * Has to be the memory-operand variant because only that 329 * guarantees the CPU buffer flush functionality according to 330 * documentation. The register-operand variant does not. 331 * Works with any segment selector, but a valid writable 332 * data segment is the fastest variant. 333 * 334 * "cc" clobber is required because VERW modifies ZF. 335 */ 336 asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc"); 337 } 338 339 /** 340 * mds_user_clear_cpu_buffers - Mitigation for MDS vulnerability 341 * 342 * Clear CPU buffers if the corresponding static key is enabled 343 */ 344 static inline void mds_user_clear_cpu_buffers(void) 345 { 346 if (static_branch_likely(&mds_user_clear)) 347 mds_clear_cpu_buffers(); 348 } 349 350 /** 351 * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability 352 * 353 * Clear CPU buffers if the corresponding static key is enabled 354 */ 355 static inline void mds_idle_clear_cpu_buffers(void) 356 { 357 if (static_branch_likely(&mds_idle_clear)) 358 mds_clear_cpu_buffers(); 359 } 360 361 #endif /* __ASSEMBLY__ */ 362 363 /* 364 * Below is used in the eBPF JIT compiler and emits the byte sequence 365 * for the following assembly: 366 * 367 * With retpolines configured: 368 * 369 * callq do_rop 370 * spec_trap: 371 * pause 372 * lfence 373 * jmp spec_trap 374 * do_rop: 375 * mov %rax,(%rsp) for x86_64 376 * mov %edx,(%esp) for x86_32 377 * retq 378 * 379 * Without retpolines configured: 380 * 381 * jmp *%rax for x86_64 382 * jmp *%edx for x86_32 383 */ 384 #ifdef CONFIG_RETPOLINE 385 # ifdef CONFIG_X86_64 386 # define RETPOLINE_RAX_BPF_JIT_SIZE 17 387 # define RETPOLINE_RAX_BPF_JIT() \ 388 do { \ 389 EMIT1_off32(0xE8, 7); /* callq do_rop */ \ 390 /* spec_trap: */ \ 391 EMIT2(0xF3, 0x90); /* pause */ \ 392 EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ 393 EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ 394 /* do_rop: */ \ 395 EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ 396 EMIT1(0xC3); /* retq */ \ 397 } while (0) 398 # else /* !CONFIG_X86_64 */ 399 # define RETPOLINE_EDX_BPF_JIT() \ 400 do { \ 401 EMIT1_off32(0xE8, 7); /* call do_rop */ \ 402 /* spec_trap: */ \ 403 EMIT2(0xF3, 0x90); /* pause */ \ 404 EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ 405 EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ 406 /* do_rop: */ \ 407 EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */ \ 408 EMIT1(0xC3); /* ret */ \ 409 } while (0) 410 # endif 411 #else /* !CONFIG_RETPOLINE */ 412 # ifdef CONFIG_X86_64 413 # define RETPOLINE_RAX_BPF_JIT_SIZE 2 414 # define RETPOLINE_RAX_BPF_JIT() \ 415 EMIT2(0xFF, 0xE0); /* jmp *%rax */ 416 # else /* !CONFIG_X86_64 */ 417 # define RETPOLINE_EDX_BPF_JIT() \ 418 EMIT2(0xFF, 0xE2) /* jmp *%edx */ 419 # endif 420 #endif 421 422 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ 423