1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
4 #define _ASM_X86_NOSPEC_BRANCH_H_
5
6 #include <linux/static_key.h>
7 #include <linux/objtool.h>
8 #include <linux/linkage.h>
9 #include <linux/types.h>
10
11 #include <asm/alternative.h>
12 #include <asm/cpufeatures.h>
13 #include <asm/msr-index.h>
14 #include <asm/unwind_hints.h>
15 #include <asm/percpu.h>
16 #include <asm/ptrace-abi.h>
17
18 /*
19 * Call depth tracking for Intel SKL CPUs to address the RSB underflow
20 * issue in software.
21 *
22 * The tracking does not use a counter. It uses uses arithmetic shift
23 * right on call entry and logical shift left on return.
24 *
25 * The depth tracking variable is initialized to 0x8000.... when the call
26 * depth is zero. The arithmetic shift right sign extends the MSB and
27 * saturates after the 12th call. The shift count is 5 for both directions
28 * so the tracking covers 12 nested calls.
29 *
30 * Call
31 * 0: 0x8000000000000000 0x0000000000000000
32 * 1: 0xfc00000000000000 0xf000000000000000
33 * ...
34 * 11: 0xfffffffffffffff8 0xfffffffffffffc00
35 * 12: 0xffffffffffffffff 0xffffffffffffffe0
36 *
37 * After a return buffer fill the depth is credited 12 calls before the
38 * next stuffing has to take place.
39 *
40 * There is a inaccuracy for situations like this:
41 *
42 * 10 calls
43 * 5 returns
44 * 3 calls
45 * 4 returns
46 * 3 calls
47 * ....
48 *
49 * The shift count might cause this to be off by one in either direction,
50 * but there is still a cushion vs. the RSB depth. The algorithm does not
51 * claim to be perfect and it can be speculated around by the CPU, but it
52 * is considered that it obfuscates the problem enough to make exploitation
53 * extremely difficult.
54 */
55 #define RET_DEPTH_SHIFT 5
56 #define RSB_RET_STUFF_LOOPS 16
57 #define RET_DEPTH_INIT 0x8000000000000000ULL
58 #define RET_DEPTH_INIT_FROM_CALL 0xfc00000000000000ULL
59 #define RET_DEPTH_CREDIT 0xffffffffffffffffULL
60
61 #ifdef CONFIG_CALL_THUNKS_DEBUG
62 # define CALL_THUNKS_DEBUG_INC_CALLS \
63 incq PER_CPU_VAR(__x86_call_count);
64 # define CALL_THUNKS_DEBUG_INC_RETS \
65 incq PER_CPU_VAR(__x86_ret_count);
66 # define CALL_THUNKS_DEBUG_INC_STUFFS \
67 incq PER_CPU_VAR(__x86_stuffs_count);
68 # define CALL_THUNKS_DEBUG_INC_CTXSW \
69 incq PER_CPU_VAR(__x86_ctxsw_count);
70 #else
71 # define CALL_THUNKS_DEBUG_INC_CALLS
72 # define CALL_THUNKS_DEBUG_INC_RETS
73 # define CALL_THUNKS_DEBUG_INC_STUFFS
74 # define CALL_THUNKS_DEBUG_INC_CTXSW
75 #endif
76
77 #if defined(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) && !defined(COMPILE_OFFSETS)
78
79 #include <asm/asm-offsets.h>
80
81 #define CREDIT_CALL_DEPTH \
82 movq $-1, PER_CPU_VAR(__x86_call_depth);
83
84 #define RESET_CALL_DEPTH \
85 xor %eax, %eax; \
86 bts $63, %rax; \
87 movq %rax, PER_CPU_VAR(__x86_call_depth);
88
89 #define RESET_CALL_DEPTH_FROM_CALL \
90 movb $0xfc, %al; \
91 shl $56, %rax; \
92 movq %rax, PER_CPU_VAR(__x86_call_depth); \
93 CALL_THUNKS_DEBUG_INC_CALLS
94
95 #define INCREMENT_CALL_DEPTH \
96 sarq $5, PER_CPU_VAR(__x86_call_depth); \
97 CALL_THUNKS_DEBUG_INC_CALLS
98
99 #else
100 #define CREDIT_CALL_DEPTH
101 #define RESET_CALL_DEPTH
102 #define RESET_CALL_DEPTH_FROM_CALL
103 #define INCREMENT_CALL_DEPTH
104 #endif
105
106 /*
107 * Fill the CPU return stack buffer.
108 *
109 * Each entry in the RSB, if used for a speculative 'ret', contains an
110 * infinite 'pause; lfence; jmp' loop to capture speculative execution.
111 *
112 * This is required in various cases for retpoline and IBRS-based
113 * mitigations for the Spectre variant 2 vulnerability. Sometimes to
114 * eliminate potentially bogus entries from the RSB, and sometimes
115 * purely to ensure that it doesn't get empty, which on some CPUs would
116 * allow predictions from other (unwanted!) sources to be used.
117 *
118 * We define a CPP macro such that it can be used from both .S files and
119 * inline assembly. It's possible to do a .macro and then include that
120 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
121 */
122
123 #define RETPOLINE_THUNK_SIZE 32
124 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
125
126 /*
127 * Common helper for __FILL_RETURN_BUFFER and __FILL_ONE_RETURN.
128 */
129 #define __FILL_RETURN_SLOT \
130 ANNOTATE_INTRA_FUNCTION_CALL; \
131 call 772f; \
132 int3; \
133 772:
134
135 /*
136 * Stuff the entire RSB.
137 *
138 * Google experimented with loop-unrolling and this turned out to be
139 * the optimal version - two calls, each with their own speculation
140 * trap should their return address end up getting used, in a loop.
141 */
142 #ifdef CONFIG_X86_64
143 #define __FILL_RETURN_BUFFER(reg, nr) \
144 mov $(nr/2), reg; \
145 771: \
146 __FILL_RETURN_SLOT \
147 __FILL_RETURN_SLOT \
148 add $(BITS_PER_LONG/8) * 2, %_ASM_SP; \
149 dec reg; \
150 jnz 771b; \
151 /* barrier for jnz misprediction */ \
152 lfence; \
153 CREDIT_CALL_DEPTH \
154 CALL_THUNKS_DEBUG_INC_CTXSW
155 #else
156 /*
157 * i386 doesn't unconditionally have LFENCE, as such it can't
158 * do a loop.
159 */
160 #define __FILL_RETURN_BUFFER(reg, nr) \
161 .rept nr; \
162 __FILL_RETURN_SLOT; \
163 .endr; \
164 add $(BITS_PER_LONG/8) * nr, %_ASM_SP;
165 #endif
166
167 /*
168 * Stuff a single RSB slot.
169 *
170 * To mitigate Post-Barrier RSB speculation, one CALL instruction must be
171 * forced to retire before letting a RET instruction execute.
172 *
173 * On PBRSB-vulnerable CPUs, it is not safe for a RET to be executed
174 * before this point.
175 */
176 #define __FILL_ONE_RETURN \
177 __FILL_RETURN_SLOT \
178 add $(BITS_PER_LONG/8), %_ASM_SP; \
179 lfence;
180
181 /*
182 * Helper for detecting if an interrupt occurred at an unsafe location within
183 * Safe-RET. If Safe-RET is interrupted after the CALL or LEA the RSB may get
184 * poisoned by the interrupt handler.
185 *
186 * The Safe-RET sequence is:
187 *
188 * CALL
189 * LEA 8(%RSP), %RSP
190 * RET
191 *
192 * The two CMPs below check whether RIP points to after the CALL or after the
193 * LEA.
194 *
195 * The LFENCE below is to address this particular speculation case:
196 *
197 * 1. Userspace runs and poisons the BTB around the safe-RET routine
198 *
199 * 2. Userspace triggers some kind of exception
200 *
201 * 3. Kernel executes error_entry() and mis-speculates the branch into thinking
202 * it actually came from kernel space
203 *
204 * 4. The kernel then further mis-speculates that the exception occurred due
205 * to an interrupted safe-RET
206 *
207 * 5. The handle_interrupted_saferet() routine speculatively executes and
208 * speculatively does a safe-RET. But this is unsafe since it was never
209 * untrained.
210 *
211 * The LFENCE fixes this by ensuring step 5 is never reached speculatively.
212 * Note that this LFENCE only occurs if safe-RET was actually interrupted (so
213 * it's outside of the normal path).
214 */
215 #define __HANDLE_INTR_SAFERET(name, pt_regs) \
216 cmpq $(name), RIP+pt_regs; \
217 jb 1f; \
218 cmpq $(name)+5, RIP+pt_regs; \
219 ja 1f; \
220 lfence; \
221 leaq pt_regs, %rdi; \
222 call handle_interrupted_saferet; \
223 1:
224
225 #ifdef __ASSEMBLER__
226
227 /*
228 * (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions
229 * vs RETBleed validation.
230 */
231 #define ANNOTATE_UNRET_SAFE ANNOTATE_RETPOLINE_SAFE
232
233 /*
234 * Abuse ANNOTATE_RETPOLINE_SAFE on a NOP to indicate UNRET_END, should
235 * eventually turn into its own annotation.
236 */
237 .macro VALIDATE_UNRET_END
238 #if defined(CONFIG_NOINSTR_VALIDATION) && \
239 (defined(CONFIG_MITIGATION_UNRET_ENTRY) || defined(CONFIG_MITIGATION_SRSO))
240 ANNOTATE_RETPOLINE_SAFE
241 nop
242 #endif
243 .endm
244
245 /*
246 * Emits a conditional CS prefix that is compatible with
247 * -mindirect-branch-cs-prefix.
248 */
249 .macro __CS_PREFIX reg:req
250 .irp rs,r8,r9,r10,r11,r12,r13,r14,r15
251 .ifc \reg,\rs
252 .byte 0x2e
253 .endif
254 .endr
255 .endm
256
257 /*
258 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
259 * indirect jmp/call which may be susceptible to the Spectre variant 2
260 * attack.
261 *
262 * NOTE: these do not take kCFI into account and are thus not comparable to C
263 * indirect calls, take care when using. The target of these should be an ENDBR
264 * instruction irrespective of kCFI.
265 */
266 .macro JMP_NOSPEC reg:req
267 #ifdef CONFIG_MITIGATION_RETPOLINE
268 __CS_PREFIX \reg
269 jmp __x86_indirect_thunk_\reg
270 #else
271 jmp *%\reg
272 int3
273 #endif
274 .endm
275
276 .macro CALL_NOSPEC reg:req
277 #ifdef CONFIG_MITIGATION_RETPOLINE
278 __CS_PREFIX \reg
279 call __x86_indirect_thunk_\reg
280 #else
281 call *%\reg
282 #endif
283 .endm
284
285 /*
286 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
287 * monstrosity above, manually.
288 */
289 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req ftr2=ALT_NOT(X86_FEATURE_ALWAYS)
290 ALTERNATIVE_2 "jmp .Lskip_rsb_\@", \
291 __stringify(__FILL_RETURN_BUFFER(\reg,\nr)), \ftr, \
292 __stringify(nop;nop;__FILL_ONE_RETURN), \ftr2
293
294 .Lskip_rsb_\@:
295 .endm
296
297 /*
298 * The CALL to srso_alias_untrain_ret() must be patched in directly at
299 * the spot where untraining must be done, ie., srso_alias_untrain_ret()
300 * must be the target of a CALL instruction instead of indirectly
301 * jumping to a wrapper which then calls it. Therefore, this macro is
302 * called outside of __UNTRAIN_RET below, for the time being, before the
303 * kernel can support nested alternatives with arbitrary nesting.
304 */
305 .macro CALL_UNTRAIN_RET
306 #if defined(CONFIG_MITIGATION_UNRET_ENTRY) || defined(CONFIG_MITIGATION_SRSO)
307 ALTERNATIVE_2 "", "call entry_untrain_ret", X86_FEATURE_UNRET, \
308 "call srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS
309 #endif
310 .endm
311
312 /*
313 * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the
314 * return thunk isn't mapped into the userspace tables (then again, AMD
315 * typically has NO_MELTDOWN).
316 *
317 * While retbleed_untrain_ret() doesn't clobber anything but requires stack,
318 * write_ibpb() will clobber AX, CX, DX.
319 *
320 * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point
321 * where we have a stack but before any RET instruction.
322 */
323 .macro __UNTRAIN_RET ibpb_feature, call_depth_insns
324 #if defined(CONFIG_MITIGATION_RETHUNK) || defined(CONFIG_MITIGATION_IBPB_ENTRY)
325 VALIDATE_UNRET_END
326 CALL_UNTRAIN_RET
327 ALTERNATIVE_2 "", \
328 "call write_ibpb", \ibpb_feature, \
329 __stringify(\call_depth_insns), X86_FEATURE_CALL_DEPTH
330 #endif
331 .endm
332
333 #define UNTRAIN_RET \
334 __UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH)
335
336 #define UNTRAIN_RET_VM \
337 __UNTRAIN_RET X86_FEATURE_IBPB_ON_VMEXIT, __stringify(RESET_CALL_DEPTH)
338
339 #define UNTRAIN_RET_FROM_CALL \
340 __UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH_FROM_CALL)
341
342 .macro HANDLE_INTR_SAFERET pt_regs
343 #ifdef CONFIG_MITIGATION_SRSO
344 ALTERNATIVE_2 "", \
345 __stringify(__HANDLE_INTR_SAFERET(srso_safe_ret, \pt_regs)), X86_FEATURE_SRSO, \
346 __stringify(__HANDLE_INTR_SAFERET(srso_alias_safe_ret, \pt_regs)), X86_FEATURE_SRSO_ALIAS
347
348 #endif
349 .endm
350
351 .macro CALL_DEPTH_ACCOUNT
352 #ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
353 ALTERNATIVE "", \
354 __stringify(INCREMENT_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
355 #endif
356 .endm
357
358 /*
359 * Macro to execute VERW insns that mitigate transient data sampling
360 * attacks such as MDS or TSA. On affected systems a microcode update
361 * overloaded VERW insns to also clear the CPU buffers. VERW clobbers
362 * CFLAGS.ZF.
363 * Note: Only the memory operand variant of VERW clears the CPU buffers.
364 */
365 #ifdef CONFIG_X86_64
366 #define VERW verw x86_verw_sel(%rip)
367 #else
368 /*
369 * In 32bit mode, the memory operand must be a %cs reference. The data segments
370 * may not be usable (vm86 mode), and the stack segment may not be flat (ESPFIX32).
371 */
372 #define VERW verw %cs:x86_verw_sel
373 #endif
374
375 /*
376 * Provide a stringified VERW macro for simple usage, and a non-stringified
377 * VERW macro for use in more elaborate sequences, e.g. to encode a conditional
378 * VERW within an ALTERNATIVE.
379 */
380 #define __CLEAR_CPU_BUFFERS __stringify(VERW)
381
382 /* If necessary, emit VERW on exit-to-userspace to clear CPU buffers. */
383 #define CLEAR_CPU_BUFFERS \
384 ALTERNATIVE "", __CLEAR_CPU_BUFFERS, X86_FEATURE_CLEAR_CPU_BUF
385
386 #ifdef CONFIG_X86_64
387 .macro CLEAR_BRANCH_HISTORY
388 ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_LOOP
389 .endm
390
391 .macro CLEAR_BRANCH_HISTORY_VMEXIT
392 ALTERNATIVE "", "call clear_bhb_loop", X86_FEATURE_CLEAR_BHB_VMEXIT
393 .endm
394 #else
395 #define CLEAR_BRANCH_HISTORY
396 #define CLEAR_BRANCH_HISTORY_VMEXIT
397 #endif
398
399 #else /* __ASSEMBLER__ */
400
401 #define ITS_THUNK_SIZE 64
402
403 typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
404 typedef u8 its_thunk_t[ITS_THUNK_SIZE];
405 extern retpoline_thunk_t __x86_indirect_thunk_array[];
406 extern retpoline_thunk_t __x86_indirect_call_thunk_array[];
407 extern retpoline_thunk_t __x86_indirect_jump_thunk_array[];
408 extern its_thunk_t __x86_indirect_its_thunk_array[];
409
410 #ifdef CONFIG_MITIGATION_RETHUNK
411 extern void __x86_return_thunk(void);
412 #else
__x86_return_thunk(void)413 static inline void __x86_return_thunk(void) {}
414 #endif
415
416 #ifdef CONFIG_MITIGATION_UNRET_ENTRY
417 extern void retbleed_return_thunk(void);
418 #else
retbleed_return_thunk(void)419 static inline void retbleed_return_thunk(void) {}
420 #endif
421
422 extern void srso_alias_untrain_ret(void);
423
424 #ifdef CONFIG_MITIGATION_SRSO
425 extern void srso_return_thunk(void);
426 extern void srso_alias_return_thunk(void);
427 #else
srso_return_thunk(void)428 static inline void srso_return_thunk(void) {}
srso_alias_return_thunk(void)429 static inline void srso_alias_return_thunk(void) {}
430 #endif
431
432 #ifdef CONFIG_MITIGATION_ITS
433 extern void its_return_thunk(void);
434 #else
its_return_thunk(void)435 static inline void its_return_thunk(void) {}
436 #endif
437
438 extern void retbleed_return_thunk(void);
439 extern void srso_return_thunk(void);
440 extern void srso_alias_return_thunk(void);
441
442 extern void entry_untrain_ret(void);
443 extern void write_ibpb(void);
444
445 #ifdef CONFIG_BPF_JIT
446 extern void bpf_arch_ibpb(void);
447 #endif
448
449 #ifdef CONFIG_X86_64
450 extern void clear_bhb_loop(void);
451 #endif
452
453 extern void (*x86_return_thunk)(void);
454
455 extern void __warn_thunk(void);
456
457 #ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
458 extern void call_depth_return_thunk(void);
459
460 #define CALL_DEPTH_ACCOUNT \
461 ALTERNATIVE("", \
462 __stringify(INCREMENT_CALL_DEPTH), \
463 X86_FEATURE_CALL_DEPTH)
464
465 DECLARE_PER_CPU_CACHE_HOT(u64, __x86_call_depth);
466
467 #ifdef CONFIG_CALL_THUNKS_DEBUG
468 DECLARE_PER_CPU(u64, __x86_call_count);
469 DECLARE_PER_CPU(u64, __x86_ret_count);
470 DECLARE_PER_CPU(u64, __x86_stuffs_count);
471 DECLARE_PER_CPU(u64, __x86_ctxsw_count);
472 #endif
473 #else /* !CONFIG_MITIGATION_CALL_DEPTH_TRACKING */
474
call_depth_return_thunk(void)475 static inline void call_depth_return_thunk(void) {}
476 #define CALL_DEPTH_ACCOUNT ""
477
478 #endif /* CONFIG_MITIGATION_CALL_DEPTH_TRACKING */
479
480 #ifdef CONFIG_MITIGATION_RETPOLINE
481
482 #define GEN(reg) \
483 extern retpoline_thunk_t __x86_indirect_thunk_ ## reg;
484 #include <asm/GEN-for-each-reg.h>
485 #undef GEN
486
487 #define GEN(reg) \
488 extern retpoline_thunk_t __x86_indirect_call_thunk_ ## reg;
489 #include <asm/GEN-for-each-reg.h>
490 #undef GEN
491
492 #define GEN(reg) \
493 extern retpoline_thunk_t __x86_indirect_jump_thunk_ ## reg;
494 #include <asm/GEN-for-each-reg.h>
495 #undef GEN
496
497 #ifdef CONFIG_X86_64
498
499 /*
500 * Emits a conditional CS prefix that is compatible with
501 * -mindirect-branch-cs-prefix.
502 */
503 #define __CS_PREFIX(reg) \
504 ".irp rs,r8,r9,r10,r11,r12,r13,r14,r15\n" \
505 ".ifc \\rs," reg "\n" \
506 ".byte 0x2e\n" \
507 ".endif\n" \
508 ".endr\n"
509
510 /*
511 * Inline asm uses the %V modifier which is only in newer GCC
512 * which is ensured when CONFIG_MITIGATION_RETPOLINE is defined.
513 */
514 #define CALL_NOSPEC __CS_PREFIX("%V[thunk_target]") \
515 "call __x86_indirect_thunk_%V[thunk_target]\n"
516
517 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
518
519 #else /* CONFIG_X86_32 */
520 /*
521 * For i386 we use the original ret-equivalent retpoline, because
522 * otherwise we'll run out of registers. We don't care about CET
523 * here, anyway.
524 */
525 # define CALL_NOSPEC \
526 ALTERNATIVE_2( \
527 ANNOTATE_RETPOLINE_SAFE "\n" \
528 "call *%[thunk_target]\n", \
529 " jmp 904f;\n" \
530 " .align 16\n" \
531 "901: call 903f;\n" \
532 "902: pause;\n" \
533 " lfence;\n" \
534 " jmp 902b;\n" \
535 " .align 16\n" \
536 "903: lea 4(%%esp), %%esp;\n" \
537 " pushl %[thunk_target];\n" \
538 " ret;\n" \
539 " .align 16\n" \
540 "904: call 901b;\n", \
541 X86_FEATURE_RETPOLINE, \
542 "lfence;\n" \
543 ANNOTATE_RETPOLINE_SAFE "\n" \
544 "call *%[thunk_target]\n", \
545 X86_FEATURE_RETPOLINE_LFENCE)
546
547 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
548 #endif
549 #else /* No retpoline for C / inline asm */
550 # define CALL_NOSPEC "call *%[thunk_target]\n"
551 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
552 #endif
553
554 /* The Spectre V2 mitigation variants */
555 enum spectre_v2_mitigation {
556 SPECTRE_V2_NONE,
557 SPECTRE_V2_RETPOLINE,
558 SPECTRE_V2_LFENCE,
559 SPECTRE_V2_EIBRS,
560 SPECTRE_V2_EIBRS_RETPOLINE,
561 SPECTRE_V2_EIBRS_LFENCE,
562 SPECTRE_V2_IBRS,
563 };
564
565 /* The indirect branch speculation control variants */
566 enum spectre_v2_user_mitigation {
567 SPECTRE_V2_USER_NONE,
568 SPECTRE_V2_USER_STRICT,
569 SPECTRE_V2_USER_STRICT_PREFERRED,
570 SPECTRE_V2_USER_PRCTL,
571 SPECTRE_V2_USER_SECCOMP,
572 };
573
574 /* The Speculative Store Bypass disable variants */
575 enum ssb_mitigation {
576 SPEC_STORE_BYPASS_NONE,
577 SPEC_STORE_BYPASS_AUTO,
578 SPEC_STORE_BYPASS_DISABLE,
579 SPEC_STORE_BYPASS_PRCTL,
580 SPEC_STORE_BYPASS_SECCOMP,
581 };
582
583 static __always_inline
alternative_msr_write(unsigned int msr,u64 val,unsigned int feature)584 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
585 {
586 asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
587 : : "c" (msr),
588 "a" ((u32)val),
589 "d" ((u32)(val >> 32)),
590 [feature] "i" (feature)
591 : "memory");
592 }
593
594 DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user);
595
indirect_branch_prediction_barrier(void)596 static inline void indirect_branch_prediction_barrier(void)
597 {
598 asm_inline volatile(ALTERNATIVE("", "call write_ibpb", X86_FEATURE_IBPB)
599 : ASM_CALL_CONSTRAINT
600 :: "rax", "rcx", "rdx", "memory");
601 }
602
603 /* The Intel SPEC CTRL MSR base value cache */
604 extern u64 x86_spec_ctrl_base;
605 DECLARE_PER_CPU(u64, x86_spec_ctrl_current);
606 extern void update_spec_ctrl_cond(u64 val);
607 extern u64 spec_ctrl_current(void);
608
609 /*
610 * With retpoline, we must use IBRS to restrict branch prediction
611 * before calling into firmware.
612 *
613 * (Implemented as CPP macros due to header hell.)
614 */
615 #define firmware_restrict_branch_speculation_start() \
616 do { \
617 preempt_disable(); \
618 alternative_msr_write(MSR_IA32_SPEC_CTRL, \
619 spec_ctrl_current() | SPEC_CTRL_IBRS, \
620 X86_FEATURE_USE_IBRS_FW); \
621 alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, \
622 X86_FEATURE_USE_IBPB_FW); \
623 } while (0)
624
625 #define firmware_restrict_branch_speculation_end() \
626 do { \
627 alternative_msr_write(MSR_IA32_SPEC_CTRL, \
628 spec_ctrl_current(), \
629 X86_FEATURE_USE_IBRS_FW); \
630 preempt_enable(); \
631 } while (0)
632
633 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
634 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
635 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
636
637 DECLARE_STATIC_KEY_FALSE(switch_vcpu_ibpb);
638
639 DECLARE_STATIC_KEY_FALSE(cpu_buf_idle_clear);
640
641 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
642
643 extern u16 x86_verw_sel;
644
645 #include <asm/segment.h>
646
647 /**
648 * x86_clear_cpu_buffers - Buffer clearing support for different x86 CPU vulns
649 *
650 * This uses the otherwise unused and obsolete VERW instruction in
651 * combination with microcode which triggers a CPU buffer flush when the
652 * instruction is executed.
653 */
x86_clear_cpu_buffers(void)654 static __always_inline void x86_clear_cpu_buffers(void)
655 {
656 static const u16 ds = __KERNEL_DS;
657
658 /*
659 * Has to be the memory-operand variant because only that
660 * guarantees the CPU buffer flush functionality according to
661 * documentation. The register-operand variant does not.
662 * Works with any segment selector, but a valid writable
663 * data segment is the fastest variant.
664 *
665 * "cc" clobber is required because VERW modifies ZF.
666 */
667 asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc");
668 }
669
670 /**
671 * x86_idle_clear_cpu_buffers - Buffer clearing support in idle for the MDS
672 * and TSA vulnerabilities.
673 *
674 * Clear CPU buffers if the corresponding static key is enabled
675 */
x86_idle_clear_cpu_buffers(void)676 static __always_inline void x86_idle_clear_cpu_buffers(void)
677 {
678 if (static_branch_likely(&cpu_buf_idle_clear))
679 x86_clear_cpu_buffers();
680 }
681
682 void srso_safe_ret(void);
683 void srso_alias_safe_ret(void);
684 void handle_interrupted_saferet(struct pt_regs *regs);
685
686 #endif /* __ASSEMBLER__ */
687
688 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
689