1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef __NOSPEC_BRANCH_H__ 4 #define __NOSPEC_BRANCH_H__ 5 6 #include <asm/alternative.h> 7 #include <asm/alternative-asm.h> 8 #include <asm/cpufeatures.h> 9 10 /* 11 * Fill the CPU return stack buffer. 12 * 13 * Each entry in the RSB, if used for a speculative 'ret', contains an 14 * infinite 'pause; lfence; jmp' loop to capture speculative execution. 15 * 16 * This is required in various cases for retpoline and IBRS-based 17 * mitigations for the Spectre variant 2 vulnerability. Sometimes to 18 * eliminate potentially bogus entries from the RSB, and sometimes 19 * purely to ensure that it doesn't get empty, which on some CPUs would 20 * allow predictions from other (unwanted!) sources to be used. 21 * 22 * We define a CPP macro such that it can be used from both .S files and 23 * inline assembly. It's possible to do a .macro and then include that 24 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. 25 */ 26 27 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ 28 #define RSB_FILL_LOOPS 16 /* To avoid underflow */ 29 30 /* 31 * Google experimented with loop-unrolling and this turned out to be 32 * the optimal version — two calls, each with their own speculation 33 * trap should their return address end up getting used, in a loop. 34 */ 35 #define __FILL_RETURN_BUFFER(reg, nr, sp) \ 36 mov $(nr/2), reg; \ 37 771: \ 38 call 772f; \ 39 773: /* speculation trap */ \ 40 pause; \ 41 lfence; \ 42 jmp 773b; \ 43 772: \ 44 call 774f; \ 45 775: /* speculation trap */ \ 46 pause; \ 47 lfence; \ 48 jmp 775b; \ 49 774: \ 50 dec reg; \ 51 jnz 771b; \ 52 add $(BITS_PER_LONG/8) * nr, sp; 53 54 #ifdef __ASSEMBLY__ 55 56 /* 57 * This should be used immediately before a retpoline alternative. It tells 58 * objtool where the retpolines are so that it can make sense of the control 59 * flow by just reading the original instruction(s) and ignoring the 60 * alternatives. 61 */ 62 .macro ANNOTATE_NOSPEC_ALTERNATIVE 63 .Lannotate_\@: 64 .pushsection .discard.nospec 65 .long .Lannotate_\@ - . 66 .popsection 67 .endm 68 69 /* 70 * These are the bare retpoline primitives for indirect jmp and call. 71 * Do not use these directly; they only exist to make the ALTERNATIVE 72 * invocation below less ugly. 73 */ 74 .macro RETPOLINE_JMP reg:req 75 call .Ldo_rop_\@ 76 .Lspec_trap_\@: 77 pause 78 lfence 79 jmp .Lspec_trap_\@ 80 .Ldo_rop_\@: 81 mov \reg, (%_ASM_SP) 82 ret 83 .endm 84 85 /* 86 * This is a wrapper around RETPOLINE_JMP so the called function in reg 87 * returns to the instruction after the macro. 88 */ 89 .macro RETPOLINE_CALL reg:req 90 jmp .Ldo_call_\@ 91 .Ldo_retpoline_jmp_\@: 92 RETPOLINE_JMP \reg 93 .Ldo_call_\@: 94 call .Ldo_retpoline_jmp_\@ 95 .endm 96 97 /* 98 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple 99 * indirect jmp/call which may be susceptible to the Spectre variant 2 100 * attack. 101 */ 102 .macro JMP_NOSPEC reg:req 103 #ifdef CONFIG_RETPOLINE 104 ANNOTATE_NOSPEC_ALTERNATIVE 105 ALTERNATIVE_2 __stringify(jmp *\reg), \ 106 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ 107 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD 108 #else 109 jmp *\reg 110 #endif 111 .endm 112 113 .macro CALL_NOSPEC reg:req 114 #ifdef CONFIG_RETPOLINE 115 ANNOTATE_NOSPEC_ALTERNATIVE 116 ALTERNATIVE_2 __stringify(call *\reg), \ 117 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ 118 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD 119 #else 120 call *\reg 121 #endif 122 .endm 123 124 /* 125 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP 126 * monstrosity above, manually. 127 */ 128 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req 129 #ifdef CONFIG_RETPOLINE 130 ANNOTATE_NOSPEC_ALTERNATIVE 131 ALTERNATIVE "jmp .Lskip_rsb_\@", \ 132 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ 133 \ftr 134 .Lskip_rsb_\@: 135 #endif 136 .endm 137 138 #else /* __ASSEMBLY__ */ 139 140 #define ANNOTATE_NOSPEC_ALTERNATIVE \ 141 "999:\n\t" \ 142 ".pushsection .discard.nospec\n\t" \ 143 ".long 999b - .\n\t" \ 144 ".popsection\n\t" 145 146 #if defined(CONFIG_X86_64) && defined(RETPOLINE) 147 148 /* 149 * Since the inline asm uses the %V modifier which is only in newer GCC, 150 * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. 151 */ 152 # define CALL_NOSPEC \ 153 ANNOTATE_NOSPEC_ALTERNATIVE \ 154 ALTERNATIVE( \ 155 "call *%[thunk_target]\n", \ 156 "call __x86_indirect_thunk_%V[thunk_target]\n", \ 157 X86_FEATURE_RETPOLINE) 158 # define THUNK_TARGET(addr) [thunk_target] "r" (addr) 159 160 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE) 161 /* 162 * For i386 we use the original ret-equivalent retpoline, because 163 * otherwise we'll run out of registers. We don't care about CET 164 * here, anyway. 165 */ 166 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \ 167 " jmp 904f;\n" \ 168 " .align 16\n" \ 169 "901: call 903f;\n" \ 170 "902: pause;\n" \ 171 " lfence;\n" \ 172 " jmp 902b;\n" \ 173 " .align 16\n" \ 174 "903: addl $4, %%esp;\n" \ 175 " pushl %[thunk_target];\n" \ 176 " ret;\n" \ 177 " .align 16\n" \ 178 "904: call 901b;\n", \ 179 X86_FEATURE_RETPOLINE) 180 181 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 182 #else /* No retpoline for C / inline asm */ 183 # define CALL_NOSPEC "call *%[thunk_target]\n" 184 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) 185 #endif 186 187 /* The Spectre V2 mitigation variants */ 188 enum spectre_v2_mitigation { 189 SPECTRE_V2_NONE, 190 SPECTRE_V2_RETPOLINE_MINIMAL, 191 SPECTRE_V2_RETPOLINE_MINIMAL_AMD, 192 SPECTRE_V2_RETPOLINE_GENERIC, 193 SPECTRE_V2_RETPOLINE_AMD, 194 SPECTRE_V2_IBRS, 195 }; 196 197 extern char __indirect_thunk_start[]; 198 extern char __indirect_thunk_end[]; 199 200 /* 201 * On VMEXIT we must ensure that no RSB predictions learned in the guest 202 * can be followed in the host, by overwriting the RSB completely. Both 203 * retpoline and IBRS mitigations for Spectre v2 need this; only on future 204 * CPUs with IBRS_ATT *might* it be avoided. 205 */ 206 static inline void vmexit_fill_RSB(void) 207 { 208 #ifdef CONFIG_RETPOLINE 209 unsigned long loops; 210 211 asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE 212 ALTERNATIVE("jmp 910f", 213 __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), 214 X86_FEATURE_RETPOLINE) 215 "910:" 216 : "=r" (loops), ASM_CALL_CONSTRAINT 217 : : "memory" ); 218 #endif 219 } 220 221 #endif /* __ASSEMBLY__ */ 222 #endif /* __NOSPEC_BRANCH_H__ */ 223