1 #ifndef _ASM_X86_ALTERNATIVE_H 2 #define _ASM_X86_ALTERNATIVE_H 3 4 #include <linux/types.h> 5 #include <linux/stddef.h> 6 #include <linux/stringify.h> 7 #include <asm/asm.h> 8 9 /* 10 * Alternative inline assembly for SMP. 11 * 12 * The LOCK_PREFIX macro defined here replaces the LOCK and 13 * LOCK_PREFIX macros used everywhere in the source tree. 14 * 15 * SMP alternatives use the same data structures as the other 16 * alternatives and the X86_FEATURE_UP flag to indicate the case of a 17 * UP system running a SMP kernel. The existing apply_alternatives() 18 * works fine for patching a SMP kernel for UP. 19 * 20 * The SMP alternative tables can be kept after boot and contain both 21 * UP and SMP versions of the instructions to allow switching back to 22 * SMP at runtime, when hotplugging in a new CPU, which is especially 23 * useful in virtualized environments. 24 * 25 * The very common lock prefix is handled as special case in a 26 * separate table which is a pure address list without replacement ptr 27 * and size information. That keeps the table sizes small. 28 */ 29 30 #ifdef CONFIG_SMP 31 #define LOCK_PREFIX_HERE \ 32 ".pushsection .smp_locks,\"a\"\n" \ 33 ".balign 4\n" \ 34 ".long 671f - .\n" /* offset */ \ 35 ".popsection\n" \ 36 "671:" 37 38 #define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; " 39 40 #else /* ! CONFIG_SMP */ 41 #define LOCK_PREFIX_HERE "" 42 #define LOCK_PREFIX "" 43 #endif 44 45 struct alt_instr { 46 s32 instr_offset; /* original instruction */ 47 s32 repl_offset; /* offset to replacement instruction */ 48 u16 cpuid; /* cpuid bit set for replacement */ 49 u8 instrlen; /* length of original instruction */ 50 u8 replacementlen; /* length of new instruction, <= instrlen */ 51 }; 52 53 extern void alternative_instructions(void); 54 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); 55 56 struct module; 57 58 #ifdef CONFIG_SMP 59 extern void alternatives_smp_module_add(struct module *mod, char *name, 60 void *locks, void *locks_end, 61 void *text, void *text_end); 62 extern void alternatives_smp_module_del(struct module *mod); 63 extern void alternatives_enable_smp(void); 64 extern int alternatives_text_reserved(void *start, void *end); 65 extern bool skip_smp_alternatives; 66 #else 67 static inline void alternatives_smp_module_add(struct module *mod, char *name, 68 void *locks, void *locks_end, 69 void *text, void *text_end) {} 70 static inline void alternatives_smp_module_del(struct module *mod) {} 71 static inline void alternatives_enable_smp(void) {} 72 static inline int alternatives_text_reserved(void *start, void *end) 73 { 74 return 0; 75 } 76 #endif /* CONFIG_SMP */ 77 78 #define OLDINSTR(oldinstr) "661:\n\t" oldinstr "\n662:\n" 79 80 #define b_replacement(number) "663"#number 81 #define e_replacement(number) "664"#number 82 83 #define alt_slen "662b-661b" 84 #define alt_rlen(number) e_replacement(number)"f-"b_replacement(number)"f" 85 86 #define ALTINSTR_ENTRY(feature, number) \ 87 " .long 661b - .\n" /* label */ \ 88 " .long " b_replacement(number)"f - .\n" /* new instruction */ \ 89 " .word " __stringify(feature) "\n" /* feature bit */ \ 90 " .byte " alt_slen "\n" /* source len */ \ 91 " .byte " alt_rlen(number) "\n" /* replacement len */ 92 93 #define DISCARD_ENTRY(number) /* rlen <= slen */ \ 94 " .byte 0xff + (" alt_rlen(number) ") - (" alt_slen ")\n" 95 96 #define ALTINSTR_REPLACEMENT(newinstr, feature, number) /* replacement */ \ 97 b_replacement(number)":\n\t" newinstr "\n" e_replacement(number) ":\n\t" 98 99 /* alternative assembly primitive: */ 100 #define ALTERNATIVE(oldinstr, newinstr, feature) \ 101 OLDINSTR(oldinstr) \ 102 ".pushsection .altinstructions,\"a\"\n" \ 103 ALTINSTR_ENTRY(feature, 1) \ 104 ".popsection\n" \ 105 ".pushsection .discard,\"aw\",@progbits\n" \ 106 DISCARD_ENTRY(1) \ 107 ".popsection\n" \ 108 ".pushsection .altinstr_replacement, \"ax\"\n" \ 109 ALTINSTR_REPLACEMENT(newinstr, feature, 1) \ 110 ".popsection" 111 112 #define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ 113 OLDINSTR(oldinstr) \ 114 ".pushsection .altinstructions,\"a\"\n" \ 115 ALTINSTR_ENTRY(feature1, 1) \ 116 ALTINSTR_ENTRY(feature2, 2) \ 117 ".popsection\n" \ 118 ".pushsection .discard,\"aw\",@progbits\n" \ 119 DISCARD_ENTRY(1) \ 120 DISCARD_ENTRY(2) \ 121 ".popsection\n" \ 122 ".pushsection .altinstr_replacement, \"ax\"\n" \ 123 ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \ 124 ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ 125 ".popsection" 126 127 /* 128 * This must be included *after* the definition of ALTERNATIVE due to 129 * <asm/arch_hweight.h> 130 */ 131 #include <asm/cpufeature.h> 132 133 /* 134 * Alternative instructions for different CPU types or capabilities. 135 * 136 * This allows to use optimized instructions even on generic binary 137 * kernels. 138 * 139 * length of oldinstr must be longer or equal the length of newinstr 140 * It can be padded with nops as needed. 141 * 142 * For non barrier like inlines please define new variants 143 * without volatile and memory clobber. 144 */ 145 #define alternative(oldinstr, newinstr, feature) \ 146 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") 147 148 /* 149 * Alternative inline assembly with input. 150 * 151 * Pecularities: 152 * No memory clobber here. 153 * Argument numbers start with 1. 154 * Best is to use constraints that are fixed size (like (%1) ... "r") 155 * If you use variable sized constraints like "m" or "g" in the 156 * replacement make sure to pad to the worst case length. 157 * Leaving an unused argument 0 to keep API compatibility. 158 */ 159 #define alternative_input(oldinstr, newinstr, feature, input...) \ 160 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 161 : : "i" (0), ## input) 162 163 /* Like alternative_input, but with a single output argument */ 164 #define alternative_io(oldinstr, newinstr, feature, output, input...) \ 165 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 166 : output : "i" (0), ## input) 167 168 /* Like alternative_io, but for replacing a direct call with another one. */ 169 #define alternative_call(oldfunc, newfunc, feature, output, input...) \ 170 asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \ 171 : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) 172 173 /* 174 * Like alternative_call, but there are two features and respective functions. 175 * If CPU has feature2, function2 is used. 176 * Otherwise, if CPU has feature1, function1 is used. 177 * Otherwise, old function is used. 178 */ 179 #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \ 180 output, input...) \ 181 asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\ 182 "call %P[new2]", feature2) \ 183 : output : [old] "i" (oldfunc), [new1] "i" (newfunc1), \ 184 [new2] "i" (newfunc2), ## input) 185 186 /* 187 * use this macro(s) if you need more than one output parameter 188 * in alternative_io 189 */ 190 #define ASM_OUTPUT2(a...) a 191 192 /* 193 * use this macro if you need clobbers but no inputs in 194 * alternative_{input,io,call}() 195 */ 196 #define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr 197 198 struct paravirt_patch_site; 199 #ifdef CONFIG_PARAVIRT 200 void apply_paravirt(struct paravirt_patch_site *start, 201 struct paravirt_patch_site *end); 202 #else 203 static inline void apply_paravirt(struct paravirt_patch_site *start, 204 struct paravirt_patch_site *end) 205 {} 206 #define __parainstructions NULL 207 #define __parainstructions_end NULL 208 #endif 209 210 extern void *text_poke_early(void *addr, const void *opcode, size_t len); 211 212 /* 213 * Clear and restore the kernel write-protection flag on the local CPU. 214 * Allows the kernel to edit read-only pages. 215 * Side-effect: any interrupt handler running between save and restore will have 216 * the ability to write to read-only pages. 217 * 218 * Warning: 219 * Code patching in the UP case is safe if NMIs and MCE handlers are stopped and 220 * no thread can be preempted in the instructions being modified (no iret to an 221 * invalid instruction possible) or if the instructions are changed from a 222 * consistent state to another consistent state atomically. 223 * More care must be taken when modifying code in the SMP case because of 224 * Intel's errata. text_poke_smp() takes care that errata, but still 225 * doesn't support NMI/MCE handler code modifying. 226 * On the local CPU you need to be protected again NMI or MCE handlers seeing an 227 * inconsistent instruction while you patch. 228 */ 229 struct text_poke_param { 230 void *addr; 231 const void *opcode; 232 size_t len; 233 }; 234 235 extern void *text_poke(void *addr, const void *opcode, size_t len); 236 extern void *text_poke_smp(void *addr, const void *opcode, size_t len); 237 extern void text_poke_smp_batch(struct text_poke_param *params, int n); 238 239 #endif /* _ASM_X86_ALTERNATIVE_H */ 240