1 #include <linux/module.h> 2 #include <linux/sched.h> 3 #include <linux/spinlock.h> 4 #include <linux/list.h> 5 #include <linux/kprobes.h> 6 #include <linux/mm.h> 7 #include <linux/vmalloc.h> 8 #include <asm/alternative.h> 9 #include <asm/sections.h> 10 #include <asm/pgtable.h> 11 #include <asm/mce.h> 12 #include <asm/nmi.h> 13 14 #define MAX_PATCH_LEN (255-1) 15 16 #ifdef CONFIG_HOTPLUG_CPU 17 static int smp_alt_once; 18 19 static int __init bootonly(char *str) 20 { 21 smp_alt_once = 1; 22 return 1; 23 } 24 __setup("smp-alt-boot", bootonly); 25 #else 26 #define smp_alt_once 1 27 #endif 28 29 static int debug_alternative; 30 31 static int __init debug_alt(char *str) 32 { 33 debug_alternative = 1; 34 return 1; 35 } 36 __setup("debug-alternative", debug_alt); 37 38 static int noreplace_smp; 39 40 static int __init setup_noreplace_smp(char *str) 41 { 42 noreplace_smp = 1; 43 return 1; 44 } 45 __setup("noreplace-smp", setup_noreplace_smp); 46 47 #ifdef CONFIG_PARAVIRT 48 static int noreplace_paravirt = 0; 49 50 static int __init setup_noreplace_paravirt(char *str) 51 { 52 noreplace_paravirt = 1; 53 return 1; 54 } 55 __setup("noreplace-paravirt", setup_noreplace_paravirt); 56 #endif 57 58 #define DPRINTK(fmt, args...) if (debug_alternative) \ 59 printk(KERN_DEBUG fmt, args) 60 61 #ifdef GENERIC_NOP1 62 /* Use inline assembly to define this because the nops are defined 63 as inline assembly strings in the include files and we cannot 64 get them easily into strings. */ 65 asm("\t.data\nintelnops: " 66 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6 67 GENERIC_NOP7 GENERIC_NOP8); 68 extern unsigned char intelnops[]; 69 static unsigned char *intel_nops[ASM_NOP_MAX+1] = { 70 NULL, 71 intelnops, 72 intelnops + 1, 73 intelnops + 1 + 2, 74 intelnops + 1 + 2 + 3, 75 intelnops + 1 + 2 + 3 + 4, 76 intelnops + 1 + 2 + 3 + 4 + 5, 77 intelnops + 1 + 2 + 3 + 4 + 5 + 6, 78 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 79 }; 80 #endif 81 82 #ifdef K8_NOP1 83 asm("\t.data\nk8nops: " 84 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6 85 K8_NOP7 K8_NOP8); 86 extern unsigned char k8nops[]; 87 static unsigned char *k8_nops[ASM_NOP_MAX+1] = { 88 NULL, 89 k8nops, 90 k8nops + 1, 91 k8nops + 1 + 2, 92 k8nops + 1 + 2 + 3, 93 k8nops + 1 + 2 + 3 + 4, 94 k8nops + 1 + 2 + 3 + 4 + 5, 95 k8nops + 1 + 2 + 3 + 4 + 5 + 6, 96 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 97 }; 98 #endif 99 100 #ifdef K7_NOP1 101 asm("\t.data\nk7nops: " 102 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6 103 K7_NOP7 K7_NOP8); 104 extern unsigned char k7nops[]; 105 static unsigned char *k7_nops[ASM_NOP_MAX+1] = { 106 NULL, 107 k7nops, 108 k7nops + 1, 109 k7nops + 1 + 2, 110 k7nops + 1 + 2 + 3, 111 k7nops + 1 + 2 + 3 + 4, 112 k7nops + 1 + 2 + 3 + 4 + 5, 113 k7nops + 1 + 2 + 3 + 4 + 5 + 6, 114 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 115 }; 116 #endif 117 118 #ifdef CONFIG_X86_64 119 120 extern char __vsyscall_0; 121 static inline unsigned char** find_nop_table(void) 122 { 123 return k8_nops; 124 } 125 126 #else /* CONFIG_X86_64 */ 127 128 static struct nop { 129 int cpuid; 130 unsigned char **noptable; 131 } noptypes[] = { 132 { X86_FEATURE_K8, k8_nops }, 133 { X86_FEATURE_K7, k7_nops }, 134 { -1, NULL } 135 }; 136 137 static unsigned char** find_nop_table(void) 138 { 139 unsigned char **noptable = intel_nops; 140 int i; 141 142 for (i = 0; noptypes[i].cpuid >= 0; i++) { 143 if (boot_cpu_has(noptypes[i].cpuid)) { 144 noptable = noptypes[i].noptable; 145 break; 146 } 147 } 148 return noptable; 149 } 150 151 #endif /* CONFIG_X86_64 */ 152 153 /* Use this to add nops to a buffer, then text_poke the whole buffer. */ 154 static void add_nops(void *insns, unsigned int len) 155 { 156 unsigned char **noptable = find_nop_table(); 157 158 while (len > 0) { 159 unsigned int noplen = len; 160 if (noplen > ASM_NOP_MAX) 161 noplen = ASM_NOP_MAX; 162 memcpy(insns, noptable[noplen], noplen); 163 insns += noplen; 164 len -= noplen; 165 } 166 } 167 168 extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 169 extern u8 *__smp_locks[], *__smp_locks_end[]; 170 171 /* Replace instructions with better alternatives for this CPU type. 172 This runs before SMP is initialized to avoid SMP problems with 173 self modifying code. This implies that assymetric systems where 174 APs have less capabilities than the boot processor are not handled. 175 Tough. Make sure you disable such features by hand. */ 176 177 void apply_alternatives(struct alt_instr *start, struct alt_instr *end) 178 { 179 struct alt_instr *a; 180 char insnbuf[MAX_PATCH_LEN]; 181 182 DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end); 183 for (a = start; a < end; a++) { 184 u8 *instr = a->instr; 185 BUG_ON(a->replacementlen > a->instrlen); 186 BUG_ON(a->instrlen > sizeof(insnbuf)); 187 if (!boot_cpu_has(a->cpuid)) 188 continue; 189 #ifdef CONFIG_X86_64 190 /* vsyscall code is not mapped yet. resolve it manually. */ 191 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) { 192 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0)); 193 DPRINTK("%s: vsyscall fixup: %p => %p\n", 194 __FUNCTION__, a->instr, instr); 195 } 196 #endif 197 memcpy(insnbuf, a->replacement, a->replacementlen); 198 add_nops(insnbuf + a->replacementlen, 199 a->instrlen - a->replacementlen); 200 text_poke(instr, insnbuf, a->instrlen); 201 } 202 } 203 204 #ifdef CONFIG_SMP 205 206 static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end) 207 { 208 u8 **ptr; 209 210 for (ptr = start; ptr < end; ptr++) { 211 if (*ptr < text) 212 continue; 213 if (*ptr > text_end) 214 continue; 215 text_poke(*ptr, ((unsigned char []){0xf0}), 1); /* add lock prefix */ 216 }; 217 } 218 219 static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end) 220 { 221 u8 **ptr; 222 char insn[1]; 223 224 if (noreplace_smp) 225 return; 226 227 add_nops(insn, 1); 228 for (ptr = start; ptr < end; ptr++) { 229 if (*ptr < text) 230 continue; 231 if (*ptr > text_end) 232 continue; 233 text_poke(*ptr, insn, 1); 234 }; 235 } 236 237 struct smp_alt_module { 238 /* what is this ??? */ 239 struct module *mod; 240 char *name; 241 242 /* ptrs to lock prefixes */ 243 u8 **locks; 244 u8 **locks_end; 245 246 /* .text segment, needed to avoid patching init code ;) */ 247 u8 *text; 248 u8 *text_end; 249 250 struct list_head next; 251 }; 252 static LIST_HEAD(smp_alt_modules); 253 static DEFINE_SPINLOCK(smp_alt); 254 255 void alternatives_smp_module_add(struct module *mod, char *name, 256 void *locks, void *locks_end, 257 void *text, void *text_end) 258 { 259 struct smp_alt_module *smp; 260 unsigned long flags; 261 262 if (noreplace_smp) 263 return; 264 265 if (smp_alt_once) { 266 if (boot_cpu_has(X86_FEATURE_UP)) 267 alternatives_smp_unlock(locks, locks_end, 268 text, text_end); 269 return; 270 } 271 272 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 273 if (NULL == smp) 274 return; /* we'll run the (safe but slow) SMP code then ... */ 275 276 smp->mod = mod; 277 smp->name = name; 278 smp->locks = locks; 279 smp->locks_end = locks_end; 280 smp->text = text; 281 smp->text_end = text_end; 282 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n", 283 __FUNCTION__, smp->locks, smp->locks_end, 284 smp->text, smp->text_end, smp->name); 285 286 spin_lock_irqsave(&smp_alt, flags); 287 list_add_tail(&smp->next, &smp_alt_modules); 288 if (boot_cpu_has(X86_FEATURE_UP)) 289 alternatives_smp_unlock(smp->locks, smp->locks_end, 290 smp->text, smp->text_end); 291 spin_unlock_irqrestore(&smp_alt, flags); 292 } 293 294 void alternatives_smp_module_del(struct module *mod) 295 { 296 struct smp_alt_module *item; 297 unsigned long flags; 298 299 if (smp_alt_once || noreplace_smp) 300 return; 301 302 spin_lock_irqsave(&smp_alt, flags); 303 list_for_each_entry(item, &smp_alt_modules, next) { 304 if (mod != item->mod) 305 continue; 306 list_del(&item->next); 307 spin_unlock_irqrestore(&smp_alt, flags); 308 DPRINTK("%s: %s\n", __FUNCTION__, item->name); 309 kfree(item); 310 return; 311 } 312 spin_unlock_irqrestore(&smp_alt, flags); 313 } 314 315 void alternatives_smp_switch(int smp) 316 { 317 struct smp_alt_module *mod; 318 unsigned long flags; 319 320 #ifdef CONFIG_LOCKDEP 321 /* 322 * A not yet fixed binutils section handling bug prevents 323 * alternatives-replacement from working reliably, so turn 324 * it off: 325 */ 326 printk("lockdep: not fixing up alternatives.\n"); 327 return; 328 #endif 329 330 if (noreplace_smp || smp_alt_once) 331 return; 332 BUG_ON(!smp && (num_online_cpus() > 1)); 333 334 spin_lock_irqsave(&smp_alt, flags); 335 if (smp) { 336 printk(KERN_INFO "SMP alternatives: switching to SMP code\n"); 337 clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability); 338 clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability); 339 list_for_each_entry(mod, &smp_alt_modules, next) 340 alternatives_smp_lock(mod->locks, mod->locks_end, 341 mod->text, mod->text_end); 342 } else { 343 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 344 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability); 345 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability); 346 list_for_each_entry(mod, &smp_alt_modules, next) 347 alternatives_smp_unlock(mod->locks, mod->locks_end, 348 mod->text, mod->text_end); 349 } 350 spin_unlock_irqrestore(&smp_alt, flags); 351 } 352 353 #endif 354 355 #ifdef CONFIG_PARAVIRT 356 void apply_paravirt(struct paravirt_patch_site *start, 357 struct paravirt_patch_site *end) 358 { 359 struct paravirt_patch_site *p; 360 char insnbuf[MAX_PATCH_LEN]; 361 362 if (noreplace_paravirt) 363 return; 364 365 for (p = start; p < end; p++) { 366 unsigned int used; 367 368 BUG_ON(p->len > MAX_PATCH_LEN); 369 /* prep the buffer with the original instructions */ 370 memcpy(insnbuf, p->instr, p->len); 371 used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf, 372 (unsigned long)p->instr, p->len); 373 374 BUG_ON(used > p->len); 375 376 /* Pad the rest with nops */ 377 add_nops(insnbuf + used, p->len - used); 378 text_poke(p->instr, insnbuf, p->len); 379 } 380 } 381 extern struct paravirt_patch_site __start_parainstructions[], 382 __stop_parainstructions[]; 383 #endif /* CONFIG_PARAVIRT */ 384 385 void __init alternative_instructions(void) 386 { 387 unsigned long flags; 388 389 /* The patching is not fully atomic, so try to avoid local interruptions 390 that might execute the to be patched code. 391 Other CPUs are not running. */ 392 stop_nmi(); 393 #ifdef CONFIG_X86_MCE 394 stop_mce(); 395 #endif 396 397 local_irq_save(flags); 398 apply_alternatives(__alt_instructions, __alt_instructions_end); 399 400 /* switch to patch-once-at-boottime-only mode and free the 401 * tables in case we know the number of CPUs will never ever 402 * change */ 403 #ifdef CONFIG_HOTPLUG_CPU 404 if (num_possible_cpus() < 2) 405 smp_alt_once = 1; 406 #endif 407 408 #ifdef CONFIG_SMP 409 if (smp_alt_once) { 410 if (1 == num_possible_cpus()) { 411 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 412 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability); 413 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability); 414 alternatives_smp_unlock(__smp_locks, __smp_locks_end, 415 _text, _etext); 416 } 417 free_init_pages("SMP alternatives", 418 (unsigned long)__smp_locks, 419 (unsigned long)__smp_locks_end); 420 } else { 421 alternatives_smp_module_add(NULL, "core kernel", 422 __smp_locks, __smp_locks_end, 423 _text, _etext); 424 alternatives_smp_switch(0); 425 } 426 #endif 427 apply_paravirt(__parainstructions, __parainstructions_end); 428 local_irq_restore(flags); 429 430 restart_nmi(); 431 #ifdef CONFIG_X86_MCE 432 restart_mce(); 433 #endif 434 } 435 436 /* 437 * Warning: 438 * When you use this code to patch more than one byte of an instruction 439 * you need to make sure that other CPUs cannot execute this code in parallel. 440 * Also no thread must be currently preempted in the middle of these instructions. 441 * And on the local CPU you need to be protected again NMI or MCE handlers 442 * seeing an inconsistent instruction while you patch. 443 */ 444 void __kprobes text_poke(void *addr, unsigned char *opcode, int len) 445 { 446 memcpy(addr, opcode, len); 447 sync_core(); 448 /* Could also do a CLFLUSH here to speed up CPU recovery; but 449 that causes hangs on some VIA CPUs. */ 450 } 451