1 #include <linux/module.h> 2 #include <linux/sched.h> 3 #include <linux/mutex.h> 4 #include <linux/list.h> 5 #include <linux/stringify.h> 6 #include <linux/kprobes.h> 7 #include <linux/mm.h> 8 #include <linux/vmalloc.h> 9 #include <linux/memory.h> 10 #include <asm/alternative.h> 11 #include <asm/sections.h> 12 #include <asm/pgtable.h> 13 #include <asm/mce.h> 14 #include <asm/nmi.h> 15 #include <asm/vsyscall.h> 16 #include <asm/cacheflush.h> 17 #include <asm/tlbflush.h> 18 #include <asm/io.h> 19 #include <asm/fixmap.h> 20 21 #define MAX_PATCH_LEN (255-1) 22 23 #ifdef CONFIG_HOTPLUG_CPU 24 static int smp_alt_once; 25 26 static int __init bootonly(char *str) 27 { 28 smp_alt_once = 1; 29 return 1; 30 } 31 __setup("smp-alt-boot", bootonly); 32 #else 33 #define smp_alt_once 1 34 #endif 35 36 static int __initdata_or_module debug_alternative; 37 38 static int __init debug_alt(char *str) 39 { 40 debug_alternative = 1; 41 return 1; 42 } 43 __setup("debug-alternative", debug_alt); 44 45 static int noreplace_smp; 46 47 static int __init setup_noreplace_smp(char *str) 48 { 49 noreplace_smp = 1; 50 return 1; 51 } 52 __setup("noreplace-smp", setup_noreplace_smp); 53 54 #ifdef CONFIG_PARAVIRT 55 static int __initdata_or_module noreplace_paravirt = 0; 56 57 static int __init setup_noreplace_paravirt(char *str) 58 { 59 noreplace_paravirt = 1; 60 return 1; 61 } 62 __setup("noreplace-paravirt", setup_noreplace_paravirt); 63 #endif 64 65 #define DPRINTK(fmt, args...) if (debug_alternative) \ 66 printk(KERN_DEBUG fmt, args) 67 68 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64) 69 /* Use inline assembly to define this because the nops are defined 70 as inline assembly strings in the include files and we cannot 71 get them easily into strings. */ 72 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: " 73 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6 74 GENERIC_NOP7 GENERIC_NOP8 75 "\t.previous"); 76 extern const unsigned char intelnops[]; 77 static const unsigned char *const __initconst_or_module 78 intel_nops[ASM_NOP_MAX+1] = { 79 NULL, 80 intelnops, 81 intelnops + 1, 82 intelnops + 1 + 2, 83 intelnops + 1 + 2 + 3, 84 intelnops + 1 + 2 + 3 + 4, 85 intelnops + 1 + 2 + 3 + 4 + 5, 86 intelnops + 1 + 2 + 3 + 4 + 5 + 6, 87 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 88 }; 89 #endif 90 91 #ifdef K8_NOP1 92 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: " 93 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6 94 K8_NOP7 K8_NOP8 95 "\t.previous"); 96 extern const unsigned char k8nops[]; 97 static const unsigned char *const __initconst_or_module 98 k8_nops[ASM_NOP_MAX+1] = { 99 NULL, 100 k8nops, 101 k8nops + 1, 102 k8nops + 1 + 2, 103 k8nops + 1 + 2 + 3, 104 k8nops + 1 + 2 + 3 + 4, 105 k8nops + 1 + 2 + 3 + 4 + 5, 106 k8nops + 1 + 2 + 3 + 4 + 5 + 6, 107 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 108 }; 109 #endif 110 111 #if defined(K7_NOP1) && !defined(CONFIG_X86_64) 112 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: " 113 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6 114 K7_NOP7 K7_NOP8 115 "\t.previous"); 116 extern const unsigned char k7nops[]; 117 static const unsigned char *const __initconst_or_module 118 k7_nops[ASM_NOP_MAX+1] = { 119 NULL, 120 k7nops, 121 k7nops + 1, 122 k7nops + 1 + 2, 123 k7nops + 1 + 2 + 3, 124 k7nops + 1 + 2 + 3 + 4, 125 k7nops + 1 + 2 + 3 + 4 + 5, 126 k7nops + 1 + 2 + 3 + 4 + 5 + 6, 127 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 128 }; 129 #endif 130 131 #ifdef P6_NOP1 132 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: " 133 P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6 134 P6_NOP7 P6_NOP8 135 "\t.previous"); 136 extern const unsigned char p6nops[]; 137 static const unsigned char *const __initconst_or_module 138 p6_nops[ASM_NOP_MAX+1] = { 139 NULL, 140 p6nops, 141 p6nops + 1, 142 p6nops + 1 + 2, 143 p6nops + 1 + 2 + 3, 144 p6nops + 1 + 2 + 3 + 4, 145 p6nops + 1 + 2 + 3 + 4 + 5, 146 p6nops + 1 + 2 + 3 + 4 + 5 + 6, 147 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 148 }; 149 #endif 150 151 #ifdef CONFIG_X86_64 152 153 extern char __vsyscall_0; 154 static const unsigned char *const *__init_or_module find_nop_table(void) 155 { 156 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 157 boot_cpu_has(X86_FEATURE_NOPL)) 158 return p6_nops; 159 else 160 return k8_nops; 161 } 162 163 #else /* CONFIG_X86_64 */ 164 165 static const unsigned char *const *__init_or_module find_nop_table(void) 166 { 167 if (boot_cpu_has(X86_FEATURE_K8)) 168 return k8_nops; 169 else if (boot_cpu_has(X86_FEATURE_K7)) 170 return k7_nops; 171 else if (boot_cpu_has(X86_FEATURE_NOPL)) 172 return p6_nops; 173 else 174 return intel_nops; 175 } 176 177 #endif /* CONFIG_X86_64 */ 178 179 /* Use this to add nops to a buffer, then text_poke the whole buffer. */ 180 static void __init_or_module add_nops(void *insns, unsigned int len) 181 { 182 const unsigned char *const *noptable = find_nop_table(); 183 184 while (len > 0) { 185 unsigned int noplen = len; 186 if (noplen > ASM_NOP_MAX) 187 noplen = ASM_NOP_MAX; 188 memcpy(insns, noptable[noplen], noplen); 189 insns += noplen; 190 len -= noplen; 191 } 192 } 193 194 extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 195 extern u8 *__smp_locks[], *__smp_locks_end[]; 196 static void *text_poke_early(void *addr, const void *opcode, size_t len); 197 198 /* Replace instructions with better alternatives for this CPU type. 199 This runs before SMP is initialized to avoid SMP problems with 200 self modifying code. This implies that assymetric systems where 201 APs have less capabilities than the boot processor are not handled. 202 Tough. Make sure you disable such features by hand. */ 203 204 void __init_or_module apply_alternatives(struct alt_instr *start, 205 struct alt_instr *end) 206 { 207 struct alt_instr *a; 208 u8 insnbuf[MAX_PATCH_LEN]; 209 210 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); 211 for (a = start; a < end; a++) { 212 u8 *instr = a->instr; 213 BUG_ON(a->replacementlen > a->instrlen); 214 BUG_ON(a->instrlen > sizeof(insnbuf)); 215 if (!boot_cpu_has(a->cpuid)) 216 continue; 217 #ifdef CONFIG_X86_64 218 /* vsyscall code is not mapped yet. resolve it manually. */ 219 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) { 220 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0)); 221 DPRINTK("%s: vsyscall fixup: %p => %p\n", 222 __func__, a->instr, instr); 223 } 224 #endif 225 memcpy(insnbuf, a->replacement, a->replacementlen); 226 if (*insnbuf == 0xe8 && a->replacementlen == 5) 227 *(s32 *)(insnbuf + 1) += a->replacement - a->instr; 228 add_nops(insnbuf + a->replacementlen, 229 a->instrlen - a->replacementlen); 230 text_poke_early(instr, insnbuf, a->instrlen); 231 } 232 } 233 234 #ifdef CONFIG_SMP 235 236 static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end) 237 { 238 u8 **ptr; 239 240 mutex_lock(&text_mutex); 241 for (ptr = start; ptr < end; ptr++) { 242 if (*ptr < text) 243 continue; 244 if (*ptr > text_end) 245 continue; 246 /* turn DS segment override prefix into lock prefix */ 247 text_poke(*ptr, ((unsigned char []){0xf0}), 1); 248 }; 249 mutex_unlock(&text_mutex); 250 } 251 252 static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end) 253 { 254 u8 **ptr; 255 256 if (noreplace_smp) 257 return; 258 259 mutex_lock(&text_mutex); 260 for (ptr = start; ptr < end; ptr++) { 261 if (*ptr < text) 262 continue; 263 if (*ptr > text_end) 264 continue; 265 /* turn lock prefix into DS segment override prefix */ 266 text_poke(*ptr, ((unsigned char []){0x3E}), 1); 267 }; 268 mutex_unlock(&text_mutex); 269 } 270 271 struct smp_alt_module { 272 /* what is this ??? */ 273 struct module *mod; 274 char *name; 275 276 /* ptrs to lock prefixes */ 277 u8 **locks; 278 u8 **locks_end; 279 280 /* .text segment, needed to avoid patching init code ;) */ 281 u8 *text; 282 u8 *text_end; 283 284 struct list_head next; 285 }; 286 static LIST_HEAD(smp_alt_modules); 287 static DEFINE_MUTEX(smp_alt); 288 static int smp_mode = 1; /* protected by smp_alt */ 289 290 void __init_or_module alternatives_smp_module_add(struct module *mod, 291 char *name, 292 void *locks, void *locks_end, 293 void *text, void *text_end) 294 { 295 struct smp_alt_module *smp; 296 297 if (noreplace_smp) 298 return; 299 300 if (smp_alt_once) { 301 if (boot_cpu_has(X86_FEATURE_UP)) 302 alternatives_smp_unlock(locks, locks_end, 303 text, text_end); 304 return; 305 } 306 307 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 308 if (NULL == smp) 309 return; /* we'll run the (safe but slow) SMP code then ... */ 310 311 smp->mod = mod; 312 smp->name = name; 313 smp->locks = locks; 314 smp->locks_end = locks_end; 315 smp->text = text; 316 smp->text_end = text_end; 317 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n", 318 __func__, smp->locks, smp->locks_end, 319 smp->text, smp->text_end, smp->name); 320 321 mutex_lock(&smp_alt); 322 list_add_tail(&smp->next, &smp_alt_modules); 323 if (boot_cpu_has(X86_FEATURE_UP)) 324 alternatives_smp_unlock(smp->locks, smp->locks_end, 325 smp->text, smp->text_end); 326 mutex_unlock(&smp_alt); 327 } 328 329 void __init_or_module alternatives_smp_module_del(struct module *mod) 330 { 331 struct smp_alt_module *item; 332 333 if (smp_alt_once || noreplace_smp) 334 return; 335 336 mutex_lock(&smp_alt); 337 list_for_each_entry(item, &smp_alt_modules, next) { 338 if (mod != item->mod) 339 continue; 340 list_del(&item->next); 341 mutex_unlock(&smp_alt); 342 DPRINTK("%s: %s\n", __func__, item->name); 343 kfree(item); 344 return; 345 } 346 mutex_unlock(&smp_alt); 347 } 348 349 void alternatives_smp_switch(int smp) 350 { 351 struct smp_alt_module *mod; 352 353 #ifdef CONFIG_LOCKDEP 354 /* 355 * Older binutils section handling bug prevented 356 * alternatives-replacement from working reliably. 357 * 358 * If this still occurs then you should see a hang 359 * or crash shortly after this line: 360 */ 361 printk("lockdep: fixing up alternatives.\n"); 362 #endif 363 364 if (noreplace_smp || smp_alt_once) 365 return; 366 BUG_ON(!smp && (num_online_cpus() > 1)); 367 368 mutex_lock(&smp_alt); 369 370 /* 371 * Avoid unnecessary switches because it forces JIT based VMs to 372 * throw away all cached translations, which can be quite costly. 373 */ 374 if (smp == smp_mode) { 375 /* nothing */ 376 } else if (smp) { 377 printk(KERN_INFO "SMP alternatives: switching to SMP code\n"); 378 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 379 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 380 list_for_each_entry(mod, &smp_alt_modules, next) 381 alternatives_smp_lock(mod->locks, mod->locks_end, 382 mod->text, mod->text_end); 383 } else { 384 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 385 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 386 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 387 list_for_each_entry(mod, &smp_alt_modules, next) 388 alternatives_smp_unlock(mod->locks, mod->locks_end, 389 mod->text, mod->text_end); 390 } 391 smp_mode = smp; 392 mutex_unlock(&smp_alt); 393 } 394 395 /* Return 1 if the address range is reserved for smp-alternatives */ 396 int alternatives_text_reserved(void *start, void *end) 397 { 398 struct smp_alt_module *mod; 399 u8 **ptr; 400 u8 *text_start = start; 401 u8 *text_end = end; 402 403 list_for_each_entry(mod, &smp_alt_modules, next) { 404 if (mod->text > text_end || mod->text_end < text_start) 405 continue; 406 for (ptr = mod->locks; ptr < mod->locks_end; ptr++) 407 if (text_start <= *ptr && text_end >= *ptr) 408 return 1; 409 } 410 411 return 0; 412 } 413 #endif 414 415 #ifdef CONFIG_PARAVIRT 416 void __init_or_module apply_paravirt(struct paravirt_patch_site *start, 417 struct paravirt_patch_site *end) 418 { 419 struct paravirt_patch_site *p; 420 char insnbuf[MAX_PATCH_LEN]; 421 422 if (noreplace_paravirt) 423 return; 424 425 for (p = start; p < end; p++) { 426 unsigned int used; 427 428 BUG_ON(p->len > MAX_PATCH_LEN); 429 /* prep the buffer with the original instructions */ 430 memcpy(insnbuf, p->instr, p->len); 431 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf, 432 (unsigned long)p->instr, p->len); 433 434 BUG_ON(used > p->len); 435 436 /* Pad the rest with nops */ 437 add_nops(insnbuf + used, p->len - used); 438 text_poke_early(p->instr, insnbuf, p->len); 439 } 440 } 441 extern struct paravirt_patch_site __start_parainstructions[], 442 __stop_parainstructions[]; 443 #endif /* CONFIG_PARAVIRT */ 444 445 void __init alternative_instructions(void) 446 { 447 /* The patching is not fully atomic, so try to avoid local interruptions 448 that might execute the to be patched code. 449 Other CPUs are not running. */ 450 stop_nmi(); 451 452 /* 453 * Don't stop machine check exceptions while patching. 454 * MCEs only happen when something got corrupted and in this 455 * case we must do something about the corruption. 456 * Ignoring it is worse than a unlikely patching race. 457 * Also machine checks tend to be broadcast and if one CPU 458 * goes into machine check the others follow quickly, so we don't 459 * expect a machine check to cause undue problems during to code 460 * patching. 461 */ 462 463 apply_alternatives(__alt_instructions, __alt_instructions_end); 464 465 /* switch to patch-once-at-boottime-only mode and free the 466 * tables in case we know the number of CPUs will never ever 467 * change */ 468 #ifdef CONFIG_HOTPLUG_CPU 469 if (num_possible_cpus() < 2) 470 smp_alt_once = 1; 471 #endif 472 473 #ifdef CONFIG_SMP 474 if (smp_alt_once) { 475 if (1 == num_possible_cpus()) { 476 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 477 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 478 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 479 480 alternatives_smp_unlock(__smp_locks, __smp_locks_end, 481 _text, _etext); 482 } 483 } else { 484 alternatives_smp_module_add(NULL, "core kernel", 485 __smp_locks, __smp_locks_end, 486 _text, _etext); 487 488 /* Only switch to UP mode if we don't immediately boot others */ 489 if (num_present_cpus() == 1 || setup_max_cpus <= 1) 490 alternatives_smp_switch(0); 491 } 492 #endif 493 apply_paravirt(__parainstructions, __parainstructions_end); 494 495 if (smp_alt_once) 496 free_init_pages("SMP alternatives", 497 (unsigned long)__smp_locks, 498 (unsigned long)__smp_locks_end); 499 500 restart_nmi(); 501 } 502 503 /** 504 * text_poke_early - Update instructions on a live kernel at boot time 505 * @addr: address to modify 506 * @opcode: source of the copy 507 * @len: length to copy 508 * 509 * When you use this code to patch more than one byte of an instruction 510 * you need to make sure that other CPUs cannot execute this code in parallel. 511 * Also no thread must be currently preempted in the middle of these 512 * instructions. And on the local CPU you need to be protected again NMI or MCE 513 * handlers seeing an inconsistent instruction while you patch. 514 */ 515 static void *__init_or_module text_poke_early(void *addr, const void *opcode, 516 size_t len) 517 { 518 unsigned long flags; 519 local_irq_save(flags); 520 memcpy(addr, opcode, len); 521 sync_core(); 522 local_irq_restore(flags); 523 /* Could also do a CLFLUSH here to speed up CPU recovery; but 524 that causes hangs on some VIA CPUs. */ 525 return addr; 526 } 527 528 /** 529 * text_poke - Update instructions on a live kernel 530 * @addr: address to modify 531 * @opcode: source of the copy 532 * @len: length to copy 533 * 534 * Only atomic text poke/set should be allowed when not doing early patching. 535 * It means the size must be writable atomically and the address must be aligned 536 * in a way that permits an atomic write. It also makes sure we fit on a single 537 * page. 538 * 539 * Note: Must be called under text_mutex. 540 */ 541 void *__kprobes text_poke(void *addr, const void *opcode, size_t len) 542 { 543 unsigned long flags; 544 char *vaddr; 545 struct page *pages[2]; 546 int i; 547 548 if (!core_kernel_text((unsigned long)addr)) { 549 pages[0] = vmalloc_to_page(addr); 550 pages[1] = vmalloc_to_page(addr + PAGE_SIZE); 551 } else { 552 pages[0] = virt_to_page(addr); 553 WARN_ON(!PageReserved(pages[0])); 554 pages[1] = virt_to_page(addr + PAGE_SIZE); 555 } 556 BUG_ON(!pages[0]); 557 local_irq_save(flags); 558 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0])); 559 if (pages[1]) 560 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1])); 561 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0); 562 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len); 563 clear_fixmap(FIX_TEXT_POKE0); 564 if (pages[1]) 565 clear_fixmap(FIX_TEXT_POKE1); 566 local_flush_tlb(); 567 sync_core(); 568 /* Could also do a CLFLUSH here to speed up CPU recovery; but 569 that causes hangs on some VIA CPUs. */ 570 for (i = 0; i < len; i++) 571 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]); 572 local_irq_restore(flags); 573 return addr; 574 } 575