1 #include <linux/module.h> 2 #include <linux/sched.h> 3 #include <linux/mutex.h> 4 #include <linux/list.h> 5 #include <linux/stringify.h> 6 #include <linux/kprobes.h> 7 #include <linux/mm.h> 8 #include <linux/vmalloc.h> 9 #include <linux/memory.h> 10 #include <linux/stop_machine.h> 11 #include <asm/alternative.h> 12 #include <asm/sections.h> 13 #include <asm/pgtable.h> 14 #include <asm/mce.h> 15 #include <asm/nmi.h> 16 #include <asm/vsyscall.h> 17 #include <asm/cacheflush.h> 18 #include <asm/tlbflush.h> 19 #include <asm/io.h> 20 #include <asm/fixmap.h> 21 22 #define MAX_PATCH_LEN (255-1) 23 24 #ifdef CONFIG_HOTPLUG_CPU 25 static int smp_alt_once; 26 27 static int __init bootonly(char *str) 28 { 29 smp_alt_once = 1; 30 return 1; 31 } 32 __setup("smp-alt-boot", bootonly); 33 #else 34 #define smp_alt_once 1 35 #endif 36 37 static int __initdata_or_module debug_alternative; 38 39 static int __init debug_alt(char *str) 40 { 41 debug_alternative = 1; 42 return 1; 43 } 44 __setup("debug-alternative", debug_alt); 45 46 static int noreplace_smp; 47 48 static int __init setup_noreplace_smp(char *str) 49 { 50 noreplace_smp = 1; 51 return 1; 52 } 53 __setup("noreplace-smp", setup_noreplace_smp); 54 55 #ifdef CONFIG_PARAVIRT 56 static int __initdata_or_module noreplace_paravirt = 0; 57 58 static int __init setup_noreplace_paravirt(char *str) 59 { 60 noreplace_paravirt = 1; 61 return 1; 62 } 63 __setup("noreplace-paravirt", setup_noreplace_paravirt); 64 #endif 65 66 #define DPRINTK(fmt, args...) if (debug_alternative) \ 67 printk(KERN_DEBUG fmt, args) 68 69 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64) 70 /* Use inline assembly to define this because the nops are defined 71 as inline assembly strings in the include files and we cannot 72 get them easily into strings. */ 73 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: " 74 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6 75 GENERIC_NOP7 GENERIC_NOP8 76 "\t.previous"); 77 extern const unsigned char intelnops[]; 78 static const unsigned char *const __initconst_or_module 79 intel_nops[ASM_NOP_MAX+1] = { 80 NULL, 81 intelnops, 82 intelnops + 1, 83 intelnops + 1 + 2, 84 intelnops + 1 + 2 + 3, 85 intelnops + 1 + 2 + 3 + 4, 86 intelnops + 1 + 2 + 3 + 4 + 5, 87 intelnops + 1 + 2 + 3 + 4 + 5 + 6, 88 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 89 }; 90 #endif 91 92 #ifdef K8_NOP1 93 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: " 94 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6 95 K8_NOP7 K8_NOP8 96 "\t.previous"); 97 extern const unsigned char k8nops[]; 98 static const unsigned char *const __initconst_or_module 99 k8_nops[ASM_NOP_MAX+1] = { 100 NULL, 101 k8nops, 102 k8nops + 1, 103 k8nops + 1 + 2, 104 k8nops + 1 + 2 + 3, 105 k8nops + 1 + 2 + 3 + 4, 106 k8nops + 1 + 2 + 3 + 4 + 5, 107 k8nops + 1 + 2 + 3 + 4 + 5 + 6, 108 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 109 }; 110 #endif 111 112 #if defined(K7_NOP1) && !defined(CONFIG_X86_64) 113 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: " 114 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6 115 K7_NOP7 K7_NOP8 116 "\t.previous"); 117 extern const unsigned char k7nops[]; 118 static const unsigned char *const __initconst_or_module 119 k7_nops[ASM_NOP_MAX+1] = { 120 NULL, 121 k7nops, 122 k7nops + 1, 123 k7nops + 1 + 2, 124 k7nops + 1 + 2 + 3, 125 k7nops + 1 + 2 + 3 + 4, 126 k7nops + 1 + 2 + 3 + 4 + 5, 127 k7nops + 1 + 2 + 3 + 4 + 5 + 6, 128 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 129 }; 130 #endif 131 132 #ifdef P6_NOP1 133 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: " 134 P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6 135 P6_NOP7 P6_NOP8 136 "\t.previous"); 137 extern const unsigned char p6nops[]; 138 static const unsigned char *const __initconst_or_module 139 p6_nops[ASM_NOP_MAX+1] = { 140 NULL, 141 p6nops, 142 p6nops + 1, 143 p6nops + 1 + 2, 144 p6nops + 1 + 2 + 3, 145 p6nops + 1 + 2 + 3 + 4, 146 p6nops + 1 + 2 + 3 + 4 + 5, 147 p6nops + 1 + 2 + 3 + 4 + 5 + 6, 148 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 149 }; 150 #endif 151 152 #ifdef CONFIG_X86_64 153 154 extern char __vsyscall_0; 155 static const unsigned char *const *__init_or_module find_nop_table(void) 156 { 157 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 158 boot_cpu_has(X86_FEATURE_NOPL)) 159 return p6_nops; 160 else 161 return k8_nops; 162 } 163 164 #else /* CONFIG_X86_64 */ 165 166 static const unsigned char *const *__init_or_module find_nop_table(void) 167 { 168 if (boot_cpu_has(X86_FEATURE_K8)) 169 return k8_nops; 170 else if (boot_cpu_has(X86_FEATURE_K7)) 171 return k7_nops; 172 else if (boot_cpu_has(X86_FEATURE_NOPL)) 173 return p6_nops; 174 else 175 return intel_nops; 176 } 177 178 #endif /* CONFIG_X86_64 */ 179 180 /* Use this to add nops to a buffer, then text_poke the whole buffer. */ 181 static void __init_or_module add_nops(void *insns, unsigned int len) 182 { 183 const unsigned char *const *noptable = find_nop_table(); 184 185 while (len > 0) { 186 unsigned int noplen = len; 187 if (noplen > ASM_NOP_MAX) 188 noplen = ASM_NOP_MAX; 189 memcpy(insns, noptable[noplen], noplen); 190 insns += noplen; 191 len -= noplen; 192 } 193 } 194 195 extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 196 extern u8 *__smp_locks[], *__smp_locks_end[]; 197 static void *text_poke_early(void *addr, const void *opcode, size_t len); 198 199 /* Replace instructions with better alternatives for this CPU type. 200 This runs before SMP is initialized to avoid SMP problems with 201 self modifying code. This implies that assymetric systems where 202 APs have less capabilities than the boot processor are not handled. 203 Tough. Make sure you disable such features by hand. */ 204 205 void __init_or_module apply_alternatives(struct alt_instr *start, 206 struct alt_instr *end) 207 { 208 struct alt_instr *a; 209 u8 insnbuf[MAX_PATCH_LEN]; 210 211 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); 212 for (a = start; a < end; a++) { 213 u8 *instr = a->instr; 214 BUG_ON(a->replacementlen > a->instrlen); 215 BUG_ON(a->instrlen > sizeof(insnbuf)); 216 if (!boot_cpu_has(a->cpuid)) 217 continue; 218 #ifdef CONFIG_X86_64 219 /* vsyscall code is not mapped yet. resolve it manually. */ 220 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) { 221 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0)); 222 DPRINTK("%s: vsyscall fixup: %p => %p\n", 223 __func__, a->instr, instr); 224 } 225 #endif 226 memcpy(insnbuf, a->replacement, a->replacementlen); 227 if (*insnbuf == 0xe8 && a->replacementlen == 5) 228 *(s32 *)(insnbuf + 1) += a->replacement - a->instr; 229 add_nops(insnbuf + a->replacementlen, 230 a->instrlen - a->replacementlen); 231 text_poke_early(instr, insnbuf, a->instrlen); 232 } 233 } 234 235 #ifdef CONFIG_SMP 236 237 static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end) 238 { 239 u8 **ptr; 240 241 mutex_lock(&text_mutex); 242 for (ptr = start; ptr < end; ptr++) { 243 if (*ptr < text) 244 continue; 245 if (*ptr > text_end) 246 continue; 247 /* turn DS segment override prefix into lock prefix */ 248 text_poke(*ptr, ((unsigned char []){0xf0}), 1); 249 }; 250 mutex_unlock(&text_mutex); 251 } 252 253 static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end) 254 { 255 u8 **ptr; 256 257 if (noreplace_smp) 258 return; 259 260 mutex_lock(&text_mutex); 261 for (ptr = start; ptr < end; ptr++) { 262 if (*ptr < text) 263 continue; 264 if (*ptr > text_end) 265 continue; 266 /* turn lock prefix into DS segment override prefix */ 267 text_poke(*ptr, ((unsigned char []){0x3E}), 1); 268 }; 269 mutex_unlock(&text_mutex); 270 } 271 272 struct smp_alt_module { 273 /* what is this ??? */ 274 struct module *mod; 275 char *name; 276 277 /* ptrs to lock prefixes */ 278 u8 **locks; 279 u8 **locks_end; 280 281 /* .text segment, needed to avoid patching init code ;) */ 282 u8 *text; 283 u8 *text_end; 284 285 struct list_head next; 286 }; 287 static LIST_HEAD(smp_alt_modules); 288 static DEFINE_MUTEX(smp_alt); 289 static int smp_mode = 1; /* protected by smp_alt */ 290 291 void __init_or_module alternatives_smp_module_add(struct module *mod, 292 char *name, 293 void *locks, void *locks_end, 294 void *text, void *text_end) 295 { 296 struct smp_alt_module *smp; 297 298 if (noreplace_smp) 299 return; 300 301 if (smp_alt_once) { 302 if (boot_cpu_has(X86_FEATURE_UP)) 303 alternatives_smp_unlock(locks, locks_end, 304 text, text_end); 305 return; 306 } 307 308 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 309 if (NULL == smp) 310 return; /* we'll run the (safe but slow) SMP code then ... */ 311 312 smp->mod = mod; 313 smp->name = name; 314 smp->locks = locks; 315 smp->locks_end = locks_end; 316 smp->text = text; 317 smp->text_end = text_end; 318 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n", 319 __func__, smp->locks, smp->locks_end, 320 smp->text, smp->text_end, smp->name); 321 322 mutex_lock(&smp_alt); 323 list_add_tail(&smp->next, &smp_alt_modules); 324 if (boot_cpu_has(X86_FEATURE_UP)) 325 alternatives_smp_unlock(smp->locks, smp->locks_end, 326 smp->text, smp->text_end); 327 mutex_unlock(&smp_alt); 328 } 329 330 void __init_or_module alternatives_smp_module_del(struct module *mod) 331 { 332 struct smp_alt_module *item; 333 334 if (smp_alt_once || noreplace_smp) 335 return; 336 337 mutex_lock(&smp_alt); 338 list_for_each_entry(item, &smp_alt_modules, next) { 339 if (mod != item->mod) 340 continue; 341 list_del(&item->next); 342 mutex_unlock(&smp_alt); 343 DPRINTK("%s: %s\n", __func__, item->name); 344 kfree(item); 345 return; 346 } 347 mutex_unlock(&smp_alt); 348 } 349 350 void alternatives_smp_switch(int smp) 351 { 352 struct smp_alt_module *mod; 353 354 #ifdef CONFIG_LOCKDEP 355 /* 356 * Older binutils section handling bug prevented 357 * alternatives-replacement from working reliably. 358 * 359 * If this still occurs then you should see a hang 360 * or crash shortly after this line: 361 */ 362 printk("lockdep: fixing up alternatives.\n"); 363 #endif 364 365 if (noreplace_smp || smp_alt_once) 366 return; 367 BUG_ON(!smp && (num_online_cpus() > 1)); 368 369 mutex_lock(&smp_alt); 370 371 /* 372 * Avoid unnecessary switches because it forces JIT based VMs to 373 * throw away all cached translations, which can be quite costly. 374 */ 375 if (smp == smp_mode) { 376 /* nothing */ 377 } else if (smp) { 378 printk(KERN_INFO "SMP alternatives: switching to SMP code\n"); 379 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 380 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 381 list_for_each_entry(mod, &smp_alt_modules, next) 382 alternatives_smp_lock(mod->locks, mod->locks_end, 383 mod->text, mod->text_end); 384 } else { 385 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 386 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 387 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 388 list_for_each_entry(mod, &smp_alt_modules, next) 389 alternatives_smp_unlock(mod->locks, mod->locks_end, 390 mod->text, mod->text_end); 391 } 392 smp_mode = smp; 393 mutex_unlock(&smp_alt); 394 } 395 396 /* Return 1 if the address range is reserved for smp-alternatives */ 397 int alternatives_text_reserved(void *start, void *end) 398 { 399 struct smp_alt_module *mod; 400 u8 **ptr; 401 u8 *text_start = start; 402 u8 *text_end = end; 403 404 list_for_each_entry(mod, &smp_alt_modules, next) { 405 if (mod->text > text_end || mod->text_end < text_start) 406 continue; 407 for (ptr = mod->locks; ptr < mod->locks_end; ptr++) 408 if (text_start <= *ptr && text_end >= *ptr) 409 return 1; 410 } 411 412 return 0; 413 } 414 #endif 415 416 #ifdef CONFIG_PARAVIRT 417 void __init_or_module apply_paravirt(struct paravirt_patch_site *start, 418 struct paravirt_patch_site *end) 419 { 420 struct paravirt_patch_site *p; 421 char insnbuf[MAX_PATCH_LEN]; 422 423 if (noreplace_paravirt) 424 return; 425 426 for (p = start; p < end; p++) { 427 unsigned int used; 428 429 BUG_ON(p->len > MAX_PATCH_LEN); 430 /* prep the buffer with the original instructions */ 431 memcpy(insnbuf, p->instr, p->len); 432 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf, 433 (unsigned long)p->instr, p->len); 434 435 BUG_ON(used > p->len); 436 437 /* Pad the rest with nops */ 438 add_nops(insnbuf + used, p->len - used); 439 text_poke_early(p->instr, insnbuf, p->len); 440 } 441 } 442 extern struct paravirt_patch_site __start_parainstructions[], 443 __stop_parainstructions[]; 444 #endif /* CONFIG_PARAVIRT */ 445 446 void __init alternative_instructions(void) 447 { 448 /* The patching is not fully atomic, so try to avoid local interruptions 449 that might execute the to be patched code. 450 Other CPUs are not running. */ 451 stop_nmi(); 452 453 /* 454 * Don't stop machine check exceptions while patching. 455 * MCEs only happen when something got corrupted and in this 456 * case we must do something about the corruption. 457 * Ignoring it is worse than a unlikely patching race. 458 * Also machine checks tend to be broadcast and if one CPU 459 * goes into machine check the others follow quickly, so we don't 460 * expect a machine check to cause undue problems during to code 461 * patching. 462 */ 463 464 apply_alternatives(__alt_instructions, __alt_instructions_end); 465 466 /* switch to patch-once-at-boottime-only mode and free the 467 * tables in case we know the number of CPUs will never ever 468 * change */ 469 #ifdef CONFIG_HOTPLUG_CPU 470 if (num_possible_cpus() < 2) 471 smp_alt_once = 1; 472 #endif 473 474 #ifdef CONFIG_SMP 475 if (smp_alt_once) { 476 if (1 == num_possible_cpus()) { 477 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 478 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 479 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 480 481 alternatives_smp_unlock(__smp_locks, __smp_locks_end, 482 _text, _etext); 483 } 484 } else { 485 alternatives_smp_module_add(NULL, "core kernel", 486 __smp_locks, __smp_locks_end, 487 _text, _etext); 488 489 /* Only switch to UP mode if we don't immediately boot others */ 490 if (num_present_cpus() == 1 || setup_max_cpus <= 1) 491 alternatives_smp_switch(0); 492 } 493 #endif 494 apply_paravirt(__parainstructions, __parainstructions_end); 495 496 if (smp_alt_once) 497 free_init_pages("SMP alternatives", 498 (unsigned long)__smp_locks, 499 (unsigned long)__smp_locks_end); 500 501 restart_nmi(); 502 } 503 504 /** 505 * text_poke_early - Update instructions on a live kernel at boot time 506 * @addr: address to modify 507 * @opcode: source of the copy 508 * @len: length to copy 509 * 510 * When you use this code to patch more than one byte of an instruction 511 * you need to make sure that other CPUs cannot execute this code in parallel. 512 * Also no thread must be currently preempted in the middle of these 513 * instructions. And on the local CPU you need to be protected again NMI or MCE 514 * handlers seeing an inconsistent instruction while you patch. 515 */ 516 static void *__init_or_module text_poke_early(void *addr, const void *opcode, 517 size_t len) 518 { 519 unsigned long flags; 520 local_irq_save(flags); 521 memcpy(addr, opcode, len); 522 sync_core(); 523 local_irq_restore(flags); 524 /* Could also do a CLFLUSH here to speed up CPU recovery; but 525 that causes hangs on some VIA CPUs. */ 526 return addr; 527 } 528 529 /** 530 * text_poke - Update instructions on a live kernel 531 * @addr: address to modify 532 * @opcode: source of the copy 533 * @len: length to copy 534 * 535 * Only atomic text poke/set should be allowed when not doing early patching. 536 * It means the size must be writable atomically and the address must be aligned 537 * in a way that permits an atomic write. It also makes sure we fit on a single 538 * page. 539 * 540 * Note: Must be called under text_mutex. 541 */ 542 void *__kprobes text_poke(void *addr, const void *opcode, size_t len) 543 { 544 unsigned long flags; 545 char *vaddr; 546 struct page *pages[2]; 547 int i; 548 549 if (!core_kernel_text((unsigned long)addr)) { 550 pages[0] = vmalloc_to_page(addr); 551 pages[1] = vmalloc_to_page(addr + PAGE_SIZE); 552 } else { 553 pages[0] = virt_to_page(addr); 554 WARN_ON(!PageReserved(pages[0])); 555 pages[1] = virt_to_page(addr + PAGE_SIZE); 556 } 557 BUG_ON(!pages[0]); 558 local_irq_save(flags); 559 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0])); 560 if (pages[1]) 561 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1])); 562 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0); 563 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len); 564 clear_fixmap(FIX_TEXT_POKE0); 565 if (pages[1]) 566 clear_fixmap(FIX_TEXT_POKE1); 567 local_flush_tlb(); 568 sync_core(); 569 /* Could also do a CLFLUSH here to speed up CPU recovery; but 570 that causes hangs on some VIA CPUs. */ 571 for (i = 0; i < len; i++) 572 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]); 573 local_irq_restore(flags); 574 return addr; 575 } 576 577 /* 578 * Cross-modifying kernel text with stop_machine(). 579 * This code originally comes from immediate value. 580 */ 581 static atomic_t stop_machine_first; 582 static int wrote_text; 583 584 struct text_poke_params { 585 void *addr; 586 const void *opcode; 587 size_t len; 588 }; 589 590 static int __kprobes stop_machine_text_poke(void *data) 591 { 592 struct text_poke_params *tpp = data; 593 594 if (atomic_dec_and_test(&stop_machine_first)) { 595 text_poke(tpp->addr, tpp->opcode, tpp->len); 596 smp_wmb(); /* Make sure other cpus see that this has run */ 597 wrote_text = 1; 598 } else { 599 while (!wrote_text) 600 cpu_relax(); 601 smp_mb(); /* Load wrote_text before following execution */ 602 } 603 604 flush_icache_range((unsigned long)tpp->addr, 605 (unsigned long)tpp->addr + tpp->len); 606 return 0; 607 } 608 609 /** 610 * text_poke_smp - Update instructions on a live kernel on SMP 611 * @addr: address to modify 612 * @opcode: source of the copy 613 * @len: length to copy 614 * 615 * Modify multi-byte instruction by using stop_machine() on SMP. This allows 616 * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying 617 * should be allowed, since stop_machine() does _not_ protect code against 618 * NMI and MCE. 619 * 620 * Note: Must be called under get_online_cpus() and text_mutex. 621 */ 622 void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len) 623 { 624 struct text_poke_params tpp; 625 626 tpp.addr = addr; 627 tpp.opcode = opcode; 628 tpp.len = len; 629 atomic_set(&stop_machine_first, 1); 630 wrote_text = 0; 631 stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); 632 return addr; 633 } 634 635