1 #include <linux/module.h> 2 #include <linux/sched.h> 3 #include <linux/mutex.h> 4 #include <linux/list.h> 5 #include <linux/stringify.h> 6 #include <linux/kprobes.h> 7 #include <linux/mm.h> 8 #include <linux/vmalloc.h> 9 #include <linux/memory.h> 10 #include <linux/stop_machine.h> 11 #include <linux/slab.h> 12 #include <asm/alternative.h> 13 #include <asm/sections.h> 14 #include <asm/pgtable.h> 15 #include <asm/mce.h> 16 #include <asm/nmi.h> 17 #include <asm/vsyscall.h> 18 #include <asm/cacheflush.h> 19 #include <asm/tlbflush.h> 20 #include <asm/io.h> 21 #include <asm/fixmap.h> 22 23 #define MAX_PATCH_LEN (255-1) 24 25 #ifdef CONFIG_HOTPLUG_CPU 26 static int smp_alt_once; 27 28 static int __init bootonly(char *str) 29 { 30 smp_alt_once = 1; 31 return 1; 32 } 33 __setup("smp-alt-boot", bootonly); 34 #else 35 #define smp_alt_once 1 36 #endif 37 38 static int __initdata_or_module debug_alternative; 39 40 static int __init debug_alt(char *str) 41 { 42 debug_alternative = 1; 43 return 1; 44 } 45 __setup("debug-alternative", debug_alt); 46 47 static int noreplace_smp; 48 49 static int __init setup_noreplace_smp(char *str) 50 { 51 noreplace_smp = 1; 52 return 1; 53 } 54 __setup("noreplace-smp", setup_noreplace_smp); 55 56 #ifdef CONFIG_PARAVIRT 57 static int __initdata_or_module noreplace_paravirt = 0; 58 59 static int __init setup_noreplace_paravirt(char *str) 60 { 61 noreplace_paravirt = 1; 62 return 1; 63 } 64 __setup("noreplace-paravirt", setup_noreplace_paravirt); 65 #endif 66 67 #define DPRINTK(fmt, args...) if (debug_alternative) \ 68 printk(KERN_DEBUG fmt, args) 69 70 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64) 71 /* Use inline assembly to define this because the nops are defined 72 as inline assembly strings in the include files and we cannot 73 get them easily into strings. */ 74 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: " 75 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6 76 GENERIC_NOP7 GENERIC_NOP8 77 "\t.previous"); 78 extern const unsigned char intelnops[]; 79 static const unsigned char *const __initconst_or_module 80 intel_nops[ASM_NOP_MAX+1] = { 81 NULL, 82 intelnops, 83 intelnops + 1, 84 intelnops + 1 + 2, 85 intelnops + 1 + 2 + 3, 86 intelnops + 1 + 2 + 3 + 4, 87 intelnops + 1 + 2 + 3 + 4 + 5, 88 intelnops + 1 + 2 + 3 + 4 + 5 + 6, 89 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 90 }; 91 #endif 92 93 #ifdef K8_NOP1 94 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: " 95 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6 96 K8_NOP7 K8_NOP8 97 "\t.previous"); 98 extern const unsigned char k8nops[]; 99 static const unsigned char *const __initconst_or_module 100 k8_nops[ASM_NOP_MAX+1] = { 101 NULL, 102 k8nops, 103 k8nops + 1, 104 k8nops + 1 + 2, 105 k8nops + 1 + 2 + 3, 106 k8nops + 1 + 2 + 3 + 4, 107 k8nops + 1 + 2 + 3 + 4 + 5, 108 k8nops + 1 + 2 + 3 + 4 + 5 + 6, 109 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 110 }; 111 #endif 112 113 #if defined(K7_NOP1) && !defined(CONFIG_X86_64) 114 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: " 115 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6 116 K7_NOP7 K7_NOP8 117 "\t.previous"); 118 extern const unsigned char k7nops[]; 119 static const unsigned char *const __initconst_or_module 120 k7_nops[ASM_NOP_MAX+1] = { 121 NULL, 122 k7nops, 123 k7nops + 1, 124 k7nops + 1 + 2, 125 k7nops + 1 + 2 + 3, 126 k7nops + 1 + 2 + 3 + 4, 127 k7nops + 1 + 2 + 3 + 4 + 5, 128 k7nops + 1 + 2 + 3 + 4 + 5 + 6, 129 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 130 }; 131 #endif 132 133 #ifdef P6_NOP1 134 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: " 135 P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6 136 P6_NOP7 P6_NOP8 137 "\t.previous"); 138 extern const unsigned char p6nops[]; 139 static const unsigned char *const __initconst_or_module 140 p6_nops[ASM_NOP_MAX+1] = { 141 NULL, 142 p6nops, 143 p6nops + 1, 144 p6nops + 1 + 2, 145 p6nops + 1 + 2 + 3, 146 p6nops + 1 + 2 + 3 + 4, 147 p6nops + 1 + 2 + 3 + 4 + 5, 148 p6nops + 1 + 2 + 3 + 4 + 5 + 6, 149 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 150 }; 151 #endif 152 153 #ifdef CONFIG_X86_64 154 155 extern char __vsyscall_0; 156 static const unsigned char *const *__init_or_module find_nop_table(void) 157 { 158 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 159 boot_cpu_has(X86_FEATURE_NOPL)) 160 return p6_nops; 161 else 162 return k8_nops; 163 } 164 165 #else /* CONFIG_X86_64 */ 166 167 static const unsigned char *const *__init_or_module find_nop_table(void) 168 { 169 if (boot_cpu_has(X86_FEATURE_K8)) 170 return k8_nops; 171 else if (boot_cpu_has(X86_FEATURE_K7)) 172 return k7_nops; 173 else if (boot_cpu_has(X86_FEATURE_NOPL)) 174 return p6_nops; 175 else 176 return intel_nops; 177 } 178 179 #endif /* CONFIG_X86_64 */ 180 181 /* Use this to add nops to a buffer, then text_poke the whole buffer. */ 182 static void __init_or_module add_nops(void *insns, unsigned int len) 183 { 184 const unsigned char *const *noptable = find_nop_table(); 185 186 while (len > 0) { 187 unsigned int noplen = len; 188 if (noplen > ASM_NOP_MAX) 189 noplen = ASM_NOP_MAX; 190 memcpy(insns, noptable[noplen], noplen); 191 insns += noplen; 192 len -= noplen; 193 } 194 } 195 196 extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 197 extern s32 __smp_locks[], __smp_locks_end[]; 198 static void *text_poke_early(void *addr, const void *opcode, size_t len); 199 200 /* Replace instructions with better alternatives for this CPU type. 201 This runs before SMP is initialized to avoid SMP problems with 202 self modifying code. This implies that assymetric systems where 203 APs have less capabilities than the boot processor are not handled. 204 Tough. Make sure you disable such features by hand. */ 205 206 void __init_or_module apply_alternatives(struct alt_instr *start, 207 struct alt_instr *end) 208 { 209 struct alt_instr *a; 210 u8 insnbuf[MAX_PATCH_LEN]; 211 212 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); 213 for (a = start; a < end; a++) { 214 u8 *instr = a->instr; 215 BUG_ON(a->replacementlen > a->instrlen); 216 BUG_ON(a->instrlen > sizeof(insnbuf)); 217 if (!boot_cpu_has(a->cpuid)) 218 continue; 219 #ifdef CONFIG_X86_64 220 /* vsyscall code is not mapped yet. resolve it manually. */ 221 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) { 222 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0)); 223 DPRINTK("%s: vsyscall fixup: %p => %p\n", 224 __func__, a->instr, instr); 225 } 226 #endif 227 memcpy(insnbuf, a->replacement, a->replacementlen); 228 if (*insnbuf == 0xe8 && a->replacementlen == 5) 229 *(s32 *)(insnbuf + 1) += a->replacement - a->instr; 230 add_nops(insnbuf + a->replacementlen, 231 a->instrlen - a->replacementlen); 232 text_poke_early(instr, insnbuf, a->instrlen); 233 } 234 } 235 236 #ifdef CONFIG_SMP 237 238 static void alternatives_smp_lock(const s32 *start, const s32 *end, 239 u8 *text, u8 *text_end) 240 { 241 const s32 *poff; 242 243 mutex_lock(&text_mutex); 244 for (poff = start; poff < end; poff++) { 245 u8 *ptr = (u8 *)poff + *poff; 246 247 if (!*poff || ptr < text || ptr >= text_end) 248 continue; 249 /* turn DS segment override prefix into lock prefix */ 250 if (*ptr == 0x3e) 251 text_poke(ptr, ((unsigned char []){0xf0}), 1); 252 }; 253 mutex_unlock(&text_mutex); 254 } 255 256 static void alternatives_smp_unlock(const s32 *start, const s32 *end, 257 u8 *text, u8 *text_end) 258 { 259 const s32 *poff; 260 261 if (noreplace_smp) 262 return; 263 264 mutex_lock(&text_mutex); 265 for (poff = start; poff < end; poff++) { 266 u8 *ptr = (u8 *)poff + *poff; 267 268 if (!*poff || ptr < text || ptr >= text_end) 269 continue; 270 /* turn lock prefix into DS segment override prefix */ 271 if (*ptr == 0xf0) 272 text_poke(ptr, ((unsigned char []){0x3E}), 1); 273 }; 274 mutex_unlock(&text_mutex); 275 } 276 277 struct smp_alt_module { 278 /* what is this ??? */ 279 struct module *mod; 280 char *name; 281 282 /* ptrs to lock prefixes */ 283 const s32 *locks; 284 const s32 *locks_end; 285 286 /* .text segment, needed to avoid patching init code ;) */ 287 u8 *text; 288 u8 *text_end; 289 290 struct list_head next; 291 }; 292 static LIST_HEAD(smp_alt_modules); 293 static DEFINE_MUTEX(smp_alt); 294 static int smp_mode = 1; /* protected by smp_alt */ 295 296 void __init_or_module alternatives_smp_module_add(struct module *mod, 297 char *name, 298 void *locks, void *locks_end, 299 void *text, void *text_end) 300 { 301 struct smp_alt_module *smp; 302 303 if (noreplace_smp) 304 return; 305 306 if (smp_alt_once) { 307 if (boot_cpu_has(X86_FEATURE_UP)) 308 alternatives_smp_unlock(locks, locks_end, 309 text, text_end); 310 return; 311 } 312 313 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 314 if (NULL == smp) 315 return; /* we'll run the (safe but slow) SMP code then ... */ 316 317 smp->mod = mod; 318 smp->name = name; 319 smp->locks = locks; 320 smp->locks_end = locks_end; 321 smp->text = text; 322 smp->text_end = text_end; 323 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n", 324 __func__, smp->locks, smp->locks_end, 325 smp->text, smp->text_end, smp->name); 326 327 mutex_lock(&smp_alt); 328 list_add_tail(&smp->next, &smp_alt_modules); 329 if (boot_cpu_has(X86_FEATURE_UP)) 330 alternatives_smp_unlock(smp->locks, smp->locks_end, 331 smp->text, smp->text_end); 332 mutex_unlock(&smp_alt); 333 } 334 335 void __init_or_module alternatives_smp_module_del(struct module *mod) 336 { 337 struct smp_alt_module *item; 338 339 if (smp_alt_once || noreplace_smp) 340 return; 341 342 mutex_lock(&smp_alt); 343 list_for_each_entry(item, &smp_alt_modules, next) { 344 if (mod != item->mod) 345 continue; 346 list_del(&item->next); 347 mutex_unlock(&smp_alt); 348 DPRINTK("%s: %s\n", __func__, item->name); 349 kfree(item); 350 return; 351 } 352 mutex_unlock(&smp_alt); 353 } 354 355 void alternatives_smp_switch(int smp) 356 { 357 struct smp_alt_module *mod; 358 359 #ifdef CONFIG_LOCKDEP 360 /* 361 * Older binutils section handling bug prevented 362 * alternatives-replacement from working reliably. 363 * 364 * If this still occurs then you should see a hang 365 * or crash shortly after this line: 366 */ 367 printk("lockdep: fixing up alternatives.\n"); 368 #endif 369 370 if (noreplace_smp || smp_alt_once) 371 return; 372 BUG_ON(!smp && (num_online_cpus() > 1)); 373 374 mutex_lock(&smp_alt); 375 376 /* 377 * Avoid unnecessary switches because it forces JIT based VMs to 378 * throw away all cached translations, which can be quite costly. 379 */ 380 if (smp == smp_mode) { 381 /* nothing */ 382 } else if (smp) { 383 printk(KERN_INFO "SMP alternatives: switching to SMP code\n"); 384 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 385 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 386 list_for_each_entry(mod, &smp_alt_modules, next) 387 alternatives_smp_lock(mod->locks, mod->locks_end, 388 mod->text, mod->text_end); 389 } else { 390 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 391 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 392 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 393 list_for_each_entry(mod, &smp_alt_modules, next) 394 alternatives_smp_unlock(mod->locks, mod->locks_end, 395 mod->text, mod->text_end); 396 } 397 smp_mode = smp; 398 mutex_unlock(&smp_alt); 399 } 400 401 /* Return 1 if the address range is reserved for smp-alternatives */ 402 int alternatives_text_reserved(void *start, void *end) 403 { 404 struct smp_alt_module *mod; 405 const s32 *poff; 406 u8 *text_start = start; 407 u8 *text_end = end; 408 409 list_for_each_entry(mod, &smp_alt_modules, next) { 410 if (mod->text > text_end || mod->text_end < text_start) 411 continue; 412 for (poff = mod->locks; poff < mod->locks_end; poff++) { 413 const u8 *ptr = (const u8 *)poff + *poff; 414 415 if (text_start <= ptr && text_end > ptr) 416 return 1; 417 } 418 } 419 420 return 0; 421 } 422 #endif 423 424 #ifdef CONFIG_PARAVIRT 425 void __init_or_module apply_paravirt(struct paravirt_patch_site *start, 426 struct paravirt_patch_site *end) 427 { 428 struct paravirt_patch_site *p; 429 char insnbuf[MAX_PATCH_LEN]; 430 431 if (noreplace_paravirt) 432 return; 433 434 for (p = start; p < end; p++) { 435 unsigned int used; 436 437 BUG_ON(p->len > MAX_PATCH_LEN); 438 /* prep the buffer with the original instructions */ 439 memcpy(insnbuf, p->instr, p->len); 440 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf, 441 (unsigned long)p->instr, p->len); 442 443 BUG_ON(used > p->len); 444 445 /* Pad the rest with nops */ 446 add_nops(insnbuf + used, p->len - used); 447 text_poke_early(p->instr, insnbuf, p->len); 448 } 449 } 450 extern struct paravirt_patch_site __start_parainstructions[], 451 __stop_parainstructions[]; 452 #endif /* CONFIG_PARAVIRT */ 453 454 void __init alternative_instructions(void) 455 { 456 /* The patching is not fully atomic, so try to avoid local interruptions 457 that might execute the to be patched code. 458 Other CPUs are not running. */ 459 stop_nmi(); 460 461 /* 462 * Don't stop machine check exceptions while patching. 463 * MCEs only happen when something got corrupted and in this 464 * case we must do something about the corruption. 465 * Ignoring it is worse than a unlikely patching race. 466 * Also machine checks tend to be broadcast and if one CPU 467 * goes into machine check the others follow quickly, so we don't 468 * expect a machine check to cause undue problems during to code 469 * patching. 470 */ 471 472 apply_alternatives(__alt_instructions, __alt_instructions_end); 473 474 /* switch to patch-once-at-boottime-only mode and free the 475 * tables in case we know the number of CPUs will never ever 476 * change */ 477 #ifdef CONFIG_HOTPLUG_CPU 478 if (num_possible_cpus() < 2) 479 smp_alt_once = 1; 480 #endif 481 482 #ifdef CONFIG_SMP 483 if (smp_alt_once) { 484 if (1 == num_possible_cpus()) { 485 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 486 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 487 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 488 489 alternatives_smp_unlock(__smp_locks, __smp_locks_end, 490 _text, _etext); 491 } 492 } else { 493 alternatives_smp_module_add(NULL, "core kernel", 494 __smp_locks, __smp_locks_end, 495 _text, _etext); 496 497 /* Only switch to UP mode if we don't immediately boot others */ 498 if (num_present_cpus() == 1 || setup_max_cpus <= 1) 499 alternatives_smp_switch(0); 500 } 501 #endif 502 apply_paravirt(__parainstructions, __parainstructions_end); 503 504 if (smp_alt_once) 505 free_init_pages("SMP alternatives", 506 (unsigned long)__smp_locks, 507 (unsigned long)__smp_locks_end); 508 509 restart_nmi(); 510 } 511 512 /** 513 * text_poke_early - Update instructions on a live kernel at boot time 514 * @addr: address to modify 515 * @opcode: source of the copy 516 * @len: length to copy 517 * 518 * When you use this code to patch more than one byte of an instruction 519 * you need to make sure that other CPUs cannot execute this code in parallel. 520 * Also no thread must be currently preempted in the middle of these 521 * instructions. And on the local CPU you need to be protected again NMI or MCE 522 * handlers seeing an inconsistent instruction while you patch. 523 */ 524 static void *__init_or_module text_poke_early(void *addr, const void *opcode, 525 size_t len) 526 { 527 unsigned long flags; 528 local_irq_save(flags); 529 memcpy(addr, opcode, len); 530 sync_core(); 531 local_irq_restore(flags); 532 /* Could also do a CLFLUSH here to speed up CPU recovery; but 533 that causes hangs on some VIA CPUs. */ 534 return addr; 535 } 536 537 /** 538 * text_poke - Update instructions on a live kernel 539 * @addr: address to modify 540 * @opcode: source of the copy 541 * @len: length to copy 542 * 543 * Only atomic text poke/set should be allowed when not doing early patching. 544 * It means the size must be writable atomically and the address must be aligned 545 * in a way that permits an atomic write. It also makes sure we fit on a single 546 * page. 547 * 548 * Note: Must be called under text_mutex. 549 */ 550 void *__kprobes text_poke(void *addr, const void *opcode, size_t len) 551 { 552 unsigned long flags; 553 char *vaddr; 554 struct page *pages[2]; 555 int i; 556 557 if (!core_kernel_text((unsigned long)addr)) { 558 pages[0] = vmalloc_to_page(addr); 559 pages[1] = vmalloc_to_page(addr + PAGE_SIZE); 560 } else { 561 pages[0] = virt_to_page(addr); 562 WARN_ON(!PageReserved(pages[0])); 563 pages[1] = virt_to_page(addr + PAGE_SIZE); 564 } 565 BUG_ON(!pages[0]); 566 local_irq_save(flags); 567 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0])); 568 if (pages[1]) 569 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1])); 570 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0); 571 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len); 572 clear_fixmap(FIX_TEXT_POKE0); 573 if (pages[1]) 574 clear_fixmap(FIX_TEXT_POKE1); 575 local_flush_tlb(); 576 sync_core(); 577 /* Could also do a CLFLUSH here to speed up CPU recovery; but 578 that causes hangs on some VIA CPUs. */ 579 for (i = 0; i < len; i++) 580 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]); 581 local_irq_restore(flags); 582 return addr; 583 } 584 585 /* 586 * Cross-modifying kernel text with stop_machine(). 587 * This code originally comes from immediate value. 588 */ 589 static atomic_t stop_machine_first; 590 static int wrote_text; 591 592 struct text_poke_params { 593 void *addr; 594 const void *opcode; 595 size_t len; 596 }; 597 598 static int __kprobes stop_machine_text_poke(void *data) 599 { 600 struct text_poke_params *tpp = data; 601 602 if (atomic_dec_and_test(&stop_machine_first)) { 603 text_poke(tpp->addr, tpp->opcode, tpp->len); 604 smp_wmb(); /* Make sure other cpus see that this has run */ 605 wrote_text = 1; 606 } else { 607 while (!wrote_text) 608 cpu_relax(); 609 smp_mb(); /* Load wrote_text before following execution */ 610 } 611 612 flush_icache_range((unsigned long)tpp->addr, 613 (unsigned long)tpp->addr + tpp->len); 614 return 0; 615 } 616 617 /** 618 * text_poke_smp - Update instructions on a live kernel on SMP 619 * @addr: address to modify 620 * @opcode: source of the copy 621 * @len: length to copy 622 * 623 * Modify multi-byte instruction by using stop_machine() on SMP. This allows 624 * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying 625 * should be allowed, since stop_machine() does _not_ protect code against 626 * NMI and MCE. 627 * 628 * Note: Must be called under get_online_cpus() and text_mutex. 629 */ 630 void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len) 631 { 632 struct text_poke_params tpp; 633 634 tpp.addr = addr; 635 tpp.opcode = opcode; 636 tpp.len = len; 637 atomic_set(&stop_machine_first, 1); 638 wrote_text = 0; 639 stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); 640 return addr; 641 } 642 643