1 #include <linux/module.h> 2 #include <linux/sched.h> 3 #include <linux/mutex.h> 4 #include <linux/list.h> 5 #include <linux/stringify.h> 6 #include <linux/kprobes.h> 7 #include <linux/mm.h> 8 #include <linux/vmalloc.h> 9 #include <linux/memory.h> 10 #include <linux/stop_machine.h> 11 #include <linux/slab.h> 12 #include <asm/alternative.h> 13 #include <asm/sections.h> 14 #include <asm/pgtable.h> 15 #include <asm/mce.h> 16 #include <asm/nmi.h> 17 #include <asm/cacheflush.h> 18 #include <asm/tlbflush.h> 19 #include <asm/io.h> 20 #include <asm/fixmap.h> 21 22 #define MAX_PATCH_LEN (255-1) 23 24 #ifdef CONFIG_HOTPLUG_CPU 25 static int smp_alt_once; 26 27 static int __init bootonly(char *str) 28 { 29 smp_alt_once = 1; 30 return 1; 31 } 32 __setup("smp-alt-boot", bootonly); 33 #else 34 #define smp_alt_once 1 35 #endif 36 37 static int __initdata_or_module debug_alternative; 38 39 static int __init debug_alt(char *str) 40 { 41 debug_alternative = 1; 42 return 1; 43 } 44 __setup("debug-alternative", debug_alt); 45 46 static int noreplace_smp; 47 48 static int __init setup_noreplace_smp(char *str) 49 { 50 noreplace_smp = 1; 51 return 1; 52 } 53 __setup("noreplace-smp", setup_noreplace_smp); 54 55 #ifdef CONFIG_PARAVIRT 56 static int __initdata_or_module noreplace_paravirt = 0; 57 58 static int __init setup_noreplace_paravirt(char *str) 59 { 60 noreplace_paravirt = 1; 61 return 1; 62 } 63 __setup("noreplace-paravirt", setup_noreplace_paravirt); 64 #endif 65 66 #define DPRINTK(fmt, args...) if (debug_alternative) \ 67 printk(KERN_DEBUG fmt, args) 68 69 /* 70 * Each GENERIC_NOPX is of X bytes, and defined as an array of bytes 71 * that correspond to that nop. Getting from one nop to the next, we 72 * add to the array the offset that is equal to the sum of all sizes of 73 * nops preceding the one we are after. 74 * 75 * Note: The GENERIC_NOP5_ATOMIC is at the end, as it breaks the 76 * nice symmetry of sizes of the previous nops. 77 */ 78 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64) 79 static const unsigned char intelnops[] = 80 { 81 GENERIC_NOP1, 82 GENERIC_NOP2, 83 GENERIC_NOP3, 84 GENERIC_NOP4, 85 GENERIC_NOP5, 86 GENERIC_NOP6, 87 GENERIC_NOP7, 88 GENERIC_NOP8, 89 GENERIC_NOP5_ATOMIC 90 }; 91 static const unsigned char * const intel_nops[ASM_NOP_MAX+2] = 92 { 93 NULL, 94 intelnops, 95 intelnops + 1, 96 intelnops + 1 + 2, 97 intelnops + 1 + 2 + 3, 98 intelnops + 1 + 2 + 3 + 4, 99 intelnops + 1 + 2 + 3 + 4 + 5, 100 intelnops + 1 + 2 + 3 + 4 + 5 + 6, 101 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 102 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 103 }; 104 #endif 105 106 #ifdef K8_NOP1 107 static const unsigned char k8nops[] = 108 { 109 K8_NOP1, 110 K8_NOP2, 111 K8_NOP3, 112 K8_NOP4, 113 K8_NOP5, 114 K8_NOP6, 115 K8_NOP7, 116 K8_NOP8, 117 K8_NOP5_ATOMIC 118 }; 119 static const unsigned char * const k8_nops[ASM_NOP_MAX+2] = 120 { 121 NULL, 122 k8nops, 123 k8nops + 1, 124 k8nops + 1 + 2, 125 k8nops + 1 + 2 + 3, 126 k8nops + 1 + 2 + 3 + 4, 127 k8nops + 1 + 2 + 3 + 4 + 5, 128 k8nops + 1 + 2 + 3 + 4 + 5 + 6, 129 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 130 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 131 }; 132 #endif 133 134 #if defined(K7_NOP1) && !defined(CONFIG_X86_64) 135 static const unsigned char k7nops[] = 136 { 137 K7_NOP1, 138 K7_NOP2, 139 K7_NOP3, 140 K7_NOP4, 141 K7_NOP5, 142 K7_NOP6, 143 K7_NOP7, 144 K7_NOP8, 145 K7_NOP5_ATOMIC 146 }; 147 static const unsigned char * const k7_nops[ASM_NOP_MAX+2] = 148 { 149 NULL, 150 k7nops, 151 k7nops + 1, 152 k7nops + 1 + 2, 153 k7nops + 1 + 2 + 3, 154 k7nops + 1 + 2 + 3 + 4, 155 k7nops + 1 + 2 + 3 + 4 + 5, 156 k7nops + 1 + 2 + 3 + 4 + 5 + 6, 157 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 158 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 159 }; 160 #endif 161 162 #ifdef P6_NOP1 163 static const unsigned char __initconst_or_module p6nops[] = 164 { 165 P6_NOP1, 166 P6_NOP2, 167 P6_NOP3, 168 P6_NOP4, 169 P6_NOP5, 170 P6_NOP6, 171 P6_NOP7, 172 P6_NOP8, 173 P6_NOP5_ATOMIC 174 }; 175 static const unsigned char * const p6_nops[ASM_NOP_MAX+2] = 176 { 177 NULL, 178 p6nops, 179 p6nops + 1, 180 p6nops + 1 + 2, 181 p6nops + 1 + 2 + 3, 182 p6nops + 1 + 2 + 3 + 4, 183 p6nops + 1 + 2 + 3 + 4 + 5, 184 p6nops + 1 + 2 + 3 + 4 + 5 + 6, 185 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 186 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 187 }; 188 #endif 189 190 /* Initialize these to a safe default */ 191 #ifdef CONFIG_X86_64 192 const unsigned char * const *ideal_nops = p6_nops; 193 #else 194 const unsigned char * const *ideal_nops = intel_nops; 195 #endif 196 197 void __init arch_init_ideal_nops(void) 198 { 199 switch (boot_cpu_data.x86_vendor) { 200 case X86_VENDOR_INTEL: 201 /* 202 * Due to a decoder implementation quirk, some 203 * specific Intel CPUs actually perform better with 204 * the "k8_nops" than with the SDM-recommended NOPs. 205 */ 206 if (boot_cpu_data.x86 == 6 && 207 boot_cpu_data.x86_model >= 0x0f && 208 boot_cpu_data.x86_model != 0x1c && 209 boot_cpu_data.x86_model != 0x26 && 210 boot_cpu_data.x86_model != 0x27 && 211 boot_cpu_data.x86_model < 0x30) { 212 ideal_nops = k8_nops; 213 } else if (boot_cpu_has(X86_FEATURE_NOPL)) { 214 ideal_nops = p6_nops; 215 } else { 216 #ifdef CONFIG_X86_64 217 ideal_nops = k8_nops; 218 #else 219 ideal_nops = intel_nops; 220 #endif 221 } 222 223 default: 224 #ifdef CONFIG_X86_64 225 ideal_nops = k8_nops; 226 #else 227 if (boot_cpu_has(X86_FEATURE_K8)) 228 ideal_nops = k8_nops; 229 else if (boot_cpu_has(X86_FEATURE_K7)) 230 ideal_nops = k7_nops; 231 else 232 ideal_nops = intel_nops; 233 #endif 234 } 235 } 236 237 /* Use this to add nops to a buffer, then text_poke the whole buffer. */ 238 static void __init_or_module add_nops(void *insns, unsigned int len) 239 { 240 while (len > 0) { 241 unsigned int noplen = len; 242 if (noplen > ASM_NOP_MAX) 243 noplen = ASM_NOP_MAX; 244 memcpy(insns, ideal_nops[noplen], noplen); 245 insns += noplen; 246 len -= noplen; 247 } 248 } 249 250 extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 251 extern s32 __smp_locks[], __smp_locks_end[]; 252 void *text_poke_early(void *addr, const void *opcode, size_t len); 253 254 /* Replace instructions with better alternatives for this CPU type. 255 This runs before SMP is initialized to avoid SMP problems with 256 self modifying code. This implies that asymmetric systems where 257 APs have less capabilities than the boot processor are not handled. 258 Tough. Make sure you disable such features by hand. */ 259 260 void __init_or_module apply_alternatives(struct alt_instr *start, 261 struct alt_instr *end) 262 { 263 struct alt_instr *a; 264 u8 *instr, *replacement; 265 u8 insnbuf[MAX_PATCH_LEN]; 266 267 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); 268 /* 269 * The scan order should be from start to end. A later scanned 270 * alternative code can overwrite a previous scanned alternative code. 271 * Some kernel functions (e.g. memcpy, memset, etc) use this order to 272 * patch code. 273 * 274 * So be careful if you want to change the scan order to any other 275 * order. 276 */ 277 for (a = start; a < end; a++) { 278 instr = (u8 *)&a->instr_offset + a->instr_offset; 279 replacement = (u8 *)&a->repl_offset + a->repl_offset; 280 BUG_ON(a->replacementlen > a->instrlen); 281 BUG_ON(a->instrlen > sizeof(insnbuf)); 282 BUG_ON(a->cpuid >= NCAPINTS*32); 283 if (!boot_cpu_has(a->cpuid)) 284 continue; 285 286 memcpy(insnbuf, replacement, a->replacementlen); 287 288 /* 0xe8 is a relative jump; fix the offset. */ 289 if (*insnbuf == 0xe8 && a->replacementlen == 5) 290 *(s32 *)(insnbuf + 1) += replacement - instr; 291 292 add_nops(insnbuf + a->replacementlen, 293 a->instrlen - a->replacementlen); 294 295 text_poke_early(instr, insnbuf, a->instrlen); 296 } 297 } 298 299 #ifdef CONFIG_SMP 300 301 static void alternatives_smp_lock(const s32 *start, const s32 *end, 302 u8 *text, u8 *text_end) 303 { 304 const s32 *poff; 305 306 mutex_lock(&text_mutex); 307 for (poff = start; poff < end; poff++) { 308 u8 *ptr = (u8 *)poff + *poff; 309 310 if (!*poff || ptr < text || ptr >= text_end) 311 continue; 312 /* turn DS segment override prefix into lock prefix */ 313 if (*ptr == 0x3e) 314 text_poke(ptr, ((unsigned char []){0xf0}), 1); 315 }; 316 mutex_unlock(&text_mutex); 317 } 318 319 static void alternatives_smp_unlock(const s32 *start, const s32 *end, 320 u8 *text, u8 *text_end) 321 { 322 const s32 *poff; 323 324 if (noreplace_smp) 325 return; 326 327 mutex_lock(&text_mutex); 328 for (poff = start; poff < end; poff++) { 329 u8 *ptr = (u8 *)poff + *poff; 330 331 if (!*poff || ptr < text || ptr >= text_end) 332 continue; 333 /* turn lock prefix into DS segment override prefix */ 334 if (*ptr == 0xf0) 335 text_poke(ptr, ((unsigned char []){0x3E}), 1); 336 }; 337 mutex_unlock(&text_mutex); 338 } 339 340 struct smp_alt_module { 341 /* what is this ??? */ 342 struct module *mod; 343 char *name; 344 345 /* ptrs to lock prefixes */ 346 const s32 *locks; 347 const s32 *locks_end; 348 349 /* .text segment, needed to avoid patching init code ;) */ 350 u8 *text; 351 u8 *text_end; 352 353 struct list_head next; 354 }; 355 static LIST_HEAD(smp_alt_modules); 356 static DEFINE_MUTEX(smp_alt); 357 static int smp_mode = 1; /* protected by smp_alt */ 358 359 void __init_or_module alternatives_smp_module_add(struct module *mod, 360 char *name, 361 void *locks, void *locks_end, 362 void *text, void *text_end) 363 { 364 struct smp_alt_module *smp; 365 366 if (noreplace_smp) 367 return; 368 369 if (smp_alt_once) { 370 if (boot_cpu_has(X86_FEATURE_UP)) 371 alternatives_smp_unlock(locks, locks_end, 372 text, text_end); 373 return; 374 } 375 376 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 377 if (NULL == smp) 378 return; /* we'll run the (safe but slow) SMP code then ... */ 379 380 smp->mod = mod; 381 smp->name = name; 382 smp->locks = locks; 383 smp->locks_end = locks_end; 384 smp->text = text; 385 smp->text_end = text_end; 386 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n", 387 __func__, smp->locks, smp->locks_end, 388 smp->text, smp->text_end, smp->name); 389 390 mutex_lock(&smp_alt); 391 list_add_tail(&smp->next, &smp_alt_modules); 392 if (boot_cpu_has(X86_FEATURE_UP)) 393 alternatives_smp_unlock(smp->locks, smp->locks_end, 394 smp->text, smp->text_end); 395 mutex_unlock(&smp_alt); 396 } 397 398 void __init_or_module alternatives_smp_module_del(struct module *mod) 399 { 400 struct smp_alt_module *item; 401 402 if (smp_alt_once || noreplace_smp) 403 return; 404 405 mutex_lock(&smp_alt); 406 list_for_each_entry(item, &smp_alt_modules, next) { 407 if (mod != item->mod) 408 continue; 409 list_del(&item->next); 410 mutex_unlock(&smp_alt); 411 DPRINTK("%s: %s\n", __func__, item->name); 412 kfree(item); 413 return; 414 } 415 mutex_unlock(&smp_alt); 416 } 417 418 bool skip_smp_alternatives; 419 void alternatives_smp_switch(int smp) 420 { 421 struct smp_alt_module *mod; 422 423 #ifdef CONFIG_LOCKDEP 424 /* 425 * Older binutils section handling bug prevented 426 * alternatives-replacement from working reliably. 427 * 428 * If this still occurs then you should see a hang 429 * or crash shortly after this line: 430 */ 431 printk("lockdep: fixing up alternatives.\n"); 432 #endif 433 434 if (noreplace_smp || smp_alt_once || skip_smp_alternatives) 435 return; 436 BUG_ON(!smp && (num_online_cpus() > 1)); 437 438 mutex_lock(&smp_alt); 439 440 /* 441 * Avoid unnecessary switches because it forces JIT based VMs to 442 * throw away all cached translations, which can be quite costly. 443 */ 444 if (smp == smp_mode) { 445 /* nothing */ 446 } else if (smp) { 447 printk(KERN_INFO "SMP alternatives: switching to SMP code\n"); 448 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 449 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 450 list_for_each_entry(mod, &smp_alt_modules, next) 451 alternatives_smp_lock(mod->locks, mod->locks_end, 452 mod->text, mod->text_end); 453 } else { 454 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 455 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 456 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 457 list_for_each_entry(mod, &smp_alt_modules, next) 458 alternatives_smp_unlock(mod->locks, mod->locks_end, 459 mod->text, mod->text_end); 460 } 461 smp_mode = smp; 462 mutex_unlock(&smp_alt); 463 } 464 465 /* Return 1 if the address range is reserved for smp-alternatives */ 466 int alternatives_text_reserved(void *start, void *end) 467 { 468 struct smp_alt_module *mod; 469 const s32 *poff; 470 u8 *text_start = start; 471 u8 *text_end = end; 472 473 list_for_each_entry(mod, &smp_alt_modules, next) { 474 if (mod->text > text_end || mod->text_end < text_start) 475 continue; 476 for (poff = mod->locks; poff < mod->locks_end; poff++) { 477 const u8 *ptr = (const u8 *)poff + *poff; 478 479 if (text_start <= ptr && text_end > ptr) 480 return 1; 481 } 482 } 483 484 return 0; 485 } 486 #endif 487 488 #ifdef CONFIG_PARAVIRT 489 void __init_or_module apply_paravirt(struct paravirt_patch_site *start, 490 struct paravirt_patch_site *end) 491 { 492 struct paravirt_patch_site *p; 493 char insnbuf[MAX_PATCH_LEN]; 494 495 if (noreplace_paravirt) 496 return; 497 498 for (p = start; p < end; p++) { 499 unsigned int used; 500 501 BUG_ON(p->len > MAX_PATCH_LEN); 502 /* prep the buffer with the original instructions */ 503 memcpy(insnbuf, p->instr, p->len); 504 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf, 505 (unsigned long)p->instr, p->len); 506 507 BUG_ON(used > p->len); 508 509 /* Pad the rest with nops */ 510 add_nops(insnbuf + used, p->len - used); 511 text_poke_early(p->instr, insnbuf, p->len); 512 } 513 } 514 extern struct paravirt_patch_site __start_parainstructions[], 515 __stop_parainstructions[]; 516 #endif /* CONFIG_PARAVIRT */ 517 518 void __init alternative_instructions(void) 519 { 520 /* The patching is not fully atomic, so try to avoid local interruptions 521 that might execute the to be patched code. 522 Other CPUs are not running. */ 523 stop_nmi(); 524 525 /* 526 * Don't stop machine check exceptions while patching. 527 * MCEs only happen when something got corrupted and in this 528 * case we must do something about the corruption. 529 * Ignoring it is worse than a unlikely patching race. 530 * Also machine checks tend to be broadcast and if one CPU 531 * goes into machine check the others follow quickly, so we don't 532 * expect a machine check to cause undue problems during to code 533 * patching. 534 */ 535 536 apply_alternatives(__alt_instructions, __alt_instructions_end); 537 538 /* switch to patch-once-at-boottime-only mode and free the 539 * tables in case we know the number of CPUs will never ever 540 * change */ 541 #ifdef CONFIG_HOTPLUG_CPU 542 if (num_possible_cpus() < 2) 543 smp_alt_once = 1; 544 #endif 545 546 #ifdef CONFIG_SMP 547 if (smp_alt_once) { 548 if (1 == num_possible_cpus()) { 549 printk(KERN_INFO "SMP alternatives: switching to UP code\n"); 550 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 551 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 552 553 alternatives_smp_unlock(__smp_locks, __smp_locks_end, 554 _text, _etext); 555 } 556 } else { 557 alternatives_smp_module_add(NULL, "core kernel", 558 __smp_locks, __smp_locks_end, 559 _text, _etext); 560 561 /* Only switch to UP mode if we don't immediately boot others */ 562 if (num_present_cpus() == 1 || setup_max_cpus <= 1) 563 alternatives_smp_switch(0); 564 } 565 #endif 566 apply_paravirt(__parainstructions, __parainstructions_end); 567 568 if (smp_alt_once) 569 free_init_pages("SMP alternatives", 570 (unsigned long)__smp_locks, 571 (unsigned long)__smp_locks_end); 572 573 restart_nmi(); 574 } 575 576 /** 577 * text_poke_early - Update instructions on a live kernel at boot time 578 * @addr: address to modify 579 * @opcode: source of the copy 580 * @len: length to copy 581 * 582 * When you use this code to patch more than one byte of an instruction 583 * you need to make sure that other CPUs cannot execute this code in parallel. 584 * Also no thread must be currently preempted in the middle of these 585 * instructions. And on the local CPU you need to be protected again NMI or MCE 586 * handlers seeing an inconsistent instruction while you patch. 587 */ 588 void *__init_or_module text_poke_early(void *addr, const void *opcode, 589 size_t len) 590 { 591 unsigned long flags; 592 local_irq_save(flags); 593 memcpy(addr, opcode, len); 594 sync_core(); 595 local_irq_restore(flags); 596 /* Could also do a CLFLUSH here to speed up CPU recovery; but 597 that causes hangs on some VIA CPUs. */ 598 return addr; 599 } 600 601 /** 602 * text_poke - Update instructions on a live kernel 603 * @addr: address to modify 604 * @opcode: source of the copy 605 * @len: length to copy 606 * 607 * Only atomic text poke/set should be allowed when not doing early patching. 608 * It means the size must be writable atomically and the address must be aligned 609 * in a way that permits an atomic write. It also makes sure we fit on a single 610 * page. 611 * 612 * Note: Must be called under text_mutex. 613 */ 614 void *__kprobes text_poke(void *addr, const void *opcode, size_t len) 615 { 616 unsigned long flags; 617 char *vaddr; 618 struct page *pages[2]; 619 int i; 620 621 if (!core_kernel_text((unsigned long)addr)) { 622 pages[0] = vmalloc_to_page(addr); 623 pages[1] = vmalloc_to_page(addr + PAGE_SIZE); 624 } else { 625 pages[0] = virt_to_page(addr); 626 WARN_ON(!PageReserved(pages[0])); 627 pages[1] = virt_to_page(addr + PAGE_SIZE); 628 } 629 BUG_ON(!pages[0]); 630 local_irq_save(flags); 631 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0])); 632 if (pages[1]) 633 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1])); 634 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0); 635 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len); 636 clear_fixmap(FIX_TEXT_POKE0); 637 if (pages[1]) 638 clear_fixmap(FIX_TEXT_POKE1); 639 local_flush_tlb(); 640 sync_core(); 641 /* Could also do a CLFLUSH here to speed up CPU recovery; but 642 that causes hangs on some VIA CPUs. */ 643 for (i = 0; i < len; i++) 644 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]); 645 local_irq_restore(flags); 646 return addr; 647 } 648 649 /* 650 * Cross-modifying kernel text with stop_machine(). 651 * This code originally comes from immediate value. 652 */ 653 static atomic_t stop_machine_first; 654 static int wrote_text; 655 656 struct text_poke_params { 657 struct text_poke_param *params; 658 int nparams; 659 }; 660 661 static int __kprobes stop_machine_text_poke(void *data) 662 { 663 struct text_poke_params *tpp = data; 664 struct text_poke_param *p; 665 int i; 666 667 if (atomic_dec_and_test(&stop_machine_first)) { 668 for (i = 0; i < tpp->nparams; i++) { 669 p = &tpp->params[i]; 670 text_poke(p->addr, p->opcode, p->len); 671 } 672 smp_wmb(); /* Make sure other cpus see that this has run */ 673 wrote_text = 1; 674 } else { 675 while (!wrote_text) 676 cpu_relax(); 677 smp_mb(); /* Load wrote_text before following execution */ 678 } 679 680 for (i = 0; i < tpp->nparams; i++) { 681 p = &tpp->params[i]; 682 flush_icache_range((unsigned long)p->addr, 683 (unsigned long)p->addr + p->len); 684 } 685 /* 686 * Intel Archiecture Software Developer's Manual section 7.1.3 specifies 687 * that a core serializing instruction such as "cpuid" should be 688 * executed on _each_ core before the new instruction is made visible. 689 */ 690 sync_core(); 691 return 0; 692 } 693 694 /** 695 * text_poke_smp - Update instructions on a live kernel on SMP 696 * @addr: address to modify 697 * @opcode: source of the copy 698 * @len: length to copy 699 * 700 * Modify multi-byte instruction by using stop_machine() on SMP. This allows 701 * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying 702 * should be allowed, since stop_machine() does _not_ protect code against 703 * NMI and MCE. 704 * 705 * Note: Must be called under get_online_cpus() and text_mutex. 706 */ 707 void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len) 708 { 709 struct text_poke_params tpp; 710 struct text_poke_param p; 711 712 p.addr = addr; 713 p.opcode = opcode; 714 p.len = len; 715 tpp.params = &p; 716 tpp.nparams = 1; 717 atomic_set(&stop_machine_first, 1); 718 wrote_text = 0; 719 /* Use __stop_machine() because the caller already got online_cpus. */ 720 __stop_machine(stop_machine_text_poke, (void *)&tpp, cpu_online_mask); 721 return addr; 722 } 723 724 /** 725 * text_poke_smp_batch - Update instructions on a live kernel on SMP 726 * @params: an array of text_poke parameters 727 * @n: the number of elements in params. 728 * 729 * Modify multi-byte instruction by using stop_machine() on SMP. Since the 730 * stop_machine() is heavy task, it is better to aggregate text_poke requests 731 * and do it once if possible. 732 * 733 * Note: Must be called under get_online_cpus() and text_mutex. 734 */ 735 void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n) 736 { 737 struct text_poke_params tpp = {.params = params, .nparams = n}; 738 739 atomic_set(&stop_machine_first, 1); 740 wrote_text = 0; 741 __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); 742 } 743