1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Kernel Probes (KProbes) 4 * 5 * Copyright (C) IBM Corporation, 2002, 2004 6 * 7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 8 * Probes initial implementation (includes suggestions from 9 * Rusty Russell). 10 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 11 * hlists and exceptions notifier as suggested by Andi Kleen. 12 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 13 * interface to access function arguments. 14 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 15 * exceptions notifier to be first on the priority list. 16 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 17 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 18 * <prasanna@in.ibm.com> added function-return probes. 19 */ 20 21 #define pr_fmt(fmt) "kprobes: " fmt 22 23 #include <linux/kprobes.h> 24 #include <linux/hash.h> 25 #include <linux/init.h> 26 #include <linux/slab.h> 27 #include <linux/stddef.h> 28 #include <linux/export.h> 29 #include <linux/moduleloader.h> 30 #include <linux/kallsyms.h> 31 #include <linux/freezer.h> 32 #include <linux/seq_file.h> 33 #include <linux/debugfs.h> 34 #include <linux/sysctl.h> 35 #include <linux/kdebug.h> 36 #include <linux/memory.h> 37 #include <linux/ftrace.h> 38 #include <linux/cpu.h> 39 #include <linux/jump_label.h> 40 #include <linux/static_call.h> 41 #include <linux/perf_event.h> 42 43 #include <asm/sections.h> 44 #include <asm/cacheflush.h> 45 #include <asm/errno.h> 46 #include <linux/uaccess.h> 47 48 #define KPROBE_HASH_BITS 6 49 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 50 51 #if !defined(CONFIG_OPTPROBES) || !defined(CONFIG_SYSCTL) 52 #define kprobe_sysctls_init() do { } while (0) 53 #endif 54 55 static int kprobes_initialized; 56 /* kprobe_table can be accessed by 57 * - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held. 58 * Or 59 * - RCU hlist traversal under disabling preempt (breakpoint handlers) 60 */ 61 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 62 63 /* NOTE: change this value only with 'kprobe_mutex' held */ 64 static bool kprobes_all_disarmed; 65 66 /* This protects 'kprobe_table' and 'optimizing_list' */ 67 static DEFINE_MUTEX(kprobe_mutex); 68 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance); 69 70 kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, 71 unsigned int __unused) 72 { 73 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name))); 74 } 75 76 /* 77 * Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where 78 * kprobes can not probe. 79 */ 80 static LIST_HEAD(kprobe_blacklist); 81 82 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT 83 /* 84 * 'kprobe::ainsn.insn' points to the copy of the instruction to be 85 * single-stepped. x86_64, POWER4 and above have no-exec support and 86 * stepping on the instruction on a vmalloced/kmalloced/data page 87 * is a recipe for disaster 88 */ 89 struct kprobe_insn_page { 90 struct list_head list; 91 kprobe_opcode_t *insns; /* Page of instruction slots */ 92 struct kprobe_insn_cache *cache; 93 int nused; 94 int ngarbage; 95 char slot_used[]; 96 }; 97 98 #define KPROBE_INSN_PAGE_SIZE(slots) \ 99 (offsetof(struct kprobe_insn_page, slot_used) + \ 100 (sizeof(char) * (slots))) 101 102 static int slots_per_page(struct kprobe_insn_cache *c) 103 { 104 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t)); 105 } 106 107 enum kprobe_slot_state { 108 SLOT_CLEAN = 0, 109 SLOT_DIRTY = 1, 110 SLOT_USED = 2, 111 }; 112 113 void __weak *alloc_insn_page(void) 114 { 115 /* 116 * Use module_alloc() so this page is within +/- 2GB of where the 117 * kernel image and loaded module images reside. This is required 118 * for most of the architectures. 119 * (e.g. x86-64 needs this to handle the %rip-relative fixups.) 120 */ 121 return module_alloc(PAGE_SIZE); 122 } 123 124 static void free_insn_page(void *page) 125 { 126 module_memfree(page); 127 } 128 129 struct kprobe_insn_cache kprobe_insn_slots = { 130 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex), 131 .alloc = alloc_insn_page, 132 .free = free_insn_page, 133 .sym = KPROBE_INSN_PAGE_SYM, 134 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages), 135 .insn_size = MAX_INSN_SIZE, 136 .nr_garbage = 0, 137 }; 138 static int collect_garbage_slots(struct kprobe_insn_cache *c); 139 140 /** 141 * __get_insn_slot() - Find a slot on an executable page for an instruction. 142 * We allocate an executable page if there's no room on existing ones. 143 */ 144 kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c) 145 { 146 struct kprobe_insn_page *kip; 147 kprobe_opcode_t *slot = NULL; 148 149 /* Since the slot array is not protected by rcu, we need a mutex */ 150 mutex_lock(&c->mutex); 151 retry: 152 rcu_read_lock(); 153 list_for_each_entry_rcu(kip, &c->pages, list) { 154 if (kip->nused < slots_per_page(c)) { 155 int i; 156 157 for (i = 0; i < slots_per_page(c); i++) { 158 if (kip->slot_used[i] == SLOT_CLEAN) { 159 kip->slot_used[i] = SLOT_USED; 160 kip->nused++; 161 slot = kip->insns + (i * c->insn_size); 162 rcu_read_unlock(); 163 goto out; 164 } 165 } 166 /* kip->nused is broken. Fix it. */ 167 kip->nused = slots_per_page(c); 168 WARN_ON(1); 169 } 170 } 171 rcu_read_unlock(); 172 173 /* If there are any garbage slots, collect it and try again. */ 174 if (c->nr_garbage && collect_garbage_slots(c) == 0) 175 goto retry; 176 177 /* All out of space. Need to allocate a new page. */ 178 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL); 179 if (!kip) 180 goto out; 181 182 kip->insns = c->alloc(); 183 if (!kip->insns) { 184 kfree(kip); 185 goto out; 186 } 187 INIT_LIST_HEAD(&kip->list); 188 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c)); 189 kip->slot_used[0] = SLOT_USED; 190 kip->nused = 1; 191 kip->ngarbage = 0; 192 kip->cache = c; 193 list_add_rcu(&kip->list, &c->pages); 194 slot = kip->insns; 195 196 /* Record the perf ksymbol register event after adding the page */ 197 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns, 198 PAGE_SIZE, false, c->sym); 199 out: 200 mutex_unlock(&c->mutex); 201 return slot; 202 } 203 204 /* Return true if all garbages are collected, otherwise false. */ 205 static bool collect_one_slot(struct kprobe_insn_page *kip, int idx) 206 { 207 kip->slot_used[idx] = SLOT_CLEAN; 208 kip->nused--; 209 if (kip->nused == 0) { 210 /* 211 * Page is no longer in use. Free it unless 212 * it's the last one. We keep the last one 213 * so as not to have to set it up again the 214 * next time somebody inserts a probe. 215 */ 216 if (!list_is_singular(&kip->list)) { 217 /* 218 * Record perf ksymbol unregister event before removing 219 * the page. 220 */ 221 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, 222 (unsigned long)kip->insns, PAGE_SIZE, true, 223 kip->cache->sym); 224 list_del_rcu(&kip->list); 225 synchronize_rcu(); 226 kip->cache->free(kip->insns); 227 kfree(kip); 228 } 229 return true; 230 } 231 return false; 232 } 233 234 static int collect_garbage_slots(struct kprobe_insn_cache *c) 235 { 236 struct kprobe_insn_page *kip, *next; 237 238 /* Ensure no-one is interrupted on the garbages */ 239 synchronize_rcu(); 240 241 list_for_each_entry_safe(kip, next, &c->pages, list) { 242 int i; 243 244 if (kip->ngarbage == 0) 245 continue; 246 kip->ngarbage = 0; /* we will collect all garbages */ 247 for (i = 0; i < slots_per_page(c); i++) { 248 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i)) 249 break; 250 } 251 } 252 c->nr_garbage = 0; 253 return 0; 254 } 255 256 void __free_insn_slot(struct kprobe_insn_cache *c, 257 kprobe_opcode_t *slot, int dirty) 258 { 259 struct kprobe_insn_page *kip; 260 long idx; 261 262 mutex_lock(&c->mutex); 263 rcu_read_lock(); 264 list_for_each_entry_rcu(kip, &c->pages, list) { 265 idx = ((long)slot - (long)kip->insns) / 266 (c->insn_size * sizeof(kprobe_opcode_t)); 267 if (idx >= 0 && idx < slots_per_page(c)) 268 goto out; 269 } 270 /* Could not find this slot. */ 271 WARN_ON(1); 272 kip = NULL; 273 out: 274 rcu_read_unlock(); 275 /* Mark and sweep: this may sleep */ 276 if (kip) { 277 /* Check double free */ 278 WARN_ON(kip->slot_used[idx] != SLOT_USED); 279 if (dirty) { 280 kip->slot_used[idx] = SLOT_DIRTY; 281 kip->ngarbage++; 282 if (++c->nr_garbage > slots_per_page(c)) 283 collect_garbage_slots(c); 284 } else { 285 collect_one_slot(kip, idx); 286 } 287 } 288 mutex_unlock(&c->mutex); 289 } 290 291 /* 292 * Check given address is on the page of kprobe instruction slots. 293 * This will be used for checking whether the address on a stack 294 * is on a text area or not. 295 */ 296 bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr) 297 { 298 struct kprobe_insn_page *kip; 299 bool ret = false; 300 301 rcu_read_lock(); 302 list_for_each_entry_rcu(kip, &c->pages, list) { 303 if (addr >= (unsigned long)kip->insns && 304 addr < (unsigned long)kip->insns + PAGE_SIZE) { 305 ret = true; 306 break; 307 } 308 } 309 rcu_read_unlock(); 310 311 return ret; 312 } 313 314 int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum, 315 unsigned long *value, char *type, char *sym) 316 { 317 struct kprobe_insn_page *kip; 318 int ret = -ERANGE; 319 320 rcu_read_lock(); 321 list_for_each_entry_rcu(kip, &c->pages, list) { 322 if ((*symnum)--) 323 continue; 324 strscpy(sym, c->sym, KSYM_NAME_LEN); 325 *type = 't'; 326 *value = (unsigned long)kip->insns; 327 ret = 0; 328 break; 329 } 330 rcu_read_unlock(); 331 332 return ret; 333 } 334 335 #ifdef CONFIG_OPTPROBES 336 void __weak *alloc_optinsn_page(void) 337 { 338 return alloc_insn_page(); 339 } 340 341 void __weak free_optinsn_page(void *page) 342 { 343 free_insn_page(page); 344 } 345 346 /* For optimized_kprobe buffer */ 347 struct kprobe_insn_cache kprobe_optinsn_slots = { 348 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex), 349 .alloc = alloc_optinsn_page, 350 .free = free_optinsn_page, 351 .sym = KPROBE_OPTINSN_PAGE_SYM, 352 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages), 353 /* .insn_size is initialized later */ 354 .nr_garbage = 0, 355 }; 356 #endif 357 #endif 358 359 /* We have preemption disabled.. so it is safe to use __ versions */ 360 static inline void set_kprobe_instance(struct kprobe *kp) 361 { 362 __this_cpu_write(kprobe_instance, kp); 363 } 364 365 static inline void reset_kprobe_instance(void) 366 { 367 __this_cpu_write(kprobe_instance, NULL); 368 } 369 370 /* 371 * This routine is called either: 372 * - under the 'kprobe_mutex' - during kprobe_[un]register(). 373 * OR 374 * - with preemption disabled - from architecture specific code. 375 */ 376 struct kprobe *get_kprobe(void *addr) 377 { 378 struct hlist_head *head; 379 struct kprobe *p; 380 381 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 382 hlist_for_each_entry_rcu(p, head, hlist, 383 lockdep_is_held(&kprobe_mutex)) { 384 if (p->addr == addr) 385 return p; 386 } 387 388 return NULL; 389 } 390 NOKPROBE_SYMBOL(get_kprobe); 391 392 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs); 393 394 /* Return true if 'p' is an aggregator */ 395 static inline bool kprobe_aggrprobe(struct kprobe *p) 396 { 397 return p->pre_handler == aggr_pre_handler; 398 } 399 400 /* Return true if 'p' is unused */ 401 static inline bool kprobe_unused(struct kprobe *p) 402 { 403 return kprobe_aggrprobe(p) && kprobe_disabled(p) && 404 list_empty(&p->list); 405 } 406 407 /* Keep all fields in the kprobe consistent. */ 408 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p) 409 { 410 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t)); 411 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn)); 412 } 413 414 #ifdef CONFIG_OPTPROBES 415 /* NOTE: This is protected by 'kprobe_mutex'. */ 416 static bool kprobes_allow_optimization; 417 418 /* 419 * Call all 'kprobe::pre_handler' on the list, but ignores its return value. 420 * This must be called from arch-dep optimized caller. 421 */ 422 void opt_pre_handler(struct kprobe *p, struct pt_regs *regs) 423 { 424 struct kprobe *kp; 425 426 list_for_each_entry_rcu(kp, &p->list, list) { 427 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 428 set_kprobe_instance(kp); 429 kp->pre_handler(kp, regs); 430 } 431 reset_kprobe_instance(); 432 } 433 } 434 NOKPROBE_SYMBOL(opt_pre_handler); 435 436 /* Free optimized instructions and optimized_kprobe */ 437 static void free_aggr_kprobe(struct kprobe *p) 438 { 439 struct optimized_kprobe *op; 440 441 op = container_of(p, struct optimized_kprobe, kp); 442 arch_remove_optimized_kprobe(op); 443 arch_remove_kprobe(p); 444 kfree(op); 445 } 446 447 /* Return true if the kprobe is ready for optimization. */ 448 static inline int kprobe_optready(struct kprobe *p) 449 { 450 struct optimized_kprobe *op; 451 452 if (kprobe_aggrprobe(p)) { 453 op = container_of(p, struct optimized_kprobe, kp); 454 return arch_prepared_optinsn(&op->optinsn); 455 } 456 457 return 0; 458 } 459 460 /* Return true if the kprobe is disarmed. Note: p must be on hash list */ 461 static inline bool kprobe_disarmed(struct kprobe *p) 462 { 463 struct optimized_kprobe *op; 464 465 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */ 466 if (!kprobe_aggrprobe(p)) 467 return kprobe_disabled(p); 468 469 op = container_of(p, struct optimized_kprobe, kp); 470 471 return kprobe_disabled(p) && list_empty(&op->list); 472 } 473 474 /* Return true if the probe is queued on (un)optimizing lists */ 475 static bool kprobe_queued(struct kprobe *p) 476 { 477 struct optimized_kprobe *op; 478 479 if (kprobe_aggrprobe(p)) { 480 op = container_of(p, struct optimized_kprobe, kp); 481 if (!list_empty(&op->list)) 482 return true; 483 } 484 return false; 485 } 486 487 /* 488 * Return an optimized kprobe whose optimizing code replaces 489 * instructions including 'addr' (exclude breakpoint). 490 */ 491 static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr) 492 { 493 int i; 494 struct kprobe *p = NULL; 495 struct optimized_kprobe *op; 496 497 /* Don't check i == 0, since that is a breakpoint case. */ 498 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) 499 p = get_kprobe(addr - i); 500 501 if (p && kprobe_optready(p)) { 502 op = container_of(p, struct optimized_kprobe, kp); 503 if (arch_within_optimized_kprobe(op, addr)) 504 return p; 505 } 506 507 return NULL; 508 } 509 510 /* Optimization staging list, protected by 'kprobe_mutex' */ 511 static LIST_HEAD(optimizing_list); 512 static LIST_HEAD(unoptimizing_list); 513 static LIST_HEAD(freeing_list); 514 515 static void kprobe_optimizer(struct work_struct *work); 516 static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer); 517 #define OPTIMIZE_DELAY 5 518 519 /* 520 * Optimize (replace a breakpoint with a jump) kprobes listed on 521 * 'optimizing_list'. 522 */ 523 static void do_optimize_kprobes(void) 524 { 525 lockdep_assert_held(&text_mutex); 526 /* 527 * The optimization/unoptimization refers 'online_cpus' via 528 * stop_machine() and cpu-hotplug modifies the 'online_cpus'. 529 * And same time, 'text_mutex' will be held in cpu-hotplug and here. 530 * This combination can cause a deadlock (cpu-hotplug tries to lock 531 * 'text_mutex' but stop_machine() can not be done because 532 * the 'online_cpus' has been changed) 533 * To avoid this deadlock, caller must have locked cpu-hotplug 534 * for preventing cpu-hotplug outside of 'text_mutex' locking. 535 */ 536 lockdep_assert_cpus_held(); 537 538 /* Optimization never be done when disarmed */ 539 if (kprobes_all_disarmed || !kprobes_allow_optimization || 540 list_empty(&optimizing_list)) 541 return; 542 543 arch_optimize_kprobes(&optimizing_list); 544 } 545 546 /* 547 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint 548 * if need) kprobes listed on 'unoptimizing_list'. 549 */ 550 static void do_unoptimize_kprobes(void) 551 { 552 struct optimized_kprobe *op, *tmp; 553 554 lockdep_assert_held(&text_mutex); 555 /* See comment in do_optimize_kprobes() */ 556 lockdep_assert_cpus_held(); 557 558 /* Unoptimization must be done anytime */ 559 if (list_empty(&unoptimizing_list)) 560 return; 561 562 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list); 563 /* Loop on 'freeing_list' for disarming */ 564 list_for_each_entry_safe(op, tmp, &freeing_list, list) { 565 /* Switching from detour code to origin */ 566 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 567 /* Disarm probes if marked disabled */ 568 if (kprobe_disabled(&op->kp)) 569 arch_disarm_kprobe(&op->kp); 570 if (kprobe_unused(&op->kp)) { 571 /* 572 * Remove unused probes from hash list. After waiting 573 * for synchronization, these probes are reclaimed. 574 * (reclaiming is done by do_free_cleaned_kprobes().) 575 */ 576 hlist_del_rcu(&op->kp.hlist); 577 } else 578 list_del_init(&op->list); 579 } 580 } 581 582 /* Reclaim all kprobes on the 'freeing_list' */ 583 static void do_free_cleaned_kprobes(void) 584 { 585 struct optimized_kprobe *op, *tmp; 586 587 list_for_each_entry_safe(op, tmp, &freeing_list, list) { 588 list_del_init(&op->list); 589 if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) { 590 /* 591 * This must not happen, but if there is a kprobe 592 * still in use, keep it on kprobes hash list. 593 */ 594 continue; 595 } 596 free_aggr_kprobe(&op->kp); 597 } 598 } 599 600 /* Start optimizer after OPTIMIZE_DELAY passed */ 601 static void kick_kprobe_optimizer(void) 602 { 603 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY); 604 } 605 606 /* Kprobe jump optimizer */ 607 static void kprobe_optimizer(struct work_struct *work) 608 { 609 mutex_lock(&kprobe_mutex); 610 cpus_read_lock(); 611 mutex_lock(&text_mutex); 612 613 /* 614 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed) 615 * kprobes before waiting for quiesence period. 616 */ 617 do_unoptimize_kprobes(); 618 619 /* 620 * Step 2: Wait for quiesence period to ensure all potentially 621 * preempted tasks to have normally scheduled. Because optprobe 622 * may modify multiple instructions, there is a chance that Nth 623 * instruction is preempted. In that case, such tasks can return 624 * to 2nd-Nth byte of jump instruction. This wait is for avoiding it. 625 * Note that on non-preemptive kernel, this is transparently converted 626 * to synchronoze_sched() to wait for all interrupts to have completed. 627 */ 628 synchronize_rcu_tasks(); 629 630 /* Step 3: Optimize kprobes after quiesence period */ 631 do_optimize_kprobes(); 632 633 /* Step 4: Free cleaned kprobes after quiesence period */ 634 do_free_cleaned_kprobes(); 635 636 mutex_unlock(&text_mutex); 637 cpus_read_unlock(); 638 639 /* Step 5: Kick optimizer again if needed */ 640 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) 641 kick_kprobe_optimizer(); 642 643 mutex_unlock(&kprobe_mutex); 644 } 645 646 /* Wait for completing optimization and unoptimization */ 647 void wait_for_kprobe_optimizer(void) 648 { 649 mutex_lock(&kprobe_mutex); 650 651 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) { 652 mutex_unlock(&kprobe_mutex); 653 654 /* This will also make 'optimizing_work' execute immmediately */ 655 flush_delayed_work(&optimizing_work); 656 /* 'optimizing_work' might not have been queued yet, relax */ 657 cpu_relax(); 658 659 mutex_lock(&kprobe_mutex); 660 } 661 662 mutex_unlock(&kprobe_mutex); 663 } 664 665 static bool optprobe_queued_unopt(struct optimized_kprobe *op) 666 { 667 struct optimized_kprobe *_op; 668 669 list_for_each_entry(_op, &unoptimizing_list, list) { 670 if (op == _op) 671 return true; 672 } 673 674 return false; 675 } 676 677 /* Optimize kprobe if p is ready to be optimized */ 678 static void optimize_kprobe(struct kprobe *p) 679 { 680 struct optimized_kprobe *op; 681 682 /* Check if the kprobe is disabled or not ready for optimization. */ 683 if (!kprobe_optready(p) || !kprobes_allow_optimization || 684 (kprobe_disabled(p) || kprobes_all_disarmed)) 685 return; 686 687 /* kprobes with 'post_handler' can not be optimized */ 688 if (p->post_handler) 689 return; 690 691 op = container_of(p, struct optimized_kprobe, kp); 692 693 /* Check there is no other kprobes at the optimized instructions */ 694 if (arch_check_optimized_kprobe(op) < 0) 695 return; 696 697 /* Check if it is already optimized. */ 698 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) { 699 if (optprobe_queued_unopt(op)) { 700 /* This is under unoptimizing. Just dequeue the probe */ 701 list_del_init(&op->list); 702 } 703 return; 704 } 705 op->kp.flags |= KPROBE_FLAG_OPTIMIZED; 706 707 /* 708 * On the 'unoptimizing_list' and 'optimizing_list', 709 * 'op' must have OPTIMIZED flag 710 */ 711 if (WARN_ON_ONCE(!list_empty(&op->list))) 712 return; 713 714 list_add(&op->list, &optimizing_list); 715 kick_kprobe_optimizer(); 716 } 717 718 /* Short cut to direct unoptimizing */ 719 static void force_unoptimize_kprobe(struct optimized_kprobe *op) 720 { 721 lockdep_assert_cpus_held(); 722 arch_unoptimize_kprobe(op); 723 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 724 } 725 726 /* Unoptimize a kprobe if p is optimized */ 727 static void unoptimize_kprobe(struct kprobe *p, bool force) 728 { 729 struct optimized_kprobe *op; 730 731 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p)) 732 return; /* This is not an optprobe nor optimized */ 733 734 op = container_of(p, struct optimized_kprobe, kp); 735 if (!kprobe_optimized(p)) 736 return; 737 738 if (!list_empty(&op->list)) { 739 if (optprobe_queued_unopt(op)) { 740 /* Queued in unoptimizing queue */ 741 if (force) { 742 /* 743 * Forcibly unoptimize the kprobe here, and queue it 744 * in the freeing list for release afterwards. 745 */ 746 force_unoptimize_kprobe(op); 747 list_move(&op->list, &freeing_list); 748 } 749 } else { 750 /* Dequeue from the optimizing queue */ 751 list_del_init(&op->list); 752 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 753 } 754 return; 755 } 756 757 /* Optimized kprobe case */ 758 if (force) { 759 /* Forcibly update the code: this is a special case */ 760 force_unoptimize_kprobe(op); 761 } else { 762 list_add(&op->list, &unoptimizing_list); 763 kick_kprobe_optimizer(); 764 } 765 } 766 767 /* Cancel unoptimizing for reusing */ 768 static int reuse_unused_kprobe(struct kprobe *ap) 769 { 770 struct optimized_kprobe *op; 771 772 /* 773 * Unused kprobe MUST be on the way of delayed unoptimizing (means 774 * there is still a relative jump) and disabled. 775 */ 776 op = container_of(ap, struct optimized_kprobe, kp); 777 WARN_ON_ONCE(list_empty(&op->list)); 778 /* Enable the probe again */ 779 ap->flags &= ~KPROBE_FLAG_DISABLED; 780 /* Optimize it again. (remove from 'op->list') */ 781 if (!kprobe_optready(ap)) 782 return -EINVAL; 783 784 optimize_kprobe(ap); 785 return 0; 786 } 787 788 /* Remove optimized instructions */ 789 static void kill_optimized_kprobe(struct kprobe *p) 790 { 791 struct optimized_kprobe *op; 792 793 op = container_of(p, struct optimized_kprobe, kp); 794 if (!list_empty(&op->list)) 795 /* Dequeue from the (un)optimization queue */ 796 list_del_init(&op->list); 797 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 798 799 if (kprobe_unused(p)) { 800 /* Enqueue if it is unused */ 801 list_add(&op->list, &freeing_list); 802 /* 803 * Remove unused probes from the hash list. After waiting 804 * for synchronization, this probe is reclaimed. 805 * (reclaiming is done by do_free_cleaned_kprobes().) 806 */ 807 hlist_del_rcu(&op->kp.hlist); 808 } 809 810 /* Don't touch the code, because it is already freed. */ 811 arch_remove_optimized_kprobe(op); 812 } 813 814 static inline 815 void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p) 816 { 817 if (!kprobe_ftrace(p)) 818 arch_prepare_optimized_kprobe(op, p); 819 } 820 821 /* Try to prepare optimized instructions */ 822 static void prepare_optimized_kprobe(struct kprobe *p) 823 { 824 struct optimized_kprobe *op; 825 826 op = container_of(p, struct optimized_kprobe, kp); 827 __prepare_optimized_kprobe(op, p); 828 } 829 830 /* Allocate new optimized_kprobe and try to prepare optimized instructions. */ 831 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 832 { 833 struct optimized_kprobe *op; 834 835 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL); 836 if (!op) 837 return NULL; 838 839 INIT_LIST_HEAD(&op->list); 840 op->kp.addr = p->addr; 841 __prepare_optimized_kprobe(op, p); 842 843 return &op->kp; 844 } 845 846 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p); 847 848 /* 849 * Prepare an optimized_kprobe and optimize it. 850 * NOTE: 'p' must be a normal registered kprobe. 851 */ 852 static void try_to_optimize_kprobe(struct kprobe *p) 853 { 854 struct kprobe *ap; 855 struct optimized_kprobe *op; 856 857 /* Impossible to optimize ftrace-based kprobe. */ 858 if (kprobe_ftrace(p)) 859 return; 860 861 /* For preparing optimization, jump_label_text_reserved() is called. */ 862 cpus_read_lock(); 863 jump_label_lock(); 864 mutex_lock(&text_mutex); 865 866 ap = alloc_aggr_kprobe(p); 867 if (!ap) 868 goto out; 869 870 op = container_of(ap, struct optimized_kprobe, kp); 871 if (!arch_prepared_optinsn(&op->optinsn)) { 872 /* If failed to setup optimizing, fallback to kprobe. */ 873 arch_remove_optimized_kprobe(op); 874 kfree(op); 875 goto out; 876 } 877 878 init_aggr_kprobe(ap, p); 879 optimize_kprobe(ap); /* This just kicks optimizer thread. */ 880 881 out: 882 mutex_unlock(&text_mutex); 883 jump_label_unlock(); 884 cpus_read_unlock(); 885 } 886 887 static void optimize_all_kprobes(void) 888 { 889 struct hlist_head *head; 890 struct kprobe *p; 891 unsigned int i; 892 893 mutex_lock(&kprobe_mutex); 894 /* If optimization is already allowed, just return. */ 895 if (kprobes_allow_optimization) 896 goto out; 897 898 cpus_read_lock(); 899 kprobes_allow_optimization = true; 900 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 901 head = &kprobe_table[i]; 902 hlist_for_each_entry(p, head, hlist) 903 if (!kprobe_disabled(p)) 904 optimize_kprobe(p); 905 } 906 cpus_read_unlock(); 907 pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n"); 908 out: 909 mutex_unlock(&kprobe_mutex); 910 } 911 912 #ifdef CONFIG_SYSCTL 913 static void unoptimize_all_kprobes(void) 914 { 915 struct hlist_head *head; 916 struct kprobe *p; 917 unsigned int i; 918 919 mutex_lock(&kprobe_mutex); 920 /* If optimization is already prohibited, just return. */ 921 if (!kprobes_allow_optimization) { 922 mutex_unlock(&kprobe_mutex); 923 return; 924 } 925 926 cpus_read_lock(); 927 kprobes_allow_optimization = false; 928 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 929 head = &kprobe_table[i]; 930 hlist_for_each_entry(p, head, hlist) { 931 if (!kprobe_disabled(p)) 932 unoptimize_kprobe(p, false); 933 } 934 } 935 cpus_read_unlock(); 936 mutex_unlock(&kprobe_mutex); 937 938 /* Wait for unoptimizing completion. */ 939 wait_for_kprobe_optimizer(); 940 pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n"); 941 } 942 943 static DEFINE_MUTEX(kprobe_sysctl_mutex); 944 static int sysctl_kprobes_optimization; 945 static int proc_kprobes_optimization_handler(struct ctl_table *table, 946 int write, void *buffer, 947 size_t *length, loff_t *ppos) 948 { 949 int ret; 950 951 mutex_lock(&kprobe_sysctl_mutex); 952 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0; 953 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 954 955 if (sysctl_kprobes_optimization) 956 optimize_all_kprobes(); 957 else 958 unoptimize_all_kprobes(); 959 mutex_unlock(&kprobe_sysctl_mutex); 960 961 return ret; 962 } 963 964 static struct ctl_table kprobe_sysctls[] = { 965 { 966 .procname = "kprobes-optimization", 967 .data = &sysctl_kprobes_optimization, 968 .maxlen = sizeof(int), 969 .mode = 0644, 970 .proc_handler = proc_kprobes_optimization_handler, 971 .extra1 = SYSCTL_ZERO, 972 .extra2 = SYSCTL_ONE, 973 }, 974 {} 975 }; 976 977 static void __init kprobe_sysctls_init(void) 978 { 979 register_sysctl_init("debug", kprobe_sysctls); 980 } 981 #endif /* CONFIG_SYSCTL */ 982 983 /* Put a breakpoint for a probe. */ 984 static void __arm_kprobe(struct kprobe *p) 985 { 986 struct kprobe *_p; 987 988 lockdep_assert_held(&text_mutex); 989 990 /* Find the overlapping optimized kprobes. */ 991 _p = get_optimized_kprobe(p->addr); 992 if (unlikely(_p)) 993 /* Fallback to unoptimized kprobe */ 994 unoptimize_kprobe(_p, true); 995 996 arch_arm_kprobe(p); 997 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ 998 } 999 1000 /* Remove the breakpoint of a probe. */ 1001 static void __disarm_kprobe(struct kprobe *p, bool reopt) 1002 { 1003 struct kprobe *_p; 1004 1005 lockdep_assert_held(&text_mutex); 1006 1007 /* Try to unoptimize */ 1008 unoptimize_kprobe(p, kprobes_all_disarmed); 1009 1010 if (!kprobe_queued(p)) { 1011 arch_disarm_kprobe(p); 1012 /* If another kprobe was blocked, re-optimize it. */ 1013 _p = get_optimized_kprobe(p->addr); 1014 if (unlikely(_p) && reopt) 1015 optimize_kprobe(_p); 1016 } 1017 /* 1018 * TODO: Since unoptimization and real disarming will be done by 1019 * the worker thread, we can not check whether another probe are 1020 * unoptimized because of this probe here. It should be re-optimized 1021 * by the worker thread. 1022 */ 1023 } 1024 1025 #else /* !CONFIG_OPTPROBES */ 1026 1027 #define optimize_kprobe(p) do {} while (0) 1028 #define unoptimize_kprobe(p, f) do {} while (0) 1029 #define kill_optimized_kprobe(p) do {} while (0) 1030 #define prepare_optimized_kprobe(p) do {} while (0) 1031 #define try_to_optimize_kprobe(p) do {} while (0) 1032 #define __arm_kprobe(p) arch_arm_kprobe(p) 1033 #define __disarm_kprobe(p, o) arch_disarm_kprobe(p) 1034 #define kprobe_disarmed(p) kprobe_disabled(p) 1035 #define wait_for_kprobe_optimizer() do {} while (0) 1036 1037 static int reuse_unused_kprobe(struct kprobe *ap) 1038 { 1039 /* 1040 * If the optimized kprobe is NOT supported, the aggr kprobe is 1041 * released at the same time that the last aggregated kprobe is 1042 * unregistered. 1043 * Thus there should be no chance to reuse unused kprobe. 1044 */ 1045 WARN_ON_ONCE(1); 1046 return -EINVAL; 1047 } 1048 1049 static void free_aggr_kprobe(struct kprobe *p) 1050 { 1051 arch_remove_kprobe(p); 1052 kfree(p); 1053 } 1054 1055 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 1056 { 1057 return kzalloc(sizeof(struct kprobe), GFP_KERNEL); 1058 } 1059 #endif /* CONFIG_OPTPROBES */ 1060 1061 #ifdef CONFIG_KPROBES_ON_FTRACE 1062 static struct ftrace_ops kprobe_ftrace_ops __read_mostly = { 1063 .func = kprobe_ftrace_handler, 1064 .flags = FTRACE_OPS_FL_SAVE_REGS, 1065 }; 1066 1067 static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = { 1068 .func = kprobe_ftrace_handler, 1069 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY, 1070 }; 1071 1072 static int kprobe_ipmodify_enabled; 1073 static int kprobe_ftrace_enabled; 1074 1075 static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, 1076 int *cnt) 1077 { 1078 int ret = 0; 1079 1080 lockdep_assert_held(&kprobe_mutex); 1081 1082 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0); 1083 if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret)) 1084 return ret; 1085 1086 if (*cnt == 0) { 1087 ret = register_ftrace_function(ops); 1088 if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret)) 1089 goto err_ftrace; 1090 } 1091 1092 (*cnt)++; 1093 return ret; 1094 1095 err_ftrace: 1096 /* 1097 * At this point, sinec ops is not registered, we should be sefe from 1098 * registering empty filter. 1099 */ 1100 ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); 1101 return ret; 1102 } 1103 1104 static int arm_kprobe_ftrace(struct kprobe *p) 1105 { 1106 bool ipmodify = (p->post_handler != NULL); 1107 1108 return __arm_kprobe_ftrace(p, 1109 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops, 1110 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled); 1111 } 1112 1113 static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, 1114 int *cnt) 1115 { 1116 int ret = 0; 1117 1118 lockdep_assert_held(&kprobe_mutex); 1119 1120 if (*cnt == 1) { 1121 ret = unregister_ftrace_function(ops); 1122 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret)) 1123 return ret; 1124 } 1125 1126 (*cnt)--; 1127 1128 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); 1129 WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n", 1130 p->addr, ret); 1131 return ret; 1132 } 1133 1134 static int disarm_kprobe_ftrace(struct kprobe *p) 1135 { 1136 bool ipmodify = (p->post_handler != NULL); 1137 1138 return __disarm_kprobe_ftrace(p, 1139 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops, 1140 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled); 1141 } 1142 #else /* !CONFIG_KPROBES_ON_FTRACE */ 1143 static inline int arm_kprobe_ftrace(struct kprobe *p) 1144 { 1145 return -ENODEV; 1146 } 1147 1148 static inline int disarm_kprobe_ftrace(struct kprobe *p) 1149 { 1150 return -ENODEV; 1151 } 1152 #endif 1153 1154 static int prepare_kprobe(struct kprobe *p) 1155 { 1156 /* Must ensure p->addr is really on ftrace */ 1157 if (kprobe_ftrace(p)) 1158 return arch_prepare_kprobe_ftrace(p); 1159 1160 return arch_prepare_kprobe(p); 1161 } 1162 1163 static int arm_kprobe(struct kprobe *kp) 1164 { 1165 if (unlikely(kprobe_ftrace(kp))) 1166 return arm_kprobe_ftrace(kp); 1167 1168 cpus_read_lock(); 1169 mutex_lock(&text_mutex); 1170 __arm_kprobe(kp); 1171 mutex_unlock(&text_mutex); 1172 cpus_read_unlock(); 1173 1174 return 0; 1175 } 1176 1177 static int disarm_kprobe(struct kprobe *kp, bool reopt) 1178 { 1179 if (unlikely(kprobe_ftrace(kp))) 1180 return disarm_kprobe_ftrace(kp); 1181 1182 cpus_read_lock(); 1183 mutex_lock(&text_mutex); 1184 __disarm_kprobe(kp, reopt); 1185 mutex_unlock(&text_mutex); 1186 cpus_read_unlock(); 1187 1188 return 0; 1189 } 1190 1191 /* 1192 * Aggregate handlers for multiple kprobes support - these handlers 1193 * take care of invoking the individual kprobe handlers on p->list 1194 */ 1195 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 1196 { 1197 struct kprobe *kp; 1198 1199 list_for_each_entry_rcu(kp, &p->list, list) { 1200 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 1201 set_kprobe_instance(kp); 1202 if (kp->pre_handler(kp, regs)) 1203 return 1; 1204 } 1205 reset_kprobe_instance(); 1206 } 1207 return 0; 1208 } 1209 NOKPROBE_SYMBOL(aggr_pre_handler); 1210 1211 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 1212 unsigned long flags) 1213 { 1214 struct kprobe *kp; 1215 1216 list_for_each_entry_rcu(kp, &p->list, list) { 1217 if (kp->post_handler && likely(!kprobe_disabled(kp))) { 1218 set_kprobe_instance(kp); 1219 kp->post_handler(kp, regs, flags); 1220 reset_kprobe_instance(); 1221 } 1222 } 1223 } 1224 NOKPROBE_SYMBOL(aggr_post_handler); 1225 1226 /* Walks the list and increments 'nmissed' if 'p' has child probes. */ 1227 void kprobes_inc_nmissed_count(struct kprobe *p) 1228 { 1229 struct kprobe *kp; 1230 1231 if (!kprobe_aggrprobe(p)) { 1232 p->nmissed++; 1233 } else { 1234 list_for_each_entry_rcu(kp, &p->list, list) 1235 kp->nmissed++; 1236 } 1237 } 1238 NOKPROBE_SYMBOL(kprobes_inc_nmissed_count); 1239 1240 static void free_rp_inst_rcu(struct rcu_head *head) 1241 { 1242 struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu); 1243 1244 if (refcount_dec_and_test(&ri->rph->ref)) 1245 kfree(ri->rph); 1246 kfree(ri); 1247 } 1248 NOKPROBE_SYMBOL(free_rp_inst_rcu); 1249 1250 static void recycle_rp_inst(struct kretprobe_instance *ri) 1251 { 1252 struct kretprobe *rp = get_kretprobe(ri); 1253 1254 if (likely(rp)) 1255 freelist_add(&ri->freelist, &rp->freelist); 1256 else 1257 call_rcu(&ri->rcu, free_rp_inst_rcu); 1258 } 1259 NOKPROBE_SYMBOL(recycle_rp_inst); 1260 1261 static struct kprobe kprobe_busy = { 1262 .addr = (void *) get_kprobe, 1263 }; 1264 1265 void kprobe_busy_begin(void) 1266 { 1267 struct kprobe_ctlblk *kcb; 1268 1269 preempt_disable(); 1270 __this_cpu_write(current_kprobe, &kprobe_busy); 1271 kcb = get_kprobe_ctlblk(); 1272 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 1273 } 1274 1275 void kprobe_busy_end(void) 1276 { 1277 __this_cpu_write(current_kprobe, NULL); 1278 preempt_enable(); 1279 } 1280 1281 /* 1282 * This function is called from delayed_put_task_struct() when a task is 1283 * dead and cleaned up to recycle any kretprobe instances associated with 1284 * this task. These left over instances represent probed functions that 1285 * have been called but will never return. 1286 */ 1287 void kprobe_flush_task(struct task_struct *tk) 1288 { 1289 struct kretprobe_instance *ri; 1290 struct llist_node *node; 1291 1292 /* Early boot, not yet initialized. */ 1293 if (unlikely(!kprobes_initialized)) 1294 return; 1295 1296 kprobe_busy_begin(); 1297 1298 node = __llist_del_all(&tk->kretprobe_instances); 1299 while (node) { 1300 ri = container_of(node, struct kretprobe_instance, llist); 1301 node = node->next; 1302 1303 recycle_rp_inst(ri); 1304 } 1305 1306 kprobe_busy_end(); 1307 } 1308 NOKPROBE_SYMBOL(kprobe_flush_task); 1309 1310 static inline void free_rp_inst(struct kretprobe *rp) 1311 { 1312 struct kretprobe_instance *ri; 1313 struct freelist_node *node; 1314 int count = 0; 1315 1316 node = rp->freelist.head; 1317 while (node) { 1318 ri = container_of(node, struct kretprobe_instance, freelist); 1319 node = node->next; 1320 1321 kfree(ri); 1322 count++; 1323 } 1324 1325 if (refcount_sub_and_test(count, &rp->rph->ref)) { 1326 kfree(rp->rph); 1327 rp->rph = NULL; 1328 } 1329 } 1330 1331 /* Add the new probe to 'ap->list'. */ 1332 static int add_new_kprobe(struct kprobe *ap, struct kprobe *p) 1333 { 1334 if (p->post_handler) 1335 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */ 1336 1337 list_add_rcu(&p->list, &ap->list); 1338 if (p->post_handler && !ap->post_handler) 1339 ap->post_handler = aggr_post_handler; 1340 1341 return 0; 1342 } 1343 1344 /* 1345 * Fill in the required fields of the aggregator kprobe. Replace the 1346 * earlier kprobe in the hlist with the aggregator kprobe. 1347 */ 1348 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 1349 { 1350 /* Copy the insn slot of 'p' to 'ap'. */ 1351 copy_kprobe(p, ap); 1352 flush_insn_slot(ap); 1353 ap->addr = p->addr; 1354 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED; 1355 ap->pre_handler = aggr_pre_handler; 1356 /* We don't care the kprobe which has gone. */ 1357 if (p->post_handler && !kprobe_gone(p)) 1358 ap->post_handler = aggr_post_handler; 1359 1360 INIT_LIST_HEAD(&ap->list); 1361 INIT_HLIST_NODE(&ap->hlist); 1362 1363 list_add_rcu(&p->list, &ap->list); 1364 hlist_replace_rcu(&p->hlist, &ap->hlist); 1365 } 1366 1367 /* 1368 * This registers the second or subsequent kprobe at the same address. 1369 */ 1370 static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p) 1371 { 1372 int ret = 0; 1373 struct kprobe *ap = orig_p; 1374 1375 cpus_read_lock(); 1376 1377 /* For preparing optimization, jump_label_text_reserved() is called */ 1378 jump_label_lock(); 1379 mutex_lock(&text_mutex); 1380 1381 if (!kprobe_aggrprobe(orig_p)) { 1382 /* If 'orig_p' is not an 'aggr_kprobe', create new one. */ 1383 ap = alloc_aggr_kprobe(orig_p); 1384 if (!ap) { 1385 ret = -ENOMEM; 1386 goto out; 1387 } 1388 init_aggr_kprobe(ap, orig_p); 1389 } else if (kprobe_unused(ap)) { 1390 /* This probe is going to die. Rescue it */ 1391 ret = reuse_unused_kprobe(ap); 1392 if (ret) 1393 goto out; 1394 } 1395 1396 if (kprobe_gone(ap)) { 1397 /* 1398 * Attempting to insert new probe at the same location that 1399 * had a probe in the module vaddr area which already 1400 * freed. So, the instruction slot has already been 1401 * released. We need a new slot for the new probe. 1402 */ 1403 ret = arch_prepare_kprobe(ap); 1404 if (ret) 1405 /* 1406 * Even if fail to allocate new slot, don't need to 1407 * free the 'ap'. It will be used next time, or 1408 * freed by unregister_kprobe(). 1409 */ 1410 goto out; 1411 1412 /* Prepare optimized instructions if possible. */ 1413 prepare_optimized_kprobe(ap); 1414 1415 /* 1416 * Clear gone flag to prevent allocating new slot again, and 1417 * set disabled flag because it is not armed yet. 1418 */ 1419 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE) 1420 | KPROBE_FLAG_DISABLED; 1421 } 1422 1423 /* Copy the insn slot of 'p' to 'ap'. */ 1424 copy_kprobe(ap, p); 1425 ret = add_new_kprobe(ap, p); 1426 1427 out: 1428 mutex_unlock(&text_mutex); 1429 jump_label_unlock(); 1430 cpus_read_unlock(); 1431 1432 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) { 1433 ap->flags &= ~KPROBE_FLAG_DISABLED; 1434 if (!kprobes_all_disarmed) { 1435 /* Arm the breakpoint again. */ 1436 ret = arm_kprobe(ap); 1437 if (ret) { 1438 ap->flags |= KPROBE_FLAG_DISABLED; 1439 list_del_rcu(&p->list); 1440 synchronize_rcu(); 1441 } 1442 } 1443 } 1444 return ret; 1445 } 1446 1447 bool __weak arch_within_kprobe_blacklist(unsigned long addr) 1448 { 1449 /* The '__kprobes' functions and entry code must not be probed. */ 1450 return addr >= (unsigned long)__kprobes_text_start && 1451 addr < (unsigned long)__kprobes_text_end; 1452 } 1453 1454 static bool __within_kprobe_blacklist(unsigned long addr) 1455 { 1456 struct kprobe_blacklist_entry *ent; 1457 1458 if (arch_within_kprobe_blacklist(addr)) 1459 return true; 1460 /* 1461 * If 'kprobe_blacklist' is defined, check the address and 1462 * reject any probe registration in the prohibited area. 1463 */ 1464 list_for_each_entry(ent, &kprobe_blacklist, list) { 1465 if (addr >= ent->start_addr && addr < ent->end_addr) 1466 return true; 1467 } 1468 return false; 1469 } 1470 1471 bool within_kprobe_blacklist(unsigned long addr) 1472 { 1473 char symname[KSYM_NAME_LEN], *p; 1474 1475 if (__within_kprobe_blacklist(addr)) 1476 return true; 1477 1478 /* Check if the address is on a suffixed-symbol */ 1479 if (!lookup_symbol_name(addr, symname)) { 1480 p = strchr(symname, '.'); 1481 if (!p) 1482 return false; 1483 *p = '\0'; 1484 addr = (unsigned long)kprobe_lookup_name(symname, 0); 1485 if (addr) 1486 return __within_kprobe_blacklist(addr); 1487 } 1488 return false; 1489 } 1490 1491 /* 1492 * If 'symbol_name' is specified, look it up and add the 'offset' 1493 * to it. This way, we can specify a relative address to a symbol. 1494 * This returns encoded errors if it fails to look up symbol or invalid 1495 * combination of parameters. 1496 */ 1497 static kprobe_opcode_t *_kprobe_addr(kprobe_opcode_t *addr, 1498 const char *symbol_name, unsigned int offset) 1499 { 1500 if ((symbol_name && addr) || (!symbol_name && !addr)) 1501 goto invalid; 1502 1503 if (symbol_name) { 1504 addr = kprobe_lookup_name(symbol_name, offset); 1505 if (!addr) 1506 return ERR_PTR(-ENOENT); 1507 } 1508 1509 addr = (kprobe_opcode_t *)(((char *)addr) + offset); 1510 if (addr) 1511 return addr; 1512 1513 invalid: 1514 return ERR_PTR(-EINVAL); 1515 } 1516 1517 static kprobe_opcode_t *kprobe_addr(struct kprobe *p) 1518 { 1519 return _kprobe_addr(p->addr, p->symbol_name, p->offset); 1520 } 1521 1522 /* 1523 * Check the 'p' is valid and return the aggregator kprobe 1524 * at the same address. 1525 */ 1526 static struct kprobe *__get_valid_kprobe(struct kprobe *p) 1527 { 1528 struct kprobe *ap, *list_p; 1529 1530 lockdep_assert_held(&kprobe_mutex); 1531 1532 ap = get_kprobe(p->addr); 1533 if (unlikely(!ap)) 1534 return NULL; 1535 1536 if (p != ap) { 1537 list_for_each_entry(list_p, &ap->list, list) 1538 if (list_p == p) 1539 /* kprobe p is a valid probe */ 1540 goto valid; 1541 return NULL; 1542 } 1543 valid: 1544 return ap; 1545 } 1546 1547 /* 1548 * Warn and return error if the kprobe is being re-registered since 1549 * there must be a software bug. 1550 */ 1551 static inline int warn_kprobe_rereg(struct kprobe *p) 1552 { 1553 int ret = 0; 1554 1555 mutex_lock(&kprobe_mutex); 1556 if (WARN_ON_ONCE(__get_valid_kprobe(p))) 1557 ret = -EINVAL; 1558 mutex_unlock(&kprobe_mutex); 1559 1560 return ret; 1561 } 1562 1563 static int check_ftrace_location(struct kprobe *p) 1564 { 1565 unsigned long ftrace_addr; 1566 1567 ftrace_addr = ftrace_location((unsigned long)p->addr); 1568 if (ftrace_addr) { 1569 #ifdef CONFIG_KPROBES_ON_FTRACE 1570 /* Given address is not on the instruction boundary */ 1571 if ((unsigned long)p->addr != ftrace_addr) 1572 return -EILSEQ; 1573 p->flags |= KPROBE_FLAG_FTRACE; 1574 #else /* !CONFIG_KPROBES_ON_FTRACE */ 1575 return -EINVAL; 1576 #endif 1577 } 1578 return 0; 1579 } 1580 1581 static int check_kprobe_address_safe(struct kprobe *p, 1582 struct module **probed_mod) 1583 { 1584 int ret; 1585 1586 ret = check_ftrace_location(p); 1587 if (ret) 1588 return ret; 1589 jump_label_lock(); 1590 preempt_disable(); 1591 1592 /* Ensure it is not in reserved area nor out of text */ 1593 if (!kernel_text_address((unsigned long) p->addr) || 1594 within_kprobe_blacklist((unsigned long) p->addr) || 1595 jump_label_text_reserved(p->addr, p->addr) || 1596 static_call_text_reserved(p->addr, p->addr) || 1597 find_bug((unsigned long)p->addr)) { 1598 ret = -EINVAL; 1599 goto out; 1600 } 1601 1602 /* Check if 'p' is probing a module. */ 1603 *probed_mod = __module_text_address((unsigned long) p->addr); 1604 if (*probed_mod) { 1605 /* 1606 * We must hold a refcount of the probed module while updating 1607 * its code to prohibit unexpected unloading. 1608 */ 1609 if (unlikely(!try_module_get(*probed_mod))) { 1610 ret = -ENOENT; 1611 goto out; 1612 } 1613 1614 /* 1615 * If the module freed '.init.text', we couldn't insert 1616 * kprobes in there. 1617 */ 1618 if (within_module_init((unsigned long)p->addr, *probed_mod) && 1619 (*probed_mod)->state != MODULE_STATE_COMING) { 1620 module_put(*probed_mod); 1621 *probed_mod = NULL; 1622 ret = -ENOENT; 1623 } 1624 } 1625 out: 1626 preempt_enable(); 1627 jump_label_unlock(); 1628 1629 return ret; 1630 } 1631 1632 int register_kprobe(struct kprobe *p) 1633 { 1634 int ret; 1635 struct kprobe *old_p; 1636 struct module *probed_mod; 1637 kprobe_opcode_t *addr; 1638 1639 /* Adjust probe address from symbol */ 1640 addr = kprobe_addr(p); 1641 if (IS_ERR(addr)) 1642 return PTR_ERR(addr); 1643 p->addr = addr; 1644 1645 ret = warn_kprobe_rereg(p); 1646 if (ret) 1647 return ret; 1648 1649 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */ 1650 p->flags &= KPROBE_FLAG_DISABLED; 1651 p->nmissed = 0; 1652 INIT_LIST_HEAD(&p->list); 1653 1654 ret = check_kprobe_address_safe(p, &probed_mod); 1655 if (ret) 1656 return ret; 1657 1658 mutex_lock(&kprobe_mutex); 1659 1660 old_p = get_kprobe(p->addr); 1661 if (old_p) { 1662 /* Since this may unoptimize 'old_p', locking 'text_mutex'. */ 1663 ret = register_aggr_kprobe(old_p, p); 1664 goto out; 1665 } 1666 1667 cpus_read_lock(); 1668 /* Prevent text modification */ 1669 mutex_lock(&text_mutex); 1670 ret = prepare_kprobe(p); 1671 mutex_unlock(&text_mutex); 1672 cpus_read_unlock(); 1673 if (ret) 1674 goto out; 1675 1676 INIT_HLIST_NODE(&p->hlist); 1677 hlist_add_head_rcu(&p->hlist, 1678 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 1679 1680 if (!kprobes_all_disarmed && !kprobe_disabled(p)) { 1681 ret = arm_kprobe(p); 1682 if (ret) { 1683 hlist_del_rcu(&p->hlist); 1684 synchronize_rcu(); 1685 goto out; 1686 } 1687 } 1688 1689 /* Try to optimize kprobe */ 1690 try_to_optimize_kprobe(p); 1691 out: 1692 mutex_unlock(&kprobe_mutex); 1693 1694 if (probed_mod) 1695 module_put(probed_mod); 1696 1697 return ret; 1698 } 1699 EXPORT_SYMBOL_GPL(register_kprobe); 1700 1701 /* Check if all probes on the 'ap' are disabled. */ 1702 static bool aggr_kprobe_disabled(struct kprobe *ap) 1703 { 1704 struct kprobe *kp; 1705 1706 lockdep_assert_held(&kprobe_mutex); 1707 1708 list_for_each_entry(kp, &ap->list, list) 1709 if (!kprobe_disabled(kp)) 1710 /* 1711 * Since there is an active probe on the list, 1712 * we can't disable this 'ap'. 1713 */ 1714 return false; 1715 1716 return true; 1717 } 1718 1719 static struct kprobe *__disable_kprobe(struct kprobe *p) 1720 { 1721 struct kprobe *orig_p; 1722 int ret; 1723 1724 lockdep_assert_held(&kprobe_mutex); 1725 1726 /* Get an original kprobe for return */ 1727 orig_p = __get_valid_kprobe(p); 1728 if (unlikely(orig_p == NULL)) 1729 return ERR_PTR(-EINVAL); 1730 1731 if (!kprobe_disabled(p)) { 1732 /* Disable probe if it is a child probe */ 1733 if (p != orig_p) 1734 p->flags |= KPROBE_FLAG_DISABLED; 1735 1736 /* Try to disarm and disable this/parent probe */ 1737 if (p == orig_p || aggr_kprobe_disabled(orig_p)) { 1738 /* 1739 * If 'kprobes_all_disarmed' is set, 'orig_p' 1740 * should have already been disarmed, so 1741 * skip unneed disarming process. 1742 */ 1743 if (!kprobes_all_disarmed) { 1744 ret = disarm_kprobe(orig_p, true); 1745 if (ret) { 1746 p->flags &= ~KPROBE_FLAG_DISABLED; 1747 return ERR_PTR(ret); 1748 } 1749 } 1750 orig_p->flags |= KPROBE_FLAG_DISABLED; 1751 } 1752 } 1753 1754 return orig_p; 1755 } 1756 1757 /* 1758 * Unregister a kprobe without a scheduler synchronization. 1759 */ 1760 static int __unregister_kprobe_top(struct kprobe *p) 1761 { 1762 struct kprobe *ap, *list_p; 1763 1764 /* Disable kprobe. This will disarm it if needed. */ 1765 ap = __disable_kprobe(p); 1766 if (IS_ERR(ap)) 1767 return PTR_ERR(ap); 1768 1769 if (ap == p) 1770 /* 1771 * This probe is an independent(and non-optimized) kprobe 1772 * (not an aggrprobe). Remove from the hash list. 1773 */ 1774 goto disarmed; 1775 1776 /* Following process expects this probe is an aggrprobe */ 1777 WARN_ON(!kprobe_aggrprobe(ap)); 1778 1779 if (list_is_singular(&ap->list) && kprobe_disarmed(ap)) 1780 /* 1781 * !disarmed could be happen if the probe is under delayed 1782 * unoptimizing. 1783 */ 1784 goto disarmed; 1785 else { 1786 /* If disabling probe has special handlers, update aggrprobe */ 1787 if (p->post_handler && !kprobe_gone(p)) { 1788 list_for_each_entry(list_p, &ap->list, list) { 1789 if ((list_p != p) && (list_p->post_handler)) 1790 goto noclean; 1791 } 1792 ap->post_handler = NULL; 1793 } 1794 noclean: 1795 /* 1796 * Remove from the aggrprobe: this path will do nothing in 1797 * __unregister_kprobe_bottom(). 1798 */ 1799 list_del_rcu(&p->list); 1800 if (!kprobe_disabled(ap) && !kprobes_all_disarmed) 1801 /* 1802 * Try to optimize this probe again, because post 1803 * handler may have been changed. 1804 */ 1805 optimize_kprobe(ap); 1806 } 1807 return 0; 1808 1809 disarmed: 1810 hlist_del_rcu(&ap->hlist); 1811 return 0; 1812 } 1813 1814 static void __unregister_kprobe_bottom(struct kprobe *p) 1815 { 1816 struct kprobe *ap; 1817 1818 if (list_empty(&p->list)) 1819 /* This is an independent kprobe */ 1820 arch_remove_kprobe(p); 1821 else if (list_is_singular(&p->list)) { 1822 /* This is the last child of an aggrprobe */ 1823 ap = list_entry(p->list.next, struct kprobe, list); 1824 list_del(&p->list); 1825 free_aggr_kprobe(ap); 1826 } 1827 /* Otherwise, do nothing. */ 1828 } 1829 1830 int register_kprobes(struct kprobe **kps, int num) 1831 { 1832 int i, ret = 0; 1833 1834 if (num <= 0) 1835 return -EINVAL; 1836 for (i = 0; i < num; i++) { 1837 ret = register_kprobe(kps[i]); 1838 if (ret < 0) { 1839 if (i > 0) 1840 unregister_kprobes(kps, i); 1841 break; 1842 } 1843 } 1844 return ret; 1845 } 1846 EXPORT_SYMBOL_GPL(register_kprobes); 1847 1848 void unregister_kprobe(struct kprobe *p) 1849 { 1850 unregister_kprobes(&p, 1); 1851 } 1852 EXPORT_SYMBOL_GPL(unregister_kprobe); 1853 1854 void unregister_kprobes(struct kprobe **kps, int num) 1855 { 1856 int i; 1857 1858 if (num <= 0) 1859 return; 1860 mutex_lock(&kprobe_mutex); 1861 for (i = 0; i < num; i++) 1862 if (__unregister_kprobe_top(kps[i]) < 0) 1863 kps[i]->addr = NULL; 1864 mutex_unlock(&kprobe_mutex); 1865 1866 synchronize_rcu(); 1867 for (i = 0; i < num; i++) 1868 if (kps[i]->addr) 1869 __unregister_kprobe_bottom(kps[i]); 1870 } 1871 EXPORT_SYMBOL_GPL(unregister_kprobes); 1872 1873 int __weak kprobe_exceptions_notify(struct notifier_block *self, 1874 unsigned long val, void *data) 1875 { 1876 return NOTIFY_DONE; 1877 } 1878 NOKPROBE_SYMBOL(kprobe_exceptions_notify); 1879 1880 static struct notifier_block kprobe_exceptions_nb = { 1881 .notifier_call = kprobe_exceptions_notify, 1882 .priority = 0x7fffffff /* we need to be notified first */ 1883 }; 1884 1885 #ifdef CONFIG_KRETPROBES 1886 1887 /* This assumes the 'tsk' is the current task or the is not running. */ 1888 static kprobe_opcode_t *__kretprobe_find_ret_addr(struct task_struct *tsk, 1889 struct llist_node **cur) 1890 { 1891 struct kretprobe_instance *ri = NULL; 1892 struct llist_node *node = *cur; 1893 1894 if (!node) 1895 node = tsk->kretprobe_instances.first; 1896 else 1897 node = node->next; 1898 1899 while (node) { 1900 ri = container_of(node, struct kretprobe_instance, llist); 1901 if (ri->ret_addr != kretprobe_trampoline_addr()) { 1902 *cur = node; 1903 return ri->ret_addr; 1904 } 1905 node = node->next; 1906 } 1907 return NULL; 1908 } 1909 NOKPROBE_SYMBOL(__kretprobe_find_ret_addr); 1910 1911 /** 1912 * kretprobe_find_ret_addr -- Find correct return address modified by kretprobe 1913 * @tsk: Target task 1914 * @fp: A frame pointer 1915 * @cur: a storage of the loop cursor llist_node pointer for next call 1916 * 1917 * Find the correct return address modified by a kretprobe on @tsk in unsigned 1918 * long type. If it finds the return address, this returns that address value, 1919 * or this returns 0. 1920 * The @tsk must be 'current' or a task which is not running. @fp is a hint 1921 * to get the currect return address - which is compared with the 1922 * kretprobe_instance::fp field. The @cur is a loop cursor for searching the 1923 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the 1924 * first call, but '@cur' itself must NOT NULL. 1925 */ 1926 unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp, 1927 struct llist_node **cur) 1928 { 1929 struct kretprobe_instance *ri = NULL; 1930 kprobe_opcode_t *ret; 1931 1932 if (WARN_ON_ONCE(!cur)) 1933 return 0; 1934 1935 do { 1936 ret = __kretprobe_find_ret_addr(tsk, cur); 1937 if (!ret) 1938 break; 1939 ri = container_of(*cur, struct kretprobe_instance, llist); 1940 } while (ri->fp != fp); 1941 1942 return (unsigned long)ret; 1943 } 1944 NOKPROBE_SYMBOL(kretprobe_find_ret_addr); 1945 1946 void __weak arch_kretprobe_fixup_return(struct pt_regs *regs, 1947 kprobe_opcode_t *correct_ret_addr) 1948 { 1949 /* 1950 * Do nothing by default. Please fill this to update the fake return 1951 * address on the stack with the correct one on each arch if possible. 1952 */ 1953 } 1954 1955 unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs, 1956 void *frame_pointer) 1957 { 1958 kprobe_opcode_t *correct_ret_addr = NULL; 1959 struct kretprobe_instance *ri = NULL; 1960 struct llist_node *first, *node = NULL; 1961 struct kretprobe *rp; 1962 1963 /* Find correct address and all nodes for this frame. */ 1964 correct_ret_addr = __kretprobe_find_ret_addr(current, &node); 1965 if (!correct_ret_addr) { 1966 pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n"); 1967 BUG_ON(1); 1968 } 1969 1970 /* 1971 * Set the return address as the instruction pointer, because if the 1972 * user handler calls stack_trace_save_regs() with this 'regs', 1973 * the stack trace will start from the instruction pointer. 1974 */ 1975 instruction_pointer_set(regs, (unsigned long)correct_ret_addr); 1976 1977 /* Run the user handler of the nodes. */ 1978 first = current->kretprobe_instances.first; 1979 while (first) { 1980 ri = container_of(first, struct kretprobe_instance, llist); 1981 1982 if (WARN_ON_ONCE(ri->fp != frame_pointer)) 1983 break; 1984 1985 rp = get_kretprobe(ri); 1986 if (rp && rp->handler) { 1987 struct kprobe *prev = kprobe_running(); 1988 1989 __this_cpu_write(current_kprobe, &rp->kp); 1990 ri->ret_addr = correct_ret_addr; 1991 rp->handler(ri, regs); 1992 __this_cpu_write(current_kprobe, prev); 1993 } 1994 if (first == node) 1995 break; 1996 1997 first = first->next; 1998 } 1999 2000 arch_kretprobe_fixup_return(regs, correct_ret_addr); 2001 2002 /* Unlink all nodes for this frame. */ 2003 first = current->kretprobe_instances.first; 2004 current->kretprobe_instances.first = node->next; 2005 node->next = NULL; 2006 2007 /* Recycle free instances. */ 2008 while (first) { 2009 ri = container_of(first, struct kretprobe_instance, llist); 2010 first = first->next; 2011 2012 recycle_rp_inst(ri); 2013 } 2014 2015 return (unsigned long)correct_ret_addr; 2016 } 2017 NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) 2018 2019 /* 2020 * This kprobe pre_handler is registered with every kretprobe. When probe 2021 * hits it will set up the return probe. 2022 */ 2023 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) 2024 { 2025 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 2026 struct kretprobe_instance *ri; 2027 struct freelist_node *fn; 2028 2029 fn = freelist_try_get(&rp->freelist); 2030 if (!fn) { 2031 rp->nmissed++; 2032 return 0; 2033 } 2034 2035 ri = container_of(fn, struct kretprobe_instance, freelist); 2036 2037 if (rp->entry_handler && rp->entry_handler(ri, regs)) { 2038 freelist_add(&ri->freelist, &rp->freelist); 2039 return 0; 2040 } 2041 2042 arch_prepare_kretprobe(ri, regs); 2043 2044 __llist_add(&ri->llist, ¤t->kretprobe_instances); 2045 2046 return 0; 2047 } 2048 NOKPROBE_SYMBOL(pre_handler_kretprobe); 2049 2050 bool __weak arch_kprobe_on_func_entry(unsigned long offset) 2051 { 2052 return !offset; 2053 } 2054 2055 /** 2056 * kprobe_on_func_entry() -- check whether given address is function entry 2057 * @addr: Target address 2058 * @sym: Target symbol name 2059 * @offset: The offset from the symbol or the address 2060 * 2061 * This checks whether the given @addr+@offset or @sym+@offset is on the 2062 * function entry address or not. 2063 * This returns 0 if it is the function entry, or -EINVAL if it is not. 2064 * And also it returns -ENOENT if it fails the symbol or address lookup. 2065 * Caller must pass @addr or @sym (either one must be NULL), or this 2066 * returns -EINVAL. 2067 */ 2068 int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset) 2069 { 2070 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset); 2071 2072 if (IS_ERR(kp_addr)) 2073 return PTR_ERR(kp_addr); 2074 2075 if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset)) 2076 return -ENOENT; 2077 2078 if (!arch_kprobe_on_func_entry(offset)) 2079 return -EINVAL; 2080 2081 return 0; 2082 } 2083 2084 int register_kretprobe(struct kretprobe *rp) 2085 { 2086 int ret; 2087 struct kretprobe_instance *inst; 2088 int i; 2089 void *addr; 2090 2091 ret = kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset); 2092 if (ret) 2093 return ret; 2094 2095 /* If only 'rp->kp.addr' is specified, check reregistering kprobes */ 2096 if (rp->kp.addr && warn_kprobe_rereg(&rp->kp)) 2097 return -EINVAL; 2098 2099 if (kretprobe_blacklist_size) { 2100 addr = kprobe_addr(&rp->kp); 2101 if (IS_ERR(addr)) 2102 return PTR_ERR(addr); 2103 2104 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 2105 if (kretprobe_blacklist[i].addr == addr) 2106 return -EINVAL; 2107 } 2108 } 2109 2110 if (rp->data_size > KRETPROBE_MAX_DATA_SIZE) 2111 return -E2BIG; 2112 2113 rp->kp.pre_handler = pre_handler_kretprobe; 2114 rp->kp.post_handler = NULL; 2115 2116 /* Pre-allocate memory for max kretprobe instances */ 2117 if (rp->maxactive <= 0) { 2118 #ifdef CONFIG_PREEMPTION 2119 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus()); 2120 #else 2121 rp->maxactive = num_possible_cpus(); 2122 #endif 2123 } 2124 rp->freelist.head = NULL; 2125 rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL); 2126 if (!rp->rph) 2127 return -ENOMEM; 2128 2129 rp->rph->rp = rp; 2130 for (i = 0; i < rp->maxactive; i++) { 2131 inst = kzalloc(sizeof(struct kretprobe_instance) + 2132 rp->data_size, GFP_KERNEL); 2133 if (inst == NULL) { 2134 refcount_set(&rp->rph->ref, i); 2135 free_rp_inst(rp); 2136 return -ENOMEM; 2137 } 2138 inst->rph = rp->rph; 2139 freelist_add(&inst->freelist, &rp->freelist); 2140 } 2141 refcount_set(&rp->rph->ref, i); 2142 2143 rp->nmissed = 0; 2144 /* Establish function entry probe point */ 2145 ret = register_kprobe(&rp->kp); 2146 if (ret != 0) 2147 free_rp_inst(rp); 2148 return ret; 2149 } 2150 EXPORT_SYMBOL_GPL(register_kretprobe); 2151 2152 int register_kretprobes(struct kretprobe **rps, int num) 2153 { 2154 int ret = 0, i; 2155 2156 if (num <= 0) 2157 return -EINVAL; 2158 for (i = 0; i < num; i++) { 2159 ret = register_kretprobe(rps[i]); 2160 if (ret < 0) { 2161 if (i > 0) 2162 unregister_kretprobes(rps, i); 2163 break; 2164 } 2165 } 2166 return ret; 2167 } 2168 EXPORT_SYMBOL_GPL(register_kretprobes); 2169 2170 void unregister_kretprobe(struct kretprobe *rp) 2171 { 2172 unregister_kretprobes(&rp, 1); 2173 } 2174 EXPORT_SYMBOL_GPL(unregister_kretprobe); 2175 2176 void unregister_kretprobes(struct kretprobe **rps, int num) 2177 { 2178 int i; 2179 2180 if (num <= 0) 2181 return; 2182 mutex_lock(&kprobe_mutex); 2183 for (i = 0; i < num; i++) { 2184 if (__unregister_kprobe_top(&rps[i]->kp) < 0) 2185 rps[i]->kp.addr = NULL; 2186 rps[i]->rph->rp = NULL; 2187 } 2188 mutex_unlock(&kprobe_mutex); 2189 2190 synchronize_rcu(); 2191 for (i = 0; i < num; i++) { 2192 if (rps[i]->kp.addr) { 2193 __unregister_kprobe_bottom(&rps[i]->kp); 2194 free_rp_inst(rps[i]); 2195 } 2196 } 2197 } 2198 EXPORT_SYMBOL_GPL(unregister_kretprobes); 2199 2200 #else /* CONFIG_KRETPROBES */ 2201 int register_kretprobe(struct kretprobe *rp) 2202 { 2203 return -EOPNOTSUPP; 2204 } 2205 EXPORT_SYMBOL_GPL(register_kretprobe); 2206 2207 int register_kretprobes(struct kretprobe **rps, int num) 2208 { 2209 return -EOPNOTSUPP; 2210 } 2211 EXPORT_SYMBOL_GPL(register_kretprobes); 2212 2213 void unregister_kretprobe(struct kretprobe *rp) 2214 { 2215 } 2216 EXPORT_SYMBOL_GPL(unregister_kretprobe); 2217 2218 void unregister_kretprobes(struct kretprobe **rps, int num) 2219 { 2220 } 2221 EXPORT_SYMBOL_GPL(unregister_kretprobes); 2222 2223 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) 2224 { 2225 return 0; 2226 } 2227 NOKPROBE_SYMBOL(pre_handler_kretprobe); 2228 2229 #endif /* CONFIG_KRETPROBES */ 2230 2231 /* Set the kprobe gone and remove its instruction buffer. */ 2232 static void kill_kprobe(struct kprobe *p) 2233 { 2234 struct kprobe *kp; 2235 2236 lockdep_assert_held(&kprobe_mutex); 2237 2238 p->flags |= KPROBE_FLAG_GONE; 2239 if (kprobe_aggrprobe(p)) { 2240 /* 2241 * If this is an aggr_kprobe, we have to list all the 2242 * chained probes and mark them GONE. 2243 */ 2244 list_for_each_entry(kp, &p->list, list) 2245 kp->flags |= KPROBE_FLAG_GONE; 2246 p->post_handler = NULL; 2247 kill_optimized_kprobe(p); 2248 } 2249 /* 2250 * Here, we can remove insn_slot safely, because no thread calls 2251 * the original probed function (which will be freed soon) any more. 2252 */ 2253 arch_remove_kprobe(p); 2254 2255 /* 2256 * The module is going away. We should disarm the kprobe which 2257 * is using ftrace, because ftrace framework is still available at 2258 * 'MODULE_STATE_GOING' notification. 2259 */ 2260 if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed) 2261 disarm_kprobe_ftrace(p); 2262 } 2263 2264 /* Disable one kprobe */ 2265 int disable_kprobe(struct kprobe *kp) 2266 { 2267 int ret = 0; 2268 struct kprobe *p; 2269 2270 mutex_lock(&kprobe_mutex); 2271 2272 /* Disable this kprobe */ 2273 p = __disable_kprobe(kp); 2274 if (IS_ERR(p)) 2275 ret = PTR_ERR(p); 2276 2277 mutex_unlock(&kprobe_mutex); 2278 return ret; 2279 } 2280 EXPORT_SYMBOL_GPL(disable_kprobe); 2281 2282 /* Enable one kprobe */ 2283 int enable_kprobe(struct kprobe *kp) 2284 { 2285 int ret = 0; 2286 struct kprobe *p; 2287 2288 mutex_lock(&kprobe_mutex); 2289 2290 /* Check whether specified probe is valid. */ 2291 p = __get_valid_kprobe(kp); 2292 if (unlikely(p == NULL)) { 2293 ret = -EINVAL; 2294 goto out; 2295 } 2296 2297 if (kprobe_gone(kp)) { 2298 /* This kprobe has gone, we couldn't enable it. */ 2299 ret = -EINVAL; 2300 goto out; 2301 } 2302 2303 if (p != kp) 2304 kp->flags &= ~KPROBE_FLAG_DISABLED; 2305 2306 if (!kprobes_all_disarmed && kprobe_disabled(p)) { 2307 p->flags &= ~KPROBE_FLAG_DISABLED; 2308 ret = arm_kprobe(p); 2309 if (ret) 2310 p->flags |= KPROBE_FLAG_DISABLED; 2311 } 2312 out: 2313 mutex_unlock(&kprobe_mutex); 2314 return ret; 2315 } 2316 EXPORT_SYMBOL_GPL(enable_kprobe); 2317 2318 /* Caller must NOT call this in usual path. This is only for critical case */ 2319 void dump_kprobe(struct kprobe *kp) 2320 { 2321 pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n", 2322 kp->symbol_name, kp->offset, kp->addr); 2323 } 2324 NOKPROBE_SYMBOL(dump_kprobe); 2325 2326 int kprobe_add_ksym_blacklist(unsigned long entry) 2327 { 2328 struct kprobe_blacklist_entry *ent; 2329 unsigned long offset = 0, size = 0; 2330 2331 if (!kernel_text_address(entry) || 2332 !kallsyms_lookup_size_offset(entry, &size, &offset)) 2333 return -EINVAL; 2334 2335 ent = kmalloc(sizeof(*ent), GFP_KERNEL); 2336 if (!ent) 2337 return -ENOMEM; 2338 ent->start_addr = entry; 2339 ent->end_addr = entry + size; 2340 INIT_LIST_HEAD(&ent->list); 2341 list_add_tail(&ent->list, &kprobe_blacklist); 2342 2343 return (int)size; 2344 } 2345 2346 /* Add all symbols in given area into kprobe blacklist */ 2347 int kprobe_add_area_blacklist(unsigned long start, unsigned long end) 2348 { 2349 unsigned long entry; 2350 int ret = 0; 2351 2352 for (entry = start; entry < end; entry += ret) { 2353 ret = kprobe_add_ksym_blacklist(entry); 2354 if (ret < 0) 2355 return ret; 2356 if (ret == 0) /* In case of alias symbol */ 2357 ret = 1; 2358 } 2359 return 0; 2360 } 2361 2362 /* Remove all symbols in given area from kprobe blacklist */ 2363 static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end) 2364 { 2365 struct kprobe_blacklist_entry *ent, *n; 2366 2367 list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) { 2368 if (ent->start_addr < start || ent->start_addr >= end) 2369 continue; 2370 list_del(&ent->list); 2371 kfree(ent); 2372 } 2373 } 2374 2375 static void kprobe_remove_ksym_blacklist(unsigned long entry) 2376 { 2377 kprobe_remove_area_blacklist(entry, entry + 1); 2378 } 2379 2380 int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value, 2381 char *type, char *sym) 2382 { 2383 return -ERANGE; 2384 } 2385 2386 int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type, 2387 char *sym) 2388 { 2389 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT 2390 if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym)) 2391 return 0; 2392 #ifdef CONFIG_OPTPROBES 2393 if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym)) 2394 return 0; 2395 #endif 2396 #endif 2397 if (!arch_kprobe_get_kallsym(&symnum, value, type, sym)) 2398 return 0; 2399 return -ERANGE; 2400 } 2401 2402 int __init __weak arch_populate_kprobe_blacklist(void) 2403 { 2404 return 0; 2405 } 2406 2407 /* 2408 * Lookup and populate the kprobe_blacklist. 2409 * 2410 * Unlike the kretprobe blacklist, we'll need to determine 2411 * the range of addresses that belong to the said functions, 2412 * since a kprobe need not necessarily be at the beginning 2413 * of a function. 2414 */ 2415 static int __init populate_kprobe_blacklist(unsigned long *start, 2416 unsigned long *end) 2417 { 2418 unsigned long entry; 2419 unsigned long *iter; 2420 int ret; 2421 2422 for (iter = start; iter < end; iter++) { 2423 entry = (unsigned long)dereference_symbol_descriptor((void *)*iter); 2424 ret = kprobe_add_ksym_blacklist(entry); 2425 if (ret == -EINVAL) 2426 continue; 2427 if (ret < 0) 2428 return ret; 2429 } 2430 2431 /* Symbols in '__kprobes_text' are blacklisted */ 2432 ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start, 2433 (unsigned long)__kprobes_text_end); 2434 if (ret) 2435 return ret; 2436 2437 /* Symbols in 'noinstr' section are blacklisted */ 2438 ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start, 2439 (unsigned long)__noinstr_text_end); 2440 2441 return ret ? : arch_populate_kprobe_blacklist(); 2442 } 2443 2444 static void add_module_kprobe_blacklist(struct module *mod) 2445 { 2446 unsigned long start, end; 2447 int i; 2448 2449 if (mod->kprobe_blacklist) { 2450 for (i = 0; i < mod->num_kprobe_blacklist; i++) 2451 kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]); 2452 } 2453 2454 start = (unsigned long)mod->kprobes_text_start; 2455 if (start) { 2456 end = start + mod->kprobes_text_size; 2457 kprobe_add_area_blacklist(start, end); 2458 } 2459 2460 start = (unsigned long)mod->noinstr_text_start; 2461 if (start) { 2462 end = start + mod->noinstr_text_size; 2463 kprobe_add_area_blacklist(start, end); 2464 } 2465 } 2466 2467 static void remove_module_kprobe_blacklist(struct module *mod) 2468 { 2469 unsigned long start, end; 2470 int i; 2471 2472 if (mod->kprobe_blacklist) { 2473 for (i = 0; i < mod->num_kprobe_blacklist; i++) 2474 kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]); 2475 } 2476 2477 start = (unsigned long)mod->kprobes_text_start; 2478 if (start) { 2479 end = start + mod->kprobes_text_size; 2480 kprobe_remove_area_blacklist(start, end); 2481 } 2482 2483 start = (unsigned long)mod->noinstr_text_start; 2484 if (start) { 2485 end = start + mod->noinstr_text_size; 2486 kprobe_remove_area_blacklist(start, end); 2487 } 2488 } 2489 2490 /* Module notifier call back, checking kprobes on the module */ 2491 static int kprobes_module_callback(struct notifier_block *nb, 2492 unsigned long val, void *data) 2493 { 2494 struct module *mod = data; 2495 struct hlist_head *head; 2496 struct kprobe *p; 2497 unsigned int i; 2498 int checkcore = (val == MODULE_STATE_GOING); 2499 2500 if (val == MODULE_STATE_COMING) { 2501 mutex_lock(&kprobe_mutex); 2502 add_module_kprobe_blacklist(mod); 2503 mutex_unlock(&kprobe_mutex); 2504 } 2505 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE) 2506 return NOTIFY_DONE; 2507 2508 /* 2509 * When 'MODULE_STATE_GOING' was notified, both of module '.text' and 2510 * '.init.text' sections would be freed. When 'MODULE_STATE_LIVE' was 2511 * notified, only '.init.text' section would be freed. We need to 2512 * disable kprobes which have been inserted in the sections. 2513 */ 2514 mutex_lock(&kprobe_mutex); 2515 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2516 head = &kprobe_table[i]; 2517 hlist_for_each_entry(p, head, hlist) 2518 if (within_module_init((unsigned long)p->addr, mod) || 2519 (checkcore && 2520 within_module_core((unsigned long)p->addr, mod))) { 2521 /* 2522 * The vaddr this probe is installed will soon 2523 * be vfreed buy not synced to disk. Hence, 2524 * disarming the breakpoint isn't needed. 2525 * 2526 * Note, this will also move any optimized probes 2527 * that are pending to be removed from their 2528 * corresponding lists to the 'freeing_list' and 2529 * will not be touched by the delayed 2530 * kprobe_optimizer() work handler. 2531 */ 2532 kill_kprobe(p); 2533 } 2534 } 2535 if (val == MODULE_STATE_GOING) 2536 remove_module_kprobe_blacklist(mod); 2537 mutex_unlock(&kprobe_mutex); 2538 return NOTIFY_DONE; 2539 } 2540 2541 static struct notifier_block kprobe_module_nb = { 2542 .notifier_call = kprobes_module_callback, 2543 .priority = 0 2544 }; 2545 2546 void kprobe_free_init_mem(void) 2547 { 2548 void *start = (void *)(&__init_begin); 2549 void *end = (void *)(&__init_end); 2550 struct hlist_head *head; 2551 struct kprobe *p; 2552 int i; 2553 2554 mutex_lock(&kprobe_mutex); 2555 2556 /* Kill all kprobes on initmem because the target code has been freed. */ 2557 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2558 head = &kprobe_table[i]; 2559 hlist_for_each_entry(p, head, hlist) { 2560 if (start <= (void *)p->addr && (void *)p->addr < end) 2561 kill_kprobe(p); 2562 } 2563 } 2564 2565 mutex_unlock(&kprobe_mutex); 2566 } 2567 2568 static int __init init_kprobes(void) 2569 { 2570 int i, err = 0; 2571 2572 /* FIXME allocate the probe table, currently defined statically */ 2573 /* initialize all list heads */ 2574 for (i = 0; i < KPROBE_TABLE_SIZE; i++) 2575 INIT_HLIST_HEAD(&kprobe_table[i]); 2576 2577 err = populate_kprobe_blacklist(__start_kprobe_blacklist, 2578 __stop_kprobe_blacklist); 2579 if (err) 2580 pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err); 2581 2582 if (kretprobe_blacklist_size) { 2583 /* lookup the function address from its name */ 2584 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 2585 kretprobe_blacklist[i].addr = 2586 kprobe_lookup_name(kretprobe_blacklist[i].name, 0); 2587 if (!kretprobe_blacklist[i].addr) 2588 pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n", 2589 kretprobe_blacklist[i].name); 2590 } 2591 } 2592 2593 /* By default, kprobes are armed */ 2594 kprobes_all_disarmed = false; 2595 2596 #if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT) 2597 /* Init 'kprobe_optinsn_slots' for allocation */ 2598 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE; 2599 #endif 2600 2601 err = arch_init_kprobes(); 2602 if (!err) 2603 err = register_die_notifier(&kprobe_exceptions_nb); 2604 if (!err) 2605 err = register_module_notifier(&kprobe_module_nb); 2606 2607 kprobes_initialized = (err == 0); 2608 kprobe_sysctls_init(); 2609 return err; 2610 } 2611 early_initcall(init_kprobes); 2612 2613 #if defined(CONFIG_OPTPROBES) 2614 static int __init init_optprobes(void) 2615 { 2616 /* 2617 * Enable kprobe optimization - this kicks the optimizer which 2618 * depends on synchronize_rcu_tasks() and ksoftirqd, that is 2619 * not spawned in early initcall. So delay the optimization. 2620 */ 2621 optimize_all_kprobes(); 2622 2623 return 0; 2624 } 2625 subsys_initcall(init_optprobes); 2626 #endif 2627 2628 #ifdef CONFIG_DEBUG_FS 2629 static void report_probe(struct seq_file *pi, struct kprobe *p, 2630 const char *sym, int offset, char *modname, struct kprobe *pp) 2631 { 2632 char *kprobe_type; 2633 void *addr = p->addr; 2634 2635 if (p->pre_handler == pre_handler_kretprobe) 2636 kprobe_type = "r"; 2637 else 2638 kprobe_type = "k"; 2639 2640 if (!kallsyms_show_value(pi->file->f_cred)) 2641 addr = NULL; 2642 2643 if (sym) 2644 seq_printf(pi, "%px %s %s+0x%x %s ", 2645 addr, kprobe_type, sym, offset, 2646 (modname ? modname : " ")); 2647 else /* try to use %pS */ 2648 seq_printf(pi, "%px %s %pS ", 2649 addr, kprobe_type, p->addr); 2650 2651 if (!pp) 2652 pp = p; 2653 seq_printf(pi, "%s%s%s%s\n", 2654 (kprobe_gone(p) ? "[GONE]" : ""), 2655 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""), 2656 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""), 2657 (kprobe_ftrace(pp) ? "[FTRACE]" : "")); 2658 } 2659 2660 static void *kprobe_seq_start(struct seq_file *f, loff_t *pos) 2661 { 2662 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL; 2663 } 2664 2665 static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos) 2666 { 2667 (*pos)++; 2668 if (*pos >= KPROBE_TABLE_SIZE) 2669 return NULL; 2670 return pos; 2671 } 2672 2673 static void kprobe_seq_stop(struct seq_file *f, void *v) 2674 { 2675 /* Nothing to do */ 2676 } 2677 2678 static int show_kprobe_addr(struct seq_file *pi, void *v) 2679 { 2680 struct hlist_head *head; 2681 struct kprobe *p, *kp; 2682 const char *sym = NULL; 2683 unsigned int i = *(loff_t *) v; 2684 unsigned long offset = 0; 2685 char *modname, namebuf[KSYM_NAME_LEN]; 2686 2687 head = &kprobe_table[i]; 2688 preempt_disable(); 2689 hlist_for_each_entry_rcu(p, head, hlist) { 2690 sym = kallsyms_lookup((unsigned long)p->addr, NULL, 2691 &offset, &modname, namebuf); 2692 if (kprobe_aggrprobe(p)) { 2693 list_for_each_entry_rcu(kp, &p->list, list) 2694 report_probe(pi, kp, sym, offset, modname, p); 2695 } else 2696 report_probe(pi, p, sym, offset, modname, NULL); 2697 } 2698 preempt_enable(); 2699 return 0; 2700 } 2701 2702 static const struct seq_operations kprobes_sops = { 2703 .start = kprobe_seq_start, 2704 .next = kprobe_seq_next, 2705 .stop = kprobe_seq_stop, 2706 .show = show_kprobe_addr 2707 }; 2708 2709 DEFINE_SEQ_ATTRIBUTE(kprobes); 2710 2711 /* kprobes/blacklist -- shows which functions can not be probed */ 2712 static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos) 2713 { 2714 mutex_lock(&kprobe_mutex); 2715 return seq_list_start(&kprobe_blacklist, *pos); 2716 } 2717 2718 static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos) 2719 { 2720 return seq_list_next(v, &kprobe_blacklist, pos); 2721 } 2722 2723 static int kprobe_blacklist_seq_show(struct seq_file *m, void *v) 2724 { 2725 struct kprobe_blacklist_entry *ent = 2726 list_entry(v, struct kprobe_blacklist_entry, list); 2727 2728 /* 2729 * If '/proc/kallsyms' is not showing kernel address, we won't 2730 * show them here either. 2731 */ 2732 if (!kallsyms_show_value(m->file->f_cred)) 2733 seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL, 2734 (void *)ent->start_addr); 2735 else 2736 seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr, 2737 (void *)ent->end_addr, (void *)ent->start_addr); 2738 return 0; 2739 } 2740 2741 static void kprobe_blacklist_seq_stop(struct seq_file *f, void *v) 2742 { 2743 mutex_unlock(&kprobe_mutex); 2744 } 2745 2746 static const struct seq_operations kprobe_blacklist_sops = { 2747 .start = kprobe_blacklist_seq_start, 2748 .next = kprobe_blacklist_seq_next, 2749 .stop = kprobe_blacklist_seq_stop, 2750 .show = kprobe_blacklist_seq_show, 2751 }; 2752 DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist); 2753 2754 static int arm_all_kprobes(void) 2755 { 2756 struct hlist_head *head; 2757 struct kprobe *p; 2758 unsigned int i, total = 0, errors = 0; 2759 int err, ret = 0; 2760 2761 mutex_lock(&kprobe_mutex); 2762 2763 /* If kprobes are armed, just return */ 2764 if (!kprobes_all_disarmed) 2765 goto already_enabled; 2766 2767 /* 2768 * optimize_kprobe() called by arm_kprobe() checks 2769 * kprobes_all_disarmed, so set kprobes_all_disarmed before 2770 * arm_kprobe. 2771 */ 2772 kprobes_all_disarmed = false; 2773 /* Arming kprobes doesn't optimize kprobe itself */ 2774 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2775 head = &kprobe_table[i]; 2776 /* Arm all kprobes on a best-effort basis */ 2777 hlist_for_each_entry(p, head, hlist) { 2778 if (!kprobe_disabled(p)) { 2779 err = arm_kprobe(p); 2780 if (err) { 2781 errors++; 2782 ret = err; 2783 } 2784 total++; 2785 } 2786 } 2787 } 2788 2789 if (errors) 2790 pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n", 2791 errors, total); 2792 else 2793 pr_info("Kprobes globally enabled\n"); 2794 2795 already_enabled: 2796 mutex_unlock(&kprobe_mutex); 2797 return ret; 2798 } 2799 2800 static int disarm_all_kprobes(void) 2801 { 2802 struct hlist_head *head; 2803 struct kprobe *p; 2804 unsigned int i, total = 0, errors = 0; 2805 int err, ret = 0; 2806 2807 mutex_lock(&kprobe_mutex); 2808 2809 /* If kprobes are already disarmed, just return */ 2810 if (kprobes_all_disarmed) { 2811 mutex_unlock(&kprobe_mutex); 2812 return 0; 2813 } 2814 2815 kprobes_all_disarmed = true; 2816 2817 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2818 head = &kprobe_table[i]; 2819 /* Disarm all kprobes on a best-effort basis */ 2820 hlist_for_each_entry(p, head, hlist) { 2821 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) { 2822 err = disarm_kprobe(p, false); 2823 if (err) { 2824 errors++; 2825 ret = err; 2826 } 2827 total++; 2828 } 2829 } 2830 } 2831 2832 if (errors) 2833 pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n", 2834 errors, total); 2835 else 2836 pr_info("Kprobes globally disabled\n"); 2837 2838 mutex_unlock(&kprobe_mutex); 2839 2840 /* Wait for disarming all kprobes by optimizer */ 2841 wait_for_kprobe_optimizer(); 2842 2843 return ret; 2844 } 2845 2846 /* 2847 * XXX: The debugfs bool file interface doesn't allow for callbacks 2848 * when the bool state is switched. We can reuse that facility when 2849 * available 2850 */ 2851 static ssize_t read_enabled_file_bool(struct file *file, 2852 char __user *user_buf, size_t count, loff_t *ppos) 2853 { 2854 char buf[3]; 2855 2856 if (!kprobes_all_disarmed) 2857 buf[0] = '1'; 2858 else 2859 buf[0] = '0'; 2860 buf[1] = '\n'; 2861 buf[2] = 0x00; 2862 return simple_read_from_buffer(user_buf, count, ppos, buf, 2); 2863 } 2864 2865 static ssize_t write_enabled_file_bool(struct file *file, 2866 const char __user *user_buf, size_t count, loff_t *ppos) 2867 { 2868 bool enable; 2869 int ret; 2870 2871 ret = kstrtobool_from_user(user_buf, count, &enable); 2872 if (ret) 2873 return ret; 2874 2875 ret = enable ? arm_all_kprobes() : disarm_all_kprobes(); 2876 if (ret) 2877 return ret; 2878 2879 return count; 2880 } 2881 2882 static const struct file_operations fops_kp = { 2883 .read = read_enabled_file_bool, 2884 .write = write_enabled_file_bool, 2885 .llseek = default_llseek, 2886 }; 2887 2888 static int __init debugfs_kprobe_init(void) 2889 { 2890 struct dentry *dir; 2891 2892 dir = debugfs_create_dir("kprobes", NULL); 2893 2894 debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops); 2895 2896 debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp); 2897 2898 debugfs_create_file("blacklist", 0400, dir, NULL, 2899 &kprobe_blacklist_fops); 2900 2901 return 0; 2902 } 2903 2904 late_initcall(debugfs_kprobe_init); 2905 #endif /* CONFIG_DEBUG_FS */ 2906