1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Kernel Probes (KProbes) 4 * 5 * Copyright (C) IBM Corporation, 2002, 2004 6 * 7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 8 * Probes initial implementation (includes suggestions from 9 * Rusty Russell). 10 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 11 * hlists and exceptions notifier as suggested by Andi Kleen. 12 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 13 * interface to access function arguments. 14 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 15 * exceptions notifier to be first on the priority list. 16 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 17 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 18 * <prasanna@in.ibm.com> added function-return probes. 19 */ 20 21 #define pr_fmt(fmt) "kprobes: " fmt 22 23 #include <linux/kprobes.h> 24 #include <linux/hash.h> 25 #include <linux/init.h> 26 #include <linux/slab.h> 27 #include <linux/stddef.h> 28 #include <linux/export.h> 29 #include <linux/kallsyms.h> 30 #include <linux/freezer.h> 31 #include <linux/seq_file.h> 32 #include <linux/debugfs.h> 33 #include <linux/sysctl.h> 34 #include <linux/kdebug.h> 35 #include <linux/kthread.h> 36 #include <linux/memory.h> 37 #include <linux/ftrace.h> 38 #include <linux/cpu.h> 39 #include <linux/jump_label.h> 40 #include <linux/static_call.h> 41 #include <linux/perf_event.h> 42 #include <linux/execmem.h> 43 #include <linux/cleanup.h> 44 #include <linux/wait.h> 45 #include <linux/wait_bit.h> 46 #include <linux/rcupdate.h> 47 48 #include <asm/sections.h> 49 #include <asm/cacheflush.h> 50 #include <asm/errno.h> 51 #include <linux/uaccess.h> 52 53 #define KPROBE_HASH_BITS 6 54 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 55 56 #if !defined(CONFIG_OPTPROBES) || !defined(CONFIG_SYSCTL) 57 #define kprobe_sysctls_init() do { } while (0) 58 #endif 59 60 static int kprobes_initialized; 61 /* kprobe_table can be accessed by 62 * - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held. 63 * Or 64 * - RCU hlist traversal under disabling preempt (breakpoint handlers) 65 */ 66 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 67 68 /* NOTE: change this value only with 'kprobe_mutex' held */ 69 static bool kprobes_all_disarmed; 70 71 /* This protects 'kprobe_table' and 'optprobe_gens' */ 72 static DEFINE_MUTEX(kprobe_mutex); 73 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance); 74 75 kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, 76 unsigned int __unused) 77 { 78 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name))); 79 } 80 81 /* 82 * Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where 83 * kprobes can not probe. 84 */ 85 static LIST_HEAD(kprobe_blacklist); 86 87 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT 88 /* 89 * 'kprobe::ainsn.insn' points to the copy of the instruction to be 90 * single-stepped. x86_64, POWER4 and above have no-exec support and 91 * stepping on the instruction on a vmalloced/kmalloced/data page 92 * is a recipe for disaster 93 */ 94 struct kprobe_insn_page { 95 struct list_head list; 96 kprobe_opcode_t *insns; /* Page of instruction slots */ 97 struct kprobe_insn_cache *cache; 98 int nused; 99 int ngarbage; 100 char slot_used[]; 101 }; 102 103 static int slots_per_page(struct kprobe_insn_cache *c) 104 { 105 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t)); 106 } 107 108 enum kprobe_slot_state { 109 SLOT_CLEAN = 0, 110 SLOT_DIRTY = 1, 111 SLOT_USED = 2, 112 }; 113 114 void __weak *alloc_insn_page(void) 115 { 116 /* 117 * Use execmem_alloc() so this page is within +/- 2GB of where the 118 * kernel image and loaded module images reside. This is required 119 * for most of the architectures. 120 * (e.g. x86-64 needs this to handle the %rip-relative fixups.) 121 */ 122 return execmem_alloc(EXECMEM_KPROBES, PAGE_SIZE); 123 } 124 125 static void free_insn_page(void *page) 126 { 127 execmem_free(page); 128 } 129 130 struct kprobe_insn_cache kprobe_insn_slots = { 131 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex), 132 .alloc = alloc_insn_page, 133 .free = free_insn_page, 134 .sym = KPROBE_INSN_PAGE_SYM, 135 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages), 136 .insn_size = MAX_INSN_SIZE, 137 .nr_garbage = 0, 138 }; 139 static int collect_garbage_slots(struct kprobe_insn_cache *c); 140 141 /** 142 * __get_insn_slot - Find a slot on an executable page for an instruction. 143 * @c: Pointer to kprobe instruction cache 144 * 145 * Description: Locates available slot on existing executable pages, 146 * allocates an executable page if there's no room on existing ones. 147 * Return: Pointer to instruction slot on success, NULL on failure. 148 */ 149 kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c) 150 { 151 struct kprobe_insn_page *kip; 152 153 /* Since the slot array is not protected by rcu, we need a mutex */ 154 guard(mutex)(&c->mutex); 155 do { 156 guard(rcu)(); 157 list_for_each_entry_rcu(kip, &c->pages, list) { 158 if (kip->nused < slots_per_page(c)) { 159 int i; 160 161 for (i = 0; i < slots_per_page(c); i++) { 162 if (kip->slot_used[i] == SLOT_CLEAN) { 163 kip->slot_used[i] = SLOT_USED; 164 kip->nused++; 165 return kip->insns + (i * c->insn_size); 166 } 167 } 168 /* kip->nused is broken. Fix it. */ 169 kip->nused = slots_per_page(c); 170 WARN_ON(1); 171 } 172 } 173 /* If there are any garbage slots, collect it and try again. */ 174 } while (c->nr_garbage && collect_garbage_slots(c) == 0); 175 176 /* All out of space. Need to allocate a new page. */ 177 kip = kmalloc_flex(*kip, slot_used, slots_per_page(c)); 178 if (!kip) 179 return NULL; 180 181 kip->insns = c->alloc(); 182 if (!kip->insns) { 183 kfree(kip); 184 return NULL; 185 } 186 INIT_LIST_HEAD(&kip->list); 187 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c)); 188 kip->slot_used[0] = SLOT_USED; 189 kip->nused = 1; 190 kip->ngarbage = 0; 191 kip->cache = c; 192 list_add_rcu(&kip->list, &c->pages); 193 194 /* Record the perf ksymbol register event after adding the page */ 195 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns, 196 PAGE_SIZE, false, c->sym); 197 198 return kip->insns; 199 } 200 201 /* Return true if all garbages are collected, otherwise false. */ 202 static bool collect_one_slot(struct kprobe_insn_page *kip, int idx) 203 { 204 kip->slot_used[idx] = SLOT_CLEAN; 205 kip->nused--; 206 if (kip->nused != 0) 207 return false; 208 209 /* 210 * Page is no longer in use. Free it unless 211 * it's the last one. We keep the last one 212 * so as not to have to set it up again the 213 * next time somebody inserts a probe. 214 */ 215 if (!list_is_singular(&kip->list)) { 216 /* 217 * Record perf ksymbol unregister event before removing 218 * the page. 219 */ 220 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, 221 (unsigned long)kip->insns, PAGE_SIZE, true, 222 kip->cache->sym); 223 list_del_rcu(&kip->list); 224 synchronize_rcu(); 225 kip->cache->free(kip->insns); 226 kfree(kip); 227 } 228 return true; 229 } 230 231 static int collect_garbage_slots(struct kprobe_insn_cache *c) 232 { 233 struct kprobe_insn_page *kip, *next; 234 235 /* Ensure no-one is interrupted on the garbages */ 236 synchronize_rcu(); 237 238 list_for_each_entry_safe(kip, next, &c->pages, list) { 239 int i; 240 241 if (kip->ngarbage == 0) 242 continue; 243 kip->ngarbage = 0; /* we will collect all garbages */ 244 for (i = 0; i < slots_per_page(c); i++) { 245 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i)) 246 break; 247 } 248 } 249 c->nr_garbage = 0; 250 return 0; 251 } 252 253 static long __find_insn_page(struct kprobe_insn_cache *c, 254 kprobe_opcode_t *slot, struct kprobe_insn_page **pkip) 255 { 256 struct kprobe_insn_page *kip = NULL; 257 long idx; 258 259 guard(rcu)(); 260 list_for_each_entry_rcu(kip, &c->pages, list) { 261 idx = ((long)slot - (long)kip->insns) / 262 (c->insn_size * sizeof(kprobe_opcode_t)); 263 if (idx >= 0 && idx < slots_per_page(c)) { 264 *pkip = kip; 265 return idx; 266 } 267 } 268 /* Could not find this slot. */ 269 WARN_ON(1); 270 *pkip = NULL; 271 return -1; 272 } 273 274 void __free_insn_slot(struct kprobe_insn_cache *c, 275 kprobe_opcode_t *slot, int dirty) 276 { 277 struct kprobe_insn_page *kip = NULL; 278 long idx; 279 280 guard(mutex)(&c->mutex); 281 idx = __find_insn_page(c, slot, &kip); 282 /* Mark and sweep: this may sleep */ 283 if (kip) { 284 /* Check double free */ 285 WARN_ON(kip->slot_used[idx] != SLOT_USED); 286 if (dirty) { 287 kip->slot_used[idx] = SLOT_DIRTY; 288 kip->ngarbage++; 289 if (++c->nr_garbage > slots_per_page(c)) 290 collect_garbage_slots(c); 291 } else { 292 collect_one_slot(kip, idx); 293 } 294 } 295 } 296 297 /* 298 * Check given address is on the page of kprobe instruction slots. 299 * This will be used for checking whether the address on a stack 300 * is on a text area or not. 301 */ 302 bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr) 303 { 304 struct kprobe_insn_page *kip; 305 bool ret = false; 306 307 rcu_read_lock(); 308 list_for_each_entry_rcu(kip, &c->pages, list) { 309 if (addr >= (unsigned long)kip->insns && 310 addr < (unsigned long)kip->insns + PAGE_SIZE) { 311 ret = true; 312 break; 313 } 314 } 315 rcu_read_unlock(); 316 317 return ret; 318 } 319 320 int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum, 321 unsigned long *value, char *type, char *sym) 322 { 323 struct kprobe_insn_page *kip; 324 int ret = -ERANGE; 325 326 rcu_read_lock(); 327 list_for_each_entry_rcu(kip, &c->pages, list) { 328 if ((*symnum)--) 329 continue; 330 strscpy(sym, c->sym, KSYM_NAME_LEN); 331 *type = 't'; 332 *value = (unsigned long)kip->insns; 333 ret = 0; 334 break; 335 } 336 rcu_read_unlock(); 337 338 return ret; 339 } 340 341 #ifdef CONFIG_OPTPROBES 342 void __weak *alloc_optinsn_page(void) 343 { 344 return alloc_insn_page(); 345 } 346 347 void __weak free_optinsn_page(void *page) 348 { 349 free_insn_page(page); 350 } 351 352 /* For optimized_kprobe buffer */ 353 struct kprobe_insn_cache kprobe_optinsn_slots = { 354 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex), 355 .alloc = alloc_optinsn_page, 356 .free = free_optinsn_page, 357 .sym = KPROBE_OPTINSN_PAGE_SYM, 358 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages), 359 /* .insn_size is initialized later */ 360 .nr_garbage = 0, 361 }; 362 #endif /* CONFIG_OPTPROBES */ 363 #endif /* __ARCH_WANT_KPROBES_INSN_SLOT */ 364 365 /* We have preemption disabled.. so it is safe to use __ versions */ 366 static inline void set_kprobe_instance(struct kprobe *kp) 367 { 368 __this_cpu_write(kprobe_instance, kp); 369 } 370 371 static inline void reset_kprobe_instance(void) 372 { 373 __this_cpu_write(kprobe_instance, NULL); 374 } 375 376 /* 377 * This routine is called either: 378 * - under the 'kprobe_mutex' - during kprobe_[un]register(). 379 * OR 380 * - with preemption disabled - from architecture specific code. 381 */ 382 struct kprobe *get_kprobe(void *addr) 383 { 384 struct hlist_head *head; 385 struct kprobe *p; 386 387 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 388 hlist_for_each_entry_rcu(p, head, hlist, 389 lockdep_is_held(&kprobe_mutex)) { 390 if (p->addr == addr) 391 return p; 392 } 393 394 return NULL; 395 } 396 NOKPROBE_SYMBOL(get_kprobe); 397 398 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs); 399 400 /* Return true if 'p' is an aggregator */ 401 static inline bool kprobe_aggrprobe(struct kprobe *p) 402 { 403 return p->pre_handler == aggr_pre_handler; 404 } 405 406 /* Return true if 'p' is unused */ 407 static inline bool kprobe_unused(struct kprobe *p) 408 { 409 return kprobe_aggrprobe(p) && kprobe_disabled(p) && 410 list_empty(&p->list); 411 } 412 413 /* Keep all fields in the kprobe consistent. */ 414 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p) 415 { 416 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t)); 417 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn)); 418 } 419 420 #ifdef CONFIG_OPTPROBES 421 /* NOTE: This is protected by 'kprobe_mutex'. */ 422 static bool kprobes_allow_optimization; 423 424 /* 425 * Call all 'kprobe::pre_handler' on the list, but ignores its return value. 426 * This must be called from arch-dep optimized caller. 427 */ 428 void opt_pre_handler(struct kprobe *p, struct pt_regs *regs) 429 { 430 struct kprobe *kp; 431 432 list_for_each_entry_rcu(kp, &p->list, list) { 433 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 434 set_kprobe_instance(kp); 435 kp->pre_handler(kp, regs); 436 } 437 reset_kprobe_instance(); 438 } 439 } 440 NOKPROBE_SYMBOL(opt_pre_handler); 441 442 /* Free optimized instructions and optimized_kprobe */ 443 static void free_aggr_kprobe(struct kprobe *p) 444 { 445 struct optimized_kprobe *op; 446 447 op = container_of(p, struct optimized_kprobe, kp); 448 arch_remove_optimized_kprobe(op); 449 arch_remove_kprobe(p); 450 kfree(op); 451 } 452 453 /* Return true if the kprobe is ready for optimization. */ 454 static inline int kprobe_optready(struct kprobe *p) 455 { 456 struct optimized_kprobe *op; 457 458 if (kprobe_aggrprobe(p)) { 459 op = container_of(p, struct optimized_kprobe, kp); 460 return arch_prepared_optinsn(&op->optinsn); 461 } 462 463 return 0; 464 } 465 466 /* Return true if the kprobe is disarmed. Note: p must be on hash list */ 467 bool kprobe_disarmed(struct kprobe *p) 468 { 469 struct optimized_kprobe *op; 470 471 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */ 472 if (!kprobe_aggrprobe(p)) 473 return kprobe_disabled(p); 474 475 op = container_of(p, struct optimized_kprobe, kp); 476 477 return kprobe_disabled(p) && list_empty(&op->list); 478 } 479 480 481 /* 482 * Return an optimized kprobe whose optimizing code replaces 483 * instructions including 'addr' (exclude breakpoint). 484 */ 485 static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr) 486 { 487 int i; 488 struct kprobe *p; 489 struct optimized_kprobe *op; 490 491 /* Don't check i == 0, since that is a breakpoint case. */ 492 for (i = 1; i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) { 493 p = get_kprobe(addr - i); 494 /* A disabled probe can have prepared, but inactive, optinsns. */ 495 if (!p || !kprobe_optready(p) || kprobe_disarmed(p)) 496 continue; 497 498 op = container_of(p, struct optimized_kprobe, kp); 499 if (arch_within_optimized_kprobe(op, addr)) 500 return p; 501 } 502 503 return NULL; 504 } 505 506 #define OPTPROBE_GEN_MAX 2 507 508 struct optprobe_generation { 509 /* Probes waiting for grace period to drain tasks from original text */ 510 struct list_head optimizing_list; 511 /* Probes queued for batch unoptimization upon dispatch */ 512 struct list_head unoptimizing_list; 513 /* In-use probes waiting for grace period to drain optinsn slot */ 514 struct list_head cooling_list; 515 /* Unused probes unhashed and waiting for batch release */ 516 struct list_head freeing_list; 517 /* Tasks RCU callback head for asynchronous waiting */ 518 struct rcu_head rcu; 519 /* True if dispatched and awaiting Tasks RCU callback */ 520 bool in_flight; 521 /* True when Tasks RCU callback has fired and ready to finalize */ 522 bool ready; 523 }; 524 525 /* 526 * Generational ring of optprobes. 527 * 528 * Incoming probe requests are queued into the waiting room generation 529 * (optprobe_gens[optprobe_cur_gen]). When dispatched, the generation 530 * unoptimizes its probes, invokes call_rcu_tasks(), and optprobe_cur_gen 531 * advances to the next slot. 532 * 533 * To ensure an idle generation is always available to collect incoming 534 * requests without dynamic allocation, the last available generation slot 535 * is never dispatched until another generation has finished. 536 */ 537 static struct optprobe_generation optprobe_gens[OPTPROBE_GEN_MAX]; 538 static int optprobe_cur_gen; 539 540 static void optimize_kprobe(struct kprobe *p); 541 static struct task_struct *kprobe_optimizer_task; 542 static wait_queue_head_t kprobe_optimizer_wait; 543 static atomic_t optimizer_state; 544 enum { 545 OPTIMIZER_ST_IDLE = 0, 546 OPTIMIZER_ST_KICKED = 1, 547 OPTIMIZER_ST_FLUSHING = 2, 548 }; 549 550 /* Bumped at the end of each kprobe_optimizer() pass, under 'kprobe_mutex' */ 551 static unsigned long optimizer_passes; 552 553 #define OPTIMIZE_DELAY 5 554 555 static bool optprobe_has_queued_probes(void) 556 { 557 struct optprobe_generation *gen = &optprobe_gens[optprobe_cur_gen]; 558 559 return !list_empty(&gen->optimizing_list) || 560 !list_empty(&gen->unoptimizing_list) || 561 !list_empty(&gen->cooling_list) || 562 !list_empty(&gen->freeing_list); 563 } 564 565 static int optprobe_active_gens_count(void) 566 { 567 int count = 0; 568 int i; 569 570 for (i = 0; i < OPTPROBE_GEN_MAX; i++) { 571 if (optprobe_gens[i].in_flight || optprobe_gens[i].ready) 572 count++; 573 } 574 return count; 575 } 576 577 /* 578 * The last generation must not be fired until another generation is done. 579 * (Thus the last generation acts as the waiting room.) 580 */ 581 static bool optprobe_can_fire(void) 582 { 583 return optprobe_active_gens_count() < OPTPROBE_GEN_MAX - 1; 584 } 585 586 static bool optprobe_has_ready_gens(void) 587 { 588 int i; 589 590 for (i = 0; i < OPTPROBE_GEN_MAX; i++) { 591 if (READ_ONCE(optprobe_gens[i].ready)) 592 return true; 593 } 594 return false; 595 } 596 597 static bool optprobe_optimizer_busy(void) 598 { 599 return optprobe_has_queued_probes() || (optprobe_active_gens_count() > 0); 600 } 601 602 /* 603 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint 604 * if need) kprobes listed on 'unopt_list'. 605 */ 606 static void do_unoptimize_kprobes(struct list_head *unopt_list, 607 struct list_head *free_list, 608 struct list_head *cooling_list) 609 { 610 struct optimized_kprobe *op, *tmp; 611 LIST_HEAD(unoptimized); 612 613 lockdep_assert_held(&text_mutex); 614 /* 615 * The optimization/unoptimization refers 'online_cpus' via 616 * stop_machine() and cpu-hotplug modifies the 'online_cpus'. 617 * And same time, 'text_mutex' will be held in cpu-hotplug and here. 618 * This combination can cause a deadlock (cpu-hotplug tries to lock 619 * 'text_mutex' but stop_machine() can not be done because 620 * the 'online_cpus' has been changed) 621 * To avoid this deadlock, caller must have locked cpu-hotplug 622 * for preventing cpu-hotplug outside of 'text_mutex' locking. 623 */ 624 lockdep_assert_cpus_held(); 625 626 if (list_empty(unopt_list)) 627 return; 628 629 arch_unoptimize_kprobes(unopt_list, &unoptimized); 630 631 /* Loop on newly unoptimized probes for disarming and hash list removal */ 632 list_for_each_entry_safe(op, tmp, &unoptimized, list) { 633 /* Switching from detour code to origin */ 634 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 635 /* 636 * Disarm probes if marked disabled and not gone. Unhashed probes 637 * were disarmed before they were unhashed. 638 */ 639 if (kprobe_disabled(&op->kp) && !kprobe_gone(&op->kp) && 640 !hlist_unhashed(&op->kp.hlist)) 641 arch_disarm_kprobe(&op->kp); 642 if (kprobe_unused(&op->kp)) { 643 /* 644 * Remove unused probes from hash list. After waiting 645 * for synchronization, these probes are reclaimed. 646 * (reclaiming is done by do_free_cleaned_kprobes().) 647 */ 648 hlist_del_init_rcu(&op->kp.hlist); 649 list_move(&op->list, free_list); 650 } else { 651 /* 652 * Keep on cooling_list until the quiescence period 653 * completes so that kprobe_disarmed() remains false and 654 * unregister_kprobes() does not prematurely free it. 655 */ 656 list_move(&op->list, cooling_list); 657 } 658 } 659 } 660 661 /* Reclaim all kprobes on the 'free_list' */ 662 static void do_free_cleaned_kprobes(struct list_head *free_list) 663 { 664 struct optimized_kprobe *op, *tmp; 665 666 list_for_each_entry_safe(op, tmp, free_list, list) { 667 struct kprobe *_p; 668 669 list_del_init(&op->list); 670 if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) { 671 /* 672 * This must not happen, but if there is a kprobe 673 * still in use, keep it on kprobes hash list. 674 */ 675 continue; 676 } 677 678 /* 679 * The aggregator was holding back another probe while it sat on the 680 * unoptimizing/freeing lists. Now that the aggregator has been fully 681 * reverted we can safely retry the optimization of that sibling. 682 */ 683 _p = get_optimized_kprobe(op->kp.addr); 684 if (unlikely(_p)) 685 optimize_kprobe(_p); 686 687 free_aggr_kprobe(&op->kp); 688 } 689 } 690 691 static void kick_kprobe_optimizer(void); 692 693 static void optprobe_generation_rcu_cb(struct rcu_head *rcu) 694 { 695 struct optprobe_generation *gen; 696 697 gen = container_of(rcu, struct optprobe_generation, rcu); 698 WRITE_ONCE(gen->ready, true); 699 wake_up(&kprobe_optimizer_wait); 700 } 701 702 static void optprobe_finalize_generation(struct optprobe_generation *gen) 703 { 704 struct optimized_kprobe *op, *tmp; 705 706 lockdep_assert_held(&kprobe_mutex); 707 708 scoped_guard(cpus_read_lock) { 709 guard(mutex)(&text_mutex); 710 711 /* Optimization never be done when disarmed */ 712 if (!kprobes_all_disarmed && kprobes_allow_optimization && 713 !list_empty(&gen->optimizing_list)) 714 arch_optimize_kprobes(&gen->optimizing_list); 715 } 716 717 /* Free cleaned kprobes after quiescence period */ 718 do_free_cleaned_kprobes(&gen->freeing_list); 719 720 /* Finalize unoptimized kprobes whose quiescence period completed */ 721 list_for_each_entry_safe(op, tmp, &gen->cooling_list, list) { 722 if (kprobe_unused(&op->kp)) { 723 /* 724 * Unregistered while quiescence period was in flight. 725 * Unhash it now and move to cur_gen's freeing list so 726 * it will be reclaimed after the next quiescence period. 727 */ 728 hlist_del_init_rcu(&op->kp.hlist); 729 list_move(&op->list, &optprobe_gens[optprobe_cur_gen].freeing_list); 730 kick_kprobe_optimizer(); 731 } else { 732 struct kprobe *_p; 733 734 /* Still in use; now safely disarmed */ 735 list_del_init(&op->list); 736 if (!kprobe_disabled(&op->kp)) { 737 optimize_kprobe(&op->kp); 738 } else { 739 _p = get_optimized_kprobe(op->kp.addr); 740 if (unlikely(_p)) 741 optimize_kprobe(_p); 742 } 743 } 744 } 745 746 gen->in_flight = false; 747 WRITE_ONCE(gen->ready, false); 748 } 749 750 static bool optprobe_dispatch_generation(void) 751 { 752 struct optprobe_generation *gen; 753 754 lockdep_assert_held(&kprobe_mutex); 755 756 if (!optprobe_can_fire() || !optprobe_has_queued_probes()) 757 return false; 758 759 gen = &optprobe_gens[optprobe_cur_gen]; 760 761 scoped_guard(cpus_read_lock) { 762 guard(mutex)(&text_mutex); 763 764 /* 765 * Unoptimize kprobes and collect cleaned (unused and disarmed) 766 * kprobes before waiting for quiescence period. 767 */ 768 do_unoptimize_kprobes(&gen->unoptimizing_list, &gen->freeing_list, 769 &gen->cooling_list); 770 } 771 772 /* Advance cur_gen to the next generation slot */ 773 optprobe_cur_gen = (optprobe_cur_gen + 1) % OPTPROBE_GEN_MAX; 774 775 gen->in_flight = true; 776 WRITE_ONCE(gen->ready, false); 777 778 call_rcu_tasks(&gen->rcu, optprobe_generation_rcu_cb); 779 return true; 780 } 781 782 /* Kprobe jump optimizer */ 783 static void kprobe_optimizer(void) 784 { 785 bool progress = false; 786 int i; 787 788 guard(mutex)(&kprobe_mutex); 789 790 /* Step 1: Finalize any generation whose Tasks RCU grace period completed */ 791 for (i = 0; i < OPTPROBE_GEN_MAX; i++) { 792 if (READ_ONCE(optprobe_gens[i].ready)) { 793 optprobe_finalize_generation(&optprobe_gens[i]); 794 progress = true; 795 } 796 } 797 798 /* Step 2: Dispatch waiting room generation if allowed */ 799 if (optprobe_dispatch_generation()) 800 progress = true; 801 802 /* Step 3: Wake up flushers if progress was made */ 803 if (progress) { 804 optimizer_passes++; 805 wake_up_var_locked(&optimizer_passes, &kprobe_mutex); 806 } 807 808 if (optprobe_has_queued_probes() && optprobe_can_fire()) { 809 /* Probes remain and can be fired immediately (e.g. retried siblings) */ 810 kick_kprobe_optimizer(); 811 } 812 } 813 814 static int kprobe_optimizer_thread(void *data) 815 { 816 while (!kthread_should_stop()) { 817 /* Wait until there is work to do or a generation is ready */ 818 wait_event_interruptible(kprobe_optimizer_wait, 819 atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE || 820 optprobe_has_ready_gens() || 821 kthread_should_stop()); 822 823 if (kthread_should_stop()) 824 break; 825 826 /* 827 * If it was a normal kick and no generation is ready to finalize, 828 * wait for OPTIMIZE_DELAY to batch incoming requests. 829 * This wait can be interrupted by a flush request or a ready generation. 830 */ 831 if (atomic_read(&optimizer_state) == OPTIMIZER_ST_KICKED && 832 !optprobe_has_ready_gens()) { 833 wait_event_interruptible_timeout( 834 kprobe_optimizer_wait, 835 atomic_read(&optimizer_state) == OPTIMIZER_ST_FLUSHING || 836 optprobe_has_ready_gens() || 837 kthread_should_stop(), 838 OPTIMIZE_DELAY); 839 } 840 841 if (kthread_should_stop()) 842 break; 843 844 atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE); 845 846 kprobe_optimizer(); 847 } 848 return 0; 849 } 850 851 /* Start optimizer after OPTIMIZE_DELAY passed */ 852 static void kick_kprobe_optimizer(void) 853 { 854 lockdep_assert_held(&kprobe_mutex); 855 if (atomic_cmpxchg(&optimizer_state, 856 OPTIMIZER_ST_IDLE, OPTIMIZER_ST_KICKED) == OPTIMIZER_ST_IDLE) 857 wake_up(&kprobe_optimizer_wait); 858 } 859 860 static void wait_for_kprobe_optimizer_locked(void) 861 { 862 lockdep_assert_held(&kprobe_mutex); 863 864 while (optprobe_optimizer_busy()) { 865 unsigned long passes = optimizer_passes; 866 867 /* Wake up optimizer thread if it can make progress */ 868 if ((optprobe_can_fire() && optprobe_has_queued_probes()) || 869 optprobe_has_ready_gens()) { 870 if (atomic_xchg_acquire(&optimizer_state, 871 OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING) 872 wake_up(&kprobe_optimizer_wait); 873 } 874 875 /* 876 * kprobe_optimizer() holds 'kprobe_mutex' for a whole pass, which 877 * this drops while sleeping, so a new count means progress was made. 878 */ 879 wait_var_event_mutex(&optimizer_passes, 880 optimizer_passes != passes, &kprobe_mutex); 881 } 882 } 883 884 /* Wait for completing optimization and unoptimization */ 885 void wait_for_kprobe_optimizer(void) 886 { 887 guard(mutex)(&kprobe_mutex); 888 889 wait_for_kprobe_optimizer_locked(); 890 } 891 892 bool optprobe_queued_unopt(struct optimized_kprobe *op) 893 { 894 struct optimized_kprobe *_op; 895 int i; 896 897 for (i = 0; i < OPTPROBE_GEN_MAX; i++) { 898 list_for_each_entry(_op, &optprobe_gens[i].unoptimizing_list, list) { 899 if (op == _op) 900 return true; 901 } 902 } 903 904 return false; 905 } 906 907 static bool optprobe_queued_freeing(struct optimized_kprobe *op) 908 { 909 struct optimized_kprobe *_op; 910 int i; 911 912 for (i = 0; i < OPTPROBE_GEN_MAX; i++) { 913 list_for_each_entry(_op, &optprobe_gens[i].freeing_list, list) { 914 if (op == _op) 915 return true; 916 } 917 } 918 919 return false; 920 } 921 922 static bool optprobe_queued_cooling(struct optimized_kprobe *op) 923 { 924 struct optimized_kprobe *_op; 925 int i; 926 927 for (i = 0; i < OPTPROBE_GEN_MAX; i++) { 928 list_for_each_entry(_op, &optprobe_gens[i].cooling_list, list) { 929 if (op == _op) 930 return true; 931 } 932 } 933 934 return false; 935 } 936 937 /* Optimize kprobe if p is ready to be optimized */ 938 static void optimize_kprobe(struct kprobe *p) 939 { 940 struct optimized_kprobe *op; 941 942 /* Check if the kprobe is disabled or not ready for optimization. */ 943 if (!kprobe_optready(p) || !kprobes_allow_optimization || 944 (kprobe_disabled(p) || kprobes_all_disarmed)) 945 return; 946 947 /* kprobes with 'post_handler' can not be optimized */ 948 if (p->post_handler) 949 return; 950 951 op = container_of(p, struct optimized_kprobe, kp); 952 953 /* Check there is no other kprobes at the optimized instructions */ 954 if (arch_check_optimized_kprobe(op) < 0) 955 return; 956 957 /* Check if it is already optimized. */ 958 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) { 959 if (optprobe_queued_unopt(op)) { 960 /* This is under unoptimizing. Just dequeue the probe */ 961 list_del_init(&op->list); 962 } 963 return; 964 } 965 966 if (optprobe_queued_cooling(op)) { 967 /* Under in-flight unoptimization. It will be re-optimized upon finalize */ 968 return; 969 } 970 971 if (optprobe_queued_freeing(op)) 972 list_del_init(&op->list); 973 974 op->kp.flags |= KPROBE_FLAG_OPTIMIZED; 975 976 /* 977 * On the 'unoptimizing_list' and 'optimizing_list', 978 * 'op' must have OPTIMIZED flag 979 */ 980 if (WARN_ON_ONCE(!list_empty(&op->list))) 981 return; 982 983 list_add(&op->list, &optprobe_gens[optprobe_cur_gen].optimizing_list); 984 kick_kprobe_optimizer(); 985 } 986 987 /* Short cut to direct unoptimizing */ 988 static void force_unoptimize_kprobe(struct optimized_kprobe *op) 989 { 990 lockdep_assert_cpus_held(); 991 arch_unoptimize_kprobe(op); 992 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 993 } 994 995 /* Unoptimize a kprobe if p is optimized */ 996 static void unoptimize_kprobe(struct kprobe *p, bool force) 997 { 998 struct optprobe_generation *cur; 999 struct optimized_kprobe *op; 1000 1001 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p)) 1002 return; /* This is not an optprobe nor optimized */ 1003 1004 op = container_of(p, struct optimized_kprobe, kp); 1005 cur = &optprobe_gens[optprobe_cur_gen]; 1006 1007 if (!list_empty(&op->list)) { 1008 if (optprobe_queued_unopt(op)) { 1009 /* Queued in unoptimizing queue */ 1010 if (force) { 1011 /* 1012 * Forcibly unoptimize the kprobe here, and queue it 1013 * for cooling down (or freeing if unused). 1014 */ 1015 force_unoptimize_kprobe(op); 1016 /* Disarm probes if marked disabled and not gone */ 1017 if (kprobe_disabled(&op->kp) && !kprobe_gone(&op->kp)) 1018 arch_disarm_kprobe(&op->kp); 1019 if (kprobe_unused(&op->kp)) { 1020 hlist_del_init_rcu(&op->kp.hlist); 1021 list_move(&op->list, &cur->freeing_list); 1022 } else { 1023 list_move(&op->list, &cur->cooling_list); 1024 } 1025 kick_kprobe_optimizer(); 1026 } 1027 } else if (optprobe_queued_cooling(op)) { 1028 if (force && kprobe_unused(&op->kp)) { 1029 /* 1030 * Already unoptimized, move to freeing list for 1031 * release afterwards. 1032 */ 1033 if (!kprobe_gone(&op->kp)) 1034 arch_disarm_kprobe(&op->kp); 1035 hlist_del_init_rcu(&op->kp.hlist); 1036 list_move(&op->list, &cur->freeing_list); 1037 kick_kprobe_optimizer(); 1038 } 1039 } else { 1040 /* Dequeue from the optimizing queue */ 1041 list_del_init(&op->list); 1042 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 1043 } 1044 return; 1045 } 1046 1047 if (!kprobe_optimized(p)) 1048 return; 1049 1050 /* Optimized kprobe case */ 1051 if (force) { 1052 /* Forcibly update the code: this is a special case */ 1053 force_unoptimize_kprobe(op); 1054 list_add(&op->list, &cur->cooling_list); 1055 } else { 1056 list_add(&op->list, &cur->unoptimizing_list); 1057 } 1058 kick_kprobe_optimizer(); 1059 } 1060 1061 /* Cancel unoptimizing for reusing */ 1062 static int reuse_unused_kprobe(struct kprobe *ap) 1063 { 1064 struct optimized_kprobe *op; 1065 1066 /* 1067 * Unused kprobe MUST be on the way of delayed unoptimizing (means 1068 * there is still a relative jump) and disabled. 1069 */ 1070 op = container_of(ap, struct optimized_kprobe, kp); 1071 WARN_ON_ONCE(list_empty(&op->list)); 1072 /* Already disarmed, the caller arms it only for an enabled probe */ 1073 if (optprobe_queued_cooling(op)) 1074 return 0; 1075 1076 /* Enable the probe again */ 1077 ap->flags &= ~KPROBE_FLAG_DISABLED; 1078 /* Optimize it again. (remove from 'op->list') */ 1079 if (!kprobe_optready(ap)) 1080 return -EINVAL; 1081 1082 optimize_kprobe(ap); 1083 return 0; 1084 } 1085 1086 /* Remove optimized instructions */ 1087 static void kill_optimized_kprobe(struct kprobe *p) 1088 { 1089 struct optimized_kprobe *op; 1090 1091 op = container_of(p, struct optimized_kprobe, kp); 1092 if (!list_empty(&op->list)) { 1093 if (kprobe_unused(p)) { 1094 if (optprobe_queued_unopt(op) || optprobe_queued_cooling(op)) { 1095 hlist_del_init_rcu(&op->kp.hlist); 1096 list_move(&op->list, 1097 &optprobe_gens[optprobe_cur_gen].freeing_list); 1098 kick_kprobe_optimizer(); 1099 } else if (!optprobe_queued_freeing(op)) { 1100 list_del_init(&op->list); 1101 } 1102 } else { 1103 list_del_init(&op->list); 1104 } 1105 } 1106 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 1107 1108 /* 1109 * Don't remove the slot if it is queued for freeing or unoptimization; 1110 * the optimizer will reclaim it after the quiescence period. 1111 */ 1112 if (!optprobe_queued_freeing(op) && !optprobe_queued_unopt(op) && 1113 !optprobe_queued_cooling(op)) 1114 arch_remove_optimized_kprobe(op); 1115 } 1116 1117 static inline 1118 void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p) 1119 { 1120 if (!kprobe_ftrace(p)) 1121 arch_prepare_optimized_kprobe(op, p); 1122 } 1123 1124 /* Try to prepare optimized instructions */ 1125 static void prepare_optimized_kprobe(struct kprobe *p) 1126 { 1127 struct optimized_kprobe *op; 1128 1129 op = container_of(p, struct optimized_kprobe, kp); 1130 __prepare_optimized_kprobe(op, p); 1131 } 1132 1133 /* Allocate new optimized_kprobe and try to prepare optimized instructions. */ 1134 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 1135 { 1136 struct optimized_kprobe *op; 1137 1138 op = kzalloc_obj(struct optimized_kprobe); 1139 if (!op) 1140 return NULL; 1141 1142 INIT_LIST_HEAD(&op->list); 1143 op->kp.addr = p->addr; 1144 __prepare_optimized_kprobe(op, p); 1145 1146 return &op->kp; 1147 } 1148 1149 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p); 1150 1151 /* 1152 * Prepare an optimized_kprobe and optimize it. 1153 * NOTE: 'p' must be a normal registered kprobe. 1154 */ 1155 static void try_to_optimize_kprobe(struct kprobe *p) 1156 { 1157 struct kprobe *ap; 1158 struct optimized_kprobe *op; 1159 1160 /* Impossible to optimize ftrace-based kprobe. */ 1161 if (kprobe_ftrace(p)) 1162 return; 1163 1164 /* For preparing optimization, jump_label_text_reserved() is called. */ 1165 guard(cpus_read_lock)(); 1166 guard(jump_label_lock)(); 1167 guard(mutex)(&text_mutex); 1168 1169 ap = alloc_aggr_kprobe(p); 1170 if (!ap) 1171 return; 1172 1173 op = container_of(ap, struct optimized_kprobe, kp); 1174 if (!arch_prepared_optinsn(&op->optinsn)) { 1175 /* If failed to setup optimizing, fallback to kprobe. */ 1176 arch_remove_optimized_kprobe(op); 1177 kfree(op); 1178 return; 1179 } 1180 1181 init_aggr_kprobe(ap, p); 1182 optimize_kprobe(ap); /* This just kicks optimizer thread. */ 1183 } 1184 1185 static void optimize_all_kprobes(void) 1186 { 1187 struct hlist_head *head; 1188 struct kprobe *p; 1189 unsigned int i; 1190 1191 guard(mutex)(&kprobe_mutex); 1192 /* If optimization is already allowed, just return. */ 1193 if (kprobes_allow_optimization) 1194 return; 1195 1196 cpus_read_lock(); 1197 kprobes_allow_optimization = true; 1198 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1199 head = &kprobe_table[i]; 1200 hlist_for_each_entry(p, head, hlist) 1201 if (!kprobe_disabled(p)) 1202 optimize_kprobe(p); 1203 } 1204 cpus_read_unlock(); 1205 pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n"); 1206 } 1207 1208 #ifdef CONFIG_SYSCTL 1209 static void unoptimize_all_kprobes(void) 1210 { 1211 struct hlist_head *head; 1212 struct kprobe *p; 1213 unsigned int i; 1214 1215 guard(mutex)(&kprobe_mutex); 1216 /* If optimization is already prohibited, just return. */ 1217 if (!kprobes_allow_optimization) 1218 return; 1219 1220 cpus_read_lock(); 1221 kprobes_allow_optimization = false; 1222 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1223 head = &kprobe_table[i]; 1224 hlist_for_each_entry(p, head, hlist) { 1225 if (!kprobe_disabled(p)) 1226 unoptimize_kprobe(p, false); 1227 } 1228 } 1229 cpus_read_unlock(); 1230 /* Wait for unoptimizing completion. */ 1231 wait_for_kprobe_optimizer_locked(); 1232 pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n"); 1233 } 1234 1235 static DEFINE_MUTEX(kprobe_sysctl_mutex); 1236 static int sysctl_kprobes_optimization; 1237 static int proc_kprobes_optimization_handler(const struct ctl_table *table, 1238 int write, void *buffer, 1239 size_t *length, loff_t *ppos) 1240 { 1241 int ret; 1242 1243 guard(mutex)(&kprobe_sysctl_mutex); 1244 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0; 1245 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 1246 1247 if (sysctl_kprobes_optimization) 1248 optimize_all_kprobes(); 1249 else 1250 unoptimize_all_kprobes(); 1251 1252 return ret; 1253 } 1254 1255 static const struct ctl_table kprobe_sysctls[] = { 1256 { 1257 .procname = "kprobes-optimization", 1258 .data = &sysctl_kprobes_optimization, 1259 .maxlen = sizeof(int), 1260 .mode = 0644, 1261 .proc_handler = proc_kprobes_optimization_handler, 1262 .extra1 = SYSCTL_ZERO, 1263 .extra2 = SYSCTL_ONE, 1264 }, 1265 }; 1266 1267 static void __init kprobe_sysctls_init(void) 1268 { 1269 register_sysctl_init("debug", kprobe_sysctls); 1270 } 1271 #endif /* CONFIG_SYSCTL */ 1272 1273 /* Return true if the probe is queued on unoptimizing lists */ 1274 static bool kprobe_queued_unopt(struct kprobe *p) 1275 { 1276 struct optimized_kprobe *op; 1277 1278 if (kprobe_aggrprobe(p)) { 1279 op = container_of(p, struct optimized_kprobe, kp); 1280 if (optprobe_queued_unopt(op)) 1281 return true; 1282 } 1283 return false; 1284 } 1285 1286 /* Put a breakpoint for a probe. */ 1287 static void __arm_kprobe(struct kprobe *p) 1288 { 1289 struct kprobe *_p; 1290 1291 lockdep_assert_held(&text_mutex); 1292 1293 /* Find the overlapping optimized kprobes. */ 1294 _p = get_optimized_kprobe(p->addr); 1295 if (unlikely(_p)) 1296 /* Fallback to unoptimized kprobe */ 1297 unoptimize_kprobe(_p, true); 1298 1299 if (!kprobe_queued_unopt(p)) 1300 arch_arm_kprobe(p); 1301 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ 1302 if (unlikely(kprobe_queued_unopt(p))) 1303 unoptimize_kprobe(p, true); 1304 } 1305 1306 /* Remove the breakpoint of a probe. */ 1307 static void __disarm_kprobe(struct kprobe *p, bool reopt) 1308 { 1309 struct kprobe *_p; 1310 1311 lockdep_assert_held(&text_mutex); 1312 1313 /* Try to unoptimize */ 1314 unoptimize_kprobe(p, kprobes_all_disarmed); 1315 1316 if (!kprobe_queued_unopt(p)) { 1317 arch_disarm_kprobe(p); 1318 /* If another kprobe was blocked, re-optimize it. */ 1319 _p = get_optimized_kprobe(p->addr); 1320 if (unlikely(_p) && reopt) 1321 optimize_kprobe(_p); 1322 } 1323 } 1324 1325 static void __init init_optprobe(void) 1326 { 1327 int i; 1328 1329 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT 1330 /* Init 'kprobe_optinsn_slots' for allocation */ 1331 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE; 1332 #endif 1333 1334 for (i = 0; i < OPTPROBE_GEN_MAX; i++) { 1335 INIT_LIST_HEAD(&optprobe_gens[i].optimizing_list); 1336 INIT_LIST_HEAD(&optprobe_gens[i].unoptimizing_list); 1337 INIT_LIST_HEAD(&optprobe_gens[i].cooling_list); 1338 INIT_LIST_HEAD(&optprobe_gens[i].freeing_list); 1339 optprobe_gens[i].in_flight = false; 1340 optprobe_gens[i].ready = false; 1341 } 1342 optprobe_cur_gen = 0; 1343 1344 init_waitqueue_head(&kprobe_optimizer_wait); 1345 atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE); 1346 kprobe_optimizer_task = kthread_run(kprobe_optimizer_thread, NULL, 1347 "kprobe-optimizer"); 1348 } 1349 #else /* !CONFIG_OPTPROBES */ 1350 1351 #define init_optprobe() do {} while (0) 1352 #define optimize_kprobe(p) do {} while (0) 1353 #define unoptimize_kprobe(p, f) do {} while (0) 1354 #define kill_optimized_kprobe(p) do {} while (0) 1355 #define prepare_optimized_kprobe(p) do {} while (0) 1356 #define try_to_optimize_kprobe(p) do {} while (0) 1357 #define __arm_kprobe(p) arch_arm_kprobe(p) 1358 #define __disarm_kprobe(p, o) arch_disarm_kprobe(p) 1359 #define kprobe_disarmed(p) kprobe_disabled(p) 1360 #define wait_for_kprobe_optimizer_locked() \ 1361 lockdep_assert_held(&kprobe_mutex) 1362 1363 static int reuse_unused_kprobe(struct kprobe *ap) 1364 { 1365 /* 1366 * If the optimized kprobe is NOT supported, the aggr kprobe is 1367 * released at the same time that the last aggregated kprobe is 1368 * unregistered. 1369 * Thus there should be no chance to reuse unused kprobe. 1370 */ 1371 WARN_ON_ONCE(1); 1372 return -EINVAL; 1373 } 1374 1375 static void free_aggr_kprobe(struct kprobe *p) 1376 { 1377 arch_remove_kprobe(p); 1378 kfree(p); 1379 } 1380 1381 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 1382 { 1383 return kzalloc_obj(struct kprobe); 1384 } 1385 #endif /* CONFIG_OPTPROBES */ 1386 1387 #ifdef CONFIG_KPROBES_ON_FTRACE 1388 static struct ftrace_ops kprobe_ftrace_ops __read_mostly = { 1389 .func = kprobe_ftrace_handler, 1390 .flags = FTRACE_OPS_FL_SAVE_REGS, 1391 }; 1392 1393 static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = { 1394 .func = kprobe_ftrace_handler, 1395 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY, 1396 }; 1397 1398 static int kprobe_ipmodify_enabled; 1399 static int kprobe_ftrace_enabled; 1400 bool kprobe_ftrace_disabled; 1401 1402 static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, 1403 int *cnt) 1404 { 1405 int ret; 1406 1407 lockdep_assert_held(&kprobe_mutex); 1408 1409 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0); 1410 if (ret < 0) 1411 return ret; 1412 1413 if (*cnt == 0) { 1414 ret = register_ftrace_function(ops); 1415 if (ret < 0) { 1416 /* 1417 * At this point, sinec ops is not registered, we should be sefe from 1418 * registering empty filter. 1419 */ 1420 ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); 1421 return ret; 1422 } 1423 } 1424 1425 (*cnt)++; 1426 return ret; 1427 } 1428 1429 static int arm_kprobe_ftrace(struct kprobe *p) 1430 { 1431 bool ipmodify = (p->post_handler != NULL); 1432 1433 return __arm_kprobe_ftrace(p, 1434 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops, 1435 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled); 1436 } 1437 1438 static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, 1439 int *cnt) 1440 { 1441 int ret; 1442 1443 lockdep_assert_held(&kprobe_mutex); 1444 if (unlikely(kprobe_ftrace_disabled)) { 1445 /* Now ftrace is disabled forever, disarm is already done. */ 1446 return 0; 1447 } 1448 1449 if (*cnt == 1) { 1450 ret = unregister_ftrace_function(ops); 1451 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret)) 1452 return ret; 1453 } 1454 1455 (*cnt)--; 1456 1457 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); 1458 if (WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n", 1459 p->addr, ret)) { 1460 if (*cnt == 0) 1461 register_ftrace_function(ops); 1462 (*cnt)++; 1463 return ret; 1464 } 1465 return 0; 1466 } 1467 1468 static int disarm_kprobe_ftrace(struct kprobe *p) 1469 { 1470 bool ipmodify = (p->post_handler != NULL); 1471 1472 return __disarm_kprobe_ftrace(p, 1473 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops, 1474 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled); 1475 } 1476 1477 void kprobe_ftrace_kill(void) 1478 { 1479 kprobe_ftrace_disabled = true; 1480 } 1481 #else /* !CONFIG_KPROBES_ON_FTRACE */ 1482 static inline int arm_kprobe_ftrace(struct kprobe *p) 1483 { 1484 return -ENODEV; 1485 } 1486 1487 static inline int disarm_kprobe_ftrace(struct kprobe *p) 1488 { 1489 return -ENODEV; 1490 } 1491 #endif 1492 1493 static int prepare_kprobe(struct kprobe *p) 1494 { 1495 /* Must ensure p->addr is really on ftrace */ 1496 if (kprobe_ftrace(p)) 1497 return arch_prepare_kprobe_ftrace(p); 1498 1499 return arch_prepare_kprobe(p); 1500 } 1501 1502 static int arm_kprobe(struct kprobe *kp) 1503 { 1504 if (unlikely(kprobe_ftrace(kp))) 1505 return arm_kprobe_ftrace(kp); 1506 1507 guard(cpus_read_lock)(); 1508 guard(mutex)(&text_mutex); 1509 __arm_kprobe(kp); 1510 return 0; 1511 } 1512 1513 static int disarm_kprobe(struct kprobe *kp, bool reopt) 1514 { 1515 if (unlikely(kprobe_ftrace(kp))) 1516 return disarm_kprobe_ftrace(kp); 1517 1518 guard(cpus_read_lock)(); 1519 guard(mutex)(&text_mutex); 1520 __disarm_kprobe(kp, reopt); 1521 return 0; 1522 } 1523 1524 /* 1525 * Aggregate handlers for multiple kprobes support - these handlers 1526 * take care of invoking the individual kprobe handlers on p->list 1527 */ 1528 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 1529 { 1530 struct kprobe *kp; 1531 1532 list_for_each_entry_rcu(kp, &p->list, list) { 1533 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 1534 set_kprobe_instance(kp); 1535 if (kp->pre_handler(kp, regs)) 1536 return 1; 1537 } 1538 reset_kprobe_instance(); 1539 } 1540 return 0; 1541 } 1542 NOKPROBE_SYMBOL(aggr_pre_handler); 1543 1544 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 1545 unsigned long flags) 1546 { 1547 struct kprobe *kp; 1548 1549 list_for_each_entry_rcu(kp, &p->list, list) { 1550 if (kp->post_handler && likely(!kprobe_disabled(kp))) { 1551 set_kprobe_instance(kp); 1552 kp->post_handler(kp, regs, flags); 1553 reset_kprobe_instance(); 1554 } 1555 } 1556 } 1557 NOKPROBE_SYMBOL(aggr_post_handler); 1558 1559 /* Walks the list and increments 'nmissed' if 'p' has child probes. */ 1560 void kprobes_inc_nmissed_count(struct kprobe *p) 1561 { 1562 struct kprobe *kp; 1563 1564 if (!kprobe_aggrprobe(p)) { 1565 p->nmissed++; 1566 } else { 1567 list_for_each_entry_rcu(kp, &p->list, list) 1568 kp->nmissed++; 1569 } 1570 } 1571 NOKPROBE_SYMBOL(kprobes_inc_nmissed_count); 1572 1573 static struct kprobe kprobe_busy = { 1574 .addr = (void *) get_kprobe, 1575 }; 1576 1577 void kprobe_busy_begin(void) 1578 { 1579 struct kprobe_ctlblk *kcb; 1580 1581 preempt_disable(); 1582 __this_cpu_write(current_kprobe, &kprobe_busy); 1583 kcb = get_kprobe_ctlblk(); 1584 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 1585 } 1586 1587 void kprobe_busy_end(void) 1588 { 1589 __this_cpu_write(current_kprobe, NULL); 1590 preempt_enable(); 1591 } 1592 1593 /* Add the new probe to 'ap->list'. */ 1594 static void add_new_kprobe(struct kprobe *ap, struct kprobe *p) 1595 { 1596 /* A reused cooling probe is already unoptimized */ 1597 if (p->post_handler && !kprobe_unused(ap)) 1598 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */ 1599 1600 list_add_rcu(&p->list, &ap->list); 1601 } 1602 1603 /* 1604 * Fill in the required fields of the aggregator kprobe. Replace the 1605 * earlier kprobe in the hlist with the aggregator kprobe. 1606 */ 1607 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 1608 { 1609 /* Copy the insn slot of 'p' to 'ap'. */ 1610 copy_kprobe(p, ap); 1611 flush_insn_slot(ap); 1612 ap->addr = p->addr; 1613 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED; 1614 ap->pre_handler = aggr_pre_handler; 1615 /* We don't care the kprobe which has gone. */ 1616 if (p->post_handler && !kprobe_gone(p)) 1617 ap->post_handler = aggr_post_handler; 1618 1619 INIT_LIST_HEAD(&ap->list); 1620 INIT_HLIST_NODE(&ap->hlist); 1621 1622 list_add_rcu(&p->list, &ap->list); 1623 hlist_replace_rcu(&p->hlist, &ap->hlist); 1624 } 1625 1626 /* 1627 * This registers the second or subsequent kprobe at the same address. 1628 */ 1629 static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p) 1630 { 1631 int ret = 0; 1632 struct kprobe *ap = orig_p; 1633 bool reset_post_handler = false; 1634 bool ftrace_switch_ops = false; 1635 1636 scoped_guard(cpus_read_lock) { 1637 /* For preparing optimization, jump_label_text_reserved() is called */ 1638 guard(jump_label_lock)(); 1639 guard(mutex)(&text_mutex); 1640 1641 if (!kprobe_aggrprobe(orig_p)) { 1642 /* If 'orig_p' is not an 'aggr_kprobe', create new one. */ 1643 ap = alloc_aggr_kprobe(orig_p); 1644 if (!ap) 1645 return -ENOMEM; 1646 init_aggr_kprobe(ap, orig_p); 1647 } else if (kprobe_unused(ap)) { 1648 /* This probe is going to die. Rescue it */ 1649 ret = reuse_unused_kprobe(ap); 1650 if (ret) 1651 return ret; 1652 } 1653 1654 if (kprobe_gone(ap)) { 1655 /* 1656 * Attempting to insert new probe at the same location that 1657 * had a probe in the module vaddr area which already 1658 * freed. So, the instruction slot has already been 1659 * released. We need a new slot for the new probe. 1660 */ 1661 ret = arch_prepare_kprobe(ap); 1662 if (ret) 1663 /* 1664 * Even if fail to allocate new slot, don't need to 1665 * free the 'ap'. It will be used next time, or 1666 * freed by unregister_kprobe(). 1667 */ 1668 return ret; 1669 1670 /* Prepare optimized instructions if possible. */ 1671 prepare_optimized_kprobe(ap); 1672 1673 /* 1674 * Clear gone flag to prevent allocating new slot again, and 1675 * set disabled flag because it is not armed yet. 1676 */ 1677 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE) 1678 | KPROBE_FLAG_DISABLED; 1679 } 1680 1681 if (p->post_handler && !ap->post_handler) { 1682 reset_post_handler = true; 1683 if (kprobe_ftrace(ap) && !kprobes_all_disarmed && 1684 !kprobe_disabled(ap)) 1685 ftrace_switch_ops = true; 1686 else 1687 ap->post_handler = aggr_post_handler; 1688 } 1689 1690 /* Copy the insn slot of 'p' to 'ap'. */ 1691 copy_kprobe(ap, p); 1692 if (!ftrace_switch_ops) 1693 add_new_kprobe(ap, p); 1694 } 1695 1696 if (ftrace_switch_ops) { 1697 /* 1698 * Must switch ftrace ops outside of text_mutex and cpus_read_lock 1699 * to prevent deadlocks when ftrace code modification acquires text_mutex. 1700 */ 1701 ret = disarm_kprobe_ftrace(ap); 1702 if (ret) 1703 return ret; 1704 1705 ap->post_handler = aggr_post_handler; 1706 ret = arm_kprobe_ftrace(ap); 1707 if (ret) { 1708 ap->post_handler = NULL; 1709 if (arm_kprobe_ftrace(ap)) 1710 ap->flags |= KPROBE_FLAG_DISABLED; 1711 return ret; 1712 } 1713 add_new_kprobe(ap, p); 1714 } 1715 1716 if (kprobe_disabled(ap) && !kprobe_disabled(p)) { 1717 ap->flags &= ~KPROBE_FLAG_DISABLED; 1718 if (!kprobes_all_disarmed) { 1719 /* Arm the breakpoint again. */ 1720 ret = arm_kprobe(ap); 1721 if (ret) { 1722 ap->flags |= KPROBE_FLAG_DISABLED; 1723 if (reset_post_handler) 1724 ap->post_handler = NULL; 1725 list_del_rcu(&p->list); 1726 synchronize_rcu(); 1727 } 1728 } 1729 } 1730 return ret; 1731 } 1732 1733 bool __weak arch_within_kprobe_blacklist(unsigned long addr) 1734 { 1735 /* The '__kprobes' functions and entry code must not be probed. */ 1736 return addr >= (unsigned long)__kprobes_text_start && 1737 addr < (unsigned long)__kprobes_text_end; 1738 } 1739 1740 static bool __within_kprobe_blacklist(unsigned long addr) 1741 { 1742 struct kprobe_blacklist_entry *ent; 1743 1744 if (arch_within_kprobe_blacklist(addr)) 1745 return true; 1746 /* 1747 * If 'kprobe_blacklist' is defined, check the address and 1748 * reject any probe registration in the prohibited area. 1749 * Note: this can return true during transition period where 1750 * (start_addr, end_addr) in the black list is shrinking 1751 * but old entry has not been removed yet. This is acceptable 1752 * because the worst case is that we reject more probes than 1753 * we should. 1754 */ 1755 guard(rcu)(); 1756 list_for_each_entry_rcu(ent, &kprobe_blacklist, list) { 1757 if (addr >= ent->start_addr && addr < ent->end_addr) 1758 return true; 1759 } 1760 return false; 1761 } 1762 1763 bool within_kprobe_blacklist(unsigned long addr) 1764 { 1765 char symname[KSYM_NAME_LEN], *p; 1766 1767 if (__within_kprobe_blacklist(addr)) 1768 return true; 1769 1770 /* Check if the address is on a suffixed-symbol */ 1771 if (!lookup_symbol_name(addr, symname)) { 1772 p = strchr(symname, '.'); 1773 if (!p) 1774 return false; 1775 *p = '\0'; 1776 addr = (unsigned long)kprobe_lookup_name(symname, 0); 1777 if (addr) 1778 return __within_kprobe_blacklist(addr); 1779 } 1780 return false; 1781 } 1782 1783 /* 1784 * arch_adjust_kprobe_addr - adjust the address 1785 * @addr: symbol base address 1786 * @offset: offset within the symbol 1787 * @on_func_entry: was this @addr+@offset on the function entry 1788 * 1789 * Typically returns @addr + @offset, except for special cases where the 1790 * function might be prefixed by a CFI landing pad, in that case any offset 1791 * inside the landing pad is mapped to the first 'real' instruction of the 1792 * symbol. 1793 * 1794 * Specifically, for things like IBT/BTI, skip the resp. ENDBR/BTI.C 1795 * instruction at +0. 1796 */ 1797 kprobe_opcode_t *__weak arch_adjust_kprobe_addr(unsigned long addr, 1798 unsigned long offset, 1799 bool *on_func_entry) 1800 { 1801 *on_func_entry = !offset; 1802 return (kprobe_opcode_t *)(addr + offset); 1803 } 1804 1805 /* 1806 * If 'symbol_name' is specified, look it up and add the 'offset' 1807 * to it. This way, we can specify a relative address to a symbol. 1808 * This returns encoded errors if it fails to look up symbol or invalid 1809 * combination of parameters. 1810 */ 1811 static kprobe_opcode_t * 1812 _kprobe_addr(kprobe_opcode_t *addr, const char *symbol_name, 1813 unsigned long offset, bool *on_func_entry) 1814 { 1815 if ((symbol_name && addr) || (!symbol_name && !addr)) 1816 return ERR_PTR(-EINVAL); 1817 1818 if (symbol_name) { 1819 /* 1820 * Input: @sym + @offset 1821 * Output: @addr + @offset 1822 * 1823 * NOTE: kprobe_lookup_name() does *NOT* fold the offset 1824 * argument into it's output! 1825 */ 1826 addr = kprobe_lookup_name(symbol_name, offset); 1827 if (!addr) 1828 return ERR_PTR(-ENOENT); 1829 } 1830 1831 /* 1832 * So here we have @addr + @offset, displace it into a new 1833 * @addr' + @offset' where @addr' is the symbol start address. 1834 */ 1835 addr = (void *)addr + offset; 1836 if (!kallsyms_lookup_size_offset((unsigned long)addr, NULL, &offset)) 1837 return ERR_PTR(-ENOENT); 1838 addr = (void *)addr - offset; 1839 1840 /* 1841 * Then ask the architecture to re-combine them, taking care of 1842 * magical function entry details while telling us if this was indeed 1843 * at the start of the function. 1844 */ 1845 addr = arch_adjust_kprobe_addr((unsigned long)addr, offset, on_func_entry); 1846 if (!addr) 1847 return ERR_PTR(-EINVAL); 1848 1849 return addr; 1850 } 1851 1852 /* 1853 * Check the 'p' is valid and return the aggregator kprobe 1854 * at the same address. 1855 */ 1856 static struct kprobe *__get_valid_kprobe_at(struct kprobe *p, kprobe_opcode_t *addr) 1857 { 1858 struct hlist_head *head; 1859 struct kprobe *ap, *list_p; 1860 1861 lockdep_assert_held(&kprobe_mutex); 1862 1863 if (unlikely(!addr)) 1864 return NULL; 1865 1866 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 1867 hlist_for_each_entry_rcu(ap, head, hlist, 1868 lockdep_is_held(&kprobe_mutex)) { 1869 if (ap == p) { 1870 p->addr = addr; 1871 return ap; 1872 } 1873 if (ap->addr == addr && kprobe_aggrprobe(ap)) { 1874 list_for_each_entry(list_p, &ap->list, list) { 1875 if (list_p == p) { 1876 p->addr = addr; 1877 return ap; 1878 } 1879 } 1880 } 1881 } 1882 1883 return NULL; 1884 } 1885 1886 static inline struct kprobe *__get_valid_kprobe(struct kprobe *p) 1887 { 1888 return __get_valid_kprobe_at(p, p->addr); 1889 } 1890 1891 /* Return error if the kprobe is being re-registered */ 1892 static inline int check_kprobe_rereg_at(struct kprobe *p, kprobe_opcode_t *addr) 1893 { 1894 guard(mutex)(&kprobe_mutex); 1895 1896 if (__get_valid_kprobe_at(p, addr)) 1897 return -EINVAL; 1898 1899 return 0; 1900 } 1901 1902 static inline int check_kprobe_rereg(struct kprobe *p) 1903 { 1904 return check_kprobe_rereg_at(p, p->addr); 1905 } 1906 1907 static int check_ftrace_location(struct kprobe *p) 1908 { 1909 unsigned long addr = (unsigned long)p->addr; 1910 1911 if (ftrace_location(addr) == addr) { 1912 #ifdef CONFIG_KPROBES_ON_FTRACE 1913 p->flags |= KPROBE_FLAG_FTRACE; 1914 #else 1915 return -EINVAL; 1916 #endif 1917 } 1918 return 0; 1919 } 1920 1921 static bool is_cfi_preamble_symbol(unsigned long addr) 1922 { 1923 char symbuf[KSYM_NAME_LEN]; 1924 1925 if (lookup_symbol_name(addr, symbuf)) 1926 return false; 1927 1928 return str_has_prefix(symbuf, "__cfi_") || 1929 str_has_prefix(symbuf, "__pfx_"); 1930 } 1931 1932 static int check_kprobe_address_safe(struct kprobe *p, 1933 struct module **probed_mod) 1934 { 1935 int ret; 1936 1937 ret = check_ftrace_location(p); 1938 if (ret) 1939 return ret; 1940 1941 guard(jump_label_lock)(); 1942 1943 /* Ensure the address is in a text area, and find a module if exists. */ 1944 *probed_mod = NULL; 1945 if (!core_kernel_text((unsigned long) p->addr)) { 1946 guard(rcu)(); 1947 *probed_mod = __module_text_address((unsigned long) p->addr); 1948 if (!(*probed_mod)) 1949 return -EINVAL; 1950 1951 /* 1952 * We must hold a refcount of the probed module while updating 1953 * its code to prohibit unexpected unloading. 1954 */ 1955 if (unlikely(!try_module_get(*probed_mod))) 1956 return -ENOENT; 1957 } 1958 /* Ensure it is not in reserved area. */ 1959 if (in_gate_area_no_mm((unsigned long) p->addr) || 1960 within_kprobe_blacklist((unsigned long) p->addr) || 1961 jump_label_text_reserved(p->addr, p->addr) || 1962 static_call_text_reserved(p->addr, p->addr) || 1963 find_bug((unsigned long)p->addr) || 1964 is_cfi_preamble_symbol((unsigned long)p->addr)) { 1965 module_put(*probed_mod); 1966 return -EINVAL; 1967 } 1968 1969 /* Get module refcount and reject __init functions for loaded modules. */ 1970 if (IS_ENABLED(CONFIG_MODULES) && *probed_mod) { 1971 /* 1972 * If the module freed '.init.text', we couldn't insert 1973 * kprobes in there. 1974 */ 1975 if (within_module_init((unsigned long)p->addr, *probed_mod) && 1976 !module_is_coming(*probed_mod)) { 1977 module_put(*probed_mod); 1978 return -ENOENT; 1979 } 1980 } 1981 1982 return 0; 1983 } 1984 1985 static int __register_kprobe(struct kprobe *p) 1986 { 1987 int ret; 1988 struct kprobe *old_p; 1989 1990 guard(mutex)(&kprobe_mutex); 1991 1992 old_p = get_kprobe(p->addr); 1993 if (old_p) 1994 /* Since this may unoptimize 'old_p', locking 'text_mutex'. */ 1995 return register_aggr_kprobe(old_p, p); 1996 1997 scoped_guard(cpus_read_lock) { 1998 /* Prevent text modification */ 1999 guard(mutex)(&text_mutex); 2000 ret = prepare_kprobe(p); 2001 if (ret) 2002 return ret; 2003 } 2004 2005 INIT_HLIST_NODE(&p->hlist); 2006 hlist_add_head_rcu(&p->hlist, 2007 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 2008 2009 if (!kprobes_all_disarmed && !kprobe_disabled(p)) { 2010 ret = arm_kprobe(p); 2011 if (ret) { 2012 hlist_del_rcu(&p->hlist); 2013 synchronize_rcu(); 2014 arch_remove_kprobe(p); 2015 return ret; 2016 } 2017 } 2018 2019 /* Try to optimize kprobe */ 2020 try_to_optimize_kprobe(p); 2021 return 0; 2022 } 2023 2024 int register_kprobe(struct kprobe *p) 2025 { 2026 int ret; 2027 struct module *probed_mod; 2028 kprobe_opcode_t *addr; 2029 bool on_func_entry; 2030 2031 /* Canonicalize probe address from symbol */ 2032 addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry); 2033 if (IS_ERR(addr)) 2034 return PTR_ERR(addr); 2035 p->addr = addr; 2036 2037 ret = check_kprobe_rereg(p); 2038 if (ret) 2039 return ret; 2040 2041 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */ 2042 p->flags &= KPROBE_FLAG_DISABLED; 2043 if (on_func_entry) 2044 p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY; 2045 p->nmissed = 0; 2046 INIT_LIST_HEAD(&p->list); 2047 2048 ret = check_kprobe_address_safe(p, &probed_mod); 2049 if (ret) 2050 return ret; 2051 2052 ret = __register_kprobe(p); 2053 2054 if (probed_mod) 2055 module_put(probed_mod); 2056 2057 return ret; 2058 } 2059 EXPORT_SYMBOL_GPL(register_kprobe); 2060 2061 /* Check if all probes on the 'ap' are disabled. */ 2062 static bool aggr_kprobe_disabled(struct kprobe *ap) 2063 { 2064 struct kprobe *kp; 2065 2066 lockdep_assert_held(&kprobe_mutex); 2067 2068 list_for_each_entry(kp, &ap->list, list) 2069 if (!kprobe_disabled(kp)) 2070 /* 2071 * Since there is an active probe on the list, 2072 * we can't disable this 'ap'. 2073 */ 2074 return false; 2075 2076 return true; 2077 } 2078 2079 static struct kprobe *__disable_kprobe(struct kprobe *p) 2080 { 2081 struct kprobe *orig_p; 2082 int ret; 2083 2084 lockdep_assert_held(&kprobe_mutex); 2085 2086 /* Get an original kprobe for return */ 2087 orig_p = __get_valid_kprobe(p); 2088 if (unlikely(orig_p == NULL)) 2089 return ERR_PTR(-EINVAL); 2090 2091 if (kprobe_disabled(p)) 2092 return orig_p; 2093 2094 /* Disable probe if it is a child probe */ 2095 if (p != orig_p) 2096 p->flags |= KPROBE_FLAG_DISABLED; 2097 2098 /* Try to disarm and disable this/parent probe */ 2099 if (p == orig_p || aggr_kprobe_disabled(orig_p)) { 2100 /* 2101 * Don't be lazy here. Even if 'kprobes_all_disarmed' 2102 * is false, 'orig_p' might not have been armed yet. 2103 * Note arm_all_kprobes() __tries__ to arm all kprobes 2104 * on the best effort basis. 2105 */ 2106 if (!kprobes_all_disarmed && !kprobe_disabled(orig_p)) { 2107 ret = disarm_kprobe(orig_p, true); 2108 if (ret) { 2109 p->flags &= ~KPROBE_FLAG_DISABLED; 2110 return ERR_PTR(ret); 2111 } 2112 } 2113 orig_p->flags |= KPROBE_FLAG_DISABLED; 2114 } 2115 2116 return orig_p; 2117 } 2118 2119 /* 2120 * Unregister a kprobe without a scheduler synchronization. 2121 */ 2122 static int __unregister_kprobe_top(struct kprobe *p) 2123 { 2124 struct kprobe *ap, *list_p; 2125 2126 /* Disable kprobe. This will disarm it if needed. */ 2127 ap = __disable_kprobe(p); 2128 if (IS_ERR(ap)) 2129 return PTR_ERR(ap); 2130 2131 WARN_ON(ap != p && !kprobe_aggrprobe(ap)); 2132 2133 /* 2134 * If the probe is an independent(and non-optimized) kprobe 2135 * (not an aggrprobe), the last kprobe on the aggrprobe, or 2136 * kprobe is already disarmed, just remove from the hash list. 2137 */ 2138 if (ap == p || 2139 (list_is_singular(&ap->list) && kprobe_disarmed(ap))) { 2140 /* 2141 * !disarmed could be happen if the probe is under delayed 2142 * unoptimizing. 2143 */ 2144 hlist_del_rcu(&ap->hlist); 2145 return 0; 2146 } 2147 2148 /* If disabling probe has special handlers, update aggrprobe */ 2149 if (p->post_handler && !kprobe_gone(p)) { 2150 list_for_each_entry(list_p, &ap->list, list) { 2151 if ((list_p != p) && (list_p->post_handler)) 2152 break; 2153 } 2154 /* No other probe has post_handler */ 2155 if (list_entry_is_head(list_p, &ap->list, list)) { 2156 /* 2157 * For the kprobe-on-ftrace case, we keep the 2158 * post_handler setting to identify this aggrprobe 2159 * armed with kprobe_ipmodify_ops. 2160 */ 2161 if (!kprobe_ftrace(ap)) 2162 ap->post_handler = NULL; 2163 } 2164 } 2165 2166 /* 2167 * Remove from the aggrprobe: this path will do nothing in 2168 * __unregister_kprobe_bottom(). 2169 */ 2170 list_del_rcu(&p->list); 2171 if (!kprobe_disabled(ap) && !kprobes_all_disarmed) 2172 /* 2173 * Try to optimize this probe again, because post 2174 * handler may have been changed. 2175 */ 2176 optimize_kprobe(ap); 2177 return 0; 2178 2179 } 2180 2181 static void __unregister_kprobe_bottom(struct kprobe *p) 2182 { 2183 struct kprobe *ap; 2184 2185 if (list_empty(&p->list)) 2186 /* This is an independent kprobe */ 2187 arch_remove_kprobe(p); 2188 else if (list_is_singular(&p->list)) { 2189 /* This is the last child of an aggrprobe */ 2190 ap = list_entry(p->list.next, struct kprobe, list); 2191 list_del(&p->list); 2192 free_aggr_kprobe(ap); 2193 } 2194 /* Otherwise, do nothing. */ 2195 } 2196 2197 int register_kprobes(struct kprobe **kps, int num) 2198 { 2199 int i, ret = 0; 2200 2201 if (num <= 0) 2202 return -EINVAL; 2203 for (i = 0; i < num; i++) { 2204 ret = register_kprobe(kps[i]); 2205 if (ret < 0) { 2206 if (i > 0) 2207 unregister_kprobes(kps, i); 2208 break; 2209 } 2210 } 2211 return ret; 2212 } 2213 EXPORT_SYMBOL_GPL(register_kprobes); 2214 2215 void unregister_kprobe(struct kprobe *p) 2216 { 2217 unregister_kprobes(&p, 1); 2218 } 2219 EXPORT_SYMBOL_GPL(unregister_kprobe); 2220 2221 void unregister_kprobes(struct kprobe **kps, int num) 2222 { 2223 int i; 2224 2225 if (num <= 0) 2226 return; 2227 scoped_guard(mutex, &kprobe_mutex) { 2228 for (i = 0; i < num; i++) 2229 if (__unregister_kprobe_top(kps[i]) < 0) 2230 kps[i]->addr = NULL; 2231 } 2232 synchronize_rcu(); 2233 for (i = 0; i < num; i++) 2234 if (kps[i]->addr) 2235 __unregister_kprobe_bottom(kps[i]); 2236 } 2237 EXPORT_SYMBOL_GPL(unregister_kprobes); 2238 2239 int __weak kprobe_exceptions_notify(struct notifier_block *self, 2240 unsigned long val, void *data) 2241 { 2242 return NOTIFY_DONE; 2243 } 2244 NOKPROBE_SYMBOL(kprobe_exceptions_notify); 2245 2246 static struct notifier_block kprobe_exceptions_nb = { 2247 .notifier_call = kprobe_exceptions_notify, 2248 .priority = 0x7fffffff /* we need to be notified first */ 2249 }; 2250 2251 #ifdef CONFIG_KRETPROBES 2252 2253 #if !defined(CONFIG_KRETPROBE_ON_RETHOOK) 2254 2255 /* callbacks for objpool of kretprobe instances */ 2256 static int kretprobe_init_inst(void *nod, void *context) 2257 { 2258 struct kretprobe_instance *ri = nod; 2259 2260 ri->rph = context; 2261 return 0; 2262 } 2263 static int kretprobe_fini_pool(struct objpool_head *head, void *context) 2264 { 2265 kfree(context); 2266 return 0; 2267 } 2268 2269 static void free_rp_inst_rcu(struct rcu_head *head) 2270 { 2271 struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu); 2272 struct kretprobe_holder *rph = ri->rph; 2273 2274 objpool_drop(ri, &rph->pool); 2275 } 2276 NOKPROBE_SYMBOL(free_rp_inst_rcu); 2277 2278 static void recycle_rp_inst(struct kretprobe_instance *ri) 2279 { 2280 struct kretprobe *rp = get_kretprobe(ri); 2281 2282 if (likely(rp)) 2283 objpool_push(ri, &rp->rph->pool); 2284 else 2285 call_rcu(&ri->rcu, free_rp_inst_rcu); 2286 } 2287 NOKPROBE_SYMBOL(recycle_rp_inst); 2288 2289 /* 2290 * This function is called from delayed_put_task_struct() when a task is 2291 * dead and cleaned up to recycle any kretprobe instances associated with 2292 * this task. These left over instances represent probed functions that 2293 * have been called but will never return. 2294 */ 2295 void kprobe_flush_task(struct task_struct *tk) 2296 { 2297 struct kretprobe_instance *ri; 2298 struct llist_node *node; 2299 2300 /* Early boot, not yet initialized. */ 2301 if (unlikely(!kprobes_initialized)) 2302 return; 2303 2304 kprobe_busy_begin(); 2305 2306 node = __llist_del_all(&tk->kretprobe_instances); 2307 while (node) { 2308 ri = container_of(node, struct kretprobe_instance, llist); 2309 node = node->next; 2310 2311 recycle_rp_inst(ri); 2312 } 2313 2314 kprobe_busy_end(); 2315 } 2316 NOKPROBE_SYMBOL(kprobe_flush_task); 2317 2318 static inline void free_rp_inst(struct kretprobe *rp) 2319 { 2320 struct kretprobe_holder *rph = rp->rph; 2321 2322 if (!rph) 2323 return; 2324 rp->rph = NULL; 2325 objpool_fini(&rph->pool); 2326 } 2327 2328 /* This assumes the 'tsk' is the current task or the is not running. */ 2329 static kprobe_opcode_t *__kretprobe_find_ret_addr(struct task_struct *tsk, 2330 struct llist_node **cur) 2331 { 2332 struct kretprobe_instance *ri = NULL; 2333 struct llist_node *node = *cur; 2334 2335 if (!node) 2336 node = tsk->kretprobe_instances.first; 2337 else 2338 node = node->next; 2339 2340 while (node) { 2341 ri = container_of(node, struct kretprobe_instance, llist); 2342 if (ri->ret_addr != kretprobe_trampoline_addr()) { 2343 *cur = node; 2344 return ri->ret_addr; 2345 } 2346 node = node->next; 2347 } 2348 return NULL; 2349 } 2350 NOKPROBE_SYMBOL(__kretprobe_find_ret_addr); 2351 2352 /** 2353 * kretprobe_find_ret_addr -- Find correct return address modified by kretprobe 2354 * @tsk: Target task 2355 * @fp: A frame pointer 2356 * @cur: a storage of the loop cursor llist_node pointer for next call 2357 * 2358 * Find the correct return address modified by a kretprobe on @tsk in unsigned 2359 * long type. If it finds the return address, this returns that address value, 2360 * or this returns 0. 2361 * The @tsk must be 'current' or a task which is not running. @fp is a hint 2362 * to get the currect return address - which is compared with the 2363 * kretprobe_instance::fp field. The @cur is a loop cursor for searching the 2364 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the 2365 * first call, but '@cur' itself must NOT NULL. 2366 */ 2367 unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp, 2368 struct llist_node **cur) 2369 { 2370 struct kretprobe_instance *ri; 2371 kprobe_opcode_t *ret; 2372 2373 if (WARN_ON_ONCE(!cur)) 2374 return 0; 2375 2376 do { 2377 ret = __kretprobe_find_ret_addr(tsk, cur); 2378 if (!ret) 2379 break; 2380 ri = container_of(*cur, struct kretprobe_instance, llist); 2381 } while (ri->fp != fp); 2382 2383 return (unsigned long)ret; 2384 } 2385 NOKPROBE_SYMBOL(kretprobe_find_ret_addr); 2386 2387 void __weak arch_kretprobe_fixup_return(struct pt_regs *regs, 2388 kprobe_opcode_t *correct_ret_addr) 2389 { 2390 /* 2391 * Do nothing by default. Please fill this to update the fake return 2392 * address on the stack with the correct one on each arch if possible. 2393 */ 2394 } 2395 2396 unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs, 2397 void *frame_pointer) 2398 { 2399 struct kretprobe_instance *ri = NULL; 2400 struct llist_node *first, *node = NULL; 2401 kprobe_opcode_t *correct_ret_addr; 2402 struct kretprobe *rp; 2403 2404 /* Find correct address and all nodes for this frame. */ 2405 correct_ret_addr = __kretprobe_find_ret_addr(current, &node); 2406 if (!correct_ret_addr) { 2407 pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n"); 2408 BUG_ON(1); 2409 } 2410 2411 /* 2412 * Set the return address as the instruction pointer, because if the 2413 * user handler calls stack_trace_save_regs() with this 'regs', 2414 * the stack trace will start from the instruction pointer. 2415 */ 2416 instruction_pointer_set(regs, (unsigned long)correct_ret_addr); 2417 2418 /* Run the user handler of the nodes. */ 2419 first = current->kretprobe_instances.first; 2420 while (first) { 2421 ri = container_of(first, struct kretprobe_instance, llist); 2422 2423 if (WARN_ON_ONCE(ri->fp != frame_pointer)) 2424 break; 2425 2426 rp = get_kretprobe(ri); 2427 if (rp && rp->handler) { 2428 struct kprobe *prev = kprobe_running(); 2429 2430 __this_cpu_write(current_kprobe, &rp->kp); 2431 ri->ret_addr = correct_ret_addr; 2432 rp->handler(ri, regs); 2433 __this_cpu_write(current_kprobe, prev); 2434 } 2435 if (first == node) 2436 break; 2437 2438 first = first->next; 2439 } 2440 2441 arch_kretprobe_fixup_return(regs, correct_ret_addr); 2442 2443 /* Unlink all nodes for this frame. */ 2444 first = current->kretprobe_instances.first; 2445 current->kretprobe_instances.first = node->next; 2446 node->next = NULL; 2447 2448 /* Recycle free instances. */ 2449 while (first) { 2450 ri = container_of(first, struct kretprobe_instance, llist); 2451 first = first->next; 2452 2453 recycle_rp_inst(ri); 2454 } 2455 2456 return (unsigned long)correct_ret_addr; 2457 } 2458 NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) 2459 2460 /* 2461 * This kprobe pre_handler is registered with every kretprobe. When probe 2462 * hits it will set up the return probe. 2463 */ 2464 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) 2465 { 2466 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 2467 struct kretprobe_holder *rph = rp->rph; 2468 struct kretprobe_instance *ri; 2469 2470 ri = objpool_pop(&rph->pool); 2471 if (!ri) { 2472 rp->nmissed++; 2473 return 0; 2474 } 2475 2476 if (rp->entry_handler && rp->entry_handler(ri, regs)) { 2477 objpool_push(ri, &rph->pool); 2478 return 0; 2479 } 2480 2481 arch_prepare_kretprobe(ri, regs); 2482 2483 __llist_add(&ri->llist, ¤t->kretprobe_instances); 2484 2485 return 0; 2486 } 2487 NOKPROBE_SYMBOL(pre_handler_kretprobe); 2488 #else /* CONFIG_KRETPROBE_ON_RETHOOK */ 2489 /* 2490 * This kprobe pre_handler is registered with every kretprobe. When probe 2491 * hits it will set up the return probe. 2492 */ 2493 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) 2494 { 2495 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 2496 struct kretprobe_instance *ri; 2497 struct rethook_node *rhn; 2498 2499 rhn = rethook_try_get(rp->rh); 2500 if (!rhn) { 2501 rp->nmissed++; 2502 return 0; 2503 } 2504 2505 ri = container_of(rhn, struct kretprobe_instance, node); 2506 2507 if (rp->entry_handler && rp->entry_handler(ri, regs)) 2508 rethook_recycle(rhn); 2509 else 2510 rethook_hook(rhn, regs, kprobe_ftrace(p)); 2511 2512 return 0; 2513 } 2514 NOKPROBE_SYMBOL(pre_handler_kretprobe); 2515 2516 static void kretprobe_rethook_handler(struct rethook_node *rh, void *data, 2517 unsigned long ret_addr, 2518 struct pt_regs *regs) 2519 { 2520 struct kretprobe *rp = (struct kretprobe *)data; 2521 struct kretprobe_instance *ri; 2522 struct kprobe_ctlblk *kcb; 2523 2524 /* The data must NOT be null. This means rethook data structure is broken. */ 2525 if (WARN_ON_ONCE(!data) || !rp->handler) 2526 return; 2527 2528 __this_cpu_write(current_kprobe, &rp->kp); 2529 kcb = get_kprobe_ctlblk(); 2530 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 2531 2532 ri = container_of(rh, struct kretprobe_instance, node); 2533 rp->handler(ri, regs); 2534 2535 __this_cpu_write(current_kprobe, NULL); 2536 } 2537 NOKPROBE_SYMBOL(kretprobe_rethook_handler); 2538 2539 #endif /* !CONFIG_KRETPROBE_ON_RETHOOK */ 2540 2541 /** 2542 * kprobe_on_func_entry() -- check whether given address is function entry 2543 * @addr: Target address 2544 * @sym: Target symbol name 2545 * @offset: The offset from the symbol or the address 2546 * 2547 * This checks whether the given @addr+@offset or @sym+@offset is on the 2548 * function entry address or not. 2549 * This returns 0 if it is the function entry, or -EINVAL if it is not. 2550 * And also it returns -ENOENT if it fails the symbol or address lookup. 2551 * Caller must pass @addr or @sym (either one must be NULL), or this 2552 * returns -EINVAL. 2553 */ 2554 int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset) 2555 { 2556 bool on_func_entry; 2557 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset, &on_func_entry); 2558 2559 if (IS_ERR(kp_addr)) 2560 return PTR_ERR(kp_addr); 2561 2562 if (!on_func_entry) 2563 return -EINVAL; 2564 2565 return 0; 2566 } 2567 2568 int register_kretprobe(struct kretprobe *rp) 2569 { 2570 int ret; 2571 int i; 2572 void *addr; 2573 bool on_func_entry; 2574 2575 addr = _kprobe_addr(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset, &on_func_entry); 2576 if (IS_ERR(addr)) 2577 return PTR_ERR(addr); 2578 2579 if (!on_func_entry) 2580 return -EINVAL; 2581 2582 if (check_kprobe_rereg_at(&rp->kp, addr)) 2583 return -EINVAL; 2584 2585 if (kretprobe_blacklist_size) { 2586 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 2587 if (kretprobe_blacklist[i].addr == addr) 2588 return -EINVAL; 2589 } 2590 } 2591 2592 if (rp->data_size > KRETPROBE_MAX_DATA_SIZE) 2593 return -E2BIG; 2594 2595 rp->kp.pre_handler = pre_handler_kretprobe; 2596 rp->kp.post_handler = NULL; 2597 2598 /* Pre-allocate memory for max kretprobe instances */ 2599 if (rp->maxactive <= 0) 2600 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus()); 2601 2602 #ifdef CONFIG_KRETPROBE_ON_RETHOOK 2603 rp->rh = rethook_alloc((void *)rp, kretprobe_rethook_handler, 2604 sizeof(struct kretprobe_instance) + 2605 rp->data_size, rp->maxactive); 2606 if (IS_ERR(rp->rh)) 2607 return PTR_ERR(rp->rh); 2608 2609 rp->nmissed = 0; 2610 /* Establish function entry probe point */ 2611 ret = register_kprobe(&rp->kp); 2612 if (ret != 0) { 2613 rethook_free(rp->rh); 2614 rp->rh = NULL; 2615 } 2616 #else /* !CONFIG_KRETPROBE_ON_RETHOOK */ 2617 rp->rph = kzalloc_obj(struct kretprobe_holder); 2618 if (!rp->rph) 2619 return -ENOMEM; 2620 2621 if (objpool_init(&rp->rph->pool, rp->maxactive, rp->data_size + 2622 sizeof(struct kretprobe_instance), GFP_KERNEL, 2623 rp->rph, kretprobe_init_inst, kretprobe_fini_pool)) { 2624 kfree(rp->rph); 2625 rp->rph = NULL; 2626 return -ENOMEM; 2627 } 2628 rcu_assign_pointer(rp->rph->rp, rp); 2629 rp->nmissed = 0; 2630 /* Establish function entry probe point */ 2631 ret = register_kprobe(&rp->kp); 2632 if (ret != 0) 2633 free_rp_inst(rp); 2634 #endif 2635 return ret; 2636 } 2637 EXPORT_SYMBOL_GPL(register_kretprobe); 2638 2639 int register_kretprobes(struct kretprobe **rps, int num) 2640 { 2641 int ret = 0, i; 2642 2643 if (num <= 0) 2644 return -EINVAL; 2645 for (i = 0; i < num; i++) { 2646 ret = register_kretprobe(rps[i]); 2647 if (ret < 0) { 2648 if (i > 0) 2649 unregister_kretprobes(rps, i); 2650 break; 2651 } 2652 } 2653 return ret; 2654 } 2655 EXPORT_SYMBOL_GPL(register_kretprobes); 2656 2657 void unregister_kretprobe(struct kretprobe *rp) 2658 { 2659 unregister_kretprobes(&rp, 1); 2660 } 2661 EXPORT_SYMBOL_GPL(unregister_kretprobe); 2662 2663 void unregister_kretprobes(struct kretprobe **rps, int num) 2664 { 2665 int i; 2666 2667 if (num <= 0) 2668 return; 2669 for (i = 0; i < num; i++) { 2670 guard(mutex)(&kprobe_mutex); 2671 2672 if (__unregister_kprobe_top(&rps[i]->kp) < 0) { 2673 rps[i]->kp.addr = NULL; 2674 continue; 2675 } 2676 #ifdef CONFIG_KRETPROBE_ON_RETHOOK 2677 rethook_free(rps[i]->rh); 2678 #else 2679 rcu_assign_pointer(rps[i]->rph->rp, NULL); 2680 #endif 2681 } 2682 2683 synchronize_rcu(); 2684 for (i = 0; i < num; i++) { 2685 if (rps[i]->kp.addr) { 2686 __unregister_kprobe_bottom(&rps[i]->kp); 2687 #ifdef CONFIG_KRETPROBE_ON_RETHOOK 2688 rps[i]->rh = NULL; 2689 #else 2690 free_rp_inst(rps[i]); 2691 #endif 2692 } 2693 } 2694 } 2695 EXPORT_SYMBOL_GPL(unregister_kretprobes); 2696 2697 #else /* CONFIG_KRETPROBES */ 2698 int register_kretprobe(struct kretprobe *rp) 2699 { 2700 return -EOPNOTSUPP; 2701 } 2702 EXPORT_SYMBOL_GPL(register_kretprobe); 2703 2704 int register_kretprobes(struct kretprobe **rps, int num) 2705 { 2706 return -EOPNOTSUPP; 2707 } 2708 EXPORT_SYMBOL_GPL(register_kretprobes); 2709 2710 void unregister_kretprobe(struct kretprobe *rp) 2711 { 2712 } 2713 EXPORT_SYMBOL_GPL(unregister_kretprobe); 2714 2715 void unregister_kretprobes(struct kretprobe **rps, int num) 2716 { 2717 } 2718 EXPORT_SYMBOL_GPL(unregister_kretprobes); 2719 2720 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) 2721 { 2722 return 0; 2723 } 2724 NOKPROBE_SYMBOL(pre_handler_kretprobe); 2725 2726 #endif /* CONFIG_KRETPROBES */ 2727 2728 /* Set the kprobe gone and remove its instruction buffer. */ 2729 static void kill_kprobe(struct kprobe *p) 2730 { 2731 struct kprobe *kp; 2732 2733 lockdep_assert_held(&kprobe_mutex); 2734 2735 /* 2736 * The module is going away. We should disarm the kprobe which 2737 * is using ftrace, because ftrace framework is still available at 2738 * 'MODULE_STATE_GOING' notification. 2739 */ 2740 if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed) 2741 disarm_kprobe_ftrace(p); 2742 2743 p->flags |= KPROBE_FLAG_GONE; 2744 if (kprobe_aggrprobe(p)) { 2745 /* 2746 * If this is an aggr_kprobe, we have to list all the 2747 * chained probes and mark them GONE. 2748 */ 2749 list_for_each_entry(kp, &p->list, list) 2750 kp->flags |= KPROBE_FLAG_GONE; 2751 p->post_handler = NULL; 2752 kill_optimized_kprobe(p); 2753 } 2754 /* 2755 * Here, we can remove insn_slot safely, because no thread calls 2756 * the original probed function (which will be freed soon) any more. 2757 */ 2758 arch_remove_kprobe(p); 2759 } 2760 2761 /* Disable one kprobe */ 2762 int disable_kprobe(struct kprobe *kp) 2763 { 2764 struct kprobe *p; 2765 2766 guard(mutex)(&kprobe_mutex); 2767 2768 /* Disable this kprobe */ 2769 p = __disable_kprobe(kp); 2770 2771 return IS_ERR(p) ? PTR_ERR(p) : 0; 2772 } 2773 EXPORT_SYMBOL_GPL(disable_kprobe); 2774 2775 /* Enable one kprobe */ 2776 int enable_kprobe(struct kprobe *kp) 2777 { 2778 int ret = 0; 2779 struct kprobe *p; 2780 2781 guard(mutex)(&kprobe_mutex); 2782 2783 /* Check whether specified probe is valid. */ 2784 p = __get_valid_kprobe(kp); 2785 if (unlikely(p == NULL)) 2786 return -EINVAL; 2787 2788 if (kprobe_gone(kp)) 2789 /* This kprobe has gone, we couldn't enable it. */ 2790 return -EINVAL; 2791 2792 if (p != kp) 2793 kp->flags &= ~KPROBE_FLAG_DISABLED; 2794 2795 if (!kprobes_all_disarmed && kprobe_disabled(p)) { 2796 p->flags &= ~KPROBE_FLAG_DISABLED; 2797 ret = arm_kprobe(p); 2798 if (ret) { 2799 p->flags |= KPROBE_FLAG_DISABLED; 2800 if (p != kp) 2801 kp->flags |= KPROBE_FLAG_DISABLED; 2802 } 2803 } 2804 return ret; 2805 } 2806 EXPORT_SYMBOL_GPL(enable_kprobe); 2807 2808 /* Caller must NOT call this in usual path. This is only for critical case */ 2809 void dump_kprobe(struct kprobe *kp) 2810 { 2811 pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n", 2812 kp->symbol_name, kp->offset, kp->addr); 2813 } 2814 NOKPROBE_SYMBOL(dump_kprobe); 2815 2816 int kprobe_add_ksym_blacklist(unsigned long entry) 2817 { 2818 struct kprobe_blacklist_entry *ent; 2819 unsigned long offset = 0, size = 0; 2820 2821 if (!kernel_text_address(entry) || 2822 !kallsyms_lookup_size_offset(entry, &size, &offset)) 2823 return -EINVAL; 2824 2825 ent = kmalloc_obj(*ent); 2826 if (!ent) 2827 return -ENOMEM; 2828 ent->start_addr = entry; 2829 ent->end_addr = entry + size; 2830 INIT_LIST_HEAD(&ent->list); 2831 list_add_tail_rcu(&ent->list, &kprobe_blacklist); 2832 2833 return (int)size; 2834 } 2835 2836 /* Add all symbols in given area into kprobe blacklist */ 2837 int kprobe_add_area_blacklist(unsigned long start, unsigned long end) 2838 { 2839 unsigned long entry; 2840 int ret = 0; 2841 2842 for (entry = start; entry < end; entry += ret) { 2843 ret = kprobe_add_ksym_blacklist(entry); 2844 if (ret < 0) 2845 return ret; 2846 if (ret == 0) /* In case of alias symbol */ 2847 ret = 1; 2848 } 2849 return 0; 2850 } 2851 2852 int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value, 2853 char *type, char *sym) 2854 { 2855 return -ERANGE; 2856 } 2857 2858 int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type, 2859 char *sym) 2860 { 2861 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT 2862 if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym)) 2863 return 0; 2864 #ifdef CONFIG_OPTPROBES 2865 if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym)) 2866 return 0; 2867 #endif 2868 #endif 2869 if (!arch_kprobe_get_kallsym(&symnum, value, type, sym)) 2870 return 0; 2871 return -ERANGE; 2872 } 2873 2874 int __init __weak arch_populate_kprobe_blacklist(void) 2875 { 2876 return 0; 2877 } 2878 2879 /* 2880 * Lookup and populate the kprobe_blacklist. 2881 * 2882 * Unlike the kretprobe blacklist, we'll need to determine 2883 * the range of addresses that belong to the said functions, 2884 * since a kprobe need not necessarily be at the beginning 2885 * of a function. 2886 */ 2887 static int __init populate_kprobe_blacklist(unsigned long *start, 2888 unsigned long *end) 2889 { 2890 unsigned long entry; 2891 unsigned long *iter; 2892 int ret; 2893 2894 for (iter = start; iter < end; iter++) { 2895 entry = (unsigned long)dereference_symbol_descriptor((void *)*iter); 2896 ret = kprobe_add_ksym_blacklist(entry); 2897 if (ret == -EINVAL) 2898 continue; 2899 if (ret < 0) 2900 return ret; 2901 } 2902 2903 /* Symbols in '__kprobes_text' are blacklisted */ 2904 ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start, 2905 (unsigned long)__kprobes_text_end); 2906 if (ret) 2907 return ret; 2908 2909 /* Symbols in 'noinstr' section are blacklisted */ 2910 ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start, 2911 (unsigned long)__noinstr_text_end); 2912 2913 return ret ? : arch_populate_kprobe_blacklist(); 2914 } 2915 2916 #ifdef CONFIG_MODULES 2917 /* Remove all symbols in given area from kprobe blacklist */ 2918 static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end) 2919 { 2920 struct kprobe_blacklist_entry *ent, *n; 2921 2922 list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) { 2923 if (ent->start_addr < start || ent->start_addr >= end) 2924 continue; 2925 list_del_rcu(&ent->list); 2926 kfree_rcu(ent, rcu); 2927 } 2928 } 2929 2930 static void kprobe_remove_ksym_blacklist(unsigned long entry) 2931 { 2932 kprobe_remove_area_blacklist(entry, entry + 1); 2933 } 2934 2935 static void add_module_kprobe_blacklist(struct module *mod) 2936 { 2937 unsigned long start, end; 2938 int i; 2939 2940 if (mod->kprobe_blacklist) { 2941 for (i = 0; i < mod->num_kprobe_blacklist; i++) 2942 kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]); 2943 } 2944 2945 start = (unsigned long)mod->kprobes_text_start; 2946 if (start) { 2947 end = start + mod->kprobes_text_size; 2948 kprobe_add_area_blacklist(start, end); 2949 } 2950 2951 start = (unsigned long)mod->noinstr_text_start; 2952 if (start) { 2953 end = start + mod->noinstr_text_size; 2954 kprobe_add_area_blacklist(start, end); 2955 } 2956 } 2957 2958 static void remove_module_kprobe_blacklist(struct module *mod) 2959 { 2960 unsigned long start, end; 2961 int i; 2962 2963 if (mod->kprobe_blacklist) { 2964 for (i = 0; i < mod->num_kprobe_blacklist; i++) 2965 kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]); 2966 } 2967 2968 start = (unsigned long)mod->kprobes_text_start; 2969 if (start) { 2970 end = start + mod->kprobes_text_size; 2971 kprobe_remove_area_blacklist(start, end); 2972 } 2973 2974 start = (unsigned long)mod->noinstr_text_start; 2975 if (start) { 2976 end = start + mod->noinstr_text_size; 2977 kprobe_remove_area_blacklist(start, end); 2978 } 2979 } 2980 2981 /* Module notifier call back, checking kprobes on the module */ 2982 static int kprobes_module_callback(struct notifier_block *nb, 2983 unsigned long val, void *data) 2984 { 2985 struct module *mod = data; 2986 struct hlist_head *head; 2987 struct kprobe *p; 2988 unsigned int i; 2989 int checkcore = (val == MODULE_STATE_GOING); 2990 2991 guard(mutex)(&kprobe_mutex); 2992 2993 if (val == MODULE_STATE_COMING) 2994 add_module_kprobe_blacklist(mod); 2995 2996 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE) 2997 return NOTIFY_DONE; 2998 2999 /* 3000 * When 'MODULE_STATE_GOING' was notified, both of module '.text' and 3001 * '.init.text' sections would be freed. When 'MODULE_STATE_LIVE' was 3002 * notified, only '.init.text' section would be freed. We need to 3003 * disable kprobes which have been inserted in the sections. 3004 */ 3005 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 3006 head = &kprobe_table[i]; 3007 hlist_for_each_entry(p, head, hlist) 3008 if (within_module_init((unsigned long)p->addr, mod) || 3009 (checkcore && 3010 within_module_core((unsigned long)p->addr, mod))) { 3011 /* 3012 * The vaddr this probe is installed will soon 3013 * be vfreed buy not synced to disk. Hence, 3014 * disarming the breakpoint isn't needed. 3015 * 3016 * Note, this will also move any optimized probes 3017 * that are pending to be removed from their 3018 * corresponding lists to the 'freeing_list' and 3019 * will not be touched by the delayed 3020 * kprobe_optimizer() work handler. 3021 */ 3022 kill_kprobe(p); 3023 } 3024 } 3025 if (val == MODULE_STATE_GOING) 3026 remove_module_kprobe_blacklist(mod); 3027 return NOTIFY_DONE; 3028 } 3029 3030 static struct notifier_block kprobe_module_nb = { 3031 .notifier_call = kprobes_module_callback, 3032 .priority = 0 3033 }; 3034 3035 static int kprobe_register_module_notifier(void) 3036 { 3037 return register_module_notifier(&kprobe_module_nb); 3038 } 3039 #else 3040 static int kprobe_register_module_notifier(void) 3041 { 3042 return 0; 3043 } 3044 #endif /* CONFIG_MODULES */ 3045 3046 void kprobe_free_init_mem(void) 3047 { 3048 void *start = (void *)(&__init_begin); 3049 void *end = (void *)(&__init_end); 3050 struct hlist_head *head; 3051 struct kprobe *p; 3052 int i; 3053 3054 guard(mutex)(&kprobe_mutex); 3055 3056 /* Kill all kprobes on initmem because the target code has been freed. */ 3057 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 3058 head = &kprobe_table[i]; 3059 hlist_for_each_entry(p, head, hlist) { 3060 if (start <= (void *)p->addr && (void *)p->addr < end) 3061 kill_kprobe(p); 3062 } 3063 } 3064 } 3065 3066 static int __init init_kprobes(void) 3067 { 3068 int i, err; 3069 3070 /* FIXME allocate the probe table, currently defined statically */ 3071 /* initialize all list heads */ 3072 for (i = 0; i < KPROBE_TABLE_SIZE; i++) 3073 INIT_HLIST_HEAD(&kprobe_table[i]); 3074 3075 err = populate_kprobe_blacklist(__start_kprobe_blacklist, 3076 __stop_kprobe_blacklist); 3077 if (err) 3078 pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err); 3079 3080 if (kretprobe_blacklist_size) { 3081 /* lookup the function address from its name */ 3082 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 3083 kretprobe_blacklist[i].addr = 3084 kprobe_lookup_name(kretprobe_blacklist[i].name, 0); 3085 if (!kretprobe_blacklist[i].addr) 3086 pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n", 3087 kretprobe_blacklist[i].name); 3088 } 3089 } 3090 3091 /* By default, kprobes are armed */ 3092 kprobes_all_disarmed = false; 3093 3094 /* Initialize the optimization infrastructure */ 3095 init_optprobe(); 3096 3097 err = arch_init_kprobes(); 3098 if (!err) 3099 err = register_die_notifier(&kprobe_exceptions_nb); 3100 if (!err) 3101 err = kprobe_register_module_notifier(); 3102 3103 kprobes_initialized = (err == 0); 3104 kprobe_sysctls_init(); 3105 return err; 3106 } 3107 early_initcall(init_kprobes); 3108 3109 #if defined(CONFIG_OPTPROBES) 3110 static int __init init_optprobes(void) 3111 { 3112 /* 3113 * Enable kprobe optimization - this kicks the optimizer which 3114 * depends on synchronize_rcu_tasks() and ksoftirqd, that is 3115 * not spawned in early initcall. So delay the optimization. 3116 */ 3117 optimize_all_kprobes(); 3118 3119 return 0; 3120 } 3121 subsys_initcall(init_optprobes); 3122 #endif 3123 3124 #ifdef CONFIG_DEBUG_FS 3125 static void report_probe(struct seq_file *pi, struct kprobe *p, 3126 const char *sym, int offset, char *modname, struct kprobe *pp) 3127 { 3128 char *kprobe_type; 3129 void *addr = p->addr; 3130 3131 if (p->pre_handler == pre_handler_kretprobe) 3132 kprobe_type = "r"; 3133 else 3134 kprobe_type = "k"; 3135 3136 if (!kallsyms_show_value(pi->file->f_cred)) 3137 addr = NULL; 3138 3139 if (sym) 3140 seq_printf(pi, "%px %s %s+0x%x %s ", 3141 addr, kprobe_type, sym, offset, 3142 (modname ? modname : " ")); 3143 else /* try to use %pS */ 3144 seq_printf(pi, "%px %s %pS ", 3145 addr, kprobe_type, p->addr); 3146 3147 if (!pp) 3148 pp = p; 3149 seq_printf(pi, "%s%s%s%s\n", 3150 (kprobe_gone(p) ? "[GONE]" : ""), 3151 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""), 3152 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""), 3153 (kprobe_ftrace(pp) ? "[FTRACE]" : "")); 3154 } 3155 3156 static void *kprobe_seq_start(struct seq_file *f, loff_t *pos) 3157 { 3158 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL; 3159 } 3160 3161 static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos) 3162 { 3163 (*pos)++; 3164 if (*pos >= KPROBE_TABLE_SIZE) 3165 return NULL; 3166 return pos; 3167 } 3168 3169 static void kprobe_seq_stop(struct seq_file *f, void *v) 3170 { 3171 /* Nothing to do */ 3172 } 3173 3174 static int show_kprobe_addr(struct seq_file *pi, void *v) 3175 { 3176 struct hlist_head *head; 3177 struct kprobe *p, *kp; 3178 const char *sym; 3179 unsigned int i = *(loff_t *) v; 3180 unsigned long offset = 0; 3181 char *modname, namebuf[KSYM_NAME_LEN]; 3182 3183 head = &kprobe_table[i]; 3184 preempt_disable(); 3185 hlist_for_each_entry_rcu(p, head, hlist) { 3186 sym = kallsyms_lookup((unsigned long)p->addr, NULL, 3187 &offset, &modname, namebuf); 3188 if (kprobe_aggrprobe(p)) { 3189 list_for_each_entry_rcu(kp, &p->list, list) 3190 report_probe(pi, kp, sym, offset, modname, p); 3191 } else 3192 report_probe(pi, p, sym, offset, modname, NULL); 3193 } 3194 preempt_enable(); 3195 return 0; 3196 } 3197 3198 static const struct seq_operations kprobes_sops = { 3199 .start = kprobe_seq_start, 3200 .next = kprobe_seq_next, 3201 .stop = kprobe_seq_stop, 3202 .show = show_kprobe_addr 3203 }; 3204 3205 DEFINE_SEQ_ATTRIBUTE(kprobes); 3206 3207 /* kprobes/blacklist -- shows which functions can not be probed */ 3208 static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos) 3209 { 3210 mutex_lock(&kprobe_mutex); 3211 return seq_list_start(&kprobe_blacklist, *pos); 3212 } 3213 3214 static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos) 3215 { 3216 return seq_list_next(v, &kprobe_blacklist, pos); 3217 } 3218 3219 static int kprobe_blacklist_seq_show(struct seq_file *m, void *v) 3220 { 3221 struct kprobe_blacklist_entry *ent = 3222 list_entry(v, struct kprobe_blacklist_entry, list); 3223 3224 /* 3225 * If '/proc/kallsyms' is not showing kernel address, we won't 3226 * show them here either. 3227 */ 3228 if (!kallsyms_show_value(m->file->f_cred)) 3229 seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL, 3230 (void *)ent->start_addr); 3231 else 3232 seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr, 3233 (void *)ent->end_addr, (void *)ent->start_addr); 3234 return 0; 3235 } 3236 3237 static void kprobe_blacklist_seq_stop(struct seq_file *f, void *v) 3238 { 3239 mutex_unlock(&kprobe_mutex); 3240 } 3241 3242 static const struct seq_operations kprobe_blacklist_sops = { 3243 .start = kprobe_blacklist_seq_start, 3244 .next = kprobe_blacklist_seq_next, 3245 .stop = kprobe_blacklist_seq_stop, 3246 .show = kprobe_blacklist_seq_show, 3247 }; 3248 DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist); 3249 3250 static int arm_all_kprobes(void) 3251 { 3252 struct hlist_head *head; 3253 struct kprobe *p; 3254 unsigned int i, total = 0, errors = 0; 3255 int err, ret = 0; 3256 3257 guard(mutex)(&kprobe_mutex); 3258 3259 /* If kprobes are armed, just return */ 3260 if (!kprobes_all_disarmed) 3261 return 0; 3262 3263 /* 3264 * optimize_kprobe() called by arm_kprobe() checks 3265 * kprobes_all_disarmed, so set kprobes_all_disarmed before 3266 * arm_kprobe. 3267 */ 3268 kprobes_all_disarmed = false; 3269 /* Arming kprobes doesn't optimize kprobe itself */ 3270 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 3271 head = &kprobe_table[i]; 3272 /* Arm all kprobes on a best-effort basis */ 3273 hlist_for_each_entry(p, head, hlist) { 3274 if (!kprobe_disabled(p)) { 3275 err = arm_kprobe(p); 3276 if (err) { 3277 errors++; 3278 ret = err; 3279 } 3280 total++; 3281 } 3282 } 3283 } 3284 3285 if (errors) 3286 pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n", 3287 errors, total); 3288 else 3289 pr_info("Kprobes globally enabled\n"); 3290 3291 return ret; 3292 } 3293 3294 static int disarm_all_kprobes(void) 3295 { 3296 struct hlist_head *head; 3297 struct kprobe *p; 3298 unsigned int i, total = 0, errors = 0; 3299 int err, ret = 0; 3300 3301 guard(mutex)(&kprobe_mutex); 3302 3303 /* If kprobes are already disarmed, just return */ 3304 if (kprobes_all_disarmed) 3305 return 0; 3306 3307 kprobes_all_disarmed = true; 3308 3309 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 3310 head = &kprobe_table[i]; 3311 /* Disarm all kprobes on a best-effort basis */ 3312 hlist_for_each_entry(p, head, hlist) { 3313 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) { 3314 err = disarm_kprobe(p, false); 3315 if (err) { 3316 errors++; 3317 ret = err; 3318 } 3319 total++; 3320 } 3321 } 3322 } 3323 3324 if (errors) 3325 pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n", 3326 errors, total); 3327 else 3328 pr_info("Kprobes globally disabled\n"); 3329 3330 /* Wait for disarming all kprobes by optimizer */ 3331 wait_for_kprobe_optimizer_locked(); 3332 return ret; 3333 } 3334 3335 static int kprobes_enabled_set(void *data, u64 val) 3336 { 3337 if (val) 3338 return arm_all_kprobes(); 3339 3340 return disarm_all_kprobes(); 3341 } 3342 3343 static int kprobes_enabled_get(void *data, u64 *val) 3344 { 3345 *val = !kprobes_all_disarmed; 3346 return 0; 3347 } 3348 3349 DEFINE_DEBUGFS_ATTRIBUTE(fops_kp, kprobes_enabled_get, 3350 kprobes_enabled_set, "%llu\n"); 3351 3352 static int __init debugfs_kprobe_init(void) 3353 { 3354 struct dentry *dir; 3355 3356 dir = debugfs_create_dir("kprobes", NULL); 3357 3358 debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops); 3359 3360 debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp); 3361 3362 debugfs_create_file("blacklist", 0400, dir, NULL, 3363 &kprobe_blacklist_fops); 3364 3365 return 0; 3366 } 3367 3368 late_initcall(debugfs_kprobe_init); 3369 #endif /* CONFIG_DEBUG_FS */ 3370