1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Kprobes-based tracing events 4 * 5 * Created by Masami Hiramatsu <mhiramat@redhat.com> 6 * 7 */ 8 #define pr_fmt(fmt) "trace_kprobe: " fmt 9 10 #include <linux/bpf-cgroup.h> 11 #include <linux/cleanup.h> 12 #include <linux/error-injection.h> 13 #include <linux/module.h> 14 #include <linux/rculist.h> 15 #include <linux/security.h> 16 #include <linux/uaccess.h> 17 18 #include <asm/setup.h> /* for COMMAND_LINE_SIZE */ 19 20 #include "trace_dynevent.h" 21 #include "trace_kprobe_selftest.h" 22 #include "trace_probe.h" 23 #include "trace_probe_kernel.h" 24 #include "trace_probe_tmpl.h" 25 26 #define KPROBE_EVENT_SYSTEM "kprobes" 27 #define KRETPROBE_MAXACTIVE_MAX 4096 28 29 /* Kprobe early definition from command line */ 30 static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata; 31 32 static int __init set_kprobe_boot_events(char *str) 33 { 34 strscpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE); 35 disable_tracing_selftest("running kprobe events"); 36 37 return 1; 38 } 39 __setup("kprobe_event=", set_kprobe_boot_events); 40 41 static int trace_kprobe_create(const char *raw_command); 42 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); 43 static int trace_kprobe_release(struct dyn_event *ev); 44 static bool trace_kprobe_is_busy(struct dyn_event *ev); 45 static bool trace_kprobe_match(const char *system, const char *event, 46 int argc, const char **argv, struct dyn_event *ev); 47 48 static struct dyn_event_operations trace_kprobe_ops = { 49 .create = trace_kprobe_create, 50 .show = trace_kprobe_show, 51 .is_busy = trace_kprobe_is_busy, 52 .free = trace_kprobe_release, 53 .match = trace_kprobe_match, 54 }; 55 56 /* 57 * Kprobe event core functions 58 */ 59 struct trace_kprobe { 60 struct dyn_event devent; 61 struct kretprobe rp; /* Use rp.kp for kprobe use */ 62 unsigned long __percpu *nhit; 63 const char *symbol; /* symbol name */ 64 struct trace_probe tp; 65 }; 66 67 static bool is_trace_kprobe(struct dyn_event *ev) 68 { 69 return ev->ops == &trace_kprobe_ops; 70 } 71 72 static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev) 73 { 74 return container_of(ev, struct trace_kprobe, devent); 75 } 76 77 /** 78 * for_each_trace_kprobe - iterate over the trace_kprobe list 79 * @pos: the struct trace_kprobe * for each entry 80 * @dpos: the struct dyn_event * to use as a loop cursor 81 */ 82 #define for_each_trace_kprobe(pos, dpos) \ 83 for_each_dyn_event(dpos) \ 84 if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) 85 #define trace_kprobe_list_empty() list_empty(&dyn_event_list) 86 87 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) 88 { 89 return tk->rp.handler != NULL; 90 } 91 92 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) 93 { 94 return tk->symbol ? tk->symbol : "unknown"; 95 } 96 97 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) 98 { 99 return tk->rp.kp.offset; 100 } 101 102 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) 103 { 104 return kprobe_gone(&tk->rp.kp); 105 } 106 107 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, 108 struct module *mod) 109 { 110 int len = strlen(module_name(mod)); 111 const char *name = trace_kprobe_symbol(tk); 112 113 return strncmp(module_name(mod), name, len) == 0 && name[len] == ':'; 114 } 115 116 #ifdef CONFIG_MODULES 117 static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 118 { 119 char *p; 120 bool ret; 121 122 if (!tk->symbol) 123 return false; 124 p = strchr(tk->symbol, ':'); 125 if (!p) 126 return true; 127 *p = '\0'; 128 scoped_guard(rcu) 129 ret = !!find_module(tk->symbol); 130 *p = ':'; 131 132 return ret; 133 } 134 #else 135 static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 136 { 137 return false; 138 } 139 #endif 140 141 static bool trace_kprobe_is_busy(struct dyn_event *ev) 142 { 143 struct trace_kprobe *tk = to_trace_kprobe(ev); 144 145 return trace_probe_is_enabled(&tk->tp); 146 } 147 148 static bool trace_kprobe_match_command_head(struct trace_kprobe *tk, 149 int argc, const char **argv) 150 { 151 char buf[MAX_ARGSTR_LEN + 1]; 152 153 if (!argc) 154 return true; 155 156 if (!tk->symbol) 157 snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr); 158 else if (tk->rp.kp.offset) 159 snprintf(buf, sizeof(buf), "%s+%u", 160 trace_kprobe_symbol(tk), tk->rp.kp.offset); 161 else 162 snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk)); 163 if (strcmp(buf, argv[0])) 164 return false; 165 argc--; argv++; 166 167 return trace_probe_match_command_args(&tk->tp, argc, argv); 168 } 169 170 static bool trace_kprobe_match(const char *system, const char *event, 171 int argc, const char **argv, struct dyn_event *ev) 172 { 173 struct trace_kprobe *tk = to_trace_kprobe(ev); 174 175 return (event[0] == '\0' || 176 strcmp(trace_probe_name(&tk->tp), event) == 0) && 177 (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) && 178 trace_kprobe_match_command_head(tk, argc, argv); 179 } 180 181 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 182 { 183 unsigned long nhit = 0; 184 int cpu; 185 186 for_each_possible_cpu(cpu) 187 nhit += *per_cpu_ptr(tk->nhit, cpu); 188 189 return nhit; 190 } 191 192 static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk) 193 { 194 return !(list_empty(&tk->rp.kp.list) && 195 hlist_unhashed(&tk->rp.kp.hlist)); 196 } 197 198 /* Return 0 if it fails to find the symbol address */ 199 static nokprobe_inline 200 unsigned long trace_kprobe_address(struct trace_kprobe *tk) 201 { 202 unsigned long addr; 203 204 if (tk->symbol) { 205 addr = (unsigned long) 206 kallsyms_lookup_name(trace_kprobe_symbol(tk)); 207 if (addr) 208 addr += tk->rp.kp.offset; 209 } else { 210 addr = (unsigned long)tk->rp.kp.addr; 211 } 212 return addr; 213 } 214 215 static nokprobe_inline struct trace_kprobe * 216 trace_kprobe_primary_from_call(struct trace_event_call *call) 217 { 218 struct trace_probe *tp; 219 220 tp = trace_probe_primary_from_call(call); 221 if (WARN_ON_ONCE(!tp)) 222 return NULL; 223 224 return container_of(tp, struct trace_kprobe, tp); 225 } 226 227 bool trace_kprobe_on_func_entry(struct trace_event_call *call) 228 { 229 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 230 231 return tk ? (kprobe_on_func_entry(tk->rp.kp.addr, 232 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, 233 tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false; 234 } 235 236 bool trace_kprobe_error_injectable(struct trace_event_call *call) 237 { 238 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 239 240 return tk ? within_error_injection_list(trace_kprobe_address(tk)) : 241 false; 242 } 243 244 static int register_kprobe_event(struct trace_kprobe *tk); 245 static int unregister_kprobe_event(struct trace_kprobe *tk); 246 247 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 248 static int kretprobe_dispatcher(struct kretprobe_instance *ri, 249 struct pt_regs *regs); 250 251 static void free_trace_kprobe(struct trace_kprobe *tk) 252 { 253 if (tk) { 254 trace_probe_cleanup(&tk->tp); 255 kfree(tk->symbol); 256 free_percpu(tk->nhit); 257 kfree(tk); 258 } 259 } 260 261 DEFINE_FREE(free_trace_kprobe, struct trace_kprobe *, 262 if (!IS_ERR_OR_NULL(_T)) free_trace_kprobe(_T)) 263 264 /* 265 * Allocate new trace_probe and initialize it (including kprobes). 266 */ 267 static struct trace_kprobe *alloc_trace_kprobe(const char *group, 268 const char *event, 269 void *addr, 270 const char *symbol, 271 unsigned long offs, 272 int maxactive, 273 int nargs, bool is_return) 274 { 275 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 276 int ret = -ENOMEM; 277 278 tk = kzalloc_flex(*tk, tp.args, nargs); 279 if (!tk) 280 return ERR_PTR(ret); 281 282 tk->nhit = alloc_percpu(unsigned long); 283 if (!tk->nhit) 284 return ERR_PTR(ret); 285 286 if (symbol) { 287 tk->symbol = kstrdup(symbol, GFP_KERNEL); 288 if (!tk->symbol) 289 return ERR_PTR(ret); 290 tk->rp.kp.symbol_name = tk->symbol; 291 tk->rp.kp.offset = offs; 292 } else 293 tk->rp.kp.addr = addr; 294 295 if (is_return) 296 tk->rp.handler = kretprobe_dispatcher; 297 else 298 tk->rp.kp.pre_handler = kprobe_dispatcher; 299 300 tk->rp.maxactive = maxactive; 301 INIT_HLIST_NODE(&tk->rp.kp.hlist); 302 INIT_LIST_HEAD(&tk->rp.kp.list); 303 304 ret = trace_probe_init(&tk->tp, event, group, false, nargs); 305 if (ret < 0) 306 return ERR_PTR(ret); 307 308 dyn_event_init(&tk->devent, &trace_kprobe_ops); 309 return_ptr(tk); 310 } 311 312 static struct trace_kprobe *find_trace_kprobe(const char *event, 313 const char *group) 314 { 315 struct dyn_event *pos; 316 struct trace_kprobe *tk; 317 318 for_each_trace_kprobe(tk, pos) 319 if (strcmp(trace_probe_name(&tk->tp), event) == 0 && 320 strcmp(trace_probe_group_name(&tk->tp), group) == 0) 321 return tk; 322 return NULL; 323 } 324 325 static inline int __enable_trace_kprobe(struct trace_kprobe *tk) 326 { 327 int ret = 0; 328 329 if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) { 330 if (trace_kprobe_is_return(tk)) 331 ret = enable_kretprobe(&tk->rp); 332 else 333 ret = enable_kprobe(&tk->rp.kp); 334 } 335 336 return ret; 337 } 338 339 static void __disable_trace_kprobe(struct trace_probe *tp) 340 { 341 struct trace_kprobe *tk; 342 343 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 344 if (!trace_kprobe_is_registered(tk)) 345 continue; 346 if (trace_kprobe_is_return(tk)) 347 disable_kretprobe(&tk->rp); 348 else 349 disable_kprobe(&tk->rp.kp); 350 } 351 } 352 353 /* 354 * Enable trace_probe 355 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 356 */ 357 static int enable_trace_kprobe(struct trace_event_call *call, 358 struct trace_event_file *file) 359 { 360 struct trace_probe *tp; 361 struct trace_kprobe *tk; 362 bool enabled; 363 int ret = 0; 364 365 tp = trace_probe_primary_from_call(call); 366 if (WARN_ON_ONCE(!tp)) 367 return -ENODEV; 368 enabled = trace_probe_is_enabled(tp); 369 370 /* This also changes "enabled" state */ 371 if (file) { 372 ret = trace_probe_add_file(tp, file); 373 if (ret) 374 return ret; 375 } else 376 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 377 378 if (enabled) 379 return 0; 380 381 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 382 if (trace_kprobe_has_gone(tk)) 383 continue; 384 ret = __enable_trace_kprobe(tk); 385 if (ret) 386 break; 387 enabled = true; 388 } 389 390 if (ret) { 391 /* Failed to enable one of them. Roll back all */ 392 if (enabled) 393 __disable_trace_kprobe(tp); 394 if (file) 395 trace_probe_remove_file(tp, file); 396 else 397 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 398 } 399 400 return ret; 401 } 402 403 /* 404 * Disable trace_probe 405 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 406 */ 407 static int disable_trace_kprobe(struct trace_event_call *call, 408 struct trace_event_file *file) 409 { 410 struct trace_probe *tp; 411 412 tp = trace_probe_primary_from_call(call); 413 if (WARN_ON_ONCE(!tp)) 414 return -ENODEV; 415 416 if (file) { 417 if (!trace_probe_get_file_link(tp, file)) 418 return -ENOENT; 419 if (!trace_probe_has_single_file(tp)) 420 goto out; 421 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 422 } else 423 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 424 425 if (!trace_probe_is_enabled(tp)) 426 __disable_trace_kprobe(tp); 427 428 out: 429 if (file) 430 /* 431 * Synchronization is done in below function. For perf event, 432 * file == NULL and perf_trace_event_unreg() calls 433 * tracepoint_synchronize_unregister() to ensure synchronize 434 * event. We don't need to care about it. 435 */ 436 trace_probe_remove_file(tp, file); 437 438 return 0; 439 } 440 441 #if defined(CONFIG_DYNAMIC_FTRACE) && \ 442 !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE) 443 static bool __within_notrace_func(unsigned long addr) 444 { 445 unsigned long offset, size; 446 447 if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset)) 448 return false; 449 450 /* Get the entry address of the target function */ 451 addr -= offset; 452 453 /* 454 * Since ftrace_location_range() does inclusive range check, we need 455 * to subtract 1 byte from the end address. 456 */ 457 return !ftrace_location_range(addr, addr + size - 1); 458 } 459 460 static bool within_notrace_func(struct trace_kprobe *tk) 461 { 462 unsigned long addr = trace_kprobe_address(tk); 463 char symname[KSYM_NAME_LEN], *p; 464 465 if (!__within_notrace_func(addr)) 466 return false; 467 468 /* Check if the address is on a suffixed-symbol */ 469 if (!lookup_symbol_name(addr, symname)) { 470 p = strchr(symname, '.'); 471 if (!p) 472 return true; 473 *p = '\0'; 474 addr = (unsigned long)kprobe_lookup_name(symname, 0); 475 if (addr) 476 return __within_notrace_func(addr); 477 } 478 479 return true; 480 } 481 #else 482 #define within_notrace_func(tk) (false) 483 #endif 484 485 /* Internal register function - just handle k*probes and flags */ 486 static int __register_trace_kprobe(struct trace_kprobe *tk) 487 { 488 int i, ret; 489 490 ret = security_locked_down(LOCKDOWN_KPROBES); 491 if (ret) 492 return ret; 493 494 if (trace_kprobe_is_registered(tk)) 495 return -EINVAL; 496 497 if (within_notrace_func(tk)) { 498 pr_warn("Could not probe notrace function %ps\n", 499 (void *)trace_kprobe_address(tk)); 500 return -EINVAL; 501 } 502 503 for (i = 0; i < tk->tp.nr_args; i++) { 504 ret = traceprobe_update_arg(&tk->tp.args[i]); 505 if (ret) 506 return ret; 507 } 508 509 /* Set/clear disabled flag according to tp->flag */ 510 if (trace_probe_is_enabled(&tk->tp)) 511 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; 512 else 513 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; 514 515 if (trace_kprobe_is_return(tk)) 516 ret = register_kretprobe(&tk->rp); 517 else 518 ret = register_kprobe(&tk->rp.kp); 519 520 return ret; 521 } 522 523 /* Internal unregister function - just handle k*probes and flags */ 524 static void __unregister_trace_kprobe(struct trace_kprobe *tk) 525 { 526 if (trace_kprobe_is_registered(tk)) { 527 if (trace_kprobe_is_return(tk)) 528 unregister_kretprobe(&tk->rp); 529 else 530 unregister_kprobe(&tk->rp.kp); 531 /* Cleanup kprobe for reuse and mark it unregistered */ 532 INIT_HLIST_NODE(&tk->rp.kp.hlist); 533 INIT_LIST_HEAD(&tk->rp.kp.list); 534 if (tk->rp.kp.symbol_name) 535 tk->rp.kp.addr = NULL; 536 } 537 } 538 539 /* Unregister a trace_probe and probe_event */ 540 static int unregister_trace_kprobe(struct trace_kprobe *tk) 541 { 542 /* If other probes are on the event, just unregister kprobe */ 543 if (trace_probe_has_sibling(&tk->tp)) 544 goto unreg; 545 546 /* Enabled event can not be unregistered */ 547 if (trace_probe_is_enabled(&tk->tp)) 548 return -EBUSY; 549 550 /* If there's a reference to the dynamic event */ 551 if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp))) 552 return -EBUSY; 553 554 /* Will fail if probe is being used by ftrace or perf */ 555 if (unregister_kprobe_event(tk)) 556 return -EBUSY; 557 558 unreg: 559 __unregister_trace_kprobe(tk); 560 dyn_event_remove(&tk->devent); 561 trace_probe_unlink(&tk->tp); 562 563 return 0; 564 } 565 566 static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig, 567 struct trace_kprobe *comp) 568 { 569 struct trace_probe_event *tpe = orig->tp.event; 570 int i; 571 572 list_for_each_entry(orig, &tpe->probes, tp.list) { 573 if (strcmp(trace_kprobe_symbol(orig), 574 trace_kprobe_symbol(comp)) || 575 trace_kprobe_offset(orig) != trace_kprobe_offset(comp)) 576 continue; 577 578 /* 579 * trace_probe_compare_arg_type() ensured that nr_args and 580 * each argument name and type are same. Let's compare comm. 581 */ 582 for (i = 0; i < orig->tp.nr_args; i++) { 583 if (strcmp(orig->tp.args[i].comm, 584 comp->tp.args[i].comm)) 585 break; 586 } 587 588 if (i == orig->tp.nr_args) 589 return true; 590 } 591 592 return false; 593 } 594 595 static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to) 596 { 597 int ret; 598 599 ret = trace_probe_compare_arg_type(&tk->tp, &to->tp); 600 if (ret) { 601 /* Note that argument starts index = 2 */ 602 trace_probe_log_set_index(ret + 1); 603 trace_probe_log_err(0, DIFF_ARG_TYPE); 604 return -EEXIST; 605 } 606 if (trace_kprobe_has_same_kprobe(to, tk)) { 607 trace_probe_log_set_index(0); 608 trace_probe_log_err(0, SAME_PROBE); 609 return -EEXIST; 610 } 611 612 /* Append to existing event */ 613 ret = trace_probe_append(&tk->tp, &to->tp); 614 if (ret) 615 return ret; 616 617 /* Register k*probe */ 618 ret = __register_trace_kprobe(tk); 619 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 620 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 621 ret = 0; 622 } 623 624 if (ret) 625 trace_probe_unlink(&tk->tp); 626 else 627 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 628 629 return ret; 630 } 631 632 /* Register a trace_probe and probe_event */ 633 static int register_trace_kprobe(struct trace_kprobe *tk) 634 { 635 struct trace_kprobe *old_tk; 636 int ret; 637 638 guard(mutex)(&event_mutex); 639 640 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), 641 trace_probe_group_name(&tk->tp)); 642 if (old_tk) { 643 if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) { 644 trace_probe_log_set_index(0); 645 trace_probe_log_err(0, DIFF_PROBE_TYPE); 646 return -EEXIST; 647 } 648 return append_trace_kprobe(tk, old_tk); 649 } 650 651 /* Register new event */ 652 ret = register_kprobe_event(tk); 653 if (ret) { 654 if (ret == -EEXIST) { 655 trace_probe_log_set_index(0); 656 trace_probe_log_err(0, EVENT_EXIST); 657 } else 658 pr_warn("Failed to register probe event(%d)\n", ret); 659 return ret; 660 } 661 662 /* Register k*probe */ 663 ret = __register_trace_kprobe(tk); 664 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 665 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 666 ret = 0; 667 } 668 669 if (ret < 0) 670 unregister_kprobe_event(tk); 671 else 672 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 673 674 return ret; 675 } 676 677 #ifdef CONFIG_MODULES 678 static int validate_module_probe_symbol(const char *modname, const char *symbol); 679 680 static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk) 681 { 682 const char *p; 683 int ret = 0; 684 685 p = strchr(trace_kprobe_symbol(tk), ':'); 686 if (p) 687 ret = validate_module_probe_symbol(module_name(mod), p + 1); 688 if (!ret) 689 ret = __register_trace_kprobe(tk); 690 return ret; 691 } 692 693 /* Module notifier call back, checking event on the module */ 694 static int trace_kprobe_module_callback(struct notifier_block *nb, 695 unsigned long val, void *data) 696 { 697 struct module *mod = data; 698 struct dyn_event *pos; 699 struct trace_kprobe *tk; 700 int ret; 701 702 if (val != MODULE_STATE_COMING) 703 return NOTIFY_DONE; 704 705 /* Update probes on coming module */ 706 guard(mutex)(&event_mutex); 707 for_each_trace_kprobe(tk, pos) { 708 if (trace_kprobe_within_module(tk, mod)) { 709 /* Don't need to check busy - this should have gone. */ 710 __unregister_trace_kprobe(tk); 711 ret = register_module_trace_kprobe(mod, tk); 712 if (ret) 713 pr_warn("Failed to re-register probe %s on %s: %d\n", 714 trace_probe_name(&tk->tp), 715 module_name(mod), ret); 716 } 717 } 718 719 return NOTIFY_DONE; 720 } 721 722 static struct notifier_block trace_kprobe_module_nb = { 723 .notifier_call = trace_kprobe_module_callback, 724 .priority = 2 /* Invoked after kprobe and jump_label module callback */ 725 }; 726 static int trace_kprobe_register_module_notifier(void) 727 { 728 return register_module_notifier(&trace_kprobe_module_nb); 729 } 730 #else 731 static int trace_kprobe_register_module_notifier(void) 732 { 733 return 0; 734 } 735 #endif /* CONFIG_MODULES */ 736 737 static int count_symbols(void *data, unsigned long unused) 738 { 739 unsigned int *count = data; 740 741 (*count)++; 742 743 return 0; 744 } 745 746 struct sym_count_ctx { 747 unsigned int count; 748 const char *name; 749 }; 750 751 static int count_mod_symbols(void *data, const char *name, unsigned long unused) 752 { 753 struct sym_count_ctx *ctx = data; 754 755 if (strcmp(name, ctx->name) == 0) 756 ctx->count++; 757 758 return 0; 759 } 760 761 static unsigned int number_of_same_symbols(const char *mod, const char *func_name) 762 { 763 struct sym_count_ctx ctx = { .count = 0, .name = func_name }; 764 765 if (!mod) 766 kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count); 767 768 /* 769 * If the symbol is found in vmlinux, use vmlinux resolution only. 770 * This prevents module symbols from shadowing vmlinux symbols 771 * and causing -EADDRNOTAVAIL for unqualified kprobe targets. 772 */ 773 if (!mod && ctx.count > 0) 774 return ctx.count; 775 776 module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx); 777 778 return ctx.count; 779 } 780 781 static int validate_module_probe_symbol(const char *modname, const char *symbol) 782 { 783 unsigned int count = number_of_same_symbols(modname, symbol); 784 785 if (count > 1) { 786 /* 787 * Users should use ADDR to remove the ambiguity of 788 * using KSYM only. 789 */ 790 return -EADDRNOTAVAIL; 791 } else if (count == 0) { 792 /* 793 * We can return ENOENT earlier than when register the 794 * kprobe. 795 */ 796 return -ENOENT; 797 } 798 return 0; 799 } 800 801 #ifdef CONFIG_MODULES 802 /* Return NULL if the module is not loaded or under unloading. */ 803 static struct module *try_module_get_by_name(const char *name) 804 { 805 struct module *mod; 806 807 guard(rcu)(); 808 mod = find_module(name); 809 if (mod && !try_module_get(mod)) 810 mod = NULL; 811 return mod; 812 } 813 #else 814 #define try_module_get_by_name(name) (NULL) 815 #endif 816 817 static int validate_probe_symbol(char *symbol) 818 { 819 struct module *mod = NULL; 820 char *modname = NULL, *p; 821 int ret = 0; 822 823 p = strchr(symbol, ':'); 824 if (p) { 825 modname = symbol; 826 symbol = p + 1; 827 *p = '\0'; 828 mod = try_module_get_by_name(modname); 829 if (!mod) 830 goto out; 831 } 832 833 ret = validate_module_probe_symbol(modname, symbol); 834 out: 835 if (p) 836 *p = ':'; 837 if (mod) 838 module_put(mod); 839 return ret; 840 } 841 842 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 843 struct pt_regs *regs); 844 845 static int trace_kprobe_create_internal(int argc, const char *argv[], 846 struct traceprobe_parse_context *ctx) 847 { 848 /* 849 * Argument syntax: 850 * - Add kprobe: 851 * p[:[GRP/][EVENT]] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS] 852 * - Add kretprobe: 853 * r[MAXACTIVE][:[GRP/][EVENT]] [MOD:]KSYM[+0] [FETCHARGS] 854 * Or 855 * p[:[GRP/][EVENT]] [MOD:]KSYM[+0]%return [FETCHARGS] 856 * 857 * Fetch args: 858 * $retval : fetch return value 859 * $stack : fetch stack address 860 * $stackN : fetch Nth of stack (N:0-) 861 * $comm : fetch current task comm 862 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 863 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 864 * %REG : fetch register REG 865 * Dereferencing memory fetch: 866 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 867 * Alias name of args: 868 * NAME=FETCHARG : set NAME as alias of FETCHARG. 869 * Type of args: 870 * FETCHARG:TYPE : use TYPE instead of unsigned long. 871 */ 872 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 873 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM; 874 const char **new_argv __free(kfree) = NULL; 875 int i, len, new_argc = 0, ret = 0; 876 char *symbol __free(kfree) = NULL; 877 char *ebuf __free(kfree) = NULL; 878 char *gbuf __free(kfree) = NULL; 879 char *abuf __free(kfree) = NULL; 880 char *dbuf __free(kfree) = NULL; 881 enum probe_print_type ptype; 882 bool is_return = false; 883 int maxactive = 0; 884 void *addr = NULL; 885 char *tmp = NULL; 886 long offset = 0; 887 888 switch (argv[0][0]) { 889 case 'r': 890 is_return = true; 891 break; 892 case 'p': 893 break; 894 default: 895 return -ECANCELED; 896 } 897 if (argc < 2) 898 return -ECANCELED; 899 900 event = strchr(&argv[0][1], ':'); 901 if (event) 902 event++; 903 904 if (isdigit(argv[0][1])) { 905 char *buf __free(kfree) = NULL; 906 907 if (!is_return) { 908 trace_probe_log_err(1, BAD_MAXACT_TYPE); 909 return -EINVAL; 910 } 911 if (event) 912 len = event - &argv[0][1] - 1; 913 else 914 len = strlen(&argv[0][1]); 915 if (len > MAX_EVENT_NAME_LEN - 1) { 916 trace_probe_log_err(1, BAD_MAXACT); 917 return -EINVAL; 918 } 919 buf = kmemdup(&argv[0][1], len + 1, GFP_KERNEL); 920 if (!buf) 921 return -ENOMEM; 922 buf[len] = '\0'; 923 ret = kstrtouint(buf, 0, &maxactive); 924 if (ret || !maxactive) { 925 trace_probe_log_err(1, BAD_MAXACT); 926 return -EINVAL; 927 } 928 /* kretprobes instances are iterated over via a list. The 929 * maximum should stay reasonable. 930 */ 931 if (maxactive > KRETPROBE_MAXACTIVE_MAX) { 932 trace_probe_log_err(1, MAXACT_TOO_BIG); 933 return -EINVAL; 934 } 935 } 936 937 /* try to parse an address. if that fails, try to read the 938 * input as a symbol. */ 939 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { 940 trace_probe_log_set_index(1); 941 /* Check whether uprobe event specified */ 942 if (strchr(argv[1], '/') && strchr(argv[1], ':')) 943 return -ECANCELED; 944 945 /* a symbol specified */ 946 symbol = kstrdup(argv[1], GFP_KERNEL); 947 if (!symbol) 948 return -ENOMEM; 949 950 tmp = strchr(symbol, '%'); 951 if (tmp) { 952 if (!strcmp(tmp, "%return")) { 953 *tmp = '\0'; 954 is_return = true; 955 } else { 956 trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX); 957 return -EINVAL; 958 } 959 } 960 961 /* TODO: support .init module functions */ 962 ret = traceprobe_split_symbol_offset(symbol, &offset); 963 if (ret || offset < 0 || offset > UINT_MAX) { 964 trace_probe_log_err(0, BAD_PROBE_ADDR); 965 return -EINVAL; 966 } 967 ret = validate_probe_symbol(symbol); 968 if (ret) { 969 if (ret == -EADDRNOTAVAIL) 970 trace_probe_log_err(0, NON_UNIQ_SYMBOL); 971 else 972 trace_probe_log_err(0, BAD_PROBE_ADDR); 973 return -EINVAL; 974 } 975 if (is_return) 976 ctx->flags |= TPARG_FL_RETURN; 977 ret = kprobe_on_func_entry(NULL, symbol, offset); 978 if (ret == 0 && !is_return) 979 ctx->flags |= TPARG_FL_FENTRY; 980 /* Defer the ENOENT case until register kprobe */ 981 if (ret == -EINVAL && is_return) { 982 trace_probe_log_err(0, BAD_RETPROBE); 983 return -EINVAL; 984 } 985 } 986 987 trace_probe_log_set_index(0); 988 if (event) { 989 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 990 if (!gbuf) 991 return -ENOMEM; 992 ret = traceprobe_parse_event_name(&event, &group, gbuf, 993 event - argv[0]); 994 if (ret) 995 return ret; 996 } 997 998 if (!event) { 999 /* Make a new event name */ 1000 ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 1001 if (!ebuf) 1002 return -ENOMEM; 1003 if (symbol) 1004 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 1005 is_return ? 'r' : 'p', symbol, offset); 1006 else 1007 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_0x%p", 1008 is_return ? 'r' : 'p', addr); 1009 sanitize_event_name(ebuf); 1010 event = ebuf; 1011 } 1012 1013 abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL); 1014 if (!abuf) 1015 return -ENOMEM; 1016 argc -= 2; argv += 2; 1017 ctx->funcname = symbol; 1018 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, 1019 abuf, MAX_BTF_ARGS_LEN, ctx); 1020 if (IS_ERR(new_argv)) { 1021 ret = PTR_ERR(new_argv); 1022 new_argv = NULL; 1023 return ret; 1024 } 1025 if (new_argv) { 1026 argc = new_argc; 1027 argv = new_argv; 1028 } 1029 if (argc > MAX_TRACE_ARGS) { 1030 trace_probe_log_set_index(2); 1031 trace_probe_log_err(0, TOO_MANY_ARGS); 1032 return -E2BIG; 1033 } 1034 1035 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); 1036 if (ret) 1037 return ret; 1038 1039 /* setup a probe */ 1040 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, 1041 argc, is_return); 1042 if (IS_ERR(tk)) { 1043 ret = PTR_ERR(tk); 1044 /* This must return -ENOMEM, else there is a bug */ 1045 WARN_ON_ONCE(ret != -ENOMEM); 1046 return ret; /* We know tk is not allocated */ 1047 } 1048 1049 /* parse arguments */ 1050 for (i = 0; i < argc; i++) { 1051 trace_probe_log_set_index(i + 2); 1052 ctx->offset = 0; 1053 ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], ctx); 1054 if (ret) 1055 return ret; /* This can be -ENOMEM */ 1056 } 1057 /* entry handler for kretprobe */ 1058 if (is_return && tk->tp.entry_arg) { 1059 tk->rp.entry_handler = trace_kprobe_entry_handler; 1060 tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp); 1061 } 1062 1063 ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1064 ret = traceprobe_set_print_fmt(&tk->tp, ptype); 1065 if (ret < 0) 1066 return ret; 1067 1068 ret = register_trace_kprobe(tk); 1069 if (ret) { 1070 trace_probe_log_set_index(1); 1071 if (ret == -EILSEQ) 1072 trace_probe_log_err(0, BAD_INSN_BNDRY); 1073 else if (ret == -ENOENT) 1074 trace_probe_log_err(0, BAD_PROBE_ADDR); 1075 else if (ret != -ENOMEM && ret != -EEXIST) 1076 trace_probe_log_err(0, FAIL_REG_PROBE); 1077 return ret; 1078 } 1079 /* 1080 * Here, 'tk' has been registered to the list successfully, 1081 * so we don't need to free it. 1082 */ 1083 tk = NULL; 1084 1085 return 0; 1086 } 1087 1088 static int trace_kprobe_create_cb(int argc, const char *argv[]) 1089 { 1090 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 1091 int ret; 1092 1093 ctx = kzalloc_obj(*ctx); 1094 if (!ctx) 1095 return -ENOMEM; 1096 ctx->flags = TPARG_FL_KERNEL; 1097 1098 trace_probe_log_init("trace_kprobe", argc, argv); 1099 1100 ret = trace_kprobe_create_internal(argc, argv, ctx); 1101 1102 trace_probe_log_clear(); 1103 return ret; 1104 } 1105 1106 static int trace_kprobe_create(const char *raw_command) 1107 { 1108 return trace_probe_create(raw_command, trace_kprobe_create_cb); 1109 } 1110 1111 static int create_or_delete_trace_kprobe(const char *raw_command) 1112 { 1113 int ret; 1114 1115 if (raw_command[0] == '-') 1116 return dyn_event_release(raw_command, &trace_kprobe_ops); 1117 1118 ret = dyn_event_create(raw_command, &trace_kprobe_ops); 1119 return ret == -ECANCELED ? -EINVAL : ret; 1120 } 1121 1122 static int trace_kprobe_run_command(struct dynevent_cmd *cmd) 1123 { 1124 return create_or_delete_trace_kprobe(cmd->seq.buffer); 1125 } 1126 1127 /** 1128 * kprobe_event_cmd_init - Initialize a kprobe event command object 1129 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1130 * @buf: A pointer to the buffer used to build the command 1131 * @maxlen: The length of the buffer passed in @buf 1132 * 1133 * Initialize a synthetic event command object. Use this before 1134 * calling any of the other kprobe_event functions. 1135 */ 1136 void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen) 1137 { 1138 dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE, 1139 trace_kprobe_run_command); 1140 } 1141 EXPORT_SYMBOL_GPL(kprobe_event_cmd_init); 1142 1143 /** 1144 * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list 1145 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1146 * @kretprobe: Is this a return probe? 1147 * @name: The name of the kprobe event 1148 * @loc: The location of the kprobe event 1149 * @...: Variable number of arg (pairs), one pair for each field 1150 * 1151 * NOTE: Users normally won't want to call this function directly, but 1152 * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically 1153 * adds a NULL to the end of the arg list. If this function is used 1154 * directly, make sure the last arg in the variable arg list is NULL. 1155 * 1156 * Generate a kprobe event command to be executed by 1157 * kprobe_event_gen_cmd_end(). This function can be used to generate the 1158 * complete command or only the first part of it; in the latter case, 1159 * kprobe_event_add_fields() can be used to add more fields following this. 1160 * 1161 * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This 1162 * returns -EINVAL if @loc == NULL. 1163 * 1164 * Return: 0 if successful, error otherwise. 1165 */ 1166 int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe, 1167 const char *name, const char *loc, ...) 1168 { 1169 char buf[MAX_EVENT_NAME_LEN]; 1170 struct dynevent_arg arg; 1171 va_list args; 1172 int ret; 1173 1174 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1175 return -EINVAL; 1176 1177 if (!loc) 1178 return -EINVAL; 1179 1180 if (kretprobe) 1181 snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name); 1182 else 1183 snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name); 1184 1185 ret = dynevent_str_add(cmd, buf); 1186 if (ret) 1187 return ret; 1188 1189 dynevent_arg_init(&arg, 0); 1190 arg.str = loc; 1191 ret = dynevent_arg_add(cmd, &arg, NULL); 1192 if (ret) 1193 return ret; 1194 1195 va_start(args, loc); 1196 for (;;) { 1197 const char *field; 1198 1199 field = va_arg(args, const char *); 1200 if (!field) 1201 break; 1202 1203 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1204 ret = -EINVAL; 1205 break; 1206 } 1207 1208 arg.str = field; 1209 ret = dynevent_arg_add(cmd, &arg, NULL); 1210 if (ret) 1211 break; 1212 } 1213 va_end(args); 1214 1215 return ret; 1216 } 1217 EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start); 1218 1219 /** 1220 * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list 1221 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1222 * @...: Variable number of arg (pairs), one pair for each field 1223 * 1224 * NOTE: Users normally won't want to call this function directly, but 1225 * rather use the kprobe_event_add_fields() wrapper, which 1226 * automatically adds a NULL to the end of the arg list. If this 1227 * function is used directly, make sure the last arg in the variable 1228 * arg list is NULL. 1229 * 1230 * Add probe fields to an existing kprobe command using a variable 1231 * list of args. Fields are added in the same order they're listed. 1232 * 1233 * Return: 0 if successful, error otherwise. 1234 */ 1235 int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...) 1236 { 1237 struct dynevent_arg arg; 1238 va_list args; 1239 int ret = 0; 1240 1241 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1242 return -EINVAL; 1243 1244 dynevent_arg_init(&arg, 0); 1245 1246 va_start(args, cmd); 1247 for (;;) { 1248 const char *field; 1249 1250 field = va_arg(args, const char *); 1251 if (!field) 1252 break; 1253 1254 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1255 ret = -EINVAL; 1256 break; 1257 } 1258 1259 arg.str = field; 1260 ret = dynevent_arg_add(cmd, &arg, NULL); 1261 if (ret) 1262 break; 1263 } 1264 va_end(args); 1265 1266 return ret; 1267 } 1268 EXPORT_SYMBOL_GPL(__kprobe_event_add_fields); 1269 1270 /** 1271 * kprobe_event_delete - Delete a kprobe event 1272 * @name: The name of the kprobe event to delete 1273 * 1274 * Delete a kprobe event with the give @name from kernel code rather 1275 * than directly from the command line. 1276 * 1277 * Return: 0 if successful, error otherwise. 1278 */ 1279 int kprobe_event_delete(const char *name) 1280 { 1281 char buf[MAX_EVENT_NAME_LEN]; 1282 1283 snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name); 1284 1285 return create_or_delete_trace_kprobe(buf); 1286 } 1287 EXPORT_SYMBOL_GPL(kprobe_event_delete); 1288 1289 static int trace_kprobe_release(struct dyn_event *ev) 1290 { 1291 struct trace_kprobe *tk = to_trace_kprobe(ev); 1292 int ret = unregister_trace_kprobe(tk); 1293 1294 if (!ret) 1295 free_trace_kprobe(tk); 1296 return ret; 1297 } 1298 1299 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) 1300 { 1301 struct trace_kprobe *tk = to_trace_kprobe(ev); 1302 int i; 1303 1304 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); 1305 if (trace_kprobe_is_return(tk) && tk->rp.maxactive) 1306 seq_printf(m, "%d", tk->rp.maxactive); 1307 seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp), 1308 trace_probe_name(&tk->tp)); 1309 1310 if (!tk->symbol) 1311 seq_printf(m, " 0x%p", tk->rp.kp.addr); 1312 else if (tk->rp.kp.offset) 1313 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk), 1314 tk->rp.kp.offset); 1315 else 1316 seq_printf(m, " %s", trace_kprobe_symbol(tk)); 1317 1318 for (i = 0; i < tk->tp.nr_args; i++) 1319 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); 1320 seq_putc(m, '\n'); 1321 1322 return 0; 1323 } 1324 1325 static int probes_seq_show(struct seq_file *m, void *v) 1326 { 1327 struct dyn_event *ev = v; 1328 1329 if (!is_trace_kprobe(ev)) 1330 return 0; 1331 1332 return trace_kprobe_show(m, ev); 1333 } 1334 1335 static const struct seq_operations probes_seq_op = { 1336 .start = dyn_event_seq_start, 1337 .next = dyn_event_seq_next, 1338 .stop = dyn_event_seq_stop, 1339 .show = probes_seq_show 1340 }; 1341 1342 static int probes_open(struct inode *inode, struct file *file) 1343 { 1344 int ret; 1345 1346 ret = security_locked_down(LOCKDOWN_TRACEFS); 1347 if (ret) 1348 return ret; 1349 1350 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1351 ret = dyn_events_release_all(&trace_kprobe_ops); 1352 if (ret < 0) 1353 return ret; 1354 } 1355 1356 return seq_open(file, &probes_seq_op); 1357 } 1358 1359 static ssize_t probes_write(struct file *file, const char __user *buffer, 1360 size_t count, loff_t *ppos) 1361 { 1362 return trace_parse_run_command(file, buffer, count, ppos, 1363 create_or_delete_trace_kprobe); 1364 } 1365 1366 static const struct file_operations kprobe_events_ops = { 1367 .owner = THIS_MODULE, 1368 .open = probes_open, 1369 .read = seq_read, 1370 .llseek = seq_lseek, 1371 .release = seq_release, 1372 .write = probes_write, 1373 }; 1374 1375 static unsigned long trace_kprobe_missed(struct trace_kprobe *tk) 1376 { 1377 return trace_kprobe_is_return(tk) ? 1378 tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed; 1379 } 1380 1381 /* Probes profiling interfaces */ 1382 static int probes_profile_seq_show(struct seq_file *m, void *v) 1383 { 1384 struct dyn_event *ev = v; 1385 struct trace_kprobe *tk; 1386 unsigned long nmissed; 1387 1388 if (!is_trace_kprobe(ev)) 1389 return 0; 1390 1391 tk = to_trace_kprobe(ev); 1392 nmissed = trace_kprobe_missed(tk); 1393 seq_printf(m, " %-44s %15lu %15lu\n", 1394 trace_probe_name(&tk->tp), 1395 trace_kprobe_nhit(tk), 1396 nmissed); 1397 1398 return 0; 1399 } 1400 1401 static const struct seq_operations profile_seq_op = { 1402 .start = dyn_event_seq_start, 1403 .next = dyn_event_seq_next, 1404 .stop = dyn_event_seq_stop, 1405 .show = probes_profile_seq_show 1406 }; 1407 1408 static int profile_open(struct inode *inode, struct file *file) 1409 { 1410 int ret; 1411 1412 ret = security_locked_down(LOCKDOWN_TRACEFS); 1413 if (ret) 1414 return ret; 1415 1416 return seq_open(file, &profile_seq_op); 1417 } 1418 1419 static const struct file_operations kprobe_profile_ops = { 1420 .owner = THIS_MODULE, 1421 .open = profile_open, 1422 .read = seq_read, 1423 .llseek = seq_lseek, 1424 .release = seq_release, 1425 }; 1426 1427 /* Note that we don't verify it, since the code does not come from user space */ 1428 static int 1429 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 1430 void *dest, void *base) 1431 { 1432 struct pt_regs *regs = rec; 1433 unsigned long val; 1434 int ret; 1435 1436 retry: 1437 /* 1st stage: get value from context */ 1438 switch (code->op) { 1439 case FETCH_OP_REG: 1440 val = regs_get_register(regs, code->param); 1441 break; 1442 case FETCH_OP_STACK: 1443 val = regs_get_kernel_stack_nth(regs, code->param); 1444 break; 1445 case FETCH_OP_STACKP: 1446 val = kernel_stack_pointer(regs); 1447 break; 1448 case FETCH_OP_RETVAL: 1449 val = regs_return_value(regs); 1450 break; 1451 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 1452 case FETCH_OP_ARG: 1453 val = regs_get_kernel_argument(regs, code->param); 1454 break; 1455 case FETCH_OP_EDATA: 1456 val = *(unsigned long *)((unsigned long)edata + code->offset); 1457 break; 1458 #endif 1459 case FETCH_NOP_SYMBOL: /* Ignore a place holder */ 1460 code++; 1461 goto retry; 1462 default: 1463 ret = process_common_fetch_insn(code, &val); 1464 if (ret < 0) 1465 return ret; 1466 } 1467 code++; 1468 1469 return process_fetch_insn_bottom(code, val, dest, base); 1470 } 1471 NOKPROBE_SYMBOL(process_fetch_insn) 1472 1473 /* Kprobe handler */ 1474 static nokprobe_inline void 1475 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 1476 struct trace_event_file *trace_file) 1477 { 1478 struct kprobe_trace_entry_head *entry; 1479 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1480 struct trace_event_buffer fbuffer; 1481 int dsize; 1482 1483 WARN_ON(call != trace_file->event_call); 1484 1485 if (trace_trigger_soft_disabled(trace_file)) 1486 return; 1487 1488 dsize = __get_data_size(&tk->tp, regs, NULL); 1489 1490 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1491 sizeof(*entry) + tk->tp.size + dsize); 1492 if (!entry) 1493 return; 1494 1495 fbuffer.regs = regs; 1496 entry->ip = (unsigned long)tk->rp.kp.addr; 1497 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1498 1499 trace_event_buffer_commit(&fbuffer); 1500 } 1501 1502 static void 1503 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) 1504 { 1505 struct event_file_link *link; 1506 1507 trace_probe_for_each_link_rcu(link, &tk->tp) 1508 __kprobe_trace_func(tk, regs, link->file); 1509 } 1510 NOKPROBE_SYMBOL(kprobe_trace_func); 1511 1512 /* Kretprobe handler */ 1513 1514 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 1515 struct pt_regs *regs) 1516 { 1517 struct kretprobe *rp = get_kretprobe(ri); 1518 struct trace_kprobe *tk; 1519 1520 /* 1521 * There is a small chance that get_kretprobe(ri) returns NULL when 1522 * the kretprobe is unregister on another CPU between kretprobe's 1523 * trampoline_handler and this function. 1524 */ 1525 if (unlikely(!rp)) 1526 return -ENOENT; 1527 1528 tk = container_of(rp, struct trace_kprobe, rp); 1529 1530 /* store argument values into ri->data as entry data */ 1531 if (tk->tp.entry_arg) 1532 store_trace_entry_data(ri->data, &tk->tp, regs); 1533 1534 return 0; 1535 } 1536 1537 1538 static nokprobe_inline void 1539 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1540 struct pt_regs *regs, 1541 struct trace_event_file *trace_file) 1542 { 1543 struct kretprobe_trace_entry_head *entry; 1544 struct trace_event_buffer fbuffer; 1545 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1546 int dsize; 1547 1548 WARN_ON(call != trace_file->event_call); 1549 1550 if (trace_trigger_soft_disabled(trace_file)) 1551 return; 1552 1553 dsize = __get_data_size(&tk->tp, regs, ri->data); 1554 1555 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1556 sizeof(*entry) + tk->tp.size + dsize); 1557 if (!entry) 1558 return; 1559 1560 fbuffer.regs = regs; 1561 entry->func = (unsigned long)tk->rp.kp.addr; 1562 entry->ret_ip = get_kretprobe_retaddr(ri); 1563 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1564 1565 trace_event_buffer_commit(&fbuffer); 1566 } 1567 1568 static void 1569 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1570 struct pt_regs *regs) 1571 { 1572 struct event_file_link *link; 1573 1574 trace_probe_for_each_link_rcu(link, &tk->tp) 1575 __kretprobe_trace_func(tk, ri, regs, link->file); 1576 } 1577 NOKPROBE_SYMBOL(kretprobe_trace_func); 1578 1579 /* Event entry printers */ 1580 static enum print_line_t 1581 print_kprobe_event(struct trace_iterator *iter, int flags, 1582 struct trace_event *event) 1583 { 1584 struct kprobe_trace_entry_head *field; 1585 struct trace_seq *s = &iter->seq; 1586 struct trace_probe *tp; 1587 1588 field = (struct kprobe_trace_entry_head *)iter->ent; 1589 tp = trace_probe_primary_from_call( 1590 container_of(event, struct trace_event_call, event)); 1591 if (WARN_ON_ONCE(!tp)) 1592 goto out; 1593 1594 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1595 1596 if (!seq_print_ip_sym_offset(s, field->ip, flags)) 1597 goto out; 1598 1599 trace_seq_putc(s, ')'); 1600 1601 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1602 (u8 *)&field[1], field) < 0) 1603 goto out; 1604 1605 trace_seq_putc(s, '\n'); 1606 out: 1607 return trace_handle_return(s); 1608 } 1609 1610 static enum print_line_t 1611 print_kretprobe_event(struct trace_iterator *iter, int flags, 1612 struct trace_event *event) 1613 { 1614 struct kretprobe_trace_entry_head *field; 1615 struct trace_seq *s = &iter->seq; 1616 struct trace_probe *tp; 1617 1618 field = (struct kretprobe_trace_entry_head *)iter->ent; 1619 tp = trace_probe_primary_from_call( 1620 container_of(event, struct trace_event_call, event)); 1621 if (WARN_ON_ONCE(!tp)) 1622 goto out; 1623 1624 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1625 1626 if (!seq_print_ip_sym_offset(s, field->ret_ip, flags)) 1627 goto out; 1628 1629 trace_seq_puts(s, " <- "); 1630 1631 if (!seq_print_ip_sym_no_offset(s, field->func, flags)) 1632 goto out; 1633 1634 trace_seq_putc(s, ')'); 1635 1636 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1637 (u8 *)&field[1], field) < 0) 1638 goto out; 1639 1640 trace_seq_putc(s, '\n'); 1641 1642 out: 1643 return trace_handle_return(s); 1644 } 1645 1646 1647 static int kprobe_event_define_fields(struct trace_event_call *event_call) 1648 { 1649 int ret; 1650 struct kprobe_trace_entry_head field; 1651 struct trace_probe *tp; 1652 1653 tp = trace_probe_primary_from_call(event_call); 1654 if (WARN_ON_ONCE(!tp)) 1655 return -ENOENT; 1656 1657 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1658 1659 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1660 } 1661 1662 static int kretprobe_event_define_fields(struct trace_event_call *event_call) 1663 { 1664 int ret; 1665 struct kretprobe_trace_entry_head field; 1666 struct trace_probe *tp; 1667 1668 tp = trace_probe_primary_from_call(event_call); 1669 if (WARN_ON_ONCE(!tp)) 1670 return -ENOENT; 1671 1672 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1673 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1674 1675 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1676 } 1677 1678 #ifdef CONFIG_PERF_EVENTS 1679 1680 /* Kprobe profile handler */ 1681 static int 1682 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) 1683 { 1684 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1685 struct kprobe_trace_entry_head *entry; 1686 struct hlist_head *head; 1687 int size, __size, dsize; 1688 int rctx; 1689 1690 if (bpf_prog_array_valid(call)) { 1691 unsigned long orig_ip = instruction_pointer(regs); 1692 int ret; 1693 1694 ret = trace_call_bpf(call, regs); 1695 1696 /* 1697 * We need to check and see if we modified the pc of the 1698 * pt_regs, and if so return 1 so that we don't do the 1699 * single stepping. 1700 */ 1701 if (orig_ip != instruction_pointer(regs)) 1702 return 1; 1703 if (!ret) 1704 return 0; 1705 } 1706 1707 head = this_cpu_ptr(call->perf_events); 1708 if (hlist_empty(head)) 1709 return 0; 1710 1711 dsize = __get_data_size(&tk->tp, regs, NULL); 1712 __size = sizeof(*entry) + tk->tp.size + dsize; 1713 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1714 size -= sizeof(u32); 1715 1716 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1717 if (!entry) 1718 return 0; 1719 1720 entry->ip = (unsigned long)tk->rp.kp.addr; 1721 memset(&entry[1], 0, dsize); 1722 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1723 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1724 head, NULL); 1725 return 0; 1726 } 1727 NOKPROBE_SYMBOL(kprobe_perf_func); 1728 1729 /* Kretprobe profile handler */ 1730 static void 1731 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1732 struct pt_regs *regs) 1733 { 1734 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1735 struct kretprobe_trace_entry_head *entry; 1736 struct hlist_head *head; 1737 int size, __size, dsize; 1738 int rctx; 1739 1740 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs)) 1741 return; 1742 1743 head = this_cpu_ptr(call->perf_events); 1744 if (hlist_empty(head)) 1745 return; 1746 1747 dsize = __get_data_size(&tk->tp, regs, ri->data); 1748 __size = sizeof(*entry) + tk->tp.size + dsize; 1749 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1750 size -= sizeof(u32); 1751 1752 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1753 if (!entry) 1754 return; 1755 1756 entry->func = (unsigned long)tk->rp.kp.addr; 1757 entry->ret_ip = get_kretprobe_retaddr(ri); 1758 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1759 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1760 head, NULL); 1761 } 1762 NOKPROBE_SYMBOL(kretprobe_perf_func); 1763 1764 int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, 1765 const char **symbol, u64 *probe_offset, 1766 u64 *probe_addr, unsigned long *missed, 1767 bool perf_type_tracepoint) 1768 { 1769 const char *pevent = trace_event_name(event->tp_event); 1770 const char *group = event->tp_event->class->system; 1771 struct trace_kprobe *tk; 1772 1773 if (perf_type_tracepoint) 1774 tk = find_trace_kprobe(pevent, group); 1775 else 1776 tk = trace_kprobe_primary_from_call(event->tp_event); 1777 if (!tk) 1778 return -EINVAL; 1779 1780 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE 1781 : BPF_FD_TYPE_KPROBE; 1782 *probe_offset = tk->rp.kp.offset; 1783 *probe_addr = kallsyms_show_value(current_cred()) ? 1784 (unsigned long)tk->rp.kp.addr : 0; 1785 *symbol = tk->symbol; 1786 if (missed) 1787 *missed = trace_kprobe_missed(tk); 1788 return 0; 1789 } 1790 #endif /* CONFIG_PERF_EVENTS */ 1791 1792 /* 1793 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1794 * 1795 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe 1796 * lockless, but we can't race with this __init function. 1797 */ 1798 static int kprobe_register(struct trace_event_call *event, 1799 enum trace_reg type, void *data) 1800 { 1801 struct trace_event_file *file = data; 1802 1803 switch (type) { 1804 case TRACE_REG_REGISTER: 1805 return enable_trace_kprobe(event, file); 1806 case TRACE_REG_UNREGISTER: 1807 return disable_trace_kprobe(event, file); 1808 1809 #ifdef CONFIG_PERF_EVENTS 1810 case TRACE_REG_PERF_REGISTER: 1811 return enable_trace_kprobe(event, NULL); 1812 case TRACE_REG_PERF_UNREGISTER: 1813 return disable_trace_kprobe(event, NULL); 1814 case TRACE_REG_PERF_OPEN: 1815 case TRACE_REG_PERF_CLOSE: 1816 case TRACE_REG_PERF_ADD: 1817 case TRACE_REG_PERF_DEL: 1818 return 0; 1819 #endif 1820 } 1821 return 0; 1822 } 1823 1824 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) 1825 { 1826 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); 1827 unsigned int flags = trace_probe_load_flag(&tk->tp); 1828 int ret = 0; 1829 1830 raw_cpu_inc(*tk->nhit); 1831 1832 if (flags & TP_FLAG_TRACE) 1833 kprobe_trace_func(tk, regs); 1834 #ifdef CONFIG_PERF_EVENTS 1835 if (flags & TP_FLAG_PROFILE) 1836 ret = kprobe_perf_func(tk, regs); 1837 #endif 1838 return ret; 1839 } 1840 NOKPROBE_SYMBOL(kprobe_dispatcher); 1841 1842 static int 1843 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) 1844 { 1845 struct kretprobe *rp = get_kretprobe(ri); 1846 struct trace_kprobe *tk; 1847 unsigned int flags; 1848 1849 /* 1850 * There is a small chance that get_kretprobe(ri) returns NULL when 1851 * the kretprobe is unregister on another CPU between kretprobe's 1852 * trampoline_handler and this function. 1853 */ 1854 if (unlikely(!rp)) 1855 return 0; 1856 1857 tk = container_of(rp, struct trace_kprobe, rp); 1858 raw_cpu_inc(*tk->nhit); 1859 1860 flags = trace_probe_load_flag(&tk->tp); 1861 if (flags & TP_FLAG_TRACE) 1862 kretprobe_trace_func(tk, ri, regs); 1863 #ifdef CONFIG_PERF_EVENTS 1864 if (flags & TP_FLAG_PROFILE) 1865 kretprobe_perf_func(tk, ri, regs); 1866 #endif 1867 return 0; /* We don't tweak kernel, so just return 0 */ 1868 } 1869 NOKPROBE_SYMBOL(kretprobe_dispatcher); 1870 1871 static struct trace_event_functions kretprobe_funcs = { 1872 .trace = print_kretprobe_event 1873 }; 1874 1875 static struct trace_event_functions kprobe_funcs = { 1876 .trace = print_kprobe_event 1877 }; 1878 1879 static struct trace_event_fields kretprobe_fields_array[] = { 1880 { .type = TRACE_FUNCTION_TYPE, 1881 .define_fields = kretprobe_event_define_fields }, 1882 {} 1883 }; 1884 1885 static struct trace_event_fields kprobe_fields_array[] = { 1886 { .type = TRACE_FUNCTION_TYPE, 1887 .define_fields = kprobe_event_define_fields }, 1888 {} 1889 }; 1890 1891 static inline void init_trace_event_call(struct trace_kprobe *tk) 1892 { 1893 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1894 1895 if (trace_kprobe_is_return(tk)) { 1896 call->event.funcs = &kretprobe_funcs; 1897 call->class->fields_array = kretprobe_fields_array; 1898 } else { 1899 call->event.funcs = &kprobe_funcs; 1900 call->class->fields_array = kprobe_fields_array; 1901 } 1902 1903 call->flags = TRACE_EVENT_FL_KPROBE; 1904 call->class->reg = kprobe_register; 1905 } 1906 1907 static int register_kprobe_event(struct trace_kprobe *tk) 1908 { 1909 init_trace_event_call(tk); 1910 1911 return trace_probe_register_event_call(&tk->tp); 1912 } 1913 1914 static int unregister_kprobe_event(struct trace_kprobe *tk) 1915 { 1916 return trace_probe_unregister_event_call(&tk->tp); 1917 } 1918 1919 #ifdef CONFIG_PERF_EVENTS 1920 1921 /* create a trace_kprobe, but don't add it to global lists */ 1922 struct trace_event_call * 1923 create_local_trace_kprobe(char *func, void *addr, unsigned long offs, 1924 bool is_return) 1925 { 1926 enum probe_print_type ptype; 1927 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 1928 int ret; 1929 char *event; 1930 1931 if (func) { 1932 ret = validate_probe_symbol(func); 1933 if (ret) 1934 return ERR_PTR(ret); 1935 } 1936 1937 /* 1938 * local trace_kprobes are not added to dyn_event, so they are never 1939 * searched in find_trace_kprobe(). Therefore, there is no concern of 1940 * duplicated name here. 1941 */ 1942 event = func ? func : "DUMMY_EVENT"; 1943 1944 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func, 1945 offs, 0 /* maxactive */, 0 /* nargs */, 1946 is_return); 1947 1948 if (IS_ERR(tk)) { 1949 pr_info("Failed to allocate trace_probe.(%d)\n", 1950 (int)PTR_ERR(tk)); 1951 return ERR_CAST(tk); 1952 } 1953 1954 init_trace_event_call(tk); 1955 1956 ptype = trace_kprobe_is_return(tk) ? 1957 PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1958 if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) 1959 return ERR_PTR(-ENOMEM); 1960 1961 ret = __register_trace_kprobe(tk); 1962 if (ret < 0) 1963 return ERR_PTR(ret); 1964 1965 return trace_probe_event_call(&(no_free_ptr(tk)->tp)); 1966 } 1967 1968 void destroy_local_trace_kprobe(struct trace_event_call *event_call) 1969 { 1970 struct trace_kprobe *tk; 1971 1972 tk = trace_kprobe_primary_from_call(event_call); 1973 if (unlikely(!tk)) 1974 return; 1975 1976 if (trace_probe_is_enabled(&tk->tp)) { 1977 WARN_ON(1); 1978 return; 1979 } 1980 1981 __unregister_trace_kprobe(tk); 1982 1983 free_trace_kprobe(tk); 1984 } 1985 #endif /* CONFIG_PERF_EVENTS */ 1986 1987 static __init void enable_boot_kprobe_events(void) 1988 { 1989 struct trace_array *tr = top_trace_array(); 1990 struct trace_event_file *file; 1991 struct trace_kprobe *tk; 1992 struct dyn_event *pos; 1993 1994 if (trace_kprobe_list_empty()) 1995 return; 1996 1997 guard(mutex)(&event_mutex); 1998 for_each_trace_kprobe(tk, pos) { 1999 list_for_each_entry(file, &tr->events, list) 2000 if (file->event_call == trace_probe_event_call(&tk->tp)) 2001 trace_event_enable_disable(file, 1, 0); 2002 } 2003 } 2004 2005 static __init void setup_boot_kprobe_events(void) 2006 { 2007 char *p, *cmd = kprobe_boot_events_buf; 2008 int ret; 2009 2010 strreplace(kprobe_boot_events_buf, ',', ' '); 2011 2012 while (cmd && *cmd != '\0') { 2013 p = strchr(cmd, ';'); 2014 if (p) 2015 *p++ = '\0'; 2016 2017 ret = create_or_delete_trace_kprobe(cmd); 2018 if (ret) 2019 pr_warn("Failed to add event(%d): %s\n", ret, cmd); 2020 2021 cmd = p; 2022 } 2023 2024 enable_boot_kprobe_events(); 2025 } 2026 2027 /* 2028 * Register dynevent at core_initcall. This allows kernel to setup kprobe 2029 * events in postcore_initcall without tracefs. 2030 */ 2031 static __init int init_kprobe_trace_early(void) 2032 { 2033 int ret; 2034 2035 ret = dyn_event_register(&trace_kprobe_ops); 2036 if (ret) 2037 return ret; 2038 2039 if (trace_kprobe_register_module_notifier()) 2040 return -EINVAL; 2041 2042 return 0; 2043 } 2044 core_initcall(init_kprobe_trace_early); 2045 2046 /* Make a tracefs interface for controlling probe points */ 2047 static __init int init_kprobe_trace(void) 2048 { 2049 int ret; 2050 2051 ret = tracing_init_dentry(); 2052 if (ret) 2053 return 0; 2054 2055 /* Event list interface */ 2056 trace_create_file("kprobe_events", TRACE_MODE_WRITE, 2057 NULL, NULL, &kprobe_events_ops); 2058 2059 /* Profile interface */ 2060 trace_create_file("kprobe_profile", TRACE_MODE_READ, 2061 NULL, NULL, &kprobe_profile_ops); 2062 2063 /* If no 'kprobe_event=' cmd is provided, return directly. */ 2064 if (kprobe_boot_events_buf[0] == '\0') 2065 return 0; 2066 2067 setup_boot_kprobe_events(); 2068 2069 return 0; 2070 } 2071 fs_initcall(init_kprobe_trace); 2072 2073 2074 #ifdef CONFIG_FTRACE_STARTUP_TEST 2075 static __init struct trace_event_file * 2076 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) 2077 { 2078 struct trace_event_file *file; 2079 2080 list_for_each_entry(file, &tr->events, list) 2081 if (file->event_call == trace_probe_event_call(&tk->tp)) 2082 return file; 2083 2084 return NULL; 2085 } 2086 2087 /* 2088 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this 2089 * stage, we can do this lockless. 2090 */ 2091 static __init int kprobe_trace_self_tests_init(void) 2092 { 2093 int ret, warn = 0; 2094 int (*target)(int, int, int, int, int, int); 2095 struct trace_kprobe *tk; 2096 struct trace_event_file *file; 2097 2098 if (unlikely(tracing_disabled)) 2099 return -ENODEV; 2100 2101 if (tracing_selftest_disabled) 2102 return 0; 2103 2104 target = kprobe_trace_selftest_target; 2105 2106 pr_info("Testing kprobe tracing: "); 2107 2108 ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)"); 2109 if (WARN_ONCE(ret, "error on probing function entry.")) { 2110 warn++; 2111 } else { 2112 /* Enable trace point */ 2113 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2114 if (WARN_ONCE(tk == NULL, "error on probing function entry.")) { 2115 warn++; 2116 } else { 2117 file = find_trace_probe_file(tk, top_trace_array()); 2118 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2119 warn++; 2120 } else 2121 enable_trace_kprobe( 2122 trace_probe_event_call(&tk->tp), file); 2123 } 2124 } 2125 2126 ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval"); 2127 if (WARN_ONCE(ret, "error on probing function return.")) { 2128 warn++; 2129 } else { 2130 /* Enable trace point */ 2131 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2132 if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) { 2133 warn++; 2134 } else { 2135 file = find_trace_probe_file(tk, top_trace_array()); 2136 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2137 warn++; 2138 } else 2139 enable_trace_kprobe( 2140 trace_probe_event_call(&tk->tp), file); 2141 } 2142 } 2143 2144 if (warn) 2145 goto end; 2146 2147 ret = target(1, 2, 3, 4, 5, 6); 2148 2149 /* 2150 * Not expecting an error here, the check is only to prevent the 2151 * optimizer from removing the call to target() as otherwise there 2152 * are no side-effects and the call is never performed. 2153 */ 2154 if (ret != 21) 2155 warn++; 2156 2157 /* Disable trace points before removing it */ 2158 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2159 if (WARN_ONCE(tk == NULL, "error on getting test probe.")) { 2160 warn++; 2161 } else { 2162 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2163 "incorrect number of testprobe hits.")) 2164 warn++; 2165 2166 file = find_trace_probe_file(tk, top_trace_array()); 2167 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2168 warn++; 2169 } else 2170 disable_trace_kprobe( 2171 trace_probe_event_call(&tk->tp), file); 2172 } 2173 2174 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2175 if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) { 2176 warn++; 2177 } else { 2178 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2179 "incorrect number of testprobe2 hits.")) 2180 warn++; 2181 2182 file = find_trace_probe_file(tk, top_trace_array()); 2183 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2184 warn++; 2185 } else 2186 disable_trace_kprobe( 2187 trace_probe_event_call(&tk->tp), file); 2188 } 2189 2190 ret = create_or_delete_trace_kprobe("-:testprobe"); 2191 if (WARN_ONCE(ret, "error on deleting a probe.")) 2192 warn++; 2193 2194 ret = create_or_delete_trace_kprobe("-:testprobe2"); 2195 if (WARN_ONCE(ret, "error on deleting a probe.")) 2196 warn++; 2197 2198 2199 end: 2200 /* 2201 * Wait for the optimizer work to finish. Otherwise it might fiddle 2202 * with probes in already freed __init text. 2203 */ 2204 wait_for_kprobe_optimizer(); 2205 if (warn) 2206 pr_cont("NG: Some tests are failed. Please check them.\n"); 2207 else 2208 pr_cont("OK\n"); 2209 return 0; 2210 } 2211 2212 late_initcall(kprobe_trace_self_tests_init); 2213 2214 #endif 2215