1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Kprobes-based tracing events 4 * 5 * Created by Masami Hiramatsu <mhiramat@redhat.com> 6 * 7 */ 8 #define pr_fmt(fmt) "trace_kprobe: " fmt 9 10 #include <linux/bpf-cgroup.h> 11 #include <linux/cleanup.h> 12 #include <linux/error-injection.h> 13 #include <linux/module.h> 14 #include <linux/rculist.h> 15 #include <linux/security.h> 16 #include <linux/uaccess.h> 17 18 #include <asm/setup.h> /* for COMMAND_LINE_SIZE */ 19 20 #include "trace_dynevent.h" 21 #include "trace_kprobe_selftest.h" 22 #include "trace_probe.h" 23 #include "trace_probe_kernel.h" 24 #include "trace_probe_tmpl.h" 25 26 #define KPROBE_EVENT_SYSTEM "kprobes" 27 #define KRETPROBE_MAXACTIVE_MAX 4096 28 29 /* Kprobe early definition from command line */ 30 static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata; 31 32 static int __init set_kprobe_boot_events(char *str) 33 { 34 trace_append_boot_param(kprobe_boot_events_buf, str, ';', 35 COMMAND_LINE_SIZE); 36 disable_tracing_selftest("running kprobe events"); 37 38 return 1; 39 } 40 __setup("kprobe_event=", set_kprobe_boot_events); 41 42 static int trace_kprobe_create(const char *raw_command); 43 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); 44 static int trace_kprobe_release(struct dyn_event *ev); 45 static bool trace_kprobe_is_busy(struct dyn_event *ev); 46 static bool trace_kprobe_match(const char *system, const char *event, 47 int argc, const char **argv, struct dyn_event *ev); 48 49 static struct dyn_event_operations trace_kprobe_ops = { 50 .create = trace_kprobe_create, 51 .show = trace_kprobe_show, 52 .is_busy = trace_kprobe_is_busy, 53 .free = trace_kprobe_release, 54 .match = trace_kprobe_match, 55 }; 56 57 /* 58 * Kprobe event core functions 59 */ 60 struct trace_kprobe { 61 struct dyn_event devent; 62 struct kretprobe rp; /* Use rp.kp for kprobe use */ 63 unsigned long __percpu *nhit; 64 const char *symbol; /* symbol name */ 65 struct trace_probe tp; 66 }; 67 68 static bool is_trace_kprobe(struct dyn_event *ev) 69 { 70 return ev->ops == &trace_kprobe_ops; 71 } 72 73 static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev) 74 { 75 return container_of(ev, struct trace_kprobe, devent); 76 } 77 78 /** 79 * for_each_trace_kprobe - iterate over the trace_kprobe list 80 * @pos: the struct trace_kprobe * for each entry 81 * @dpos: the struct dyn_event * to use as a loop cursor 82 */ 83 #define for_each_trace_kprobe(pos, dpos) \ 84 for_each_dyn_event(dpos) \ 85 if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) 86 #define trace_kprobe_list_empty() list_empty(&dyn_event_list) 87 88 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) 89 { 90 return tk->rp.handler != NULL; 91 } 92 93 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) 94 { 95 return tk->symbol ? tk->symbol : "unknown"; 96 } 97 98 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) 99 { 100 return tk->rp.kp.offset; 101 } 102 103 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) 104 { 105 return kprobe_gone(&tk->rp.kp); 106 } 107 108 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, 109 struct module *mod) 110 { 111 int len = strlen(module_name(mod)); 112 const char *name = trace_kprobe_symbol(tk); 113 114 return strncmp(module_name(mod), name, len) == 0 && name[len] == ':'; 115 } 116 117 #ifdef CONFIG_MODULES 118 static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 119 { 120 char *p; 121 bool ret; 122 123 if (!tk->symbol) 124 return false; 125 p = strchr(tk->symbol, ':'); 126 if (!p) 127 return true; 128 *p = '\0'; 129 scoped_guard(rcu) 130 ret = !!find_module(tk->symbol); 131 *p = ':'; 132 133 return ret; 134 } 135 #else 136 static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 137 { 138 return false; 139 } 140 #endif 141 142 static bool trace_kprobe_is_busy(struct dyn_event *ev) 143 { 144 struct trace_kprobe *tk = to_trace_kprobe(ev); 145 146 return trace_probe_is_enabled(&tk->tp); 147 } 148 149 static bool trace_kprobe_match_command_head(struct trace_kprobe *tk, 150 int argc, const char **argv) 151 { 152 char buf[MAX_ARGSTR_LEN + 1]; 153 154 if (!argc) 155 return true; 156 157 if (!tk->symbol) 158 snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr); 159 else if (tk->rp.kp.offset) 160 snprintf(buf, sizeof(buf), "%s+%u", 161 trace_kprobe_symbol(tk), tk->rp.kp.offset); 162 else 163 snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk)); 164 if (strcmp(buf, argv[0])) 165 return false; 166 argc--; argv++; 167 168 return trace_probe_match_command_args(&tk->tp, argc, argv); 169 } 170 171 static bool trace_kprobe_match(const char *system, const char *event, 172 int argc, const char **argv, struct dyn_event *ev) 173 { 174 struct trace_kprobe *tk = to_trace_kprobe(ev); 175 176 return (event[0] == '\0' || 177 strcmp(trace_probe_name(&tk->tp), event) == 0) && 178 (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) && 179 trace_kprobe_match_command_head(tk, argc, argv); 180 } 181 182 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 183 { 184 unsigned long nhit = 0; 185 int cpu; 186 187 for_each_possible_cpu(cpu) 188 nhit += *per_cpu_ptr(tk->nhit, cpu); 189 190 return nhit; 191 } 192 193 static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk) 194 { 195 return !(list_empty(&tk->rp.kp.list) && 196 hlist_unhashed(&tk->rp.kp.hlist)); 197 } 198 199 /* Return 0 if it fails to find the symbol address */ 200 static nokprobe_inline 201 unsigned long trace_kprobe_address(struct trace_kprobe *tk) 202 { 203 unsigned long addr; 204 205 if (tk->symbol) { 206 addr = (unsigned long) 207 kallsyms_lookup_name(trace_kprobe_symbol(tk)); 208 if (addr) 209 addr += tk->rp.kp.offset; 210 } else { 211 addr = (unsigned long)tk->rp.kp.addr; 212 } 213 return addr; 214 } 215 216 static nokprobe_inline struct trace_kprobe * 217 trace_kprobe_primary_from_call(struct trace_event_call *call) 218 { 219 struct trace_probe *tp; 220 221 tp = trace_probe_primary_from_call(call); 222 if (WARN_ON_ONCE(!tp)) 223 return NULL; 224 225 return container_of(tp, struct trace_kprobe, tp); 226 } 227 228 bool trace_kprobe_on_func_entry(struct trace_event_call *call) 229 { 230 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 231 232 return tk ? (kprobe_on_func_entry(tk->rp.kp.addr, 233 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, 234 tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false; 235 } 236 237 bool trace_kprobe_error_injectable(struct trace_event_call *call) 238 { 239 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 240 241 return tk ? within_error_injection_list(trace_kprobe_address(tk)) : 242 false; 243 } 244 245 static int register_kprobe_event(struct trace_kprobe *tk); 246 static int unregister_kprobe_event(struct trace_kprobe *tk); 247 248 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 249 static int kretprobe_dispatcher(struct kretprobe_instance *ri, 250 struct pt_regs *regs); 251 252 static void free_trace_kprobe(struct trace_kprobe *tk) 253 { 254 if (tk) { 255 trace_probe_cleanup(&tk->tp); 256 kfree(tk->symbol); 257 free_percpu(tk->nhit); 258 kfree(tk); 259 } 260 } 261 262 DEFINE_FREE(free_trace_kprobe, struct trace_kprobe *, 263 if (!IS_ERR_OR_NULL(_T)) free_trace_kprobe(_T)) 264 265 /* 266 * Allocate new trace_probe and initialize it (including kprobes). 267 */ 268 static struct trace_kprobe *alloc_trace_kprobe(const char *group, 269 const char *event, 270 void *addr, 271 const char *symbol, 272 unsigned long offs, 273 int maxactive, 274 int nargs, bool is_return) 275 { 276 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 277 int ret = -ENOMEM; 278 279 tk = kzalloc_flex(*tk, tp.args, nargs); 280 if (!tk) 281 return ERR_PTR(ret); 282 283 tk->nhit = alloc_percpu(unsigned long); 284 if (!tk->nhit) 285 return ERR_PTR(ret); 286 287 if (symbol) { 288 tk->symbol = kstrdup(symbol, GFP_KERNEL); 289 if (!tk->symbol) 290 return ERR_PTR(ret); 291 tk->rp.kp.symbol_name = tk->symbol; 292 tk->rp.kp.offset = offs; 293 } else 294 tk->rp.kp.addr = addr; 295 296 if (is_return) 297 tk->rp.handler = kretprobe_dispatcher; 298 else 299 tk->rp.kp.pre_handler = kprobe_dispatcher; 300 301 tk->rp.maxactive = maxactive; 302 INIT_HLIST_NODE(&tk->rp.kp.hlist); 303 INIT_LIST_HEAD(&tk->rp.kp.list); 304 305 ret = trace_probe_init(&tk->tp, event, group, false, nargs); 306 if (ret < 0) 307 return ERR_PTR(ret); 308 309 dyn_event_init(&tk->devent, &trace_kprobe_ops); 310 return_ptr(tk); 311 } 312 313 static struct trace_kprobe *find_trace_kprobe(const char *event, 314 const char *group) 315 { 316 struct dyn_event *pos; 317 struct trace_kprobe *tk; 318 319 for_each_trace_kprobe(tk, pos) 320 if (strcmp(trace_probe_name(&tk->tp), event) == 0 && 321 strcmp(trace_probe_group_name(&tk->tp), group) == 0) 322 return tk; 323 return NULL; 324 } 325 326 static inline int __enable_trace_kprobe(struct trace_kprobe *tk) 327 { 328 int ret = 0; 329 330 if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) { 331 if (trace_kprobe_is_return(tk)) 332 ret = enable_kretprobe(&tk->rp); 333 else 334 ret = enable_kprobe(&tk->rp.kp); 335 } 336 337 return ret; 338 } 339 340 static void __disable_trace_kprobe(struct trace_probe *tp) 341 { 342 struct trace_kprobe *tk; 343 344 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 345 if (!trace_kprobe_is_registered(tk)) 346 continue; 347 if (trace_kprobe_is_return(tk)) 348 disable_kretprobe(&tk->rp); 349 else 350 disable_kprobe(&tk->rp.kp); 351 } 352 } 353 354 /* 355 * Enable trace_probe 356 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 357 */ 358 static int enable_trace_kprobe(struct trace_event_call *call, 359 struct trace_event_file *file) 360 { 361 struct trace_probe *tp; 362 struct trace_kprobe *tk; 363 bool enabled; 364 int ret = 0; 365 366 tp = trace_probe_primary_from_call(call); 367 if (WARN_ON_ONCE(!tp)) 368 return -ENODEV; 369 enabled = trace_probe_is_enabled(tp); 370 371 /* This also changes "enabled" state */ 372 if (file) { 373 ret = trace_probe_add_file(tp, file); 374 if (ret) 375 return ret; 376 } else 377 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 378 379 if (enabled) 380 return 0; 381 382 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 383 if (trace_kprobe_has_gone(tk)) 384 continue; 385 ret = __enable_trace_kprobe(tk); 386 if (ret) 387 break; 388 enabled = true; 389 } 390 391 if (ret) { 392 /* Failed to enable one of them. Roll back all */ 393 if (enabled) 394 __disable_trace_kprobe(tp); 395 if (file) 396 trace_probe_remove_file(tp, file); 397 else 398 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 399 } 400 401 return ret; 402 } 403 404 /* 405 * Disable trace_probe 406 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 407 */ 408 static int disable_trace_kprobe(struct trace_event_call *call, 409 struct trace_event_file *file) 410 { 411 struct trace_probe *tp; 412 413 tp = trace_probe_primary_from_call(call); 414 if (WARN_ON_ONCE(!tp)) 415 return -ENODEV; 416 417 if (file) { 418 if (!trace_probe_get_file_link(tp, file)) 419 return -ENOENT; 420 if (!trace_probe_has_single_file(tp)) 421 goto out; 422 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 423 } else 424 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 425 426 if (!trace_probe_is_enabled(tp)) 427 __disable_trace_kprobe(tp); 428 429 out: 430 if (file) 431 /* 432 * Synchronization is done in below function. For perf event, 433 * file == NULL and perf_trace_event_unreg() calls 434 * tracepoint_synchronize_unregister() to ensure synchronize 435 * event. We don't need to care about it. 436 */ 437 trace_probe_remove_file(tp, file); 438 439 return 0; 440 } 441 442 #if defined(CONFIG_DYNAMIC_FTRACE) && \ 443 !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE) 444 static bool __within_notrace_func(unsigned long addr) 445 { 446 unsigned long offset, size; 447 448 if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset)) 449 return false; 450 451 /* Get the entry address of the target function */ 452 addr -= offset; 453 454 /* 455 * Since ftrace_location_range() does inclusive range check, we need 456 * to subtract 1 byte from the end address. 457 */ 458 return !ftrace_location_range(addr, addr + size - 1); 459 } 460 461 static bool within_notrace_func(struct trace_kprobe *tk) 462 { 463 unsigned long addr = trace_kprobe_address(tk); 464 char symname[KSYM_NAME_LEN], *p; 465 466 if (!__within_notrace_func(addr)) 467 return false; 468 469 /* Check if the address is on a suffixed-symbol */ 470 if (!lookup_symbol_name(addr, symname)) { 471 p = strchr(symname, '.'); 472 if (!p) 473 return true; 474 *p = '\0'; 475 addr = (unsigned long)kprobe_lookup_name(symname, 0); 476 if (addr) 477 return __within_notrace_func(addr); 478 } 479 480 return true; 481 } 482 #else 483 #define within_notrace_func(tk) (false) 484 #endif 485 486 /* Internal register function - just handle k*probes and flags */ 487 static int __register_trace_kprobe(struct trace_kprobe *tk) 488 { 489 int i, ret; 490 491 ret = security_locked_down(LOCKDOWN_KPROBES); 492 if (ret) 493 return ret; 494 495 if (trace_kprobe_is_registered(tk)) 496 return -EINVAL; 497 498 if (within_notrace_func(tk)) { 499 pr_warn("Could not probe notrace function %ps\n", 500 (void *)trace_kprobe_address(tk)); 501 return -EINVAL; 502 } 503 504 for (i = 0; i < tk->tp.nr_args; i++) { 505 ret = traceprobe_update_arg(&tk->tp.args[i]); 506 if (ret) 507 return ret; 508 } 509 510 /* Set/clear disabled flag according to tp->flag */ 511 if (trace_probe_is_enabled(&tk->tp)) 512 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; 513 else 514 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; 515 516 if (trace_kprobe_is_return(tk)) 517 ret = register_kretprobe(&tk->rp); 518 else 519 ret = register_kprobe(&tk->rp.kp); 520 521 return ret; 522 } 523 524 /* Internal unregister function - just handle k*probes and flags */ 525 static void __unregister_trace_kprobe(struct trace_kprobe *tk) 526 { 527 if (trace_kprobe_is_registered(tk)) { 528 if (trace_kprobe_is_return(tk)) 529 unregister_kretprobe(&tk->rp); 530 else 531 unregister_kprobe(&tk->rp.kp); 532 /* Cleanup kprobe for reuse and mark it unregistered */ 533 INIT_HLIST_NODE(&tk->rp.kp.hlist); 534 INIT_LIST_HEAD(&tk->rp.kp.list); 535 if (tk->rp.kp.symbol_name) 536 tk->rp.kp.addr = NULL; 537 } 538 } 539 540 /* Unregister a trace_probe and probe_event */ 541 static int unregister_trace_kprobe(struct trace_kprobe *tk) 542 { 543 /* If other probes are on the event, just unregister kprobe */ 544 if (trace_probe_has_sibling(&tk->tp)) 545 goto unreg; 546 547 /* Enabled event can not be unregistered */ 548 if (trace_probe_is_enabled(&tk->tp)) 549 return -EBUSY; 550 551 /* If there's a reference to the dynamic event */ 552 if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp))) 553 return -EBUSY; 554 555 /* Will fail if probe is being used by ftrace or perf */ 556 if (unregister_kprobe_event(tk)) 557 return -EBUSY; 558 559 unreg: 560 __unregister_trace_kprobe(tk); 561 dyn_event_remove(&tk->devent); 562 trace_probe_unlink(&tk->tp); 563 564 return 0; 565 } 566 567 static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig, 568 struct trace_kprobe *comp) 569 { 570 struct trace_probe_event *tpe = orig->tp.event; 571 int i; 572 573 list_for_each_entry(orig, &tpe->probes, tp.list) { 574 if (strcmp(trace_kprobe_symbol(orig), 575 trace_kprobe_symbol(comp)) || 576 trace_kprobe_offset(orig) != trace_kprobe_offset(comp)) 577 continue; 578 579 /* 580 * trace_probe_compare_arg_type() ensured that nr_args and 581 * each argument name and type are same. Let's compare comm. 582 */ 583 for (i = 0; i < orig->tp.nr_args; i++) { 584 if (strcmp(orig->tp.args[i].comm, 585 comp->tp.args[i].comm)) 586 break; 587 } 588 589 if (i == orig->tp.nr_args) 590 return true; 591 } 592 593 return false; 594 } 595 596 static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to) 597 { 598 int ret; 599 600 ret = trace_probe_compare_arg_type(&tk->tp, &to->tp); 601 if (ret) { 602 /* Note that argument starts index = 2 */ 603 trace_probe_log_set_index(ret + 1); 604 trace_probe_log_err(0, DIFF_ARG_TYPE); 605 return -EEXIST; 606 } 607 if (trace_kprobe_has_same_kprobe(to, tk)) { 608 trace_probe_log_set_index(0); 609 trace_probe_log_err(0, SAME_PROBE); 610 return -EEXIST; 611 } 612 613 /* Append to existing event */ 614 ret = trace_probe_append(&tk->tp, &to->tp); 615 if (ret) 616 return ret; 617 618 /* Register k*probe */ 619 ret = __register_trace_kprobe(tk); 620 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 621 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 622 ret = 0; 623 } 624 625 if (ret) 626 trace_probe_unlink(&tk->tp); 627 else 628 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 629 630 return ret; 631 } 632 633 /* Register a trace_probe and probe_event */ 634 static int register_trace_kprobe(struct trace_kprobe *tk) 635 { 636 struct trace_kprobe *old_tk; 637 int ret; 638 639 guard(mutex)(&event_mutex); 640 641 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), 642 trace_probe_group_name(&tk->tp)); 643 if (old_tk) { 644 if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) { 645 trace_probe_log_set_index(0); 646 trace_probe_log_err(0, DIFF_PROBE_TYPE); 647 return -EEXIST; 648 } 649 return append_trace_kprobe(tk, old_tk); 650 } 651 652 /* Register new event */ 653 ret = register_kprobe_event(tk); 654 if (ret) { 655 if (ret == -EEXIST) { 656 trace_probe_log_set_index(0); 657 trace_probe_log_err(0, EVENT_EXIST); 658 } else 659 pr_warn("Failed to register probe event(%d)\n", ret); 660 return ret; 661 } 662 663 /* Register k*probe */ 664 ret = __register_trace_kprobe(tk); 665 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 666 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 667 ret = 0; 668 } 669 670 if (ret < 0) 671 unregister_kprobe_event(tk); 672 else 673 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 674 675 return ret; 676 } 677 678 #ifdef CONFIG_MODULES 679 static int validate_module_probe_symbol(const char *modname, const char *symbol); 680 681 static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk) 682 { 683 const char *p; 684 int ret = 0; 685 686 p = strchr(trace_kprobe_symbol(tk), ':'); 687 if (p) 688 ret = validate_module_probe_symbol(module_name(mod), p + 1); 689 if (!ret) 690 ret = __register_trace_kprobe(tk); 691 return ret; 692 } 693 694 /* Module notifier call back, checking event on the module */ 695 static int trace_kprobe_module_callback(struct notifier_block *nb, 696 unsigned long val, void *data) 697 { 698 struct module *mod = data; 699 struct dyn_event *pos; 700 struct trace_kprobe *tk; 701 int ret; 702 703 if (val != MODULE_STATE_COMING) 704 return NOTIFY_DONE; 705 706 /* Update probes on coming module */ 707 guard(mutex)(&event_mutex); 708 for_each_trace_kprobe(tk, pos) { 709 if (trace_kprobe_within_module(tk, mod)) { 710 /* Don't need to check busy - this should have gone. */ 711 __unregister_trace_kprobe(tk); 712 ret = register_module_trace_kprobe(mod, tk); 713 if (ret) 714 pr_warn("Failed to re-register probe %s on %s: %d\n", 715 trace_probe_name(&tk->tp), 716 module_name(mod), ret); 717 } 718 } 719 720 return NOTIFY_DONE; 721 } 722 723 static struct notifier_block trace_kprobe_module_nb = { 724 .notifier_call = trace_kprobe_module_callback, 725 .priority = 2 /* Invoked after kprobe and jump_label module callback */ 726 }; 727 static int trace_kprobe_register_module_notifier(void) 728 { 729 return register_module_notifier(&trace_kprobe_module_nb); 730 } 731 #else 732 static int trace_kprobe_register_module_notifier(void) 733 { 734 return 0; 735 } 736 #endif /* CONFIG_MODULES */ 737 738 static int count_symbols(void *data, unsigned long unused) 739 { 740 unsigned int *count = data; 741 742 (*count)++; 743 744 return 0; 745 } 746 747 struct sym_count_ctx { 748 unsigned int count; 749 const char *name; 750 }; 751 752 static int count_mod_symbols(void *data, const char *name, unsigned long unused) 753 { 754 struct sym_count_ctx *ctx = data; 755 756 if (strcmp(name, ctx->name) == 0) 757 ctx->count++; 758 759 return 0; 760 } 761 762 static unsigned int number_of_same_symbols(const char *mod, const char *func_name) 763 { 764 struct sym_count_ctx ctx = { .count = 0, .name = func_name }; 765 766 if (!mod) 767 kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count); 768 769 /* 770 * If the symbol is found in vmlinux, use vmlinux resolution only. 771 * This prevents module symbols from shadowing vmlinux symbols 772 * and causing -EADDRNOTAVAIL for unqualified kprobe targets. 773 */ 774 if (!mod && ctx.count > 0) 775 return ctx.count; 776 777 module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx); 778 779 return ctx.count; 780 } 781 782 static int validate_module_probe_symbol(const char *modname, const char *symbol) 783 { 784 unsigned int count = number_of_same_symbols(modname, symbol); 785 786 if (count > 1) { 787 /* 788 * Users should use ADDR to remove the ambiguity of 789 * using KSYM only. 790 */ 791 return -EADDRNOTAVAIL; 792 } else if (count == 0) { 793 /* 794 * We can return ENOENT earlier than when register the 795 * kprobe. 796 */ 797 return -ENOENT; 798 } 799 return 0; 800 } 801 802 #ifdef CONFIG_MODULES 803 /* Return NULL if the module is not loaded or under unloading. */ 804 static struct module *try_module_get_by_name(const char *name) 805 { 806 struct module *mod; 807 808 guard(rcu)(); 809 mod = find_module(name); 810 if (mod && !try_module_get(mod)) 811 mod = NULL; 812 return mod; 813 } 814 #else 815 #define try_module_get_by_name(name) (NULL) 816 #endif 817 818 static int validate_probe_symbol(char *symbol) 819 { 820 struct module *mod = NULL; 821 char *modname = NULL, *p; 822 int ret = 0; 823 824 p = strchr(symbol, ':'); 825 if (p) { 826 modname = symbol; 827 symbol = p + 1; 828 *p = '\0'; 829 mod = try_module_get_by_name(modname); 830 if (!mod) 831 goto out; 832 } 833 834 ret = validate_module_probe_symbol(modname, symbol); 835 out: 836 if (p) 837 *p = ':'; 838 if (mod) 839 module_put(mod); 840 return ret; 841 } 842 843 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 844 struct pt_regs *regs); 845 846 static int trace_kprobe_create_internal(int argc, const char *argv[], 847 struct traceprobe_parse_context *ctx) 848 { 849 /* 850 * Argument syntax: 851 * - Add kprobe: 852 * p[:[GRP/][EVENT]] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS] 853 * - Add kretprobe: 854 * r[MAXACTIVE][:[GRP/][EVENT]] [MOD:]KSYM[+0] [FETCHARGS] 855 * Or 856 * p[:[GRP/][EVENT]] [MOD:]KSYM[+0]%return [FETCHARGS] 857 * 858 * Fetch args: 859 * $retval : fetch return value 860 * $stack : fetch stack address 861 * $stackN : fetch Nth of stack (N:0-) 862 * $comm : fetch current task comm 863 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 864 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 865 * %REG : fetch register REG 866 * Dereferencing memory fetch: 867 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 868 * Alias name of args: 869 * NAME=FETCHARG : set NAME as alias of FETCHARG. 870 * Type of args: 871 * FETCHARG:TYPE : use TYPE instead of unsigned long. 872 */ 873 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 874 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM; 875 const char **new_argv __free(kfree) = NULL; 876 int i, len, new_argc = 0, ret = 0; 877 char *symbol __free(kfree) = NULL; 878 char *ebuf __free(kfree) = NULL; 879 char *gbuf __free(kfree) = NULL; 880 char *abuf __free(kfree) = NULL; 881 char *dbuf __free(kfree) = NULL; 882 enum probe_print_type ptype; 883 bool is_return = false; 884 int maxactive = 0; 885 void *addr = NULL; 886 char *tmp = NULL; 887 long offset = 0; 888 889 switch (argv[0][0]) { 890 case 'r': 891 is_return = true; 892 break; 893 case 'p': 894 break; 895 default: 896 return -ECANCELED; 897 } 898 if (argc < 2) 899 return -ECANCELED; 900 901 event = strchr(&argv[0][1], ':'); 902 if (event) 903 event++; 904 905 if (isdigit(argv[0][1])) { 906 char *buf __free(kfree) = NULL; 907 908 if (!is_return) { 909 trace_probe_log_err(1, BAD_MAXACT_TYPE); 910 return -EINVAL; 911 } 912 if (event) 913 len = event - &argv[0][1] - 1; 914 else 915 len = strlen(&argv[0][1]); 916 if (len > MAX_EVENT_NAME_LEN - 1) { 917 trace_probe_log_err(1, BAD_MAXACT); 918 return -EINVAL; 919 } 920 buf = kmemdup(&argv[0][1], len + 1, GFP_KERNEL); 921 if (!buf) 922 return -ENOMEM; 923 buf[len] = '\0'; 924 ret = kstrtouint(buf, 0, &maxactive); 925 if (ret || !maxactive) { 926 trace_probe_log_err(1, BAD_MAXACT); 927 return -EINVAL; 928 } 929 /* kretprobes instances are iterated over via a list. The 930 * maximum should stay reasonable. 931 */ 932 if (maxactive > KRETPROBE_MAXACTIVE_MAX) { 933 trace_probe_log_err(1, MAXACT_TOO_BIG); 934 return -EINVAL; 935 } 936 } 937 938 /* try to parse an address. if that fails, try to read the 939 * input as a symbol. */ 940 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { 941 trace_probe_log_set_index(1); 942 /* Check whether uprobe event specified */ 943 if (strchr(argv[1], '/') && strchr(argv[1], ':')) 944 return -ECANCELED; 945 946 /* a symbol specified */ 947 symbol = kstrdup(argv[1], GFP_KERNEL); 948 if (!symbol) 949 return -ENOMEM; 950 951 tmp = strchr(symbol, '%'); 952 if (tmp) { 953 if (!strcmp(tmp, "%return")) { 954 *tmp = '\0'; 955 is_return = true; 956 } else { 957 trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX); 958 return -EINVAL; 959 } 960 } 961 962 /* TODO: support .init module functions */ 963 ret = traceprobe_split_symbol_offset(symbol, &offset); 964 if (ret || offset < 0 || offset > UINT_MAX) { 965 trace_probe_log_err(0, BAD_PROBE_ADDR); 966 return -EINVAL; 967 } 968 ret = validate_probe_symbol(symbol); 969 if (ret) { 970 if (ret == -EADDRNOTAVAIL) 971 trace_probe_log_err(0, NON_UNIQ_SYMBOL); 972 else 973 trace_probe_log_err(0, BAD_PROBE_ADDR); 974 return -EINVAL; 975 } 976 if (is_return) 977 ctx->flags |= TPARG_FL_RETURN; 978 ret = kprobe_on_func_entry(NULL, symbol, offset); 979 if (ret == 0 && !is_return) 980 ctx->flags |= TPARG_FL_FENTRY; 981 /* Defer the ENOENT case until register kprobe */ 982 if (ret == -EINVAL && is_return) { 983 trace_probe_log_err(0, BAD_RETPROBE); 984 return -EINVAL; 985 } 986 } 987 988 trace_probe_log_set_index(0); 989 if (event) { 990 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 991 if (!gbuf) 992 return -ENOMEM; 993 ret = traceprobe_parse_event_name(&event, &group, gbuf, 994 event - argv[0]); 995 if (ret) 996 return ret; 997 } 998 999 if (!event) { 1000 /* Make a new event name */ 1001 ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 1002 if (!ebuf) 1003 return -ENOMEM; 1004 if (symbol) 1005 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 1006 is_return ? 'r' : 'p', symbol, offset); 1007 else 1008 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_0x%p", 1009 is_return ? 'r' : 'p', addr); 1010 sanitize_event_name(ebuf); 1011 event = ebuf; 1012 } 1013 1014 abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL); 1015 if (!abuf) 1016 return -ENOMEM; 1017 argc -= 2; argv += 2; 1018 ctx->funcname = symbol; 1019 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, 1020 abuf, MAX_BTF_ARGS_LEN, ctx); 1021 if (IS_ERR(new_argv)) { 1022 ret = PTR_ERR(new_argv); 1023 new_argv = NULL; 1024 return ret; 1025 } 1026 if (new_argv) { 1027 argc = new_argc; 1028 argv = new_argv; 1029 } 1030 if (argc > MAX_TRACE_ARGS) { 1031 trace_probe_log_set_index(2); 1032 trace_probe_log_err(0, TOO_MANY_ARGS); 1033 return -E2BIG; 1034 } 1035 1036 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); 1037 if (ret) 1038 return ret; 1039 1040 /* setup a probe */ 1041 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, 1042 argc, is_return); 1043 if (IS_ERR(tk)) { 1044 ret = PTR_ERR(tk); 1045 /* This must return -ENOMEM, else there is a bug */ 1046 WARN_ON_ONCE(ret != -ENOMEM); 1047 return ret; /* We know tk is not allocated */ 1048 } 1049 1050 /* parse arguments */ 1051 for (i = 0; i < argc; i++) { 1052 trace_probe_log_set_index(i + 2); 1053 ctx->offset = 0; 1054 ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], ctx); 1055 if (ret) 1056 return ret; /* This can be -ENOMEM */ 1057 } 1058 /* entry handler for kretprobe */ 1059 if (is_return && tk->tp.entry_arg) { 1060 tk->rp.entry_handler = trace_kprobe_entry_handler; 1061 tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp); 1062 } 1063 1064 ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1065 ret = traceprobe_set_print_fmt(&tk->tp, ptype); 1066 if (ret < 0) 1067 return ret; 1068 1069 ret = register_trace_kprobe(tk); 1070 if (ret) { 1071 trace_probe_log_set_index(1); 1072 if (ret == -EILSEQ) 1073 trace_probe_log_err(0, BAD_INSN_BNDRY); 1074 else if (ret == -ENOENT) 1075 trace_probe_log_err(0, BAD_PROBE_ADDR); 1076 else if (ret != -ENOMEM && ret != -EEXIST) 1077 trace_probe_log_err(0, FAIL_REG_PROBE); 1078 return ret; 1079 } 1080 /* 1081 * Here, 'tk' has been registered to the list successfully, 1082 * so we don't need to free it. 1083 */ 1084 tk = NULL; 1085 1086 return 0; 1087 } 1088 1089 static int trace_kprobe_create_cb(int argc, const char *argv[]) 1090 { 1091 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 1092 int ret; 1093 1094 ctx = kzalloc_obj(*ctx); 1095 if (!ctx) 1096 return -ENOMEM; 1097 ctx->flags = TPARG_FL_KERNEL; 1098 1099 trace_probe_log_init("trace_kprobe", argc, argv); 1100 1101 ret = trace_kprobe_create_internal(argc, argv, ctx); 1102 1103 trace_probe_log_clear(); 1104 return ret; 1105 } 1106 1107 static int trace_kprobe_create(const char *raw_command) 1108 { 1109 return trace_probe_create(raw_command, trace_kprobe_create_cb); 1110 } 1111 1112 static int create_or_delete_trace_kprobe(const char *raw_command) 1113 { 1114 int ret; 1115 1116 if (raw_command[0] == '-') 1117 return dyn_event_release(raw_command, &trace_kprobe_ops); 1118 1119 ret = dyn_event_create(raw_command, &trace_kprobe_ops); 1120 return ret == -ECANCELED ? -EINVAL : ret; 1121 } 1122 1123 static int trace_kprobe_run_command(struct dynevent_cmd *cmd) 1124 { 1125 return create_or_delete_trace_kprobe(cmd->seq.buffer); 1126 } 1127 1128 /** 1129 * kprobe_event_cmd_init - Initialize a kprobe event command object 1130 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1131 * @buf: A pointer to the buffer used to build the command 1132 * @maxlen: The length of the buffer passed in @buf 1133 * 1134 * Initialize a synthetic event command object. Use this before 1135 * calling any of the other kprobe_event functions. 1136 */ 1137 void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen) 1138 { 1139 dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE, 1140 trace_kprobe_run_command); 1141 } 1142 EXPORT_SYMBOL_GPL(kprobe_event_cmd_init); 1143 1144 /** 1145 * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list 1146 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1147 * @kretprobe: Is this a return probe? 1148 * @name: The name of the kprobe event 1149 * @loc: The location of the kprobe event 1150 * @...: Variable number of arg (pairs), one pair for each field 1151 * 1152 * NOTE: Users normally won't want to call this function directly, but 1153 * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically 1154 * adds a NULL to the end of the arg list. If this function is used 1155 * directly, make sure the last arg in the variable arg list is NULL. 1156 * 1157 * Generate a kprobe event command to be executed by 1158 * kprobe_event_gen_cmd_end(). This function can be used to generate the 1159 * complete command or only the first part of it; in the latter case, 1160 * kprobe_event_add_fields() can be used to add more fields following this. 1161 * 1162 * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This 1163 * returns -EINVAL if @loc == NULL. 1164 * 1165 * Return: 0 if successful, error otherwise. 1166 */ 1167 int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe, 1168 const char *name, const char *loc, ...) 1169 { 1170 char buf[MAX_EVENT_NAME_LEN]; 1171 struct dynevent_arg arg; 1172 va_list args; 1173 int ret; 1174 1175 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1176 return -EINVAL; 1177 1178 if (!loc) 1179 return -EINVAL; 1180 1181 if (kretprobe) 1182 snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name); 1183 else 1184 snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name); 1185 1186 ret = dynevent_str_add(cmd, buf); 1187 if (ret) 1188 return ret; 1189 1190 dynevent_arg_init(&arg, 0); 1191 arg.str = loc; 1192 ret = dynevent_arg_add(cmd, &arg, NULL); 1193 if (ret) 1194 return ret; 1195 1196 va_start(args, loc); 1197 for (;;) { 1198 const char *field; 1199 1200 field = va_arg(args, const char *); 1201 if (!field) 1202 break; 1203 1204 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1205 ret = -EINVAL; 1206 break; 1207 } 1208 1209 arg.str = field; 1210 ret = dynevent_arg_add(cmd, &arg, NULL); 1211 if (ret) 1212 break; 1213 } 1214 va_end(args); 1215 1216 return ret; 1217 } 1218 EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start); 1219 1220 /** 1221 * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list 1222 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1223 * @...: Variable number of arg (pairs), one pair for each field 1224 * 1225 * NOTE: Users normally won't want to call this function directly, but 1226 * rather use the kprobe_event_add_fields() wrapper, which 1227 * automatically adds a NULL to the end of the arg list. If this 1228 * function is used directly, make sure the last arg in the variable 1229 * arg list is NULL. 1230 * 1231 * Add probe fields to an existing kprobe command using a variable 1232 * list of args. Fields are added in the same order they're listed. 1233 * 1234 * Return: 0 if successful, error otherwise. 1235 */ 1236 int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...) 1237 { 1238 struct dynevent_arg arg; 1239 va_list args; 1240 int ret = 0; 1241 1242 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1243 return -EINVAL; 1244 1245 dynevent_arg_init(&arg, 0); 1246 1247 va_start(args, cmd); 1248 for (;;) { 1249 const char *field; 1250 1251 field = va_arg(args, const char *); 1252 if (!field) 1253 break; 1254 1255 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1256 ret = -EINVAL; 1257 break; 1258 } 1259 1260 arg.str = field; 1261 ret = dynevent_arg_add(cmd, &arg, NULL); 1262 if (ret) 1263 break; 1264 } 1265 va_end(args); 1266 1267 return ret; 1268 } 1269 EXPORT_SYMBOL_GPL(__kprobe_event_add_fields); 1270 1271 /** 1272 * kprobe_event_delete - Delete a kprobe event 1273 * @name: The name of the kprobe event to delete 1274 * 1275 * Delete a kprobe event with the give @name from kernel code rather 1276 * than directly from the command line. 1277 * 1278 * Return: 0 if successful, error otherwise. 1279 */ 1280 int kprobe_event_delete(const char *name) 1281 { 1282 char buf[MAX_EVENT_NAME_LEN]; 1283 1284 snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name); 1285 1286 return create_or_delete_trace_kprobe(buf); 1287 } 1288 EXPORT_SYMBOL_GPL(kprobe_event_delete); 1289 1290 static int trace_kprobe_release(struct dyn_event *ev) 1291 { 1292 struct trace_kprobe *tk = to_trace_kprobe(ev); 1293 int ret = unregister_trace_kprobe(tk); 1294 1295 if (!ret) 1296 free_trace_kprobe(tk); 1297 return ret; 1298 } 1299 1300 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) 1301 { 1302 struct trace_kprobe *tk = to_trace_kprobe(ev); 1303 int i; 1304 1305 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); 1306 if (trace_kprobe_is_return(tk) && tk->rp.maxactive) 1307 seq_printf(m, "%d", tk->rp.maxactive); 1308 seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp), 1309 trace_probe_name(&tk->tp)); 1310 1311 if (!tk->symbol) 1312 seq_printf(m, " 0x%p", tk->rp.kp.addr); 1313 else if (tk->rp.kp.offset) 1314 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk), 1315 tk->rp.kp.offset); 1316 else 1317 seq_printf(m, " %s", trace_kprobe_symbol(tk)); 1318 1319 for (i = 0; i < tk->tp.nr_args; i++) 1320 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); 1321 seq_putc(m, '\n'); 1322 1323 trace_probe_dump_args(m, &tk->tp); 1324 1325 return 0; 1326 } 1327 1328 static int probes_seq_show(struct seq_file *m, void *v) 1329 { 1330 struct dyn_event *ev = v; 1331 1332 if (!is_trace_kprobe(ev)) 1333 return 0; 1334 1335 return trace_kprobe_show(m, ev); 1336 } 1337 1338 static const struct seq_operations probes_seq_op = { 1339 .start = dyn_event_seq_start, 1340 .next = dyn_event_seq_next, 1341 .stop = dyn_event_seq_stop, 1342 .show = probes_seq_show 1343 }; 1344 1345 static int probes_open(struct inode *inode, struct file *file) 1346 { 1347 int ret; 1348 1349 ret = security_locked_down(LOCKDOWN_TRACEFS); 1350 if (ret) 1351 return ret; 1352 1353 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1354 ret = dyn_events_release_all(&trace_kprobe_ops); 1355 if (ret < 0) 1356 return ret; 1357 } 1358 1359 return seq_open(file, &probes_seq_op); 1360 } 1361 1362 static ssize_t probes_write(struct file *file, const char __user *buffer, 1363 size_t count, loff_t *ppos) 1364 { 1365 return trace_parse_run_command(file, buffer, count, ppos, 1366 create_or_delete_trace_kprobe); 1367 } 1368 1369 static const struct file_operations kprobe_events_ops = { 1370 .owner = THIS_MODULE, 1371 .open = probes_open, 1372 .read = seq_read, 1373 .llseek = seq_lseek, 1374 .release = seq_release, 1375 .write = probes_write, 1376 }; 1377 1378 static unsigned long trace_kprobe_missed(struct trace_kprobe *tk) 1379 { 1380 return trace_kprobe_is_return(tk) ? 1381 tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed; 1382 } 1383 1384 /* Probes profiling interfaces */ 1385 static int probes_profile_seq_show(struct seq_file *m, void *v) 1386 { 1387 struct dyn_event *ev = v; 1388 struct trace_kprobe *tk; 1389 unsigned long nmissed; 1390 1391 if (!is_trace_kprobe(ev)) 1392 return 0; 1393 1394 tk = to_trace_kprobe(ev); 1395 nmissed = trace_kprobe_missed(tk); 1396 seq_printf(m, " %-44s %15lu %15lu\n", 1397 trace_probe_name(&tk->tp), 1398 trace_kprobe_nhit(tk), 1399 nmissed); 1400 1401 return 0; 1402 } 1403 1404 static const struct seq_operations profile_seq_op = { 1405 .start = dyn_event_seq_start, 1406 .next = dyn_event_seq_next, 1407 .stop = dyn_event_seq_stop, 1408 .show = probes_profile_seq_show 1409 }; 1410 1411 static int profile_open(struct inode *inode, struct file *file) 1412 { 1413 int ret; 1414 1415 ret = security_locked_down(LOCKDOWN_TRACEFS); 1416 if (ret) 1417 return ret; 1418 1419 return seq_open(file, &profile_seq_op); 1420 } 1421 1422 static const struct file_operations kprobe_profile_ops = { 1423 .owner = THIS_MODULE, 1424 .open = profile_open, 1425 .read = seq_read, 1426 .llseek = seq_lseek, 1427 .release = seq_release, 1428 }; 1429 1430 /* Note that we don't verify it, since the code does not come from user space */ 1431 static int 1432 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 1433 void *dest, void *base) 1434 { 1435 struct pt_regs *regs = rec; 1436 unsigned long val; 1437 int ret; 1438 1439 retry: 1440 /* 1st stage: get value from context */ 1441 switch (code->op) { 1442 case FETCH_OP_REG: 1443 val = regs_get_register(regs, code->param); 1444 break; 1445 case FETCH_OP_STACK: 1446 val = regs_get_kernel_stack_nth(regs, code->param); 1447 break; 1448 case FETCH_OP_STACKP: 1449 val = kernel_stack_pointer(regs); 1450 break; 1451 case FETCH_OP_RETVAL: 1452 val = regs_return_value(regs); 1453 break; 1454 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 1455 case FETCH_OP_ARG: 1456 val = regs_get_kernel_argument(regs, code->param); 1457 break; 1458 case FETCH_OP_EDATA: 1459 val = *(unsigned long *)((unsigned long)edata + code->offset); 1460 break; 1461 #endif 1462 case FETCH_NOP_SYMBOL: /* Ignore a place holder */ 1463 code++; 1464 goto retry; 1465 default: 1466 ret = process_common_fetch_insn(code, &val); 1467 if (ret < 0) 1468 return ret; 1469 } 1470 code++; 1471 1472 return process_fetch_insn_bottom(code, val, dest, base); 1473 } 1474 NOKPROBE_SYMBOL(process_fetch_insn) 1475 1476 /* Kprobe handler */ 1477 static nokprobe_inline void 1478 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 1479 struct trace_event_file *trace_file) 1480 { 1481 struct kprobe_trace_entry_head *entry; 1482 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1483 struct trace_event_buffer fbuffer; 1484 int dsize; 1485 1486 WARN_ON(call != trace_file->event_call); 1487 1488 if (trace_trigger_soft_disabled(trace_file)) 1489 return; 1490 1491 dsize = __get_data_size(&tk->tp, regs, NULL); 1492 1493 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1494 sizeof(*entry) + tk->tp.size + dsize); 1495 if (!entry) 1496 return; 1497 1498 fbuffer.regs = regs; 1499 entry->ip = (unsigned long)tk->rp.kp.addr; 1500 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1501 1502 trace_event_buffer_commit(&fbuffer); 1503 } 1504 1505 static void 1506 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) 1507 { 1508 struct event_file_link *link; 1509 1510 trace_probe_for_each_link_rcu(link, &tk->tp) 1511 __kprobe_trace_func(tk, regs, link->file); 1512 } 1513 NOKPROBE_SYMBOL(kprobe_trace_func); 1514 1515 /* Kretprobe handler */ 1516 1517 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 1518 struct pt_regs *regs) 1519 { 1520 struct kretprobe *rp = get_kretprobe(ri); 1521 struct trace_kprobe *tk; 1522 1523 /* 1524 * There is a small chance that get_kretprobe(ri) returns NULL when 1525 * the kretprobe is unregister on another CPU between kretprobe's 1526 * trampoline_handler and this function. 1527 */ 1528 if (unlikely(!rp)) 1529 return -ENOENT; 1530 1531 tk = container_of(rp, struct trace_kprobe, rp); 1532 1533 /* store argument values into ri->data as entry data */ 1534 if (tk->tp.entry_arg) 1535 store_trace_entry_data(ri->data, &tk->tp, regs); 1536 1537 return 0; 1538 } 1539 1540 1541 static nokprobe_inline void 1542 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1543 struct pt_regs *regs, 1544 struct trace_event_file *trace_file) 1545 { 1546 struct kretprobe_trace_entry_head *entry; 1547 struct trace_event_buffer fbuffer; 1548 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1549 int dsize; 1550 1551 WARN_ON(call != trace_file->event_call); 1552 1553 if (trace_trigger_soft_disabled(trace_file)) 1554 return; 1555 1556 dsize = __get_data_size(&tk->tp, regs, ri->data); 1557 1558 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1559 sizeof(*entry) + tk->tp.size + dsize); 1560 if (!entry) 1561 return; 1562 1563 fbuffer.regs = regs; 1564 entry->func = (unsigned long)tk->rp.kp.addr; 1565 entry->ret_ip = get_kretprobe_retaddr(ri); 1566 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1567 1568 trace_event_buffer_commit(&fbuffer); 1569 } 1570 1571 static void 1572 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1573 struct pt_regs *regs) 1574 { 1575 struct event_file_link *link; 1576 1577 trace_probe_for_each_link_rcu(link, &tk->tp) 1578 __kretprobe_trace_func(tk, ri, regs, link->file); 1579 } 1580 NOKPROBE_SYMBOL(kretprobe_trace_func); 1581 1582 /* Event entry printers */ 1583 static enum print_line_t 1584 print_kprobe_event(struct trace_iterator *iter, int flags, 1585 struct trace_event *event) 1586 { 1587 struct kprobe_trace_entry_head *field; 1588 struct trace_seq *s = &iter->seq; 1589 struct trace_probe *tp; 1590 1591 field = (struct kprobe_trace_entry_head *)iter->ent; 1592 tp = trace_probe_primary_from_call( 1593 container_of(event, struct trace_event_call, event)); 1594 if (WARN_ON_ONCE(!tp)) 1595 goto out; 1596 1597 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1598 1599 if (!seq_print_ip_sym_offset(s, field->ip, flags)) 1600 goto out; 1601 1602 trace_seq_putc(s, ')'); 1603 1604 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1605 (u8 *)&field[1], field) < 0) 1606 goto out; 1607 1608 trace_seq_putc(s, '\n'); 1609 out: 1610 return trace_handle_return(s); 1611 } 1612 1613 static enum print_line_t 1614 print_kretprobe_event(struct trace_iterator *iter, int flags, 1615 struct trace_event *event) 1616 { 1617 struct kretprobe_trace_entry_head *field; 1618 struct trace_seq *s = &iter->seq; 1619 struct trace_probe *tp; 1620 1621 field = (struct kretprobe_trace_entry_head *)iter->ent; 1622 tp = trace_probe_primary_from_call( 1623 container_of(event, struct trace_event_call, event)); 1624 if (WARN_ON_ONCE(!tp)) 1625 goto out; 1626 1627 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1628 1629 if (!seq_print_ip_sym_offset(s, field->ret_ip, flags)) 1630 goto out; 1631 1632 trace_seq_puts(s, " <- "); 1633 1634 if (!seq_print_ip_sym_no_offset(s, field->func, flags)) 1635 goto out; 1636 1637 trace_seq_putc(s, ')'); 1638 1639 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1640 (u8 *)&field[1], field) < 0) 1641 goto out; 1642 1643 trace_seq_putc(s, '\n'); 1644 1645 out: 1646 return trace_handle_return(s); 1647 } 1648 1649 1650 static int kprobe_event_define_fields(struct trace_event_call *event_call) 1651 { 1652 int ret; 1653 struct kprobe_trace_entry_head field; 1654 struct trace_probe *tp; 1655 1656 tp = trace_probe_primary_from_call(event_call); 1657 if (WARN_ON_ONCE(!tp)) 1658 return -ENOENT; 1659 1660 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1661 1662 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1663 } 1664 1665 static int kretprobe_event_define_fields(struct trace_event_call *event_call) 1666 { 1667 int ret; 1668 struct kretprobe_trace_entry_head field; 1669 struct trace_probe *tp; 1670 1671 tp = trace_probe_primary_from_call(event_call); 1672 if (WARN_ON_ONCE(!tp)) 1673 return -ENOENT; 1674 1675 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1676 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1677 1678 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1679 } 1680 1681 #ifdef CONFIG_PERF_EVENTS 1682 1683 /* Kprobe profile handler */ 1684 static int 1685 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) 1686 { 1687 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1688 struct kprobe_trace_entry_head *entry; 1689 struct hlist_head *head; 1690 int size, __size, dsize; 1691 int rctx; 1692 1693 if (bpf_prog_array_valid(call)) { 1694 unsigned long orig_ip = instruction_pointer(regs); 1695 int ret; 1696 1697 ret = trace_call_bpf(call, regs); 1698 1699 /* 1700 * We need to check and see if we modified the pc of the 1701 * pt_regs, and if so return 1 so that we don't do the 1702 * single stepping. 1703 */ 1704 if (orig_ip != instruction_pointer(regs)) 1705 return 1; 1706 if (!ret) 1707 return 0; 1708 } 1709 1710 head = this_cpu_ptr(call->perf_events); 1711 if (hlist_empty(head)) 1712 return 0; 1713 1714 dsize = __get_data_size(&tk->tp, regs, NULL); 1715 __size = sizeof(*entry) + tk->tp.size + dsize; 1716 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1717 size -= sizeof(u32); 1718 1719 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1720 if (!entry) 1721 return 0; 1722 1723 entry->ip = (unsigned long)tk->rp.kp.addr; 1724 memset(&entry[1], 0, dsize); 1725 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1726 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1727 head, NULL); 1728 return 0; 1729 } 1730 NOKPROBE_SYMBOL(kprobe_perf_func); 1731 1732 /* Kretprobe profile handler */ 1733 static void 1734 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1735 struct pt_regs *regs) 1736 { 1737 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1738 struct kretprobe_trace_entry_head *entry; 1739 struct hlist_head *head; 1740 int size, __size, dsize; 1741 int rctx; 1742 1743 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs)) 1744 return; 1745 1746 head = this_cpu_ptr(call->perf_events); 1747 if (hlist_empty(head)) 1748 return; 1749 1750 dsize = __get_data_size(&tk->tp, regs, ri->data); 1751 __size = sizeof(*entry) + tk->tp.size + dsize; 1752 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1753 size -= sizeof(u32); 1754 1755 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1756 if (!entry) 1757 return; 1758 1759 entry->func = (unsigned long)tk->rp.kp.addr; 1760 entry->ret_ip = get_kretprobe_retaddr(ri); 1761 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1762 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1763 head, NULL); 1764 } 1765 NOKPROBE_SYMBOL(kretprobe_perf_func); 1766 1767 int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, 1768 const char **symbol, u64 *probe_offset, 1769 u64 *probe_addr, unsigned long *missed, 1770 bool perf_type_tracepoint) 1771 { 1772 const char *pevent = trace_event_name(event->tp_event); 1773 const char *group = event->tp_event->class->system; 1774 struct trace_kprobe *tk; 1775 1776 if (perf_type_tracepoint) 1777 tk = find_trace_kprobe(pevent, group); 1778 else 1779 tk = trace_kprobe_primary_from_call(event->tp_event); 1780 if (!tk) 1781 return -EINVAL; 1782 1783 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE 1784 : BPF_FD_TYPE_KPROBE; 1785 *probe_offset = tk->rp.kp.offset; 1786 *probe_addr = kallsyms_show_value(current_cred()) ? 1787 (unsigned long)tk->rp.kp.addr : 0; 1788 *symbol = tk->symbol; 1789 if (missed) 1790 *missed = trace_kprobe_missed(tk); 1791 return 0; 1792 } 1793 #endif /* CONFIG_PERF_EVENTS */ 1794 1795 /* 1796 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1797 * 1798 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe 1799 * lockless, but we can't race with this __init function. 1800 */ 1801 static int kprobe_register(struct trace_event_call *event, 1802 enum trace_reg type, void *data) 1803 { 1804 struct trace_event_file *file = data; 1805 1806 switch (type) { 1807 case TRACE_REG_REGISTER: 1808 return enable_trace_kprobe(event, file); 1809 case TRACE_REG_UNREGISTER: 1810 return disable_trace_kprobe(event, file); 1811 1812 #ifdef CONFIG_PERF_EVENTS 1813 case TRACE_REG_PERF_REGISTER: 1814 return enable_trace_kprobe(event, NULL); 1815 case TRACE_REG_PERF_UNREGISTER: 1816 return disable_trace_kprobe(event, NULL); 1817 case TRACE_REG_PERF_OPEN: 1818 case TRACE_REG_PERF_CLOSE: 1819 case TRACE_REG_PERF_ADD: 1820 case TRACE_REG_PERF_DEL: 1821 return 0; 1822 #endif 1823 } 1824 return 0; 1825 } 1826 1827 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) 1828 { 1829 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); 1830 unsigned int flags = trace_probe_load_flag(&tk->tp); 1831 int ret = 0; 1832 1833 raw_cpu_inc(*tk->nhit); 1834 1835 if (flags & TP_FLAG_TRACE) 1836 kprobe_trace_func(tk, regs); 1837 #ifdef CONFIG_PERF_EVENTS 1838 if (flags & TP_FLAG_PROFILE) 1839 ret = kprobe_perf_func(tk, regs); 1840 #endif 1841 return ret; 1842 } 1843 NOKPROBE_SYMBOL(kprobe_dispatcher); 1844 1845 static int 1846 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) 1847 { 1848 struct kretprobe *rp = get_kretprobe(ri); 1849 struct trace_kprobe *tk; 1850 unsigned int flags; 1851 1852 /* 1853 * There is a small chance that get_kretprobe(ri) returns NULL when 1854 * the kretprobe is unregister on another CPU between kretprobe's 1855 * trampoline_handler and this function. 1856 */ 1857 if (unlikely(!rp)) 1858 return 0; 1859 1860 tk = container_of(rp, struct trace_kprobe, rp); 1861 raw_cpu_inc(*tk->nhit); 1862 1863 flags = trace_probe_load_flag(&tk->tp); 1864 if (flags & TP_FLAG_TRACE) 1865 kretprobe_trace_func(tk, ri, regs); 1866 #ifdef CONFIG_PERF_EVENTS 1867 if (flags & TP_FLAG_PROFILE) 1868 kretprobe_perf_func(tk, ri, regs); 1869 #endif 1870 return 0; /* We don't tweak kernel, so just return 0 */ 1871 } 1872 NOKPROBE_SYMBOL(kretprobe_dispatcher); 1873 1874 static struct trace_event_functions kretprobe_funcs = { 1875 .trace = print_kretprobe_event 1876 }; 1877 1878 static struct trace_event_functions kprobe_funcs = { 1879 .trace = print_kprobe_event 1880 }; 1881 1882 static struct trace_event_fields kretprobe_fields_array[] = { 1883 { .type = TRACE_FUNCTION_TYPE, 1884 .define_fields = kretprobe_event_define_fields }, 1885 {} 1886 }; 1887 1888 static struct trace_event_fields kprobe_fields_array[] = { 1889 { .type = TRACE_FUNCTION_TYPE, 1890 .define_fields = kprobe_event_define_fields }, 1891 {} 1892 }; 1893 1894 static inline void init_trace_event_call(struct trace_kprobe *tk) 1895 { 1896 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1897 1898 if (trace_kprobe_is_return(tk)) { 1899 call->event.funcs = &kretprobe_funcs; 1900 call->class->fields_array = kretprobe_fields_array; 1901 } else { 1902 call->event.funcs = &kprobe_funcs; 1903 call->class->fields_array = kprobe_fields_array; 1904 } 1905 1906 call->flags = TRACE_EVENT_FL_KPROBE; 1907 call->class->reg = kprobe_register; 1908 } 1909 1910 static int register_kprobe_event(struct trace_kprobe *tk) 1911 { 1912 init_trace_event_call(tk); 1913 1914 return trace_probe_register_event_call(&tk->tp); 1915 } 1916 1917 static int unregister_kprobe_event(struct trace_kprobe *tk) 1918 { 1919 return trace_probe_unregister_event_call(&tk->tp); 1920 } 1921 1922 #ifdef CONFIG_PERF_EVENTS 1923 1924 /* create a trace_kprobe, but don't add it to global lists */ 1925 struct trace_event_call * 1926 create_local_trace_kprobe(char *func, void *addr, unsigned long offs, 1927 bool is_return) 1928 { 1929 enum probe_print_type ptype; 1930 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 1931 int ret; 1932 char *event; 1933 1934 if (func) { 1935 ret = validate_probe_symbol(func); 1936 if (ret) 1937 return ERR_PTR(ret); 1938 } 1939 1940 /* 1941 * local trace_kprobes are not added to dyn_event, so they are never 1942 * searched in find_trace_kprobe(). Therefore, there is no concern of 1943 * duplicated name here. 1944 */ 1945 event = func ? func : "DUMMY_EVENT"; 1946 1947 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func, 1948 offs, 0 /* maxactive */, 0 /* nargs */, 1949 is_return); 1950 1951 if (IS_ERR(tk)) { 1952 pr_info("Failed to allocate trace_probe.(%d)\n", 1953 (int)PTR_ERR(tk)); 1954 return ERR_CAST(tk); 1955 } 1956 1957 init_trace_event_call(tk); 1958 1959 ptype = trace_kprobe_is_return(tk) ? 1960 PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1961 if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) 1962 return ERR_PTR(-ENOMEM); 1963 1964 ret = __register_trace_kprobe(tk); 1965 if (ret < 0) 1966 return ERR_PTR(ret); 1967 1968 return trace_probe_event_call(&(no_free_ptr(tk)->tp)); 1969 } 1970 1971 void destroy_local_trace_kprobe(struct trace_event_call *event_call) 1972 { 1973 struct trace_kprobe *tk; 1974 1975 tk = trace_kprobe_primary_from_call(event_call); 1976 if (unlikely(!tk)) 1977 return; 1978 1979 if (trace_probe_is_enabled(&tk->tp)) { 1980 WARN_ON(1); 1981 return; 1982 } 1983 1984 __unregister_trace_kprobe(tk); 1985 1986 free_trace_kprobe(tk); 1987 } 1988 #endif /* CONFIG_PERF_EVENTS */ 1989 1990 static __init void enable_boot_kprobe_events(void) 1991 { 1992 struct trace_array *tr = top_trace_array(); 1993 struct trace_event_file *file; 1994 struct trace_kprobe *tk; 1995 struct dyn_event *pos; 1996 1997 if (trace_kprobe_list_empty()) 1998 return; 1999 2000 guard(mutex)(&event_mutex); 2001 for_each_trace_kprobe(tk, pos) { 2002 list_for_each_entry(file, &tr->events, list) 2003 if (file->event_call == trace_probe_event_call(&tk->tp)) 2004 trace_event_enable_disable(file, 1, 0); 2005 } 2006 } 2007 2008 static __init void setup_boot_kprobe_events(void) 2009 { 2010 char *p, *cmd = kprobe_boot_events_buf; 2011 int ret; 2012 2013 strreplace(kprobe_boot_events_buf, ',', ' '); 2014 2015 while (cmd && *cmd != '\0') { 2016 p = strchr(cmd, ';'); 2017 if (p) 2018 *p++ = '\0'; 2019 2020 ret = create_or_delete_trace_kprobe(cmd); 2021 if (ret) 2022 pr_warn("Failed to add event(%d): %s\n", ret, cmd); 2023 2024 cmd = p; 2025 } 2026 2027 enable_boot_kprobe_events(); 2028 } 2029 2030 /* 2031 * Register dynevent at core_initcall. This allows kernel to setup kprobe 2032 * events in postcore_initcall without tracefs. 2033 */ 2034 static __init int init_kprobe_trace_early(void) 2035 { 2036 int ret; 2037 2038 ret = dyn_event_register(&trace_kprobe_ops); 2039 if (ret) 2040 return ret; 2041 2042 if (trace_kprobe_register_module_notifier()) 2043 return -EINVAL; 2044 2045 return 0; 2046 } 2047 core_initcall(init_kprobe_trace_early); 2048 2049 /* Make a tracefs interface for controlling probe points */ 2050 static __init int init_kprobe_trace(void) 2051 { 2052 int ret; 2053 2054 ret = tracing_init_dentry(); 2055 if (ret) 2056 return 0; 2057 2058 /* Event list interface */ 2059 trace_create_file("kprobe_events", TRACE_MODE_WRITE, 2060 NULL, NULL, &kprobe_events_ops); 2061 2062 /* Profile interface */ 2063 trace_create_file("kprobe_profile", TRACE_MODE_READ, 2064 NULL, NULL, &kprobe_profile_ops); 2065 2066 /* If no 'kprobe_event=' cmd is provided, return directly. */ 2067 if (kprobe_boot_events_buf[0] == '\0') 2068 return 0; 2069 2070 setup_boot_kprobe_events(); 2071 2072 return 0; 2073 } 2074 fs_initcall(init_kprobe_trace); 2075 2076 2077 #ifdef CONFIG_FTRACE_STARTUP_TEST 2078 static __init struct trace_event_file * 2079 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) 2080 { 2081 struct trace_event_file *file; 2082 2083 list_for_each_entry(file, &tr->events, list) 2084 if (file->event_call == trace_probe_event_call(&tk->tp)) 2085 return file; 2086 2087 return NULL; 2088 } 2089 2090 /* 2091 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this 2092 * stage, we can do this lockless. 2093 */ 2094 static __init int kprobe_trace_self_tests_init(void) 2095 { 2096 int ret, warn = 0; 2097 int (*target)(int, int, int, int, int, int); 2098 struct trace_kprobe *tk; 2099 struct trace_event_file *file; 2100 2101 if (unlikely(tracing_disabled)) 2102 return -ENODEV; 2103 2104 if (tracing_selftest_disabled) 2105 return 0; 2106 2107 target = kprobe_trace_selftest_target; 2108 2109 pr_info("Testing kprobe tracing: "); 2110 2111 ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)"); 2112 if (WARN_ONCE(ret, "error on probing function entry.")) { 2113 warn++; 2114 } else { 2115 /* Enable trace point */ 2116 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2117 if (WARN_ONCE(tk == NULL, "error on probing function entry.")) { 2118 warn++; 2119 } else { 2120 file = find_trace_probe_file(tk, top_trace_array()); 2121 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2122 warn++; 2123 } else 2124 enable_trace_kprobe( 2125 trace_probe_event_call(&tk->tp), file); 2126 } 2127 } 2128 2129 ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval"); 2130 if (WARN_ONCE(ret, "error on probing function return.")) { 2131 warn++; 2132 } else { 2133 /* Enable trace point */ 2134 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2135 if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) { 2136 warn++; 2137 } else { 2138 file = find_trace_probe_file(tk, top_trace_array()); 2139 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2140 warn++; 2141 } else 2142 enable_trace_kprobe( 2143 trace_probe_event_call(&tk->tp), file); 2144 } 2145 } 2146 2147 if (warn) 2148 goto end; 2149 2150 ret = target(1, 2, 3, 4, 5, 6); 2151 2152 /* 2153 * Not expecting an error here, the check is only to prevent the 2154 * optimizer from removing the call to target() as otherwise there 2155 * are no side-effects and the call is never performed. 2156 */ 2157 if (ret != 21) 2158 warn++; 2159 2160 /* Disable trace points before removing it */ 2161 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2162 if (WARN_ONCE(tk == NULL, "error on getting test probe.")) { 2163 warn++; 2164 } else { 2165 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2166 "incorrect number of testprobe hits.")) 2167 warn++; 2168 2169 file = find_trace_probe_file(tk, top_trace_array()); 2170 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2171 warn++; 2172 } else 2173 disable_trace_kprobe( 2174 trace_probe_event_call(&tk->tp), file); 2175 } 2176 2177 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2178 if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) { 2179 warn++; 2180 } else { 2181 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2182 "incorrect number of testprobe2 hits.")) 2183 warn++; 2184 2185 file = find_trace_probe_file(tk, top_trace_array()); 2186 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2187 warn++; 2188 } else 2189 disable_trace_kprobe( 2190 trace_probe_event_call(&tk->tp), file); 2191 } 2192 2193 ret = create_or_delete_trace_kprobe("-:testprobe"); 2194 if (WARN_ONCE(ret, "error on deleting a probe.")) 2195 warn++; 2196 2197 ret = create_or_delete_trace_kprobe("-:testprobe2"); 2198 if (WARN_ONCE(ret, "error on deleting a probe.")) 2199 warn++; 2200 2201 2202 end: 2203 /* 2204 * Wait for the optimizer work to finish. Otherwise it might fiddle 2205 * with probes in already freed __init text. 2206 */ 2207 wait_for_kprobe_optimizer(); 2208 if (warn) 2209 pr_cont("NG: Some tests are failed. Please check them.\n"); 2210 else 2211 pr_cont("OK\n"); 2212 return 0; 2213 } 2214 2215 late_initcall(kprobe_trace_self_tests_init); 2216 2217 #endif 2218