1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Kprobes-based tracing events 4 * 5 * Created by Masami Hiramatsu <mhiramat@redhat.com> 6 * 7 */ 8 #define pr_fmt(fmt) "trace_kprobe: " fmt 9 10 #include <linux/bpf-cgroup.h> 11 #include <linux/security.h> 12 #include <linux/module.h> 13 #include <linux/uaccess.h> 14 #include <linux/rculist.h> 15 #include <linux/error-injection.h> 16 17 #include <asm/setup.h> /* for COMMAND_LINE_SIZE */ 18 19 #include "trace_dynevent.h" 20 #include "trace_kprobe_selftest.h" 21 #include "trace_probe.h" 22 #include "trace_probe_tmpl.h" 23 #include "trace_probe_kernel.h" 24 25 #define KPROBE_EVENT_SYSTEM "kprobes" 26 #define KRETPROBE_MAXACTIVE_MAX 4096 27 28 /* Kprobe early definition from command line */ 29 static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata; 30 31 static int __init set_kprobe_boot_events(char *str) 32 { 33 strscpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE); 34 disable_tracing_selftest("running kprobe events"); 35 36 return 1; 37 } 38 __setup("kprobe_event=", set_kprobe_boot_events); 39 40 static int trace_kprobe_create(const char *raw_command); 41 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); 42 static int trace_kprobe_release(struct dyn_event *ev); 43 static bool trace_kprobe_is_busy(struct dyn_event *ev); 44 static bool trace_kprobe_match(const char *system, const char *event, 45 int argc, const char **argv, struct dyn_event *ev); 46 47 static struct dyn_event_operations trace_kprobe_ops = { 48 .create = trace_kprobe_create, 49 .show = trace_kprobe_show, 50 .is_busy = trace_kprobe_is_busy, 51 .free = trace_kprobe_release, 52 .match = trace_kprobe_match, 53 }; 54 55 /* 56 * Kprobe event core functions 57 */ 58 struct trace_kprobe { 59 struct dyn_event devent; 60 struct kretprobe rp; /* Use rp.kp for kprobe use */ 61 unsigned long __percpu *nhit; 62 const char *symbol; /* symbol name */ 63 struct trace_probe tp; 64 }; 65 66 static bool is_trace_kprobe(struct dyn_event *ev) 67 { 68 return ev->ops == &trace_kprobe_ops; 69 } 70 71 static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev) 72 { 73 return container_of(ev, struct trace_kprobe, devent); 74 } 75 76 /** 77 * for_each_trace_kprobe - iterate over the trace_kprobe list 78 * @pos: the struct trace_kprobe * for each entry 79 * @dpos: the struct dyn_event * to use as a loop cursor 80 */ 81 #define for_each_trace_kprobe(pos, dpos) \ 82 for_each_dyn_event(dpos) \ 83 if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) 84 85 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) 86 { 87 return tk->rp.handler != NULL; 88 } 89 90 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) 91 { 92 return tk->symbol ? tk->symbol : "unknown"; 93 } 94 95 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) 96 { 97 return tk->rp.kp.offset; 98 } 99 100 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) 101 { 102 return kprobe_gone(&tk->rp.kp); 103 } 104 105 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, 106 struct module *mod) 107 { 108 int len = strlen(module_name(mod)); 109 const char *name = trace_kprobe_symbol(tk); 110 111 return strncmp(module_name(mod), name, len) == 0 && name[len] == ':'; 112 } 113 114 #ifdef CONFIG_MODULES 115 static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 116 { 117 char *p; 118 bool ret; 119 120 if (!tk->symbol) 121 return false; 122 p = strchr(tk->symbol, ':'); 123 if (!p) 124 return true; 125 *p = '\0'; 126 rcu_read_lock_sched(); 127 ret = !!find_module(tk->symbol); 128 rcu_read_unlock_sched(); 129 *p = ':'; 130 131 return ret; 132 } 133 #else 134 static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 135 { 136 return false; 137 } 138 #endif 139 140 static bool trace_kprobe_is_busy(struct dyn_event *ev) 141 { 142 struct trace_kprobe *tk = to_trace_kprobe(ev); 143 144 return trace_probe_is_enabled(&tk->tp); 145 } 146 147 static bool trace_kprobe_match_command_head(struct trace_kprobe *tk, 148 int argc, const char **argv) 149 { 150 char buf[MAX_ARGSTR_LEN + 1]; 151 152 if (!argc) 153 return true; 154 155 if (!tk->symbol) 156 snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr); 157 else if (tk->rp.kp.offset) 158 snprintf(buf, sizeof(buf), "%s+%u", 159 trace_kprobe_symbol(tk), tk->rp.kp.offset); 160 else 161 snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk)); 162 if (strcmp(buf, argv[0])) 163 return false; 164 argc--; argv++; 165 166 return trace_probe_match_command_args(&tk->tp, argc, argv); 167 } 168 169 static bool trace_kprobe_match(const char *system, const char *event, 170 int argc, const char **argv, struct dyn_event *ev) 171 { 172 struct trace_kprobe *tk = to_trace_kprobe(ev); 173 174 return (event[0] == '\0' || 175 strcmp(trace_probe_name(&tk->tp), event) == 0) && 176 (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) && 177 trace_kprobe_match_command_head(tk, argc, argv); 178 } 179 180 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 181 { 182 unsigned long nhit = 0; 183 int cpu; 184 185 for_each_possible_cpu(cpu) 186 nhit += *per_cpu_ptr(tk->nhit, cpu); 187 188 return nhit; 189 } 190 191 static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk) 192 { 193 return !(list_empty(&tk->rp.kp.list) && 194 hlist_unhashed(&tk->rp.kp.hlist)); 195 } 196 197 /* Return 0 if it fails to find the symbol address */ 198 static nokprobe_inline 199 unsigned long trace_kprobe_address(struct trace_kprobe *tk) 200 { 201 unsigned long addr; 202 203 if (tk->symbol) { 204 addr = (unsigned long) 205 kallsyms_lookup_name(trace_kprobe_symbol(tk)); 206 if (addr) 207 addr += tk->rp.kp.offset; 208 } else { 209 addr = (unsigned long)tk->rp.kp.addr; 210 } 211 return addr; 212 } 213 214 static nokprobe_inline struct trace_kprobe * 215 trace_kprobe_primary_from_call(struct trace_event_call *call) 216 { 217 struct trace_probe *tp; 218 219 tp = trace_probe_primary_from_call(call); 220 if (WARN_ON_ONCE(!tp)) 221 return NULL; 222 223 return container_of(tp, struct trace_kprobe, tp); 224 } 225 226 bool trace_kprobe_on_func_entry(struct trace_event_call *call) 227 { 228 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 229 230 return tk ? (kprobe_on_func_entry(tk->rp.kp.addr, 231 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, 232 tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false; 233 } 234 235 bool trace_kprobe_error_injectable(struct trace_event_call *call) 236 { 237 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 238 239 return tk ? within_error_injection_list(trace_kprobe_address(tk)) : 240 false; 241 } 242 243 static int register_kprobe_event(struct trace_kprobe *tk); 244 static int unregister_kprobe_event(struct trace_kprobe *tk); 245 246 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 247 static int kretprobe_dispatcher(struct kretprobe_instance *ri, 248 struct pt_regs *regs); 249 250 static void free_trace_kprobe(struct trace_kprobe *tk) 251 { 252 if (tk) { 253 trace_probe_cleanup(&tk->tp); 254 kfree(tk->symbol); 255 free_percpu(tk->nhit); 256 kfree(tk); 257 } 258 } 259 260 /* 261 * Allocate new trace_probe and initialize it (including kprobes). 262 */ 263 static struct trace_kprobe *alloc_trace_kprobe(const char *group, 264 const char *event, 265 void *addr, 266 const char *symbol, 267 unsigned long offs, 268 int maxactive, 269 int nargs, bool is_return) 270 { 271 struct trace_kprobe *tk; 272 int ret = -ENOMEM; 273 274 tk = kzalloc(struct_size(tk, tp.args, nargs), GFP_KERNEL); 275 if (!tk) 276 return ERR_PTR(ret); 277 278 tk->nhit = alloc_percpu(unsigned long); 279 if (!tk->nhit) 280 goto error; 281 282 if (symbol) { 283 tk->symbol = kstrdup(symbol, GFP_KERNEL); 284 if (!tk->symbol) 285 goto error; 286 tk->rp.kp.symbol_name = tk->symbol; 287 tk->rp.kp.offset = offs; 288 } else 289 tk->rp.kp.addr = addr; 290 291 if (is_return) 292 tk->rp.handler = kretprobe_dispatcher; 293 else 294 tk->rp.kp.pre_handler = kprobe_dispatcher; 295 296 tk->rp.maxactive = maxactive; 297 INIT_HLIST_NODE(&tk->rp.kp.hlist); 298 INIT_LIST_HEAD(&tk->rp.kp.list); 299 300 ret = trace_probe_init(&tk->tp, event, group, false, nargs); 301 if (ret < 0) 302 goto error; 303 304 dyn_event_init(&tk->devent, &trace_kprobe_ops); 305 return tk; 306 error: 307 free_trace_kprobe(tk); 308 return ERR_PTR(ret); 309 } 310 311 static struct trace_kprobe *find_trace_kprobe(const char *event, 312 const char *group) 313 { 314 struct dyn_event *pos; 315 struct trace_kprobe *tk; 316 317 for_each_trace_kprobe(tk, pos) 318 if (strcmp(trace_probe_name(&tk->tp), event) == 0 && 319 strcmp(trace_probe_group_name(&tk->tp), group) == 0) 320 return tk; 321 return NULL; 322 } 323 324 static inline int __enable_trace_kprobe(struct trace_kprobe *tk) 325 { 326 int ret = 0; 327 328 if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) { 329 if (trace_kprobe_is_return(tk)) 330 ret = enable_kretprobe(&tk->rp); 331 else 332 ret = enable_kprobe(&tk->rp.kp); 333 } 334 335 return ret; 336 } 337 338 static void __disable_trace_kprobe(struct trace_probe *tp) 339 { 340 struct trace_kprobe *tk; 341 342 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 343 if (!trace_kprobe_is_registered(tk)) 344 continue; 345 if (trace_kprobe_is_return(tk)) 346 disable_kretprobe(&tk->rp); 347 else 348 disable_kprobe(&tk->rp.kp); 349 } 350 } 351 352 /* 353 * Enable trace_probe 354 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 355 */ 356 static int enable_trace_kprobe(struct trace_event_call *call, 357 struct trace_event_file *file) 358 { 359 struct trace_probe *tp; 360 struct trace_kprobe *tk; 361 bool enabled; 362 int ret = 0; 363 364 tp = trace_probe_primary_from_call(call); 365 if (WARN_ON_ONCE(!tp)) 366 return -ENODEV; 367 enabled = trace_probe_is_enabled(tp); 368 369 /* This also changes "enabled" state */ 370 if (file) { 371 ret = trace_probe_add_file(tp, file); 372 if (ret) 373 return ret; 374 } else 375 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 376 377 if (enabled) 378 return 0; 379 380 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 381 if (trace_kprobe_has_gone(tk)) 382 continue; 383 ret = __enable_trace_kprobe(tk); 384 if (ret) 385 break; 386 enabled = true; 387 } 388 389 if (ret) { 390 /* Failed to enable one of them. Roll back all */ 391 if (enabled) 392 __disable_trace_kprobe(tp); 393 if (file) 394 trace_probe_remove_file(tp, file); 395 else 396 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 397 } 398 399 return ret; 400 } 401 402 /* 403 * Disable trace_probe 404 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 405 */ 406 static int disable_trace_kprobe(struct trace_event_call *call, 407 struct trace_event_file *file) 408 { 409 struct trace_probe *tp; 410 411 tp = trace_probe_primary_from_call(call); 412 if (WARN_ON_ONCE(!tp)) 413 return -ENODEV; 414 415 if (file) { 416 if (!trace_probe_get_file_link(tp, file)) 417 return -ENOENT; 418 if (!trace_probe_has_single_file(tp)) 419 goto out; 420 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 421 } else 422 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 423 424 if (!trace_probe_is_enabled(tp)) 425 __disable_trace_kprobe(tp); 426 427 out: 428 if (file) 429 /* 430 * Synchronization is done in below function. For perf event, 431 * file == NULL and perf_trace_event_unreg() calls 432 * tracepoint_synchronize_unregister() to ensure synchronize 433 * event. We don't need to care about it. 434 */ 435 trace_probe_remove_file(tp, file); 436 437 return 0; 438 } 439 440 #if defined(CONFIG_DYNAMIC_FTRACE) && \ 441 !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE) 442 static bool __within_notrace_func(unsigned long addr) 443 { 444 unsigned long offset, size; 445 446 if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset)) 447 return false; 448 449 /* Get the entry address of the target function */ 450 addr -= offset; 451 452 /* 453 * Since ftrace_location_range() does inclusive range check, we need 454 * to subtract 1 byte from the end address. 455 */ 456 return !ftrace_location_range(addr, addr + size - 1); 457 } 458 459 static bool within_notrace_func(struct trace_kprobe *tk) 460 { 461 unsigned long addr = trace_kprobe_address(tk); 462 char symname[KSYM_NAME_LEN], *p; 463 464 if (!__within_notrace_func(addr)) 465 return false; 466 467 /* Check if the address is on a suffixed-symbol */ 468 if (!lookup_symbol_name(addr, symname)) { 469 p = strchr(symname, '.'); 470 if (!p) 471 return true; 472 *p = '\0'; 473 addr = (unsigned long)kprobe_lookup_name(symname, 0); 474 if (addr) 475 return __within_notrace_func(addr); 476 } 477 478 return true; 479 } 480 #else 481 #define within_notrace_func(tk) (false) 482 #endif 483 484 /* Internal register function - just handle k*probes and flags */ 485 static int __register_trace_kprobe(struct trace_kprobe *tk) 486 { 487 int i, ret; 488 489 ret = security_locked_down(LOCKDOWN_KPROBES); 490 if (ret) 491 return ret; 492 493 if (trace_kprobe_is_registered(tk)) 494 return -EINVAL; 495 496 if (within_notrace_func(tk)) { 497 pr_warn("Could not probe notrace function %ps\n", 498 (void *)trace_kprobe_address(tk)); 499 return -EINVAL; 500 } 501 502 for (i = 0; i < tk->tp.nr_args; i++) { 503 ret = traceprobe_update_arg(&tk->tp.args[i]); 504 if (ret) 505 return ret; 506 } 507 508 /* Set/clear disabled flag according to tp->flag */ 509 if (trace_probe_is_enabled(&tk->tp)) 510 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; 511 else 512 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; 513 514 if (trace_kprobe_is_return(tk)) 515 ret = register_kretprobe(&tk->rp); 516 else 517 ret = register_kprobe(&tk->rp.kp); 518 519 return ret; 520 } 521 522 /* Internal unregister function - just handle k*probes and flags */ 523 static void __unregister_trace_kprobe(struct trace_kprobe *tk) 524 { 525 if (trace_kprobe_is_registered(tk)) { 526 if (trace_kprobe_is_return(tk)) 527 unregister_kretprobe(&tk->rp); 528 else 529 unregister_kprobe(&tk->rp.kp); 530 /* Cleanup kprobe for reuse and mark it unregistered */ 531 INIT_HLIST_NODE(&tk->rp.kp.hlist); 532 INIT_LIST_HEAD(&tk->rp.kp.list); 533 if (tk->rp.kp.symbol_name) 534 tk->rp.kp.addr = NULL; 535 } 536 } 537 538 /* Unregister a trace_probe and probe_event */ 539 static int unregister_trace_kprobe(struct trace_kprobe *tk) 540 { 541 /* If other probes are on the event, just unregister kprobe */ 542 if (trace_probe_has_sibling(&tk->tp)) 543 goto unreg; 544 545 /* Enabled event can not be unregistered */ 546 if (trace_probe_is_enabled(&tk->tp)) 547 return -EBUSY; 548 549 /* If there's a reference to the dynamic event */ 550 if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp))) 551 return -EBUSY; 552 553 /* Will fail if probe is being used by ftrace or perf */ 554 if (unregister_kprobe_event(tk)) 555 return -EBUSY; 556 557 unreg: 558 __unregister_trace_kprobe(tk); 559 dyn_event_remove(&tk->devent); 560 trace_probe_unlink(&tk->tp); 561 562 return 0; 563 } 564 565 static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig, 566 struct trace_kprobe *comp) 567 { 568 struct trace_probe_event *tpe = orig->tp.event; 569 int i; 570 571 list_for_each_entry(orig, &tpe->probes, tp.list) { 572 if (strcmp(trace_kprobe_symbol(orig), 573 trace_kprobe_symbol(comp)) || 574 trace_kprobe_offset(orig) != trace_kprobe_offset(comp)) 575 continue; 576 577 /* 578 * trace_probe_compare_arg_type() ensured that nr_args and 579 * each argument name and type are same. Let's compare comm. 580 */ 581 for (i = 0; i < orig->tp.nr_args; i++) { 582 if (strcmp(orig->tp.args[i].comm, 583 comp->tp.args[i].comm)) 584 break; 585 } 586 587 if (i == orig->tp.nr_args) 588 return true; 589 } 590 591 return false; 592 } 593 594 static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to) 595 { 596 int ret; 597 598 ret = trace_probe_compare_arg_type(&tk->tp, &to->tp); 599 if (ret) { 600 /* Note that argument starts index = 2 */ 601 trace_probe_log_set_index(ret + 1); 602 trace_probe_log_err(0, DIFF_ARG_TYPE); 603 return -EEXIST; 604 } 605 if (trace_kprobe_has_same_kprobe(to, tk)) { 606 trace_probe_log_set_index(0); 607 trace_probe_log_err(0, SAME_PROBE); 608 return -EEXIST; 609 } 610 611 /* Append to existing event */ 612 ret = trace_probe_append(&tk->tp, &to->tp); 613 if (ret) 614 return ret; 615 616 /* Register k*probe */ 617 ret = __register_trace_kprobe(tk); 618 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 619 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 620 ret = 0; 621 } 622 623 if (ret) 624 trace_probe_unlink(&tk->tp); 625 else 626 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 627 628 return ret; 629 } 630 631 /* Register a trace_probe and probe_event */ 632 static int register_trace_kprobe(struct trace_kprobe *tk) 633 { 634 struct trace_kprobe *old_tk; 635 int ret; 636 637 mutex_lock(&event_mutex); 638 639 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), 640 trace_probe_group_name(&tk->tp)); 641 if (old_tk) { 642 if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) { 643 trace_probe_log_set_index(0); 644 trace_probe_log_err(0, DIFF_PROBE_TYPE); 645 ret = -EEXIST; 646 } else { 647 ret = append_trace_kprobe(tk, old_tk); 648 } 649 goto end; 650 } 651 652 /* Register new event */ 653 ret = register_kprobe_event(tk); 654 if (ret) { 655 if (ret == -EEXIST) { 656 trace_probe_log_set_index(0); 657 trace_probe_log_err(0, EVENT_EXIST); 658 } else 659 pr_warn("Failed to register probe event(%d)\n", ret); 660 goto end; 661 } 662 663 /* Register k*probe */ 664 ret = __register_trace_kprobe(tk); 665 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 666 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 667 ret = 0; 668 } 669 670 if (ret < 0) 671 unregister_kprobe_event(tk); 672 else 673 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 674 675 end: 676 mutex_unlock(&event_mutex); 677 return ret; 678 } 679 680 #ifdef CONFIG_MODULES 681 static int validate_module_probe_symbol(const char *modname, const char *symbol); 682 683 static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk) 684 { 685 const char *p; 686 int ret = 0; 687 688 p = strchr(trace_kprobe_symbol(tk), ':'); 689 if (p) 690 ret = validate_module_probe_symbol(module_name(mod), p + 1); 691 if (!ret) 692 ret = __register_trace_kprobe(tk); 693 return ret; 694 } 695 696 /* Module notifier call back, checking event on the module */ 697 static int trace_kprobe_module_callback(struct notifier_block *nb, 698 unsigned long val, void *data) 699 { 700 struct module *mod = data; 701 struct dyn_event *pos; 702 struct trace_kprobe *tk; 703 int ret; 704 705 if (val != MODULE_STATE_COMING) 706 return NOTIFY_DONE; 707 708 /* Update probes on coming module */ 709 mutex_lock(&event_mutex); 710 for_each_trace_kprobe(tk, pos) { 711 if (trace_kprobe_within_module(tk, mod)) { 712 /* Don't need to check busy - this should have gone. */ 713 __unregister_trace_kprobe(tk); 714 ret = register_module_trace_kprobe(mod, tk); 715 if (ret) 716 pr_warn("Failed to re-register probe %s on %s: %d\n", 717 trace_probe_name(&tk->tp), 718 module_name(mod), ret); 719 } 720 } 721 mutex_unlock(&event_mutex); 722 723 return NOTIFY_DONE; 724 } 725 726 static struct notifier_block trace_kprobe_module_nb = { 727 .notifier_call = trace_kprobe_module_callback, 728 .priority = 1 /* Invoked after kprobe module callback */ 729 }; 730 static int trace_kprobe_register_module_notifier(void) 731 { 732 return register_module_notifier(&trace_kprobe_module_nb); 733 } 734 #else 735 static int trace_kprobe_register_module_notifier(void) 736 { 737 return 0; 738 } 739 #endif /* CONFIG_MODULES */ 740 741 static int count_symbols(void *data, unsigned long unused) 742 { 743 unsigned int *count = data; 744 745 (*count)++; 746 747 return 0; 748 } 749 750 struct sym_count_ctx { 751 unsigned int count; 752 const char *name; 753 }; 754 755 static int count_mod_symbols(void *data, const char *name, unsigned long unused) 756 { 757 struct sym_count_ctx *ctx = data; 758 759 if (strcmp(name, ctx->name) == 0) 760 ctx->count++; 761 762 return 0; 763 } 764 765 static unsigned int number_of_same_symbols(const char *mod, const char *func_name) 766 { 767 struct sym_count_ctx ctx = { .count = 0, .name = func_name }; 768 769 if (!mod) 770 kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count); 771 772 module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx); 773 774 return ctx.count; 775 } 776 777 static int validate_module_probe_symbol(const char *modname, const char *symbol) 778 { 779 unsigned int count = number_of_same_symbols(modname, symbol); 780 781 if (count > 1) { 782 /* 783 * Users should use ADDR to remove the ambiguity of 784 * using KSYM only. 785 */ 786 return -EADDRNOTAVAIL; 787 } else if (count == 0) { 788 /* 789 * We can return ENOENT earlier than when register the 790 * kprobe. 791 */ 792 return -ENOENT; 793 } 794 return 0; 795 } 796 797 #ifdef CONFIG_MODULES 798 /* Return NULL if the module is not loaded or under unloading. */ 799 static struct module *try_module_get_by_name(const char *name) 800 { 801 struct module *mod; 802 803 rcu_read_lock_sched(); 804 mod = find_module(name); 805 if (mod && !try_module_get(mod)) 806 mod = NULL; 807 rcu_read_unlock_sched(); 808 809 return mod; 810 } 811 #else 812 #define try_module_get_by_name(name) (NULL) 813 #endif 814 815 static int validate_probe_symbol(char *symbol) 816 { 817 struct module *mod = NULL; 818 char *modname = NULL, *p; 819 int ret = 0; 820 821 p = strchr(symbol, ':'); 822 if (p) { 823 modname = symbol; 824 symbol = p + 1; 825 *p = '\0'; 826 mod = try_module_get_by_name(modname); 827 if (!mod) 828 goto out; 829 } 830 831 ret = validate_module_probe_symbol(modname, symbol); 832 out: 833 if (p) 834 *p = ':'; 835 if (mod) 836 module_put(mod); 837 return ret; 838 } 839 840 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 841 struct pt_regs *regs); 842 843 static int __trace_kprobe_create(int argc, const char *argv[]) 844 { 845 /* 846 * Argument syntax: 847 * - Add kprobe: 848 * p[:[GRP/][EVENT]] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS] 849 * - Add kretprobe: 850 * r[MAXACTIVE][:[GRP/][EVENT]] [MOD:]KSYM[+0] [FETCHARGS] 851 * Or 852 * p[:[GRP/][EVENT]] [MOD:]KSYM[+0]%return [FETCHARGS] 853 * 854 * Fetch args: 855 * $retval : fetch return value 856 * $stack : fetch stack address 857 * $stackN : fetch Nth of stack (N:0-) 858 * $comm : fetch current task comm 859 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 860 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 861 * %REG : fetch register REG 862 * Dereferencing memory fetch: 863 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 864 * Alias name of args: 865 * NAME=FETCHARG : set NAME as alias of FETCHARG. 866 * Type of args: 867 * FETCHARG:TYPE : use TYPE instead of unsigned long. 868 */ 869 struct trace_kprobe *tk = NULL; 870 int i, len, new_argc = 0, ret = 0; 871 bool is_return = false; 872 char *symbol = NULL, *tmp = NULL; 873 const char **new_argv = NULL; 874 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM; 875 enum probe_print_type ptype; 876 int maxactive = 0; 877 long offset = 0; 878 void *addr = NULL; 879 char buf[MAX_EVENT_NAME_LEN]; 880 char gbuf[MAX_EVENT_NAME_LEN]; 881 char abuf[MAX_BTF_ARGS_LEN]; 882 char *dbuf = NULL; 883 struct traceprobe_parse_context ctx = { .flags = TPARG_FL_KERNEL }; 884 885 switch (argv[0][0]) { 886 case 'r': 887 is_return = true; 888 break; 889 case 'p': 890 break; 891 default: 892 return -ECANCELED; 893 } 894 if (argc < 2) 895 return -ECANCELED; 896 897 trace_probe_log_init("trace_kprobe", argc, argv); 898 899 event = strchr(&argv[0][1], ':'); 900 if (event) 901 event++; 902 903 if (isdigit(argv[0][1])) { 904 if (!is_return) { 905 trace_probe_log_err(1, BAD_MAXACT_TYPE); 906 goto parse_error; 907 } 908 if (event) 909 len = event - &argv[0][1] - 1; 910 else 911 len = strlen(&argv[0][1]); 912 if (len > MAX_EVENT_NAME_LEN - 1) { 913 trace_probe_log_err(1, BAD_MAXACT); 914 goto parse_error; 915 } 916 memcpy(buf, &argv[0][1], len); 917 buf[len] = '\0'; 918 ret = kstrtouint(buf, 0, &maxactive); 919 if (ret || !maxactive) { 920 trace_probe_log_err(1, BAD_MAXACT); 921 goto parse_error; 922 } 923 /* kretprobes instances are iterated over via a list. The 924 * maximum should stay reasonable. 925 */ 926 if (maxactive > KRETPROBE_MAXACTIVE_MAX) { 927 trace_probe_log_err(1, MAXACT_TOO_BIG); 928 goto parse_error; 929 } 930 } 931 932 /* try to parse an address. if that fails, try to read the 933 * input as a symbol. */ 934 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { 935 trace_probe_log_set_index(1); 936 /* Check whether uprobe event specified */ 937 if (strchr(argv[1], '/') && strchr(argv[1], ':')) { 938 ret = -ECANCELED; 939 goto error; 940 } 941 /* a symbol specified */ 942 symbol = kstrdup(argv[1], GFP_KERNEL); 943 if (!symbol) 944 return -ENOMEM; 945 946 tmp = strchr(symbol, '%'); 947 if (tmp) { 948 if (!strcmp(tmp, "%return")) { 949 *tmp = '\0'; 950 is_return = true; 951 } else { 952 trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX); 953 goto parse_error; 954 } 955 } 956 957 /* TODO: support .init module functions */ 958 ret = traceprobe_split_symbol_offset(symbol, &offset); 959 if (ret || offset < 0 || offset > UINT_MAX) { 960 trace_probe_log_err(0, BAD_PROBE_ADDR); 961 goto parse_error; 962 } 963 ret = validate_probe_symbol(symbol); 964 if (ret) { 965 if (ret == -EADDRNOTAVAIL) 966 trace_probe_log_err(0, NON_UNIQ_SYMBOL); 967 else 968 trace_probe_log_err(0, BAD_PROBE_ADDR); 969 goto parse_error; 970 } 971 if (is_return) 972 ctx.flags |= TPARG_FL_RETURN; 973 ret = kprobe_on_func_entry(NULL, symbol, offset); 974 if (ret == 0 && !is_return) 975 ctx.flags |= TPARG_FL_FENTRY; 976 /* Defer the ENOENT case until register kprobe */ 977 if (ret == -EINVAL && is_return) { 978 trace_probe_log_err(0, BAD_RETPROBE); 979 goto parse_error; 980 } 981 } 982 983 trace_probe_log_set_index(0); 984 if (event) { 985 ret = traceprobe_parse_event_name(&event, &group, gbuf, 986 event - argv[0]); 987 if (ret) 988 goto parse_error; 989 } 990 991 if (!event) { 992 /* Make a new event name */ 993 if (symbol) 994 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 995 is_return ? 'r' : 'p', symbol, offset); 996 else 997 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p", 998 is_return ? 'r' : 'p', addr); 999 sanitize_event_name(buf); 1000 event = buf; 1001 } 1002 1003 argc -= 2; argv += 2; 1004 ctx.funcname = symbol; 1005 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, 1006 abuf, MAX_BTF_ARGS_LEN, &ctx); 1007 if (IS_ERR(new_argv)) { 1008 ret = PTR_ERR(new_argv); 1009 new_argv = NULL; 1010 goto out; 1011 } 1012 if (new_argv) { 1013 argc = new_argc; 1014 argv = new_argv; 1015 } 1016 if (argc > MAX_TRACE_ARGS) { 1017 ret = -E2BIG; 1018 goto out; 1019 } 1020 1021 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); 1022 if (ret) 1023 goto out; 1024 1025 /* setup a probe */ 1026 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, 1027 argc, is_return); 1028 if (IS_ERR(tk)) { 1029 ret = PTR_ERR(tk); 1030 /* This must return -ENOMEM, else there is a bug */ 1031 WARN_ON_ONCE(ret != -ENOMEM); 1032 goto out; /* We know tk is not allocated */ 1033 } 1034 1035 /* parse arguments */ 1036 for (i = 0; i < argc; i++) { 1037 trace_probe_log_set_index(i + 2); 1038 ctx.offset = 0; 1039 ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], &ctx); 1040 if (ret) 1041 goto error; /* This can be -ENOMEM */ 1042 } 1043 /* entry handler for kretprobe */ 1044 if (is_return && tk->tp.entry_arg) { 1045 tk->rp.entry_handler = trace_kprobe_entry_handler; 1046 tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp); 1047 } 1048 1049 ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1050 ret = traceprobe_set_print_fmt(&tk->tp, ptype); 1051 if (ret < 0) 1052 goto error; 1053 1054 ret = register_trace_kprobe(tk); 1055 if (ret) { 1056 trace_probe_log_set_index(1); 1057 if (ret == -EILSEQ) 1058 trace_probe_log_err(0, BAD_INSN_BNDRY); 1059 else if (ret == -ENOENT) 1060 trace_probe_log_err(0, BAD_PROBE_ADDR); 1061 else if (ret != -ENOMEM && ret != -EEXIST) 1062 trace_probe_log_err(0, FAIL_REG_PROBE); 1063 goto error; 1064 } 1065 1066 out: 1067 traceprobe_finish_parse(&ctx); 1068 trace_probe_log_clear(); 1069 kfree(new_argv); 1070 kfree(symbol); 1071 kfree(dbuf); 1072 return ret; 1073 1074 parse_error: 1075 ret = -EINVAL; 1076 error: 1077 free_trace_kprobe(tk); 1078 goto out; 1079 } 1080 1081 static int trace_kprobe_create(const char *raw_command) 1082 { 1083 return trace_probe_create(raw_command, __trace_kprobe_create); 1084 } 1085 1086 static int create_or_delete_trace_kprobe(const char *raw_command) 1087 { 1088 int ret; 1089 1090 if (raw_command[0] == '-') 1091 return dyn_event_release(raw_command, &trace_kprobe_ops); 1092 1093 ret = trace_kprobe_create(raw_command); 1094 return ret == -ECANCELED ? -EINVAL : ret; 1095 } 1096 1097 static int trace_kprobe_run_command(struct dynevent_cmd *cmd) 1098 { 1099 return create_or_delete_trace_kprobe(cmd->seq.buffer); 1100 } 1101 1102 /** 1103 * kprobe_event_cmd_init - Initialize a kprobe event command object 1104 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1105 * @buf: A pointer to the buffer used to build the command 1106 * @maxlen: The length of the buffer passed in @buf 1107 * 1108 * Initialize a synthetic event command object. Use this before 1109 * calling any of the other kprobe_event functions. 1110 */ 1111 void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen) 1112 { 1113 dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE, 1114 trace_kprobe_run_command); 1115 } 1116 EXPORT_SYMBOL_GPL(kprobe_event_cmd_init); 1117 1118 /** 1119 * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list 1120 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1121 * @kretprobe: Is this a return probe? 1122 * @name: The name of the kprobe event 1123 * @loc: The location of the kprobe event 1124 * @...: Variable number of arg (pairs), one pair for each field 1125 * 1126 * NOTE: Users normally won't want to call this function directly, but 1127 * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically 1128 * adds a NULL to the end of the arg list. If this function is used 1129 * directly, make sure the last arg in the variable arg list is NULL. 1130 * 1131 * Generate a kprobe event command to be executed by 1132 * kprobe_event_gen_cmd_end(). This function can be used to generate the 1133 * complete command or only the first part of it; in the latter case, 1134 * kprobe_event_add_fields() can be used to add more fields following this. 1135 * 1136 * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This 1137 * returns -EINVAL if @loc == NULL. 1138 * 1139 * Return: 0 if successful, error otherwise. 1140 */ 1141 int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe, 1142 const char *name, const char *loc, ...) 1143 { 1144 char buf[MAX_EVENT_NAME_LEN]; 1145 struct dynevent_arg arg; 1146 va_list args; 1147 int ret; 1148 1149 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1150 return -EINVAL; 1151 1152 if (!loc) 1153 return -EINVAL; 1154 1155 if (kretprobe) 1156 snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name); 1157 else 1158 snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name); 1159 1160 ret = dynevent_str_add(cmd, buf); 1161 if (ret) 1162 return ret; 1163 1164 dynevent_arg_init(&arg, 0); 1165 arg.str = loc; 1166 ret = dynevent_arg_add(cmd, &arg, NULL); 1167 if (ret) 1168 return ret; 1169 1170 va_start(args, loc); 1171 for (;;) { 1172 const char *field; 1173 1174 field = va_arg(args, const char *); 1175 if (!field) 1176 break; 1177 1178 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1179 ret = -EINVAL; 1180 break; 1181 } 1182 1183 arg.str = field; 1184 ret = dynevent_arg_add(cmd, &arg, NULL); 1185 if (ret) 1186 break; 1187 } 1188 va_end(args); 1189 1190 return ret; 1191 } 1192 EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start); 1193 1194 /** 1195 * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list 1196 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1197 * @...: Variable number of arg (pairs), one pair for each field 1198 * 1199 * NOTE: Users normally won't want to call this function directly, but 1200 * rather use the kprobe_event_add_fields() wrapper, which 1201 * automatically adds a NULL to the end of the arg list. If this 1202 * function is used directly, make sure the last arg in the variable 1203 * arg list is NULL. 1204 * 1205 * Add probe fields to an existing kprobe command using a variable 1206 * list of args. Fields are added in the same order they're listed. 1207 * 1208 * Return: 0 if successful, error otherwise. 1209 */ 1210 int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...) 1211 { 1212 struct dynevent_arg arg; 1213 va_list args; 1214 int ret = 0; 1215 1216 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1217 return -EINVAL; 1218 1219 dynevent_arg_init(&arg, 0); 1220 1221 va_start(args, cmd); 1222 for (;;) { 1223 const char *field; 1224 1225 field = va_arg(args, const char *); 1226 if (!field) 1227 break; 1228 1229 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1230 ret = -EINVAL; 1231 break; 1232 } 1233 1234 arg.str = field; 1235 ret = dynevent_arg_add(cmd, &arg, NULL); 1236 if (ret) 1237 break; 1238 } 1239 va_end(args); 1240 1241 return ret; 1242 } 1243 EXPORT_SYMBOL_GPL(__kprobe_event_add_fields); 1244 1245 /** 1246 * kprobe_event_delete - Delete a kprobe event 1247 * @name: The name of the kprobe event to delete 1248 * 1249 * Delete a kprobe event with the give @name from kernel code rather 1250 * than directly from the command line. 1251 * 1252 * Return: 0 if successful, error otherwise. 1253 */ 1254 int kprobe_event_delete(const char *name) 1255 { 1256 char buf[MAX_EVENT_NAME_LEN]; 1257 1258 snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name); 1259 1260 return create_or_delete_trace_kprobe(buf); 1261 } 1262 EXPORT_SYMBOL_GPL(kprobe_event_delete); 1263 1264 static int trace_kprobe_release(struct dyn_event *ev) 1265 { 1266 struct trace_kprobe *tk = to_trace_kprobe(ev); 1267 int ret = unregister_trace_kprobe(tk); 1268 1269 if (!ret) 1270 free_trace_kprobe(tk); 1271 return ret; 1272 } 1273 1274 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) 1275 { 1276 struct trace_kprobe *tk = to_trace_kprobe(ev); 1277 int i; 1278 1279 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); 1280 if (trace_kprobe_is_return(tk) && tk->rp.maxactive) 1281 seq_printf(m, "%d", tk->rp.maxactive); 1282 seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp), 1283 trace_probe_name(&tk->tp)); 1284 1285 if (!tk->symbol) 1286 seq_printf(m, " 0x%p", tk->rp.kp.addr); 1287 else if (tk->rp.kp.offset) 1288 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk), 1289 tk->rp.kp.offset); 1290 else 1291 seq_printf(m, " %s", trace_kprobe_symbol(tk)); 1292 1293 for (i = 0; i < tk->tp.nr_args; i++) 1294 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); 1295 seq_putc(m, '\n'); 1296 1297 return 0; 1298 } 1299 1300 static int probes_seq_show(struct seq_file *m, void *v) 1301 { 1302 struct dyn_event *ev = v; 1303 1304 if (!is_trace_kprobe(ev)) 1305 return 0; 1306 1307 return trace_kprobe_show(m, ev); 1308 } 1309 1310 static const struct seq_operations probes_seq_op = { 1311 .start = dyn_event_seq_start, 1312 .next = dyn_event_seq_next, 1313 .stop = dyn_event_seq_stop, 1314 .show = probes_seq_show 1315 }; 1316 1317 static int probes_open(struct inode *inode, struct file *file) 1318 { 1319 int ret; 1320 1321 ret = security_locked_down(LOCKDOWN_TRACEFS); 1322 if (ret) 1323 return ret; 1324 1325 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1326 ret = dyn_events_release_all(&trace_kprobe_ops); 1327 if (ret < 0) 1328 return ret; 1329 } 1330 1331 return seq_open(file, &probes_seq_op); 1332 } 1333 1334 static ssize_t probes_write(struct file *file, const char __user *buffer, 1335 size_t count, loff_t *ppos) 1336 { 1337 return trace_parse_run_command(file, buffer, count, ppos, 1338 create_or_delete_trace_kprobe); 1339 } 1340 1341 static const struct file_operations kprobe_events_ops = { 1342 .owner = THIS_MODULE, 1343 .open = probes_open, 1344 .read = seq_read, 1345 .llseek = seq_lseek, 1346 .release = seq_release, 1347 .write = probes_write, 1348 }; 1349 1350 static unsigned long trace_kprobe_missed(struct trace_kprobe *tk) 1351 { 1352 return trace_kprobe_is_return(tk) ? 1353 tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed; 1354 } 1355 1356 /* Probes profiling interfaces */ 1357 static int probes_profile_seq_show(struct seq_file *m, void *v) 1358 { 1359 struct dyn_event *ev = v; 1360 struct trace_kprobe *tk; 1361 unsigned long nmissed; 1362 1363 if (!is_trace_kprobe(ev)) 1364 return 0; 1365 1366 tk = to_trace_kprobe(ev); 1367 nmissed = trace_kprobe_missed(tk); 1368 seq_printf(m, " %-44s %15lu %15lu\n", 1369 trace_probe_name(&tk->tp), 1370 trace_kprobe_nhit(tk), 1371 nmissed); 1372 1373 return 0; 1374 } 1375 1376 static const struct seq_operations profile_seq_op = { 1377 .start = dyn_event_seq_start, 1378 .next = dyn_event_seq_next, 1379 .stop = dyn_event_seq_stop, 1380 .show = probes_profile_seq_show 1381 }; 1382 1383 static int profile_open(struct inode *inode, struct file *file) 1384 { 1385 int ret; 1386 1387 ret = security_locked_down(LOCKDOWN_TRACEFS); 1388 if (ret) 1389 return ret; 1390 1391 return seq_open(file, &profile_seq_op); 1392 } 1393 1394 static const struct file_operations kprobe_profile_ops = { 1395 .owner = THIS_MODULE, 1396 .open = profile_open, 1397 .read = seq_read, 1398 .llseek = seq_lseek, 1399 .release = seq_release, 1400 }; 1401 1402 /* Note that we don't verify it, since the code does not come from user space */ 1403 static int 1404 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 1405 void *dest, void *base) 1406 { 1407 struct pt_regs *regs = rec; 1408 unsigned long val; 1409 int ret; 1410 1411 retry: 1412 /* 1st stage: get value from context */ 1413 switch (code->op) { 1414 case FETCH_OP_REG: 1415 val = regs_get_register(regs, code->param); 1416 break; 1417 case FETCH_OP_STACK: 1418 val = regs_get_kernel_stack_nth(regs, code->param); 1419 break; 1420 case FETCH_OP_STACKP: 1421 val = kernel_stack_pointer(regs); 1422 break; 1423 case FETCH_OP_RETVAL: 1424 val = regs_return_value(regs); 1425 break; 1426 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 1427 case FETCH_OP_ARG: 1428 val = regs_get_kernel_argument(regs, code->param); 1429 break; 1430 case FETCH_OP_EDATA: 1431 val = *(unsigned long *)((unsigned long)edata + code->offset); 1432 break; 1433 #endif 1434 case FETCH_NOP_SYMBOL: /* Ignore a place holder */ 1435 code++; 1436 goto retry; 1437 default: 1438 ret = process_common_fetch_insn(code, &val); 1439 if (ret < 0) 1440 return ret; 1441 } 1442 code++; 1443 1444 return process_fetch_insn_bottom(code, val, dest, base); 1445 } 1446 NOKPROBE_SYMBOL(process_fetch_insn) 1447 1448 /* Kprobe handler */ 1449 static nokprobe_inline void 1450 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 1451 struct trace_event_file *trace_file) 1452 { 1453 struct kprobe_trace_entry_head *entry; 1454 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1455 struct trace_event_buffer fbuffer; 1456 int dsize; 1457 1458 WARN_ON(call != trace_file->event_call); 1459 1460 if (trace_trigger_soft_disabled(trace_file)) 1461 return; 1462 1463 dsize = __get_data_size(&tk->tp, regs, NULL); 1464 1465 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1466 sizeof(*entry) + tk->tp.size + dsize); 1467 if (!entry) 1468 return; 1469 1470 fbuffer.regs = regs; 1471 entry->ip = (unsigned long)tk->rp.kp.addr; 1472 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1473 1474 trace_event_buffer_commit(&fbuffer); 1475 } 1476 1477 static void 1478 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) 1479 { 1480 struct event_file_link *link; 1481 1482 trace_probe_for_each_link_rcu(link, &tk->tp) 1483 __kprobe_trace_func(tk, regs, link->file); 1484 } 1485 NOKPROBE_SYMBOL(kprobe_trace_func); 1486 1487 /* Kretprobe handler */ 1488 1489 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 1490 struct pt_regs *regs) 1491 { 1492 struct kretprobe *rp = get_kretprobe(ri); 1493 struct trace_kprobe *tk; 1494 1495 /* 1496 * There is a small chance that get_kretprobe(ri) returns NULL when 1497 * the kretprobe is unregister on another CPU between kretprobe's 1498 * trampoline_handler and this function. 1499 */ 1500 if (unlikely(!rp)) 1501 return -ENOENT; 1502 1503 tk = container_of(rp, struct trace_kprobe, rp); 1504 1505 /* store argument values into ri->data as entry data */ 1506 if (tk->tp.entry_arg) 1507 store_trace_entry_data(ri->data, &tk->tp, regs); 1508 1509 return 0; 1510 } 1511 1512 1513 static nokprobe_inline void 1514 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1515 struct pt_regs *regs, 1516 struct trace_event_file *trace_file) 1517 { 1518 struct kretprobe_trace_entry_head *entry; 1519 struct trace_event_buffer fbuffer; 1520 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1521 int dsize; 1522 1523 WARN_ON(call != trace_file->event_call); 1524 1525 if (trace_trigger_soft_disabled(trace_file)) 1526 return; 1527 1528 dsize = __get_data_size(&tk->tp, regs, ri->data); 1529 1530 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1531 sizeof(*entry) + tk->tp.size + dsize); 1532 if (!entry) 1533 return; 1534 1535 fbuffer.regs = regs; 1536 entry->func = (unsigned long)tk->rp.kp.addr; 1537 entry->ret_ip = get_kretprobe_retaddr(ri); 1538 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1539 1540 trace_event_buffer_commit(&fbuffer); 1541 } 1542 1543 static void 1544 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1545 struct pt_regs *regs) 1546 { 1547 struct event_file_link *link; 1548 1549 trace_probe_for_each_link_rcu(link, &tk->tp) 1550 __kretprobe_trace_func(tk, ri, regs, link->file); 1551 } 1552 NOKPROBE_SYMBOL(kretprobe_trace_func); 1553 1554 /* Event entry printers */ 1555 static enum print_line_t 1556 print_kprobe_event(struct trace_iterator *iter, int flags, 1557 struct trace_event *event) 1558 { 1559 struct kprobe_trace_entry_head *field; 1560 struct trace_seq *s = &iter->seq; 1561 struct trace_probe *tp; 1562 1563 field = (struct kprobe_trace_entry_head *)iter->ent; 1564 tp = trace_probe_primary_from_call( 1565 container_of(event, struct trace_event_call, event)); 1566 if (WARN_ON_ONCE(!tp)) 1567 goto out; 1568 1569 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1570 1571 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) 1572 goto out; 1573 1574 trace_seq_putc(s, ')'); 1575 1576 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1577 (u8 *)&field[1], field) < 0) 1578 goto out; 1579 1580 trace_seq_putc(s, '\n'); 1581 out: 1582 return trace_handle_return(s); 1583 } 1584 1585 static enum print_line_t 1586 print_kretprobe_event(struct trace_iterator *iter, int flags, 1587 struct trace_event *event) 1588 { 1589 struct kretprobe_trace_entry_head *field; 1590 struct trace_seq *s = &iter->seq; 1591 struct trace_probe *tp; 1592 1593 field = (struct kretprobe_trace_entry_head *)iter->ent; 1594 tp = trace_probe_primary_from_call( 1595 container_of(event, struct trace_event_call, event)); 1596 if (WARN_ON_ONCE(!tp)) 1597 goto out; 1598 1599 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1600 1601 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) 1602 goto out; 1603 1604 trace_seq_puts(s, " <- "); 1605 1606 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) 1607 goto out; 1608 1609 trace_seq_putc(s, ')'); 1610 1611 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1612 (u8 *)&field[1], field) < 0) 1613 goto out; 1614 1615 trace_seq_putc(s, '\n'); 1616 1617 out: 1618 return trace_handle_return(s); 1619 } 1620 1621 1622 static int kprobe_event_define_fields(struct trace_event_call *event_call) 1623 { 1624 int ret; 1625 struct kprobe_trace_entry_head field; 1626 struct trace_probe *tp; 1627 1628 tp = trace_probe_primary_from_call(event_call); 1629 if (WARN_ON_ONCE(!tp)) 1630 return -ENOENT; 1631 1632 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1633 1634 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1635 } 1636 1637 static int kretprobe_event_define_fields(struct trace_event_call *event_call) 1638 { 1639 int ret; 1640 struct kretprobe_trace_entry_head field; 1641 struct trace_probe *tp; 1642 1643 tp = trace_probe_primary_from_call(event_call); 1644 if (WARN_ON_ONCE(!tp)) 1645 return -ENOENT; 1646 1647 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1648 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1649 1650 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1651 } 1652 1653 #ifdef CONFIG_PERF_EVENTS 1654 1655 /* Kprobe profile handler */ 1656 static int 1657 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) 1658 { 1659 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1660 struct kprobe_trace_entry_head *entry; 1661 struct hlist_head *head; 1662 int size, __size, dsize; 1663 int rctx; 1664 1665 if (bpf_prog_array_valid(call)) { 1666 unsigned long orig_ip = instruction_pointer(regs); 1667 int ret; 1668 1669 ret = trace_call_bpf(call, regs); 1670 1671 /* 1672 * We need to check and see if we modified the pc of the 1673 * pt_regs, and if so return 1 so that we don't do the 1674 * single stepping. 1675 */ 1676 if (orig_ip != instruction_pointer(regs)) 1677 return 1; 1678 if (!ret) 1679 return 0; 1680 } 1681 1682 head = this_cpu_ptr(call->perf_events); 1683 if (hlist_empty(head)) 1684 return 0; 1685 1686 dsize = __get_data_size(&tk->tp, regs, NULL); 1687 __size = sizeof(*entry) + tk->tp.size + dsize; 1688 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1689 size -= sizeof(u32); 1690 1691 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1692 if (!entry) 1693 return 0; 1694 1695 entry->ip = (unsigned long)tk->rp.kp.addr; 1696 memset(&entry[1], 0, dsize); 1697 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1698 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1699 head, NULL); 1700 return 0; 1701 } 1702 NOKPROBE_SYMBOL(kprobe_perf_func); 1703 1704 /* Kretprobe profile handler */ 1705 static void 1706 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1707 struct pt_regs *regs) 1708 { 1709 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1710 struct kretprobe_trace_entry_head *entry; 1711 struct hlist_head *head; 1712 int size, __size, dsize; 1713 int rctx; 1714 1715 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs)) 1716 return; 1717 1718 head = this_cpu_ptr(call->perf_events); 1719 if (hlist_empty(head)) 1720 return; 1721 1722 dsize = __get_data_size(&tk->tp, regs, ri->data); 1723 __size = sizeof(*entry) + tk->tp.size + dsize; 1724 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1725 size -= sizeof(u32); 1726 1727 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1728 if (!entry) 1729 return; 1730 1731 entry->func = (unsigned long)tk->rp.kp.addr; 1732 entry->ret_ip = get_kretprobe_retaddr(ri); 1733 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1734 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1735 head, NULL); 1736 } 1737 NOKPROBE_SYMBOL(kretprobe_perf_func); 1738 1739 int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, 1740 const char **symbol, u64 *probe_offset, 1741 u64 *probe_addr, unsigned long *missed, 1742 bool perf_type_tracepoint) 1743 { 1744 const char *pevent = trace_event_name(event->tp_event); 1745 const char *group = event->tp_event->class->system; 1746 struct trace_kprobe *tk; 1747 1748 if (perf_type_tracepoint) 1749 tk = find_trace_kprobe(pevent, group); 1750 else 1751 tk = trace_kprobe_primary_from_call(event->tp_event); 1752 if (!tk) 1753 return -EINVAL; 1754 1755 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE 1756 : BPF_FD_TYPE_KPROBE; 1757 *probe_offset = tk->rp.kp.offset; 1758 *probe_addr = kallsyms_show_value(current_cred()) ? 1759 (unsigned long)tk->rp.kp.addr : 0; 1760 *symbol = tk->symbol; 1761 if (missed) 1762 *missed = trace_kprobe_missed(tk); 1763 return 0; 1764 } 1765 #endif /* CONFIG_PERF_EVENTS */ 1766 1767 /* 1768 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1769 * 1770 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe 1771 * lockless, but we can't race with this __init function. 1772 */ 1773 static int kprobe_register(struct trace_event_call *event, 1774 enum trace_reg type, void *data) 1775 { 1776 struct trace_event_file *file = data; 1777 1778 switch (type) { 1779 case TRACE_REG_REGISTER: 1780 return enable_trace_kprobe(event, file); 1781 case TRACE_REG_UNREGISTER: 1782 return disable_trace_kprobe(event, file); 1783 1784 #ifdef CONFIG_PERF_EVENTS 1785 case TRACE_REG_PERF_REGISTER: 1786 return enable_trace_kprobe(event, NULL); 1787 case TRACE_REG_PERF_UNREGISTER: 1788 return disable_trace_kprobe(event, NULL); 1789 case TRACE_REG_PERF_OPEN: 1790 case TRACE_REG_PERF_CLOSE: 1791 case TRACE_REG_PERF_ADD: 1792 case TRACE_REG_PERF_DEL: 1793 return 0; 1794 #endif 1795 } 1796 return 0; 1797 } 1798 1799 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) 1800 { 1801 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); 1802 int ret = 0; 1803 1804 raw_cpu_inc(*tk->nhit); 1805 1806 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) 1807 kprobe_trace_func(tk, regs); 1808 #ifdef CONFIG_PERF_EVENTS 1809 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) 1810 ret = kprobe_perf_func(tk, regs); 1811 #endif 1812 return ret; 1813 } 1814 NOKPROBE_SYMBOL(kprobe_dispatcher); 1815 1816 static int 1817 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) 1818 { 1819 struct kretprobe *rp = get_kretprobe(ri); 1820 struct trace_kprobe *tk; 1821 1822 /* 1823 * There is a small chance that get_kretprobe(ri) returns NULL when 1824 * the kretprobe is unregister on another CPU between kretprobe's 1825 * trampoline_handler and this function. 1826 */ 1827 if (unlikely(!rp)) 1828 return 0; 1829 1830 tk = container_of(rp, struct trace_kprobe, rp); 1831 raw_cpu_inc(*tk->nhit); 1832 1833 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) 1834 kretprobe_trace_func(tk, ri, regs); 1835 #ifdef CONFIG_PERF_EVENTS 1836 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) 1837 kretprobe_perf_func(tk, ri, regs); 1838 #endif 1839 return 0; /* We don't tweak kernel, so just return 0 */ 1840 } 1841 NOKPROBE_SYMBOL(kretprobe_dispatcher); 1842 1843 static struct trace_event_functions kretprobe_funcs = { 1844 .trace = print_kretprobe_event 1845 }; 1846 1847 static struct trace_event_functions kprobe_funcs = { 1848 .trace = print_kprobe_event 1849 }; 1850 1851 static struct trace_event_fields kretprobe_fields_array[] = { 1852 { .type = TRACE_FUNCTION_TYPE, 1853 .define_fields = kretprobe_event_define_fields }, 1854 {} 1855 }; 1856 1857 static struct trace_event_fields kprobe_fields_array[] = { 1858 { .type = TRACE_FUNCTION_TYPE, 1859 .define_fields = kprobe_event_define_fields }, 1860 {} 1861 }; 1862 1863 static inline void init_trace_event_call(struct trace_kprobe *tk) 1864 { 1865 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1866 1867 if (trace_kprobe_is_return(tk)) { 1868 call->event.funcs = &kretprobe_funcs; 1869 call->class->fields_array = kretprobe_fields_array; 1870 } else { 1871 call->event.funcs = &kprobe_funcs; 1872 call->class->fields_array = kprobe_fields_array; 1873 } 1874 1875 call->flags = TRACE_EVENT_FL_KPROBE; 1876 call->class->reg = kprobe_register; 1877 } 1878 1879 static int register_kprobe_event(struct trace_kprobe *tk) 1880 { 1881 init_trace_event_call(tk); 1882 1883 return trace_probe_register_event_call(&tk->tp); 1884 } 1885 1886 static int unregister_kprobe_event(struct trace_kprobe *tk) 1887 { 1888 return trace_probe_unregister_event_call(&tk->tp); 1889 } 1890 1891 #ifdef CONFIG_PERF_EVENTS 1892 1893 /* create a trace_kprobe, but don't add it to global lists */ 1894 struct trace_event_call * 1895 create_local_trace_kprobe(char *func, void *addr, unsigned long offs, 1896 bool is_return) 1897 { 1898 enum probe_print_type ptype; 1899 struct trace_kprobe *tk; 1900 int ret; 1901 char *event; 1902 1903 if (func) { 1904 ret = validate_probe_symbol(func); 1905 if (ret) 1906 return ERR_PTR(ret); 1907 } 1908 1909 /* 1910 * local trace_kprobes are not added to dyn_event, so they are never 1911 * searched in find_trace_kprobe(). Therefore, there is no concern of 1912 * duplicated name here. 1913 */ 1914 event = func ? func : "DUMMY_EVENT"; 1915 1916 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func, 1917 offs, 0 /* maxactive */, 0 /* nargs */, 1918 is_return); 1919 1920 if (IS_ERR(tk)) { 1921 pr_info("Failed to allocate trace_probe.(%d)\n", 1922 (int)PTR_ERR(tk)); 1923 return ERR_CAST(tk); 1924 } 1925 1926 init_trace_event_call(tk); 1927 1928 ptype = trace_kprobe_is_return(tk) ? 1929 PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1930 if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) { 1931 ret = -ENOMEM; 1932 goto error; 1933 } 1934 1935 ret = __register_trace_kprobe(tk); 1936 if (ret < 0) 1937 goto error; 1938 1939 return trace_probe_event_call(&tk->tp); 1940 error: 1941 free_trace_kprobe(tk); 1942 return ERR_PTR(ret); 1943 } 1944 1945 void destroy_local_trace_kprobe(struct trace_event_call *event_call) 1946 { 1947 struct trace_kprobe *tk; 1948 1949 tk = trace_kprobe_primary_from_call(event_call); 1950 if (unlikely(!tk)) 1951 return; 1952 1953 if (trace_probe_is_enabled(&tk->tp)) { 1954 WARN_ON(1); 1955 return; 1956 } 1957 1958 __unregister_trace_kprobe(tk); 1959 1960 free_trace_kprobe(tk); 1961 } 1962 #endif /* CONFIG_PERF_EVENTS */ 1963 1964 static __init void enable_boot_kprobe_events(void) 1965 { 1966 struct trace_array *tr = top_trace_array(); 1967 struct trace_event_file *file; 1968 struct trace_kprobe *tk; 1969 struct dyn_event *pos; 1970 1971 mutex_lock(&event_mutex); 1972 for_each_trace_kprobe(tk, pos) { 1973 list_for_each_entry(file, &tr->events, list) 1974 if (file->event_call == trace_probe_event_call(&tk->tp)) 1975 trace_event_enable_disable(file, 1, 0); 1976 } 1977 mutex_unlock(&event_mutex); 1978 } 1979 1980 static __init void setup_boot_kprobe_events(void) 1981 { 1982 char *p, *cmd = kprobe_boot_events_buf; 1983 int ret; 1984 1985 strreplace(kprobe_boot_events_buf, ',', ' '); 1986 1987 while (cmd && *cmd != '\0') { 1988 p = strchr(cmd, ';'); 1989 if (p) 1990 *p++ = '\0'; 1991 1992 ret = create_or_delete_trace_kprobe(cmd); 1993 if (ret) 1994 pr_warn("Failed to add event(%d): %s\n", ret, cmd); 1995 1996 cmd = p; 1997 } 1998 1999 enable_boot_kprobe_events(); 2000 } 2001 2002 /* 2003 * Register dynevent at core_initcall. This allows kernel to setup kprobe 2004 * events in postcore_initcall without tracefs. 2005 */ 2006 static __init int init_kprobe_trace_early(void) 2007 { 2008 int ret; 2009 2010 ret = dyn_event_register(&trace_kprobe_ops); 2011 if (ret) 2012 return ret; 2013 2014 if (trace_kprobe_register_module_notifier()) 2015 return -EINVAL; 2016 2017 return 0; 2018 } 2019 core_initcall(init_kprobe_trace_early); 2020 2021 /* Make a tracefs interface for controlling probe points */ 2022 static __init int init_kprobe_trace(void) 2023 { 2024 int ret; 2025 2026 ret = tracing_init_dentry(); 2027 if (ret) 2028 return 0; 2029 2030 /* Event list interface */ 2031 trace_create_file("kprobe_events", TRACE_MODE_WRITE, 2032 NULL, NULL, &kprobe_events_ops); 2033 2034 /* Profile interface */ 2035 trace_create_file("kprobe_profile", TRACE_MODE_READ, 2036 NULL, NULL, &kprobe_profile_ops); 2037 2038 setup_boot_kprobe_events(); 2039 2040 return 0; 2041 } 2042 fs_initcall(init_kprobe_trace); 2043 2044 2045 #ifdef CONFIG_FTRACE_STARTUP_TEST 2046 static __init struct trace_event_file * 2047 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) 2048 { 2049 struct trace_event_file *file; 2050 2051 list_for_each_entry(file, &tr->events, list) 2052 if (file->event_call == trace_probe_event_call(&tk->tp)) 2053 return file; 2054 2055 return NULL; 2056 } 2057 2058 /* 2059 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this 2060 * stage, we can do this lockless. 2061 */ 2062 static __init int kprobe_trace_self_tests_init(void) 2063 { 2064 int ret, warn = 0; 2065 int (*target)(int, int, int, int, int, int); 2066 struct trace_kprobe *tk; 2067 struct trace_event_file *file; 2068 2069 if (tracing_is_disabled()) 2070 return -ENODEV; 2071 2072 if (tracing_selftest_disabled) 2073 return 0; 2074 2075 target = kprobe_trace_selftest_target; 2076 2077 pr_info("Testing kprobe tracing: "); 2078 2079 ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)"); 2080 if (WARN_ONCE(ret, "error on probing function entry.")) { 2081 warn++; 2082 } else { 2083 /* Enable trace point */ 2084 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2085 if (WARN_ONCE(tk == NULL, "error on probing function entry.")) { 2086 warn++; 2087 } else { 2088 file = find_trace_probe_file(tk, top_trace_array()); 2089 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2090 warn++; 2091 } else 2092 enable_trace_kprobe( 2093 trace_probe_event_call(&tk->tp), file); 2094 } 2095 } 2096 2097 ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval"); 2098 if (WARN_ONCE(ret, "error on probing function return.")) { 2099 warn++; 2100 } else { 2101 /* Enable trace point */ 2102 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2103 if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) { 2104 warn++; 2105 } else { 2106 file = find_trace_probe_file(tk, top_trace_array()); 2107 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2108 warn++; 2109 } else 2110 enable_trace_kprobe( 2111 trace_probe_event_call(&tk->tp), file); 2112 } 2113 } 2114 2115 if (warn) 2116 goto end; 2117 2118 ret = target(1, 2, 3, 4, 5, 6); 2119 2120 /* 2121 * Not expecting an error here, the check is only to prevent the 2122 * optimizer from removing the call to target() as otherwise there 2123 * are no side-effects and the call is never performed. 2124 */ 2125 if (ret != 21) 2126 warn++; 2127 2128 /* Disable trace points before removing it */ 2129 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2130 if (WARN_ONCE(tk == NULL, "error on getting test probe.")) { 2131 warn++; 2132 } else { 2133 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2134 "incorrect number of testprobe hits.")) 2135 warn++; 2136 2137 file = find_trace_probe_file(tk, top_trace_array()); 2138 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2139 warn++; 2140 } else 2141 disable_trace_kprobe( 2142 trace_probe_event_call(&tk->tp), file); 2143 } 2144 2145 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2146 if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) { 2147 warn++; 2148 } else { 2149 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2150 "incorrect number of testprobe2 hits.")) 2151 warn++; 2152 2153 file = find_trace_probe_file(tk, top_trace_array()); 2154 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2155 warn++; 2156 } else 2157 disable_trace_kprobe( 2158 trace_probe_event_call(&tk->tp), file); 2159 } 2160 2161 ret = create_or_delete_trace_kprobe("-:testprobe"); 2162 if (WARN_ONCE(ret, "error on deleting a probe.")) 2163 warn++; 2164 2165 ret = create_or_delete_trace_kprobe("-:testprobe2"); 2166 if (WARN_ONCE(ret, "error on deleting a probe.")) 2167 warn++; 2168 2169 2170 end: 2171 /* 2172 * Wait for the optimizer work to finish. Otherwise it might fiddle 2173 * with probes in already freed __init text. 2174 */ 2175 wait_for_kprobe_optimizer(); 2176 if (warn) 2177 pr_cont("NG: Some tests are failed. Please check them.\n"); 2178 else 2179 pr_cont("OK\n"); 2180 return 0; 2181 } 2182 2183 late_initcall(kprobe_trace_self_tests_init); 2184 2185 #endif 2186