1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Kprobes-based tracing events 4 * 5 * Created by Masami Hiramatsu <mhiramat@redhat.com> 6 * 7 */ 8 #define pr_fmt(fmt) "trace_kprobe: " fmt 9 10 #include <linux/bpf-cgroup.h> 11 #include <linux/cleanup.h> 12 #include <linux/error-injection.h> 13 #include <linux/module.h> 14 #include <linux/rculist.h> 15 #include <linux/security.h> 16 #include <linux/uaccess.h> 17 18 #include <asm/setup.h> /* for COMMAND_LINE_SIZE */ 19 20 #include "trace_dynevent.h" 21 #include "trace_kprobe_selftest.h" 22 #include "trace_probe.h" 23 #include "trace_probe_kernel.h" 24 #include "trace_probe_tmpl.h" 25 26 #define KPROBE_EVENT_SYSTEM "kprobes" 27 #define KRETPROBE_MAXACTIVE_MAX 4096 28 29 /* Kprobe early definition from command line */ 30 static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata; 31 32 static int __init set_kprobe_boot_events(char *str) 33 { 34 strscpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE); 35 disable_tracing_selftest("running kprobe events"); 36 37 return 1; 38 } 39 __setup("kprobe_event=", set_kprobe_boot_events); 40 41 static int trace_kprobe_create(const char *raw_command); 42 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); 43 static int trace_kprobe_release(struct dyn_event *ev); 44 static bool trace_kprobe_is_busy(struct dyn_event *ev); 45 static bool trace_kprobe_match(const char *system, const char *event, 46 int argc, const char **argv, struct dyn_event *ev); 47 48 static struct dyn_event_operations trace_kprobe_ops = { 49 .create = trace_kprobe_create, 50 .show = trace_kprobe_show, 51 .is_busy = trace_kprobe_is_busy, 52 .free = trace_kprobe_release, 53 .match = trace_kprobe_match, 54 }; 55 56 /* 57 * Kprobe event core functions 58 */ 59 struct trace_kprobe { 60 struct dyn_event devent; 61 struct kretprobe rp; /* Use rp.kp for kprobe use */ 62 unsigned long __percpu *nhit; 63 const char *symbol; /* symbol name */ 64 struct trace_probe tp; 65 }; 66 67 static bool is_trace_kprobe(struct dyn_event *ev) 68 { 69 return ev->ops == &trace_kprobe_ops; 70 } 71 72 static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev) 73 { 74 return container_of(ev, struct trace_kprobe, devent); 75 } 76 77 /** 78 * for_each_trace_kprobe - iterate over the trace_kprobe list 79 * @pos: the struct trace_kprobe * for each entry 80 * @dpos: the struct dyn_event * to use as a loop cursor 81 */ 82 #define for_each_trace_kprobe(pos, dpos) \ 83 for_each_dyn_event(dpos) \ 84 if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) 85 86 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) 87 { 88 return tk->rp.handler != NULL; 89 } 90 91 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) 92 { 93 return tk->symbol ? tk->symbol : "unknown"; 94 } 95 96 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) 97 { 98 return tk->rp.kp.offset; 99 } 100 101 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) 102 { 103 return kprobe_gone(&tk->rp.kp); 104 } 105 106 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, 107 struct module *mod) 108 { 109 int len = strlen(module_name(mod)); 110 const char *name = trace_kprobe_symbol(tk); 111 112 return strncmp(module_name(mod), name, len) == 0 && name[len] == ':'; 113 } 114 115 #ifdef CONFIG_MODULES 116 static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 117 { 118 char *p; 119 bool ret; 120 121 if (!tk->symbol) 122 return false; 123 p = strchr(tk->symbol, ':'); 124 if (!p) 125 return true; 126 *p = '\0'; 127 scoped_guard(rcu) 128 ret = !!find_module(tk->symbol); 129 *p = ':'; 130 131 return ret; 132 } 133 #else 134 static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 135 { 136 return false; 137 } 138 #endif 139 140 static bool trace_kprobe_is_busy(struct dyn_event *ev) 141 { 142 struct trace_kprobe *tk = to_trace_kprobe(ev); 143 144 return trace_probe_is_enabled(&tk->tp); 145 } 146 147 static bool trace_kprobe_match_command_head(struct trace_kprobe *tk, 148 int argc, const char **argv) 149 { 150 char buf[MAX_ARGSTR_LEN + 1]; 151 152 if (!argc) 153 return true; 154 155 if (!tk->symbol) 156 snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr); 157 else if (tk->rp.kp.offset) 158 snprintf(buf, sizeof(buf), "%s+%u", 159 trace_kprobe_symbol(tk), tk->rp.kp.offset); 160 else 161 snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk)); 162 if (strcmp(buf, argv[0])) 163 return false; 164 argc--; argv++; 165 166 return trace_probe_match_command_args(&tk->tp, argc, argv); 167 } 168 169 static bool trace_kprobe_match(const char *system, const char *event, 170 int argc, const char **argv, struct dyn_event *ev) 171 { 172 struct trace_kprobe *tk = to_trace_kprobe(ev); 173 174 return (event[0] == '\0' || 175 strcmp(trace_probe_name(&tk->tp), event) == 0) && 176 (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) && 177 trace_kprobe_match_command_head(tk, argc, argv); 178 } 179 180 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 181 { 182 unsigned long nhit = 0; 183 int cpu; 184 185 for_each_possible_cpu(cpu) 186 nhit += *per_cpu_ptr(tk->nhit, cpu); 187 188 return nhit; 189 } 190 191 static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk) 192 { 193 return !(list_empty(&tk->rp.kp.list) && 194 hlist_unhashed(&tk->rp.kp.hlist)); 195 } 196 197 /* Return 0 if it fails to find the symbol address */ 198 static nokprobe_inline 199 unsigned long trace_kprobe_address(struct trace_kprobe *tk) 200 { 201 unsigned long addr; 202 203 if (tk->symbol) { 204 addr = (unsigned long) 205 kallsyms_lookup_name(trace_kprobe_symbol(tk)); 206 if (addr) 207 addr += tk->rp.kp.offset; 208 } else { 209 addr = (unsigned long)tk->rp.kp.addr; 210 } 211 return addr; 212 } 213 214 static nokprobe_inline struct trace_kprobe * 215 trace_kprobe_primary_from_call(struct trace_event_call *call) 216 { 217 struct trace_probe *tp; 218 219 tp = trace_probe_primary_from_call(call); 220 if (WARN_ON_ONCE(!tp)) 221 return NULL; 222 223 return container_of(tp, struct trace_kprobe, tp); 224 } 225 226 bool trace_kprobe_on_func_entry(struct trace_event_call *call) 227 { 228 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 229 230 return tk ? (kprobe_on_func_entry(tk->rp.kp.addr, 231 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, 232 tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false; 233 } 234 235 bool trace_kprobe_error_injectable(struct trace_event_call *call) 236 { 237 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 238 239 return tk ? within_error_injection_list(trace_kprobe_address(tk)) : 240 false; 241 } 242 243 static int register_kprobe_event(struct trace_kprobe *tk); 244 static int unregister_kprobe_event(struct trace_kprobe *tk); 245 246 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 247 static int kretprobe_dispatcher(struct kretprobe_instance *ri, 248 struct pt_regs *regs); 249 250 static void free_trace_kprobe(struct trace_kprobe *tk) 251 { 252 if (tk) { 253 trace_probe_cleanup(&tk->tp); 254 kfree(tk->symbol); 255 free_percpu(tk->nhit); 256 kfree(tk); 257 } 258 } 259 260 DEFINE_FREE(free_trace_kprobe, struct trace_kprobe *, 261 if (!IS_ERR_OR_NULL(_T)) free_trace_kprobe(_T)) 262 263 /* 264 * Allocate new trace_probe and initialize it (including kprobes). 265 */ 266 static struct trace_kprobe *alloc_trace_kprobe(const char *group, 267 const char *event, 268 void *addr, 269 const char *symbol, 270 unsigned long offs, 271 int maxactive, 272 int nargs, bool is_return) 273 { 274 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 275 int ret = -ENOMEM; 276 277 tk = kzalloc(struct_size(tk, tp.args, nargs), GFP_KERNEL); 278 if (!tk) 279 return ERR_PTR(ret); 280 281 tk->nhit = alloc_percpu(unsigned long); 282 if (!tk->nhit) 283 return ERR_PTR(ret); 284 285 if (symbol) { 286 tk->symbol = kstrdup(symbol, GFP_KERNEL); 287 if (!tk->symbol) 288 return ERR_PTR(ret); 289 tk->rp.kp.symbol_name = tk->symbol; 290 tk->rp.kp.offset = offs; 291 } else 292 tk->rp.kp.addr = addr; 293 294 if (is_return) 295 tk->rp.handler = kretprobe_dispatcher; 296 else 297 tk->rp.kp.pre_handler = kprobe_dispatcher; 298 299 tk->rp.maxactive = maxactive; 300 INIT_HLIST_NODE(&tk->rp.kp.hlist); 301 INIT_LIST_HEAD(&tk->rp.kp.list); 302 303 ret = trace_probe_init(&tk->tp, event, group, false, nargs); 304 if (ret < 0) 305 return ERR_PTR(ret); 306 307 dyn_event_init(&tk->devent, &trace_kprobe_ops); 308 return_ptr(tk); 309 } 310 311 static struct trace_kprobe *find_trace_kprobe(const char *event, 312 const char *group) 313 { 314 struct dyn_event *pos; 315 struct trace_kprobe *tk; 316 317 for_each_trace_kprobe(tk, pos) 318 if (strcmp(trace_probe_name(&tk->tp), event) == 0 && 319 strcmp(trace_probe_group_name(&tk->tp), group) == 0) 320 return tk; 321 return NULL; 322 } 323 324 static inline int __enable_trace_kprobe(struct trace_kprobe *tk) 325 { 326 int ret = 0; 327 328 if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) { 329 if (trace_kprobe_is_return(tk)) 330 ret = enable_kretprobe(&tk->rp); 331 else 332 ret = enable_kprobe(&tk->rp.kp); 333 } 334 335 return ret; 336 } 337 338 static void __disable_trace_kprobe(struct trace_probe *tp) 339 { 340 struct trace_kprobe *tk; 341 342 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 343 if (!trace_kprobe_is_registered(tk)) 344 continue; 345 if (trace_kprobe_is_return(tk)) 346 disable_kretprobe(&tk->rp); 347 else 348 disable_kprobe(&tk->rp.kp); 349 } 350 } 351 352 /* 353 * Enable trace_probe 354 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 355 */ 356 static int enable_trace_kprobe(struct trace_event_call *call, 357 struct trace_event_file *file) 358 { 359 struct trace_probe *tp; 360 struct trace_kprobe *tk; 361 bool enabled; 362 int ret = 0; 363 364 tp = trace_probe_primary_from_call(call); 365 if (WARN_ON_ONCE(!tp)) 366 return -ENODEV; 367 enabled = trace_probe_is_enabled(tp); 368 369 /* This also changes "enabled" state */ 370 if (file) { 371 ret = trace_probe_add_file(tp, file); 372 if (ret) 373 return ret; 374 } else 375 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 376 377 if (enabled) 378 return 0; 379 380 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 381 if (trace_kprobe_has_gone(tk)) 382 continue; 383 ret = __enable_trace_kprobe(tk); 384 if (ret) 385 break; 386 enabled = true; 387 } 388 389 if (ret) { 390 /* Failed to enable one of them. Roll back all */ 391 if (enabled) 392 __disable_trace_kprobe(tp); 393 if (file) 394 trace_probe_remove_file(tp, file); 395 else 396 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 397 } 398 399 return ret; 400 } 401 402 /* 403 * Disable trace_probe 404 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 405 */ 406 static int disable_trace_kprobe(struct trace_event_call *call, 407 struct trace_event_file *file) 408 { 409 struct trace_probe *tp; 410 411 tp = trace_probe_primary_from_call(call); 412 if (WARN_ON_ONCE(!tp)) 413 return -ENODEV; 414 415 if (file) { 416 if (!trace_probe_get_file_link(tp, file)) 417 return -ENOENT; 418 if (!trace_probe_has_single_file(tp)) 419 goto out; 420 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 421 } else 422 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 423 424 if (!trace_probe_is_enabled(tp)) 425 __disable_trace_kprobe(tp); 426 427 out: 428 if (file) 429 /* 430 * Synchronization is done in below function. For perf event, 431 * file == NULL and perf_trace_event_unreg() calls 432 * tracepoint_synchronize_unregister() to ensure synchronize 433 * event. We don't need to care about it. 434 */ 435 trace_probe_remove_file(tp, file); 436 437 return 0; 438 } 439 440 #if defined(CONFIG_DYNAMIC_FTRACE) && \ 441 !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE) 442 static bool __within_notrace_func(unsigned long addr) 443 { 444 unsigned long offset, size; 445 446 if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset)) 447 return false; 448 449 /* Get the entry address of the target function */ 450 addr -= offset; 451 452 /* 453 * Since ftrace_location_range() does inclusive range check, we need 454 * to subtract 1 byte from the end address. 455 */ 456 return !ftrace_location_range(addr, addr + size - 1); 457 } 458 459 static bool within_notrace_func(struct trace_kprobe *tk) 460 { 461 unsigned long addr = trace_kprobe_address(tk); 462 char symname[KSYM_NAME_LEN], *p; 463 464 if (!__within_notrace_func(addr)) 465 return false; 466 467 /* Check if the address is on a suffixed-symbol */ 468 if (!lookup_symbol_name(addr, symname)) { 469 p = strchr(symname, '.'); 470 if (!p) 471 return true; 472 *p = '\0'; 473 addr = (unsigned long)kprobe_lookup_name(symname, 0); 474 if (addr) 475 return __within_notrace_func(addr); 476 } 477 478 return true; 479 } 480 #else 481 #define within_notrace_func(tk) (false) 482 #endif 483 484 /* Internal register function - just handle k*probes and flags */ 485 static int __register_trace_kprobe(struct trace_kprobe *tk) 486 { 487 int i, ret; 488 489 ret = security_locked_down(LOCKDOWN_KPROBES); 490 if (ret) 491 return ret; 492 493 if (trace_kprobe_is_registered(tk)) 494 return -EINVAL; 495 496 if (within_notrace_func(tk)) { 497 pr_warn("Could not probe notrace function %ps\n", 498 (void *)trace_kprobe_address(tk)); 499 return -EINVAL; 500 } 501 502 for (i = 0; i < tk->tp.nr_args; i++) { 503 ret = traceprobe_update_arg(&tk->tp.args[i]); 504 if (ret) 505 return ret; 506 } 507 508 /* Set/clear disabled flag according to tp->flag */ 509 if (trace_probe_is_enabled(&tk->tp)) 510 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; 511 else 512 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; 513 514 if (trace_kprobe_is_return(tk)) 515 ret = register_kretprobe(&tk->rp); 516 else 517 ret = register_kprobe(&tk->rp.kp); 518 519 return ret; 520 } 521 522 /* Internal unregister function - just handle k*probes and flags */ 523 static void __unregister_trace_kprobe(struct trace_kprobe *tk) 524 { 525 if (trace_kprobe_is_registered(tk)) { 526 if (trace_kprobe_is_return(tk)) 527 unregister_kretprobe(&tk->rp); 528 else 529 unregister_kprobe(&tk->rp.kp); 530 /* Cleanup kprobe for reuse and mark it unregistered */ 531 INIT_HLIST_NODE(&tk->rp.kp.hlist); 532 INIT_LIST_HEAD(&tk->rp.kp.list); 533 if (tk->rp.kp.symbol_name) 534 tk->rp.kp.addr = NULL; 535 } 536 } 537 538 /* Unregister a trace_probe and probe_event */ 539 static int unregister_trace_kprobe(struct trace_kprobe *tk) 540 { 541 /* If other probes are on the event, just unregister kprobe */ 542 if (trace_probe_has_sibling(&tk->tp)) 543 goto unreg; 544 545 /* Enabled event can not be unregistered */ 546 if (trace_probe_is_enabled(&tk->tp)) 547 return -EBUSY; 548 549 /* If there's a reference to the dynamic event */ 550 if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp))) 551 return -EBUSY; 552 553 /* Will fail if probe is being used by ftrace or perf */ 554 if (unregister_kprobe_event(tk)) 555 return -EBUSY; 556 557 unreg: 558 __unregister_trace_kprobe(tk); 559 dyn_event_remove(&tk->devent); 560 trace_probe_unlink(&tk->tp); 561 562 return 0; 563 } 564 565 static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig, 566 struct trace_kprobe *comp) 567 { 568 struct trace_probe_event *tpe = orig->tp.event; 569 int i; 570 571 list_for_each_entry(orig, &tpe->probes, tp.list) { 572 if (strcmp(trace_kprobe_symbol(orig), 573 trace_kprobe_symbol(comp)) || 574 trace_kprobe_offset(orig) != trace_kprobe_offset(comp)) 575 continue; 576 577 /* 578 * trace_probe_compare_arg_type() ensured that nr_args and 579 * each argument name and type are same. Let's compare comm. 580 */ 581 for (i = 0; i < orig->tp.nr_args; i++) { 582 if (strcmp(orig->tp.args[i].comm, 583 comp->tp.args[i].comm)) 584 break; 585 } 586 587 if (i == orig->tp.nr_args) 588 return true; 589 } 590 591 return false; 592 } 593 594 static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to) 595 { 596 int ret; 597 598 ret = trace_probe_compare_arg_type(&tk->tp, &to->tp); 599 if (ret) { 600 /* Note that argument starts index = 2 */ 601 trace_probe_log_set_index(ret + 1); 602 trace_probe_log_err(0, DIFF_ARG_TYPE); 603 return -EEXIST; 604 } 605 if (trace_kprobe_has_same_kprobe(to, tk)) { 606 trace_probe_log_set_index(0); 607 trace_probe_log_err(0, SAME_PROBE); 608 return -EEXIST; 609 } 610 611 /* Append to existing event */ 612 ret = trace_probe_append(&tk->tp, &to->tp); 613 if (ret) 614 return ret; 615 616 /* Register k*probe */ 617 ret = __register_trace_kprobe(tk); 618 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 619 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 620 ret = 0; 621 } 622 623 if (ret) 624 trace_probe_unlink(&tk->tp); 625 else 626 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 627 628 return ret; 629 } 630 631 /* Register a trace_probe and probe_event */ 632 static int register_trace_kprobe(struct trace_kprobe *tk) 633 { 634 struct trace_kprobe *old_tk; 635 int ret; 636 637 guard(mutex)(&event_mutex); 638 639 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), 640 trace_probe_group_name(&tk->tp)); 641 if (old_tk) { 642 if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) { 643 trace_probe_log_set_index(0); 644 trace_probe_log_err(0, DIFF_PROBE_TYPE); 645 return -EEXIST; 646 } 647 return append_trace_kprobe(tk, old_tk); 648 } 649 650 /* Register new event */ 651 ret = register_kprobe_event(tk); 652 if (ret) { 653 if (ret == -EEXIST) { 654 trace_probe_log_set_index(0); 655 trace_probe_log_err(0, EVENT_EXIST); 656 } else 657 pr_warn("Failed to register probe event(%d)\n", ret); 658 return ret; 659 } 660 661 /* Register k*probe */ 662 ret = __register_trace_kprobe(tk); 663 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 664 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 665 ret = 0; 666 } 667 668 if (ret < 0) 669 unregister_kprobe_event(tk); 670 else 671 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 672 673 return ret; 674 } 675 676 #ifdef CONFIG_MODULES 677 static int validate_module_probe_symbol(const char *modname, const char *symbol); 678 679 static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk) 680 { 681 const char *p; 682 int ret = 0; 683 684 p = strchr(trace_kprobe_symbol(tk), ':'); 685 if (p) 686 ret = validate_module_probe_symbol(module_name(mod), p + 1); 687 if (!ret) 688 ret = __register_trace_kprobe(tk); 689 return ret; 690 } 691 692 /* Module notifier call back, checking event on the module */ 693 static int trace_kprobe_module_callback(struct notifier_block *nb, 694 unsigned long val, void *data) 695 { 696 struct module *mod = data; 697 struct dyn_event *pos; 698 struct trace_kprobe *tk; 699 int ret; 700 701 if (val != MODULE_STATE_COMING) 702 return NOTIFY_DONE; 703 704 /* Update probes on coming module */ 705 guard(mutex)(&event_mutex); 706 for_each_trace_kprobe(tk, pos) { 707 if (trace_kprobe_within_module(tk, mod)) { 708 /* Don't need to check busy - this should have gone. */ 709 __unregister_trace_kprobe(tk); 710 ret = register_module_trace_kprobe(mod, tk); 711 if (ret) 712 pr_warn("Failed to re-register probe %s on %s: %d\n", 713 trace_probe_name(&tk->tp), 714 module_name(mod), ret); 715 } 716 } 717 718 return NOTIFY_DONE; 719 } 720 721 static struct notifier_block trace_kprobe_module_nb = { 722 .notifier_call = trace_kprobe_module_callback, 723 .priority = 2 /* Invoked after kprobe and jump_label module callback */ 724 }; 725 static int trace_kprobe_register_module_notifier(void) 726 { 727 return register_module_notifier(&trace_kprobe_module_nb); 728 } 729 #else 730 static int trace_kprobe_register_module_notifier(void) 731 { 732 return 0; 733 } 734 #endif /* CONFIG_MODULES */ 735 736 static int count_symbols(void *data, unsigned long unused) 737 { 738 unsigned int *count = data; 739 740 (*count)++; 741 742 return 0; 743 } 744 745 struct sym_count_ctx { 746 unsigned int count; 747 const char *name; 748 }; 749 750 static int count_mod_symbols(void *data, const char *name, unsigned long unused) 751 { 752 struct sym_count_ctx *ctx = data; 753 754 if (strcmp(name, ctx->name) == 0) 755 ctx->count++; 756 757 return 0; 758 } 759 760 static unsigned int number_of_same_symbols(const char *mod, const char *func_name) 761 { 762 struct sym_count_ctx ctx = { .count = 0, .name = func_name }; 763 764 if (!mod) 765 kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count); 766 767 module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx); 768 769 return ctx.count; 770 } 771 772 static int validate_module_probe_symbol(const char *modname, const char *symbol) 773 { 774 unsigned int count = number_of_same_symbols(modname, symbol); 775 776 if (count > 1) { 777 /* 778 * Users should use ADDR to remove the ambiguity of 779 * using KSYM only. 780 */ 781 return -EADDRNOTAVAIL; 782 } else if (count == 0) { 783 /* 784 * We can return ENOENT earlier than when register the 785 * kprobe. 786 */ 787 return -ENOENT; 788 } 789 return 0; 790 } 791 792 #ifdef CONFIG_MODULES 793 /* Return NULL if the module is not loaded or under unloading. */ 794 static struct module *try_module_get_by_name(const char *name) 795 { 796 struct module *mod; 797 798 guard(rcu)(); 799 mod = find_module(name); 800 if (mod && !try_module_get(mod)) 801 mod = NULL; 802 return mod; 803 } 804 #else 805 #define try_module_get_by_name(name) (NULL) 806 #endif 807 808 static int validate_probe_symbol(char *symbol) 809 { 810 struct module *mod = NULL; 811 char *modname = NULL, *p; 812 int ret = 0; 813 814 p = strchr(symbol, ':'); 815 if (p) { 816 modname = symbol; 817 symbol = p + 1; 818 *p = '\0'; 819 mod = try_module_get_by_name(modname); 820 if (!mod) 821 goto out; 822 } 823 824 ret = validate_module_probe_symbol(modname, symbol); 825 out: 826 if (p) 827 *p = ':'; 828 if (mod) 829 module_put(mod); 830 return ret; 831 } 832 833 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 834 struct pt_regs *regs); 835 836 static int trace_kprobe_create_internal(int argc, const char *argv[], 837 struct traceprobe_parse_context *ctx) 838 { 839 /* 840 * Argument syntax: 841 * - Add kprobe: 842 * p[:[GRP/][EVENT]] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS] 843 * - Add kretprobe: 844 * r[MAXACTIVE][:[GRP/][EVENT]] [MOD:]KSYM[+0] [FETCHARGS] 845 * Or 846 * p[:[GRP/][EVENT]] [MOD:]KSYM[+0]%return [FETCHARGS] 847 * 848 * Fetch args: 849 * $retval : fetch return value 850 * $stack : fetch stack address 851 * $stackN : fetch Nth of stack (N:0-) 852 * $comm : fetch current task comm 853 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 854 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 855 * %REG : fetch register REG 856 * Dereferencing memory fetch: 857 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 858 * Alias name of args: 859 * NAME=FETCHARG : set NAME as alias of FETCHARG. 860 * Type of args: 861 * FETCHARG:TYPE : use TYPE instead of unsigned long. 862 */ 863 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 864 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM; 865 const char **new_argv __free(kfree) = NULL; 866 int i, len, new_argc = 0, ret = 0; 867 char *symbol __free(kfree) = NULL; 868 char *ebuf __free(kfree) = NULL; 869 char *gbuf __free(kfree) = NULL; 870 char *abuf __free(kfree) = NULL; 871 char *dbuf __free(kfree) = NULL; 872 enum probe_print_type ptype; 873 bool is_return = false; 874 int maxactive = 0; 875 void *addr = NULL; 876 char *tmp = NULL; 877 long offset = 0; 878 879 switch (argv[0][0]) { 880 case 'r': 881 is_return = true; 882 break; 883 case 'p': 884 break; 885 default: 886 return -ECANCELED; 887 } 888 if (argc < 2) 889 return -ECANCELED; 890 891 event = strchr(&argv[0][1], ':'); 892 if (event) 893 event++; 894 895 if (isdigit(argv[0][1])) { 896 char *buf __free(kfree) = NULL; 897 898 if (!is_return) { 899 trace_probe_log_err(1, BAD_MAXACT_TYPE); 900 return -EINVAL; 901 } 902 if (event) 903 len = event - &argv[0][1] - 1; 904 else 905 len = strlen(&argv[0][1]); 906 if (len > MAX_EVENT_NAME_LEN - 1) { 907 trace_probe_log_err(1, BAD_MAXACT); 908 return -EINVAL; 909 } 910 buf = kmemdup(&argv[0][1], len + 1, GFP_KERNEL); 911 if (!buf) 912 return -ENOMEM; 913 buf[len] = '\0'; 914 ret = kstrtouint(buf, 0, &maxactive); 915 if (ret || !maxactive) { 916 trace_probe_log_err(1, BAD_MAXACT); 917 return -EINVAL; 918 } 919 /* kretprobes instances are iterated over via a list. The 920 * maximum should stay reasonable. 921 */ 922 if (maxactive > KRETPROBE_MAXACTIVE_MAX) { 923 trace_probe_log_err(1, MAXACT_TOO_BIG); 924 return -EINVAL; 925 } 926 } 927 928 /* try to parse an address. if that fails, try to read the 929 * input as a symbol. */ 930 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { 931 trace_probe_log_set_index(1); 932 /* Check whether uprobe event specified */ 933 if (strchr(argv[1], '/') && strchr(argv[1], ':')) 934 return -ECANCELED; 935 936 /* a symbol specified */ 937 symbol = kstrdup(argv[1], GFP_KERNEL); 938 if (!symbol) 939 return -ENOMEM; 940 941 tmp = strchr(symbol, '%'); 942 if (tmp) { 943 if (!strcmp(tmp, "%return")) { 944 *tmp = '\0'; 945 is_return = true; 946 } else { 947 trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX); 948 return -EINVAL; 949 } 950 } 951 952 /* TODO: support .init module functions */ 953 ret = traceprobe_split_symbol_offset(symbol, &offset); 954 if (ret || offset < 0 || offset > UINT_MAX) { 955 trace_probe_log_err(0, BAD_PROBE_ADDR); 956 return -EINVAL; 957 } 958 ret = validate_probe_symbol(symbol); 959 if (ret) { 960 if (ret == -EADDRNOTAVAIL) 961 trace_probe_log_err(0, NON_UNIQ_SYMBOL); 962 else 963 trace_probe_log_err(0, BAD_PROBE_ADDR); 964 return -EINVAL; 965 } 966 if (is_return) 967 ctx->flags |= TPARG_FL_RETURN; 968 ret = kprobe_on_func_entry(NULL, symbol, offset); 969 if (ret == 0 && !is_return) 970 ctx->flags |= TPARG_FL_FENTRY; 971 /* Defer the ENOENT case until register kprobe */ 972 if (ret == -EINVAL && is_return) { 973 trace_probe_log_err(0, BAD_RETPROBE); 974 return -EINVAL; 975 } 976 } 977 978 trace_probe_log_set_index(0); 979 if (event) { 980 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 981 if (!gbuf) 982 return -ENOMEM; 983 ret = traceprobe_parse_event_name(&event, &group, gbuf, 984 event - argv[0]); 985 if (ret) 986 return ret; 987 } 988 989 if (!event) { 990 /* Make a new event name */ 991 ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 992 if (!ebuf) 993 return -ENOMEM; 994 if (symbol) 995 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 996 is_return ? 'r' : 'p', symbol, offset); 997 else 998 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_0x%p", 999 is_return ? 'r' : 'p', addr); 1000 sanitize_event_name(ebuf); 1001 event = ebuf; 1002 } 1003 1004 abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL); 1005 if (!abuf) 1006 return -ENOMEM; 1007 argc -= 2; argv += 2; 1008 ctx->funcname = symbol; 1009 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, 1010 abuf, MAX_BTF_ARGS_LEN, ctx); 1011 if (IS_ERR(new_argv)) { 1012 ret = PTR_ERR(new_argv); 1013 new_argv = NULL; 1014 return ret; 1015 } 1016 if (new_argv) { 1017 argc = new_argc; 1018 argv = new_argv; 1019 } 1020 if (argc > MAX_TRACE_ARGS) { 1021 trace_probe_log_set_index(2); 1022 trace_probe_log_err(0, TOO_MANY_ARGS); 1023 return -E2BIG; 1024 } 1025 1026 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); 1027 if (ret) 1028 return ret; 1029 1030 /* setup a probe */ 1031 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, 1032 argc, is_return); 1033 if (IS_ERR(tk)) { 1034 ret = PTR_ERR(tk); 1035 /* This must return -ENOMEM, else there is a bug */ 1036 WARN_ON_ONCE(ret != -ENOMEM); 1037 return ret; /* We know tk is not allocated */ 1038 } 1039 1040 /* parse arguments */ 1041 for (i = 0; i < argc; i++) { 1042 trace_probe_log_set_index(i + 2); 1043 ctx->offset = 0; 1044 ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], ctx); 1045 if (ret) 1046 return ret; /* This can be -ENOMEM */ 1047 } 1048 /* entry handler for kretprobe */ 1049 if (is_return && tk->tp.entry_arg) { 1050 tk->rp.entry_handler = trace_kprobe_entry_handler; 1051 tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp); 1052 } 1053 1054 ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1055 ret = traceprobe_set_print_fmt(&tk->tp, ptype); 1056 if (ret < 0) 1057 return ret; 1058 1059 ret = register_trace_kprobe(tk); 1060 if (ret) { 1061 trace_probe_log_set_index(1); 1062 if (ret == -EILSEQ) 1063 trace_probe_log_err(0, BAD_INSN_BNDRY); 1064 else if (ret == -ENOENT) 1065 trace_probe_log_err(0, BAD_PROBE_ADDR); 1066 else if (ret != -ENOMEM && ret != -EEXIST) 1067 trace_probe_log_err(0, FAIL_REG_PROBE); 1068 return ret; 1069 } 1070 /* 1071 * Here, 'tk' has been registered to the list successfully, 1072 * so we don't need to free it. 1073 */ 1074 tk = NULL; 1075 1076 return 0; 1077 } 1078 1079 static int trace_kprobe_create_cb(int argc, const char *argv[]) 1080 { 1081 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 1082 int ret; 1083 1084 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 1085 if (!ctx) 1086 return -ENOMEM; 1087 ctx->flags = TPARG_FL_KERNEL; 1088 1089 trace_probe_log_init("trace_kprobe", argc, argv); 1090 1091 ret = trace_kprobe_create_internal(argc, argv, ctx); 1092 1093 trace_probe_log_clear(); 1094 return ret; 1095 } 1096 1097 static int trace_kprobe_create(const char *raw_command) 1098 { 1099 return trace_probe_create(raw_command, trace_kprobe_create_cb); 1100 } 1101 1102 static int create_or_delete_trace_kprobe(const char *raw_command) 1103 { 1104 int ret; 1105 1106 if (raw_command[0] == '-') 1107 return dyn_event_release(raw_command, &trace_kprobe_ops); 1108 1109 ret = dyn_event_create(raw_command, &trace_kprobe_ops); 1110 return ret == -ECANCELED ? -EINVAL : ret; 1111 } 1112 1113 static int trace_kprobe_run_command(struct dynevent_cmd *cmd) 1114 { 1115 return create_or_delete_trace_kprobe(cmd->seq.buffer); 1116 } 1117 1118 /** 1119 * kprobe_event_cmd_init - Initialize a kprobe event command object 1120 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1121 * @buf: A pointer to the buffer used to build the command 1122 * @maxlen: The length of the buffer passed in @buf 1123 * 1124 * Initialize a synthetic event command object. Use this before 1125 * calling any of the other kprobe_event functions. 1126 */ 1127 void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen) 1128 { 1129 dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE, 1130 trace_kprobe_run_command); 1131 } 1132 EXPORT_SYMBOL_GPL(kprobe_event_cmd_init); 1133 1134 /** 1135 * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list 1136 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1137 * @kretprobe: Is this a return probe? 1138 * @name: The name of the kprobe event 1139 * @loc: The location of the kprobe event 1140 * @...: Variable number of arg (pairs), one pair for each field 1141 * 1142 * NOTE: Users normally won't want to call this function directly, but 1143 * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically 1144 * adds a NULL to the end of the arg list. If this function is used 1145 * directly, make sure the last arg in the variable arg list is NULL. 1146 * 1147 * Generate a kprobe event command to be executed by 1148 * kprobe_event_gen_cmd_end(). This function can be used to generate the 1149 * complete command or only the first part of it; in the latter case, 1150 * kprobe_event_add_fields() can be used to add more fields following this. 1151 * 1152 * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This 1153 * returns -EINVAL if @loc == NULL. 1154 * 1155 * Return: 0 if successful, error otherwise. 1156 */ 1157 int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe, 1158 const char *name, const char *loc, ...) 1159 { 1160 char buf[MAX_EVENT_NAME_LEN]; 1161 struct dynevent_arg arg; 1162 va_list args; 1163 int ret; 1164 1165 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1166 return -EINVAL; 1167 1168 if (!loc) 1169 return -EINVAL; 1170 1171 if (kretprobe) 1172 snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name); 1173 else 1174 snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name); 1175 1176 ret = dynevent_str_add(cmd, buf); 1177 if (ret) 1178 return ret; 1179 1180 dynevent_arg_init(&arg, 0); 1181 arg.str = loc; 1182 ret = dynevent_arg_add(cmd, &arg, NULL); 1183 if (ret) 1184 return ret; 1185 1186 va_start(args, loc); 1187 for (;;) { 1188 const char *field; 1189 1190 field = va_arg(args, const char *); 1191 if (!field) 1192 break; 1193 1194 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1195 ret = -EINVAL; 1196 break; 1197 } 1198 1199 arg.str = field; 1200 ret = dynevent_arg_add(cmd, &arg, NULL); 1201 if (ret) 1202 break; 1203 } 1204 va_end(args); 1205 1206 return ret; 1207 } 1208 EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start); 1209 1210 /** 1211 * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list 1212 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1213 * @...: Variable number of arg (pairs), one pair for each field 1214 * 1215 * NOTE: Users normally won't want to call this function directly, but 1216 * rather use the kprobe_event_add_fields() wrapper, which 1217 * automatically adds a NULL to the end of the arg list. If this 1218 * function is used directly, make sure the last arg in the variable 1219 * arg list is NULL. 1220 * 1221 * Add probe fields to an existing kprobe command using a variable 1222 * list of args. Fields are added in the same order they're listed. 1223 * 1224 * Return: 0 if successful, error otherwise. 1225 */ 1226 int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...) 1227 { 1228 struct dynevent_arg arg; 1229 va_list args; 1230 int ret = 0; 1231 1232 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1233 return -EINVAL; 1234 1235 dynevent_arg_init(&arg, 0); 1236 1237 va_start(args, cmd); 1238 for (;;) { 1239 const char *field; 1240 1241 field = va_arg(args, const char *); 1242 if (!field) 1243 break; 1244 1245 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1246 ret = -EINVAL; 1247 break; 1248 } 1249 1250 arg.str = field; 1251 ret = dynevent_arg_add(cmd, &arg, NULL); 1252 if (ret) 1253 break; 1254 } 1255 va_end(args); 1256 1257 return ret; 1258 } 1259 EXPORT_SYMBOL_GPL(__kprobe_event_add_fields); 1260 1261 /** 1262 * kprobe_event_delete - Delete a kprobe event 1263 * @name: The name of the kprobe event to delete 1264 * 1265 * Delete a kprobe event with the give @name from kernel code rather 1266 * than directly from the command line. 1267 * 1268 * Return: 0 if successful, error otherwise. 1269 */ 1270 int kprobe_event_delete(const char *name) 1271 { 1272 char buf[MAX_EVENT_NAME_LEN]; 1273 1274 snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name); 1275 1276 return create_or_delete_trace_kprobe(buf); 1277 } 1278 EXPORT_SYMBOL_GPL(kprobe_event_delete); 1279 1280 static int trace_kprobe_release(struct dyn_event *ev) 1281 { 1282 struct trace_kprobe *tk = to_trace_kprobe(ev); 1283 int ret = unregister_trace_kprobe(tk); 1284 1285 if (!ret) 1286 free_trace_kprobe(tk); 1287 return ret; 1288 } 1289 1290 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) 1291 { 1292 struct trace_kprobe *tk = to_trace_kprobe(ev); 1293 int i; 1294 1295 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); 1296 if (trace_kprobe_is_return(tk) && tk->rp.maxactive) 1297 seq_printf(m, "%d", tk->rp.maxactive); 1298 seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp), 1299 trace_probe_name(&tk->tp)); 1300 1301 if (!tk->symbol) 1302 seq_printf(m, " 0x%p", tk->rp.kp.addr); 1303 else if (tk->rp.kp.offset) 1304 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk), 1305 tk->rp.kp.offset); 1306 else 1307 seq_printf(m, " %s", trace_kprobe_symbol(tk)); 1308 1309 for (i = 0; i < tk->tp.nr_args; i++) 1310 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); 1311 seq_putc(m, '\n'); 1312 1313 return 0; 1314 } 1315 1316 static int probes_seq_show(struct seq_file *m, void *v) 1317 { 1318 struct dyn_event *ev = v; 1319 1320 if (!is_trace_kprobe(ev)) 1321 return 0; 1322 1323 return trace_kprobe_show(m, ev); 1324 } 1325 1326 static const struct seq_operations probes_seq_op = { 1327 .start = dyn_event_seq_start, 1328 .next = dyn_event_seq_next, 1329 .stop = dyn_event_seq_stop, 1330 .show = probes_seq_show 1331 }; 1332 1333 static int probes_open(struct inode *inode, struct file *file) 1334 { 1335 int ret; 1336 1337 ret = security_locked_down(LOCKDOWN_TRACEFS); 1338 if (ret) 1339 return ret; 1340 1341 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1342 ret = dyn_events_release_all(&trace_kprobe_ops); 1343 if (ret < 0) 1344 return ret; 1345 } 1346 1347 return seq_open(file, &probes_seq_op); 1348 } 1349 1350 static ssize_t probes_write(struct file *file, const char __user *buffer, 1351 size_t count, loff_t *ppos) 1352 { 1353 return trace_parse_run_command(file, buffer, count, ppos, 1354 create_or_delete_trace_kprobe); 1355 } 1356 1357 static const struct file_operations kprobe_events_ops = { 1358 .owner = THIS_MODULE, 1359 .open = probes_open, 1360 .read = seq_read, 1361 .llseek = seq_lseek, 1362 .release = seq_release, 1363 .write = probes_write, 1364 }; 1365 1366 static unsigned long trace_kprobe_missed(struct trace_kprobe *tk) 1367 { 1368 return trace_kprobe_is_return(tk) ? 1369 tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed; 1370 } 1371 1372 /* Probes profiling interfaces */ 1373 static int probes_profile_seq_show(struct seq_file *m, void *v) 1374 { 1375 struct dyn_event *ev = v; 1376 struct trace_kprobe *tk; 1377 unsigned long nmissed; 1378 1379 if (!is_trace_kprobe(ev)) 1380 return 0; 1381 1382 tk = to_trace_kprobe(ev); 1383 nmissed = trace_kprobe_missed(tk); 1384 seq_printf(m, " %-44s %15lu %15lu\n", 1385 trace_probe_name(&tk->tp), 1386 trace_kprobe_nhit(tk), 1387 nmissed); 1388 1389 return 0; 1390 } 1391 1392 static const struct seq_operations profile_seq_op = { 1393 .start = dyn_event_seq_start, 1394 .next = dyn_event_seq_next, 1395 .stop = dyn_event_seq_stop, 1396 .show = probes_profile_seq_show 1397 }; 1398 1399 static int profile_open(struct inode *inode, struct file *file) 1400 { 1401 int ret; 1402 1403 ret = security_locked_down(LOCKDOWN_TRACEFS); 1404 if (ret) 1405 return ret; 1406 1407 return seq_open(file, &profile_seq_op); 1408 } 1409 1410 static const struct file_operations kprobe_profile_ops = { 1411 .owner = THIS_MODULE, 1412 .open = profile_open, 1413 .read = seq_read, 1414 .llseek = seq_lseek, 1415 .release = seq_release, 1416 }; 1417 1418 /* Note that we don't verify it, since the code does not come from user space */ 1419 static int 1420 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 1421 void *dest, void *base) 1422 { 1423 struct pt_regs *regs = rec; 1424 unsigned long val; 1425 int ret; 1426 1427 retry: 1428 /* 1st stage: get value from context */ 1429 switch (code->op) { 1430 case FETCH_OP_REG: 1431 val = regs_get_register(regs, code->param); 1432 break; 1433 case FETCH_OP_STACK: 1434 val = regs_get_kernel_stack_nth(regs, code->param); 1435 break; 1436 case FETCH_OP_STACKP: 1437 val = kernel_stack_pointer(regs); 1438 break; 1439 case FETCH_OP_RETVAL: 1440 val = regs_return_value(regs); 1441 break; 1442 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 1443 case FETCH_OP_ARG: 1444 val = regs_get_kernel_argument(regs, code->param); 1445 break; 1446 case FETCH_OP_EDATA: 1447 val = *(unsigned long *)((unsigned long)edata + code->offset); 1448 break; 1449 #endif 1450 case FETCH_NOP_SYMBOL: /* Ignore a place holder */ 1451 code++; 1452 goto retry; 1453 default: 1454 ret = process_common_fetch_insn(code, &val); 1455 if (ret < 0) 1456 return ret; 1457 } 1458 code++; 1459 1460 return process_fetch_insn_bottom(code, val, dest, base); 1461 } 1462 NOKPROBE_SYMBOL(process_fetch_insn) 1463 1464 /* Kprobe handler */ 1465 static nokprobe_inline void 1466 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 1467 struct trace_event_file *trace_file) 1468 { 1469 struct kprobe_trace_entry_head *entry; 1470 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1471 struct trace_event_buffer fbuffer; 1472 int dsize; 1473 1474 WARN_ON(call != trace_file->event_call); 1475 1476 if (trace_trigger_soft_disabled(trace_file)) 1477 return; 1478 1479 dsize = __get_data_size(&tk->tp, regs, NULL); 1480 1481 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1482 sizeof(*entry) + tk->tp.size + dsize); 1483 if (!entry) 1484 return; 1485 1486 fbuffer.regs = regs; 1487 entry->ip = (unsigned long)tk->rp.kp.addr; 1488 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1489 1490 trace_event_buffer_commit(&fbuffer); 1491 } 1492 1493 static void 1494 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) 1495 { 1496 struct event_file_link *link; 1497 1498 trace_probe_for_each_link_rcu(link, &tk->tp) 1499 __kprobe_trace_func(tk, regs, link->file); 1500 } 1501 NOKPROBE_SYMBOL(kprobe_trace_func); 1502 1503 /* Kretprobe handler */ 1504 1505 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 1506 struct pt_regs *regs) 1507 { 1508 struct kretprobe *rp = get_kretprobe(ri); 1509 struct trace_kprobe *tk; 1510 1511 /* 1512 * There is a small chance that get_kretprobe(ri) returns NULL when 1513 * the kretprobe is unregister on another CPU between kretprobe's 1514 * trampoline_handler and this function. 1515 */ 1516 if (unlikely(!rp)) 1517 return -ENOENT; 1518 1519 tk = container_of(rp, struct trace_kprobe, rp); 1520 1521 /* store argument values into ri->data as entry data */ 1522 if (tk->tp.entry_arg) 1523 store_trace_entry_data(ri->data, &tk->tp, regs); 1524 1525 return 0; 1526 } 1527 1528 1529 static nokprobe_inline void 1530 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1531 struct pt_regs *regs, 1532 struct trace_event_file *trace_file) 1533 { 1534 struct kretprobe_trace_entry_head *entry; 1535 struct trace_event_buffer fbuffer; 1536 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1537 int dsize; 1538 1539 WARN_ON(call != trace_file->event_call); 1540 1541 if (trace_trigger_soft_disabled(trace_file)) 1542 return; 1543 1544 dsize = __get_data_size(&tk->tp, regs, ri->data); 1545 1546 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1547 sizeof(*entry) + tk->tp.size + dsize); 1548 if (!entry) 1549 return; 1550 1551 fbuffer.regs = regs; 1552 entry->func = (unsigned long)tk->rp.kp.addr; 1553 entry->ret_ip = get_kretprobe_retaddr(ri); 1554 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1555 1556 trace_event_buffer_commit(&fbuffer); 1557 } 1558 1559 static void 1560 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1561 struct pt_regs *regs) 1562 { 1563 struct event_file_link *link; 1564 1565 trace_probe_for_each_link_rcu(link, &tk->tp) 1566 __kretprobe_trace_func(tk, ri, regs, link->file); 1567 } 1568 NOKPROBE_SYMBOL(kretprobe_trace_func); 1569 1570 /* Event entry printers */ 1571 static enum print_line_t 1572 print_kprobe_event(struct trace_iterator *iter, int flags, 1573 struct trace_event *event) 1574 { 1575 struct kprobe_trace_entry_head *field; 1576 struct trace_seq *s = &iter->seq; 1577 struct trace_probe *tp; 1578 1579 field = (struct kprobe_trace_entry_head *)iter->ent; 1580 tp = trace_probe_primary_from_call( 1581 container_of(event, struct trace_event_call, event)); 1582 if (WARN_ON_ONCE(!tp)) 1583 goto out; 1584 1585 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1586 1587 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) 1588 goto out; 1589 1590 trace_seq_putc(s, ')'); 1591 1592 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1593 (u8 *)&field[1], field) < 0) 1594 goto out; 1595 1596 trace_seq_putc(s, '\n'); 1597 out: 1598 return trace_handle_return(s); 1599 } 1600 1601 static enum print_line_t 1602 print_kretprobe_event(struct trace_iterator *iter, int flags, 1603 struct trace_event *event) 1604 { 1605 struct kretprobe_trace_entry_head *field; 1606 struct trace_seq *s = &iter->seq; 1607 struct trace_probe *tp; 1608 1609 field = (struct kretprobe_trace_entry_head *)iter->ent; 1610 tp = trace_probe_primary_from_call( 1611 container_of(event, struct trace_event_call, event)); 1612 if (WARN_ON_ONCE(!tp)) 1613 goto out; 1614 1615 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1616 1617 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) 1618 goto out; 1619 1620 trace_seq_puts(s, " <- "); 1621 1622 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) 1623 goto out; 1624 1625 trace_seq_putc(s, ')'); 1626 1627 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1628 (u8 *)&field[1], field) < 0) 1629 goto out; 1630 1631 trace_seq_putc(s, '\n'); 1632 1633 out: 1634 return trace_handle_return(s); 1635 } 1636 1637 1638 static int kprobe_event_define_fields(struct trace_event_call *event_call) 1639 { 1640 int ret; 1641 struct kprobe_trace_entry_head field; 1642 struct trace_probe *tp; 1643 1644 tp = trace_probe_primary_from_call(event_call); 1645 if (WARN_ON_ONCE(!tp)) 1646 return -ENOENT; 1647 1648 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1649 1650 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1651 } 1652 1653 static int kretprobe_event_define_fields(struct trace_event_call *event_call) 1654 { 1655 int ret; 1656 struct kretprobe_trace_entry_head field; 1657 struct trace_probe *tp; 1658 1659 tp = trace_probe_primary_from_call(event_call); 1660 if (WARN_ON_ONCE(!tp)) 1661 return -ENOENT; 1662 1663 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1664 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1665 1666 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1667 } 1668 1669 #ifdef CONFIG_PERF_EVENTS 1670 1671 /* Kprobe profile handler */ 1672 static int 1673 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) 1674 { 1675 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1676 struct kprobe_trace_entry_head *entry; 1677 struct hlist_head *head; 1678 int size, __size, dsize; 1679 int rctx; 1680 1681 if (bpf_prog_array_valid(call)) { 1682 unsigned long orig_ip = instruction_pointer(regs); 1683 int ret; 1684 1685 ret = trace_call_bpf(call, regs); 1686 1687 /* 1688 * We need to check and see if we modified the pc of the 1689 * pt_regs, and if so return 1 so that we don't do the 1690 * single stepping. 1691 */ 1692 if (orig_ip != instruction_pointer(regs)) 1693 return 1; 1694 if (!ret) 1695 return 0; 1696 } 1697 1698 head = this_cpu_ptr(call->perf_events); 1699 if (hlist_empty(head)) 1700 return 0; 1701 1702 dsize = __get_data_size(&tk->tp, regs, NULL); 1703 __size = sizeof(*entry) + tk->tp.size + dsize; 1704 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1705 size -= sizeof(u32); 1706 1707 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1708 if (!entry) 1709 return 0; 1710 1711 entry->ip = (unsigned long)tk->rp.kp.addr; 1712 memset(&entry[1], 0, dsize); 1713 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1714 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1715 head, NULL); 1716 return 0; 1717 } 1718 NOKPROBE_SYMBOL(kprobe_perf_func); 1719 1720 /* Kretprobe profile handler */ 1721 static void 1722 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1723 struct pt_regs *regs) 1724 { 1725 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1726 struct kretprobe_trace_entry_head *entry; 1727 struct hlist_head *head; 1728 int size, __size, dsize; 1729 int rctx; 1730 1731 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs)) 1732 return; 1733 1734 head = this_cpu_ptr(call->perf_events); 1735 if (hlist_empty(head)) 1736 return; 1737 1738 dsize = __get_data_size(&tk->tp, regs, ri->data); 1739 __size = sizeof(*entry) + tk->tp.size + dsize; 1740 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1741 size -= sizeof(u32); 1742 1743 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1744 if (!entry) 1745 return; 1746 1747 entry->func = (unsigned long)tk->rp.kp.addr; 1748 entry->ret_ip = get_kretprobe_retaddr(ri); 1749 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1750 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1751 head, NULL); 1752 } 1753 NOKPROBE_SYMBOL(kretprobe_perf_func); 1754 1755 int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, 1756 const char **symbol, u64 *probe_offset, 1757 u64 *probe_addr, unsigned long *missed, 1758 bool perf_type_tracepoint) 1759 { 1760 const char *pevent = trace_event_name(event->tp_event); 1761 const char *group = event->tp_event->class->system; 1762 struct trace_kprobe *tk; 1763 1764 if (perf_type_tracepoint) 1765 tk = find_trace_kprobe(pevent, group); 1766 else 1767 tk = trace_kprobe_primary_from_call(event->tp_event); 1768 if (!tk) 1769 return -EINVAL; 1770 1771 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE 1772 : BPF_FD_TYPE_KPROBE; 1773 *probe_offset = tk->rp.kp.offset; 1774 *probe_addr = kallsyms_show_value(current_cred()) ? 1775 (unsigned long)tk->rp.kp.addr : 0; 1776 *symbol = tk->symbol; 1777 if (missed) 1778 *missed = trace_kprobe_missed(tk); 1779 return 0; 1780 } 1781 #endif /* CONFIG_PERF_EVENTS */ 1782 1783 /* 1784 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1785 * 1786 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe 1787 * lockless, but we can't race with this __init function. 1788 */ 1789 static int kprobe_register(struct trace_event_call *event, 1790 enum trace_reg type, void *data) 1791 { 1792 struct trace_event_file *file = data; 1793 1794 switch (type) { 1795 case TRACE_REG_REGISTER: 1796 return enable_trace_kprobe(event, file); 1797 case TRACE_REG_UNREGISTER: 1798 return disable_trace_kprobe(event, file); 1799 1800 #ifdef CONFIG_PERF_EVENTS 1801 case TRACE_REG_PERF_REGISTER: 1802 return enable_trace_kprobe(event, NULL); 1803 case TRACE_REG_PERF_UNREGISTER: 1804 return disable_trace_kprobe(event, NULL); 1805 case TRACE_REG_PERF_OPEN: 1806 case TRACE_REG_PERF_CLOSE: 1807 case TRACE_REG_PERF_ADD: 1808 case TRACE_REG_PERF_DEL: 1809 return 0; 1810 #endif 1811 } 1812 return 0; 1813 } 1814 1815 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) 1816 { 1817 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); 1818 int ret = 0; 1819 1820 raw_cpu_inc(*tk->nhit); 1821 1822 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) 1823 kprobe_trace_func(tk, regs); 1824 #ifdef CONFIG_PERF_EVENTS 1825 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) 1826 ret = kprobe_perf_func(tk, regs); 1827 #endif 1828 return ret; 1829 } 1830 NOKPROBE_SYMBOL(kprobe_dispatcher); 1831 1832 static int 1833 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) 1834 { 1835 struct kretprobe *rp = get_kretprobe(ri); 1836 struct trace_kprobe *tk; 1837 1838 /* 1839 * There is a small chance that get_kretprobe(ri) returns NULL when 1840 * the kretprobe is unregister on another CPU between kretprobe's 1841 * trampoline_handler and this function. 1842 */ 1843 if (unlikely(!rp)) 1844 return 0; 1845 1846 tk = container_of(rp, struct trace_kprobe, rp); 1847 raw_cpu_inc(*tk->nhit); 1848 1849 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) 1850 kretprobe_trace_func(tk, ri, regs); 1851 #ifdef CONFIG_PERF_EVENTS 1852 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) 1853 kretprobe_perf_func(tk, ri, regs); 1854 #endif 1855 return 0; /* We don't tweak kernel, so just return 0 */ 1856 } 1857 NOKPROBE_SYMBOL(kretprobe_dispatcher); 1858 1859 static struct trace_event_functions kretprobe_funcs = { 1860 .trace = print_kretprobe_event 1861 }; 1862 1863 static struct trace_event_functions kprobe_funcs = { 1864 .trace = print_kprobe_event 1865 }; 1866 1867 static struct trace_event_fields kretprobe_fields_array[] = { 1868 { .type = TRACE_FUNCTION_TYPE, 1869 .define_fields = kretprobe_event_define_fields }, 1870 {} 1871 }; 1872 1873 static struct trace_event_fields kprobe_fields_array[] = { 1874 { .type = TRACE_FUNCTION_TYPE, 1875 .define_fields = kprobe_event_define_fields }, 1876 {} 1877 }; 1878 1879 static inline void init_trace_event_call(struct trace_kprobe *tk) 1880 { 1881 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1882 1883 if (trace_kprobe_is_return(tk)) { 1884 call->event.funcs = &kretprobe_funcs; 1885 call->class->fields_array = kretprobe_fields_array; 1886 } else { 1887 call->event.funcs = &kprobe_funcs; 1888 call->class->fields_array = kprobe_fields_array; 1889 } 1890 1891 call->flags = TRACE_EVENT_FL_KPROBE; 1892 call->class->reg = kprobe_register; 1893 } 1894 1895 static int register_kprobe_event(struct trace_kprobe *tk) 1896 { 1897 init_trace_event_call(tk); 1898 1899 return trace_probe_register_event_call(&tk->tp); 1900 } 1901 1902 static int unregister_kprobe_event(struct trace_kprobe *tk) 1903 { 1904 return trace_probe_unregister_event_call(&tk->tp); 1905 } 1906 1907 #ifdef CONFIG_PERF_EVENTS 1908 1909 /* create a trace_kprobe, but don't add it to global lists */ 1910 struct trace_event_call * 1911 create_local_trace_kprobe(char *func, void *addr, unsigned long offs, 1912 bool is_return) 1913 { 1914 enum probe_print_type ptype; 1915 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 1916 int ret; 1917 char *event; 1918 1919 if (func) { 1920 ret = validate_probe_symbol(func); 1921 if (ret) 1922 return ERR_PTR(ret); 1923 } 1924 1925 /* 1926 * local trace_kprobes are not added to dyn_event, so they are never 1927 * searched in find_trace_kprobe(). Therefore, there is no concern of 1928 * duplicated name here. 1929 */ 1930 event = func ? func : "DUMMY_EVENT"; 1931 1932 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func, 1933 offs, 0 /* maxactive */, 0 /* nargs */, 1934 is_return); 1935 1936 if (IS_ERR(tk)) { 1937 pr_info("Failed to allocate trace_probe.(%d)\n", 1938 (int)PTR_ERR(tk)); 1939 return ERR_CAST(tk); 1940 } 1941 1942 init_trace_event_call(tk); 1943 1944 ptype = trace_kprobe_is_return(tk) ? 1945 PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1946 if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) 1947 return ERR_PTR(-ENOMEM); 1948 1949 ret = __register_trace_kprobe(tk); 1950 if (ret < 0) 1951 return ERR_PTR(ret); 1952 1953 return trace_probe_event_call(&(no_free_ptr(tk)->tp)); 1954 } 1955 1956 void destroy_local_trace_kprobe(struct trace_event_call *event_call) 1957 { 1958 struct trace_kprobe *tk; 1959 1960 tk = trace_kprobe_primary_from_call(event_call); 1961 if (unlikely(!tk)) 1962 return; 1963 1964 if (trace_probe_is_enabled(&tk->tp)) { 1965 WARN_ON(1); 1966 return; 1967 } 1968 1969 __unregister_trace_kprobe(tk); 1970 1971 free_trace_kprobe(tk); 1972 } 1973 #endif /* CONFIG_PERF_EVENTS */ 1974 1975 static __init void enable_boot_kprobe_events(void) 1976 { 1977 struct trace_array *tr = top_trace_array(); 1978 struct trace_event_file *file; 1979 struct trace_kprobe *tk; 1980 struct dyn_event *pos; 1981 1982 guard(mutex)(&event_mutex); 1983 for_each_trace_kprobe(tk, pos) { 1984 list_for_each_entry(file, &tr->events, list) 1985 if (file->event_call == trace_probe_event_call(&tk->tp)) 1986 trace_event_enable_disable(file, 1, 0); 1987 } 1988 } 1989 1990 static __init void setup_boot_kprobe_events(void) 1991 { 1992 char *p, *cmd = kprobe_boot_events_buf; 1993 int ret; 1994 1995 strreplace(kprobe_boot_events_buf, ',', ' '); 1996 1997 while (cmd && *cmd != '\0') { 1998 p = strchr(cmd, ';'); 1999 if (p) 2000 *p++ = '\0'; 2001 2002 ret = create_or_delete_trace_kprobe(cmd); 2003 if (ret) 2004 pr_warn("Failed to add event(%d): %s\n", ret, cmd); 2005 2006 cmd = p; 2007 } 2008 2009 enable_boot_kprobe_events(); 2010 } 2011 2012 /* 2013 * Register dynevent at core_initcall. This allows kernel to setup kprobe 2014 * events in postcore_initcall without tracefs. 2015 */ 2016 static __init int init_kprobe_trace_early(void) 2017 { 2018 int ret; 2019 2020 ret = dyn_event_register(&trace_kprobe_ops); 2021 if (ret) 2022 return ret; 2023 2024 if (trace_kprobe_register_module_notifier()) 2025 return -EINVAL; 2026 2027 return 0; 2028 } 2029 core_initcall(init_kprobe_trace_early); 2030 2031 /* Make a tracefs interface for controlling probe points */ 2032 static __init int init_kprobe_trace(void) 2033 { 2034 int ret; 2035 2036 ret = tracing_init_dentry(); 2037 if (ret) 2038 return 0; 2039 2040 /* Event list interface */ 2041 trace_create_file("kprobe_events", TRACE_MODE_WRITE, 2042 NULL, NULL, &kprobe_events_ops); 2043 2044 /* Profile interface */ 2045 trace_create_file("kprobe_profile", TRACE_MODE_READ, 2046 NULL, NULL, &kprobe_profile_ops); 2047 2048 setup_boot_kprobe_events(); 2049 2050 return 0; 2051 } 2052 fs_initcall(init_kprobe_trace); 2053 2054 2055 #ifdef CONFIG_FTRACE_STARTUP_TEST 2056 static __init struct trace_event_file * 2057 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) 2058 { 2059 struct trace_event_file *file; 2060 2061 list_for_each_entry(file, &tr->events, list) 2062 if (file->event_call == trace_probe_event_call(&tk->tp)) 2063 return file; 2064 2065 return NULL; 2066 } 2067 2068 /* 2069 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this 2070 * stage, we can do this lockless. 2071 */ 2072 static __init int kprobe_trace_self_tests_init(void) 2073 { 2074 int ret, warn = 0; 2075 int (*target)(int, int, int, int, int, int); 2076 struct trace_kprobe *tk; 2077 struct trace_event_file *file; 2078 2079 if (tracing_is_disabled()) 2080 return -ENODEV; 2081 2082 if (tracing_selftest_disabled) 2083 return 0; 2084 2085 target = kprobe_trace_selftest_target; 2086 2087 pr_info("Testing kprobe tracing: "); 2088 2089 ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)"); 2090 if (WARN_ONCE(ret, "error on probing function entry.")) { 2091 warn++; 2092 } else { 2093 /* Enable trace point */ 2094 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2095 if (WARN_ONCE(tk == NULL, "error on probing function entry.")) { 2096 warn++; 2097 } else { 2098 file = find_trace_probe_file(tk, top_trace_array()); 2099 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2100 warn++; 2101 } else 2102 enable_trace_kprobe( 2103 trace_probe_event_call(&tk->tp), file); 2104 } 2105 } 2106 2107 ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval"); 2108 if (WARN_ONCE(ret, "error on probing function return.")) { 2109 warn++; 2110 } else { 2111 /* Enable trace point */ 2112 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2113 if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) { 2114 warn++; 2115 } else { 2116 file = find_trace_probe_file(tk, top_trace_array()); 2117 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2118 warn++; 2119 } else 2120 enable_trace_kprobe( 2121 trace_probe_event_call(&tk->tp), file); 2122 } 2123 } 2124 2125 if (warn) 2126 goto end; 2127 2128 ret = target(1, 2, 3, 4, 5, 6); 2129 2130 /* 2131 * Not expecting an error here, the check is only to prevent the 2132 * optimizer from removing the call to target() as otherwise there 2133 * are no side-effects and the call is never performed. 2134 */ 2135 if (ret != 21) 2136 warn++; 2137 2138 /* Disable trace points before removing it */ 2139 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2140 if (WARN_ONCE(tk == NULL, "error on getting test probe.")) { 2141 warn++; 2142 } else { 2143 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2144 "incorrect number of testprobe hits.")) 2145 warn++; 2146 2147 file = find_trace_probe_file(tk, top_trace_array()); 2148 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2149 warn++; 2150 } else 2151 disable_trace_kprobe( 2152 trace_probe_event_call(&tk->tp), file); 2153 } 2154 2155 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2156 if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) { 2157 warn++; 2158 } else { 2159 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2160 "incorrect number of testprobe2 hits.")) 2161 warn++; 2162 2163 file = find_trace_probe_file(tk, top_trace_array()); 2164 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2165 warn++; 2166 } else 2167 disable_trace_kprobe( 2168 trace_probe_event_call(&tk->tp), file); 2169 } 2170 2171 ret = create_or_delete_trace_kprobe("-:testprobe"); 2172 if (WARN_ONCE(ret, "error on deleting a probe.")) 2173 warn++; 2174 2175 ret = create_or_delete_trace_kprobe("-:testprobe2"); 2176 if (WARN_ONCE(ret, "error on deleting a probe.")) 2177 warn++; 2178 2179 2180 end: 2181 /* 2182 * Wait for the optimizer work to finish. Otherwise it might fiddle 2183 * with probes in already freed __init text. 2184 */ 2185 wait_for_kprobe_optimizer(); 2186 if (warn) 2187 pr_cont("NG: Some tests are failed. Please check them.\n"); 2188 else 2189 pr_cont("OK\n"); 2190 return 0; 2191 } 2192 2193 late_initcall(kprobe_trace_self_tests_init); 2194 2195 #endif 2196