1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Kprobes-based tracing events 4 * 5 * Created by Masami Hiramatsu <mhiramat@redhat.com> 6 * 7 */ 8 #define pr_fmt(fmt) "trace_kprobe: " fmt 9 10 #include <linux/bpf-cgroup.h> 11 #include <linux/cleanup.h> 12 #include <linux/error-injection.h> 13 #include <linux/module.h> 14 #include <linux/rculist.h> 15 #include <linux/security.h> 16 #include <linux/uaccess.h> 17 18 #include <asm/setup.h> /* for COMMAND_LINE_SIZE */ 19 20 #include "trace_dynevent.h" 21 #include "trace_kprobe_selftest.h" 22 #include "trace_probe.h" 23 #include "trace_probe_kernel.h" 24 #include "trace_probe_tmpl.h" 25 26 #define KPROBE_EVENT_SYSTEM "kprobes" 27 #define KRETPROBE_MAXACTIVE_MAX 4096 28 29 /* Kprobe early definition from command line */ 30 static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata; 31 32 static int __init set_kprobe_boot_events(char *str) 33 { 34 trace_append_boot_param(kprobe_boot_events_buf, str, ';', 35 COMMAND_LINE_SIZE); 36 disable_tracing_selftest("running kprobe events"); 37 38 return 1; 39 } 40 __setup("kprobe_event=", set_kprobe_boot_events); 41 42 static int trace_kprobe_create(const char *raw_command); 43 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); 44 static int trace_kprobe_release(struct dyn_event *ev); 45 static bool trace_kprobe_is_busy(struct dyn_event *ev); 46 static bool trace_kprobe_match(const char *system, const char *event, 47 int argc, const char **argv, struct dyn_event *ev); 48 49 static struct dyn_event_operations trace_kprobe_ops = { 50 .create = trace_kprobe_create, 51 .show = trace_kprobe_show, 52 .is_busy = trace_kprobe_is_busy, 53 .free = trace_kprobe_release, 54 .match = trace_kprobe_match, 55 }; 56 57 /* 58 * Kprobe event core functions 59 */ 60 struct trace_kprobe { 61 struct dyn_event devent; 62 struct kretprobe rp; /* Use rp.kp for kprobe use */ 63 unsigned long __percpu *nhit; 64 const char *symbol; /* symbol name */ 65 struct trace_probe tp; 66 }; 67 68 static bool is_trace_kprobe(struct dyn_event *ev) 69 { 70 return ev->ops == &trace_kprobe_ops; 71 } 72 73 static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev) 74 { 75 return container_of(ev, struct trace_kprobe, devent); 76 } 77 78 /** 79 * for_each_trace_kprobe - iterate over the trace_kprobe list 80 * @pos: the struct trace_kprobe * for each entry 81 * @dpos: the struct dyn_event * to use as a loop cursor 82 */ 83 #define for_each_trace_kprobe(pos, dpos) \ 84 for_each_dyn_event(dpos) \ 85 if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) 86 #define trace_kprobe_list_empty() list_empty(&dyn_event_list) 87 88 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) 89 { 90 return tk->rp.handler != NULL; 91 } 92 93 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) 94 { 95 return tk->symbol ? tk->symbol : "unknown"; 96 } 97 98 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) 99 { 100 return tk->rp.kp.offset; 101 } 102 103 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) 104 { 105 return kprobe_gone(&tk->rp.kp); 106 } 107 108 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, 109 struct module *mod) 110 { 111 int len = strlen(module_name(mod)); 112 const char *name = trace_kprobe_symbol(tk); 113 114 return strncmp(module_name(mod), name, len) == 0 && name[len] == ':'; 115 } 116 117 #ifdef CONFIG_MODULES 118 static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 119 { 120 char *p; 121 bool ret; 122 123 if (!tk->symbol) 124 return false; 125 p = strchr(tk->symbol, ':'); 126 if (!p) 127 return true; 128 *p = '\0'; 129 scoped_guard(rcu) 130 ret = !!find_module(tk->symbol); 131 *p = ':'; 132 133 return ret; 134 } 135 #else 136 static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) 137 { 138 return false; 139 } 140 #endif 141 142 static bool trace_kprobe_is_busy(struct dyn_event *ev) 143 { 144 struct trace_kprobe *tk = to_trace_kprobe(ev); 145 146 return trace_probe_is_enabled(&tk->tp); 147 } 148 149 static bool trace_kprobe_match_command_head(struct trace_kprobe *tk, 150 int argc, const char **argv) 151 { 152 char buf[32]; 153 int len; 154 155 if (!argc) 156 return true; 157 158 if (!tk->symbol) { 159 snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr); 160 if (strcmp(buf, argv[0])) 161 return false; 162 } else if (tk->rp.kp.offset) { 163 len = strlen(trace_kprobe_symbol(tk)); 164 if (strncmp(trace_kprobe_symbol(tk), argv[0], len) || 165 argv[0][len] != '+') 166 return false; 167 168 snprintf(buf, sizeof(buf), "%u", tk->rp.kp.offset); 169 if (strcmp(buf, &argv[0][len + 1])) 170 return false; 171 } else if (strcmp(trace_kprobe_symbol(tk), argv[0])) 172 return false; 173 174 argc--; argv++; 175 176 return trace_probe_match_command_args(&tk->tp, argc, argv); 177 } 178 179 static bool trace_kprobe_match(const char *system, const char *event, 180 int argc, const char **argv, struct dyn_event *ev) 181 { 182 struct trace_kprobe *tk = to_trace_kprobe(ev); 183 184 return (event[0] == '\0' || 185 strcmp(trace_probe_name(&tk->tp), event) == 0) && 186 (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) && 187 trace_kprobe_match_command_head(tk, argc, argv); 188 } 189 190 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 191 { 192 unsigned long nhit = 0; 193 int cpu; 194 195 for_each_possible_cpu(cpu) 196 nhit += *per_cpu_ptr(tk->nhit, cpu); 197 198 return nhit; 199 } 200 201 static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk) 202 { 203 return !(list_empty(&tk->rp.kp.list) && 204 hlist_unhashed(&tk->rp.kp.hlist)); 205 } 206 207 /* Return 0 if it fails to find the symbol address */ 208 static nokprobe_inline 209 unsigned long trace_kprobe_address(struct trace_kprobe *tk) 210 { 211 unsigned long addr; 212 213 if (tk->symbol) { 214 addr = (unsigned long) 215 kallsyms_lookup_name(trace_kprobe_symbol(tk)); 216 if (addr) 217 addr += tk->rp.kp.offset; 218 } else { 219 addr = (unsigned long)tk->rp.kp.addr; 220 } 221 return addr; 222 } 223 224 static nokprobe_inline struct trace_kprobe * 225 trace_kprobe_primary_from_call(struct trace_event_call *call) 226 { 227 struct trace_probe *tp; 228 229 tp = trace_probe_primary_from_call(call); 230 if (WARN_ON_ONCE(!tp)) 231 return NULL; 232 233 return container_of(tp, struct trace_kprobe, tp); 234 } 235 236 bool trace_kprobe_on_func_entry(struct trace_event_call *call) 237 { 238 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 239 240 return tk ? (kprobe_on_func_entry(tk->rp.kp.addr, 241 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, 242 tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false; 243 } 244 245 bool trace_kprobe_error_injectable(struct trace_event_call *call) 246 { 247 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); 248 249 return tk ? within_error_injection_list(trace_kprobe_address(tk)) : 250 false; 251 } 252 253 static int register_kprobe_event(struct trace_kprobe *tk); 254 static int unregister_kprobe_event(struct trace_kprobe *tk); 255 256 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 257 static int kretprobe_dispatcher(struct kretprobe_instance *ri, 258 struct pt_regs *regs); 259 260 static void free_trace_kprobe(struct trace_kprobe *tk) 261 { 262 if (tk) { 263 trace_probe_cleanup(&tk->tp); 264 kfree(tk->symbol); 265 free_percpu(tk->nhit); 266 kfree(tk); 267 } 268 } 269 270 DEFINE_FREE(free_trace_kprobe, struct trace_kprobe *, 271 if (!IS_ERR_OR_NULL(_T)) free_trace_kprobe(_T)) 272 273 /* 274 * Allocate new trace_probe and initialize it (including kprobes). 275 */ 276 static struct trace_kprobe *alloc_trace_kprobe(const char *group, 277 const char *event, 278 void *addr, 279 const char *symbol, 280 unsigned long offs, 281 int maxactive, 282 int nargs, bool is_return) 283 { 284 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 285 int ret = -ENOMEM; 286 287 tk = kzalloc_flex(*tk, tp.args, nargs); 288 if (!tk) 289 return ERR_PTR(ret); 290 291 tk->nhit = alloc_percpu(unsigned long); 292 if (!tk->nhit) 293 return ERR_PTR(ret); 294 295 if (symbol) { 296 tk->symbol = kstrdup(symbol, GFP_KERNEL); 297 if (!tk->symbol) 298 return ERR_PTR(ret); 299 tk->rp.kp.symbol_name = tk->symbol; 300 tk->rp.kp.offset = offs; 301 } else 302 tk->rp.kp.addr = addr; 303 304 if (is_return) 305 tk->rp.handler = kretprobe_dispatcher; 306 else 307 tk->rp.kp.pre_handler = kprobe_dispatcher; 308 309 tk->rp.maxactive = maxactive; 310 INIT_HLIST_NODE(&tk->rp.kp.hlist); 311 INIT_LIST_HEAD(&tk->rp.kp.list); 312 313 ret = trace_probe_init(&tk->tp, event, group, false, nargs); 314 if (ret < 0) 315 return ERR_PTR(ret); 316 317 dyn_event_init(&tk->devent, &trace_kprobe_ops); 318 return_ptr(tk); 319 } 320 321 static struct trace_kprobe *find_trace_kprobe(const char *event, 322 const char *group) 323 { 324 struct dyn_event *pos; 325 struct trace_kprobe *tk; 326 327 for_each_trace_kprobe(tk, pos) 328 if (strcmp(trace_probe_name(&tk->tp), event) == 0 && 329 strcmp(trace_probe_group_name(&tk->tp), group) == 0) 330 return tk; 331 return NULL; 332 } 333 334 static inline int __enable_trace_kprobe(struct trace_kprobe *tk) 335 { 336 int ret = 0; 337 338 if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) { 339 if (trace_kprobe_is_return(tk)) 340 ret = enable_kretprobe(&tk->rp); 341 else 342 ret = enable_kprobe(&tk->rp.kp); 343 } 344 345 return ret; 346 } 347 348 static void __disable_trace_kprobe(struct trace_probe *tp) 349 { 350 struct trace_kprobe *tk; 351 352 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 353 if (!trace_kprobe_is_registered(tk)) 354 continue; 355 if (trace_kprobe_is_return(tk)) 356 disable_kretprobe(&tk->rp); 357 else 358 disable_kprobe(&tk->rp.kp); 359 } 360 } 361 362 /* 363 * Enable trace_probe 364 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 365 */ 366 static int enable_trace_kprobe(struct trace_event_call *call, 367 struct trace_event_file *file) 368 { 369 struct trace_probe *tp; 370 struct trace_kprobe *tk; 371 bool enabled; 372 int ret = 0; 373 374 tp = trace_probe_primary_from_call(call); 375 if (WARN_ON_ONCE(!tp)) 376 return -ENODEV; 377 enabled = trace_probe_is_enabled(tp); 378 379 /* This also changes "enabled" state */ 380 if (file) { 381 ret = trace_probe_add_file(tp, file); 382 if (ret) 383 return ret; 384 } else 385 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 386 387 if (enabled) 388 return 0; 389 390 list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) { 391 if (trace_kprobe_has_gone(tk)) 392 continue; 393 ret = __enable_trace_kprobe(tk); 394 if (ret) 395 break; 396 enabled = true; 397 } 398 399 if (ret) { 400 /* Failed to enable one of them. Roll back all */ 401 if (enabled) 402 __disable_trace_kprobe(tp); 403 if (file) 404 trace_probe_remove_file(tp, file); 405 else 406 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 407 } 408 409 return ret; 410 } 411 412 /* 413 * Disable trace_probe 414 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 415 */ 416 static int disable_trace_kprobe(struct trace_event_call *call, 417 struct trace_event_file *file) 418 { 419 struct trace_probe *tp; 420 421 tp = trace_probe_primary_from_call(call); 422 if (WARN_ON_ONCE(!tp)) 423 return -ENODEV; 424 425 if (file) { 426 if (!trace_probe_get_file_link(tp, file)) 427 return -ENOENT; 428 if (!trace_probe_has_single_file(tp)) 429 goto out; 430 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 431 } else 432 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 433 434 if (!trace_probe_is_enabled(tp)) 435 __disable_trace_kprobe(tp); 436 437 out: 438 if (file) 439 /* 440 * Synchronization is done in below function. For perf event, 441 * file == NULL and perf_trace_event_unreg() calls 442 * tracepoint_synchronize_unregister() to ensure synchronize 443 * event. We don't need to care about it. 444 */ 445 trace_probe_remove_file(tp, file); 446 447 return 0; 448 } 449 450 #if defined(CONFIG_DYNAMIC_FTRACE) && \ 451 !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE) 452 static bool __within_notrace_func(unsigned long addr) 453 { 454 unsigned long offset, size; 455 456 if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset)) 457 return false; 458 459 /* Get the entry address of the target function */ 460 addr -= offset; 461 462 /* 463 * Since ftrace_location_range() does inclusive range check, we need 464 * to subtract 1 byte from the end address. 465 */ 466 return !ftrace_location_range(addr, addr + size - 1); 467 } 468 469 static bool within_notrace_func(struct trace_kprobe *tk) 470 { 471 unsigned long addr = trace_kprobe_address(tk); 472 char symname[KSYM_NAME_LEN], *p; 473 474 if (!__within_notrace_func(addr)) 475 return false; 476 477 /* Check if the address is on a suffixed-symbol */ 478 if (!lookup_symbol_name(addr, symname)) { 479 p = strchr(symname, '.'); 480 if (!p) 481 return true; 482 *p = '\0'; 483 addr = (unsigned long)kprobe_lookup_name(symname, 0); 484 if (addr) 485 return __within_notrace_func(addr); 486 } 487 488 return true; 489 } 490 #else 491 #define within_notrace_func(tk) (false) 492 #endif 493 494 /* Internal register function - just handle k*probes and flags */ 495 static int __register_trace_kprobe(struct trace_kprobe *tk) 496 { 497 int i, ret; 498 499 ret = security_locked_down(LOCKDOWN_KPROBES); 500 if (ret) 501 return ret; 502 503 if (trace_kprobe_is_registered(tk)) 504 return -EINVAL; 505 506 if (within_notrace_func(tk)) { 507 pr_warn("Could not probe notrace function %ps\n", 508 (void *)trace_kprobe_address(tk)); 509 return -EINVAL; 510 } 511 512 for (i = 0; i < tk->tp.nr_args; i++) { 513 ret = traceprobe_update_arg(&tk->tp.args[i]); 514 if (ret) 515 return ret; 516 } 517 518 /* Set/clear disabled flag according to tp->flag */ 519 if (trace_probe_is_enabled(&tk->tp)) 520 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; 521 else 522 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; 523 524 if (trace_kprobe_is_return(tk)) 525 ret = register_kretprobe(&tk->rp); 526 else 527 ret = register_kprobe(&tk->rp.kp); 528 529 return ret; 530 } 531 532 /* Internal unregister function - just handle k*probes and flags */ 533 static void __unregister_trace_kprobe(struct trace_kprobe *tk) 534 { 535 if (trace_kprobe_is_registered(tk)) { 536 if (trace_kprobe_is_return(tk)) 537 unregister_kretprobe(&tk->rp); 538 else 539 unregister_kprobe(&tk->rp.kp); 540 /* Cleanup kprobe for reuse and mark it unregistered */ 541 INIT_HLIST_NODE(&tk->rp.kp.hlist); 542 INIT_LIST_HEAD(&tk->rp.kp.list); 543 if (tk->rp.kp.symbol_name) 544 tk->rp.kp.addr = NULL; 545 } 546 } 547 548 /* Unregister a trace_probe and probe_event */ 549 static int unregister_trace_kprobe(struct trace_kprobe *tk) 550 { 551 /* If other probes are on the event, just unregister kprobe */ 552 if (trace_probe_has_sibling(&tk->tp)) 553 goto unreg; 554 555 /* Enabled event can not be unregistered */ 556 if (trace_probe_is_enabled(&tk->tp)) 557 return -EBUSY; 558 559 /* If there's a reference to the dynamic event */ 560 if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp))) 561 return -EBUSY; 562 563 /* Will fail if probe is being used by ftrace or perf */ 564 if (unregister_kprobe_event(tk)) 565 return -EBUSY; 566 567 unreg: 568 __unregister_trace_kprobe(tk); 569 dyn_event_remove(&tk->devent); 570 trace_probe_unlink(&tk->tp); 571 572 return 0; 573 } 574 575 static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig, 576 struct trace_kprobe *comp) 577 { 578 struct trace_probe_event *tpe = orig->tp.event; 579 int i; 580 581 list_for_each_entry(orig, &tpe->probes, tp.list) { 582 if (strcmp(trace_kprobe_symbol(orig), 583 trace_kprobe_symbol(comp)) || 584 trace_kprobe_offset(orig) != trace_kprobe_offset(comp)) 585 continue; 586 587 /* 588 * trace_probe_compare_arg_type() ensured that nr_args and 589 * each argument name and type are same. Let's compare comm. 590 */ 591 for (i = 0; i < orig->tp.nr_args; i++) { 592 if (strcmp(orig->tp.args[i].comm, 593 comp->tp.args[i].comm)) 594 break; 595 } 596 597 if (i == orig->tp.nr_args) 598 return true; 599 } 600 601 return false; 602 } 603 604 static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to) 605 { 606 int ret; 607 608 ret = trace_probe_compare_arg_type(&tk->tp, &to->tp); 609 if (ret) { 610 /* Note that argument starts index = 2 */ 611 trace_probe_log_set_index(ret + 1); 612 trace_probe_log_err(0, DIFF_ARG_TYPE); 613 return -EEXIST; 614 } 615 if (trace_kprobe_has_same_kprobe(to, tk)) { 616 trace_probe_log_set_index(0); 617 trace_probe_log_err(0, SAME_PROBE); 618 return -EEXIST; 619 } 620 621 /* Append to existing event */ 622 ret = trace_probe_append(&tk->tp, &to->tp); 623 if (ret) 624 return ret; 625 626 /* Register k*probe */ 627 ret = __register_trace_kprobe(tk); 628 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 629 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 630 ret = 0; 631 } 632 633 if (ret) 634 trace_probe_unlink(&tk->tp); 635 else 636 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 637 638 return ret; 639 } 640 641 /* Register a trace_probe and probe_event */ 642 static int register_trace_kprobe(struct trace_kprobe *tk) 643 { 644 struct trace_kprobe *old_tk; 645 int ret; 646 647 guard(mutex)(&event_mutex); 648 649 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), 650 trace_probe_group_name(&tk->tp)); 651 if (old_tk) { 652 if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) { 653 trace_probe_log_set_index(0); 654 trace_probe_log_err(0, DIFF_PROBE_TYPE); 655 return -EEXIST; 656 } 657 return append_trace_kprobe(tk, old_tk); 658 } 659 660 /* Register new event */ 661 ret = register_kprobe_event(tk); 662 if (ret) { 663 if (ret == -EEXIST) { 664 trace_probe_log_set_index(0); 665 trace_probe_log_err(0, EVENT_EXIST); 666 } else 667 pr_warn("Failed to register probe event(%d)\n", ret); 668 return ret; 669 } 670 671 /* Register k*probe */ 672 ret = __register_trace_kprobe(tk); 673 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { 674 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 675 ret = 0; 676 } 677 678 if (ret < 0) 679 unregister_kprobe_event(tk); 680 else 681 dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp)); 682 683 return ret; 684 } 685 686 #ifdef CONFIG_MODULES 687 static int validate_module_probe_symbol(const char *modname, const char *symbol); 688 689 static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk) 690 { 691 const char *p; 692 int ret = 0; 693 694 p = strchr(trace_kprobe_symbol(tk), ':'); 695 if (p) 696 ret = validate_module_probe_symbol(module_name(mod), p + 1); 697 if (!ret) 698 ret = __register_trace_kprobe(tk); 699 return ret; 700 } 701 702 /* Module notifier call back, checking event on the module */ 703 static int trace_kprobe_module_callback(struct notifier_block *nb, 704 unsigned long val, void *data) 705 { 706 struct module *mod = data; 707 struct dyn_event *pos; 708 struct trace_kprobe *tk; 709 int ret; 710 711 if (val != MODULE_STATE_COMING) 712 return NOTIFY_DONE; 713 714 /* Update probes on coming module */ 715 guard(mutex)(&event_mutex); 716 for_each_trace_kprobe(tk, pos) { 717 if (trace_kprobe_within_module(tk, mod)) { 718 /* Don't need to check busy - this should have gone. */ 719 __unregister_trace_kprobe(tk); 720 ret = register_module_trace_kprobe(mod, tk); 721 if (ret) 722 pr_warn("Failed to re-register probe %s on %s: %d\n", 723 trace_probe_name(&tk->tp), 724 module_name(mod), ret); 725 } 726 } 727 728 return NOTIFY_DONE; 729 } 730 731 static struct notifier_block trace_kprobe_module_nb = { 732 .notifier_call = trace_kprobe_module_callback, 733 .priority = 2 /* Invoked after kprobe and jump_label module callback */ 734 }; 735 static int trace_kprobe_register_module_notifier(void) 736 { 737 return register_module_notifier(&trace_kprobe_module_nb); 738 } 739 #else 740 static int trace_kprobe_register_module_notifier(void) 741 { 742 return 0; 743 } 744 #endif /* CONFIG_MODULES */ 745 746 static int count_symbols(void *data, unsigned long unused) 747 { 748 unsigned int *count = data; 749 750 (*count)++; 751 752 return 0; 753 } 754 755 struct sym_count_ctx { 756 unsigned int count; 757 const char *name; 758 }; 759 760 static int count_mod_symbols(void *data, const char *name, unsigned long unused) 761 { 762 struct sym_count_ctx *ctx = data; 763 764 if (strcmp(name, ctx->name) == 0) 765 ctx->count++; 766 767 return 0; 768 } 769 770 static unsigned int number_of_same_symbols(const char *mod, const char *func_name) 771 { 772 struct sym_count_ctx ctx = { .count = 0, .name = func_name }; 773 774 if (!mod) 775 kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count); 776 777 /* 778 * If the symbol is found in vmlinux, use vmlinux resolution only. 779 * This prevents module symbols from shadowing vmlinux symbols 780 * and causing -EADDRNOTAVAIL for unqualified kprobe targets. 781 */ 782 if (!mod && ctx.count > 0) 783 return ctx.count; 784 785 module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx); 786 787 return ctx.count; 788 } 789 790 static int validate_module_probe_symbol(const char *modname, const char *symbol) 791 { 792 unsigned int count = number_of_same_symbols(modname, symbol); 793 794 if (count > 1) { 795 /* 796 * Users should use ADDR to remove the ambiguity of 797 * using KSYM only. 798 */ 799 return -EADDRNOTAVAIL; 800 } else if (count == 0) { 801 /* 802 * We can return ENOENT earlier than when register the 803 * kprobe. 804 */ 805 return -ENOENT; 806 } 807 return 0; 808 } 809 810 #ifdef CONFIG_MODULES 811 /* Return NULL if the module is not loaded or under unloading. */ 812 static struct module *try_module_get_by_name(const char *name) 813 { 814 struct module *mod; 815 816 guard(rcu)(); 817 mod = find_module(name); 818 if (mod && !try_module_get(mod)) 819 mod = NULL; 820 return mod; 821 } 822 #else 823 #define try_module_get_by_name(name) (NULL) 824 #endif 825 826 static int validate_probe_symbol(char *symbol) 827 { 828 struct module *mod = NULL; 829 char *modname = NULL, *p; 830 int ret = 0; 831 832 p = strchr(symbol, ':'); 833 if (p) { 834 modname = symbol; 835 symbol = p + 1; 836 *p = '\0'; 837 mod = try_module_get_by_name(modname); 838 if (!mod) 839 goto out; 840 } 841 842 ret = validate_module_probe_symbol(modname, symbol); 843 out: 844 if (p) 845 *p = ':'; 846 if (mod) 847 module_put(mod); 848 return ret; 849 } 850 851 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 852 struct pt_regs *regs); 853 854 static int trace_kprobe_create_internal(int argc, const char *argv[], 855 struct traceprobe_parse_context *ctx) 856 { 857 /* 858 * Argument syntax: 859 * - Add kprobe: 860 * p[:[GRP/][EVENT]] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS] 861 * - Add kretprobe: 862 * r[MAXACTIVE][:[GRP/][EVENT]] [MOD:]KSYM[+0] [FETCHARGS] 863 * Or 864 * p[:[GRP/][EVENT]] [MOD:]KSYM[+0]%return [FETCHARGS] 865 * 866 * Fetch args: 867 * $retval : fetch return value 868 * $stack : fetch stack address 869 * $stackN : fetch Nth of stack (N:0-) 870 * $comm : fetch current task comm 871 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 872 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 873 * %REG : fetch register REG 874 * Dereferencing memory fetch: 875 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 876 * Alias name of args: 877 * NAME=FETCHARG : set NAME as alias of FETCHARG. 878 * Type of args: 879 * FETCHARG:TYPE : use TYPE instead of unsigned long. 880 */ 881 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 882 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM; 883 const char **new_argv __free(kfree) = NULL; 884 int i, len, new_argc = 0, ret = 0; 885 char *symbol __free(kfree) = NULL; 886 char *ebuf __free(kfree) = NULL; 887 char *gbuf __free(kfree) = NULL; 888 char *abuf __free(kfree) = NULL; 889 char *dbuf __free(kfree) = NULL; 890 enum probe_print_type ptype; 891 bool is_return = false; 892 int maxactive = 0; 893 void *addr = NULL; 894 char *tmp = NULL; 895 long offset = 0; 896 897 switch (argv[0][0]) { 898 case 'r': 899 is_return = true; 900 break; 901 case 'p': 902 break; 903 default: 904 return -ECANCELED; 905 } 906 if (argc < 2) 907 return -ECANCELED; 908 909 event = strchr(&argv[0][1], ':'); 910 if (event) 911 event++; 912 913 if (isdigit(argv[0][1])) { 914 char *buf __free(kfree) = NULL; 915 916 if (!is_return) { 917 trace_probe_log_err(1, BAD_MAXACT_TYPE); 918 return -EINVAL; 919 } 920 if (event) 921 len = event - &argv[0][1] - 1; 922 else 923 len = strlen(&argv[0][1]); 924 if (len > MAX_EVENT_NAME_LEN - 1) { 925 trace_probe_log_err(1, BAD_MAXACT); 926 return -EINVAL; 927 } 928 buf = kmemdup(&argv[0][1], len + 1, GFP_KERNEL); 929 if (!buf) 930 return -ENOMEM; 931 buf[len] = '\0'; 932 ret = kstrtouint(buf, 0, &maxactive); 933 if (ret || !maxactive) { 934 trace_probe_log_err(1, BAD_MAXACT); 935 return -EINVAL; 936 } 937 /* kretprobes instances are iterated over via a list. The 938 * maximum should stay reasonable. 939 */ 940 if (maxactive > KRETPROBE_MAXACTIVE_MAX) { 941 trace_probe_log_err(1, MAXACT_TOO_BIG); 942 return -EINVAL; 943 } 944 } 945 946 /* try to parse an address. if that fails, try to read the 947 * input as a symbol. */ 948 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { 949 trace_probe_log_set_index(1); 950 /* Check whether uprobe event specified */ 951 if (strchr(argv[1], '/') && strchr(argv[1], ':')) 952 return -ECANCELED; 953 954 /* a symbol specified */ 955 symbol = kstrdup(argv[1], GFP_KERNEL); 956 if (!symbol) 957 return -ENOMEM; 958 959 tmp = strchr(symbol, '%'); 960 if (tmp) { 961 if (!strcmp(tmp, "%return")) { 962 *tmp = '\0'; 963 is_return = true; 964 } else { 965 trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX); 966 return -EINVAL; 967 } 968 } 969 970 /* TODO: support .init module functions */ 971 ret = traceprobe_split_symbol_offset(symbol, &offset); 972 if (ret || offset < 0 || offset > UINT_MAX) { 973 trace_probe_log_err(0, BAD_PROBE_ADDR); 974 return -EINVAL; 975 } 976 ret = validate_probe_symbol(symbol); 977 if (ret) { 978 if (ret == -EADDRNOTAVAIL) 979 trace_probe_log_err(0, NON_UNIQ_SYMBOL); 980 else 981 trace_probe_log_err(0, BAD_PROBE_ADDR); 982 return -EINVAL; 983 } 984 if (is_return) 985 ctx->flags |= TPARG_FL_RETURN; 986 ret = kprobe_on_func_entry(NULL, symbol, offset); 987 if (ret == 0 && !is_return) 988 ctx->flags |= TPARG_FL_FENTRY; 989 /* Defer the ENOENT case until register kprobe */ 990 if (ret == -EINVAL && is_return) { 991 trace_probe_log_err(0, BAD_RETPROBE); 992 return -EINVAL; 993 } 994 } 995 996 trace_probe_log_set_index(0); 997 if (event) { 998 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 999 if (!gbuf) 1000 return -ENOMEM; 1001 ret = traceprobe_parse_event_name(&event, &group, gbuf, 1002 event - argv[0]); 1003 if (ret) 1004 return ret; 1005 } 1006 1007 if (!event) { 1008 /* Make a new event name */ 1009 ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 1010 if (!ebuf) 1011 return -ENOMEM; 1012 if (symbol) 1013 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 1014 is_return ? 'r' : 'p', symbol, offset); 1015 else 1016 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_0x%p", 1017 is_return ? 'r' : 'p', addr); 1018 sanitize_event_name(ebuf); 1019 event = ebuf; 1020 } 1021 1022 abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL); 1023 if (!abuf) 1024 return -ENOMEM; 1025 argc -= 2; argv += 2; 1026 ctx->funcname = symbol; 1027 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, 1028 abuf, MAX_BTF_ARGS_LEN, ctx); 1029 if (IS_ERR(new_argv)) { 1030 ret = PTR_ERR(new_argv); 1031 new_argv = NULL; 1032 return ret; 1033 } 1034 if (new_argv) { 1035 argc = new_argc; 1036 argv = new_argv; 1037 } 1038 if (argc > MAX_TRACE_ARGS) { 1039 trace_probe_log_set_index(2); 1040 trace_probe_log_err(0, TOO_MANY_ARGS); 1041 return -E2BIG; 1042 } 1043 1044 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); 1045 if (ret) 1046 return ret; 1047 1048 /* setup a probe */ 1049 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, 1050 argc, is_return); 1051 if (IS_ERR(tk)) { 1052 ret = PTR_ERR(tk); 1053 /* This must return -ENOMEM, else there is a bug */ 1054 WARN_ON_ONCE(ret != -ENOMEM); 1055 return ret; /* We know tk is not allocated */ 1056 } 1057 1058 /* parse arguments */ 1059 for (i = 0; i < argc; i++) { 1060 trace_probe_log_set_index(i + 2); 1061 ctx->offset = 0; 1062 ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], ctx); 1063 if (ret) 1064 return ret; /* This can be -ENOMEM */ 1065 } 1066 /* entry handler for kretprobe */ 1067 if (is_return && tk->tp.entry_arg) { 1068 tk->rp.entry_handler = trace_kprobe_entry_handler; 1069 tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp); 1070 } 1071 1072 ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1073 ret = traceprobe_set_print_fmt(&tk->tp, ptype); 1074 if (ret < 0) 1075 return ret; 1076 1077 ret = register_trace_kprobe(tk); 1078 if (ret) { 1079 trace_probe_log_set_index(1); 1080 if (ret == -EILSEQ) 1081 trace_probe_log_err(0, BAD_INSN_BNDRY); 1082 else if (ret == -ENOENT) 1083 trace_probe_log_err(0, BAD_PROBE_ADDR); 1084 else if (ret != -ENOMEM && ret != -EEXIST) 1085 trace_probe_log_err(0, FAIL_REG_PROBE); 1086 return ret; 1087 } 1088 /* 1089 * Here, 'tk' has been registered to the list successfully, 1090 * so we don't need to free it. 1091 */ 1092 tk = NULL; 1093 1094 return 0; 1095 } 1096 1097 static int trace_kprobe_create_cb(int argc, const char *argv[]) 1098 { 1099 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 1100 int ret; 1101 1102 ctx = kzalloc_obj(*ctx); 1103 if (!ctx) 1104 return -ENOMEM; 1105 ctx->flags = TPARG_FL_KERNEL; 1106 1107 trace_probe_log_init("trace_kprobe", argc, argv); 1108 1109 ret = trace_kprobe_create_internal(argc, argv, ctx); 1110 1111 trace_probe_log_clear(); 1112 return ret; 1113 } 1114 1115 static int trace_kprobe_create(const char *raw_command) 1116 { 1117 return trace_probe_create(raw_command, trace_kprobe_create_cb); 1118 } 1119 1120 static int create_or_delete_trace_kprobe(const char *raw_command) 1121 { 1122 int ret; 1123 1124 if (raw_command[0] == '-') 1125 return dyn_event_release(raw_command, &trace_kprobe_ops); 1126 1127 ret = dyn_event_create(raw_command, &trace_kprobe_ops); 1128 return ret == -ECANCELED ? -EINVAL : ret; 1129 } 1130 1131 static int trace_kprobe_run_command(struct dynevent_cmd *cmd) 1132 { 1133 return create_or_delete_trace_kprobe(cmd->seq.buffer); 1134 } 1135 1136 /** 1137 * kprobe_event_cmd_init - Initialize a kprobe event command object 1138 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1139 * @buf: A pointer to the buffer used to build the command 1140 * @maxlen: The length of the buffer passed in @buf 1141 * 1142 * Initialize a synthetic event command object. Use this before 1143 * calling any of the other kprobe_event functions. 1144 */ 1145 void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen) 1146 { 1147 dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE, 1148 trace_kprobe_run_command); 1149 } 1150 EXPORT_SYMBOL_GPL(kprobe_event_cmd_init); 1151 1152 /** 1153 * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list 1154 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1155 * @kretprobe: Is this a return probe? 1156 * @name: The name of the kprobe event 1157 * @loc: The location of the kprobe event 1158 * @...: Variable number of arg (pairs), one pair for each field 1159 * 1160 * NOTE: Users normally won't want to call this function directly, but 1161 * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically 1162 * adds a NULL to the end of the arg list. If this function is used 1163 * directly, make sure the last arg in the variable arg list is NULL. 1164 * 1165 * Generate a kprobe event command to be executed by 1166 * kprobe_event_gen_cmd_end(). This function can be used to generate the 1167 * complete command or only the first part of it; in the latter case, 1168 * kprobe_event_add_fields() can be used to add more fields following this. 1169 * 1170 * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This 1171 * returns -EINVAL if @loc == NULL. 1172 * 1173 * Return: 0 if successful, error otherwise. 1174 */ 1175 int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe, 1176 const char *name, const char *loc, ...) 1177 { 1178 char buf[MAX_EVENT_NAME_LEN]; 1179 struct dynevent_arg arg; 1180 va_list args; 1181 int ret; 1182 1183 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1184 return -EINVAL; 1185 1186 if (!loc) 1187 return -EINVAL; 1188 1189 if (kretprobe) 1190 snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name); 1191 else 1192 snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name); 1193 1194 ret = dynevent_str_add(cmd, buf); 1195 if (ret) 1196 return ret; 1197 1198 dynevent_arg_init(&arg, 0); 1199 arg.str = loc; 1200 ret = dynevent_arg_add(cmd, &arg, NULL); 1201 if (ret) 1202 return ret; 1203 1204 va_start(args, loc); 1205 for (;;) { 1206 const char *field; 1207 1208 field = va_arg(args, const char *); 1209 if (!field) 1210 break; 1211 1212 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1213 ret = -EINVAL; 1214 break; 1215 } 1216 1217 arg.str = field; 1218 ret = dynevent_arg_add(cmd, &arg, NULL); 1219 if (ret) 1220 break; 1221 } 1222 va_end(args); 1223 1224 return ret; 1225 } 1226 EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start); 1227 1228 /** 1229 * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list 1230 * @cmd: A pointer to the dynevent_cmd struct representing the new event 1231 * @...: Variable number of arg (pairs), one pair for each field 1232 * 1233 * NOTE: Users normally won't want to call this function directly, but 1234 * rather use the kprobe_event_add_fields() wrapper, which 1235 * automatically adds a NULL to the end of the arg list. If this 1236 * function is used directly, make sure the last arg in the variable 1237 * arg list is NULL. 1238 * 1239 * Add probe fields to an existing kprobe command using a variable 1240 * list of args. Fields are added in the same order they're listed. 1241 * 1242 * Return: 0 if successful, error otherwise. 1243 */ 1244 int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...) 1245 { 1246 struct dynevent_arg arg; 1247 va_list args; 1248 int ret = 0; 1249 1250 if (cmd->type != DYNEVENT_TYPE_KPROBE) 1251 return -EINVAL; 1252 1253 dynevent_arg_init(&arg, 0); 1254 1255 va_start(args, cmd); 1256 for (;;) { 1257 const char *field; 1258 1259 field = va_arg(args, const char *); 1260 if (!field) 1261 break; 1262 1263 if (++cmd->n_fields > MAX_TRACE_ARGS) { 1264 ret = -EINVAL; 1265 break; 1266 } 1267 1268 arg.str = field; 1269 ret = dynevent_arg_add(cmd, &arg, NULL); 1270 if (ret) 1271 break; 1272 } 1273 va_end(args); 1274 1275 return ret; 1276 } 1277 EXPORT_SYMBOL_GPL(__kprobe_event_add_fields); 1278 1279 /** 1280 * kprobe_event_delete - Delete a kprobe event 1281 * @name: The name of the kprobe event to delete 1282 * 1283 * Delete a kprobe event with the give @name from kernel code rather 1284 * than directly from the command line. 1285 * 1286 * Return: 0 if successful, error otherwise. 1287 */ 1288 int kprobe_event_delete(const char *name) 1289 { 1290 char buf[MAX_EVENT_NAME_LEN]; 1291 1292 snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name); 1293 1294 return create_or_delete_trace_kprobe(buf); 1295 } 1296 EXPORT_SYMBOL_GPL(kprobe_event_delete); 1297 1298 static int trace_kprobe_release(struct dyn_event *ev) 1299 { 1300 struct trace_kprobe *tk = to_trace_kprobe(ev); 1301 int ret = unregister_trace_kprobe(tk); 1302 1303 if (!ret) 1304 free_trace_kprobe(tk); 1305 return ret; 1306 } 1307 1308 static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) 1309 { 1310 struct trace_kprobe *tk = to_trace_kprobe(ev); 1311 int i; 1312 1313 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); 1314 if (trace_kprobe_is_return(tk) && tk->rp.maxactive) 1315 seq_printf(m, "%d", tk->rp.maxactive); 1316 seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp), 1317 trace_probe_name(&tk->tp)); 1318 1319 if (!tk->symbol) 1320 seq_printf(m, " 0x%p", tk->rp.kp.addr); 1321 else if (tk->rp.kp.offset) 1322 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk), 1323 tk->rp.kp.offset); 1324 else 1325 seq_printf(m, " %s", trace_kprobe_symbol(tk)); 1326 1327 for (i = 0; i < tk->tp.nr_args; i++) 1328 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); 1329 seq_putc(m, '\n'); 1330 1331 trace_probe_dump_args(m, &tk->tp); 1332 1333 return 0; 1334 } 1335 1336 static int probes_seq_show(struct seq_file *m, void *v) 1337 { 1338 struct dyn_event *ev = v; 1339 1340 if (!is_trace_kprobe(ev)) 1341 return 0; 1342 1343 return trace_kprobe_show(m, ev); 1344 } 1345 1346 static const struct seq_operations probes_seq_op = { 1347 .start = dyn_event_seq_start, 1348 .next = dyn_event_seq_next, 1349 .stop = dyn_event_seq_stop, 1350 .show = probes_seq_show 1351 }; 1352 1353 static int probes_open(struct inode *inode, struct file *file) 1354 { 1355 int ret; 1356 1357 ret = security_locked_down(LOCKDOWN_TRACEFS); 1358 if (ret) 1359 return ret; 1360 1361 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1362 ret = dyn_events_release_all(&trace_kprobe_ops); 1363 if (ret < 0) 1364 return ret; 1365 } 1366 1367 return seq_open(file, &probes_seq_op); 1368 } 1369 1370 static ssize_t probes_write(struct file *file, const char __user *buffer, 1371 size_t count, loff_t *ppos) 1372 { 1373 return trace_parse_run_command(file, buffer, count, ppos, 1374 create_or_delete_trace_kprobe); 1375 } 1376 1377 static const struct file_operations kprobe_events_ops = { 1378 .owner = THIS_MODULE, 1379 .open = probes_open, 1380 .read = seq_read, 1381 .llseek = seq_lseek, 1382 .release = seq_release, 1383 .write = probes_write, 1384 }; 1385 1386 static unsigned long trace_kprobe_missed(struct trace_kprobe *tk) 1387 { 1388 return trace_kprobe_is_return(tk) ? 1389 tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed; 1390 } 1391 1392 /* Probes profiling interfaces */ 1393 static int probes_profile_seq_show(struct seq_file *m, void *v) 1394 { 1395 struct dyn_event *ev = v; 1396 struct trace_kprobe *tk; 1397 unsigned long nmissed; 1398 1399 if (!is_trace_kprobe(ev)) 1400 return 0; 1401 1402 tk = to_trace_kprobe(ev); 1403 nmissed = trace_kprobe_missed(tk); 1404 seq_printf(m, " %-44s %15lu %15lu\n", 1405 trace_probe_name(&tk->tp), 1406 trace_kprobe_nhit(tk), 1407 nmissed); 1408 1409 return 0; 1410 } 1411 1412 static const struct seq_operations profile_seq_op = { 1413 .start = dyn_event_seq_start, 1414 .next = dyn_event_seq_next, 1415 .stop = dyn_event_seq_stop, 1416 .show = probes_profile_seq_show 1417 }; 1418 1419 static int profile_open(struct inode *inode, struct file *file) 1420 { 1421 int ret; 1422 1423 ret = security_locked_down(LOCKDOWN_TRACEFS); 1424 if (ret) 1425 return ret; 1426 1427 return seq_open(file, &profile_seq_op); 1428 } 1429 1430 static const struct file_operations kprobe_profile_ops = { 1431 .owner = THIS_MODULE, 1432 .open = profile_open, 1433 .read = seq_read, 1434 .llseek = seq_lseek, 1435 .release = seq_release, 1436 }; 1437 1438 /* Note that we don't verify it, since the code does not come from user space */ 1439 static int 1440 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 1441 void *dest, void *base) 1442 { 1443 struct pt_regs *regs = rec; 1444 unsigned long val; 1445 int ret; 1446 1447 retry: 1448 /* 1st stage: get value from context */ 1449 switch (code->op) { 1450 case FETCH_OP_REG: 1451 val = regs_get_register(regs, code->param); 1452 break; 1453 case FETCH_OP_STACK: 1454 val = regs_get_kernel_stack_nth(regs, code->param); 1455 break; 1456 case FETCH_OP_STACKP: 1457 val = kernel_stack_pointer(regs); 1458 break; 1459 case FETCH_OP_RETVAL: 1460 val = regs_return_value(regs); 1461 break; 1462 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 1463 case FETCH_OP_ARG: 1464 val = regs_get_kernel_argument(regs, code->param); 1465 break; 1466 case FETCH_OP_EDATA: 1467 val = *(unsigned long *)((unsigned long)edata + code->offset); 1468 break; 1469 #endif 1470 case FETCH_NOP_SYMBOL: /* Ignore a place holder */ 1471 code++; 1472 goto retry; 1473 default: 1474 ret = process_common_fetch_insn(code, &val); 1475 if (ret < 0) 1476 return ret; 1477 } 1478 code++; 1479 1480 return process_fetch_insn_bottom(code, val, dest, base); 1481 } 1482 NOKPROBE_SYMBOL(process_fetch_insn) 1483 1484 /* Kprobe handler */ 1485 static nokprobe_inline void 1486 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 1487 struct trace_event_file *trace_file) 1488 { 1489 struct kprobe_trace_entry_head *entry; 1490 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1491 struct trace_event_buffer fbuffer; 1492 int dsize; 1493 1494 WARN_ON(call != trace_file->event_call); 1495 1496 if (trace_trigger_soft_disabled(trace_file)) 1497 return; 1498 1499 dsize = __get_data_size(&tk->tp, regs, NULL); 1500 1501 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1502 sizeof(*entry) + tk->tp.size + dsize); 1503 if (!entry) 1504 return; 1505 1506 fbuffer.regs = regs; 1507 entry->ip = (unsigned long)tk->rp.kp.addr; 1508 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1509 1510 trace_event_buffer_commit(&fbuffer); 1511 } 1512 1513 static void 1514 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) 1515 { 1516 struct event_file_link *link; 1517 1518 trace_probe_for_each_link_rcu(link, &tk->tp) 1519 __kprobe_trace_func(tk, regs, link->file); 1520 } 1521 NOKPROBE_SYMBOL(kprobe_trace_func); 1522 1523 /* Kretprobe handler */ 1524 1525 static int trace_kprobe_entry_handler(struct kretprobe_instance *ri, 1526 struct pt_regs *regs) 1527 { 1528 struct kretprobe *rp = get_kretprobe(ri); 1529 struct trace_kprobe *tk; 1530 1531 /* 1532 * There is a small chance that get_kretprobe(ri) returns NULL when 1533 * the kretprobe is unregister on another CPU between kretprobe's 1534 * trampoline_handler and this function. 1535 */ 1536 if (unlikely(!rp)) 1537 return -ENOENT; 1538 1539 tk = container_of(rp, struct trace_kprobe, rp); 1540 1541 /* store argument values into ri->data as entry data */ 1542 if (tk->tp.entry_arg) 1543 store_trace_entry_data(ri->data, &tk->tp, regs); 1544 1545 return 0; 1546 } 1547 1548 1549 static nokprobe_inline void 1550 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1551 struct pt_regs *regs, 1552 struct trace_event_file *trace_file) 1553 { 1554 struct kretprobe_trace_entry_head *entry; 1555 struct trace_event_buffer fbuffer; 1556 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1557 int dsize; 1558 1559 WARN_ON(call != trace_file->event_call); 1560 1561 if (trace_trigger_soft_disabled(trace_file)) 1562 return; 1563 1564 dsize = __get_data_size(&tk->tp, regs, ri->data); 1565 1566 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 1567 sizeof(*entry) + tk->tp.size + dsize); 1568 if (!entry) 1569 return; 1570 1571 fbuffer.regs = regs; 1572 entry->func = (unsigned long)tk->rp.kp.addr; 1573 entry->ret_ip = get_kretprobe_retaddr(ri); 1574 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1575 1576 trace_event_buffer_commit(&fbuffer); 1577 } 1578 1579 static void 1580 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1581 struct pt_regs *regs) 1582 { 1583 struct event_file_link *link; 1584 1585 trace_probe_for_each_link_rcu(link, &tk->tp) 1586 __kretprobe_trace_func(tk, ri, regs, link->file); 1587 } 1588 NOKPROBE_SYMBOL(kretprobe_trace_func); 1589 1590 /* Event entry printers */ 1591 static enum print_line_t 1592 print_kprobe_event(struct trace_iterator *iter, int flags, 1593 struct trace_event *event) 1594 { 1595 struct kprobe_trace_entry_head *field; 1596 struct trace_seq *s = &iter->seq; 1597 struct trace_probe *tp; 1598 1599 field = (struct kprobe_trace_entry_head *)iter->ent; 1600 tp = trace_probe_primary_from_call( 1601 container_of(event, struct trace_event_call, event)); 1602 if (WARN_ON_ONCE(!tp)) 1603 goto out; 1604 1605 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1606 1607 if (!seq_print_ip_sym_offset(s, field->ip, flags)) 1608 goto out; 1609 1610 trace_seq_putc(s, ')'); 1611 1612 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1613 (u8 *)&field[1], field) < 0) 1614 goto out; 1615 1616 trace_seq_putc(s, '\n'); 1617 out: 1618 return trace_handle_return(s); 1619 } 1620 1621 static enum print_line_t 1622 print_kretprobe_event(struct trace_iterator *iter, int flags, 1623 struct trace_event *event) 1624 { 1625 struct kretprobe_trace_entry_head *field; 1626 struct trace_seq *s = &iter->seq; 1627 struct trace_probe *tp; 1628 1629 field = (struct kretprobe_trace_entry_head *)iter->ent; 1630 tp = trace_probe_primary_from_call( 1631 container_of(event, struct trace_event_call, event)); 1632 if (WARN_ON_ONCE(!tp)) 1633 goto out; 1634 1635 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 1636 1637 if (!seq_print_ip_sym_offset(s, field->ret_ip, flags)) 1638 goto out; 1639 1640 trace_seq_puts(s, " <- "); 1641 1642 if (!seq_print_ip_sym_no_offset(s, field->func, flags)) 1643 goto out; 1644 1645 trace_seq_putc(s, ')'); 1646 1647 if (trace_probe_print_args(s, tp->args, tp->nr_args, 1648 (u8 *)&field[1], field) < 0) 1649 goto out; 1650 1651 trace_seq_putc(s, '\n'); 1652 1653 out: 1654 return trace_handle_return(s); 1655 } 1656 1657 1658 static int kprobe_event_define_fields(struct trace_event_call *event_call) 1659 { 1660 int ret; 1661 struct kprobe_trace_entry_head field; 1662 struct trace_probe *tp; 1663 1664 tp = trace_probe_primary_from_call(event_call); 1665 if (WARN_ON_ONCE(!tp)) 1666 return -ENOENT; 1667 1668 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1669 1670 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1671 } 1672 1673 static int kretprobe_event_define_fields(struct trace_event_call *event_call) 1674 { 1675 int ret; 1676 struct kretprobe_trace_entry_head field; 1677 struct trace_probe *tp; 1678 1679 tp = trace_probe_primary_from_call(event_call); 1680 if (WARN_ON_ONCE(!tp)) 1681 return -ENOENT; 1682 1683 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1684 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1685 1686 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 1687 } 1688 1689 #ifdef CONFIG_PERF_EVENTS 1690 1691 /* Kprobe profile handler */ 1692 static int 1693 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) 1694 { 1695 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1696 struct kprobe_trace_entry_head *entry; 1697 struct hlist_head *head; 1698 int size, __size, dsize; 1699 int rctx; 1700 1701 if (bpf_prog_array_valid(call)) { 1702 unsigned long orig_ip = instruction_pointer(regs); 1703 int ret; 1704 1705 ret = trace_call_bpf(call, regs); 1706 1707 /* 1708 * We need to check and see if we modified the pc of the 1709 * pt_regs, and if so return 1 so that we don't do the 1710 * single stepping. 1711 */ 1712 if (orig_ip != instruction_pointer(regs)) 1713 return 1; 1714 if (!ret) 1715 return 0; 1716 } 1717 1718 head = this_cpu_ptr(call->perf_events); 1719 if (hlist_empty(head)) 1720 return 0; 1721 1722 dsize = __get_data_size(&tk->tp, regs, NULL); 1723 __size = sizeof(*entry) + tk->tp.size + dsize; 1724 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1725 size -= sizeof(u32); 1726 1727 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1728 if (!entry) 1729 return 0; 1730 1731 entry->ip = (unsigned long)tk->rp.kp.addr; 1732 store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize); 1733 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1734 head, NULL); 1735 return 0; 1736 } 1737 NOKPROBE_SYMBOL(kprobe_perf_func); 1738 1739 /* Kretprobe profile handler */ 1740 static void 1741 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, 1742 struct pt_regs *regs) 1743 { 1744 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1745 struct kretprobe_trace_entry_head *entry; 1746 struct hlist_head *head; 1747 int size, __size, dsize; 1748 int rctx; 1749 1750 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs)) 1751 return; 1752 1753 head = this_cpu_ptr(call->perf_events); 1754 if (hlist_empty(head)) 1755 return; 1756 1757 dsize = __get_data_size(&tk->tp, regs, ri->data); 1758 __size = sizeof(*entry) + tk->tp.size + dsize; 1759 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1760 size -= sizeof(u32); 1761 1762 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1763 if (!entry) 1764 return; 1765 1766 entry->func = (unsigned long)tk->rp.kp.addr; 1767 entry->ret_ip = get_kretprobe_retaddr(ri); 1768 store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize); 1769 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1770 head, NULL); 1771 } 1772 NOKPROBE_SYMBOL(kretprobe_perf_func); 1773 1774 int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, 1775 const char **symbol, u64 *probe_offset, 1776 u64 *probe_addr, unsigned long *missed, 1777 bool perf_type_tracepoint) 1778 { 1779 const char *pevent = trace_event_name(event->tp_event); 1780 const char *group = event->tp_event->class->system; 1781 struct trace_kprobe *tk; 1782 1783 if (perf_type_tracepoint) 1784 tk = find_trace_kprobe(pevent, group); 1785 else 1786 tk = trace_kprobe_primary_from_call(event->tp_event); 1787 if (!tk) 1788 return -EINVAL; 1789 1790 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE 1791 : BPF_FD_TYPE_KPROBE; 1792 *probe_offset = tk->rp.kp.offset; 1793 *probe_addr = kallsyms_show_value(current_cred()) ? 1794 (unsigned long)tk->rp.kp.addr : 0; 1795 *symbol = tk->symbol; 1796 if (missed) 1797 *missed = trace_kprobe_missed(tk); 1798 return 0; 1799 } 1800 #endif /* CONFIG_PERF_EVENTS */ 1801 1802 /* 1803 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1804 * 1805 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe 1806 * lockless, but we can't race with this __init function. 1807 */ 1808 static int kprobe_register(struct trace_event_call *event, 1809 enum trace_reg type, void *data) 1810 { 1811 struct trace_event_file *file = data; 1812 1813 switch (type) { 1814 case TRACE_REG_REGISTER: 1815 return enable_trace_kprobe(event, file); 1816 case TRACE_REG_UNREGISTER: 1817 return disable_trace_kprobe(event, file); 1818 1819 #ifdef CONFIG_PERF_EVENTS 1820 case TRACE_REG_PERF_REGISTER: 1821 return enable_trace_kprobe(event, NULL); 1822 case TRACE_REG_PERF_UNREGISTER: 1823 return disable_trace_kprobe(event, NULL); 1824 case TRACE_REG_PERF_OPEN: 1825 case TRACE_REG_PERF_CLOSE: 1826 case TRACE_REG_PERF_ADD: 1827 case TRACE_REG_PERF_DEL: 1828 return 0; 1829 #endif 1830 } 1831 return 0; 1832 } 1833 1834 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) 1835 { 1836 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); 1837 unsigned int flags = trace_probe_load_flag(&tk->tp); 1838 int ret = 0; 1839 1840 raw_cpu_inc(*tk->nhit); 1841 1842 if (flags & TP_FLAG_TRACE) 1843 kprobe_trace_func(tk, regs); 1844 #ifdef CONFIG_PERF_EVENTS 1845 if (flags & TP_FLAG_PROFILE) 1846 ret = kprobe_perf_func(tk, regs); 1847 #endif 1848 return ret; 1849 } 1850 NOKPROBE_SYMBOL(kprobe_dispatcher); 1851 1852 static int 1853 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) 1854 { 1855 struct kretprobe *rp = get_kretprobe(ri); 1856 struct trace_kprobe *tk; 1857 unsigned int flags; 1858 1859 /* 1860 * There is a small chance that get_kretprobe(ri) returns NULL when 1861 * the kretprobe is unregister on another CPU between kretprobe's 1862 * trampoline_handler and this function. 1863 */ 1864 if (unlikely(!rp)) 1865 return 0; 1866 1867 tk = container_of(rp, struct trace_kprobe, rp); 1868 raw_cpu_inc(*tk->nhit); 1869 1870 flags = trace_probe_load_flag(&tk->tp); 1871 if (flags & TP_FLAG_TRACE) 1872 kretprobe_trace_func(tk, ri, regs); 1873 #ifdef CONFIG_PERF_EVENTS 1874 if (flags & TP_FLAG_PROFILE) 1875 kretprobe_perf_func(tk, ri, regs); 1876 #endif 1877 return 0; /* We don't tweak kernel, so just return 0 */ 1878 } 1879 NOKPROBE_SYMBOL(kretprobe_dispatcher); 1880 1881 static struct trace_event_functions kretprobe_funcs = { 1882 .trace = print_kretprobe_event 1883 }; 1884 1885 static struct trace_event_functions kprobe_funcs = { 1886 .trace = print_kprobe_event 1887 }; 1888 1889 static struct trace_event_fields kretprobe_fields_array[] = { 1890 { .type = TRACE_FUNCTION_TYPE, 1891 .define_fields = kretprobe_event_define_fields }, 1892 {} 1893 }; 1894 1895 static struct trace_event_fields kprobe_fields_array[] = { 1896 { .type = TRACE_FUNCTION_TYPE, 1897 .define_fields = kprobe_event_define_fields }, 1898 {} 1899 }; 1900 1901 static inline void init_trace_event_call(struct trace_kprobe *tk) 1902 { 1903 struct trace_event_call *call = trace_probe_event_call(&tk->tp); 1904 1905 if (trace_kprobe_is_return(tk)) { 1906 call->event.funcs = &kretprobe_funcs; 1907 call->class->fields_array = kretprobe_fields_array; 1908 } else { 1909 call->event.funcs = &kprobe_funcs; 1910 call->class->fields_array = kprobe_fields_array; 1911 } 1912 1913 call->flags = TRACE_EVENT_FL_KPROBE; 1914 call->class->reg = kprobe_register; 1915 } 1916 1917 static int register_kprobe_event(struct trace_kprobe *tk) 1918 { 1919 init_trace_event_call(tk); 1920 1921 return trace_probe_register_event_call(&tk->tp); 1922 } 1923 1924 static int unregister_kprobe_event(struct trace_kprobe *tk) 1925 { 1926 return trace_probe_unregister_event_call(&tk->tp); 1927 } 1928 1929 #ifdef CONFIG_PERF_EVENTS 1930 1931 /* create a trace_kprobe, but don't add it to global lists */ 1932 struct trace_event_call * 1933 create_local_trace_kprobe(char *func, void *addr, unsigned long offs, 1934 bool is_return) 1935 { 1936 enum probe_print_type ptype; 1937 struct trace_kprobe *tk __free(free_trace_kprobe) = NULL; 1938 int ret; 1939 char *event; 1940 1941 if (func) { 1942 ret = validate_probe_symbol(func); 1943 if (ret) 1944 return ERR_PTR(ret); 1945 } 1946 1947 /* 1948 * local trace_kprobes are not added to dyn_event, so they are never 1949 * searched in find_trace_kprobe(). Therefore, there is no concern of 1950 * duplicated name here. 1951 */ 1952 event = func ? func : "DUMMY_EVENT"; 1953 1954 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func, 1955 offs, 0 /* maxactive */, 0 /* nargs */, 1956 is_return); 1957 1958 if (IS_ERR(tk)) { 1959 pr_info("Failed to allocate trace_probe.(%d)\n", 1960 (int)PTR_ERR(tk)); 1961 return ERR_CAST(tk); 1962 } 1963 1964 init_trace_event_call(tk); 1965 1966 ptype = trace_kprobe_is_return(tk) ? 1967 PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1968 if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) 1969 return ERR_PTR(-ENOMEM); 1970 1971 ret = __register_trace_kprobe(tk); 1972 if (ret < 0) 1973 return ERR_PTR(ret); 1974 1975 return trace_probe_event_call(&(no_free_ptr(tk)->tp)); 1976 } 1977 1978 void destroy_local_trace_kprobe(struct trace_event_call *event_call) 1979 { 1980 struct trace_kprobe *tk; 1981 1982 tk = trace_kprobe_primary_from_call(event_call); 1983 if (unlikely(!tk)) 1984 return; 1985 1986 if (trace_probe_is_enabled(&tk->tp)) { 1987 WARN_ON(1); 1988 return; 1989 } 1990 1991 __unregister_trace_kprobe(tk); 1992 1993 free_trace_kprobe(tk); 1994 } 1995 #endif /* CONFIG_PERF_EVENTS */ 1996 1997 static __init void enable_boot_kprobe_events(void) 1998 { 1999 struct trace_array *tr = top_trace_array(); 2000 struct trace_event_file *file; 2001 struct trace_kprobe *tk; 2002 struct dyn_event *pos; 2003 2004 if (trace_kprobe_list_empty()) 2005 return; 2006 2007 guard(mutex)(&event_mutex); 2008 for_each_trace_kprobe(tk, pos) { 2009 list_for_each_entry(file, &tr->events, list) 2010 if (file->event_call == trace_probe_event_call(&tk->tp)) 2011 trace_event_enable_disable(file, 1, 0); 2012 } 2013 } 2014 2015 static __init void setup_boot_kprobe_events(void) 2016 { 2017 char *p, *cmd = kprobe_boot_events_buf; 2018 int ret; 2019 2020 strreplace(kprobe_boot_events_buf, ',', ' '); 2021 2022 while (cmd && *cmd != '\0') { 2023 p = strchr(cmd, ';'); 2024 if (p) 2025 *p++ = '\0'; 2026 2027 ret = create_or_delete_trace_kprobe(cmd); 2028 if (ret) 2029 pr_warn("Failed to add event(%d): %s\n", ret, cmd); 2030 2031 cmd = p; 2032 } 2033 2034 enable_boot_kprobe_events(); 2035 } 2036 2037 /* 2038 * Register dynevent at core_initcall. This allows kernel to setup kprobe 2039 * events in postcore_initcall without tracefs. 2040 */ 2041 static __init int init_kprobe_trace_early(void) 2042 { 2043 int ret; 2044 2045 ret = dyn_event_register(&trace_kprobe_ops); 2046 if (ret) 2047 return ret; 2048 2049 if (trace_kprobe_register_module_notifier()) 2050 return -EINVAL; 2051 2052 return 0; 2053 } 2054 core_initcall(init_kprobe_trace_early); 2055 2056 /* Make a tracefs interface for controlling probe points */ 2057 static __init int init_kprobe_trace(void) 2058 { 2059 int ret; 2060 2061 ret = tracing_init_dentry(); 2062 if (ret) 2063 return 0; 2064 2065 /* Event list interface */ 2066 trace_create_file("kprobe_events", TRACE_MODE_WRITE, 2067 NULL, NULL, &kprobe_events_ops); 2068 2069 /* Profile interface */ 2070 trace_create_file("kprobe_profile", TRACE_MODE_READ, 2071 NULL, NULL, &kprobe_profile_ops); 2072 2073 /* If no 'kprobe_event=' cmd is provided, return directly. */ 2074 if (kprobe_boot_events_buf[0] == '\0') 2075 return 0; 2076 2077 setup_boot_kprobe_events(); 2078 2079 return 0; 2080 } 2081 fs_initcall(init_kprobe_trace); 2082 2083 2084 #ifdef CONFIG_FTRACE_STARTUP_TEST 2085 static __init struct trace_event_file * 2086 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) 2087 { 2088 struct trace_event_file *file; 2089 2090 list_for_each_entry(file, &tr->events, list) 2091 if (file->event_call == trace_probe_event_call(&tk->tp)) 2092 return file; 2093 2094 return NULL; 2095 } 2096 2097 /* 2098 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this 2099 * stage, we can do this lockless. 2100 */ 2101 static __init int kprobe_trace_self_tests_init(void) 2102 { 2103 int ret, warn = 0; 2104 int (*target)(int, int, int, int, int, int); 2105 struct trace_kprobe *tk; 2106 struct trace_event_file *file; 2107 2108 if (unlikely(tracing_disabled)) 2109 return -ENODEV; 2110 2111 if (tracing_selftest_disabled) 2112 return 0; 2113 2114 target = kprobe_trace_selftest_target; 2115 2116 pr_info("Testing kprobe tracing: "); 2117 2118 ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)"); 2119 if (WARN_ONCE(ret, "error on probing function entry.")) { 2120 warn++; 2121 } else { 2122 /* Enable trace point */ 2123 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2124 if (WARN_ONCE(tk == NULL, "error on probing function entry.")) { 2125 warn++; 2126 } else { 2127 file = find_trace_probe_file(tk, top_trace_array()); 2128 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2129 warn++; 2130 } else 2131 enable_trace_kprobe( 2132 trace_probe_event_call(&tk->tp), file); 2133 } 2134 } 2135 2136 ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval"); 2137 if (WARN_ONCE(ret, "error on probing function return.")) { 2138 warn++; 2139 } else { 2140 /* Enable trace point */ 2141 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2142 if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) { 2143 warn++; 2144 } else { 2145 file = find_trace_probe_file(tk, top_trace_array()); 2146 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2147 warn++; 2148 } else 2149 enable_trace_kprobe( 2150 trace_probe_event_call(&tk->tp), file); 2151 } 2152 } 2153 2154 if (warn) 2155 goto end; 2156 2157 ret = target(1, 2, 3, 4, 5, 6); 2158 2159 /* 2160 * Not expecting an error here, the check is only to prevent the 2161 * optimizer from removing the call to target() as otherwise there 2162 * are no side-effects and the call is never performed. 2163 */ 2164 if (ret != 21) 2165 warn++; 2166 2167 /* Disable trace points before removing it */ 2168 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); 2169 if (WARN_ONCE(tk == NULL, "error on getting test probe.")) { 2170 warn++; 2171 } else { 2172 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2173 "incorrect number of testprobe hits.")) 2174 warn++; 2175 2176 file = find_trace_probe_file(tk, top_trace_array()); 2177 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2178 warn++; 2179 } else 2180 disable_trace_kprobe( 2181 trace_probe_event_call(&tk->tp), file); 2182 } 2183 2184 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); 2185 if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) { 2186 warn++; 2187 } else { 2188 if (WARN_ONCE(trace_kprobe_nhit(tk) != 1, 2189 "incorrect number of testprobe2 hits.")) 2190 warn++; 2191 2192 file = find_trace_probe_file(tk, top_trace_array()); 2193 if (WARN_ONCE(file == NULL, "error on getting probe file.")) { 2194 warn++; 2195 } else 2196 disable_trace_kprobe( 2197 trace_probe_event_call(&tk->tp), file); 2198 } 2199 2200 ret = create_or_delete_trace_kprobe("-:testprobe"); 2201 if (WARN_ONCE(ret, "error on deleting a probe.")) 2202 warn++; 2203 2204 ret = create_or_delete_trace_kprobe("-:testprobe2"); 2205 if (WARN_ONCE(ret, "error on deleting a probe.")) 2206 warn++; 2207 2208 2209 end: 2210 /* 2211 * Wait for the optimizer work to finish. Otherwise it might fiddle 2212 * with probes in already freed __init text. 2213 */ 2214 wait_for_kprobe_optimizer(); 2215 if (warn) 2216 pr_cont("NG: Some tests are failed. Please check them.\n"); 2217 else 2218 pr_cont("OK\n"); 2219 return 0; 2220 } 2221 2222 late_initcall(kprobe_trace_self_tests_init); 2223 2224 #endif 2225