1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * uprobes-based tracing events 4 * 5 * Copyright (C) IBM Corporation, 2010-2012 6 * Author: Srikar Dronamraju <srikar@linux.vnet.ibm.com> 7 */ 8 #define pr_fmt(fmt) "trace_uprobe: " fmt 9 10 #include <linux/bpf-cgroup.h> 11 #include <linux/cleanup.h> 12 #include <linux/ctype.h> 13 #include <linux/filter.h> 14 #include <linux/module.h> 15 #include <linux/namei.h> 16 #include <linux/percpu.h> 17 #include <linux/rculist.h> 18 #include <linux/security.h> 19 #include <linux/string.h> 20 #include <linux/uaccess.h> 21 #include <linux/uprobes.h> 22 23 #include "trace.h" 24 #include "trace_dynevent.h" 25 #include "trace_probe.h" 26 #include "trace_probe_tmpl.h" 27 28 #define UPROBE_EVENT_SYSTEM "uprobes" 29 30 struct uprobe_trace_entry_head { 31 struct trace_entry ent; 32 unsigned long vaddr[]; 33 }; 34 35 #define SIZEOF_TRACE_ENTRY(is_return) \ 36 (sizeof(struct uprobe_trace_entry_head) + \ 37 sizeof(unsigned long) * (is_return ? 2 : 1)) 38 39 #define DATAOF_TRACE_ENTRY(entry, is_return) \ 40 ((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return)) 41 42 static int trace_uprobe_create(const char *raw_command); 43 static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev); 44 static int trace_uprobe_release(struct dyn_event *ev); 45 static bool trace_uprobe_is_busy(struct dyn_event *ev); 46 static bool trace_uprobe_match(const char *system, const char *event, 47 int argc, const char **argv, struct dyn_event *ev); 48 49 static struct dyn_event_operations trace_uprobe_ops = { 50 .create = trace_uprobe_create, 51 .show = trace_uprobe_show, 52 .is_busy = trace_uprobe_is_busy, 53 .free = trace_uprobe_release, 54 .match = trace_uprobe_match, 55 }; 56 57 /* 58 * uprobe event core functions 59 */ 60 struct trace_uprobe { 61 struct dyn_event devent; 62 struct uprobe_consumer consumer; 63 struct path path; 64 char *filename; 65 struct uprobe *uprobe; 66 unsigned long offset; 67 unsigned long ref_ctr_offset; 68 unsigned long __percpu *nhits; 69 struct trace_probe tp; 70 }; 71 72 static bool is_trace_uprobe(struct dyn_event *ev) 73 { 74 return ev->ops == &trace_uprobe_ops; 75 } 76 77 static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev) 78 { 79 return container_of(ev, struct trace_uprobe, devent); 80 } 81 82 /** 83 * for_each_trace_uprobe - iterate over the trace_uprobe list 84 * @pos: the struct trace_uprobe * for each entry 85 * @dpos: the struct dyn_event * to use as a loop cursor 86 */ 87 #define for_each_trace_uprobe(pos, dpos) \ 88 for_each_dyn_event(dpos) \ 89 if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos))) 90 91 static int register_uprobe_event(struct trace_uprobe *tu); 92 static int unregister_uprobe_event(struct trace_uprobe *tu); 93 94 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs, 95 __u64 *data); 96 static int uretprobe_dispatcher(struct uprobe_consumer *con, 97 unsigned long func, struct pt_regs *regs, 98 __u64 *data); 99 100 #ifdef CONFIG_STACK_GROWSUP 101 static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n) 102 { 103 return addr - (n * sizeof(long)); 104 } 105 #else 106 static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n) 107 { 108 return addr + (n * sizeof(long)); 109 } 110 #endif 111 112 static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n) 113 { 114 unsigned long ret; 115 unsigned long addr = user_stack_pointer(regs); 116 117 addr = adjust_stack_addr(addr, n); 118 119 if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret))) 120 return 0; 121 122 return ret; 123 } 124 125 /* 126 * Uprobes-specific fetch functions 127 */ 128 static nokprobe_inline int 129 probe_mem_read(void *dest, void *src, size_t size) 130 { 131 void __user *vaddr = (void __force __user *)src; 132 133 return copy_from_user(dest, vaddr, size) ? -EFAULT : 0; 134 } 135 136 static nokprobe_inline int 137 probe_mem_read_user(void *dest, void *src, size_t size) 138 { 139 return probe_mem_read(dest, src, size); 140 } 141 142 /* 143 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max 144 * length and relative data location. 145 */ 146 static nokprobe_inline int 147 fetch_store_string(unsigned long addr, void *dest, void *base) 148 { 149 long ret; 150 u32 loc = *(u32 *)dest; 151 int maxlen = get_loc_len(loc); 152 u8 *dst = get_loc_data(dest, base); 153 void __user *src = (void __force __user *) addr; 154 155 if (unlikely(!maxlen)) 156 return -ENOMEM; 157 158 if (addr == FETCH_TOKEN_COMM) 159 ret = strscpy(dst, current->comm, maxlen); 160 else 161 ret = strncpy_from_user(dst, src, maxlen); 162 if (ret >= 0) { 163 if (ret == maxlen) 164 dst[ret - 1] = '\0'; 165 else 166 /* 167 * Include the terminating null byte. In this case it 168 * was copied by strncpy_from_user but not accounted 169 * for in ret. 170 */ 171 ret++; 172 *(u32 *)dest = make_data_loc(ret, (void *)dst - base); 173 } else 174 *(u32 *)dest = make_data_loc(0, (void *)dst - base); 175 176 return ret; 177 } 178 179 static nokprobe_inline int 180 fetch_store_string_user(unsigned long addr, void *dest, void *base) 181 { 182 return fetch_store_string(addr, dest, base); 183 } 184 185 /* Return the length of string -- including null terminal byte */ 186 static nokprobe_inline int 187 fetch_store_strlen(unsigned long addr) 188 { 189 int len; 190 void __user *vaddr = (void __force __user *) addr; 191 192 if (addr == FETCH_TOKEN_COMM) 193 len = strlen(current->comm) + 1; 194 else 195 len = strnlen_user(vaddr, MAX_STRING_SIZE); 196 197 return (len > MAX_STRING_SIZE) ? 0 : len; 198 } 199 200 static nokprobe_inline int 201 fetch_store_strlen_user(unsigned long addr) 202 { 203 return fetch_store_strlen(addr); 204 } 205 206 static unsigned long translate_user_vaddr(unsigned long file_offset) 207 { 208 unsigned long base_addr; 209 struct uprobe_dispatch_data *udd; 210 211 udd = (void *) current->utask->vaddr; 212 213 base_addr = udd->bp_addr - udd->tu->offset; 214 return base_addr + file_offset; 215 } 216 217 /* Note that we don't verify it, since the code does not come from user space */ 218 static int 219 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 220 void *dest, void *base) 221 { 222 struct pt_regs *regs = rec; 223 unsigned long val; 224 int ret; 225 226 /* 1st stage: get value from context */ 227 switch (code->op) { 228 case FETCH_OP_REG: 229 val = regs_get_register(regs, code->param); 230 break; 231 case FETCH_OP_STACK: 232 val = get_user_stack_nth(regs, code->param); 233 break; 234 case FETCH_OP_STACKP: 235 val = user_stack_pointer(regs); 236 break; 237 case FETCH_OP_RETVAL: 238 val = regs_return_value(regs); 239 break; 240 case FETCH_OP_COMM: 241 val = FETCH_TOKEN_COMM; 242 break; 243 case FETCH_OP_FOFFS: 244 val = translate_user_vaddr(code->immediate); 245 break; 246 default: 247 ret = process_common_fetch_insn(code, &val); 248 if (ret < 0) 249 return ret; 250 } 251 code++; 252 253 return process_fetch_insn_bottom(code, val, dest, base); 254 } 255 NOKPROBE_SYMBOL(process_fetch_insn) 256 257 static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter) 258 { 259 rwlock_init(&filter->rwlock); 260 filter->nr_systemwide = 0; 261 INIT_LIST_HEAD(&filter->perf_events); 262 } 263 264 static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter) 265 { 266 return !filter->nr_systemwide && list_empty(&filter->perf_events); 267 } 268 269 static inline bool is_ret_probe(struct trace_uprobe *tu) 270 { 271 return tu->consumer.ret_handler != NULL; 272 } 273 274 static bool trace_uprobe_is_busy(struct dyn_event *ev) 275 { 276 struct trace_uprobe *tu = to_trace_uprobe(ev); 277 278 return trace_probe_is_enabled(&tu->tp); 279 } 280 281 static bool trace_uprobe_match_command_head(struct trace_uprobe *tu, 282 int argc, const char **argv) 283 { 284 char buf[MAX_ARGSTR_LEN + 1]; 285 int len; 286 287 if (!argc) 288 return true; 289 290 len = strlen(tu->filename); 291 if (strncmp(tu->filename, argv[0], len) || argv[0][len] != ':') 292 return false; 293 294 if (tu->ref_ctr_offset == 0) 295 snprintf(buf, sizeof(buf), "0x%0*lx", 296 (int)(sizeof(void *) * 2), tu->offset); 297 else 298 snprintf(buf, sizeof(buf), "0x%0*lx(0x%lx)", 299 (int)(sizeof(void *) * 2), tu->offset, 300 tu->ref_ctr_offset); 301 if (strcmp(buf, &argv[0][len + 1])) 302 return false; 303 304 argc--; argv++; 305 306 return trace_probe_match_command_args(&tu->tp, argc, argv); 307 } 308 309 static bool trace_uprobe_match(const char *system, const char *event, 310 int argc, const char **argv, struct dyn_event *ev) 311 { 312 struct trace_uprobe *tu = to_trace_uprobe(ev); 313 314 return (event[0] == '\0' || 315 strcmp(trace_probe_name(&tu->tp), event) == 0) && 316 (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0) && 317 trace_uprobe_match_command_head(tu, argc, argv); 318 } 319 320 static nokprobe_inline struct trace_uprobe * 321 trace_uprobe_primary_from_call(struct trace_event_call *call) 322 { 323 struct trace_probe *tp; 324 325 tp = trace_probe_primary_from_call(call); 326 if (WARN_ON_ONCE(!tp)) 327 return NULL; 328 329 return container_of(tp, struct trace_uprobe, tp); 330 } 331 332 /* 333 * Allocate new trace_uprobe and initialize it (including uprobes). 334 */ 335 static struct trace_uprobe * 336 alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret) 337 { 338 struct trace_uprobe *tu; 339 int ret; 340 341 tu = kzalloc_flex(*tu, tp.args, nargs); 342 if (!tu) 343 return ERR_PTR(-ENOMEM); 344 345 tu->nhits = alloc_percpu(unsigned long); 346 if (!tu->nhits) { 347 ret = -ENOMEM; 348 goto error; 349 } 350 351 ret = trace_probe_init(&tu->tp, event, group, true, nargs); 352 if (ret < 0) 353 goto error; 354 355 dyn_event_init(&tu->devent, &trace_uprobe_ops); 356 tu->consumer.handler = uprobe_dispatcher; 357 if (is_ret) 358 tu->consumer.ret_handler = uretprobe_dispatcher; 359 init_trace_uprobe_filter(tu->tp.event->filter); 360 return tu; 361 362 error: 363 free_percpu(tu->nhits); 364 kfree(tu); 365 366 return ERR_PTR(ret); 367 } 368 369 static void free_trace_uprobe(struct trace_uprobe *tu) 370 { 371 if (!tu) 372 return; 373 374 path_put(&tu->path); 375 trace_probe_cleanup(&tu->tp); 376 kfree(tu->filename); 377 free_percpu(tu->nhits); 378 kfree(tu); 379 } 380 381 static struct trace_uprobe *find_probe_event(const char *event, const char *group) 382 { 383 struct dyn_event *pos; 384 struct trace_uprobe *tu; 385 386 for_each_trace_uprobe(tu, pos) 387 if (strcmp(trace_probe_name(&tu->tp), event) == 0 && 388 strcmp(trace_probe_group_name(&tu->tp), group) == 0) 389 return tu; 390 391 return NULL; 392 } 393 394 /* Unregister a trace_uprobe and probe_event */ 395 static int unregister_trace_uprobe(struct trace_uprobe *tu) 396 { 397 int ret; 398 399 if (trace_probe_has_sibling(&tu->tp)) 400 goto unreg; 401 402 /* If there's a reference to the dynamic event */ 403 if (trace_event_dyn_busy(trace_probe_event_call(&tu->tp))) 404 return -EBUSY; 405 406 ret = unregister_uprobe_event(tu); 407 if (ret) 408 return ret; 409 410 unreg: 411 dyn_event_remove(&tu->devent); 412 trace_probe_unlink(&tu->tp); 413 free_trace_uprobe(tu); 414 return 0; 415 } 416 417 static bool trace_uprobe_has_same_uprobe(struct trace_uprobe *orig, 418 struct trace_uprobe *comp) 419 { 420 struct trace_probe_event *tpe = orig->tp.event; 421 struct inode *comp_inode = d_real_inode(comp->path.dentry); 422 int i; 423 424 list_for_each_entry(orig, &tpe->probes, tp.list) { 425 if (comp_inode != d_real_inode(orig->path.dentry) || 426 comp->offset != orig->offset) 427 continue; 428 429 /* 430 * trace_probe_compare_arg_type() ensured that nr_args and 431 * each argument name and type are same. Let's compare comm. 432 */ 433 for (i = 0; i < orig->tp.nr_args; i++) { 434 if (strcmp(orig->tp.args[i].comm, 435 comp->tp.args[i].comm)) 436 break; 437 } 438 439 if (i == orig->tp.nr_args) 440 return true; 441 } 442 443 return false; 444 } 445 446 static int append_trace_uprobe(struct trace_uprobe *tu, struct trace_uprobe *to) 447 { 448 int ret; 449 450 ret = trace_probe_compare_arg_type(&tu->tp, &to->tp); 451 if (ret) { 452 /* Note that argument starts index = 2 */ 453 trace_probe_log_set_index(ret + 1); 454 trace_probe_log_err(0, DIFF_ARG_TYPE); 455 return -EEXIST; 456 } 457 if (trace_uprobe_has_same_uprobe(to, tu)) { 458 trace_probe_log_set_index(0); 459 trace_probe_log_err(0, SAME_PROBE); 460 return -EEXIST; 461 } 462 463 /* Append to existing event */ 464 ret = trace_probe_append(&tu->tp, &to->tp); 465 if (!ret) 466 dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp)); 467 468 return ret; 469 } 470 471 /* 472 * Uprobe with multiple reference counter is not allowed. i.e. 473 * If inode and offset matches, reference counter offset *must* 474 * match as well. Though, there is one exception: If user is 475 * replacing old trace_uprobe with new one(same group/event), 476 * then we allow same uprobe with new reference counter as far 477 * as the new one does not conflict with any other existing 478 * ones. 479 */ 480 static int validate_ref_ctr_offset(struct trace_uprobe *new) 481 { 482 struct dyn_event *pos; 483 struct trace_uprobe *tmp; 484 struct inode *new_inode = d_real_inode(new->path.dentry); 485 486 for_each_trace_uprobe(tmp, pos) { 487 if (new_inode == d_real_inode(tmp->path.dentry) && 488 new->offset == tmp->offset && 489 new->ref_ctr_offset != tmp->ref_ctr_offset) { 490 pr_warn("Reference counter offset mismatch."); 491 return -EINVAL; 492 } 493 } 494 return 0; 495 } 496 497 /* Register a trace_uprobe and probe_event */ 498 static int register_trace_uprobe(struct trace_uprobe *tu) 499 { 500 struct trace_uprobe *old_tu; 501 int ret; 502 503 guard(mutex)(&event_mutex); 504 505 ret = validate_ref_ctr_offset(tu); 506 if (ret) 507 return ret; 508 509 /* register as an event */ 510 old_tu = find_probe_event(trace_probe_name(&tu->tp), 511 trace_probe_group_name(&tu->tp)); 512 if (old_tu) { 513 if (is_ret_probe(tu) != is_ret_probe(old_tu)) { 514 trace_probe_log_set_index(0); 515 trace_probe_log_err(0, DIFF_PROBE_TYPE); 516 return -EEXIST; 517 } 518 return append_trace_uprobe(tu, old_tu); 519 } 520 521 ret = register_uprobe_event(tu); 522 if (ret) { 523 if (ret == -EEXIST) { 524 trace_probe_log_set_index(0); 525 trace_probe_log_err(0, EVENT_EXIST); 526 } else 527 pr_warn("Failed to register probe event(%d)\n", ret); 528 return ret; 529 } 530 531 dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp)); 532 533 return ret; 534 } 535 536 DEFINE_FREE(free_trace_uprobe, struct trace_uprobe *, if (_T) free_trace_uprobe(_T)) 537 538 /* 539 * Argument syntax: 540 * - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS] 541 */ 542 static int __trace_uprobe_create(int argc, const char **argv) 543 { 544 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 545 struct trace_uprobe *tu __free(free_trace_uprobe) = NULL; 546 const char *trlog __free(trace_probe_log_clear) = NULL; 547 const char *event = NULL, *group = UPROBE_EVENT_SYSTEM; 548 struct path path __free(path_put) = {}; 549 unsigned long offset, ref_ctr_offset; 550 char *filename __free(kfree) = NULL; 551 char *arg, *rctr, *rctr_end, *tmp; 552 char *gbuf __free(kfree) = NULL; 553 char *buf __free(kfree) = NULL; 554 enum probe_print_type ptype; 555 bool is_return = false; 556 int i, ret; 557 558 ref_ctr_offset = 0; 559 560 switch (argv[0][0]) { 561 case 'r': 562 is_return = true; 563 break; 564 case 'p': 565 break; 566 default: 567 return -ECANCELED; 568 } 569 570 if (argc < 2) 571 return -ECANCELED; 572 573 trlog = trace_probe_log_init("trace_uprobe", argc, argv); 574 575 if (argc - 2 > MAX_TRACE_ARGS) { 576 trace_probe_log_set_index(2); 577 trace_probe_log_err(0, TOO_MANY_ARGS); 578 return -E2BIG; 579 } 580 581 if (argv[0][1] == ':') 582 event = &argv[0][2]; 583 584 if (!strchr(argv[1], '/')) 585 return -ECANCELED; 586 587 filename = kstrdup(argv[1], GFP_KERNEL); 588 if (!filename) 589 return -ENOMEM; 590 591 /* Find the last occurrence, in case the path contains ':' too. */ 592 arg = strrchr(filename, ':'); 593 if (!arg || !isdigit(arg[1])) 594 return -ECANCELED; 595 596 trace_probe_log_set_index(1); /* filename is the 2nd argument */ 597 598 *arg++ = '\0'; 599 ret = kern_path(filename, LOOKUP_FOLLOW, &path); 600 if (ret) { 601 trace_probe_log_err(0, FILE_NOT_FOUND); 602 return ret; 603 } 604 if (!d_is_reg(path.dentry)) { 605 trace_probe_log_err(0, NO_REGULAR_FILE); 606 return -EINVAL; 607 } 608 609 /* Parse reference counter offset if specified. */ 610 rctr = strchr(arg, '('); 611 if (rctr) { 612 rctr_end = strchr(rctr, ')'); 613 if (!rctr_end) { 614 rctr_end = rctr + strlen(rctr); 615 trace_probe_log_err(rctr_end - filename, 616 REFCNT_OPEN_BRACE); 617 return -EINVAL; 618 } else if (rctr_end[1] != '\0') { 619 trace_probe_log_err(rctr_end + 1 - filename, 620 BAD_REFCNT_SUFFIX); 621 return -EINVAL; 622 } 623 624 *rctr++ = '\0'; 625 *rctr_end = '\0'; 626 ret = kstrtoul(rctr, 0, &ref_ctr_offset); 627 if (ret) { 628 trace_probe_log_err(rctr - filename, BAD_REFCNT); 629 return ret; 630 } 631 } 632 633 /* Check if there is %return suffix */ 634 tmp = strchr(arg, '%'); 635 if (tmp) { 636 if (!strcmp(tmp, "%return")) { 637 *tmp = '\0'; 638 is_return = true; 639 } else { 640 trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX); 641 return -EINVAL; 642 } 643 } 644 645 /* Parse uprobe offset. */ 646 ret = kstrtoul(arg, 0, &offset); 647 if (ret) { 648 trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS); 649 return ret; 650 } 651 652 /* setup a probe */ 653 trace_probe_log_set_index(0); 654 if (event) { 655 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 656 if (!gbuf) 657 return -ENOMEM; 658 659 ret = traceprobe_parse_event_name(&event, &group, gbuf, 660 event - argv[0]); 661 if (ret) 662 return ret; 663 } 664 665 if (!event) { 666 char *tail; 667 char *ptr; 668 669 tail = kstrdup(kbasename(filename), GFP_KERNEL); 670 if (!tail) 671 return -ENOMEM; 672 673 ptr = strpbrk(tail, ".-_"); 674 if (ptr) 675 *ptr = '\0'; 676 677 buf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 678 if (!buf) 679 return -ENOMEM; 680 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset); 681 event = buf; 682 kfree(tail); 683 } 684 685 argc -= 2; 686 argv += 2; 687 688 tu = alloc_trace_uprobe(group, event, argc, is_return); 689 if (IS_ERR(tu)) { 690 ret = PTR_ERR(tu); 691 /* This must return -ENOMEM otherwise there is a bug */ 692 WARN_ON_ONCE(ret != -ENOMEM); 693 return ret; 694 } 695 tu->offset = offset; 696 tu->ref_ctr_offset = ref_ctr_offset; 697 tu->path = path; 698 /* Clear @path so that it will not freed by path_put() */ 699 memset(&path, 0, sizeof(path)); 700 tu->filename = no_free_ptr(filename); 701 702 ctx = kzalloc_obj(*ctx); 703 if (!ctx) 704 return -ENOMEM; 705 ctx->flags = (is_return ? TPARG_FL_RETURN : 0) | TPARG_FL_USER; 706 707 /* parse arguments */ 708 for (i = 0; i < argc; i++) { 709 trace_probe_log_set_index(i + 2); 710 ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx); 711 if (ret) 712 return ret; 713 } 714 715 ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 716 ret = traceprobe_set_print_fmt(&tu->tp, ptype); 717 if (ret < 0) 718 return ret; 719 720 ret = register_trace_uprobe(tu); 721 if (!ret) 722 tu = NULL; 723 724 return ret; 725 } 726 727 int trace_uprobe_create(const char *raw_command) 728 { 729 return trace_probe_create(raw_command, __trace_uprobe_create); 730 } 731 732 static int create_or_delete_trace_uprobe(const char *raw_command) 733 { 734 int ret; 735 736 if (raw_command[0] == '-') 737 return dyn_event_release(raw_command, &trace_uprobe_ops); 738 739 ret = dyn_event_create(raw_command, &trace_uprobe_ops); 740 return ret == -ECANCELED ? -EINVAL : ret; 741 } 742 743 static int trace_uprobe_release(struct dyn_event *ev) 744 { 745 struct trace_uprobe *tu = to_trace_uprobe(ev); 746 747 return unregister_trace_uprobe(tu); 748 } 749 750 /* Probes listing interfaces */ 751 static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev) 752 { 753 struct trace_uprobe *tu = to_trace_uprobe(ev); 754 char c = is_ret_probe(tu) ? 'r' : 'p'; 755 int i; 756 757 seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp), 758 trace_probe_name(&tu->tp), tu->filename, 759 (int)(sizeof(void *) * 2), tu->offset); 760 761 if (tu->ref_ctr_offset) 762 seq_printf(m, "(0x%lx)", tu->ref_ctr_offset); 763 764 for (i = 0; i < tu->tp.nr_args; i++) 765 seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm); 766 767 seq_putc(m, '\n'); 768 769 trace_probe_dump_args(m, &tu->tp); 770 771 return 0; 772 } 773 774 static int probes_seq_show(struct seq_file *m, void *v) 775 { 776 struct dyn_event *ev = v; 777 778 if (!is_trace_uprobe(ev)) 779 return 0; 780 781 return trace_uprobe_show(m, ev); 782 } 783 784 static const struct seq_operations probes_seq_op = { 785 .start = dyn_event_seq_start, 786 .next = dyn_event_seq_next, 787 .stop = dyn_event_seq_stop, 788 .show = probes_seq_show 789 }; 790 791 static int probes_open(struct inode *inode, struct file *file) 792 { 793 int ret; 794 795 ret = security_locked_down(LOCKDOWN_TRACEFS); 796 if (ret) 797 return ret; 798 799 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 800 ret = dyn_events_release_all(&trace_uprobe_ops); 801 if (ret) 802 return ret; 803 } 804 805 return seq_open(file, &probes_seq_op); 806 } 807 808 static ssize_t probes_write(struct file *file, const char __user *buffer, 809 size_t count, loff_t *ppos) 810 { 811 return trace_parse_run_command(file, buffer, count, ppos, 812 create_or_delete_trace_uprobe); 813 } 814 815 static const struct file_operations uprobe_events_ops = { 816 .owner = THIS_MODULE, 817 .open = probes_open, 818 .read = seq_read, 819 .llseek = seq_lseek, 820 .release = seq_release, 821 .write = probes_write, 822 }; 823 824 /* Probes profiling interfaces */ 825 static int probes_profile_seq_show(struct seq_file *m, void *v) 826 { 827 struct dyn_event *ev = v; 828 struct trace_uprobe *tu; 829 unsigned long nhits; 830 int cpu; 831 832 if (!is_trace_uprobe(ev)) 833 return 0; 834 835 tu = to_trace_uprobe(ev); 836 837 nhits = 0; 838 for_each_possible_cpu(cpu) { 839 nhits += per_cpu(*tu->nhits, cpu); 840 } 841 842 seq_printf(m, " %s %-44s %15lu\n", tu->filename, 843 trace_probe_name(&tu->tp), nhits); 844 return 0; 845 } 846 847 static const struct seq_operations profile_seq_op = { 848 .start = dyn_event_seq_start, 849 .next = dyn_event_seq_next, 850 .stop = dyn_event_seq_stop, 851 .show = probes_profile_seq_show 852 }; 853 854 static int profile_open(struct inode *inode, struct file *file) 855 { 856 int ret; 857 858 ret = security_locked_down(LOCKDOWN_TRACEFS); 859 if (ret) 860 return ret; 861 862 return seq_open(file, &profile_seq_op); 863 } 864 865 static const struct file_operations uprobe_profile_ops = { 866 .owner = THIS_MODULE, 867 .open = profile_open, 868 .read = seq_read, 869 .llseek = seq_lseek, 870 .release = seq_release, 871 }; 872 873 struct uprobe_cpu_buffer { 874 struct mutex mutex; 875 void *buf; 876 int dsize; 877 }; 878 static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer; 879 static int uprobe_buffer_refcnt; 880 #define MAX_UCB_BUFFER_SIZE PAGE_SIZE 881 882 static int uprobe_buffer_init(void) 883 { 884 int cpu, err_cpu; 885 886 uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer); 887 if (uprobe_cpu_buffer == NULL) 888 return -ENOMEM; 889 890 for_each_possible_cpu(cpu) { 891 struct page *p = alloc_pages_node(cpu_to_node(cpu), 892 GFP_KERNEL, 0); 893 if (p == NULL) { 894 err_cpu = cpu; 895 goto err; 896 } 897 per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p); 898 mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex); 899 } 900 901 return 0; 902 903 err: 904 for_each_possible_cpu(cpu) { 905 if (cpu == err_cpu) 906 break; 907 free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf); 908 } 909 910 free_percpu(uprobe_cpu_buffer); 911 return -ENOMEM; 912 } 913 914 static int uprobe_buffer_enable(void) 915 { 916 int ret = 0; 917 918 lockdep_assert_held(&event_mutex); 919 920 if (uprobe_buffer_refcnt++ == 0) { 921 ret = uprobe_buffer_init(); 922 if (ret < 0) 923 uprobe_buffer_refcnt--; 924 } 925 926 return ret; 927 } 928 929 static void uprobe_buffer_disable(void) 930 { 931 int cpu; 932 933 lockdep_assert_held(&event_mutex); 934 935 if (--uprobe_buffer_refcnt == 0) { 936 for_each_possible_cpu(cpu) 937 free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, 938 cpu)->buf); 939 940 free_percpu(uprobe_cpu_buffer); 941 uprobe_cpu_buffer = NULL; 942 } 943 } 944 945 static struct uprobe_cpu_buffer *uprobe_buffer_get(void) 946 { 947 struct uprobe_cpu_buffer *ucb; 948 int cpu; 949 950 cpu = raw_smp_processor_id(); 951 ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu); 952 953 /* 954 * Use per-cpu buffers for fastest access, but we might migrate 955 * so the mutex makes sure we have sole access to it. 956 */ 957 mutex_lock(&ucb->mutex); 958 959 return ucb; 960 } 961 962 static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb) 963 { 964 if (!ucb) 965 return; 966 mutex_unlock(&ucb->mutex); 967 } 968 969 static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu, 970 struct pt_regs *regs, 971 struct uprobe_cpu_buffer **ucbp) 972 { 973 struct uprobe_cpu_buffer *ucb; 974 int dsize, esize; 975 976 if (*ucbp) 977 return *ucbp; 978 979 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 980 dsize = __get_data_size(&tu->tp, regs, NULL); 981 982 ucb = uprobe_buffer_get(); 983 ucb->dsize = tu->tp.size + dsize; 984 985 BUILD_BUG_ON(MAX_UCB_BUFFER_SIZE < MAX_PROBE_EVENT_SIZE); 986 if (WARN_ON_ONCE(ucb->dsize > MAX_UCB_BUFFER_SIZE)) { 987 ucb->dsize = MAX_UCB_BUFFER_SIZE; 988 dsize = MAX_UCB_BUFFER_SIZE - tu->tp.size; 989 } 990 991 store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize); 992 993 *ucbp = ucb; 994 return ucb; 995 } 996 997 static void __uprobe_trace_func(struct trace_uprobe *tu, 998 unsigned long func, struct pt_regs *regs, 999 struct uprobe_cpu_buffer *ucb, 1000 struct trace_event_file *trace_file) 1001 { 1002 struct uprobe_trace_entry_head *entry; 1003 struct trace_event_buffer fbuffer; 1004 void *data; 1005 int size, esize; 1006 struct trace_event_call *call = trace_probe_event_call(&tu->tp); 1007 1008 WARN_ON(call != trace_file->event_call); 1009 1010 if (trace_trigger_soft_disabled(trace_file)) 1011 return; 1012 1013 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 1014 size = esize + ucb->dsize; 1015 entry = trace_event_buffer_reserve(&fbuffer, trace_file, size); 1016 if (!entry) 1017 return; 1018 1019 if (is_ret_probe(tu)) { 1020 entry->vaddr[0] = func; 1021 entry->vaddr[1] = instruction_pointer(regs); 1022 data = DATAOF_TRACE_ENTRY(entry, true); 1023 } else { 1024 entry->vaddr[0] = instruction_pointer(regs); 1025 data = DATAOF_TRACE_ENTRY(entry, false); 1026 } 1027 1028 memcpy(data, ucb->buf, ucb->dsize); 1029 1030 trace_event_buffer_commit(&fbuffer); 1031 } 1032 1033 /* uprobe handler */ 1034 static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, 1035 struct uprobe_cpu_buffer **ucbp) 1036 { 1037 struct event_file_link *link; 1038 struct uprobe_cpu_buffer *ucb; 1039 1040 if (is_ret_probe(tu)) 1041 return 0; 1042 1043 ucb = prepare_uprobe_buffer(tu, regs, ucbp); 1044 1045 rcu_read_lock(); 1046 trace_probe_for_each_link_rcu(link, &tu->tp) 1047 __uprobe_trace_func(tu, 0, regs, ucb, link->file); 1048 rcu_read_unlock(); 1049 1050 return 0; 1051 } 1052 1053 static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func, 1054 struct pt_regs *regs, 1055 struct uprobe_cpu_buffer **ucbp) 1056 { 1057 struct event_file_link *link; 1058 struct uprobe_cpu_buffer *ucb; 1059 1060 ucb = prepare_uprobe_buffer(tu, regs, ucbp); 1061 1062 rcu_read_lock(); 1063 trace_probe_for_each_link_rcu(link, &tu->tp) 1064 __uprobe_trace_func(tu, func, regs, ucb, link->file); 1065 rcu_read_unlock(); 1066 } 1067 1068 /* Event entry printers */ 1069 static enum print_line_t 1070 print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event) 1071 { 1072 struct uprobe_trace_entry_head *entry; 1073 struct trace_seq *s = &iter->seq; 1074 struct trace_uprobe *tu; 1075 u8 *data; 1076 1077 entry = (struct uprobe_trace_entry_head *)iter->ent; 1078 tu = trace_uprobe_primary_from_call( 1079 container_of(event, struct trace_event_call, event)); 1080 if (unlikely(!tu)) 1081 goto out; 1082 1083 if (is_ret_probe(tu)) { 1084 trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)", 1085 trace_probe_name(&tu->tp), 1086 entry->vaddr[1], entry->vaddr[0]); 1087 data = DATAOF_TRACE_ENTRY(entry, true); 1088 } else { 1089 trace_seq_printf(s, "%s: (0x%lx)", 1090 trace_probe_name(&tu->tp), 1091 entry->vaddr[0]); 1092 data = DATAOF_TRACE_ENTRY(entry, false); 1093 } 1094 1095 if (trace_probe_print_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0) 1096 goto out; 1097 1098 trace_seq_putc(s, '\n'); 1099 1100 out: 1101 return trace_handle_return(s); 1102 } 1103 1104 typedef bool (*filter_func_t)(struct uprobe_consumer *self, struct mm_struct *mm); 1105 1106 static int trace_uprobe_enable(struct trace_uprobe *tu, filter_func_t filter) 1107 { 1108 struct inode *inode = d_real_inode(tu->path.dentry); 1109 struct uprobe *uprobe; 1110 1111 tu->consumer.filter = filter; 1112 uprobe = uprobe_register(inode, tu->offset, tu->ref_ctr_offset, &tu->consumer); 1113 if (IS_ERR(uprobe)) 1114 return PTR_ERR(uprobe); 1115 1116 tu->uprobe = uprobe; 1117 return 0; 1118 } 1119 1120 static void __probe_event_disable(struct trace_probe *tp) 1121 { 1122 struct trace_uprobe *tu; 1123 bool sync = false; 1124 1125 tu = container_of(tp, struct trace_uprobe, tp); 1126 WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter)); 1127 1128 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) { 1129 if (!tu->uprobe) 1130 continue; 1131 1132 uprobe_unregister_nosync(tu->uprobe, &tu->consumer); 1133 sync = true; 1134 tu->uprobe = NULL; 1135 } 1136 if (sync) 1137 uprobe_unregister_sync(); 1138 } 1139 1140 static int probe_event_enable(struct trace_event_call *call, 1141 struct trace_event_file *file, filter_func_t filter) 1142 { 1143 struct trace_probe *tp; 1144 struct trace_uprobe *tu; 1145 bool enabled; 1146 int ret; 1147 1148 tp = trace_probe_primary_from_call(call); 1149 if (WARN_ON_ONCE(!tp)) 1150 return -ENODEV; 1151 enabled = trace_probe_is_enabled(tp); 1152 1153 /* This may also change "enabled" state */ 1154 if (file) { 1155 if (trace_probe_test_flag(tp, TP_FLAG_PROFILE)) 1156 return -EINTR; 1157 1158 ret = trace_probe_add_file(tp, file); 1159 if (ret < 0) 1160 return ret; 1161 } else { 1162 if (trace_probe_test_flag(tp, TP_FLAG_TRACE)) 1163 return -EINTR; 1164 1165 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 1166 } 1167 1168 tu = container_of(tp, struct trace_uprobe, tp); 1169 WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter)); 1170 1171 if (enabled) 1172 return 0; 1173 1174 ret = uprobe_buffer_enable(); 1175 if (ret) 1176 goto err_flags; 1177 1178 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) { 1179 ret = trace_uprobe_enable(tu, filter); 1180 if (ret) { 1181 __probe_event_disable(tp); 1182 goto err_buffer; 1183 } 1184 } 1185 1186 return 0; 1187 1188 err_buffer: 1189 uprobe_buffer_disable(); 1190 1191 err_flags: 1192 if (file) 1193 trace_probe_remove_file(tp, file); 1194 else 1195 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 1196 1197 return ret; 1198 } 1199 1200 static void probe_event_disable(struct trace_event_call *call, 1201 struct trace_event_file *file) 1202 { 1203 struct trace_probe *tp; 1204 1205 tp = trace_probe_primary_from_call(call); 1206 if (WARN_ON_ONCE(!tp)) 1207 return; 1208 1209 if (!trace_probe_is_enabled(tp)) 1210 return; 1211 1212 if (file) { 1213 if (trace_probe_remove_file(tp, file) < 0) 1214 return; 1215 1216 if (trace_probe_is_enabled(tp)) 1217 return; 1218 } else 1219 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 1220 1221 __probe_event_disable(tp); 1222 uprobe_buffer_disable(); 1223 } 1224 1225 static int uprobe_event_define_fields(struct trace_event_call *event_call) 1226 { 1227 int ret, size; 1228 struct uprobe_trace_entry_head field; 1229 struct trace_uprobe *tu; 1230 1231 tu = trace_uprobe_primary_from_call(event_call); 1232 if (unlikely(!tu)) 1233 return -ENODEV; 1234 1235 if (is_ret_probe(tu)) { 1236 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0); 1237 DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0); 1238 size = SIZEOF_TRACE_ENTRY(true); 1239 } else { 1240 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0); 1241 size = SIZEOF_TRACE_ENTRY(false); 1242 } 1243 1244 return traceprobe_define_arg_fields(event_call, size, &tu->tp); 1245 } 1246 1247 #ifdef CONFIG_PERF_EVENTS 1248 static bool 1249 __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm) 1250 { 1251 struct perf_event *event; 1252 1253 list_for_each_entry(event, &filter->perf_events, hw.tp_list) { 1254 if (event->hw.target->mm == mm) 1255 return true; 1256 } 1257 1258 return false; 1259 } 1260 1261 static inline bool 1262 trace_uprobe_filter_event(struct trace_uprobe_filter *filter, 1263 struct perf_event *event) 1264 { 1265 return __uprobe_perf_filter(filter, event->hw.target->mm); 1266 } 1267 1268 static bool trace_uprobe_filter_remove(struct trace_uprobe_filter *filter, 1269 struct perf_event *event) 1270 { 1271 bool done; 1272 1273 write_lock(&filter->rwlock); 1274 if (event->hw.target) { 1275 list_del(&event->hw.tp_list); 1276 done = filter->nr_systemwide || 1277 (event->hw.target->flags & PF_EXITING) || 1278 trace_uprobe_filter_event(filter, event); 1279 } else { 1280 filter->nr_systemwide--; 1281 done = filter->nr_systemwide; 1282 } 1283 write_unlock(&filter->rwlock); 1284 1285 return done; 1286 } 1287 1288 /* This returns true if the filter always covers target mm */ 1289 static bool trace_uprobe_filter_add(struct trace_uprobe_filter *filter, 1290 struct perf_event *event) 1291 { 1292 bool done; 1293 1294 write_lock(&filter->rwlock); 1295 if (event->hw.target) { 1296 /* 1297 * event->parent != NULL means copy_process(), we can avoid 1298 * uprobe_apply(). current->mm must be probed and we can rely 1299 * on dup_mmap() which preserves the already installed bp's. 1300 * 1301 * attr.enable_on_exec means that exec/mmap will install the 1302 * breakpoints we need. 1303 */ 1304 done = filter->nr_systemwide || 1305 event->parent || event->attr.enable_on_exec || 1306 trace_uprobe_filter_event(filter, event); 1307 list_add(&event->hw.tp_list, &filter->perf_events); 1308 } else { 1309 done = filter->nr_systemwide; 1310 filter->nr_systemwide++; 1311 } 1312 write_unlock(&filter->rwlock); 1313 1314 return done; 1315 } 1316 1317 static int uprobe_perf_close(struct trace_event_call *call, 1318 struct perf_event *event) 1319 { 1320 struct trace_probe *tp; 1321 struct trace_uprobe *tu; 1322 int ret = 0; 1323 1324 tp = trace_probe_primary_from_call(call); 1325 if (WARN_ON_ONCE(!tp)) 1326 return -ENODEV; 1327 1328 tu = container_of(tp, struct trace_uprobe, tp); 1329 if (trace_uprobe_filter_remove(tu->tp.event->filter, event)) 1330 return 0; 1331 1332 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) { 1333 ret = uprobe_apply(tu->uprobe, &tu->consumer, false); 1334 if (ret) 1335 break; 1336 } 1337 1338 return ret; 1339 } 1340 1341 static int uprobe_perf_open(struct trace_event_call *call, 1342 struct perf_event *event) 1343 { 1344 struct trace_probe *tp; 1345 struct trace_uprobe *tu; 1346 int err = 0; 1347 1348 tp = trace_probe_primary_from_call(call); 1349 if (WARN_ON_ONCE(!tp)) 1350 return -ENODEV; 1351 1352 tu = container_of(tp, struct trace_uprobe, tp); 1353 if (trace_uprobe_filter_add(tu->tp.event->filter, event)) 1354 return 0; 1355 1356 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) { 1357 err = uprobe_apply(tu->uprobe, &tu->consumer, true); 1358 if (err) { 1359 uprobe_perf_close(call, event); 1360 break; 1361 } 1362 } 1363 1364 return err; 1365 } 1366 1367 static bool uprobe_perf_filter(struct uprobe_consumer *uc, struct mm_struct *mm) 1368 { 1369 struct trace_uprobe_filter *filter; 1370 struct trace_uprobe *tu; 1371 int ret; 1372 1373 tu = container_of(uc, struct trace_uprobe, consumer); 1374 filter = tu->tp.event->filter; 1375 1376 /* 1377 * speculative short-circuiting check to avoid unnecessarily taking 1378 * filter->rwlock below, if the uprobe has system-wide consumer 1379 */ 1380 if (READ_ONCE(filter->nr_systemwide)) 1381 return true; 1382 1383 read_lock(&filter->rwlock); 1384 ret = __uprobe_perf_filter(filter, mm); 1385 read_unlock(&filter->rwlock); 1386 1387 return ret; 1388 } 1389 1390 static void __uprobe_perf_func(struct trace_uprobe *tu, 1391 unsigned long func, struct pt_regs *regs, 1392 struct uprobe_cpu_buffer **ucbp) 1393 { 1394 struct trace_event_call *call = trace_probe_event_call(&tu->tp); 1395 struct uprobe_trace_entry_head *entry; 1396 struct uprobe_cpu_buffer *ucb; 1397 struct hlist_head *head; 1398 void *data; 1399 int size, esize; 1400 int rctx; 1401 1402 #ifdef CONFIG_BPF_EVENTS 1403 if (bpf_prog_array_valid(call)) { 1404 const struct bpf_prog_array *array; 1405 u32 ret; 1406 1407 rcu_read_lock_trace(); 1408 array = rcu_dereference_check(call->prog_array, rcu_read_lock_trace_held()); 1409 ret = bpf_prog_run_array_uprobe(array, regs, bpf_prog_run); 1410 rcu_read_unlock_trace(); 1411 if (!ret) 1412 return; 1413 } 1414 #endif /* CONFIG_BPF_EVENTS */ 1415 1416 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 1417 1418 ucb = prepare_uprobe_buffer(tu, regs, ucbp); 1419 size = esize + ucb->dsize; 1420 size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32); 1421 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough")) 1422 return; 1423 1424 preempt_disable(); 1425 head = this_cpu_ptr(call->perf_events); 1426 if (hlist_empty(head)) 1427 goto out; 1428 1429 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1430 if (!entry) 1431 goto out; 1432 1433 if (is_ret_probe(tu)) { 1434 entry->vaddr[0] = func; 1435 entry->vaddr[1] = instruction_pointer(regs); 1436 data = DATAOF_TRACE_ENTRY(entry, true); 1437 } else { 1438 entry->vaddr[0] = instruction_pointer(regs); 1439 data = DATAOF_TRACE_ENTRY(entry, false); 1440 } 1441 1442 memcpy(data, ucb->buf, ucb->dsize); 1443 1444 if (size - esize > ucb->dsize) 1445 memset(data + ucb->dsize, 0, size - esize - ucb->dsize); 1446 1447 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1448 head, NULL); 1449 out: 1450 preempt_enable(); 1451 } 1452 1453 /* uprobe profile handler */ 1454 static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs, 1455 struct uprobe_cpu_buffer **ucbp) 1456 { 1457 if (!uprobe_perf_filter(&tu->consumer, current->mm)) 1458 return UPROBE_HANDLER_REMOVE; 1459 1460 if (!is_ret_probe(tu)) 1461 __uprobe_perf_func(tu, 0, regs, ucbp); 1462 return 0; 1463 } 1464 1465 static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func, 1466 struct pt_regs *regs, 1467 struct uprobe_cpu_buffer **ucbp) 1468 { 1469 __uprobe_perf_func(tu, func, regs, ucbp); 1470 } 1471 1472 int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, 1473 const char **filename, u64 *probe_offset, 1474 u64 *probe_addr, bool perf_type_tracepoint) 1475 { 1476 const char *pevent = trace_event_name(event->tp_event); 1477 const char *group = event->tp_event->class->system; 1478 struct trace_uprobe *tu; 1479 1480 if (perf_type_tracepoint) 1481 tu = find_probe_event(pevent, group); 1482 else 1483 tu = trace_uprobe_primary_from_call(event->tp_event); 1484 if (!tu) 1485 return -EINVAL; 1486 1487 *fd_type = is_ret_probe(tu) ? BPF_FD_TYPE_URETPROBE 1488 : BPF_FD_TYPE_UPROBE; 1489 *filename = tu->filename; 1490 *probe_offset = tu->offset; 1491 *probe_addr = tu->ref_ctr_offset; 1492 return 0; 1493 } 1494 #endif /* CONFIG_PERF_EVENTS */ 1495 1496 static int 1497 trace_uprobe_register(struct trace_event_call *event, enum trace_reg type, 1498 void *data) 1499 { 1500 struct trace_event_file *file = data; 1501 1502 switch (type) { 1503 case TRACE_REG_REGISTER: 1504 return probe_event_enable(event, file, NULL); 1505 1506 case TRACE_REG_UNREGISTER: 1507 probe_event_disable(event, file); 1508 return 0; 1509 1510 #ifdef CONFIG_PERF_EVENTS 1511 case TRACE_REG_PERF_REGISTER: 1512 return probe_event_enable(event, NULL, uprobe_perf_filter); 1513 1514 case TRACE_REG_PERF_UNREGISTER: 1515 probe_event_disable(event, NULL); 1516 return 0; 1517 1518 case TRACE_REG_PERF_OPEN: 1519 return uprobe_perf_open(event, data); 1520 1521 case TRACE_REG_PERF_CLOSE: 1522 return uprobe_perf_close(event, data); 1523 1524 #endif 1525 default: 1526 return 0; 1527 } 1528 } 1529 1530 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs, 1531 __u64 *data) 1532 { 1533 struct trace_uprobe *tu; 1534 struct uprobe_dispatch_data udd; 1535 struct uprobe_cpu_buffer *ucb = NULL; 1536 unsigned int flags; 1537 int ret = 0; 1538 1539 tu = container_of(con, struct trace_uprobe, consumer); 1540 1541 this_cpu_inc(*tu->nhits); 1542 1543 udd.tu = tu; 1544 udd.bp_addr = instruction_pointer(regs); 1545 1546 current->utask->vaddr = (unsigned long) &udd; 1547 1548 if (WARN_ON_ONCE(!uprobe_cpu_buffer)) 1549 return 0; 1550 1551 flags = trace_probe_load_flag(&tu->tp); 1552 if (flags & TP_FLAG_TRACE) 1553 ret |= uprobe_trace_func(tu, regs, &ucb); 1554 1555 #ifdef CONFIG_PERF_EVENTS 1556 if (flags & TP_FLAG_PROFILE) 1557 ret |= uprobe_perf_func(tu, regs, &ucb); 1558 #endif 1559 uprobe_buffer_put(ucb); 1560 return ret; 1561 } 1562 1563 static int uretprobe_dispatcher(struct uprobe_consumer *con, 1564 unsigned long func, struct pt_regs *regs, 1565 __u64 *data) 1566 { 1567 struct trace_uprobe *tu; 1568 struct uprobe_dispatch_data udd; 1569 struct uprobe_cpu_buffer *ucb = NULL; 1570 unsigned int flags; 1571 1572 tu = container_of(con, struct trace_uprobe, consumer); 1573 1574 udd.tu = tu; 1575 udd.bp_addr = func; 1576 1577 current->utask->vaddr = (unsigned long) &udd; 1578 1579 if (WARN_ON_ONCE(!uprobe_cpu_buffer)) 1580 return 0; 1581 1582 flags = trace_probe_load_flag(&tu->tp); 1583 if (flags & TP_FLAG_TRACE) 1584 uretprobe_trace_func(tu, func, regs, &ucb); 1585 1586 #ifdef CONFIG_PERF_EVENTS 1587 if (flags & TP_FLAG_PROFILE) 1588 uretprobe_perf_func(tu, func, regs, &ucb); 1589 #endif 1590 uprobe_buffer_put(ucb); 1591 return 0; 1592 } 1593 1594 static struct trace_event_functions uprobe_funcs = { 1595 .trace = print_uprobe_event 1596 }; 1597 1598 static struct trace_event_fields uprobe_fields_array[] = { 1599 { .type = TRACE_FUNCTION_TYPE, 1600 .define_fields = uprobe_event_define_fields }, 1601 {} 1602 }; 1603 1604 static inline void init_trace_event_call(struct trace_uprobe *tu) 1605 { 1606 struct trace_event_call *call = trace_probe_event_call(&tu->tp); 1607 call->event.funcs = &uprobe_funcs; 1608 call->class->fields_array = uprobe_fields_array; 1609 1610 call->flags = TRACE_EVENT_FL_UPROBE | TRACE_EVENT_FL_CAP_ANY; 1611 call->class->reg = trace_uprobe_register; 1612 } 1613 1614 static int register_uprobe_event(struct trace_uprobe *tu) 1615 { 1616 init_trace_event_call(tu); 1617 1618 return trace_probe_register_event_call(&tu->tp); 1619 } 1620 1621 static int unregister_uprobe_event(struct trace_uprobe *tu) 1622 { 1623 return trace_probe_unregister_event_call(&tu->tp); 1624 } 1625 1626 #ifdef CONFIG_PERF_EVENTS 1627 struct trace_event_call * 1628 create_local_trace_uprobe(char *name, unsigned long offs, 1629 unsigned long ref_ctr_offset, bool is_return) 1630 { 1631 enum probe_print_type ptype; 1632 struct trace_uprobe *tu; 1633 struct path path; 1634 int ret; 1635 1636 ret = kern_path(name, LOOKUP_FOLLOW, &path); 1637 if (ret) 1638 return ERR_PTR(ret); 1639 1640 if (!d_is_reg(path.dentry)) { 1641 path_put(&path); 1642 return ERR_PTR(-EINVAL); 1643 } 1644 1645 /* 1646 * local trace_kprobes are not added to dyn_event, so they are never 1647 * searched in find_trace_kprobe(). Therefore, there is no concern of 1648 * duplicated name "DUMMY_EVENT" here. 1649 */ 1650 tu = alloc_trace_uprobe(UPROBE_EVENT_SYSTEM, "DUMMY_EVENT", 0, 1651 is_return); 1652 1653 if (IS_ERR(tu)) { 1654 pr_info("Failed to allocate trace_uprobe.(%d)\n", 1655 (int)PTR_ERR(tu)); 1656 path_put(&path); 1657 return ERR_CAST(tu); 1658 } 1659 1660 tu->offset = offs; 1661 tu->path = path; 1662 tu->ref_ctr_offset = ref_ctr_offset; 1663 tu->filename = kstrdup(name, GFP_KERNEL); 1664 if (!tu->filename) { 1665 ret = -ENOMEM; 1666 goto error; 1667 } 1668 1669 init_trace_event_call(tu); 1670 1671 ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1672 if (traceprobe_set_print_fmt(&tu->tp, ptype) < 0) { 1673 ret = -ENOMEM; 1674 goto error; 1675 } 1676 1677 return trace_probe_event_call(&tu->tp); 1678 error: 1679 free_trace_uprobe(tu); 1680 return ERR_PTR(ret); 1681 } 1682 1683 void destroy_local_trace_uprobe(struct trace_event_call *event_call) 1684 { 1685 struct trace_uprobe *tu; 1686 1687 tu = trace_uprobe_primary_from_call(event_call); 1688 1689 free_trace_uprobe(tu); 1690 } 1691 #endif /* CONFIG_PERF_EVENTS */ 1692 1693 /* Make a trace interface for controlling probe points */ 1694 static __init int init_uprobe_trace(void) 1695 { 1696 int ret; 1697 1698 ret = dyn_event_register(&trace_uprobe_ops); 1699 if (ret) 1700 return ret; 1701 1702 ret = tracing_init_dentry(); 1703 if (ret) 1704 return 0; 1705 1706 trace_create_file("uprobe_events", TRACE_MODE_WRITE, NULL, 1707 NULL, &uprobe_events_ops); 1708 /* Profile interface */ 1709 trace_create_file("uprobe_profile", TRACE_MODE_READ, NULL, 1710 NULL, &uprobe_profile_ops); 1711 return 0; 1712 } 1713 1714 fs_initcall(init_uprobe_trace); 1715