1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * uprobes-based tracing events 4 * 5 * Copyright (C) IBM Corporation, 2010-2012 6 * Author: Srikar Dronamraju <srikar@linux.vnet.ibm.com> 7 */ 8 #define pr_fmt(fmt) "trace_uprobe: " fmt 9 10 #include <linux/bpf-cgroup.h> 11 #include <linux/cleanup.h> 12 #include <linux/ctype.h> 13 #include <linux/filter.h> 14 #include <linux/module.h> 15 #include <linux/namei.h> 16 #include <linux/percpu.h> 17 #include <linux/rculist.h> 18 #include <linux/security.h> 19 #include <linux/string.h> 20 #include <linux/uaccess.h> 21 #include <linux/uprobes.h> 22 23 #include "trace.h" 24 #include "trace_dynevent.h" 25 #include "trace_probe.h" 26 #include "trace_probe_tmpl.h" 27 28 #define UPROBE_EVENT_SYSTEM "uprobes" 29 30 struct uprobe_trace_entry_head { 31 struct trace_entry ent; 32 unsigned long vaddr[]; 33 }; 34 35 #define SIZEOF_TRACE_ENTRY(is_return) \ 36 (sizeof(struct uprobe_trace_entry_head) + \ 37 sizeof(unsigned long) * (is_return ? 2 : 1)) 38 39 #define DATAOF_TRACE_ENTRY(entry, is_return) \ 40 ((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return)) 41 42 static int trace_uprobe_create(const char *raw_command); 43 static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev); 44 static int trace_uprobe_release(struct dyn_event *ev); 45 static bool trace_uprobe_is_busy(struct dyn_event *ev); 46 static bool trace_uprobe_match(const char *system, const char *event, 47 int argc, const char **argv, struct dyn_event *ev); 48 49 static struct dyn_event_operations trace_uprobe_ops = { 50 .create = trace_uprobe_create, 51 .show = trace_uprobe_show, 52 .is_busy = trace_uprobe_is_busy, 53 .free = trace_uprobe_release, 54 .match = trace_uprobe_match, 55 }; 56 57 /* 58 * uprobe event core functions 59 */ 60 struct trace_uprobe { 61 struct dyn_event devent; 62 struct uprobe_consumer consumer; 63 struct path path; 64 char *filename; 65 struct uprobe *uprobe; 66 unsigned long offset; 67 unsigned long ref_ctr_offset; 68 unsigned long __percpu *nhits; 69 struct trace_probe tp; 70 }; 71 72 static bool is_trace_uprobe(struct dyn_event *ev) 73 { 74 return ev->ops == &trace_uprobe_ops; 75 } 76 77 static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev) 78 { 79 return container_of(ev, struct trace_uprobe, devent); 80 } 81 82 /** 83 * for_each_trace_uprobe - iterate over the trace_uprobe list 84 * @pos: the struct trace_uprobe * for each entry 85 * @dpos: the struct dyn_event * to use as a loop cursor 86 */ 87 #define for_each_trace_uprobe(pos, dpos) \ 88 for_each_dyn_event(dpos) \ 89 if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos))) 90 91 static int register_uprobe_event(struct trace_uprobe *tu); 92 static int unregister_uprobe_event(struct trace_uprobe *tu); 93 94 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs, 95 __u64 *data); 96 static int uretprobe_dispatcher(struct uprobe_consumer *con, 97 unsigned long func, struct pt_regs *regs, 98 __u64 *data); 99 100 #ifdef CONFIG_STACK_GROWSUP 101 static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n) 102 { 103 return addr - (n * sizeof(long)); 104 } 105 #else 106 static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n) 107 { 108 return addr + (n * sizeof(long)); 109 } 110 #endif 111 112 static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n) 113 { 114 unsigned long ret; 115 unsigned long addr = user_stack_pointer(regs); 116 117 addr = adjust_stack_addr(addr, n); 118 119 if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret))) 120 return 0; 121 122 return ret; 123 } 124 125 /* 126 * Uprobes-specific fetch functions 127 */ 128 static nokprobe_inline int 129 probe_mem_read(void *dest, void *src, size_t size) 130 { 131 void __user *vaddr = (void __force __user *)src; 132 133 return copy_from_user(dest, vaddr, size) ? -EFAULT : 0; 134 } 135 136 static nokprobe_inline int 137 probe_mem_read_user(void *dest, void *src, size_t size) 138 { 139 return probe_mem_read(dest, src, size); 140 } 141 142 /* 143 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max 144 * length and relative data location. 145 */ 146 static nokprobe_inline int 147 fetch_store_string(unsigned long addr, void *dest, void *base) 148 { 149 long ret; 150 u32 loc = *(u32 *)dest; 151 int maxlen = get_loc_len(loc); 152 u8 *dst = get_loc_data(dest, base); 153 void __user *src = (void __force __user *) addr; 154 155 if (unlikely(!maxlen)) 156 return -ENOMEM; 157 158 if (addr == FETCH_TOKEN_COMM) 159 ret = strscpy(dst, current->comm, maxlen); 160 else 161 ret = strncpy_from_user(dst, src, maxlen); 162 if (ret >= 0) { 163 if (ret == maxlen) 164 dst[ret - 1] = '\0'; 165 else 166 /* 167 * Include the terminating null byte. In this case it 168 * was copied by strncpy_from_user but not accounted 169 * for in ret. 170 */ 171 ret++; 172 *(u32 *)dest = make_data_loc(ret, (void *)dst - base); 173 } else 174 *(u32 *)dest = make_data_loc(0, (void *)dst - base); 175 176 return ret; 177 } 178 179 static nokprobe_inline int 180 fetch_store_string_user(unsigned long addr, void *dest, void *base) 181 { 182 return fetch_store_string(addr, dest, base); 183 } 184 185 /* Return the length of string -- including null terminal byte */ 186 static nokprobe_inline int 187 fetch_store_strlen(unsigned long addr) 188 { 189 int len; 190 void __user *vaddr = (void __force __user *) addr; 191 192 if (addr == FETCH_TOKEN_COMM) 193 len = strlen(current->comm) + 1; 194 else 195 len = strnlen_user(vaddr, MAX_STRING_SIZE); 196 197 return (len > MAX_STRING_SIZE) ? 0 : len; 198 } 199 200 static nokprobe_inline int 201 fetch_store_strlen_user(unsigned long addr) 202 { 203 return fetch_store_strlen(addr); 204 } 205 206 static unsigned long translate_user_vaddr(unsigned long file_offset) 207 { 208 unsigned long base_addr; 209 struct uprobe_dispatch_data *udd; 210 211 udd = (void *) current->utask->vaddr; 212 213 base_addr = udd->bp_addr - udd->tu->offset; 214 return base_addr + file_offset; 215 } 216 217 /* Note that we don't verify it, since the code does not come from user space */ 218 static int 219 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 220 void *dest, void *base) 221 { 222 struct pt_regs *regs = rec; 223 unsigned long val; 224 int ret; 225 226 /* 1st stage: get value from context */ 227 switch (code->op) { 228 case FETCH_OP_REG: 229 val = regs_get_register(regs, code->param); 230 break; 231 case FETCH_OP_STACK: 232 val = get_user_stack_nth(regs, code->param); 233 break; 234 case FETCH_OP_STACKP: 235 val = user_stack_pointer(regs); 236 break; 237 case FETCH_OP_RETVAL: 238 val = regs_return_value(regs); 239 break; 240 case FETCH_OP_COMM: 241 val = FETCH_TOKEN_COMM; 242 break; 243 case FETCH_OP_FOFFS: 244 val = translate_user_vaddr(code->immediate); 245 break; 246 default: 247 ret = process_common_fetch_insn(code, &val); 248 if (ret < 0) 249 return ret; 250 } 251 code++; 252 253 return process_fetch_insn_bottom(code, val, dest, base); 254 } 255 NOKPROBE_SYMBOL(process_fetch_insn) 256 257 static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter) 258 { 259 rwlock_init(&filter->rwlock); 260 filter->nr_systemwide = 0; 261 INIT_LIST_HEAD(&filter->perf_events); 262 } 263 264 static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter) 265 { 266 return !filter->nr_systemwide && list_empty(&filter->perf_events); 267 } 268 269 static inline bool is_ret_probe(struct trace_uprobe *tu) 270 { 271 return tu->consumer.ret_handler != NULL; 272 } 273 274 static bool trace_uprobe_is_busy(struct dyn_event *ev) 275 { 276 struct trace_uprobe *tu = to_trace_uprobe(ev); 277 278 return trace_probe_is_enabled(&tu->tp); 279 } 280 281 static bool trace_uprobe_match_command_head(struct trace_uprobe *tu, 282 int argc, const char **argv) 283 { 284 char buf[64]; 285 int len; 286 287 if (!argc) 288 return true; 289 290 len = strlen(tu->filename); 291 if (strncmp(tu->filename, argv[0], len) || argv[0][len] != ':') 292 return false; 293 294 if (tu->ref_ctr_offset == 0) 295 snprintf(buf, sizeof(buf), "0x%0*lx", 296 (int)(sizeof(void *) * 2), tu->offset); 297 else 298 snprintf(buf, sizeof(buf), "0x%0*lx(0x%lx)", 299 (int)(sizeof(void *) * 2), tu->offset, 300 tu->ref_ctr_offset); 301 if (strcmp(buf, &argv[0][len + 1])) 302 return false; 303 304 argc--; argv++; 305 306 return trace_probe_match_command_args(&tu->tp, argc, argv); 307 } 308 309 static bool trace_uprobe_match(const char *system, const char *event, 310 int argc, const char **argv, struct dyn_event *ev) 311 { 312 struct trace_uprobe *tu = to_trace_uprobe(ev); 313 314 return (event[0] == '\0' || 315 strcmp(trace_probe_name(&tu->tp), event) == 0) && 316 (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0) && 317 trace_uprobe_match_command_head(tu, argc, argv); 318 } 319 320 static nokprobe_inline struct trace_uprobe * 321 trace_uprobe_primary_from_call(struct trace_event_call *call) 322 { 323 struct trace_probe *tp; 324 325 tp = trace_probe_primary_from_call(call); 326 if (WARN_ON_ONCE(!tp)) 327 return NULL; 328 329 return container_of(tp, struct trace_uprobe, tp); 330 } 331 332 /* 333 * Allocate new trace_uprobe and initialize it (including uprobes). 334 */ 335 static struct trace_uprobe * 336 alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret) 337 { 338 struct trace_uprobe *tu; 339 int ret; 340 341 tu = kzalloc_flex(*tu, tp.args, nargs); 342 if (!tu) 343 return ERR_PTR(-ENOMEM); 344 345 tu->nhits = alloc_percpu(unsigned long); 346 if (!tu->nhits) { 347 ret = -ENOMEM; 348 goto error; 349 } 350 351 ret = trace_probe_init(&tu->tp, event, group, true, nargs); 352 if (ret < 0) 353 goto error; 354 355 dyn_event_init(&tu->devent, &trace_uprobe_ops); 356 tu->consumer.handler = uprobe_dispatcher; 357 if (is_ret) 358 tu->consumer.ret_handler = uretprobe_dispatcher; 359 init_trace_uprobe_filter(tu->tp.event->filter); 360 return tu; 361 362 error: 363 free_percpu(tu->nhits); 364 kfree(tu); 365 366 return ERR_PTR(ret); 367 } 368 369 static void free_trace_uprobe(struct trace_uprobe *tu) 370 { 371 if (IS_ERR_OR_NULL(tu)) 372 return; 373 374 path_put(&tu->path); 375 trace_probe_cleanup(&tu->tp); 376 kfree(tu->filename); 377 free_percpu(tu->nhits); 378 kfree(tu); 379 } 380 381 static struct trace_uprobe *find_probe_event(const char *event, const char *group) 382 { 383 struct dyn_event *pos; 384 struct trace_uprobe *tu; 385 386 for_each_trace_uprobe(tu, pos) 387 if (strcmp(trace_probe_name(&tu->tp), event) == 0 && 388 strcmp(trace_probe_group_name(&tu->tp), group) == 0) 389 return tu; 390 391 return NULL; 392 } 393 394 /* Unregister a trace_uprobe and probe_event */ 395 static int unregister_trace_uprobe(struct trace_uprobe *tu) 396 { 397 int ret; 398 399 /* Enabled event can not be unregistered */ 400 if (trace_probe_is_enabled(&tu->tp)) 401 return -EBUSY; 402 403 /* If there's a reference to the dynamic event */ 404 if (trace_event_dyn_busy(trace_probe_event_call(&tu->tp))) 405 return -EBUSY; 406 407 if (trace_probe_has_sibling(&tu->tp)) 408 goto unreg; 409 410 ret = unregister_uprobe_event(tu); 411 if (ret) 412 return ret; 413 414 unreg: 415 dyn_event_remove(&tu->devent); 416 trace_probe_unlink(&tu->tp); 417 free_trace_uprobe(tu); 418 return 0; 419 } 420 421 static bool trace_uprobe_has_same_uprobe(struct trace_uprobe *orig, 422 struct trace_uprobe *comp) 423 { 424 struct trace_probe_event *tpe = orig->tp.event; 425 struct inode *comp_inode = d_real_inode(comp->path.dentry); 426 int i; 427 428 list_for_each_entry(orig, &tpe->probes, tp.list) { 429 if (comp_inode != d_real_inode(orig->path.dentry) || 430 comp->offset != orig->offset) 431 continue; 432 433 /* 434 * trace_probe_compare_arg_type() ensured that nr_args and 435 * each argument name and type are same. Let's compare comm. 436 */ 437 for (i = 0; i < orig->tp.nr_args; i++) { 438 if (strcmp(orig->tp.args[i].comm, 439 comp->tp.args[i].comm)) 440 break; 441 } 442 443 if (i == orig->tp.nr_args) 444 return true; 445 } 446 447 return false; 448 } 449 450 typedef bool (*filter_func_t)(struct uprobe_consumer *self, struct mm_struct *mm); 451 452 #ifdef CONFIG_PERF_EVENTS 453 static bool uprobe_perf_filter(struct uprobe_consumer *uc, struct mm_struct *mm); 454 #else 455 #define uprobe_perf_filter NULL 456 #endif 457 458 static int trace_uprobe_enable(struct trace_uprobe *tu, filter_func_t filter) 459 { 460 struct inode *inode = d_real_inode(tu->path.dentry); 461 struct uprobe *uprobe; 462 463 tu->consumer.filter = filter; 464 uprobe = uprobe_register(inode, tu->offset, tu->ref_ctr_offset, &tu->consumer); 465 if (IS_ERR(uprobe)) 466 return PTR_ERR(uprobe); 467 468 tu->uprobe = uprobe; 469 return 0; 470 } 471 472 static int append_trace_uprobe(struct trace_uprobe *tu, struct trace_uprobe *to) 473 { 474 int ret; 475 476 ret = trace_probe_compare_arg_type(&tu->tp, &to->tp); 477 if (ret) { 478 /* Note that argument starts index = 2 */ 479 trace_probe_log_set_index(ret + 1); 480 trace_probe_log_err(0, DIFF_ARG_TYPE); 481 return -EEXIST; 482 } 483 if (trace_uprobe_has_same_uprobe(to, tu)) { 484 trace_probe_log_set_index(0); 485 trace_probe_log_err(0, SAME_PROBE); 486 return -EEXIST; 487 } 488 489 /* Append to existing event */ 490 ret = trace_probe_append(&tu->tp, &to->tp); 491 if (ret) 492 return ret; 493 494 if (trace_probe_is_enabled(&tu->tp)) { 495 ret = trace_uprobe_enable(tu, 496 trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE) ? 497 uprobe_perf_filter : NULL); 498 if (ret) { 499 trace_probe_unlink(&tu->tp); 500 return ret; 501 } 502 } 503 504 dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp)); 505 506 return 0; 507 } 508 509 /* 510 * Uprobe with multiple reference counter is not allowed. i.e. 511 * If inode and offset matches, reference counter offset *must* 512 * match as well. Though, there is one exception: If user is 513 * replacing old trace_uprobe with new one(same group/event), 514 * then we allow same uprobe with new reference counter as far 515 * as the new one does not conflict with any other existing 516 * ones. 517 */ 518 static int validate_ref_ctr_offset(struct trace_uprobe *new) 519 { 520 struct dyn_event *pos; 521 struct trace_uprobe *tmp; 522 struct inode *new_inode = d_real_inode(new->path.dentry); 523 524 for_each_trace_uprobe(tmp, pos) { 525 if (new_inode == d_real_inode(tmp->path.dentry) && 526 new->offset == tmp->offset && 527 new->ref_ctr_offset != tmp->ref_ctr_offset) { 528 pr_warn("Reference counter offset mismatch."); 529 return -EINVAL; 530 } 531 } 532 return 0; 533 } 534 535 /* Register a trace_uprobe and probe_event */ 536 static int register_trace_uprobe(struct trace_uprobe *tu) 537 { 538 struct trace_uprobe *old_tu; 539 int ret; 540 541 guard(mutex)(&event_mutex); 542 543 ret = validate_ref_ctr_offset(tu); 544 if (ret) 545 return ret; 546 547 /* register as an event */ 548 old_tu = find_probe_event(trace_probe_name(&tu->tp), 549 trace_probe_group_name(&tu->tp)); 550 if (old_tu) { 551 if (is_ret_probe(tu) != is_ret_probe(old_tu)) { 552 trace_probe_log_set_index(0); 553 trace_probe_log_err(0, DIFF_PROBE_TYPE); 554 return -EEXIST; 555 } 556 return append_trace_uprobe(tu, old_tu); 557 } 558 559 ret = register_uprobe_event(tu); 560 if (ret) { 561 if (ret == -EEXIST) { 562 trace_probe_log_set_index(0); 563 trace_probe_log_err(0, EVENT_EXIST); 564 } else 565 pr_warn("Failed to register probe event(%d)\n", ret); 566 return ret; 567 } 568 569 dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp)); 570 571 return ret; 572 } 573 574 DEFINE_FREE(free_trace_uprobe, struct trace_uprobe *, free_trace_uprobe(_T)) 575 576 /* 577 * Argument syntax: 578 * - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS] 579 */ 580 static int __trace_uprobe_create(int argc, const char **argv) 581 { 582 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 583 struct trace_uprobe *tu __free(free_trace_uprobe) = NULL; 584 const char *trlog __free(trace_probe_log_clear) = NULL; 585 const char *event = NULL, *group = UPROBE_EVENT_SYSTEM; 586 struct path path __free(path_put) = {}; 587 unsigned long offset, ref_ctr_offset; 588 char *filename __free(kfree) = NULL; 589 char *arg, *rctr, *rctr_end, *tmp; 590 char *gbuf __free(kfree) = NULL; 591 char *buf __free(kfree) = NULL; 592 enum probe_print_type ptype; 593 bool is_return = false; 594 int i, ret; 595 596 ref_ctr_offset = 0; 597 598 switch (argv[0][0]) { 599 case 'r': 600 is_return = true; 601 break; 602 case 'p': 603 break; 604 default: 605 return -ECANCELED; 606 } 607 608 if (argc < 2) 609 return -ECANCELED; 610 611 trlog = trace_probe_log_init("trace_uprobe", argc, argv); 612 613 if (argc - 2 > MAX_TRACE_ARGS) { 614 trace_probe_log_set_index(2); 615 trace_probe_log_err(0, TOO_MANY_ARGS); 616 return -E2BIG; 617 } 618 619 if (argv[0][1] == ':') 620 event = &argv[0][2]; 621 622 if (!strchr(argv[1], '/')) 623 return -ECANCELED; 624 625 filename = kstrdup(argv[1], GFP_KERNEL); 626 if (!filename) 627 return -ENOMEM; 628 629 /* Find the last occurrence, in case the path contains ':' too. */ 630 arg = strrchr(filename, ':'); 631 if (!arg || !isdigit(arg[1])) 632 return -ECANCELED; 633 634 trace_probe_log_set_index(1); /* filename is the 2nd argument */ 635 636 *arg++ = '\0'; 637 ret = kern_path(filename, LOOKUP_FOLLOW, &path); 638 if (ret) { 639 trace_probe_log_err(0, FILE_NOT_FOUND); 640 return ret; 641 } 642 if (!d_is_reg(path.dentry)) { 643 trace_probe_log_err(0, NO_REGULAR_FILE); 644 return -EINVAL; 645 } 646 647 /* Parse reference counter offset if specified. */ 648 rctr = strchr(arg, '('); 649 if (rctr) { 650 rctr_end = strchr(rctr, ')'); 651 if (!rctr_end) { 652 rctr_end = rctr + strlen(rctr); 653 trace_probe_log_err(rctr_end - filename, 654 REFCNT_OPEN_BRACE); 655 return -EINVAL; 656 } else if (rctr_end[1] != '\0') { 657 trace_probe_log_err(rctr_end + 1 - filename, 658 BAD_REFCNT_SUFFIX); 659 return -EINVAL; 660 } 661 662 *rctr++ = '\0'; 663 *rctr_end = '\0'; 664 ret = kstrtoul(rctr, 0, &ref_ctr_offset); 665 if (ret) { 666 trace_probe_log_err(rctr - filename, BAD_REFCNT); 667 return ret; 668 } 669 } 670 671 /* Check if there is %return suffix */ 672 tmp = strchr(arg, '%'); 673 if (tmp) { 674 if (!strcmp(tmp, "%return")) { 675 *tmp = '\0'; 676 is_return = true; 677 } else { 678 trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX); 679 return -EINVAL; 680 } 681 } 682 683 /* Parse uprobe offset. */ 684 ret = kstrtoul(arg, 0, &offset); 685 if (ret) { 686 trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS); 687 return ret; 688 } 689 690 /* setup a probe */ 691 trace_probe_log_set_index(0); 692 if (event) { 693 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 694 if (!gbuf) 695 return -ENOMEM; 696 697 ret = traceprobe_parse_event_name(&event, &group, gbuf, 698 event - argv[0]); 699 if (ret) 700 return ret; 701 } 702 703 if (!event) { 704 char *tail __free(kfree) = NULL; 705 char *ptr; 706 707 tail = kstrdup(kbasename(filename), GFP_KERNEL); 708 if (!tail) 709 return -ENOMEM; 710 711 ptr = strpbrk(tail, ".-_"); 712 if (ptr) 713 *ptr = '\0'; 714 715 buf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 716 if (!buf) 717 return -ENOMEM; 718 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset); 719 event = buf; 720 } 721 722 argc -= 2; 723 argv += 2; 724 725 tu = alloc_trace_uprobe(group, event, argc, is_return); 726 if (IS_ERR(tu)) { 727 ret = PTR_ERR(tu); 728 /* This must return -ENOMEM otherwise there is a bug */ 729 WARN_ON_ONCE(ret != -ENOMEM); 730 return ret; 731 } 732 tu->offset = offset; 733 tu->ref_ctr_offset = ref_ctr_offset; 734 tu->path = path; 735 /* Clear @path so that it will not freed by path_put() */ 736 memset(&path, 0, sizeof(path)); 737 tu->filename = no_free_ptr(filename); 738 739 ctx = kzalloc_obj(*ctx); 740 if (!ctx) 741 return -ENOMEM; 742 ctx->flags = (is_return ? TPARG_FL_RETURN : 0) | TPARG_FL_USER; 743 744 /* parse arguments */ 745 for (i = 0; i < argc; i++) { 746 trace_probe_log_set_index(i + 2); 747 ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx); 748 if (ret) 749 return ret; 750 } 751 752 ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 753 ret = traceprobe_set_print_fmt(&tu->tp, ptype); 754 if (ret < 0) 755 return ret; 756 757 ret = register_trace_uprobe(tu); 758 if (!ret) 759 tu = NULL; 760 761 return ret; 762 } 763 764 int trace_uprobe_create(const char *raw_command) 765 { 766 return trace_probe_create(raw_command, __trace_uprobe_create); 767 } 768 769 static int create_or_delete_trace_uprobe(const char *raw_command) 770 { 771 int ret; 772 773 if (raw_command[0] == '-') 774 return dyn_event_release(raw_command, &trace_uprobe_ops); 775 776 ret = dyn_event_create(raw_command, &trace_uprobe_ops); 777 return ret == -ECANCELED ? -EINVAL : ret; 778 } 779 780 static int trace_uprobe_release(struct dyn_event *ev) 781 { 782 struct trace_uprobe *tu = to_trace_uprobe(ev); 783 784 return unregister_trace_uprobe(tu); 785 } 786 787 /* Probes listing interfaces */ 788 static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev) 789 { 790 struct trace_uprobe *tu = to_trace_uprobe(ev); 791 char c = is_ret_probe(tu) ? 'r' : 'p'; 792 int i; 793 794 seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp), 795 trace_probe_name(&tu->tp), tu->filename, 796 (int)(sizeof(void *) * 2), tu->offset); 797 798 if (tu->ref_ctr_offset) 799 seq_printf(m, "(0x%lx)", tu->ref_ctr_offset); 800 801 for (i = 0; i < tu->tp.nr_args; i++) 802 seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm); 803 804 seq_putc(m, '\n'); 805 806 trace_probe_dump_args(m, &tu->tp); 807 808 return 0; 809 } 810 811 static int probes_seq_show(struct seq_file *m, void *v) 812 { 813 struct dyn_event *ev = v; 814 815 if (!is_trace_uprobe(ev)) 816 return 0; 817 818 return trace_uprobe_show(m, ev); 819 } 820 821 static const struct seq_operations probes_seq_op = { 822 .start = dyn_event_seq_start, 823 .next = dyn_event_seq_next, 824 .stop = dyn_event_seq_stop, 825 .show = probes_seq_show 826 }; 827 828 static int probes_open(struct inode *inode, struct file *file) 829 { 830 int ret; 831 832 ret = security_locked_down(LOCKDOWN_TRACEFS); 833 if (ret) 834 return ret; 835 836 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 837 ret = dyn_events_release_all(&trace_uprobe_ops); 838 if (ret) 839 return ret; 840 } 841 842 return seq_open(file, &probes_seq_op); 843 } 844 845 static ssize_t probes_write(struct file *file, const char __user *buffer, 846 size_t count, loff_t *ppos) 847 { 848 return trace_parse_run_command(file, buffer, count, ppos, 849 create_or_delete_trace_uprobe); 850 } 851 852 static const struct file_operations uprobe_events_ops = { 853 .owner = THIS_MODULE, 854 .open = probes_open, 855 .read = seq_read, 856 .llseek = seq_lseek, 857 .release = seq_release, 858 .write = probes_write, 859 }; 860 861 /* Probes profiling interfaces */ 862 static int probes_profile_seq_show(struct seq_file *m, void *v) 863 { 864 struct dyn_event *ev = v; 865 struct trace_uprobe *tu; 866 unsigned long nhits; 867 int cpu; 868 869 if (!is_trace_uprobe(ev)) 870 return 0; 871 872 tu = to_trace_uprobe(ev); 873 874 nhits = 0; 875 for_each_possible_cpu(cpu) { 876 nhits += per_cpu(*tu->nhits, cpu); 877 } 878 879 seq_printf(m, " %s %-44s %15lu\n", tu->filename, 880 trace_probe_name(&tu->tp), nhits); 881 return 0; 882 } 883 884 static const struct seq_operations profile_seq_op = { 885 .start = dyn_event_seq_start, 886 .next = dyn_event_seq_next, 887 .stop = dyn_event_seq_stop, 888 .show = probes_profile_seq_show 889 }; 890 891 static int profile_open(struct inode *inode, struct file *file) 892 { 893 int ret; 894 895 ret = security_locked_down(LOCKDOWN_TRACEFS); 896 if (ret) 897 return ret; 898 899 return seq_open(file, &profile_seq_op); 900 } 901 902 static const struct file_operations uprobe_profile_ops = { 903 .owner = THIS_MODULE, 904 .open = profile_open, 905 .read = seq_read, 906 .llseek = seq_lseek, 907 .release = seq_release, 908 }; 909 910 struct uprobe_cpu_buffer { 911 struct mutex mutex; 912 void *buf; 913 int dsize; 914 }; 915 static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer; 916 static int uprobe_buffer_refcnt; 917 #define MAX_UCB_BUFFER_SIZE PAGE_SIZE 918 919 static int uprobe_buffer_init(void) 920 { 921 int cpu, err_cpu; 922 923 uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer); 924 if (uprobe_cpu_buffer == NULL) 925 return -ENOMEM; 926 927 for_each_possible_cpu(cpu) { 928 struct page *p = alloc_pages_node(cpu_to_node(cpu), 929 GFP_KERNEL, 0); 930 if (p == NULL) { 931 err_cpu = cpu; 932 goto err; 933 } 934 per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p); 935 mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex); 936 } 937 938 return 0; 939 940 err: 941 for_each_possible_cpu(cpu) { 942 if (cpu == err_cpu) 943 break; 944 free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf); 945 } 946 947 free_percpu(uprobe_cpu_buffer); 948 return -ENOMEM; 949 } 950 951 static int uprobe_buffer_enable(void) 952 { 953 int ret = 0; 954 955 lockdep_assert_held(&event_mutex); 956 957 if (uprobe_buffer_refcnt++ == 0) { 958 ret = uprobe_buffer_init(); 959 if (ret < 0) 960 uprobe_buffer_refcnt--; 961 } 962 963 return ret; 964 } 965 966 static void uprobe_buffer_disable(void) 967 { 968 int cpu; 969 970 lockdep_assert_held(&event_mutex); 971 972 if (--uprobe_buffer_refcnt == 0) { 973 for_each_possible_cpu(cpu) 974 free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, 975 cpu)->buf); 976 977 free_percpu(uprobe_cpu_buffer); 978 uprobe_cpu_buffer = NULL; 979 } 980 } 981 982 static struct uprobe_cpu_buffer *uprobe_buffer_get(void) 983 { 984 struct uprobe_cpu_buffer *ucb; 985 int cpu; 986 987 cpu = raw_smp_processor_id(); 988 ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu); 989 990 /* 991 * Use per-cpu buffers for fastest access, but we might migrate 992 * so the mutex makes sure we have sole access to it. 993 */ 994 mutex_lock(&ucb->mutex); 995 996 return ucb; 997 } 998 999 static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb) 1000 { 1001 if (!ucb) 1002 return; 1003 mutex_unlock(&ucb->mutex); 1004 } 1005 1006 static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu, 1007 struct pt_regs *regs, 1008 struct uprobe_cpu_buffer **ucbp) 1009 { 1010 struct uprobe_cpu_buffer *ucb; 1011 int dsize, esize; 1012 1013 if (*ucbp) 1014 return *ucbp; 1015 1016 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 1017 dsize = __get_data_size(&tu->tp, regs, NULL); 1018 1019 ucb = uprobe_buffer_get(); 1020 ucb->dsize = tu->tp.size + dsize; 1021 1022 BUILD_BUG_ON(MAX_UCB_BUFFER_SIZE < MAX_PROBE_EVENT_SIZE); 1023 if (WARN_ON_ONCE(ucb->dsize > MAX_UCB_BUFFER_SIZE)) { 1024 ucb->dsize = MAX_UCB_BUFFER_SIZE; 1025 dsize = MAX_UCB_BUFFER_SIZE - tu->tp.size; 1026 } 1027 1028 store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize); 1029 1030 *ucbp = ucb; 1031 return ucb; 1032 } 1033 1034 static void __uprobe_trace_func(struct trace_uprobe *tu, 1035 unsigned long func, struct pt_regs *regs, 1036 struct uprobe_cpu_buffer *ucb, 1037 struct trace_event_file *trace_file) 1038 { 1039 struct uprobe_trace_entry_head *entry; 1040 struct trace_event_buffer fbuffer; 1041 void *data; 1042 int size, esize; 1043 struct trace_event_call *call = trace_probe_event_call(&tu->tp); 1044 1045 WARN_ON(call != trace_file->event_call); 1046 1047 if (trace_trigger_soft_disabled(trace_file)) 1048 return; 1049 1050 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 1051 size = esize + ucb->dsize; 1052 entry = trace_event_buffer_reserve(&fbuffer, trace_file, size); 1053 if (!entry) 1054 return; 1055 1056 if (is_ret_probe(tu)) { 1057 entry->vaddr[0] = func; 1058 entry->vaddr[1] = instruction_pointer(regs); 1059 data = DATAOF_TRACE_ENTRY(entry, true); 1060 } else { 1061 entry->vaddr[0] = instruction_pointer(regs); 1062 data = DATAOF_TRACE_ENTRY(entry, false); 1063 } 1064 1065 memcpy(data, ucb->buf, ucb->dsize); 1066 1067 trace_event_buffer_commit(&fbuffer); 1068 } 1069 1070 /* uprobe handler */ 1071 static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, 1072 struct uprobe_cpu_buffer **ucbp) 1073 { 1074 struct event_file_link *link; 1075 struct uprobe_cpu_buffer *ucb; 1076 1077 if (is_ret_probe(tu)) 1078 return 0; 1079 1080 ucb = prepare_uprobe_buffer(tu, regs, ucbp); 1081 1082 rcu_read_lock(); 1083 trace_probe_for_each_link_rcu(link, &tu->tp) 1084 __uprobe_trace_func(tu, 0, regs, ucb, link->file); 1085 rcu_read_unlock(); 1086 1087 return 0; 1088 } 1089 1090 static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func, 1091 struct pt_regs *regs, 1092 struct uprobe_cpu_buffer **ucbp) 1093 { 1094 struct event_file_link *link; 1095 struct uprobe_cpu_buffer *ucb; 1096 1097 ucb = prepare_uprobe_buffer(tu, regs, ucbp); 1098 1099 rcu_read_lock(); 1100 trace_probe_for_each_link_rcu(link, &tu->tp) 1101 __uprobe_trace_func(tu, func, regs, ucb, link->file); 1102 rcu_read_unlock(); 1103 } 1104 1105 /* Event entry printers */ 1106 static enum print_line_t 1107 print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event) 1108 { 1109 struct uprobe_trace_entry_head *entry; 1110 struct trace_seq *s = &iter->seq; 1111 struct trace_uprobe *tu; 1112 u8 *data; 1113 1114 entry = (struct uprobe_trace_entry_head *)iter->ent; 1115 tu = trace_uprobe_primary_from_call( 1116 container_of(event, struct trace_event_call, event)); 1117 if (unlikely(!tu)) 1118 goto out; 1119 1120 if (is_ret_probe(tu)) { 1121 trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)", 1122 trace_probe_name(&tu->tp), 1123 entry->vaddr[1], entry->vaddr[0]); 1124 data = DATAOF_TRACE_ENTRY(entry, true); 1125 } else { 1126 trace_seq_printf(s, "%s: (0x%lx)", 1127 trace_probe_name(&tu->tp), 1128 entry->vaddr[0]); 1129 data = DATAOF_TRACE_ENTRY(entry, false); 1130 } 1131 1132 if (trace_probe_print_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0) 1133 goto out; 1134 1135 trace_seq_putc(s, '\n'); 1136 1137 out: 1138 return trace_handle_return(s); 1139 } 1140 1141 static void __probe_event_disable(struct trace_probe *tp) 1142 { 1143 struct trace_uprobe *tu; 1144 bool sync = false; 1145 1146 tu = container_of(tp, struct trace_uprobe, tp); 1147 WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter)); 1148 1149 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) { 1150 if (!tu->uprobe) 1151 continue; 1152 1153 uprobe_unregister_nosync(tu->uprobe, &tu->consumer); 1154 sync = true; 1155 tu->uprobe = NULL; 1156 } 1157 if (sync) 1158 uprobe_unregister_sync(); 1159 } 1160 1161 static int probe_event_enable(struct trace_event_call *call, 1162 struct trace_event_file *file, filter_func_t filter) 1163 { 1164 struct trace_probe *tp; 1165 struct trace_uprobe *tu; 1166 bool enabled; 1167 int ret; 1168 1169 tp = trace_probe_primary_from_call(call); 1170 if (WARN_ON_ONCE(!tp)) 1171 return -ENODEV; 1172 enabled = trace_probe_is_enabled(tp); 1173 1174 /* This may also change "enabled" state */ 1175 if (file) { 1176 if (trace_probe_test_flag(tp, TP_FLAG_PROFILE)) 1177 return -EINTR; 1178 1179 ret = trace_probe_add_file(tp, file); 1180 if (ret < 0) 1181 return ret; 1182 } else { 1183 if (trace_probe_test_flag(tp, TP_FLAG_TRACE)) 1184 return -EINTR; 1185 1186 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 1187 } 1188 1189 tu = container_of(tp, struct trace_uprobe, tp); 1190 WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter)); 1191 1192 if (enabled) 1193 return 0; 1194 1195 ret = uprobe_buffer_enable(); 1196 if (ret) 1197 goto err_flags; 1198 1199 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) { 1200 ret = trace_uprobe_enable(tu, filter); 1201 if (ret) { 1202 __probe_event_disable(tp); 1203 goto err_buffer; 1204 } 1205 } 1206 1207 return 0; 1208 1209 err_buffer: 1210 uprobe_buffer_disable(); 1211 1212 err_flags: 1213 if (file) 1214 trace_probe_remove_file(tp, file); 1215 else 1216 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 1217 1218 return ret; 1219 } 1220 1221 static void probe_event_disable(struct trace_event_call *call, 1222 struct trace_event_file *file) 1223 { 1224 struct trace_probe *tp; 1225 1226 tp = trace_probe_primary_from_call(call); 1227 if (WARN_ON_ONCE(!tp)) 1228 return; 1229 1230 if (!trace_probe_is_enabled(tp)) 1231 return; 1232 1233 if (file) { 1234 if (trace_probe_remove_file(tp, file) < 0) 1235 return; 1236 1237 if (trace_probe_is_enabled(tp)) 1238 return; 1239 } else 1240 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 1241 1242 __probe_event_disable(tp); 1243 uprobe_buffer_disable(); 1244 } 1245 1246 static int uprobe_event_define_fields(struct trace_event_call *event_call) 1247 { 1248 int ret, size; 1249 struct uprobe_trace_entry_head field; 1250 struct trace_uprobe *tu; 1251 1252 tu = trace_uprobe_primary_from_call(event_call); 1253 if (unlikely(!tu)) 1254 return -ENODEV; 1255 1256 if (is_ret_probe(tu)) { 1257 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0); 1258 DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0); 1259 size = SIZEOF_TRACE_ENTRY(true); 1260 } else { 1261 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0); 1262 size = SIZEOF_TRACE_ENTRY(false); 1263 } 1264 1265 return traceprobe_define_arg_fields(event_call, size, &tu->tp); 1266 } 1267 1268 #ifdef CONFIG_PERF_EVENTS 1269 static bool 1270 __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm) 1271 { 1272 struct perf_event *event; 1273 1274 list_for_each_entry(event, &filter->perf_events, hw.tp_list) { 1275 if (event->hw.target->mm == mm) 1276 return true; 1277 } 1278 1279 return false; 1280 } 1281 1282 static inline bool 1283 trace_uprobe_filter_event(struct trace_uprobe_filter *filter, 1284 struct perf_event *event) 1285 { 1286 return __uprobe_perf_filter(filter, event->hw.target->mm); 1287 } 1288 1289 static bool trace_uprobe_filter_remove(struct trace_uprobe_filter *filter, 1290 struct perf_event *event) 1291 { 1292 bool done; 1293 1294 write_lock(&filter->rwlock); 1295 if (event->hw.target) { 1296 list_del(&event->hw.tp_list); 1297 done = filter->nr_systemwide || 1298 (event->hw.target->flags & PF_EXITING) || 1299 trace_uprobe_filter_event(filter, event); 1300 } else { 1301 filter->nr_systemwide--; 1302 done = filter->nr_systemwide; 1303 } 1304 write_unlock(&filter->rwlock); 1305 1306 return done; 1307 } 1308 1309 /* This returns true if the filter always covers target mm */ 1310 static bool trace_uprobe_filter_add(struct trace_uprobe_filter *filter, 1311 struct perf_event *event) 1312 { 1313 bool done; 1314 1315 write_lock(&filter->rwlock); 1316 if (event->hw.target) { 1317 /* 1318 * event->parent != NULL means copy_process(), we can avoid 1319 * uprobe_apply(). current->mm must be probed and we can rely 1320 * on dup_mmap() which preserves the already installed bp's. 1321 * 1322 * attr.enable_on_exec means that exec/mmap will install the 1323 * breakpoints we need. 1324 */ 1325 done = filter->nr_systemwide || 1326 event->parent || event->attr.enable_on_exec || 1327 trace_uprobe_filter_event(filter, event); 1328 list_add(&event->hw.tp_list, &filter->perf_events); 1329 } else { 1330 done = filter->nr_systemwide; 1331 filter->nr_systemwide++; 1332 } 1333 write_unlock(&filter->rwlock); 1334 1335 return done; 1336 } 1337 1338 static int uprobe_perf_close(struct trace_event_call *call, 1339 struct perf_event *event) 1340 { 1341 struct trace_probe *tp; 1342 struct trace_uprobe *tu; 1343 int ret = 0; 1344 1345 tp = trace_probe_primary_from_call(call); 1346 if (WARN_ON_ONCE(!tp)) 1347 return -ENODEV; 1348 1349 tu = container_of(tp, struct trace_uprobe, tp); 1350 if (trace_uprobe_filter_remove(tu->tp.event->filter, event)) 1351 return 0; 1352 1353 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) { 1354 ret = uprobe_apply(tu->uprobe, &tu->consumer, false); 1355 if (ret) 1356 break; 1357 } 1358 1359 return ret; 1360 } 1361 1362 static int uprobe_perf_open(struct trace_event_call *call, 1363 struct perf_event *event) 1364 { 1365 struct trace_probe *tp; 1366 struct trace_uprobe *tu; 1367 int err = 0; 1368 1369 tp = trace_probe_primary_from_call(call); 1370 if (WARN_ON_ONCE(!tp)) 1371 return -ENODEV; 1372 1373 tu = container_of(tp, struct trace_uprobe, tp); 1374 if (trace_uprobe_filter_add(tu->tp.event->filter, event)) 1375 return 0; 1376 1377 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) { 1378 err = uprobe_apply(tu->uprobe, &tu->consumer, true); 1379 if (err) { 1380 uprobe_perf_close(call, event); 1381 break; 1382 } 1383 } 1384 1385 return err; 1386 } 1387 1388 static bool uprobe_perf_filter(struct uprobe_consumer *uc, struct mm_struct *mm) 1389 { 1390 struct trace_uprobe_filter *filter; 1391 struct trace_uprobe *tu; 1392 int ret; 1393 1394 tu = container_of(uc, struct trace_uprobe, consumer); 1395 filter = tu->tp.event->filter; 1396 1397 /* 1398 * speculative short-circuiting check to avoid unnecessarily taking 1399 * filter->rwlock below, if the uprobe has system-wide consumer 1400 */ 1401 if (READ_ONCE(filter->nr_systemwide)) 1402 return true; 1403 1404 read_lock(&filter->rwlock); 1405 ret = __uprobe_perf_filter(filter, mm); 1406 read_unlock(&filter->rwlock); 1407 1408 return ret; 1409 } 1410 1411 static void __uprobe_perf_func(struct trace_uprobe *tu, 1412 unsigned long func, struct pt_regs *regs, 1413 struct uprobe_cpu_buffer **ucbp) 1414 { 1415 struct trace_event_call *call = trace_probe_event_call(&tu->tp); 1416 struct uprobe_trace_entry_head *entry; 1417 struct uprobe_cpu_buffer *ucb; 1418 struct hlist_head *head; 1419 void *data; 1420 int size, esize; 1421 int rctx; 1422 1423 #ifdef CONFIG_BPF_EVENTS 1424 if (bpf_prog_array_valid(call)) { 1425 const struct bpf_prog_array *array; 1426 u32 ret; 1427 1428 rcu_read_lock_trace(); 1429 array = rcu_dereference_check(call->prog_array, rcu_read_lock_trace_held()); 1430 ret = bpf_prog_run_array_uprobe(array, regs, bpf_prog_run); 1431 rcu_read_unlock_trace(); 1432 if (!ret) 1433 return; 1434 } 1435 #endif /* CONFIG_BPF_EVENTS */ 1436 1437 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 1438 1439 ucb = prepare_uprobe_buffer(tu, regs, ucbp); 1440 size = esize + ucb->dsize; 1441 size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32); 1442 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough")) 1443 return; 1444 1445 preempt_disable(); 1446 head = this_cpu_ptr(call->perf_events); 1447 if (hlist_empty(head)) 1448 goto out; 1449 1450 entry = perf_trace_buf_alloc(size, NULL, &rctx); 1451 if (!entry) 1452 goto out; 1453 1454 if (is_ret_probe(tu)) { 1455 entry->vaddr[0] = func; 1456 entry->vaddr[1] = instruction_pointer(regs); 1457 data = DATAOF_TRACE_ENTRY(entry, true); 1458 } else { 1459 entry->vaddr[0] = instruction_pointer(regs); 1460 data = DATAOF_TRACE_ENTRY(entry, false); 1461 } 1462 1463 memcpy(data, ucb->buf, ucb->dsize); 1464 1465 if (size - esize > ucb->dsize) 1466 memset(data + ucb->dsize, 0, size - esize - ucb->dsize); 1467 1468 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1469 head, NULL); 1470 out: 1471 preempt_enable(); 1472 } 1473 1474 /* uprobe profile handler */ 1475 static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs, 1476 struct uprobe_cpu_buffer **ucbp) 1477 { 1478 if (!uprobe_perf_filter(&tu->consumer, current->mm)) 1479 return UPROBE_HANDLER_REMOVE; 1480 1481 if (!is_ret_probe(tu)) 1482 __uprobe_perf_func(tu, 0, regs, ucbp); 1483 return 0; 1484 } 1485 1486 static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func, 1487 struct pt_regs *regs, 1488 struct uprobe_cpu_buffer **ucbp) 1489 { 1490 __uprobe_perf_func(tu, func, regs, ucbp); 1491 } 1492 1493 int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, 1494 const char **filename, u64 *probe_offset, 1495 u64 *probe_addr, bool perf_type_tracepoint) 1496 { 1497 const char *pevent = trace_event_name(event->tp_event); 1498 const char *group = event->tp_event->class->system; 1499 struct trace_uprobe *tu; 1500 1501 if (perf_type_tracepoint) 1502 tu = find_probe_event(pevent, group); 1503 else 1504 tu = trace_uprobe_primary_from_call(event->tp_event); 1505 if (!tu) 1506 return -EINVAL; 1507 1508 *fd_type = is_ret_probe(tu) ? BPF_FD_TYPE_URETPROBE 1509 : BPF_FD_TYPE_UPROBE; 1510 *filename = tu->filename; 1511 *probe_offset = tu->offset; 1512 *probe_addr = tu->ref_ctr_offset; 1513 return 0; 1514 } 1515 #endif /* CONFIG_PERF_EVENTS */ 1516 1517 static int 1518 trace_uprobe_register(struct trace_event_call *event, enum trace_reg type, 1519 void *data) 1520 { 1521 struct trace_event_file *file = data; 1522 1523 switch (type) { 1524 case TRACE_REG_REGISTER: 1525 return probe_event_enable(event, file, NULL); 1526 1527 case TRACE_REG_UNREGISTER: 1528 probe_event_disable(event, file); 1529 return 0; 1530 1531 #ifdef CONFIG_PERF_EVENTS 1532 case TRACE_REG_PERF_REGISTER: 1533 return probe_event_enable(event, NULL, uprobe_perf_filter); 1534 1535 case TRACE_REG_PERF_UNREGISTER: 1536 probe_event_disable(event, NULL); 1537 return 0; 1538 1539 case TRACE_REG_PERF_OPEN: 1540 return uprobe_perf_open(event, data); 1541 1542 case TRACE_REG_PERF_CLOSE: 1543 return uprobe_perf_close(event, data); 1544 1545 #endif 1546 default: 1547 return 0; 1548 } 1549 } 1550 1551 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs, 1552 __u64 *data) 1553 { 1554 struct trace_uprobe *tu; 1555 struct uprobe_dispatch_data udd; 1556 struct uprobe_cpu_buffer *ucb = NULL; 1557 unsigned int flags; 1558 int ret = 0; 1559 1560 tu = container_of(con, struct trace_uprobe, consumer); 1561 1562 this_cpu_inc(*tu->nhits); 1563 1564 udd.tu = tu; 1565 udd.bp_addr = instruction_pointer(regs); 1566 1567 current->utask->vaddr = (unsigned long) &udd; 1568 1569 if (WARN_ON_ONCE(!uprobe_cpu_buffer)) 1570 return 0; 1571 1572 flags = trace_probe_load_flag(&tu->tp); 1573 if (flags & TP_FLAG_TRACE) 1574 ret |= uprobe_trace_func(tu, regs, &ucb); 1575 1576 #ifdef CONFIG_PERF_EVENTS 1577 if (flags & TP_FLAG_PROFILE) 1578 ret |= uprobe_perf_func(tu, regs, &ucb); 1579 #endif 1580 uprobe_buffer_put(ucb); 1581 return ret; 1582 } 1583 1584 static int uretprobe_dispatcher(struct uprobe_consumer *con, 1585 unsigned long func, struct pt_regs *regs, 1586 __u64 *data) 1587 { 1588 struct trace_uprobe *tu; 1589 struct uprobe_dispatch_data udd; 1590 struct uprobe_cpu_buffer *ucb = NULL; 1591 unsigned int flags; 1592 1593 tu = container_of(con, struct trace_uprobe, consumer); 1594 1595 udd.tu = tu; 1596 udd.bp_addr = func; 1597 1598 current->utask->vaddr = (unsigned long) &udd; 1599 1600 if (WARN_ON_ONCE(!uprobe_cpu_buffer)) 1601 return 0; 1602 1603 flags = trace_probe_load_flag(&tu->tp); 1604 if (flags & TP_FLAG_TRACE) 1605 uretprobe_trace_func(tu, func, regs, &ucb); 1606 1607 #ifdef CONFIG_PERF_EVENTS 1608 if (flags & TP_FLAG_PROFILE) 1609 uretprobe_perf_func(tu, func, regs, &ucb); 1610 #endif 1611 uprobe_buffer_put(ucb); 1612 return 0; 1613 } 1614 1615 static struct trace_event_functions uprobe_funcs = { 1616 .trace = print_uprobe_event 1617 }; 1618 1619 static struct trace_event_fields uprobe_fields_array[] = { 1620 { .type = TRACE_FUNCTION_TYPE, 1621 .define_fields = uprobe_event_define_fields }, 1622 {} 1623 }; 1624 1625 static inline void init_trace_event_call(struct trace_uprobe *tu) 1626 { 1627 struct trace_event_call *call = trace_probe_event_call(&tu->tp); 1628 call->event.funcs = &uprobe_funcs; 1629 call->class->fields_array = uprobe_fields_array; 1630 1631 call->flags = TRACE_EVENT_FL_UPROBE | TRACE_EVENT_FL_CAP_ANY; 1632 call->class->reg = trace_uprobe_register; 1633 } 1634 1635 static int register_uprobe_event(struct trace_uprobe *tu) 1636 { 1637 init_trace_event_call(tu); 1638 1639 return trace_probe_register_event_call(&tu->tp); 1640 } 1641 1642 static int unregister_uprobe_event(struct trace_uprobe *tu) 1643 { 1644 return trace_probe_unregister_event_call(&tu->tp); 1645 } 1646 1647 #ifdef CONFIG_PERF_EVENTS 1648 struct trace_event_call * 1649 create_local_trace_uprobe(char *name, unsigned long offs, 1650 unsigned long ref_ctr_offset, bool is_return) 1651 { 1652 enum probe_print_type ptype; 1653 struct trace_uprobe *tu; 1654 struct path path; 1655 int ret; 1656 1657 ret = kern_path(name, LOOKUP_FOLLOW, &path); 1658 if (ret) 1659 return ERR_PTR(ret); 1660 1661 if (!d_is_reg(path.dentry)) { 1662 path_put(&path); 1663 return ERR_PTR(-EINVAL); 1664 } 1665 1666 /* 1667 * local trace_kprobes are not added to dyn_event, so they are never 1668 * searched in find_trace_kprobe(). Therefore, there is no concern of 1669 * duplicated name "DUMMY_EVENT" here. 1670 */ 1671 tu = alloc_trace_uprobe(UPROBE_EVENT_SYSTEM, "DUMMY_EVENT", 0, 1672 is_return); 1673 1674 if (IS_ERR(tu)) { 1675 pr_info("Failed to allocate trace_uprobe.(%d)\n", 1676 (int)PTR_ERR(tu)); 1677 path_put(&path); 1678 return ERR_CAST(tu); 1679 } 1680 1681 tu->offset = offs; 1682 tu->path = path; 1683 tu->ref_ctr_offset = ref_ctr_offset; 1684 tu->filename = kstrdup(name, GFP_KERNEL); 1685 if (!tu->filename) { 1686 ret = -ENOMEM; 1687 goto error; 1688 } 1689 1690 init_trace_event_call(tu); 1691 1692 ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL; 1693 if (traceprobe_set_print_fmt(&tu->tp, ptype) < 0) { 1694 ret = -ENOMEM; 1695 goto error; 1696 } 1697 1698 return trace_probe_event_call(&tu->tp); 1699 error: 1700 free_trace_uprobe(tu); 1701 return ERR_PTR(ret); 1702 } 1703 1704 void destroy_local_trace_uprobe(struct trace_event_call *event_call) 1705 { 1706 struct trace_uprobe *tu; 1707 1708 tu = trace_uprobe_primary_from_call(event_call); 1709 1710 free_trace_uprobe(tu); 1711 } 1712 #endif /* CONFIG_PERF_EVENTS */ 1713 1714 /* Make a trace interface for controlling probe points */ 1715 static __init int init_uprobe_trace(void) 1716 { 1717 int ret; 1718 1719 ret = dyn_event_register(&trace_uprobe_ops); 1720 if (ret) 1721 return ret; 1722 1723 ret = tracing_init_dentry(); 1724 if (ret) 1725 return 0; 1726 1727 trace_create_file("uprobe_events", TRACE_MODE_WRITE, NULL, 1728 NULL, &uprobe_events_ops); 1729 /* Profile interface */ 1730 trace_create_file("uprobe_profile", TRACE_MODE_READ, NULL, 1731 NULL, &uprobe_profile_ops); 1732 return 0; 1733 } 1734 1735 fs_initcall(init_uprobe_trace); 1736