1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Fprobe-based tracing events 4 * Copyright (C) 2022 Google LLC. 5 */ 6 #define pr_fmt(fmt) "trace_fprobe: " fmt 7 8 #include <linux/fprobe.h> 9 #include <linux/list.h> 10 #include <linux/module.h> 11 #include <linux/mutex.h> 12 #include <linux/rculist.h> 13 #include <linux/security.h> 14 #include <linux/tracepoint.h> 15 #include <linux/uaccess.h> 16 17 #include <asm/ptrace.h> 18 19 #include "trace_dynevent.h" 20 #include "trace_probe.h" 21 #include "trace_probe_kernel.h" 22 #include "trace_probe_tmpl.h" 23 24 #define FPROBE_EVENT_SYSTEM "fprobes" 25 #define TRACEPOINT_EVENT_SYSTEM "tracepoints" 26 #define RETHOOK_MAXACTIVE_MAX 4096 27 28 static int trace_fprobe_create(const char *raw_command); 29 static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev); 30 static int trace_fprobe_release(struct dyn_event *ev); 31 static bool trace_fprobe_is_busy(struct dyn_event *ev); 32 static bool trace_fprobe_match(const char *system, const char *event, 33 int argc, const char **argv, struct dyn_event *ev); 34 35 static struct dyn_event_operations trace_fprobe_ops = { 36 .create = trace_fprobe_create, 37 .show = trace_fprobe_show, 38 .is_busy = trace_fprobe_is_busy, 39 .free = trace_fprobe_release, 40 .match = trace_fprobe_match, 41 }; 42 43 /* List of tracepoint_user */ 44 static LIST_HEAD(tracepoint_user_list); 45 static DEFINE_MUTEX(tracepoint_user_mutex); 46 47 /* While living tracepoint_user, @tpoint can be NULL and @refcount != 0. */ 48 struct tracepoint_user { 49 struct list_head list; 50 const char *name; 51 struct tracepoint *tpoint; 52 unsigned int refcount; 53 }; 54 55 /* NOTE: you must lock tracepoint_user_mutex. */ 56 #define for_each_tracepoint_user(tuser) \ 57 list_for_each_entry(tuser, &tracepoint_user_list, list) 58 59 static int tracepoint_user_register(struct tracepoint_user *tuser) 60 { 61 struct tracepoint *tpoint = tuser->tpoint; 62 63 if (!tpoint) 64 return 0; 65 66 return tracepoint_probe_register_prio_may_exist(tpoint, 67 tpoint->probestub, NULL, 0); 68 } 69 70 static void tracepoint_user_unregister(struct tracepoint_user *tuser) 71 { 72 if (!tuser->tpoint) 73 return; 74 75 WARN_ON_ONCE(tracepoint_probe_unregister(tuser->tpoint, tuser->tpoint->probestub, NULL)); 76 tuser->tpoint = NULL; 77 } 78 79 static unsigned long tracepoint_user_ip(struct tracepoint_user *tuser) 80 { 81 if (!tuser->tpoint) 82 return 0UL; 83 84 return (unsigned long)tuser->tpoint->probestub; 85 } 86 87 static void __tracepoint_user_free(struct tracepoint_user *tuser) 88 { 89 if (!tuser) 90 return; 91 kfree(tuser->name); 92 kfree(tuser); 93 } 94 95 DEFINE_FREE(tuser_free, struct tracepoint_user *, __tracepoint_user_free(_T)) 96 97 static struct tracepoint_user *__tracepoint_user_init(const char *name, struct tracepoint *tpoint) 98 { 99 struct tracepoint_user *tuser __free(tuser_free) = NULL; 100 int ret; 101 102 tuser = kzalloc_obj(*tuser); 103 if (!tuser) 104 return NULL; 105 tuser->name = kstrdup(name, GFP_KERNEL); 106 if (!tuser->name) 107 return NULL; 108 109 /* Register tracepoint if it is loaded. */ 110 if (tpoint) { 111 tuser->tpoint = tpoint; 112 ret = tracepoint_user_register(tuser); 113 if (ret) 114 return ERR_PTR(ret); 115 } 116 117 tuser->refcount = 1; 118 INIT_LIST_HEAD(&tuser->list); 119 list_add(&tuser->list, &tracepoint_user_list); 120 121 return_ptr(tuser); 122 } 123 124 static struct tracepoint *find_tracepoint(const char *tp_name, 125 struct module **tp_mod); 126 127 /* 128 * Get tracepoint_user if exist, or allocate new one and register it. 129 * If tracepoint is on a module, get its refcounter too. 130 * This returns errno or NULL (not loaded yet) or tracepoint_user. 131 */ 132 static struct tracepoint_user *tracepoint_user_find_get(const char *name, struct module **pmod) 133 { 134 struct module *mod __free(module_put) = NULL; 135 struct tracepoint_user *tuser; 136 struct tracepoint *tpoint; 137 138 if (!name || !pmod) 139 return ERR_PTR(-EINVAL); 140 141 /* Get and lock the module which has tracepoint. */ 142 tpoint = find_tracepoint(name, &mod); 143 144 guard(mutex)(&tracepoint_user_mutex); 145 /* Search existing tracepoint_user */ 146 for_each_tracepoint_user(tuser) { 147 if (!strcmp(tuser->name, name)) { 148 tuser->refcount++; 149 *pmod = no_free_ptr(mod); 150 return tuser; 151 } 152 } 153 154 /* The corresponding tracepoint_user is not found. */ 155 tuser = __tracepoint_user_init(name, tpoint); 156 if (!IS_ERR_OR_NULL(tuser)) 157 *pmod = no_free_ptr(mod); 158 159 return tuser; 160 } 161 162 static void tracepoint_user_put(struct tracepoint_user *tuser) 163 { 164 scoped_guard(mutex, &tracepoint_user_mutex) { 165 if (--tuser->refcount > 0) 166 return; 167 168 list_del(&tuser->list); 169 tracepoint_user_unregister(tuser); 170 } 171 172 __tracepoint_user_free(tuser); 173 } 174 175 DEFINE_FREE(tuser_put, struct tracepoint_user *, 176 if (!IS_ERR_OR_NULL(_T)) 177 tracepoint_user_put(_T)) 178 179 /* 180 * Fprobe event core functions 181 */ 182 183 /* 184 * @tprobe is true for tracepoint probe. 185 * @tuser can be NULL if the trace_fprobe is disabled or the tracepoint is not 186 * loaded with a module. If @tuser != NULL, this trace_fprobe is enabled. 187 */ 188 struct trace_fprobe { 189 struct dyn_event devent; 190 struct fprobe fp; 191 const char *symbol; 192 bool tprobe; 193 struct tracepoint_user *tuser; 194 struct trace_probe tp; 195 }; 196 197 static bool is_trace_fprobe(struct dyn_event *ev) 198 { 199 return ev->ops == &trace_fprobe_ops; 200 } 201 202 static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev) 203 { 204 return container_of(ev, struct trace_fprobe, devent); 205 } 206 207 /** 208 * for_each_trace_fprobe - iterate over the trace_fprobe list 209 * @pos: the struct trace_fprobe * for each entry 210 * @dpos: the struct dyn_event * to use as a loop cursor 211 */ 212 #define for_each_trace_fprobe(pos, dpos) \ 213 for_each_dyn_event(dpos) \ 214 if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos))) 215 216 static bool trace_fprobe_is_return(struct trace_fprobe *tf) 217 { 218 return tf->fp.exit_handler != NULL; 219 } 220 221 static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf) 222 { 223 return tf->tprobe; 224 } 225 226 static const char *trace_fprobe_symbol(struct trace_fprobe *tf) 227 { 228 return tf->symbol ? tf->symbol : "unknown"; 229 } 230 231 static bool trace_fprobe_is_busy(struct dyn_event *ev) 232 { 233 struct trace_fprobe *tf = to_trace_fprobe(ev); 234 235 return trace_probe_is_enabled(&tf->tp); 236 } 237 238 static bool trace_fprobe_match_command_head(struct trace_fprobe *tf, 239 int argc, const char **argv) 240 { 241 if (!argc) 242 return true; 243 244 if (strcmp(trace_fprobe_symbol(tf), argv[0])) 245 return false; 246 argc--; argv++; 247 248 return trace_probe_match_command_args(&tf->tp, argc, argv); 249 } 250 251 static bool trace_fprobe_match(const char *system, const char *event, 252 int argc, const char **argv, struct dyn_event *ev) 253 { 254 struct trace_fprobe *tf = to_trace_fprobe(ev); 255 256 if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event)) 257 return false; 258 259 if (system && strcmp(trace_probe_group_name(&tf->tp), system)) 260 return false; 261 262 return trace_fprobe_match_command_head(tf, argc, argv); 263 } 264 265 static bool trace_fprobe_is_registered(struct trace_fprobe *tf) 266 { 267 return fprobe_is_registered(&tf->fp); 268 } 269 270 /* 271 * Note that we don't verify the fetch_insn code, since it does not come 272 * from user space. 273 */ 274 static int 275 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 276 void *dest, void *base) 277 { 278 struct ftrace_regs *fregs = rec; 279 unsigned long val; 280 int ret; 281 282 retry: 283 /* 1st stage: get value from context */ 284 switch (code->op) { 285 case FETCH_OP_STACK: 286 val = ftrace_regs_get_kernel_stack_nth(fregs, code->param); 287 break; 288 case FETCH_OP_STACKP: 289 val = ftrace_regs_get_stack_pointer(fregs); 290 break; 291 case FETCH_OP_RETVAL: 292 val = ftrace_regs_get_return_value(fregs); 293 break; 294 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 295 case FETCH_OP_ARG: 296 val = ftrace_regs_get_argument(fregs, code->param); 297 break; 298 case FETCH_OP_EDATA: 299 val = *(unsigned long *)((unsigned long)edata + code->offset); 300 break; 301 #endif 302 case FETCH_NOP_SYMBOL: /* Ignore a place holder */ 303 code++; 304 goto retry; 305 default: 306 ret = process_common_fetch_insn(code, &val); 307 if (ret < 0) 308 return ret; 309 } 310 code++; 311 312 return process_fetch_insn_bottom(code, val, dest, base); 313 } 314 NOKPROBE_SYMBOL(process_fetch_insn) 315 316 /* function entry handler */ 317 static nokprobe_inline void 318 __fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 319 struct ftrace_regs *fregs, 320 struct trace_event_file *trace_file) 321 { 322 struct fentry_trace_entry_head *entry; 323 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 324 struct trace_event_buffer fbuffer; 325 int dsize; 326 327 if (WARN_ON_ONCE(call != trace_file->event_call)) 328 return; 329 330 if (trace_trigger_soft_disabled(trace_file)) 331 return; 332 333 dsize = __get_data_size(&tf->tp, fregs, NULL); 334 335 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 336 sizeof(*entry) + tf->tp.size + dsize); 337 if (!entry) 338 return; 339 340 fbuffer.regs = ftrace_get_regs(fregs); 341 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); 342 entry->ip = entry_ip; 343 store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize); 344 345 trace_event_buffer_commit(&fbuffer); 346 } 347 348 static void 349 fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 350 struct ftrace_regs *fregs) 351 { 352 struct event_file_link *link; 353 354 trace_probe_for_each_link_rcu(link, &tf->tp) 355 __fentry_trace_func(tf, entry_ip, fregs, link->file); 356 } 357 NOKPROBE_SYMBOL(fentry_trace_func); 358 359 static nokprobe_inline 360 void store_fprobe_entry_data(void *edata, struct trace_probe *tp, struct ftrace_regs *fregs) 361 { 362 struct probe_entry_arg *earg = tp->entry_arg; 363 unsigned long val = 0; 364 int i; 365 366 if (!earg) 367 return; 368 369 for (i = 0; i < earg->size; i++) { 370 struct fetch_insn *code = &earg->code[i]; 371 372 switch (code->op) { 373 case FETCH_OP_ARG: 374 val = ftrace_regs_get_argument(fregs, code->param); 375 break; 376 case FETCH_OP_ST_EDATA: 377 *(unsigned long *)((unsigned long)edata + code->offset) = val; 378 break; 379 case FETCH_OP_END: 380 goto end; 381 default: 382 break; 383 } 384 } 385 end: 386 return; 387 } 388 389 /* function exit handler */ 390 static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip, 391 unsigned long ret_ip, struct ftrace_regs *fregs, 392 void *entry_data) 393 { 394 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); 395 396 if (tf->tp.entry_arg) 397 store_fprobe_entry_data(entry_data, &tf->tp, fregs); 398 399 return 0; 400 } 401 NOKPROBE_SYMBOL(trace_fprobe_entry_handler) 402 403 static nokprobe_inline void 404 __fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 405 unsigned long ret_ip, struct ftrace_regs *fregs, 406 void *entry_data, struct trace_event_file *trace_file) 407 { 408 struct fexit_trace_entry_head *entry; 409 struct trace_event_buffer fbuffer; 410 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 411 int dsize; 412 413 if (WARN_ON_ONCE(call != trace_file->event_call)) 414 return; 415 416 if (trace_trigger_soft_disabled(trace_file)) 417 return; 418 419 dsize = __get_data_size(&tf->tp, fregs, entry_data); 420 421 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 422 sizeof(*entry) + tf->tp.size + dsize); 423 if (!entry) 424 return; 425 426 fbuffer.regs = ftrace_get_regs(fregs); 427 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); 428 entry->func = entry_ip; 429 entry->ret_ip = ret_ip; 430 store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize); 431 432 trace_event_buffer_commit(&fbuffer); 433 } 434 435 static void 436 fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 437 unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data) 438 { 439 struct event_file_link *link; 440 441 trace_probe_for_each_link_rcu(link, &tf->tp) 442 __fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data, link->file); 443 } 444 NOKPROBE_SYMBOL(fexit_trace_func); 445 446 #ifdef CONFIG_PERF_EVENTS 447 448 static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, 449 struct ftrace_regs *fregs) 450 { 451 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 452 struct fentry_trace_entry_head *entry; 453 struct hlist_head *head; 454 int size, __size, dsize; 455 struct pt_regs *regs; 456 int rctx; 457 458 head = this_cpu_ptr(call->perf_events); 459 if (hlist_empty(head)) 460 return 0; 461 462 dsize = __get_data_size(&tf->tp, fregs, NULL); 463 __size = sizeof(*entry) + tf->tp.size + dsize; 464 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 465 size -= sizeof(u32); 466 467 entry = perf_trace_buf_alloc(size, ®s, &rctx); 468 if (!entry) 469 return 0; 470 471 regs = ftrace_fill_perf_regs(fregs, regs); 472 473 entry->ip = entry_ip; 474 store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize); 475 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 476 head, NULL); 477 return 0; 478 } 479 NOKPROBE_SYMBOL(fentry_perf_func); 480 481 static void 482 fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, 483 unsigned long ret_ip, struct ftrace_regs *fregs, 484 void *entry_data) 485 { 486 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 487 struct fexit_trace_entry_head *entry; 488 struct hlist_head *head; 489 int size, __size, dsize; 490 struct pt_regs *regs; 491 int rctx; 492 493 head = this_cpu_ptr(call->perf_events); 494 if (hlist_empty(head)) 495 return; 496 497 dsize = __get_data_size(&tf->tp, fregs, entry_data); 498 __size = sizeof(*entry) + tf->tp.size + dsize; 499 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 500 size -= sizeof(u32); 501 502 entry = perf_trace_buf_alloc(size, ®s, &rctx); 503 if (!entry) 504 return; 505 506 regs = ftrace_fill_perf_regs(fregs, regs); 507 508 entry->func = entry_ip; 509 entry->ret_ip = ret_ip; 510 store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize); 511 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 512 head, NULL); 513 } 514 NOKPROBE_SYMBOL(fexit_perf_func); 515 #endif /* CONFIG_PERF_EVENTS */ 516 517 static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip, 518 unsigned long ret_ip, struct ftrace_regs *fregs, 519 void *entry_data) 520 { 521 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); 522 unsigned int flags = trace_probe_load_flag(&tf->tp); 523 int ret = 0; 524 525 if (flags & TP_FLAG_TRACE) 526 fentry_trace_func(tf, entry_ip, fregs); 527 528 #ifdef CONFIG_PERF_EVENTS 529 if (flags & TP_FLAG_PROFILE) 530 ret = fentry_perf_func(tf, entry_ip, fregs); 531 #endif 532 return ret; 533 } 534 NOKPROBE_SYMBOL(fentry_dispatcher); 535 536 static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip, 537 unsigned long ret_ip, struct ftrace_regs *fregs, 538 void *entry_data) 539 { 540 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); 541 unsigned int flags = trace_probe_load_flag(&tf->tp); 542 543 if (flags & TP_FLAG_TRACE) 544 fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data); 545 #ifdef CONFIG_PERF_EVENTS 546 if (flags & TP_FLAG_PROFILE) 547 fexit_perf_func(tf, entry_ip, ret_ip, fregs, entry_data); 548 #endif 549 } 550 NOKPROBE_SYMBOL(fexit_dispatcher); 551 552 static void free_trace_fprobe(struct trace_fprobe *tf) 553 { 554 if (tf) { 555 trace_probe_cleanup(&tf->tp); 556 if (tf->tuser) 557 tracepoint_user_put(tf->tuser); 558 kfree(tf->symbol); 559 kfree(tf); 560 } 561 } 562 563 /* Since alloc_trace_fprobe() can return error, check the pointer is ERR too. */ 564 DEFINE_FREE(free_trace_fprobe, struct trace_fprobe *, if (!IS_ERR_OR_NULL(_T)) free_trace_fprobe(_T)) 565 566 /* 567 * Allocate new trace_probe and initialize it (including fprobe). 568 */ 569 static struct trace_fprobe *alloc_trace_fprobe(const char *group, 570 const char *event, 571 const char *symbol, 572 int nargs, bool is_return, 573 bool is_tracepoint) 574 { 575 struct trace_fprobe *tf __free(free_trace_fprobe) = NULL; 576 int ret = -ENOMEM; 577 578 tf = kzalloc_flex(*tf, tp.args, nargs); 579 if (!tf) 580 return ERR_PTR(ret); 581 582 tf->symbol = kstrdup(symbol, GFP_KERNEL); 583 if (!tf->symbol) 584 return ERR_PTR(-ENOMEM); 585 586 if (is_return) 587 tf->fp.exit_handler = fexit_dispatcher; 588 else 589 tf->fp.entry_handler = fentry_dispatcher; 590 591 tf->tprobe = is_tracepoint; 592 593 ret = trace_probe_init(&tf->tp, event, group, false, nargs); 594 if (ret < 0) 595 return ERR_PTR(ret); 596 597 dyn_event_init(&tf->devent, &trace_fprobe_ops); 598 return_ptr(tf); 599 } 600 601 static struct trace_fprobe *find_trace_fprobe(const char *event, 602 const char *group) 603 { 604 struct dyn_event *pos; 605 struct trace_fprobe *tf; 606 607 for_each_trace_fprobe(tf, pos) 608 if (strcmp(trace_probe_name(&tf->tp), event) == 0 && 609 strcmp(trace_probe_group_name(&tf->tp), group) == 0) 610 return tf; 611 return NULL; 612 } 613 614 /* Event entry printers */ 615 static enum print_line_t 616 print_fentry_event(struct trace_iterator *iter, int flags, 617 struct trace_event *event) 618 { 619 struct fentry_trace_entry_head *field; 620 struct trace_seq *s = &iter->seq; 621 struct trace_probe *tp; 622 623 field = (struct fentry_trace_entry_head *)iter->ent; 624 tp = trace_probe_primary_from_call( 625 container_of(event, struct trace_event_call, event)); 626 if (WARN_ON_ONCE(!tp)) 627 goto out; 628 629 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 630 631 if (!seq_print_ip_sym_offset(s, field->ip, flags)) 632 goto out; 633 634 trace_seq_putc(s, ')'); 635 636 if (trace_probe_print_args(s, tp->args, tp->nr_args, 637 (u8 *)&field[1], field) < 0) 638 goto out; 639 640 trace_seq_putc(s, '\n'); 641 out: 642 return trace_handle_return(s); 643 } 644 645 static enum print_line_t 646 print_fexit_event(struct trace_iterator *iter, int flags, 647 struct trace_event *event) 648 { 649 struct fexit_trace_entry_head *field; 650 struct trace_seq *s = &iter->seq; 651 struct trace_probe *tp; 652 653 field = (struct fexit_trace_entry_head *)iter->ent; 654 tp = trace_probe_primary_from_call( 655 container_of(event, struct trace_event_call, event)); 656 if (WARN_ON_ONCE(!tp)) 657 goto out; 658 659 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 660 661 if (!seq_print_ip_sym_offset(s, field->ret_ip, flags)) 662 goto out; 663 664 trace_seq_puts(s, " <- "); 665 666 if (!seq_print_ip_sym_no_offset(s, field->func, flags)) 667 goto out; 668 669 trace_seq_putc(s, ')'); 670 671 if (trace_probe_print_args(s, tp->args, tp->nr_args, 672 (u8 *)&field[1], field) < 0) 673 goto out; 674 675 trace_seq_putc(s, '\n'); 676 677 out: 678 return trace_handle_return(s); 679 } 680 681 static int fentry_event_define_fields(struct trace_event_call *event_call) 682 { 683 int ret; 684 struct fentry_trace_entry_head field; 685 struct trace_probe *tp; 686 687 tp = trace_probe_primary_from_call(event_call); 688 if (WARN_ON_ONCE(!tp)) 689 return -ENOENT; 690 691 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 692 693 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 694 } 695 696 static int fexit_event_define_fields(struct trace_event_call *event_call) 697 { 698 int ret; 699 struct fexit_trace_entry_head field; 700 struct trace_probe *tp; 701 702 tp = trace_probe_primary_from_call(event_call); 703 if (WARN_ON_ONCE(!tp)) 704 return -ENOENT; 705 706 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 707 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 708 709 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 710 } 711 712 static struct trace_event_functions fentry_funcs = { 713 .trace = print_fentry_event 714 }; 715 716 static struct trace_event_functions fexit_funcs = { 717 .trace = print_fexit_event 718 }; 719 720 static struct trace_event_fields fentry_fields_array[] = { 721 { .type = TRACE_FUNCTION_TYPE, 722 .define_fields = fentry_event_define_fields }, 723 {} 724 }; 725 726 static struct trace_event_fields fexit_fields_array[] = { 727 { .type = TRACE_FUNCTION_TYPE, 728 .define_fields = fexit_event_define_fields }, 729 {} 730 }; 731 732 static int fprobe_register(struct trace_event_call *event, 733 enum trace_reg type, void *data); 734 735 static inline void init_trace_event_call(struct trace_fprobe *tf) 736 { 737 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 738 739 if (trace_fprobe_is_return(tf)) { 740 call->event.funcs = &fexit_funcs; 741 call->class->fields_array = fexit_fields_array; 742 } else { 743 call->event.funcs = &fentry_funcs; 744 call->class->fields_array = fentry_fields_array; 745 } 746 747 call->flags = TRACE_EVENT_FL_FPROBE; 748 call->class->reg = fprobe_register; 749 } 750 751 static int register_fprobe_event(struct trace_fprobe *tf) 752 { 753 init_trace_event_call(tf); 754 755 return trace_probe_register_event_call(&tf->tp); 756 } 757 758 static int unregister_fprobe_event(struct trace_fprobe *tf) 759 { 760 return trace_probe_unregister_event_call(&tf->tp); 761 } 762 763 static int __register_tracepoint_fprobe(struct trace_fprobe *tf) 764 { 765 struct tracepoint_user *tuser __free(tuser_put) = NULL; 766 struct module *mod __free(module_put) = NULL; 767 unsigned long ip; 768 int ret; 769 770 if (WARN_ON_ONCE(tf->tuser)) 771 return -EINVAL; 772 773 /* If the tracepoint is in a module, it must be locked in this function. */ 774 tuser = tracepoint_user_find_get(tf->symbol, &mod); 775 /* This tracepoint is not loaded yet */ 776 if (IS_ERR(tuser)) 777 return PTR_ERR(tuser); 778 if (!tuser) 779 return -ENOMEM; 780 781 /* Register fprobe only if the tracepoint is loaded. */ 782 if (tuser->tpoint) { 783 ip = tracepoint_user_ip(tuser); 784 if (WARN_ON_ONCE(!ip)) 785 return -ENOENT; 786 787 ret = register_fprobe_ips(&tf->fp, &ip, 1); 788 if (ret < 0) 789 return ret; 790 } 791 792 tf->tuser = no_free_ptr(tuser); 793 return 0; 794 } 795 796 /* Returns an error if the target function is not available, or 0 */ 797 static int trace_fprobe_verify_target(struct trace_fprobe *tf) 798 { 799 int ret; 800 801 /* Tracepoint should have a stub function. */ 802 if (trace_fprobe_is_tracepoint(tf)) 803 return 0; 804 805 /* 806 * Note: since we don't lock the module, even if this succeeded, 807 * register_fprobe() later can fail. 808 */ 809 ret = fprobe_count_ips_from_filter(tf->symbol, NULL); 810 return (ret < 0) ? ret : 0; 811 } 812 813 /* Internal register function - just handle fprobe and flags */ 814 static int __register_trace_fprobe(struct trace_fprobe *tf) 815 { 816 int i, ret; 817 818 /* Should we need new LOCKDOWN flag for fprobe? */ 819 ret = security_locked_down(LOCKDOWN_KPROBES); 820 if (ret) 821 return ret; 822 823 if (trace_fprobe_is_registered(tf)) 824 return -EINVAL; 825 826 for (i = 0; i < tf->tp.nr_args; i++) { 827 ret = traceprobe_update_arg(&tf->tp.args[i]); 828 if (ret) 829 return ret; 830 } 831 832 tf->fp.flags &= ~FPROBE_FL_DISABLED; 833 834 if (trace_fprobe_is_tracepoint(tf)) 835 return __register_tracepoint_fprobe(tf); 836 837 /* TODO: handle filter, nofilter or symbol list */ 838 return register_fprobe(&tf->fp, tf->symbol, NULL); 839 } 840 841 /* Internal unregister function - just handle fprobe and flags */ 842 static void __unregister_trace_fprobe(struct trace_fprobe *tf) 843 { 844 if (trace_fprobe_is_registered(tf)) 845 unregister_fprobe(&tf->fp); 846 if (tf->tuser) { 847 tracepoint_user_put(tf->tuser); 848 tf->tuser = NULL; 849 } 850 } 851 852 /* TODO: make this trace_*probe common function */ 853 /* Unregister a trace_probe and probe_event */ 854 static int unregister_trace_fprobe(struct trace_fprobe *tf) 855 { 856 /* If other probes are on the event, just unregister fprobe */ 857 if (trace_probe_has_sibling(&tf->tp)) 858 goto unreg; 859 860 /* Enabled event can not be unregistered */ 861 if (trace_probe_is_enabled(&tf->tp)) 862 return -EBUSY; 863 864 /* If there's a reference to the dynamic event */ 865 if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp))) 866 return -EBUSY; 867 868 /* Will fail if probe is being used by ftrace or perf */ 869 if (unregister_fprobe_event(tf)) 870 return -EBUSY; 871 872 unreg: 873 __unregister_trace_fprobe(tf); 874 dyn_event_remove(&tf->devent); 875 trace_probe_unlink(&tf->tp); 876 877 return 0; 878 } 879 880 static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig, 881 struct trace_fprobe *comp) 882 { 883 struct trace_probe_event *tpe = orig->tp.event; 884 int i; 885 886 list_for_each_entry(orig, &tpe->probes, tp.list) { 887 if (strcmp(trace_fprobe_symbol(orig), 888 trace_fprobe_symbol(comp))) 889 continue; 890 891 /* 892 * trace_probe_compare_arg_type() ensured that nr_args and 893 * each argument name and type are same. Let's compare comm. 894 */ 895 for (i = 0; i < orig->tp.nr_args; i++) { 896 if (strcmp(orig->tp.args[i].comm, 897 comp->tp.args[i].comm)) 898 break; 899 } 900 901 if (i == orig->tp.nr_args) 902 return true; 903 } 904 905 return false; 906 } 907 908 static int append_trace_fprobe_event(struct trace_fprobe *tf, struct trace_fprobe *to) 909 { 910 int ret; 911 912 if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) || 913 trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) { 914 trace_probe_log_set_index(0); 915 trace_probe_log_err(0, DIFF_PROBE_TYPE); 916 return -EEXIST; 917 } 918 ret = trace_probe_compare_arg_type(&tf->tp, &to->tp); 919 if (ret) { 920 /* Note that argument starts index = 2 */ 921 trace_probe_log_set_index(ret + 1); 922 trace_probe_log_err(0, DIFF_ARG_TYPE); 923 return -EEXIST; 924 } 925 if (trace_fprobe_has_same_fprobe(to, tf)) { 926 trace_probe_log_set_index(0); 927 trace_probe_log_err(0, SAME_PROBE); 928 return -EEXIST; 929 } 930 931 /* Append to existing event */ 932 ret = trace_probe_append(&tf->tp, &to->tp); 933 if (ret) 934 return ret; 935 936 ret = trace_fprobe_verify_target(tf); 937 if (ret) 938 trace_probe_unlink(&tf->tp); 939 else 940 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp)); 941 942 return ret; 943 } 944 945 /* Register a trace_probe and probe_event, and check the fprobe is available. */ 946 static int register_trace_fprobe_event(struct trace_fprobe *tf) 947 { 948 struct trace_fprobe *old_tf; 949 int ret; 950 951 guard(mutex)(&event_mutex); 952 953 old_tf = find_trace_fprobe(trace_probe_name(&tf->tp), 954 trace_probe_group_name(&tf->tp)); 955 if (old_tf) 956 return append_trace_fprobe_event(tf, old_tf); 957 958 /* Register new event */ 959 ret = register_fprobe_event(tf); 960 if (ret) { 961 if (ret == -EEXIST) { 962 trace_probe_log_set_index(0); 963 trace_probe_log_err(0, EVENT_EXIST); 964 } else 965 pr_warn("Failed to register probe event(%d)\n", ret); 966 return ret; 967 } 968 969 /* Verify fprobe is sane. */ 970 ret = trace_fprobe_verify_target(tf); 971 if (ret < 0) 972 unregister_fprobe_event(tf); 973 else 974 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp)); 975 976 return ret; 977 } 978 979 struct __find_tracepoint_cb_data { 980 const char *tp_name; 981 struct tracepoint *tpoint; 982 struct module *mod; 983 }; 984 985 static void __find_tracepoint_module_cb(struct tracepoint *tp, struct module *mod, void *priv) 986 { 987 struct __find_tracepoint_cb_data *data = priv; 988 989 if (!data->tpoint && !strcmp(data->tp_name, tp->name)) { 990 /* If module is not specified, try getting module refcount. */ 991 if (!data->mod && mod) { 992 /* If failed to get refcount, ignore this tracepoint. */ 993 if (!try_module_get(mod)) 994 return; 995 996 data->mod = mod; 997 } 998 data->tpoint = tp; 999 } 1000 } 1001 1002 static void __find_tracepoint_cb(struct tracepoint *tp, void *priv) 1003 { 1004 struct __find_tracepoint_cb_data *data = priv; 1005 1006 if (!data->tpoint && !strcmp(data->tp_name, tp->name)) 1007 data->tpoint = tp; 1008 } 1009 1010 /* 1011 * Find a tracepoint from kernel and module. If the tracepoint is on the module, 1012 * the module's refcount is incremented and returned as *@tp_mod. Thus, if it is 1013 * not NULL, caller must call module_put(*tp_mod) after used the tracepoint. 1014 */ 1015 static struct tracepoint *find_tracepoint(const char *tp_name, 1016 struct module **tp_mod) 1017 { 1018 struct __find_tracepoint_cb_data data = { 1019 .tp_name = tp_name, 1020 .mod = NULL, 1021 }; 1022 1023 for_each_kernel_tracepoint(__find_tracepoint_cb, &data); 1024 1025 if (!data.tpoint && IS_ENABLED(CONFIG_MODULES)) { 1026 for_each_module_tracepoint(__find_tracepoint_module_cb, &data); 1027 *tp_mod = data.mod; 1028 } 1029 1030 return data.tpoint; 1031 } 1032 1033 #ifdef CONFIG_MODULES 1034 /* 1035 * Find a tracepoint from specified module. In this case, this does not get the 1036 * module's refcount. The caller must ensure the module is not freed. 1037 */ 1038 static struct tracepoint *find_tracepoint_in_module(struct module *mod, 1039 const char *tp_name) 1040 { 1041 struct __find_tracepoint_cb_data data = { 1042 .tp_name = tp_name, 1043 .mod = mod, 1044 }; 1045 1046 for_each_tracepoint_in_module(mod, __find_tracepoint_module_cb, &data); 1047 return data.tpoint; 1048 } 1049 1050 /* These are CONFIG_MODULES=y specific functions. */ 1051 static bool tracepoint_user_within_module(struct tracepoint_user *tuser, 1052 struct module *mod) 1053 { 1054 return within_module(tracepoint_user_ip(tuser), mod); 1055 } 1056 1057 static int tracepoint_user_register_again(struct tracepoint_user *tuser, 1058 struct tracepoint *tpoint) 1059 { 1060 tuser->tpoint = tpoint; 1061 return tracepoint_user_register(tuser); 1062 } 1063 1064 static void tracepoint_user_unregister_clear(struct tracepoint_user *tuser) 1065 { 1066 tracepoint_user_unregister(tuser); 1067 tuser->tpoint = NULL; 1068 } 1069 1070 /* module callback for tracepoint_user */ 1071 static int __tracepoint_probe_module_cb(struct notifier_block *self, 1072 unsigned long val, void *data) 1073 { 1074 struct tp_module *tp_mod = data; 1075 struct tracepoint_user *tuser; 1076 struct tracepoint *tpoint; 1077 1078 if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING) 1079 return NOTIFY_DONE; 1080 1081 mutex_lock(&tracepoint_user_mutex); 1082 for_each_tracepoint_user(tuser) { 1083 if (val == MODULE_STATE_COMING) { 1084 /* This is not a tracepoint in this module. Skip it. */ 1085 tpoint = find_tracepoint_in_module(tp_mod->mod, tuser->name); 1086 if (!tpoint) 1087 continue; 1088 WARN_ON_ONCE(tracepoint_user_register_again(tuser, tpoint)); 1089 } else if (val == MODULE_STATE_GOING && 1090 tracepoint_user_within_module(tuser, tp_mod->mod)) { 1091 /* Unregister all tracepoint_user in this module. */ 1092 tracepoint_user_unregister_clear(tuser); 1093 } 1094 } 1095 mutex_unlock(&tracepoint_user_mutex); 1096 1097 return NOTIFY_DONE; 1098 } 1099 1100 static struct notifier_block tracepoint_module_nb = { 1101 .notifier_call = __tracepoint_probe_module_cb, 1102 }; 1103 1104 /* module callback for tprobe events */ 1105 static int __tprobe_event_module_cb(struct notifier_block *self, 1106 unsigned long val, void *data) 1107 { 1108 struct trace_fprobe *tf; 1109 struct dyn_event *pos; 1110 struct module *mod = data; 1111 1112 if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING) 1113 return NOTIFY_DONE; 1114 1115 mutex_lock(&event_mutex); 1116 for_each_trace_fprobe(tf, pos) { 1117 /* Skip fprobe and disabled tprobe events. */ 1118 if (!trace_fprobe_is_tracepoint(tf) || !tf->tuser) 1119 continue; 1120 1121 /* Before this notification, tracepoint notifier has already done. */ 1122 if (val == MODULE_STATE_COMING && 1123 tracepoint_user_within_module(tf->tuser, mod)) { 1124 unsigned long ip = tracepoint_user_ip(tf->tuser); 1125 1126 WARN_ON_ONCE(register_fprobe_ips(&tf->fp, &ip, 1)); 1127 } else if (val == MODULE_STATE_GOING && 1128 /* 1129 * tracepoint_user_within_module() does not work here because 1130 * tracepoint_user is already unregistered and cleared tpoint. 1131 * Instead, checking whether the fprobe is registered but 1132 * tpoint is cleared(unregistered). Such unbalance probes 1133 * must be adjusted anyway. 1134 */ 1135 trace_fprobe_is_registered(tf) && 1136 !tf->tuser->tpoint) { 1137 unregister_fprobe(&tf->fp); 1138 } 1139 } 1140 mutex_unlock(&event_mutex); 1141 1142 return NOTIFY_DONE; 1143 } 1144 1145 /* NOTE: this must be called after tracepoint callback */ 1146 static struct notifier_block tprobe_event_module_nb = { 1147 .notifier_call = __tprobe_event_module_cb, 1148 /* Make sure this is later than tracepoint module notifier. */ 1149 .priority = -10, 1150 }; 1151 #endif /* CONFIG_MODULES */ 1152 1153 static int parse_symbol_and_return(int argc, const char *argv[], 1154 char **symbol, bool *is_return, 1155 bool is_tracepoint) 1156 { 1157 char *tmp = strchr(argv[1], '%'); 1158 int i; 1159 1160 if (tmp) { 1161 int len = tmp - argv[1]; 1162 1163 if (!is_tracepoint && !strcmp(tmp, "%return")) { 1164 *is_return = true; 1165 } else { 1166 trace_probe_log_err(len, BAD_ADDR_SUFFIX); 1167 return -EINVAL; 1168 } 1169 *symbol = kmemdup_nul(argv[1], len, GFP_KERNEL); 1170 } else 1171 *symbol = kstrdup(argv[1], GFP_KERNEL); 1172 if (!*symbol) 1173 return -ENOMEM; 1174 1175 if (*is_return) 1176 return 0; 1177 1178 if (is_tracepoint) { 1179 tmp = *symbol; 1180 while (*tmp && (isalnum(*tmp) || *tmp == '_')) 1181 tmp++; 1182 if (*tmp) { 1183 /* find a wrong character. */ 1184 trace_probe_log_err(tmp - *symbol, BAD_TP_NAME); 1185 kfree(*symbol); 1186 *symbol = NULL; 1187 return -EINVAL; 1188 } 1189 } 1190 1191 /* If there is $retval, this should be a return fprobe. */ 1192 for (i = 2; i < argc; i++) { 1193 tmp = strstr(argv[i], "$retval"); 1194 if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') { 1195 if (is_tracepoint) { 1196 trace_probe_log_set_index(i); 1197 trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE); 1198 kfree(*symbol); 1199 *symbol = NULL; 1200 return -EINVAL; 1201 } 1202 *is_return = true; 1203 break; 1204 } 1205 } 1206 return 0; 1207 } 1208 1209 static int trace_fprobe_create_internal(int argc, const char *argv[], 1210 struct traceprobe_parse_context *ctx) 1211 { 1212 /* 1213 * Argument syntax: 1214 * - Add fentry probe: 1215 * f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS] 1216 * - Add fexit probe: 1217 * f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS] 1218 * - Add tracepoint probe: 1219 * t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS] 1220 * 1221 * Fetch args: 1222 * $retval : fetch return value 1223 * $stack : fetch stack address 1224 * $stackN : fetch Nth entry of stack (N:0-) 1225 * $argN : fetch Nth argument (N:1-) 1226 * $comm : fetch current task comm 1227 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 1228 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 1229 * Dereferencing memory fetch: 1230 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 1231 * Alias name of args: 1232 * NAME=FETCHARG : set NAME as alias of FETCHARG. 1233 * Type of args: 1234 * FETCHARG:TYPE : use TYPE instead of unsigned long. 1235 */ 1236 struct trace_fprobe *tf __free(free_trace_fprobe) = NULL; 1237 const char *event = NULL, *group = FPROBE_EVENT_SYSTEM; 1238 struct module *mod __free(module_put) = NULL; 1239 const char **new_argv __free(kfree) = NULL; 1240 char *symbol __free(kfree) = NULL; 1241 char *ebuf __free(kfree) = NULL; 1242 char *gbuf __free(kfree) = NULL; 1243 char *sbuf __free(kfree) = NULL; 1244 char *abuf __free(kfree) = NULL; 1245 char *dbuf __free(kfree) = NULL; 1246 int i, new_argc = 0, ret = 0; 1247 bool is_tracepoint = false; 1248 bool is_return = false; 1249 1250 if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2) 1251 return -ECANCELED; 1252 1253 if (argv[0][0] == 't') { 1254 is_tracepoint = true; 1255 group = TRACEPOINT_EVENT_SYSTEM; 1256 } 1257 1258 if (argv[0][1] != '\0') { 1259 if (argv[0][1] != ':') { 1260 trace_probe_log_set_index(0); 1261 trace_probe_log_err(1, BAD_MAXACT); 1262 return -EINVAL; 1263 } 1264 event = &argv[0][2]; 1265 } 1266 1267 trace_probe_log_set_index(1); 1268 1269 /* a symbol(or tracepoint) must be specified */ 1270 ret = parse_symbol_and_return(argc, argv, &symbol, &is_return, is_tracepoint); 1271 if (ret < 0) 1272 return -EINVAL; 1273 1274 trace_probe_log_set_index(0); 1275 if (event) { 1276 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 1277 if (!gbuf) 1278 return -ENOMEM; 1279 ret = traceprobe_parse_event_name(&event, &group, gbuf, 1280 event - argv[0]); 1281 if (ret) 1282 return -EINVAL; 1283 } 1284 1285 if (!event) { 1286 ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL); 1287 if (!ebuf) 1288 return -ENOMEM; 1289 /* Make a new event name */ 1290 if (is_tracepoint) 1291 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%s%s", 1292 isdigit(*symbol) ? "_" : "", symbol); 1293 else 1294 snprintf(ebuf, MAX_EVENT_NAME_LEN, "%s__%s", symbol, 1295 is_return ? "exit" : "entry"); 1296 sanitize_event_name(ebuf); 1297 event = ebuf; 1298 } 1299 1300 if (is_return) 1301 ctx->flags |= TPARG_FL_RETURN; 1302 else 1303 ctx->flags |= TPARG_FL_FENTRY; 1304 1305 ctx->funcname = NULL; 1306 if (is_tracepoint) { 1307 /* Get tracepoint and lock its module until the end of the registration. */ 1308 struct tracepoint *tpoint; 1309 1310 ctx->flags |= TPARG_FL_TPOINT; 1311 mod = NULL; 1312 tpoint = find_tracepoint(symbol, &mod); 1313 if (tpoint) { 1314 sbuf = kmalloc(KSYM_NAME_LEN, GFP_KERNEL); 1315 if (!sbuf) 1316 return -ENOMEM; 1317 ctx->funcname = kallsyms_lookup((unsigned long)tpoint->probestub, 1318 NULL, NULL, NULL, sbuf); 1319 } 1320 } 1321 if (!ctx->funcname) 1322 ctx->funcname = symbol; 1323 1324 abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL); 1325 if (!abuf) 1326 return -ENOMEM; 1327 argc -= 2; argv += 2; 1328 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, 1329 abuf, MAX_BTF_ARGS_LEN, ctx); 1330 if (IS_ERR(new_argv)) 1331 return PTR_ERR(new_argv); 1332 if (new_argv) { 1333 argc = new_argc; 1334 argv = new_argv; 1335 } 1336 if (argc > MAX_TRACE_ARGS) { 1337 trace_probe_log_set_index(2); 1338 trace_probe_log_err(0, TOO_MANY_ARGS); 1339 return -E2BIG; 1340 } 1341 1342 ret = traceprobe_expand_dentry_args(argc, argv, &dbuf); 1343 if (ret) 1344 return ret; 1345 1346 /* setup a probe */ 1347 tf = alloc_trace_fprobe(group, event, symbol, argc, is_return, is_tracepoint); 1348 if (IS_ERR(tf)) { 1349 ret = PTR_ERR(tf); 1350 /* This must return -ENOMEM, else there is a bug */ 1351 WARN_ON_ONCE(ret != -ENOMEM); 1352 return ret; 1353 } 1354 1355 /* parse arguments */ 1356 for (i = 0; i < argc; i++) { 1357 trace_probe_log_set_index(i + 2); 1358 ctx->offset = 0; 1359 ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], ctx); 1360 if (ret) 1361 return ret; /* This can be -ENOMEM */ 1362 } 1363 1364 if (is_return && tf->tp.entry_arg) { 1365 tf->fp.entry_handler = trace_fprobe_entry_handler; 1366 tf->fp.entry_data_size = traceprobe_get_entry_data_size(&tf->tp); 1367 if (ALIGN(tf->fp.entry_data_size, sizeof(long)) > MAX_FPROBE_DATA_SIZE) { 1368 trace_probe_log_set_index(2); 1369 trace_probe_log_err(0, TOO_MANY_EARGS); 1370 return -E2BIG; 1371 } 1372 } 1373 1374 ret = traceprobe_set_print_fmt(&tf->tp, 1375 is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL); 1376 if (ret < 0) 1377 return ret; 1378 1379 ret = register_trace_fprobe_event(tf); 1380 if (ret) { 1381 trace_probe_log_set_index(1); 1382 if (ret == -EILSEQ) 1383 trace_probe_log_err(0, BAD_INSN_BNDRY); 1384 else if (ret == -ENOENT) 1385 trace_probe_log_err(0, BAD_PROBE_ADDR); 1386 else if (ret != -ENOMEM && ret != -EEXIST) 1387 trace_probe_log_err(0, FAIL_REG_PROBE); 1388 return -EINVAL; 1389 } 1390 1391 /* 'tf' is successfully registered. To avoid freeing, assign NULL. */ 1392 tf = NULL; 1393 1394 return 0; 1395 } 1396 1397 static int trace_fprobe_create_cb(int argc, const char *argv[]) 1398 { 1399 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL; 1400 int ret; 1401 1402 ctx = kzalloc_obj(*ctx); 1403 if (!ctx) 1404 return -ENOMEM; 1405 1406 ctx->flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE; 1407 1408 trace_probe_log_init("trace_fprobe", argc, argv); 1409 ret = trace_fprobe_create_internal(argc, argv, ctx); 1410 trace_probe_log_clear(); 1411 return ret; 1412 } 1413 1414 static int trace_fprobe_create(const char *raw_command) 1415 { 1416 return trace_probe_create(raw_command, trace_fprobe_create_cb); 1417 } 1418 1419 static int trace_fprobe_release(struct dyn_event *ev) 1420 { 1421 struct trace_fprobe *tf = to_trace_fprobe(ev); 1422 int ret = unregister_trace_fprobe(tf); 1423 1424 if (!ret) 1425 free_trace_fprobe(tf); 1426 return ret; 1427 } 1428 1429 static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev) 1430 { 1431 struct trace_fprobe *tf = to_trace_fprobe(ev); 1432 int i; 1433 1434 if (trace_fprobe_is_tracepoint(tf)) 1435 seq_putc(m, 't'); 1436 else 1437 seq_putc(m, 'f'); 1438 seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp), 1439 trace_probe_name(&tf->tp)); 1440 1441 seq_printf(m, " %s%s", trace_fprobe_symbol(tf), 1442 trace_fprobe_is_return(tf) ? "%return" : ""); 1443 1444 for (i = 0; i < tf->tp.nr_args; i++) 1445 seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm); 1446 seq_putc(m, '\n'); 1447 1448 trace_probe_dump_args(m, &tf->tp); 1449 1450 return 0; 1451 } 1452 1453 /* 1454 * Enable trace_probe 1455 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 1456 */ 1457 static int enable_trace_fprobe(struct trace_event_call *call, 1458 struct trace_event_file *file) 1459 { 1460 struct trace_probe *tp; 1461 struct trace_fprobe *tf; 1462 bool enabled; 1463 int ret = 0; 1464 1465 tp = trace_probe_primary_from_call(call); 1466 if (WARN_ON_ONCE(!tp)) 1467 return -ENODEV; 1468 enabled = trace_probe_is_enabled(tp); 1469 1470 /* This also changes "enabled" state */ 1471 if (file) { 1472 ret = trace_probe_add_file(tp, file); 1473 if (ret) 1474 return ret; 1475 } else 1476 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 1477 1478 if (!enabled) { 1479 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { 1480 ret = __register_trace_fprobe(tf); 1481 if (ret < 0) 1482 goto err; 1483 } 1484 } 1485 1486 return 0; 1487 1488 err: 1489 /* Failed to enable one of them. Roll back all */ 1490 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) 1491 __unregister_trace_fprobe(tf); 1492 if (file) 1493 trace_probe_remove_file(tp, file); 1494 else 1495 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 1496 return ret; 1497 } 1498 1499 /* 1500 * Disable trace_probe 1501 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 1502 */ 1503 static int disable_trace_fprobe(struct trace_event_call *call, 1504 struct trace_event_file *file) 1505 { 1506 struct trace_fprobe *tf; 1507 struct trace_probe *tp; 1508 1509 tp = trace_probe_primary_from_call(call); 1510 if (WARN_ON_ONCE(!tp)) 1511 return -ENODEV; 1512 1513 if (file) { 1514 if (!trace_probe_get_file_link(tp, file)) 1515 return -ENOENT; 1516 if (!trace_probe_has_single_file(tp)) 1517 goto out; 1518 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 1519 } else 1520 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 1521 1522 if (!trace_probe_is_enabled(tp)) { 1523 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { 1524 unregister_fprobe(&tf->fp); 1525 if (tf->tuser) { 1526 tracepoint_user_put(tf->tuser); 1527 tf->tuser = NULL; 1528 } 1529 } 1530 } 1531 1532 out: 1533 if (file) 1534 /* 1535 * Synchronization is done in below function. For perf event, 1536 * file == NULL and perf_trace_event_unreg() calls 1537 * tracepoint_synchronize_unregister() to ensure synchronize 1538 * event. We don't need to care about it. 1539 */ 1540 trace_probe_remove_file(tp, file); 1541 1542 return 0; 1543 } 1544 1545 /* 1546 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1547 */ 1548 static int fprobe_register(struct trace_event_call *event, 1549 enum trace_reg type, void *data) 1550 { 1551 struct trace_event_file *file = data; 1552 1553 switch (type) { 1554 case TRACE_REG_REGISTER: 1555 return enable_trace_fprobe(event, file); 1556 case TRACE_REG_UNREGISTER: 1557 return disable_trace_fprobe(event, file); 1558 1559 #ifdef CONFIG_PERF_EVENTS 1560 case TRACE_REG_PERF_REGISTER: 1561 return enable_trace_fprobe(event, NULL); 1562 case TRACE_REG_PERF_UNREGISTER: 1563 return disable_trace_fprobe(event, NULL); 1564 case TRACE_REG_PERF_OPEN: 1565 case TRACE_REG_PERF_CLOSE: 1566 case TRACE_REG_PERF_ADD: 1567 case TRACE_REG_PERF_DEL: 1568 return 0; 1569 #endif 1570 } 1571 return 0; 1572 } 1573 1574 /* 1575 * Register dynevent at core_initcall. This allows kernel to setup fprobe 1576 * events in postcore_initcall without tracefs. 1577 */ 1578 static __init int init_fprobe_trace_early(void) 1579 { 1580 int ret; 1581 1582 ret = dyn_event_register(&trace_fprobe_ops); 1583 if (ret) 1584 return ret; 1585 1586 #ifdef CONFIG_MODULES 1587 ret = register_tracepoint_module_notifier(&tracepoint_module_nb); 1588 if (ret) 1589 return ret; 1590 ret = register_module_notifier(&tprobe_event_module_nb); 1591 if (ret) 1592 return ret; 1593 #endif 1594 1595 return 0; 1596 } 1597 core_initcall(init_fprobe_trace_early); 1598