1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Fprobe-based tracing events 4 * Copyright (C) 2022 Google LLC. 5 */ 6 #define pr_fmt(fmt) "trace_fprobe: " fmt 7 8 #include <linux/fprobe.h> 9 #include <linux/module.h> 10 #include <linux/rculist.h> 11 #include <linux/security.h> 12 #include <linux/tracepoint.h> 13 #include <linux/uaccess.h> 14 15 #include "trace_dynevent.h" 16 #include "trace_probe.h" 17 #include "trace_probe_kernel.h" 18 #include "trace_probe_tmpl.h" 19 20 #define FPROBE_EVENT_SYSTEM "fprobes" 21 #define TRACEPOINT_EVENT_SYSTEM "tracepoints" 22 #define RETHOOK_MAXACTIVE_MAX 4096 23 24 static int trace_fprobe_create(const char *raw_command); 25 static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev); 26 static int trace_fprobe_release(struct dyn_event *ev); 27 static bool trace_fprobe_is_busy(struct dyn_event *ev); 28 static bool trace_fprobe_match(const char *system, const char *event, 29 int argc, const char **argv, struct dyn_event *ev); 30 31 static struct dyn_event_operations trace_fprobe_ops = { 32 .create = trace_fprobe_create, 33 .show = trace_fprobe_show, 34 .is_busy = trace_fprobe_is_busy, 35 .free = trace_fprobe_release, 36 .match = trace_fprobe_match, 37 }; 38 39 /* 40 * Fprobe event core functions 41 */ 42 struct trace_fprobe { 43 struct dyn_event devent; 44 struct fprobe fp; 45 const char *symbol; 46 struct tracepoint *tpoint; 47 struct module *mod; 48 struct trace_probe tp; 49 }; 50 51 static bool is_trace_fprobe(struct dyn_event *ev) 52 { 53 return ev->ops == &trace_fprobe_ops; 54 } 55 56 static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev) 57 { 58 return container_of(ev, struct trace_fprobe, devent); 59 } 60 61 /** 62 * for_each_trace_fprobe - iterate over the trace_fprobe list 63 * @pos: the struct trace_fprobe * for each entry 64 * @dpos: the struct dyn_event * to use as a loop cursor 65 */ 66 #define for_each_trace_fprobe(pos, dpos) \ 67 for_each_dyn_event(dpos) \ 68 if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos))) 69 70 static bool trace_fprobe_is_return(struct trace_fprobe *tf) 71 { 72 return tf->fp.exit_handler != NULL; 73 } 74 75 static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf) 76 { 77 return tf->tpoint != NULL; 78 } 79 80 static const char *trace_fprobe_symbol(struct trace_fprobe *tf) 81 { 82 return tf->symbol ? tf->symbol : "unknown"; 83 } 84 85 static bool trace_fprobe_is_busy(struct dyn_event *ev) 86 { 87 struct trace_fprobe *tf = to_trace_fprobe(ev); 88 89 return trace_probe_is_enabled(&tf->tp); 90 } 91 92 static bool trace_fprobe_match_command_head(struct trace_fprobe *tf, 93 int argc, const char **argv) 94 { 95 char buf[MAX_ARGSTR_LEN + 1]; 96 97 if (!argc) 98 return true; 99 100 snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf)); 101 if (strcmp(buf, argv[0])) 102 return false; 103 argc--; argv++; 104 105 return trace_probe_match_command_args(&tf->tp, argc, argv); 106 } 107 108 static bool trace_fprobe_match(const char *system, const char *event, 109 int argc, const char **argv, struct dyn_event *ev) 110 { 111 struct trace_fprobe *tf = to_trace_fprobe(ev); 112 113 if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event)) 114 return false; 115 116 if (system && strcmp(trace_probe_group_name(&tf->tp), system)) 117 return false; 118 119 return trace_fprobe_match_command_head(tf, argc, argv); 120 } 121 122 static bool trace_fprobe_is_registered(struct trace_fprobe *tf) 123 { 124 return fprobe_is_registered(&tf->fp); 125 } 126 127 /* 128 * Note that we don't verify the fetch_insn code, since it does not come 129 * from user space. 130 */ 131 static int 132 process_fetch_insn(struct fetch_insn *code, void *rec, void *dest, 133 void *base) 134 { 135 struct pt_regs *regs = rec; 136 unsigned long val; 137 int ret; 138 139 retry: 140 /* 1st stage: get value from context */ 141 switch (code->op) { 142 case FETCH_OP_STACK: 143 val = regs_get_kernel_stack_nth(regs, code->param); 144 break; 145 case FETCH_OP_STACKP: 146 val = kernel_stack_pointer(regs); 147 break; 148 case FETCH_OP_RETVAL: 149 val = regs_return_value(regs); 150 break; 151 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 152 case FETCH_OP_ARG: 153 val = regs_get_kernel_argument(regs, code->param); 154 break; 155 #endif 156 case FETCH_NOP_SYMBOL: /* Ignore a place holder */ 157 code++; 158 goto retry; 159 default: 160 ret = process_common_fetch_insn(code, &val); 161 if (ret < 0) 162 return ret; 163 } 164 code++; 165 166 return process_fetch_insn_bottom(code, val, dest, base); 167 } 168 NOKPROBE_SYMBOL(process_fetch_insn) 169 170 /* function entry handler */ 171 static nokprobe_inline void 172 __fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 173 struct pt_regs *regs, 174 struct trace_event_file *trace_file) 175 { 176 struct fentry_trace_entry_head *entry; 177 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 178 struct trace_event_buffer fbuffer; 179 int dsize; 180 181 if (WARN_ON_ONCE(call != trace_file->event_call)) 182 return; 183 184 if (trace_trigger_soft_disabled(trace_file)) 185 return; 186 187 dsize = __get_data_size(&tf->tp, regs); 188 189 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 190 sizeof(*entry) + tf->tp.size + dsize); 191 if (!entry) 192 return; 193 194 fbuffer.regs = regs; 195 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); 196 entry->ip = entry_ip; 197 store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize); 198 199 trace_event_buffer_commit(&fbuffer); 200 } 201 202 static void 203 fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 204 struct pt_regs *regs) 205 { 206 struct event_file_link *link; 207 208 trace_probe_for_each_link_rcu(link, &tf->tp) 209 __fentry_trace_func(tf, entry_ip, regs, link->file); 210 } 211 NOKPROBE_SYMBOL(fentry_trace_func); 212 213 /* Kretprobe handler */ 214 static nokprobe_inline void 215 __fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 216 unsigned long ret_ip, struct pt_regs *regs, 217 struct trace_event_file *trace_file) 218 { 219 struct fexit_trace_entry_head *entry; 220 struct trace_event_buffer fbuffer; 221 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 222 int dsize; 223 224 if (WARN_ON_ONCE(call != trace_file->event_call)) 225 return; 226 227 if (trace_trigger_soft_disabled(trace_file)) 228 return; 229 230 dsize = __get_data_size(&tf->tp, regs); 231 232 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 233 sizeof(*entry) + tf->tp.size + dsize); 234 if (!entry) 235 return; 236 237 fbuffer.regs = regs; 238 entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); 239 entry->func = entry_ip; 240 entry->ret_ip = ret_ip; 241 store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize); 242 243 trace_event_buffer_commit(&fbuffer); 244 } 245 246 static void 247 fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip, 248 unsigned long ret_ip, struct pt_regs *regs) 249 { 250 struct event_file_link *link; 251 252 trace_probe_for_each_link_rcu(link, &tf->tp) 253 __fexit_trace_func(tf, entry_ip, ret_ip, regs, link->file); 254 } 255 NOKPROBE_SYMBOL(fexit_trace_func); 256 257 #ifdef CONFIG_PERF_EVENTS 258 259 static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, 260 struct pt_regs *regs) 261 { 262 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 263 struct fentry_trace_entry_head *entry; 264 struct hlist_head *head; 265 int size, __size, dsize; 266 int rctx; 267 268 head = this_cpu_ptr(call->perf_events); 269 if (hlist_empty(head)) 270 return 0; 271 272 dsize = __get_data_size(&tf->tp, regs); 273 __size = sizeof(*entry) + tf->tp.size + dsize; 274 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 275 size -= sizeof(u32); 276 277 entry = perf_trace_buf_alloc(size, NULL, &rctx); 278 if (!entry) 279 return 0; 280 281 entry->ip = entry_ip; 282 memset(&entry[1], 0, dsize); 283 store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize); 284 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 285 head, NULL); 286 return 0; 287 } 288 NOKPROBE_SYMBOL(fentry_perf_func); 289 290 static void 291 fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip, 292 unsigned long ret_ip, struct pt_regs *regs) 293 { 294 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 295 struct fexit_trace_entry_head *entry; 296 struct hlist_head *head; 297 int size, __size, dsize; 298 int rctx; 299 300 head = this_cpu_ptr(call->perf_events); 301 if (hlist_empty(head)) 302 return; 303 304 dsize = __get_data_size(&tf->tp, regs); 305 __size = sizeof(*entry) + tf->tp.size + dsize; 306 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 307 size -= sizeof(u32); 308 309 entry = perf_trace_buf_alloc(size, NULL, &rctx); 310 if (!entry) 311 return; 312 313 entry->func = entry_ip; 314 entry->ret_ip = ret_ip; 315 store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize); 316 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 317 head, NULL); 318 } 319 NOKPROBE_SYMBOL(fexit_perf_func); 320 #endif /* CONFIG_PERF_EVENTS */ 321 322 static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip, 323 unsigned long ret_ip, struct pt_regs *regs, 324 void *entry_data) 325 { 326 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); 327 int ret = 0; 328 329 if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE)) 330 fentry_trace_func(tf, entry_ip, regs); 331 #ifdef CONFIG_PERF_EVENTS 332 if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE)) 333 ret = fentry_perf_func(tf, entry_ip, regs); 334 #endif 335 return ret; 336 } 337 NOKPROBE_SYMBOL(fentry_dispatcher); 338 339 static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip, 340 unsigned long ret_ip, struct pt_regs *regs, 341 void *entry_data) 342 { 343 struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); 344 345 if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE)) 346 fexit_trace_func(tf, entry_ip, ret_ip, regs); 347 #ifdef CONFIG_PERF_EVENTS 348 if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE)) 349 fexit_perf_func(tf, entry_ip, ret_ip, regs); 350 #endif 351 } 352 NOKPROBE_SYMBOL(fexit_dispatcher); 353 354 static void free_trace_fprobe(struct trace_fprobe *tf) 355 { 356 if (tf) { 357 trace_probe_cleanup(&tf->tp); 358 kfree(tf->symbol); 359 kfree(tf); 360 } 361 } 362 363 /* 364 * Allocate new trace_probe and initialize it (including fprobe). 365 */ 366 static struct trace_fprobe *alloc_trace_fprobe(const char *group, 367 const char *event, 368 const char *symbol, 369 struct tracepoint *tpoint, 370 int maxactive, 371 int nargs, bool is_return) 372 { 373 struct trace_fprobe *tf; 374 int ret = -ENOMEM; 375 376 tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL); 377 if (!tf) 378 return ERR_PTR(ret); 379 380 tf->symbol = kstrdup(symbol, GFP_KERNEL); 381 if (!tf->symbol) 382 goto error; 383 384 if (is_return) 385 tf->fp.exit_handler = fexit_dispatcher; 386 else 387 tf->fp.entry_handler = fentry_dispatcher; 388 389 tf->tpoint = tpoint; 390 tf->fp.nr_maxactive = maxactive; 391 392 ret = trace_probe_init(&tf->tp, event, group, false); 393 if (ret < 0) 394 goto error; 395 396 dyn_event_init(&tf->devent, &trace_fprobe_ops); 397 return tf; 398 error: 399 free_trace_fprobe(tf); 400 return ERR_PTR(ret); 401 } 402 403 static struct trace_fprobe *find_trace_fprobe(const char *event, 404 const char *group) 405 { 406 struct dyn_event *pos; 407 struct trace_fprobe *tf; 408 409 for_each_trace_fprobe(tf, pos) 410 if (strcmp(trace_probe_name(&tf->tp), event) == 0 && 411 strcmp(trace_probe_group_name(&tf->tp), group) == 0) 412 return tf; 413 return NULL; 414 } 415 416 static inline int __enable_trace_fprobe(struct trace_fprobe *tf) 417 { 418 if (trace_fprobe_is_registered(tf)) 419 enable_fprobe(&tf->fp); 420 421 return 0; 422 } 423 424 static void __disable_trace_fprobe(struct trace_probe *tp) 425 { 426 struct trace_fprobe *tf; 427 428 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { 429 if (!trace_fprobe_is_registered(tf)) 430 continue; 431 disable_fprobe(&tf->fp); 432 } 433 } 434 435 /* 436 * Enable trace_probe 437 * if the file is NULL, enable "perf" handler, or enable "trace" handler. 438 */ 439 static int enable_trace_fprobe(struct trace_event_call *call, 440 struct trace_event_file *file) 441 { 442 struct trace_probe *tp; 443 struct trace_fprobe *tf; 444 bool enabled; 445 int ret = 0; 446 447 tp = trace_probe_primary_from_call(call); 448 if (WARN_ON_ONCE(!tp)) 449 return -ENODEV; 450 enabled = trace_probe_is_enabled(tp); 451 452 /* This also changes "enabled" state */ 453 if (file) { 454 ret = trace_probe_add_file(tp, file); 455 if (ret) 456 return ret; 457 } else 458 trace_probe_set_flag(tp, TP_FLAG_PROFILE); 459 460 if (!enabled) { 461 list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { 462 /* TODO: check the fprobe is gone */ 463 __enable_trace_fprobe(tf); 464 } 465 } 466 467 return 0; 468 } 469 470 /* 471 * Disable trace_probe 472 * if the file is NULL, disable "perf" handler, or disable "trace" handler. 473 */ 474 static int disable_trace_fprobe(struct trace_event_call *call, 475 struct trace_event_file *file) 476 { 477 struct trace_probe *tp; 478 479 tp = trace_probe_primary_from_call(call); 480 if (WARN_ON_ONCE(!tp)) 481 return -ENODEV; 482 483 if (file) { 484 if (!trace_probe_get_file_link(tp, file)) 485 return -ENOENT; 486 if (!trace_probe_has_single_file(tp)) 487 goto out; 488 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 489 } else 490 trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 491 492 if (!trace_probe_is_enabled(tp)) 493 __disable_trace_fprobe(tp); 494 495 out: 496 if (file) 497 /* 498 * Synchronization is done in below function. For perf event, 499 * file == NULL and perf_trace_event_unreg() calls 500 * tracepoint_synchronize_unregister() to ensure synchronize 501 * event. We don't need to care about it. 502 */ 503 trace_probe_remove_file(tp, file); 504 505 return 0; 506 } 507 508 /* Event entry printers */ 509 static enum print_line_t 510 print_fentry_event(struct trace_iterator *iter, int flags, 511 struct trace_event *event) 512 { 513 struct fentry_trace_entry_head *field; 514 struct trace_seq *s = &iter->seq; 515 struct trace_probe *tp; 516 517 field = (struct fentry_trace_entry_head *)iter->ent; 518 tp = trace_probe_primary_from_call( 519 container_of(event, struct trace_event_call, event)); 520 if (WARN_ON_ONCE(!tp)) 521 goto out; 522 523 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 524 525 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) 526 goto out; 527 528 trace_seq_putc(s, ')'); 529 530 if (trace_probe_print_args(s, tp->args, tp->nr_args, 531 (u8 *)&field[1], field) < 0) 532 goto out; 533 534 trace_seq_putc(s, '\n'); 535 out: 536 return trace_handle_return(s); 537 } 538 539 static enum print_line_t 540 print_fexit_event(struct trace_iterator *iter, int flags, 541 struct trace_event *event) 542 { 543 struct fexit_trace_entry_head *field; 544 struct trace_seq *s = &iter->seq; 545 struct trace_probe *tp; 546 547 field = (struct fexit_trace_entry_head *)iter->ent; 548 tp = trace_probe_primary_from_call( 549 container_of(event, struct trace_event_call, event)); 550 if (WARN_ON_ONCE(!tp)) 551 goto out; 552 553 trace_seq_printf(s, "%s: (", trace_probe_name(tp)); 554 555 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) 556 goto out; 557 558 trace_seq_puts(s, " <- "); 559 560 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) 561 goto out; 562 563 trace_seq_putc(s, ')'); 564 565 if (trace_probe_print_args(s, tp->args, tp->nr_args, 566 (u8 *)&field[1], field) < 0) 567 goto out; 568 569 trace_seq_putc(s, '\n'); 570 571 out: 572 return trace_handle_return(s); 573 } 574 575 static int fentry_event_define_fields(struct trace_event_call *event_call) 576 { 577 int ret; 578 struct fentry_trace_entry_head field; 579 struct trace_probe *tp; 580 581 tp = trace_probe_primary_from_call(event_call); 582 if (WARN_ON_ONCE(!tp)) 583 return -ENOENT; 584 585 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 586 587 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 588 } 589 590 static int fexit_event_define_fields(struct trace_event_call *event_call) 591 { 592 int ret; 593 struct fexit_trace_entry_head field; 594 struct trace_probe *tp; 595 596 tp = trace_probe_primary_from_call(event_call); 597 if (WARN_ON_ONCE(!tp)) 598 return -ENOENT; 599 600 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 601 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 602 603 return traceprobe_define_arg_fields(event_call, sizeof(field), tp); 604 } 605 606 static struct trace_event_functions fentry_funcs = { 607 .trace = print_fentry_event 608 }; 609 610 static struct trace_event_functions fexit_funcs = { 611 .trace = print_fexit_event 612 }; 613 614 static struct trace_event_fields fentry_fields_array[] = { 615 { .type = TRACE_FUNCTION_TYPE, 616 .define_fields = fentry_event_define_fields }, 617 {} 618 }; 619 620 static struct trace_event_fields fexit_fields_array[] = { 621 { .type = TRACE_FUNCTION_TYPE, 622 .define_fields = fexit_event_define_fields }, 623 {} 624 }; 625 626 static int fprobe_register(struct trace_event_call *event, 627 enum trace_reg type, void *data); 628 629 static inline void init_trace_event_call(struct trace_fprobe *tf) 630 { 631 struct trace_event_call *call = trace_probe_event_call(&tf->tp); 632 633 if (trace_fprobe_is_return(tf)) { 634 call->event.funcs = &fexit_funcs; 635 call->class->fields_array = fexit_fields_array; 636 } else { 637 call->event.funcs = &fentry_funcs; 638 call->class->fields_array = fentry_fields_array; 639 } 640 641 call->flags = TRACE_EVENT_FL_FPROBE; 642 call->class->reg = fprobe_register; 643 } 644 645 static int register_fprobe_event(struct trace_fprobe *tf) 646 { 647 init_trace_event_call(tf); 648 649 return trace_probe_register_event_call(&tf->tp); 650 } 651 652 static int unregister_fprobe_event(struct trace_fprobe *tf) 653 { 654 return trace_probe_unregister_event_call(&tf->tp); 655 } 656 657 /* Internal register function - just handle fprobe and flags */ 658 static int __register_trace_fprobe(struct trace_fprobe *tf) 659 { 660 int i, ret; 661 662 /* Should we need new LOCKDOWN flag for fprobe? */ 663 ret = security_locked_down(LOCKDOWN_KPROBES); 664 if (ret) 665 return ret; 666 667 if (trace_fprobe_is_registered(tf)) 668 return -EINVAL; 669 670 for (i = 0; i < tf->tp.nr_args; i++) { 671 ret = traceprobe_update_arg(&tf->tp.args[i]); 672 if (ret) 673 return ret; 674 } 675 676 /* Set/clear disabled flag according to tp->flag */ 677 if (trace_probe_is_enabled(&tf->tp)) 678 tf->fp.flags &= ~FPROBE_FL_DISABLED; 679 else 680 tf->fp.flags |= FPROBE_FL_DISABLED; 681 682 if (trace_fprobe_is_tracepoint(tf)) { 683 struct tracepoint *tpoint = tf->tpoint; 684 unsigned long ip = (unsigned long)tpoint->probestub; 685 /* 686 * Here, we do 2 steps to enable fprobe on a tracepoint. 687 * At first, put __probestub_##TP function on the tracepoint 688 * and put a fprobe on the stub function. 689 */ 690 ret = tracepoint_probe_register_prio_may_exist(tpoint, 691 tpoint->probestub, NULL, 0); 692 if (ret < 0) 693 return ret; 694 return register_fprobe_ips(&tf->fp, &ip, 1); 695 } 696 697 /* TODO: handle filter, nofilter or symbol list */ 698 return register_fprobe(&tf->fp, tf->symbol, NULL); 699 } 700 701 /* Internal unregister function - just handle fprobe and flags */ 702 static void __unregister_trace_fprobe(struct trace_fprobe *tf) 703 { 704 if (trace_fprobe_is_registered(tf)) { 705 unregister_fprobe(&tf->fp); 706 memset(&tf->fp, 0, sizeof(tf->fp)); 707 if (trace_fprobe_is_tracepoint(tf)) { 708 tracepoint_probe_unregister(tf->tpoint, 709 tf->tpoint->probestub, NULL); 710 tf->tpoint = NULL; 711 tf->mod = NULL; 712 } 713 } 714 } 715 716 /* TODO: make this trace_*probe common function */ 717 /* Unregister a trace_probe and probe_event */ 718 static int unregister_trace_fprobe(struct trace_fprobe *tf) 719 { 720 /* If other probes are on the event, just unregister fprobe */ 721 if (trace_probe_has_sibling(&tf->tp)) 722 goto unreg; 723 724 /* Enabled event can not be unregistered */ 725 if (trace_probe_is_enabled(&tf->tp)) 726 return -EBUSY; 727 728 /* If there's a reference to the dynamic event */ 729 if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp))) 730 return -EBUSY; 731 732 /* Will fail if probe is being used by ftrace or perf */ 733 if (unregister_fprobe_event(tf)) 734 return -EBUSY; 735 736 unreg: 737 __unregister_trace_fprobe(tf); 738 dyn_event_remove(&tf->devent); 739 trace_probe_unlink(&tf->tp); 740 741 return 0; 742 } 743 744 static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig, 745 struct trace_fprobe *comp) 746 { 747 struct trace_probe_event *tpe = orig->tp.event; 748 int i; 749 750 list_for_each_entry(orig, &tpe->probes, tp.list) { 751 if (strcmp(trace_fprobe_symbol(orig), 752 trace_fprobe_symbol(comp))) 753 continue; 754 755 /* 756 * trace_probe_compare_arg_type() ensured that nr_args and 757 * each argument name and type are same. Let's compare comm. 758 */ 759 for (i = 0; i < orig->tp.nr_args; i++) { 760 if (strcmp(orig->tp.args[i].comm, 761 comp->tp.args[i].comm)) 762 break; 763 } 764 765 if (i == orig->tp.nr_args) 766 return true; 767 } 768 769 return false; 770 } 771 772 static int append_trace_fprobe(struct trace_fprobe *tf, struct trace_fprobe *to) 773 { 774 int ret; 775 776 if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) || 777 trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) { 778 trace_probe_log_set_index(0); 779 trace_probe_log_err(0, DIFF_PROBE_TYPE); 780 return -EEXIST; 781 } 782 ret = trace_probe_compare_arg_type(&tf->tp, &to->tp); 783 if (ret) { 784 /* Note that argument starts index = 2 */ 785 trace_probe_log_set_index(ret + 1); 786 trace_probe_log_err(0, DIFF_ARG_TYPE); 787 return -EEXIST; 788 } 789 if (trace_fprobe_has_same_fprobe(to, tf)) { 790 trace_probe_log_set_index(0); 791 trace_probe_log_err(0, SAME_PROBE); 792 return -EEXIST; 793 } 794 795 /* Append to existing event */ 796 ret = trace_probe_append(&tf->tp, &to->tp); 797 if (ret) 798 return ret; 799 800 ret = __register_trace_fprobe(tf); 801 if (ret) 802 trace_probe_unlink(&tf->tp); 803 else 804 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp)); 805 806 return ret; 807 } 808 809 /* Register a trace_probe and probe_event */ 810 static int register_trace_fprobe(struct trace_fprobe *tf) 811 { 812 struct trace_fprobe *old_tf; 813 int ret; 814 815 mutex_lock(&event_mutex); 816 817 old_tf = find_trace_fprobe(trace_probe_name(&tf->tp), 818 trace_probe_group_name(&tf->tp)); 819 if (old_tf) { 820 ret = append_trace_fprobe(tf, old_tf); 821 goto end; 822 } 823 824 /* Register new event */ 825 ret = register_fprobe_event(tf); 826 if (ret) { 827 if (ret == -EEXIST) { 828 trace_probe_log_set_index(0); 829 trace_probe_log_err(0, EVENT_EXIST); 830 } else 831 pr_warn("Failed to register probe event(%d)\n", ret); 832 goto end; 833 } 834 835 /* Register fprobe */ 836 ret = __register_trace_fprobe(tf); 837 if (ret < 0) 838 unregister_fprobe_event(tf); 839 else 840 dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp)); 841 842 end: 843 mutex_unlock(&event_mutex); 844 return ret; 845 } 846 847 #ifdef CONFIG_MODULES 848 static int __tracepoint_probe_module_cb(struct notifier_block *self, 849 unsigned long val, void *data) 850 { 851 struct tp_module *tp_mod = data; 852 struct trace_fprobe *tf; 853 struct dyn_event *pos; 854 855 if (val != MODULE_STATE_GOING) 856 return NOTIFY_DONE; 857 858 mutex_lock(&event_mutex); 859 for_each_trace_fprobe(tf, pos) { 860 if (tp_mod->mod == tf->mod) { 861 tracepoint_probe_unregister(tf->tpoint, 862 tf->tpoint->probestub, NULL); 863 tf->tpoint = NULL; 864 tf->mod = NULL; 865 } 866 } 867 mutex_unlock(&event_mutex); 868 869 return NOTIFY_DONE; 870 } 871 872 static struct notifier_block tracepoint_module_nb = { 873 .notifier_call = __tracepoint_probe_module_cb, 874 }; 875 #endif /* CONFIG_MODULES */ 876 877 struct __find_tracepoint_cb_data { 878 const char *tp_name; 879 struct tracepoint *tpoint; 880 }; 881 882 static void __find_tracepoint_cb(struct tracepoint *tp, void *priv) 883 { 884 struct __find_tracepoint_cb_data *data = priv; 885 886 if (!data->tpoint && !strcmp(data->tp_name, tp->name)) 887 data->tpoint = tp; 888 } 889 890 static struct tracepoint *find_tracepoint(const char *tp_name) 891 { 892 struct __find_tracepoint_cb_data data = { 893 .tp_name = tp_name, 894 }; 895 896 for_each_kernel_tracepoint(__find_tracepoint_cb, &data); 897 898 return data.tpoint; 899 } 900 901 static int parse_symbol_and_return(int argc, const char *argv[], 902 char **symbol, bool *is_return, 903 bool is_tracepoint) 904 { 905 char *tmp = strchr(argv[1], '%'); 906 int i; 907 908 if (tmp) { 909 int len = tmp - argv[1]; 910 911 if (!is_tracepoint && !strcmp(tmp, "%return")) { 912 *is_return = true; 913 } else { 914 trace_probe_log_err(len, BAD_ADDR_SUFFIX); 915 return -EINVAL; 916 } 917 *symbol = kmemdup_nul(argv[1], len, GFP_KERNEL); 918 } else 919 *symbol = kstrdup(argv[1], GFP_KERNEL); 920 if (!*symbol) 921 return -ENOMEM; 922 923 if (*is_return) 924 return 0; 925 926 /* If there is $retval, this should be a return fprobe. */ 927 for (i = 2; i < argc; i++) { 928 tmp = strstr(argv[i], "$retval"); 929 if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') { 930 if (is_tracepoint) { 931 trace_probe_log_set_index(i); 932 trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE); 933 return -EINVAL; 934 } 935 *is_return = true; 936 break; 937 } 938 } 939 return 0; 940 } 941 942 static int __trace_fprobe_create(int argc, const char *argv[]) 943 { 944 /* 945 * Argument syntax: 946 * - Add fentry probe: 947 * f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS] 948 * - Add fexit probe: 949 * f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS] 950 * - Add tracepoint probe: 951 * t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS] 952 * 953 * Fetch args: 954 * $retval : fetch return value 955 * $stack : fetch stack address 956 * $stackN : fetch Nth entry of stack (N:0-) 957 * $argN : fetch Nth argument (N:1-) 958 * $comm : fetch current task comm 959 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 960 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 961 * Dereferencing memory fetch: 962 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 963 * Alias name of args: 964 * NAME=FETCHARG : set NAME as alias of FETCHARG. 965 * Type of args: 966 * FETCHARG:TYPE : use TYPE instead of unsigned long. 967 */ 968 struct trace_fprobe *tf = NULL; 969 int i, len, new_argc = 0, ret = 0; 970 bool is_return = false; 971 char *symbol = NULL; 972 const char *event = NULL, *group = FPROBE_EVENT_SYSTEM; 973 const char **new_argv = NULL; 974 int maxactive = 0; 975 char buf[MAX_EVENT_NAME_LEN]; 976 char gbuf[MAX_EVENT_NAME_LEN]; 977 char sbuf[KSYM_NAME_LEN]; 978 char abuf[MAX_BTF_ARGS_LEN]; 979 bool is_tracepoint = false; 980 struct tracepoint *tpoint = NULL; 981 struct traceprobe_parse_context ctx = { 982 .flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE, 983 }; 984 985 if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2) 986 return -ECANCELED; 987 988 if (argv[0][0] == 't') { 989 is_tracepoint = true; 990 group = TRACEPOINT_EVENT_SYSTEM; 991 } 992 993 trace_probe_log_init("trace_fprobe", argc, argv); 994 995 event = strchr(&argv[0][1], ':'); 996 if (event) 997 event++; 998 999 if (isdigit(argv[0][1])) { 1000 if (event) 1001 len = event - &argv[0][1] - 1; 1002 else 1003 len = strlen(&argv[0][1]); 1004 if (len > MAX_EVENT_NAME_LEN - 1) { 1005 trace_probe_log_err(1, BAD_MAXACT); 1006 goto parse_error; 1007 } 1008 memcpy(buf, &argv[0][1], len); 1009 buf[len] = '\0'; 1010 ret = kstrtouint(buf, 0, &maxactive); 1011 if (ret || !maxactive) { 1012 trace_probe_log_err(1, BAD_MAXACT); 1013 goto parse_error; 1014 } 1015 /* fprobe rethook instances are iterated over via a list. The 1016 * maximum should stay reasonable. 1017 */ 1018 if (maxactive > RETHOOK_MAXACTIVE_MAX) { 1019 trace_probe_log_err(1, MAXACT_TOO_BIG); 1020 goto parse_error; 1021 } 1022 } 1023 1024 trace_probe_log_set_index(1); 1025 1026 /* a symbol(or tracepoint) must be specified */ 1027 ret = parse_symbol_and_return(argc, argv, &symbol, &is_return, is_tracepoint); 1028 if (ret < 0) 1029 goto parse_error; 1030 1031 if (!is_return && maxactive) { 1032 trace_probe_log_set_index(0); 1033 trace_probe_log_err(1, BAD_MAXACT_TYPE); 1034 goto parse_error; 1035 } 1036 1037 trace_probe_log_set_index(0); 1038 if (event) { 1039 ret = traceprobe_parse_event_name(&event, &group, gbuf, 1040 event - argv[0]); 1041 if (ret) 1042 goto parse_error; 1043 } 1044 1045 if (!event) { 1046 /* Make a new event name */ 1047 if (is_tracepoint) 1048 snprintf(buf, MAX_EVENT_NAME_LEN, "%s%s", 1049 isdigit(*symbol) ? "_" : "", symbol); 1050 else 1051 snprintf(buf, MAX_EVENT_NAME_LEN, "%s__%s", symbol, 1052 is_return ? "exit" : "entry"); 1053 sanitize_event_name(buf); 1054 event = buf; 1055 } 1056 1057 if (is_return) 1058 ctx.flags |= TPARG_FL_RETURN; 1059 else 1060 ctx.flags |= TPARG_FL_FENTRY; 1061 1062 if (is_tracepoint) { 1063 ctx.flags |= TPARG_FL_TPOINT; 1064 tpoint = find_tracepoint(symbol); 1065 if (!tpoint) { 1066 trace_probe_log_set_index(1); 1067 trace_probe_log_err(0, NO_TRACEPOINT); 1068 goto parse_error; 1069 } 1070 ctx.funcname = kallsyms_lookup( 1071 (unsigned long)tpoint->probestub, 1072 NULL, NULL, NULL, sbuf); 1073 } else 1074 ctx.funcname = symbol; 1075 1076 argc -= 2; argv += 2; 1077 new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc, 1078 abuf, MAX_BTF_ARGS_LEN, &ctx); 1079 if (IS_ERR(new_argv)) { 1080 ret = PTR_ERR(new_argv); 1081 new_argv = NULL; 1082 goto out; 1083 } 1084 if (new_argv) { 1085 argc = new_argc; 1086 argv = new_argv; 1087 } 1088 1089 /* setup a probe */ 1090 tf = alloc_trace_fprobe(group, event, symbol, tpoint, maxactive, 1091 argc, is_return); 1092 if (IS_ERR(tf)) { 1093 ret = PTR_ERR(tf); 1094 /* This must return -ENOMEM, else there is a bug */ 1095 WARN_ON_ONCE(ret != -ENOMEM); 1096 goto out; /* We know tf is not allocated */ 1097 } 1098 1099 if (is_tracepoint) 1100 tf->mod = __module_text_address( 1101 (unsigned long)tf->tpoint->probestub); 1102 1103 /* parse arguments */ 1104 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { 1105 trace_probe_log_set_index(i + 2); 1106 ctx.offset = 0; 1107 ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], &ctx); 1108 if (ret) 1109 goto error; /* This can be -ENOMEM */ 1110 } 1111 1112 ret = traceprobe_set_print_fmt(&tf->tp, 1113 is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL); 1114 if (ret < 0) 1115 goto error; 1116 1117 ret = register_trace_fprobe(tf); 1118 if (ret) { 1119 trace_probe_log_set_index(1); 1120 if (ret == -EILSEQ) 1121 trace_probe_log_err(0, BAD_INSN_BNDRY); 1122 else if (ret == -ENOENT) 1123 trace_probe_log_err(0, BAD_PROBE_ADDR); 1124 else if (ret != -ENOMEM && ret != -EEXIST) 1125 trace_probe_log_err(0, FAIL_REG_PROBE); 1126 goto error; 1127 } 1128 1129 out: 1130 traceprobe_finish_parse(&ctx); 1131 trace_probe_log_clear(); 1132 kfree(new_argv); 1133 kfree(symbol); 1134 return ret; 1135 1136 parse_error: 1137 ret = -EINVAL; 1138 error: 1139 free_trace_fprobe(tf); 1140 goto out; 1141 } 1142 1143 static int trace_fprobe_create(const char *raw_command) 1144 { 1145 return trace_probe_create(raw_command, __trace_fprobe_create); 1146 } 1147 1148 static int trace_fprobe_release(struct dyn_event *ev) 1149 { 1150 struct trace_fprobe *tf = to_trace_fprobe(ev); 1151 int ret = unregister_trace_fprobe(tf); 1152 1153 if (!ret) 1154 free_trace_fprobe(tf); 1155 return ret; 1156 } 1157 1158 static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev) 1159 { 1160 struct trace_fprobe *tf = to_trace_fprobe(ev); 1161 int i; 1162 1163 if (trace_fprobe_is_tracepoint(tf)) 1164 seq_putc(m, 't'); 1165 else 1166 seq_putc(m, 'f'); 1167 if (trace_fprobe_is_return(tf) && tf->fp.nr_maxactive) 1168 seq_printf(m, "%d", tf->fp.nr_maxactive); 1169 seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp), 1170 trace_probe_name(&tf->tp)); 1171 1172 seq_printf(m, " %s%s", trace_fprobe_symbol(tf), 1173 trace_fprobe_is_return(tf) ? "%return" : ""); 1174 1175 for (i = 0; i < tf->tp.nr_args; i++) 1176 seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm); 1177 seq_putc(m, '\n'); 1178 1179 return 0; 1180 } 1181 1182 /* 1183 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. 1184 */ 1185 static int fprobe_register(struct trace_event_call *event, 1186 enum trace_reg type, void *data) 1187 { 1188 struct trace_event_file *file = data; 1189 1190 switch (type) { 1191 case TRACE_REG_REGISTER: 1192 return enable_trace_fprobe(event, file); 1193 case TRACE_REG_UNREGISTER: 1194 return disable_trace_fprobe(event, file); 1195 1196 #ifdef CONFIG_PERF_EVENTS 1197 case TRACE_REG_PERF_REGISTER: 1198 return enable_trace_fprobe(event, NULL); 1199 case TRACE_REG_PERF_UNREGISTER: 1200 return disable_trace_fprobe(event, NULL); 1201 case TRACE_REG_PERF_OPEN: 1202 case TRACE_REG_PERF_CLOSE: 1203 case TRACE_REG_PERF_ADD: 1204 case TRACE_REG_PERF_DEL: 1205 return 0; 1206 #endif 1207 } 1208 return 0; 1209 } 1210 1211 /* 1212 * Register dynevent at core_initcall. This allows kernel to setup fprobe 1213 * events in postcore_initcall without tracefs. 1214 */ 1215 static __init int init_fprobe_trace_early(void) 1216 { 1217 int ret; 1218 1219 ret = dyn_event_register(&trace_fprobe_ops); 1220 if (ret) 1221 return ret; 1222 1223 #ifdef CONFIG_MODULES 1224 ret = register_tracepoint_module_notifier(&tracepoint_module_nb); 1225 if (ret) 1226 return ret; 1227 #endif 1228 1229 return 0; 1230 } 1231 core_initcall(init_fprobe_trace_early); 1232