1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * ring buffer based function tracer 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Based on code from the latency_tracer, that is: 9 * 10 * Copyright (C) 2004-2006 Ingo Molnar 11 * Copyright (C) 2004 Nadia Yvette Chambers 12 */ 13 #include <linux/ring_buffer.h> 14 #include <linux/debugfs.h> 15 #include <linux/uaccess.h> 16 #include <linux/ftrace.h> 17 #include <linux/slab.h> 18 #include <linux/fs.h> 19 20 #include "trace.h" 21 22 static void tracing_start_function_trace(struct trace_array *tr); 23 static void tracing_stop_function_trace(struct trace_array *tr); 24 static void 25 function_trace_call(unsigned long ip, unsigned long parent_ip, 26 struct ftrace_ops *op, struct ftrace_regs *fregs); 27 static void 28 function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 29 struct ftrace_ops *op, struct ftrace_regs *fregs); 30 static struct tracer_flags func_flags; 31 32 /* Our option */ 33 enum { 34 TRACE_FUNC_OPT_STACK = 0x1, 35 }; 36 37 int ftrace_allocate_ftrace_ops(struct trace_array *tr) 38 { 39 struct ftrace_ops *ops; 40 41 /* The top level array uses the "global_ops" */ 42 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 43 return 0; 44 45 ops = kzalloc(sizeof(*ops), GFP_KERNEL); 46 if (!ops) 47 return -ENOMEM; 48 49 /* Currently only the non stack version is supported */ 50 ops->func = function_trace_call; 51 ops->flags = FTRACE_OPS_FL_PID; 52 53 tr->ops = ops; 54 ops->private = tr; 55 56 return 0; 57 } 58 59 void ftrace_free_ftrace_ops(struct trace_array *tr) 60 { 61 kfree(tr->ops); 62 tr->ops = NULL; 63 } 64 65 int ftrace_create_function_files(struct trace_array *tr, 66 struct dentry *parent) 67 { 68 /* 69 * The top level array uses the "global_ops", and the files are 70 * created on boot up. 71 */ 72 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 73 return 0; 74 75 if (!tr->ops) 76 return -EINVAL; 77 78 ftrace_create_filter_files(tr->ops, parent); 79 80 return 0; 81 } 82 83 void ftrace_destroy_function_files(struct trace_array *tr) 84 { 85 ftrace_destroy_filter_files(tr->ops); 86 ftrace_free_ftrace_ops(tr); 87 } 88 89 static int function_trace_init(struct trace_array *tr) 90 { 91 ftrace_func_t func; 92 /* 93 * Instance trace_arrays get their ops allocated 94 * at instance creation. Unless it failed 95 * the allocation. 96 */ 97 if (!tr->ops) 98 return -ENOMEM; 99 100 /* Currently only the global instance can do stack tracing */ 101 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && 102 func_flags.val & TRACE_FUNC_OPT_STACK) 103 func = function_stack_trace_call; 104 else 105 func = function_trace_call; 106 107 ftrace_init_array_ops(tr, func); 108 109 tr->array_buffer.cpu = raw_smp_processor_id(); 110 111 tracing_start_cmdline_record(); 112 tracing_start_function_trace(tr); 113 return 0; 114 } 115 116 static void function_trace_reset(struct trace_array *tr) 117 { 118 tracing_stop_function_trace(tr); 119 tracing_stop_cmdline_record(); 120 ftrace_reset_array_ops(tr); 121 } 122 123 static void function_trace_start(struct trace_array *tr) 124 { 125 tracing_reset_online_cpus(&tr->array_buffer); 126 } 127 128 static void 129 function_trace_call(unsigned long ip, unsigned long parent_ip, 130 struct ftrace_ops *op, struct ftrace_regs *fregs) 131 { 132 struct trace_array *tr = op->private; 133 struct trace_array_cpu *data; 134 unsigned int trace_ctx; 135 int bit; 136 int cpu; 137 138 if (unlikely(!tr->function_enabled)) 139 return; 140 141 bit = ftrace_test_recursion_trylock(ip, parent_ip); 142 if (bit < 0) 143 return; 144 145 trace_ctx = tracing_gen_ctx(); 146 preempt_disable_notrace(); 147 148 cpu = smp_processor_id(); 149 data = per_cpu_ptr(tr->array_buffer.data, cpu); 150 if (!atomic_read(&data->disabled)) 151 trace_function(tr, ip, parent_ip, trace_ctx); 152 153 ftrace_test_recursion_unlock(bit); 154 preempt_enable_notrace(); 155 } 156 157 #ifdef CONFIG_UNWINDER_ORC 158 /* 159 * Skip 2: 160 * 161 * function_stack_trace_call() 162 * ftrace_call() 163 */ 164 #define STACK_SKIP 2 165 #else 166 /* 167 * Skip 3: 168 * __trace_stack() 169 * function_stack_trace_call() 170 * ftrace_call() 171 */ 172 #define STACK_SKIP 3 173 #endif 174 175 static void 176 function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 177 struct ftrace_ops *op, struct ftrace_regs *fregs) 178 { 179 struct trace_array *tr = op->private; 180 struct trace_array_cpu *data; 181 unsigned long flags; 182 long disabled; 183 int cpu; 184 unsigned int trace_ctx; 185 186 if (unlikely(!tr->function_enabled)) 187 return; 188 189 /* 190 * Need to use raw, since this must be called before the 191 * recursive protection is performed. 192 */ 193 local_irq_save(flags); 194 cpu = raw_smp_processor_id(); 195 data = per_cpu_ptr(tr->array_buffer.data, cpu); 196 disabled = atomic_inc_return(&data->disabled); 197 198 if (likely(disabled == 1)) { 199 trace_ctx = tracing_gen_ctx_flags(flags); 200 trace_function(tr, ip, parent_ip, trace_ctx); 201 __trace_stack(tr, trace_ctx, STACK_SKIP); 202 } 203 204 atomic_dec(&data->disabled); 205 local_irq_restore(flags); 206 } 207 208 static struct tracer_opt func_opts[] = { 209 #ifdef CONFIG_STACKTRACE 210 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) }, 211 #endif 212 { } /* Always set a last empty entry */ 213 }; 214 215 static struct tracer_flags func_flags = { 216 .val = 0, /* By default: all flags disabled */ 217 .opts = func_opts 218 }; 219 220 static void tracing_start_function_trace(struct trace_array *tr) 221 { 222 tr->function_enabled = 0; 223 register_ftrace_function(tr->ops); 224 tr->function_enabled = 1; 225 } 226 227 static void tracing_stop_function_trace(struct trace_array *tr) 228 { 229 tr->function_enabled = 0; 230 unregister_ftrace_function(tr->ops); 231 } 232 233 static struct tracer function_trace; 234 235 static int 236 func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 237 { 238 switch (bit) { 239 case TRACE_FUNC_OPT_STACK: 240 /* do nothing if already set */ 241 if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK)) 242 break; 243 244 /* We can change this flag when not running. */ 245 if (tr->current_trace != &function_trace) 246 break; 247 248 unregister_ftrace_function(tr->ops); 249 250 if (set) { 251 tr->ops->func = function_stack_trace_call; 252 register_ftrace_function(tr->ops); 253 } else { 254 tr->ops->func = function_trace_call; 255 register_ftrace_function(tr->ops); 256 } 257 258 break; 259 default: 260 return -EINVAL; 261 } 262 263 return 0; 264 } 265 266 static struct tracer function_trace __tracer_data = 267 { 268 .name = "function", 269 .init = function_trace_init, 270 .reset = function_trace_reset, 271 .start = function_trace_start, 272 .flags = &func_flags, 273 .set_flag = func_set_flag, 274 .allow_instances = true, 275 #ifdef CONFIG_FTRACE_SELFTEST 276 .selftest = trace_selftest_startup_function, 277 #endif 278 }; 279 280 #ifdef CONFIG_DYNAMIC_FTRACE 281 static void update_traceon_count(struct ftrace_probe_ops *ops, 282 unsigned long ip, 283 struct trace_array *tr, bool on, 284 void *data) 285 { 286 struct ftrace_func_mapper *mapper = data; 287 long *count; 288 long old_count; 289 290 /* 291 * Tracing gets disabled (or enabled) once per count. 292 * This function can be called at the same time on multiple CPUs. 293 * It is fine if both disable (or enable) tracing, as disabling 294 * (or enabling) the second time doesn't do anything as the 295 * state of the tracer is already disabled (or enabled). 296 * What needs to be synchronized in this case is that the count 297 * only gets decremented once, even if the tracer is disabled 298 * (or enabled) twice, as the second one is really a nop. 299 * 300 * The memory barriers guarantee that we only decrement the 301 * counter once. First the count is read to a local variable 302 * and a read barrier is used to make sure that it is loaded 303 * before checking if the tracer is in the state we want. 304 * If the tracer is not in the state we want, then the count 305 * is guaranteed to be the old count. 306 * 307 * Next the tracer is set to the state we want (disabled or enabled) 308 * then a write memory barrier is used to make sure that 309 * the new state is visible before changing the counter by 310 * one minus the old counter. This guarantees that another CPU 311 * executing this code will see the new state before seeing 312 * the new counter value, and would not do anything if the new 313 * counter is seen. 314 * 315 * Note, there is no synchronization between this and a user 316 * setting the tracing_on file. But we currently don't care 317 * about that. 318 */ 319 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 320 old_count = *count; 321 322 if (old_count <= 0) 323 return; 324 325 /* Make sure we see count before checking tracing state */ 326 smp_rmb(); 327 328 if (on == !!tracer_tracing_is_on(tr)) 329 return; 330 331 if (on) 332 tracer_tracing_on(tr); 333 else 334 tracer_tracing_off(tr); 335 336 /* Make sure tracing state is visible before updating count */ 337 smp_wmb(); 338 339 *count = old_count - 1; 340 } 341 342 static void 343 ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, 344 struct trace_array *tr, struct ftrace_probe_ops *ops, 345 void *data) 346 { 347 update_traceon_count(ops, ip, tr, 1, data); 348 } 349 350 static void 351 ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, 352 struct trace_array *tr, struct ftrace_probe_ops *ops, 353 void *data) 354 { 355 update_traceon_count(ops, ip, tr, 0, data); 356 } 357 358 static void 359 ftrace_traceon(unsigned long ip, unsigned long parent_ip, 360 struct trace_array *tr, struct ftrace_probe_ops *ops, 361 void *data) 362 { 363 if (tracer_tracing_is_on(tr)) 364 return; 365 366 tracer_tracing_on(tr); 367 } 368 369 static void 370 ftrace_traceoff(unsigned long ip, unsigned long parent_ip, 371 struct trace_array *tr, struct ftrace_probe_ops *ops, 372 void *data) 373 { 374 if (!tracer_tracing_is_on(tr)) 375 return; 376 377 tracer_tracing_off(tr); 378 } 379 380 #ifdef CONFIG_UNWINDER_ORC 381 /* 382 * Skip 3: 383 * 384 * function_trace_probe_call() 385 * ftrace_ops_assist_func() 386 * ftrace_call() 387 */ 388 #define FTRACE_STACK_SKIP 3 389 #else 390 /* 391 * Skip 5: 392 * 393 * __trace_stack() 394 * ftrace_stacktrace() 395 * function_trace_probe_call() 396 * ftrace_ops_assist_func() 397 * ftrace_call() 398 */ 399 #define FTRACE_STACK_SKIP 5 400 #endif 401 402 static __always_inline void trace_stack(struct trace_array *tr) 403 { 404 unsigned int trace_ctx; 405 406 trace_ctx = tracing_gen_ctx(); 407 408 __trace_stack(tr, trace_ctx, FTRACE_STACK_SKIP); 409 } 410 411 static void 412 ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, 413 struct trace_array *tr, struct ftrace_probe_ops *ops, 414 void *data) 415 { 416 trace_stack(tr); 417 } 418 419 static void 420 ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, 421 struct trace_array *tr, struct ftrace_probe_ops *ops, 422 void *data) 423 { 424 struct ftrace_func_mapper *mapper = data; 425 long *count; 426 long old_count; 427 long new_count; 428 429 if (!tracing_is_on()) 430 return; 431 432 /* unlimited? */ 433 if (!mapper) { 434 trace_stack(tr); 435 return; 436 } 437 438 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 439 440 /* 441 * Stack traces should only execute the number of times the 442 * user specified in the counter. 443 */ 444 do { 445 old_count = *count; 446 447 if (!old_count) 448 return; 449 450 new_count = old_count - 1; 451 new_count = cmpxchg(count, old_count, new_count); 452 if (new_count == old_count) 453 trace_stack(tr); 454 455 if (!tracing_is_on()) 456 return; 457 458 } while (new_count != old_count); 459 } 460 461 static int update_count(struct ftrace_probe_ops *ops, unsigned long ip, 462 void *data) 463 { 464 struct ftrace_func_mapper *mapper = data; 465 long *count = NULL; 466 467 if (mapper) 468 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 469 470 if (count) { 471 if (*count <= 0) 472 return 0; 473 (*count)--; 474 } 475 476 return 1; 477 } 478 479 static void 480 ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, 481 struct trace_array *tr, struct ftrace_probe_ops *ops, 482 void *data) 483 { 484 if (update_count(ops, ip, data)) 485 ftrace_dump(DUMP_ALL); 486 } 487 488 /* Only dump the current CPU buffer. */ 489 static void 490 ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, 491 struct trace_array *tr, struct ftrace_probe_ops *ops, 492 void *data) 493 { 494 if (update_count(ops, ip, data)) 495 ftrace_dump(DUMP_ORIG); 496 } 497 498 static int 499 ftrace_probe_print(const char *name, struct seq_file *m, 500 unsigned long ip, struct ftrace_probe_ops *ops, 501 void *data) 502 { 503 struct ftrace_func_mapper *mapper = data; 504 long *count = NULL; 505 506 seq_printf(m, "%ps:%s", (void *)ip, name); 507 508 if (mapper) 509 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 510 511 if (count) 512 seq_printf(m, ":count=%ld\n", *count); 513 else 514 seq_puts(m, ":unlimited\n"); 515 516 return 0; 517 } 518 519 static int 520 ftrace_traceon_print(struct seq_file *m, unsigned long ip, 521 struct ftrace_probe_ops *ops, 522 void *data) 523 { 524 return ftrace_probe_print("traceon", m, ip, ops, data); 525 } 526 527 static int 528 ftrace_traceoff_print(struct seq_file *m, unsigned long ip, 529 struct ftrace_probe_ops *ops, void *data) 530 { 531 return ftrace_probe_print("traceoff", m, ip, ops, data); 532 } 533 534 static int 535 ftrace_stacktrace_print(struct seq_file *m, unsigned long ip, 536 struct ftrace_probe_ops *ops, void *data) 537 { 538 return ftrace_probe_print("stacktrace", m, ip, ops, data); 539 } 540 541 static int 542 ftrace_dump_print(struct seq_file *m, unsigned long ip, 543 struct ftrace_probe_ops *ops, void *data) 544 { 545 return ftrace_probe_print("dump", m, ip, ops, data); 546 } 547 548 static int 549 ftrace_cpudump_print(struct seq_file *m, unsigned long ip, 550 struct ftrace_probe_ops *ops, void *data) 551 { 552 return ftrace_probe_print("cpudump", m, ip, ops, data); 553 } 554 555 556 static int 557 ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr, 558 unsigned long ip, void *init_data, void **data) 559 { 560 struct ftrace_func_mapper *mapper = *data; 561 562 if (!mapper) { 563 mapper = allocate_ftrace_func_mapper(); 564 if (!mapper) 565 return -ENOMEM; 566 *data = mapper; 567 } 568 569 return ftrace_func_mapper_add_ip(mapper, ip, init_data); 570 } 571 572 static void 573 ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr, 574 unsigned long ip, void *data) 575 { 576 struct ftrace_func_mapper *mapper = data; 577 578 if (!ip) { 579 free_ftrace_func_mapper(mapper, NULL); 580 return; 581 } 582 583 ftrace_func_mapper_remove_ip(mapper, ip); 584 } 585 586 static struct ftrace_probe_ops traceon_count_probe_ops = { 587 .func = ftrace_traceon_count, 588 .print = ftrace_traceon_print, 589 .init = ftrace_count_init, 590 .free = ftrace_count_free, 591 }; 592 593 static struct ftrace_probe_ops traceoff_count_probe_ops = { 594 .func = ftrace_traceoff_count, 595 .print = ftrace_traceoff_print, 596 .init = ftrace_count_init, 597 .free = ftrace_count_free, 598 }; 599 600 static struct ftrace_probe_ops stacktrace_count_probe_ops = { 601 .func = ftrace_stacktrace_count, 602 .print = ftrace_stacktrace_print, 603 .init = ftrace_count_init, 604 .free = ftrace_count_free, 605 }; 606 607 static struct ftrace_probe_ops dump_probe_ops = { 608 .func = ftrace_dump_probe, 609 .print = ftrace_dump_print, 610 .init = ftrace_count_init, 611 .free = ftrace_count_free, 612 }; 613 614 static struct ftrace_probe_ops cpudump_probe_ops = { 615 .func = ftrace_cpudump_probe, 616 .print = ftrace_cpudump_print, 617 }; 618 619 static struct ftrace_probe_ops traceon_probe_ops = { 620 .func = ftrace_traceon, 621 .print = ftrace_traceon_print, 622 }; 623 624 static struct ftrace_probe_ops traceoff_probe_ops = { 625 .func = ftrace_traceoff, 626 .print = ftrace_traceoff_print, 627 }; 628 629 static struct ftrace_probe_ops stacktrace_probe_ops = { 630 .func = ftrace_stacktrace, 631 .print = ftrace_stacktrace_print, 632 }; 633 634 static int 635 ftrace_trace_probe_callback(struct trace_array *tr, 636 struct ftrace_probe_ops *ops, 637 struct ftrace_hash *hash, char *glob, 638 char *cmd, char *param, int enable) 639 { 640 void *count = (void *)-1; 641 char *number; 642 int ret; 643 644 /* hash funcs only work with set_ftrace_filter */ 645 if (!enable) 646 return -EINVAL; 647 648 if (glob[0] == '!') 649 return unregister_ftrace_function_probe_func(glob+1, tr, ops); 650 651 if (!param) 652 goto out_reg; 653 654 number = strsep(¶m, ":"); 655 656 if (!strlen(number)) 657 goto out_reg; 658 659 /* 660 * We use the callback data field (which is a pointer) 661 * as our counter. 662 */ 663 ret = kstrtoul(number, 0, (unsigned long *)&count); 664 if (ret) 665 return ret; 666 667 out_reg: 668 ret = register_ftrace_function_probe(glob, tr, ops, count); 669 670 return ret < 0 ? ret : 0; 671 } 672 673 static int 674 ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash, 675 char *glob, char *cmd, char *param, int enable) 676 { 677 struct ftrace_probe_ops *ops; 678 679 if (!tr) 680 return -ENODEV; 681 682 /* we register both traceon and traceoff to this callback */ 683 if (strcmp(cmd, "traceon") == 0) 684 ops = param ? &traceon_count_probe_ops : &traceon_probe_ops; 685 else 686 ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops; 687 688 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 689 param, enable); 690 } 691 692 static int 693 ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash, 694 char *glob, char *cmd, char *param, int enable) 695 { 696 struct ftrace_probe_ops *ops; 697 698 if (!tr) 699 return -ENODEV; 700 701 ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops; 702 703 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 704 param, enable); 705 } 706 707 static int 708 ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash, 709 char *glob, char *cmd, char *param, int enable) 710 { 711 struct ftrace_probe_ops *ops; 712 713 if (!tr) 714 return -ENODEV; 715 716 ops = &dump_probe_ops; 717 718 /* Only dump once. */ 719 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 720 "1", enable); 721 } 722 723 static int 724 ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash, 725 char *glob, char *cmd, char *param, int enable) 726 { 727 struct ftrace_probe_ops *ops; 728 729 if (!tr) 730 return -ENODEV; 731 732 ops = &cpudump_probe_ops; 733 734 /* Only dump once. */ 735 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 736 "1", enable); 737 } 738 739 static struct ftrace_func_command ftrace_traceon_cmd = { 740 .name = "traceon", 741 .func = ftrace_trace_onoff_callback, 742 }; 743 744 static struct ftrace_func_command ftrace_traceoff_cmd = { 745 .name = "traceoff", 746 .func = ftrace_trace_onoff_callback, 747 }; 748 749 static struct ftrace_func_command ftrace_stacktrace_cmd = { 750 .name = "stacktrace", 751 .func = ftrace_stacktrace_callback, 752 }; 753 754 static struct ftrace_func_command ftrace_dump_cmd = { 755 .name = "dump", 756 .func = ftrace_dump_callback, 757 }; 758 759 static struct ftrace_func_command ftrace_cpudump_cmd = { 760 .name = "cpudump", 761 .func = ftrace_cpudump_callback, 762 }; 763 764 static int __init init_func_cmd_traceon(void) 765 { 766 int ret; 767 768 ret = register_ftrace_command(&ftrace_traceoff_cmd); 769 if (ret) 770 return ret; 771 772 ret = register_ftrace_command(&ftrace_traceon_cmd); 773 if (ret) 774 goto out_free_traceoff; 775 776 ret = register_ftrace_command(&ftrace_stacktrace_cmd); 777 if (ret) 778 goto out_free_traceon; 779 780 ret = register_ftrace_command(&ftrace_dump_cmd); 781 if (ret) 782 goto out_free_stacktrace; 783 784 ret = register_ftrace_command(&ftrace_cpudump_cmd); 785 if (ret) 786 goto out_free_dump; 787 788 return 0; 789 790 out_free_dump: 791 unregister_ftrace_command(&ftrace_dump_cmd); 792 out_free_stacktrace: 793 unregister_ftrace_command(&ftrace_stacktrace_cmd); 794 out_free_traceon: 795 unregister_ftrace_command(&ftrace_traceon_cmd); 796 out_free_traceoff: 797 unregister_ftrace_command(&ftrace_traceoff_cmd); 798 799 return ret; 800 } 801 #else 802 static inline int init_func_cmd_traceon(void) 803 { 804 return 0; 805 } 806 #endif /* CONFIG_DYNAMIC_FTRACE */ 807 808 __init int init_function_trace(void) 809 { 810 init_func_cmd_traceon(); 811 return register_tracer(&function_trace); 812 } 813