1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * ring buffer based function tracer 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Based on code from the latency_tracer, that is: 9 * 10 * Copyright (C) 2004-2006 Ingo Molnar 11 * Copyright (C) 2004 Nadia Yvette Chambers 12 */ 13 #include <linux/ring_buffer.h> 14 #include <linux/debugfs.h> 15 #include <linux/uaccess.h> 16 #include <linux/ftrace.h> 17 #include <linux/slab.h> 18 #include <linux/fs.h> 19 20 #include "trace.h" 21 22 static void tracing_start_function_trace(struct trace_array *tr); 23 static void tracing_stop_function_trace(struct trace_array *tr); 24 static void 25 function_trace_call(unsigned long ip, unsigned long parent_ip, 26 struct ftrace_ops *op, struct ftrace_regs *fregs); 27 static void 28 function_args_trace_call(unsigned long ip, unsigned long parent_ip, 29 struct ftrace_ops *op, struct ftrace_regs *fregs); 30 static void 31 function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 32 struct ftrace_ops *op, struct ftrace_regs *fregs); 33 static void 34 function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, 35 struct ftrace_ops *op, struct ftrace_regs *fregs); 36 static void 37 function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, 38 struct ftrace_ops *op, 39 struct ftrace_regs *fregs); 40 static struct tracer_flags func_flags; 41 42 /* Our option */ 43 enum { 44 45 TRACE_FUNC_NO_OPTS = 0x0, /* No flags set. */ 46 TRACE_FUNC_OPT_STACK = 0x1, 47 TRACE_FUNC_OPT_NO_REPEATS = 0x2, 48 TRACE_FUNC_OPT_ARGS = 0x4, 49 50 /* Update this to next highest bit. */ 51 TRACE_FUNC_OPT_HIGHEST_BIT = 0x8 52 }; 53 54 #define TRACE_FUNC_OPT_MASK (TRACE_FUNC_OPT_HIGHEST_BIT - 1) 55 56 int ftrace_allocate_ftrace_ops(struct trace_array *tr) 57 { 58 struct ftrace_ops *ops; 59 60 /* The top level array uses the "global_ops" */ 61 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 62 return 0; 63 64 ops = kzalloc(sizeof(*ops), GFP_KERNEL); 65 if (!ops) 66 return -ENOMEM; 67 68 /* Currently only the non stack version is supported */ 69 ops->func = function_trace_call; 70 ops->flags = FTRACE_OPS_FL_PID; 71 72 tr->ops = ops; 73 ops->private = tr; 74 75 return 0; 76 } 77 78 void ftrace_free_ftrace_ops(struct trace_array *tr) 79 { 80 kfree(tr->ops); 81 tr->ops = NULL; 82 } 83 84 int ftrace_create_function_files(struct trace_array *tr, 85 struct dentry *parent) 86 { 87 int ret; 88 /* 89 * The top level array uses the "global_ops", and the files are 90 * created on boot up. 91 */ 92 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 93 return 0; 94 95 if (!tr->ops) 96 return -EINVAL; 97 98 ret = allocate_fgraph_ops(tr, tr->ops); 99 if (ret) { 100 kfree(tr->ops); 101 return ret; 102 } 103 104 ftrace_create_filter_files(tr->ops, parent); 105 106 return 0; 107 } 108 109 void ftrace_destroy_function_files(struct trace_array *tr) 110 { 111 ftrace_destroy_filter_files(tr->ops); 112 ftrace_free_ftrace_ops(tr); 113 free_fgraph_ops(tr); 114 } 115 116 static ftrace_func_t select_trace_function(u32 flags_val) 117 { 118 switch (flags_val & TRACE_FUNC_OPT_MASK) { 119 case TRACE_FUNC_NO_OPTS: 120 return function_trace_call; 121 case TRACE_FUNC_OPT_ARGS: 122 return function_args_trace_call; 123 case TRACE_FUNC_OPT_STACK: 124 return function_stack_trace_call; 125 case TRACE_FUNC_OPT_NO_REPEATS: 126 return function_no_repeats_trace_call; 127 case TRACE_FUNC_OPT_STACK | TRACE_FUNC_OPT_NO_REPEATS: 128 return function_stack_no_repeats_trace_call; 129 default: 130 return NULL; 131 } 132 } 133 134 static bool handle_func_repeats(struct trace_array *tr, u32 flags_val) 135 { 136 if (!tr->last_func_repeats && 137 (flags_val & TRACE_FUNC_OPT_NO_REPEATS)) { 138 tr->last_func_repeats = alloc_percpu(struct trace_func_repeats); 139 if (!tr->last_func_repeats) 140 return false; 141 } 142 143 return true; 144 } 145 146 static int function_trace_init(struct trace_array *tr) 147 { 148 ftrace_func_t func; 149 /* 150 * Instance trace_arrays get their ops allocated 151 * at instance creation. Unless it failed 152 * the allocation. 153 */ 154 if (!tr->ops) 155 return -ENOMEM; 156 157 func = select_trace_function(func_flags.val); 158 if (!func) 159 return -EINVAL; 160 161 if (!handle_func_repeats(tr, func_flags.val)) 162 return -ENOMEM; 163 164 ftrace_init_array_ops(tr, func); 165 166 tr->array_buffer.cpu = raw_smp_processor_id(); 167 168 tracing_start_cmdline_record(); 169 tracing_start_function_trace(tr); 170 return 0; 171 } 172 173 static void function_trace_reset(struct trace_array *tr) 174 { 175 tracing_stop_function_trace(tr); 176 tracing_stop_cmdline_record(); 177 ftrace_reset_array_ops(tr); 178 } 179 180 static void function_trace_start(struct trace_array *tr) 181 { 182 tracing_reset_online_cpus(&tr->array_buffer); 183 } 184 185 /* fregs are guaranteed not to be NULL if HAVE_DYNAMIC_FTRACE_WITH_ARGS is set */ 186 #if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS) 187 static __always_inline unsigned long 188 function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs) 189 { 190 unsigned long true_parent_ip; 191 int idx = 0; 192 193 true_parent_ip = parent_ip; 194 if (unlikely(parent_ip == (unsigned long)&return_to_handler) && fregs) 195 true_parent_ip = ftrace_graph_ret_addr(current, &idx, parent_ip, 196 (unsigned long *)ftrace_regs_get_stack_pointer(fregs)); 197 return true_parent_ip; 198 } 199 #else 200 static __always_inline unsigned long 201 function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs) 202 { 203 return parent_ip; 204 } 205 #endif 206 207 static void 208 function_trace_call(unsigned long ip, unsigned long parent_ip, 209 struct ftrace_ops *op, struct ftrace_regs *fregs) 210 { 211 struct trace_array *tr = op->private; 212 struct trace_array_cpu *data; 213 unsigned int trace_ctx; 214 int bit; 215 216 if (unlikely(!tr->function_enabled)) 217 return; 218 219 bit = ftrace_test_recursion_trylock(ip, parent_ip); 220 if (bit < 0) 221 return; 222 223 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 224 225 trace_ctx = tracing_gen_ctx_dec(); 226 227 data = this_cpu_ptr(tr->array_buffer.data); 228 if (!atomic_read(&data->disabled)) 229 trace_function(tr, ip, parent_ip, trace_ctx, NULL); 230 231 ftrace_test_recursion_unlock(bit); 232 } 233 234 static void 235 function_args_trace_call(unsigned long ip, unsigned long parent_ip, 236 struct ftrace_ops *op, struct ftrace_regs *fregs) 237 { 238 struct trace_array *tr = op->private; 239 struct trace_array_cpu *data; 240 unsigned int trace_ctx; 241 int bit; 242 int cpu; 243 244 if (unlikely(!tr->function_enabled)) 245 return; 246 247 bit = ftrace_test_recursion_trylock(ip, parent_ip); 248 if (bit < 0) 249 return; 250 251 trace_ctx = tracing_gen_ctx(); 252 253 cpu = smp_processor_id(); 254 data = per_cpu_ptr(tr->array_buffer.data, cpu); 255 if (!atomic_read(&data->disabled)) 256 trace_function(tr, ip, parent_ip, trace_ctx, fregs); 257 258 ftrace_test_recursion_unlock(bit); 259 } 260 261 #ifdef CONFIG_UNWINDER_ORC 262 /* 263 * Skip 2: 264 * 265 * function_stack_trace_call() 266 * ftrace_call() 267 */ 268 #define STACK_SKIP 2 269 #else 270 /* 271 * Skip 3: 272 * __trace_stack() 273 * function_stack_trace_call() 274 * ftrace_call() 275 */ 276 #define STACK_SKIP 3 277 #endif 278 279 static void 280 function_stack_trace_call(unsigned long ip, unsigned long parent_ip, 281 struct ftrace_ops *op, struct ftrace_regs *fregs) 282 { 283 struct trace_array *tr = op->private; 284 struct trace_array_cpu *data; 285 unsigned long flags; 286 long disabled; 287 int cpu; 288 unsigned int trace_ctx; 289 int skip = STACK_SKIP; 290 291 if (unlikely(!tr->function_enabled)) 292 return; 293 294 /* 295 * Need to use raw, since this must be called before the 296 * recursive protection is performed. 297 */ 298 local_irq_save(flags); 299 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 300 cpu = raw_smp_processor_id(); 301 data = per_cpu_ptr(tr->array_buffer.data, cpu); 302 disabled = atomic_inc_return(&data->disabled); 303 304 if (likely(disabled == 1)) { 305 trace_ctx = tracing_gen_ctx_flags(flags); 306 trace_function(tr, ip, parent_ip, trace_ctx, NULL); 307 #ifdef CONFIG_UNWINDER_FRAME_POINTER 308 if (ftrace_pids_enabled(op)) 309 skip++; 310 #endif 311 __trace_stack(tr, trace_ctx, skip); 312 } 313 314 atomic_dec(&data->disabled); 315 local_irq_restore(flags); 316 } 317 318 static inline bool is_repeat_check(struct trace_array *tr, 319 struct trace_func_repeats *last_info, 320 unsigned long ip, unsigned long parent_ip) 321 { 322 if (last_info->ip == ip && 323 last_info->parent_ip == parent_ip && 324 last_info->count < U16_MAX) { 325 last_info->ts_last_call = 326 ring_buffer_time_stamp(tr->array_buffer.buffer); 327 last_info->count++; 328 return true; 329 } 330 331 return false; 332 } 333 334 static inline void process_repeats(struct trace_array *tr, 335 unsigned long ip, unsigned long parent_ip, 336 struct trace_func_repeats *last_info, 337 unsigned int trace_ctx) 338 { 339 if (last_info->count) { 340 trace_last_func_repeats(tr, last_info, trace_ctx); 341 last_info->count = 0; 342 } 343 344 last_info->ip = ip; 345 last_info->parent_ip = parent_ip; 346 } 347 348 static void 349 function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, 350 struct ftrace_ops *op, 351 struct ftrace_regs *fregs) 352 { 353 struct trace_func_repeats *last_info; 354 struct trace_array *tr = op->private; 355 struct trace_array_cpu *data; 356 unsigned int trace_ctx; 357 int bit; 358 359 if (unlikely(!tr->function_enabled)) 360 return; 361 362 bit = ftrace_test_recursion_trylock(ip, parent_ip); 363 if (bit < 0) 364 return; 365 366 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 367 data = this_cpu_ptr(tr->array_buffer.data); 368 if (atomic_read(&data->disabled)) 369 goto out; 370 371 /* 372 * An interrupt may happen at any place here. But as far as I can see, 373 * the only damage that this can cause is to mess up the repetition 374 * counter without valuable data being lost. 375 * TODO: think about a solution that is better than just hoping to be 376 * lucky. 377 */ 378 last_info = this_cpu_ptr(tr->last_func_repeats); 379 if (is_repeat_check(tr, last_info, ip, parent_ip)) 380 goto out; 381 382 trace_ctx = tracing_gen_ctx_dec(); 383 process_repeats(tr, ip, parent_ip, last_info, trace_ctx); 384 385 trace_function(tr, ip, parent_ip, trace_ctx, NULL); 386 387 out: 388 ftrace_test_recursion_unlock(bit); 389 } 390 391 static void 392 function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, 393 struct ftrace_ops *op, 394 struct ftrace_regs *fregs) 395 { 396 struct trace_func_repeats *last_info; 397 struct trace_array *tr = op->private; 398 struct trace_array_cpu *data; 399 unsigned long flags; 400 long disabled; 401 int cpu; 402 unsigned int trace_ctx; 403 404 if (unlikely(!tr->function_enabled)) 405 return; 406 407 /* 408 * Need to use raw, since this must be called before the 409 * recursive protection is performed. 410 */ 411 local_irq_save(flags); 412 parent_ip = function_get_true_parent_ip(parent_ip, fregs); 413 cpu = raw_smp_processor_id(); 414 data = per_cpu_ptr(tr->array_buffer.data, cpu); 415 disabled = atomic_inc_return(&data->disabled); 416 417 if (likely(disabled == 1)) { 418 last_info = per_cpu_ptr(tr->last_func_repeats, cpu); 419 if (is_repeat_check(tr, last_info, ip, parent_ip)) 420 goto out; 421 422 trace_ctx = tracing_gen_ctx_flags(flags); 423 process_repeats(tr, ip, parent_ip, last_info, trace_ctx); 424 425 trace_function(tr, ip, parent_ip, trace_ctx, NULL); 426 __trace_stack(tr, trace_ctx, STACK_SKIP); 427 } 428 429 out: 430 atomic_dec(&data->disabled); 431 local_irq_restore(flags); 432 } 433 434 static struct tracer_opt func_opts[] = { 435 #ifdef CONFIG_STACKTRACE 436 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) }, 437 #endif 438 { TRACER_OPT(func-no-repeats, TRACE_FUNC_OPT_NO_REPEATS) }, 439 #ifdef CONFIG_FUNCTION_TRACE_ARGS 440 { TRACER_OPT(func-args, TRACE_FUNC_OPT_ARGS) }, 441 #endif 442 { } /* Always set a last empty entry */ 443 }; 444 445 static struct tracer_flags func_flags = { 446 .val = TRACE_FUNC_NO_OPTS, /* By default: all flags disabled */ 447 .opts = func_opts 448 }; 449 450 static void tracing_start_function_trace(struct trace_array *tr) 451 { 452 tr->function_enabled = 0; 453 register_ftrace_function(tr->ops); 454 tr->function_enabled = 1; 455 } 456 457 static void tracing_stop_function_trace(struct trace_array *tr) 458 { 459 tr->function_enabled = 0; 460 unregister_ftrace_function(tr->ops); 461 } 462 463 static struct tracer function_trace; 464 465 static int 466 func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 467 { 468 ftrace_func_t func; 469 u32 new_flags; 470 471 /* Do nothing if already set. */ 472 if (!!set == !!(func_flags.val & bit)) 473 return 0; 474 475 /* We can change this flag only when not running. */ 476 if (tr->current_trace != &function_trace) 477 return 0; 478 479 new_flags = (func_flags.val & ~bit) | (set ? bit : 0); 480 func = select_trace_function(new_flags); 481 if (!func) 482 return -EINVAL; 483 484 /* Check if there's anything to change. */ 485 if (tr->ops->func == func) 486 return 0; 487 488 if (!handle_func_repeats(tr, new_flags)) 489 return -ENOMEM; 490 491 unregister_ftrace_function(tr->ops); 492 tr->ops->func = func; 493 register_ftrace_function(tr->ops); 494 495 return 0; 496 } 497 498 static struct tracer function_trace __tracer_data = 499 { 500 .name = "function", 501 .init = function_trace_init, 502 .reset = function_trace_reset, 503 .start = function_trace_start, 504 .flags = &func_flags, 505 .set_flag = func_set_flag, 506 .allow_instances = true, 507 #ifdef CONFIG_FTRACE_SELFTEST 508 .selftest = trace_selftest_startup_function, 509 #endif 510 }; 511 512 #ifdef CONFIG_DYNAMIC_FTRACE 513 static void update_traceon_count(struct ftrace_probe_ops *ops, 514 unsigned long ip, 515 struct trace_array *tr, bool on, 516 void *data) 517 { 518 struct ftrace_func_mapper *mapper = data; 519 long *count; 520 long old_count; 521 522 /* 523 * Tracing gets disabled (or enabled) once per count. 524 * This function can be called at the same time on multiple CPUs. 525 * It is fine if both disable (or enable) tracing, as disabling 526 * (or enabling) the second time doesn't do anything as the 527 * state of the tracer is already disabled (or enabled). 528 * What needs to be synchronized in this case is that the count 529 * only gets decremented once, even if the tracer is disabled 530 * (or enabled) twice, as the second one is really a nop. 531 * 532 * The memory barriers guarantee that we only decrement the 533 * counter once. First the count is read to a local variable 534 * and a read barrier is used to make sure that it is loaded 535 * before checking if the tracer is in the state we want. 536 * If the tracer is not in the state we want, then the count 537 * is guaranteed to be the old count. 538 * 539 * Next the tracer is set to the state we want (disabled or enabled) 540 * then a write memory barrier is used to make sure that 541 * the new state is visible before changing the counter by 542 * one minus the old counter. This guarantees that another CPU 543 * executing this code will see the new state before seeing 544 * the new counter value, and would not do anything if the new 545 * counter is seen. 546 * 547 * Note, there is no synchronization between this and a user 548 * setting the tracing_on file. But we currently don't care 549 * about that. 550 */ 551 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 552 old_count = *count; 553 554 if (old_count <= 0) 555 return; 556 557 /* Make sure we see count before checking tracing state */ 558 smp_rmb(); 559 560 if (on == !!tracer_tracing_is_on(tr)) 561 return; 562 563 if (on) 564 tracer_tracing_on(tr); 565 else 566 tracer_tracing_off(tr); 567 568 /* Make sure tracing state is visible before updating count */ 569 smp_wmb(); 570 571 *count = old_count - 1; 572 } 573 574 static void 575 ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, 576 struct trace_array *tr, struct ftrace_probe_ops *ops, 577 void *data) 578 { 579 update_traceon_count(ops, ip, tr, 1, data); 580 } 581 582 static void 583 ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, 584 struct trace_array *tr, struct ftrace_probe_ops *ops, 585 void *data) 586 { 587 update_traceon_count(ops, ip, tr, 0, data); 588 } 589 590 static void 591 ftrace_traceon(unsigned long ip, unsigned long parent_ip, 592 struct trace_array *tr, struct ftrace_probe_ops *ops, 593 void *data) 594 { 595 if (tracer_tracing_is_on(tr)) 596 return; 597 598 tracer_tracing_on(tr); 599 } 600 601 static void 602 ftrace_traceoff(unsigned long ip, unsigned long parent_ip, 603 struct trace_array *tr, struct ftrace_probe_ops *ops, 604 void *data) 605 { 606 if (!tracer_tracing_is_on(tr)) 607 return; 608 609 tracer_tracing_off(tr); 610 } 611 612 #ifdef CONFIG_UNWINDER_ORC 613 /* 614 * Skip 3: 615 * 616 * function_trace_probe_call() 617 * ftrace_ops_assist_func() 618 * ftrace_call() 619 */ 620 #define FTRACE_STACK_SKIP 3 621 #else 622 /* 623 * Skip 5: 624 * 625 * __trace_stack() 626 * ftrace_stacktrace() 627 * function_trace_probe_call() 628 * ftrace_ops_assist_func() 629 * ftrace_call() 630 */ 631 #define FTRACE_STACK_SKIP 5 632 #endif 633 634 static __always_inline void trace_stack(struct trace_array *tr) 635 { 636 unsigned int trace_ctx; 637 638 trace_ctx = tracing_gen_ctx(); 639 640 __trace_stack(tr, trace_ctx, FTRACE_STACK_SKIP); 641 } 642 643 static void 644 ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, 645 struct trace_array *tr, struct ftrace_probe_ops *ops, 646 void *data) 647 { 648 trace_stack(tr); 649 } 650 651 static void 652 ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, 653 struct trace_array *tr, struct ftrace_probe_ops *ops, 654 void *data) 655 { 656 struct ftrace_func_mapper *mapper = data; 657 long *count; 658 long old_count; 659 long new_count; 660 661 if (!tracing_is_on()) 662 return; 663 664 /* unlimited? */ 665 if (!mapper) { 666 trace_stack(tr); 667 return; 668 } 669 670 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 671 672 /* 673 * Stack traces should only execute the number of times the 674 * user specified in the counter. 675 */ 676 do { 677 old_count = *count; 678 679 if (!old_count) 680 return; 681 682 new_count = old_count - 1; 683 new_count = cmpxchg(count, old_count, new_count); 684 if (new_count == old_count) 685 trace_stack(tr); 686 687 if (!tracing_is_on()) 688 return; 689 690 } while (new_count != old_count); 691 } 692 693 static int update_count(struct ftrace_probe_ops *ops, unsigned long ip, 694 void *data) 695 { 696 struct ftrace_func_mapper *mapper = data; 697 long *count = NULL; 698 699 if (mapper) 700 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 701 702 if (count) { 703 if (*count <= 0) 704 return 0; 705 (*count)--; 706 } 707 708 return 1; 709 } 710 711 static void 712 ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, 713 struct trace_array *tr, struct ftrace_probe_ops *ops, 714 void *data) 715 { 716 if (update_count(ops, ip, data)) 717 ftrace_dump(DUMP_ALL); 718 } 719 720 /* Only dump the current CPU buffer. */ 721 static void 722 ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, 723 struct trace_array *tr, struct ftrace_probe_ops *ops, 724 void *data) 725 { 726 if (update_count(ops, ip, data)) 727 ftrace_dump(DUMP_ORIG); 728 } 729 730 static int 731 ftrace_probe_print(const char *name, struct seq_file *m, 732 unsigned long ip, struct ftrace_probe_ops *ops, 733 void *data) 734 { 735 struct ftrace_func_mapper *mapper = data; 736 long *count = NULL; 737 738 seq_printf(m, "%ps:%s", (void *)ip, name); 739 740 if (mapper) 741 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 742 743 if (count) 744 seq_printf(m, ":count=%ld\n", *count); 745 else 746 seq_puts(m, ":unlimited\n"); 747 748 return 0; 749 } 750 751 static int 752 ftrace_traceon_print(struct seq_file *m, unsigned long ip, 753 struct ftrace_probe_ops *ops, 754 void *data) 755 { 756 return ftrace_probe_print("traceon", m, ip, ops, data); 757 } 758 759 static int 760 ftrace_traceoff_print(struct seq_file *m, unsigned long ip, 761 struct ftrace_probe_ops *ops, void *data) 762 { 763 return ftrace_probe_print("traceoff", m, ip, ops, data); 764 } 765 766 static int 767 ftrace_stacktrace_print(struct seq_file *m, unsigned long ip, 768 struct ftrace_probe_ops *ops, void *data) 769 { 770 return ftrace_probe_print("stacktrace", m, ip, ops, data); 771 } 772 773 static int 774 ftrace_dump_print(struct seq_file *m, unsigned long ip, 775 struct ftrace_probe_ops *ops, void *data) 776 { 777 return ftrace_probe_print("dump", m, ip, ops, data); 778 } 779 780 static int 781 ftrace_cpudump_print(struct seq_file *m, unsigned long ip, 782 struct ftrace_probe_ops *ops, void *data) 783 { 784 return ftrace_probe_print("cpudump", m, ip, ops, data); 785 } 786 787 788 static int 789 ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr, 790 unsigned long ip, void *init_data, void **data) 791 { 792 struct ftrace_func_mapper *mapper = *data; 793 794 if (!mapper) { 795 mapper = allocate_ftrace_func_mapper(); 796 if (!mapper) 797 return -ENOMEM; 798 *data = mapper; 799 } 800 801 return ftrace_func_mapper_add_ip(mapper, ip, init_data); 802 } 803 804 static void 805 ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr, 806 unsigned long ip, void *data) 807 { 808 struct ftrace_func_mapper *mapper = data; 809 810 if (!ip) { 811 free_ftrace_func_mapper(mapper, NULL); 812 return; 813 } 814 815 ftrace_func_mapper_remove_ip(mapper, ip); 816 } 817 818 static struct ftrace_probe_ops traceon_count_probe_ops = { 819 .func = ftrace_traceon_count, 820 .print = ftrace_traceon_print, 821 .init = ftrace_count_init, 822 .free = ftrace_count_free, 823 }; 824 825 static struct ftrace_probe_ops traceoff_count_probe_ops = { 826 .func = ftrace_traceoff_count, 827 .print = ftrace_traceoff_print, 828 .init = ftrace_count_init, 829 .free = ftrace_count_free, 830 }; 831 832 static struct ftrace_probe_ops stacktrace_count_probe_ops = { 833 .func = ftrace_stacktrace_count, 834 .print = ftrace_stacktrace_print, 835 .init = ftrace_count_init, 836 .free = ftrace_count_free, 837 }; 838 839 static struct ftrace_probe_ops dump_probe_ops = { 840 .func = ftrace_dump_probe, 841 .print = ftrace_dump_print, 842 .init = ftrace_count_init, 843 .free = ftrace_count_free, 844 }; 845 846 static struct ftrace_probe_ops cpudump_probe_ops = { 847 .func = ftrace_cpudump_probe, 848 .print = ftrace_cpudump_print, 849 }; 850 851 static struct ftrace_probe_ops traceon_probe_ops = { 852 .func = ftrace_traceon, 853 .print = ftrace_traceon_print, 854 }; 855 856 static struct ftrace_probe_ops traceoff_probe_ops = { 857 .func = ftrace_traceoff, 858 .print = ftrace_traceoff_print, 859 }; 860 861 static struct ftrace_probe_ops stacktrace_probe_ops = { 862 .func = ftrace_stacktrace, 863 .print = ftrace_stacktrace_print, 864 }; 865 866 static int 867 ftrace_trace_probe_callback(struct trace_array *tr, 868 struct ftrace_probe_ops *ops, 869 struct ftrace_hash *hash, char *glob, 870 char *cmd, char *param, int enable) 871 { 872 void *count = (void *)-1; 873 char *number; 874 int ret; 875 876 /* hash funcs only work with set_ftrace_filter */ 877 if (!enable) 878 return -EINVAL; 879 880 if (glob[0] == '!') 881 return unregister_ftrace_function_probe_func(glob+1, tr, ops); 882 883 if (!param) 884 goto out_reg; 885 886 number = strsep(¶m, ":"); 887 888 if (!strlen(number)) 889 goto out_reg; 890 891 /* 892 * We use the callback data field (which is a pointer) 893 * as our counter. 894 */ 895 ret = kstrtoul(number, 0, (unsigned long *)&count); 896 if (ret) 897 return ret; 898 899 out_reg: 900 ret = register_ftrace_function_probe(glob, tr, ops, count); 901 902 return ret < 0 ? ret : 0; 903 } 904 905 static int 906 ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash, 907 char *glob, char *cmd, char *param, int enable) 908 { 909 struct ftrace_probe_ops *ops; 910 911 if (!tr) 912 return -ENODEV; 913 914 /* we register both traceon and traceoff to this callback */ 915 if (strcmp(cmd, "traceon") == 0) 916 ops = param ? &traceon_count_probe_ops : &traceon_probe_ops; 917 else 918 ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops; 919 920 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 921 param, enable); 922 } 923 924 static int 925 ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash, 926 char *glob, char *cmd, char *param, int enable) 927 { 928 struct ftrace_probe_ops *ops; 929 930 if (!tr) 931 return -ENODEV; 932 933 ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops; 934 935 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 936 param, enable); 937 } 938 939 static int 940 ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash, 941 char *glob, char *cmd, char *param, int enable) 942 { 943 struct ftrace_probe_ops *ops; 944 945 if (!tr) 946 return -ENODEV; 947 948 ops = &dump_probe_ops; 949 950 /* Only dump once. */ 951 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 952 "1", enable); 953 } 954 955 static int 956 ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash, 957 char *glob, char *cmd, char *param, int enable) 958 { 959 struct ftrace_probe_ops *ops; 960 961 if (!tr) 962 return -ENODEV; 963 964 ops = &cpudump_probe_ops; 965 966 /* Only dump once. */ 967 return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, 968 "1", enable); 969 } 970 971 static struct ftrace_func_command ftrace_traceon_cmd = { 972 .name = "traceon", 973 .func = ftrace_trace_onoff_callback, 974 }; 975 976 static struct ftrace_func_command ftrace_traceoff_cmd = { 977 .name = "traceoff", 978 .func = ftrace_trace_onoff_callback, 979 }; 980 981 static struct ftrace_func_command ftrace_stacktrace_cmd = { 982 .name = "stacktrace", 983 .func = ftrace_stacktrace_callback, 984 }; 985 986 static struct ftrace_func_command ftrace_dump_cmd = { 987 .name = "dump", 988 .func = ftrace_dump_callback, 989 }; 990 991 static struct ftrace_func_command ftrace_cpudump_cmd = { 992 .name = "cpudump", 993 .func = ftrace_cpudump_callback, 994 }; 995 996 static int __init init_func_cmd_traceon(void) 997 { 998 int ret; 999 1000 ret = register_ftrace_command(&ftrace_traceoff_cmd); 1001 if (ret) 1002 return ret; 1003 1004 ret = register_ftrace_command(&ftrace_traceon_cmd); 1005 if (ret) 1006 goto out_free_traceoff; 1007 1008 ret = register_ftrace_command(&ftrace_stacktrace_cmd); 1009 if (ret) 1010 goto out_free_traceon; 1011 1012 ret = register_ftrace_command(&ftrace_dump_cmd); 1013 if (ret) 1014 goto out_free_stacktrace; 1015 1016 ret = register_ftrace_command(&ftrace_cpudump_cmd); 1017 if (ret) 1018 goto out_free_dump; 1019 1020 return 0; 1021 1022 out_free_dump: 1023 unregister_ftrace_command(&ftrace_dump_cmd); 1024 out_free_stacktrace: 1025 unregister_ftrace_command(&ftrace_stacktrace_cmd); 1026 out_free_traceon: 1027 unregister_ftrace_command(&ftrace_traceon_cmd); 1028 out_free_traceoff: 1029 unregister_ftrace_command(&ftrace_traceoff_cmd); 1030 1031 return ret; 1032 } 1033 #else 1034 static inline int init_func_cmd_traceon(void) 1035 { 1036 return 0; 1037 } 1038 #endif /* CONFIG_DYNAMIC_FTRACE */ 1039 1040 __init int init_function_trace(void) 1041 { 1042 init_func_cmd_traceon(); 1043 return register_tracer(&function_trace); 1044 } 1045