1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * 4 * Function graph tracer. 5 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com> 6 * Mostly borrowed from function tracer which 7 * is Copyright (c) Steven Rostedt <srostedt@redhat.com> 8 * 9 */ 10 #include <linux/uaccess.h> 11 #include <linux/ftrace.h> 12 #include <linux/interrupt.h> 13 #include <linux/slab.h> 14 #include <linux/fs.h> 15 16 #include "trace.h" 17 #include "trace_output.h" 18 19 /* When set, irq functions will be ignored */ 20 static int ftrace_graph_skip_irqs; 21 22 struct fgraph_cpu_data { 23 pid_t last_pid; 24 int depth; 25 int depth_irq; 26 int ignore; 27 unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH]; 28 }; 29 30 struct fgraph_data { 31 struct fgraph_cpu_data __percpu *cpu_data; 32 33 /* Place to preserve last processed entry. */ 34 union { 35 struct ftrace_graph_ent_entry ent; 36 struct fgraph_retaddr_ent_entry rent; 37 } ent; 38 struct ftrace_graph_ret_entry ret; 39 int failed; 40 int cpu; 41 }; 42 43 #define TRACE_GRAPH_INDENT 2 44 45 unsigned int fgraph_max_depth; 46 47 static struct tracer_opt trace_opts[] = { 48 /* Display overruns? (for self-debug purpose) */ 49 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) }, 50 /* Display CPU ? */ 51 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) }, 52 /* Display Overhead ? */ 53 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) }, 54 /* Display proc name/pid */ 55 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) }, 56 /* Display duration of execution */ 57 { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) }, 58 /* Display absolute time of an entry */ 59 { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) }, 60 /* Display interrupts */ 61 { TRACER_OPT(funcgraph-irqs, TRACE_GRAPH_PRINT_IRQS) }, 62 /* Display function name after trailing } */ 63 { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) }, 64 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 65 /* Display function return value ? */ 66 { TRACER_OPT(funcgraph-retval, TRACE_GRAPH_PRINT_RETVAL) }, 67 /* Display function return value in hexadecimal format ? */ 68 { TRACER_OPT(funcgraph-retval-hex, TRACE_GRAPH_PRINT_RETVAL_HEX) }, 69 #endif 70 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 71 /* Display function return address ? */ 72 { TRACER_OPT(funcgraph-retaddr, TRACE_GRAPH_PRINT_RETADDR) }, 73 #endif 74 #ifdef CONFIG_FUNCTION_TRACE_ARGS 75 /* Display function arguments ? */ 76 { TRACER_OPT(funcgraph-args, TRACE_GRAPH_ARGS) }, 77 #endif 78 /* Include sleep time (scheduled out) between entry and return */ 79 { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, 80 81 #ifdef CONFIG_FUNCTION_PROFILER 82 /* Include time within nested functions */ 83 { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) }, 84 #endif 85 86 { } /* Empty entry */ 87 }; 88 89 static struct tracer_flags tracer_flags = { 90 /* Don't display overruns, proc, or tail by default */ 91 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD | 92 TRACE_GRAPH_PRINT_DURATION | TRACE_GRAPH_PRINT_IRQS | 93 TRACE_GRAPH_SLEEP_TIME | TRACE_GRAPH_GRAPH_TIME, 94 .opts = trace_opts 95 }; 96 97 static bool tracer_flags_is_set(u32 flags) 98 { 99 return (tracer_flags.val & flags) == flags; 100 } 101 102 /* 103 * DURATION column is being also used to display IRQ signs, 104 * following values are used by print_graph_irq and others 105 * to fill in space into DURATION column. 106 */ 107 enum { 108 FLAGS_FILL_FULL = 1 << TRACE_GRAPH_PRINT_FILL_SHIFT, 109 FLAGS_FILL_START = 2 << TRACE_GRAPH_PRINT_FILL_SHIFT, 110 FLAGS_FILL_END = 3 << TRACE_GRAPH_PRINT_FILL_SHIFT, 111 }; 112 113 static void 114 print_graph_duration(struct trace_array *tr, unsigned long long duration, 115 struct trace_seq *s, u32 flags); 116 117 static int __graph_entry(struct trace_array *tr, struct ftrace_graph_ent *trace, 118 unsigned int trace_ctx, struct ftrace_regs *fregs) 119 { 120 struct ring_buffer_event *event; 121 struct trace_buffer *buffer = tr->array_buffer.buffer; 122 struct ftrace_graph_ent_entry *entry; 123 int size; 124 125 /* If fregs is defined, add FTRACE_REGS_MAX_ARGS long size words */ 126 size = sizeof(*entry) + (FTRACE_REGS_MAX_ARGS * !!fregs * sizeof(long)); 127 128 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, size, trace_ctx); 129 if (!event) 130 return 0; 131 132 entry = ring_buffer_event_data(event); 133 entry->graph_ent = *trace; 134 135 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 136 if (fregs) { 137 for (int i = 0; i < FTRACE_REGS_MAX_ARGS; i++) 138 entry->args[i] = ftrace_regs_get_argument(fregs, i); 139 } 140 #endif 141 142 trace_buffer_unlock_commit_nostack(buffer, event); 143 144 return 1; 145 } 146 147 int __trace_graph_entry(struct trace_array *tr, 148 struct ftrace_graph_ent *trace, 149 unsigned int trace_ctx) 150 { 151 return __graph_entry(tr, trace, trace_ctx, NULL); 152 } 153 154 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 155 int __trace_graph_retaddr_entry(struct trace_array *tr, 156 struct ftrace_graph_ent *trace, 157 unsigned int trace_ctx, 158 unsigned long retaddr) 159 { 160 struct ring_buffer_event *event; 161 struct trace_buffer *buffer = tr->array_buffer.buffer; 162 struct fgraph_retaddr_ent_entry *entry; 163 164 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RETADDR_ENT, 165 sizeof(*entry), trace_ctx); 166 if (!event) 167 return 0; 168 entry = ring_buffer_event_data(event); 169 entry->graph_ent.func = trace->func; 170 entry->graph_ent.depth = trace->depth; 171 entry->graph_ent.retaddr = retaddr; 172 trace_buffer_unlock_commit_nostack(buffer, event); 173 174 return 1; 175 } 176 #else 177 int __trace_graph_retaddr_entry(struct trace_array *tr, 178 struct ftrace_graph_ent *trace, 179 unsigned int trace_ctx, 180 unsigned long retaddr) 181 { 182 return 1; 183 } 184 #endif 185 186 static inline int ftrace_graph_ignore_irqs(void) 187 { 188 if (!ftrace_graph_skip_irqs || trace_recursion_test(TRACE_IRQ_BIT)) 189 return 0; 190 191 return in_hardirq(); 192 } 193 194 struct fgraph_times { 195 unsigned long long calltime; 196 unsigned long long sleeptime; /* may be optional! */ 197 }; 198 199 static int graph_entry(struct ftrace_graph_ent *trace, 200 struct fgraph_ops *gops, 201 struct ftrace_regs *fregs) 202 { 203 unsigned long *task_var = fgraph_get_task_var(gops); 204 struct trace_array *tr = gops->private; 205 struct fgraph_times *ftimes; 206 unsigned int trace_ctx; 207 int ret = 0; 208 209 if (*task_var & TRACE_GRAPH_NOTRACE) 210 return 0; 211 212 /* 213 * Do not trace a function if it's filtered by set_graph_notrace. 214 * Make the index of ret stack negative to indicate that it should 215 * ignore further functions. But it needs its own ret stack entry 216 * to recover the original index in order to continue tracing after 217 * returning from the function. 218 */ 219 if (ftrace_graph_notrace_addr(trace->func)) { 220 *task_var |= TRACE_GRAPH_NOTRACE; 221 /* 222 * Need to return 1 to have the return called 223 * that will clear the NOTRACE bit. 224 */ 225 return 1; 226 } 227 228 if (!ftrace_trace_task(tr)) 229 return 0; 230 231 if (ftrace_graph_ignore_func(gops, trace)) 232 return 0; 233 234 if (ftrace_graph_ignore_irqs()) 235 return 0; 236 237 if (fgraph_sleep_time) { 238 /* Only need to record the calltime */ 239 ftimes = fgraph_reserve_data(gops->idx, sizeof(ftimes->calltime)); 240 } else { 241 ftimes = fgraph_reserve_data(gops->idx, sizeof(*ftimes)); 242 if (ftimes) 243 ftimes->sleeptime = current->ftrace_sleeptime; 244 } 245 if (!ftimes) 246 return 0; 247 248 ftimes->calltime = trace_clock_local(); 249 250 /* 251 * Stop here if tracing_threshold is set. We only write function return 252 * events to the ring buffer. 253 */ 254 if (tracing_thresh) 255 return 1; 256 257 trace_ctx = tracing_gen_ctx(); 258 if (IS_ENABLED(CONFIG_FUNCTION_GRAPH_RETADDR) && 259 tracer_flags_is_set(TRACE_GRAPH_PRINT_RETADDR)) { 260 unsigned long retaddr = ftrace_graph_top_ret_addr(current); 261 ret = __trace_graph_retaddr_entry(tr, trace, trace_ctx, retaddr); 262 } else { 263 ret = __graph_entry(tr, trace, trace_ctx, fregs); 264 } 265 266 return ret; 267 } 268 269 int trace_graph_entry(struct ftrace_graph_ent *trace, 270 struct fgraph_ops *gops, 271 struct ftrace_regs *fregs) 272 { 273 return graph_entry(trace, gops, NULL); 274 } 275 276 static int trace_graph_entry_args(struct ftrace_graph_ent *trace, 277 struct fgraph_ops *gops, 278 struct ftrace_regs *fregs) 279 { 280 return graph_entry(trace, gops, fregs); 281 } 282 283 static void 284 __trace_graph_function(struct trace_array *tr, 285 unsigned long ip, unsigned int trace_ctx) 286 { 287 u64 time = trace_clock_local(); 288 struct ftrace_graph_ent ent = { 289 .func = ip, 290 .depth = 0, 291 }; 292 struct ftrace_graph_ret ret = { 293 .func = ip, 294 .depth = 0, 295 }; 296 297 __trace_graph_entry(tr, &ent, trace_ctx); 298 __trace_graph_return(tr, &ret, trace_ctx, time, time); 299 } 300 301 void 302 trace_graph_function(struct trace_array *tr, 303 unsigned long ip, unsigned long parent_ip, 304 unsigned int trace_ctx) 305 { 306 __trace_graph_function(tr, ip, trace_ctx); 307 } 308 309 void __trace_graph_return(struct trace_array *tr, 310 struct ftrace_graph_ret *trace, 311 unsigned int trace_ctx, 312 u64 calltime, u64 rettime) 313 { 314 struct ring_buffer_event *event; 315 struct trace_buffer *buffer = tr->array_buffer.buffer; 316 struct ftrace_graph_ret_entry *entry; 317 318 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, 319 sizeof(*entry), trace_ctx); 320 if (!event) 321 return; 322 entry = ring_buffer_event_data(event); 323 entry->ret = *trace; 324 entry->calltime = calltime; 325 entry->rettime = rettime; 326 trace_buffer_unlock_commit_nostack(buffer, event); 327 } 328 329 static void handle_nosleeptime(struct ftrace_graph_ret *trace, 330 struct fgraph_times *ftimes, 331 int size) 332 { 333 if (fgraph_sleep_time || size < sizeof(*ftimes)) 334 return; 335 336 ftimes->calltime += current->ftrace_sleeptime - ftimes->sleeptime; 337 } 338 339 void trace_graph_return(struct ftrace_graph_ret *trace, 340 struct fgraph_ops *gops, struct ftrace_regs *fregs) 341 { 342 unsigned long *task_var = fgraph_get_task_var(gops); 343 struct trace_array *tr = gops->private; 344 struct fgraph_times *ftimes; 345 unsigned int trace_ctx; 346 u64 calltime, rettime; 347 int size; 348 349 rettime = trace_clock_local(); 350 351 ftrace_graph_addr_finish(gops, trace); 352 353 if (*task_var & TRACE_GRAPH_NOTRACE) { 354 *task_var &= ~TRACE_GRAPH_NOTRACE; 355 return; 356 } 357 358 ftimes = fgraph_retrieve_data(gops->idx, &size); 359 if (!ftimes) 360 return; 361 362 handle_nosleeptime(trace, ftimes, size); 363 364 calltime = ftimes->calltime; 365 366 trace_ctx = tracing_gen_ctx(); 367 __trace_graph_return(tr, trace, trace_ctx, calltime, rettime); 368 } 369 370 static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, 371 struct fgraph_ops *gops, 372 struct ftrace_regs *fregs) 373 { 374 struct fgraph_times *ftimes; 375 int size; 376 377 ftrace_graph_addr_finish(gops, trace); 378 379 if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { 380 trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); 381 return; 382 } 383 384 ftimes = fgraph_retrieve_data(gops->idx, &size); 385 if (!ftimes) 386 return; 387 388 handle_nosleeptime(trace, ftimes, size); 389 390 if (tracing_thresh && 391 (trace_clock_local() - ftimes->calltime < tracing_thresh)) 392 return; 393 else 394 trace_graph_return(trace, gops, fregs); 395 } 396 397 static struct fgraph_ops funcgraph_ops = { 398 .entryfunc = &trace_graph_entry, 399 .retfunc = &trace_graph_return, 400 }; 401 402 int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) 403 { 404 struct fgraph_ops *gops; 405 406 gops = kzalloc(sizeof(*gops), GFP_KERNEL); 407 if (!gops) 408 return -ENOMEM; 409 410 gops->entryfunc = &trace_graph_entry; 411 gops->retfunc = &trace_graph_return; 412 413 tr->gops = gops; 414 gops->private = tr; 415 416 fgraph_init_ops(&gops->ops, ops); 417 418 return 0; 419 } 420 421 void free_fgraph_ops(struct trace_array *tr) 422 { 423 kfree(tr->gops); 424 } 425 426 __init void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) 427 { 428 tr->gops = &funcgraph_ops; 429 funcgraph_ops.private = tr; 430 fgraph_init_ops(&tr->gops->ops, ops); 431 } 432 433 static int graph_trace_init(struct trace_array *tr) 434 { 435 int ret; 436 437 if (tracer_flags_is_set(TRACE_GRAPH_ARGS)) 438 tr->gops->entryfunc = trace_graph_entry_args; 439 else 440 tr->gops->entryfunc = trace_graph_entry; 441 442 if (tracing_thresh) 443 tr->gops->retfunc = trace_graph_thresh_return; 444 else 445 tr->gops->retfunc = trace_graph_return; 446 447 /* Make gops functions visible before we start tracing */ 448 smp_mb(); 449 450 ret = register_ftrace_graph(tr->gops); 451 if (ret) 452 return ret; 453 tracing_start_cmdline_record(); 454 455 return 0; 456 } 457 458 static struct tracer graph_trace; 459 460 static int ftrace_graph_trace_args(struct trace_array *tr, int set) 461 { 462 trace_func_graph_ent_t entry; 463 464 /* Do nothing if the current tracer is not this tracer */ 465 if (tr->current_trace != &graph_trace) 466 return 0; 467 468 if (set) 469 entry = trace_graph_entry_args; 470 else 471 entry = trace_graph_entry; 472 473 /* See if there's any changes */ 474 if (tr->gops->entryfunc == entry) 475 return 0; 476 477 unregister_ftrace_graph(tr->gops); 478 479 tr->gops->entryfunc = entry; 480 481 /* Make gops functions visible before we start tracing */ 482 smp_mb(); 483 return register_ftrace_graph(tr->gops); 484 } 485 486 static void graph_trace_reset(struct trace_array *tr) 487 { 488 tracing_stop_cmdline_record(); 489 unregister_ftrace_graph(tr->gops); 490 } 491 492 static int graph_trace_update_thresh(struct trace_array *tr) 493 { 494 graph_trace_reset(tr); 495 return graph_trace_init(tr); 496 } 497 498 static int max_bytes_for_cpu; 499 500 static void print_graph_cpu(struct trace_seq *s, int cpu) 501 { 502 /* 503 * Start with a space character - to make it stand out 504 * to the right a bit when trace output is pasted into 505 * email: 506 */ 507 trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu); 508 } 509 510 #define TRACE_GRAPH_PROCINFO_LENGTH 14 511 512 static void print_graph_proc(struct trace_seq *s, pid_t pid) 513 { 514 char comm[TASK_COMM_LEN]; 515 /* sign + log10(MAX_INT) + '\0' */ 516 char pid_str[12]; 517 int spaces = 0; 518 int len; 519 int i; 520 521 trace_find_cmdline(pid, comm); 522 comm[7] = '\0'; 523 sprintf(pid_str, "%d", pid); 524 525 /* 1 stands for the "-" character */ 526 len = strlen(comm) + strlen(pid_str) + 1; 527 528 if (len < TRACE_GRAPH_PROCINFO_LENGTH) 529 spaces = TRACE_GRAPH_PROCINFO_LENGTH - len; 530 531 /* First spaces to align center */ 532 for (i = 0; i < spaces / 2; i++) 533 trace_seq_putc(s, ' '); 534 535 trace_seq_printf(s, "%s-%s", comm, pid_str); 536 537 /* Last spaces to align center */ 538 for (i = 0; i < spaces - (spaces / 2); i++) 539 trace_seq_putc(s, ' '); 540 } 541 542 543 static void print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry) 544 { 545 trace_seq_putc(s, ' '); 546 trace_print_lat_fmt(s, entry); 547 trace_seq_puts(s, " | "); 548 } 549 550 /* If the pid changed since the last trace, output this event */ 551 static void 552 verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data) 553 { 554 pid_t prev_pid; 555 pid_t *last_pid; 556 557 if (!data) 558 return; 559 560 last_pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); 561 562 if (*last_pid == pid) 563 return; 564 565 prev_pid = *last_pid; 566 *last_pid = pid; 567 568 if (prev_pid == -1) 569 return; 570 /* 571 * Context-switch trace line: 572 573 ------------------------------------------ 574 | 1) migration/0--1 => sshd-1755 575 ------------------------------------------ 576 577 */ 578 trace_seq_puts(s, " ------------------------------------------\n"); 579 print_graph_cpu(s, cpu); 580 print_graph_proc(s, prev_pid); 581 trace_seq_puts(s, " => "); 582 print_graph_proc(s, pid); 583 trace_seq_puts(s, "\n ------------------------------------------\n\n"); 584 } 585 586 static struct ftrace_graph_ret_entry * 587 get_return_for_leaf(struct trace_iterator *iter, 588 struct ftrace_graph_ent_entry *curr) 589 { 590 struct fgraph_data *data = iter->private; 591 struct ring_buffer_iter *ring_iter = NULL; 592 struct ring_buffer_event *event; 593 struct ftrace_graph_ret_entry *next; 594 595 /* 596 * If the previous output failed to write to the seq buffer, 597 * then we just reuse the data from before. 598 */ 599 if (data && data->failed) { 600 curr = &data->ent.ent; 601 next = &data->ret; 602 } else { 603 604 ring_iter = trace_buffer_iter(iter, iter->cpu); 605 606 /* First peek to compare current entry and the next one */ 607 if (ring_iter) 608 event = ring_buffer_iter_peek(ring_iter, NULL); 609 else { 610 /* 611 * We need to consume the current entry to see 612 * the next one. 613 */ 614 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, 615 NULL, NULL); 616 event = ring_buffer_peek(iter->array_buffer->buffer, iter->cpu, 617 NULL, NULL); 618 } 619 620 if (!event) 621 return NULL; 622 623 next = ring_buffer_event_data(event); 624 625 if (data) { 626 /* 627 * Save current and next entries for later reference 628 * if the output fails. 629 */ 630 if (unlikely(curr->ent.type == TRACE_GRAPH_RETADDR_ENT)) 631 data->ent.rent = *(struct fgraph_retaddr_ent_entry *)curr; 632 else 633 data->ent.ent = *curr; 634 /* 635 * If the next event is not a return type, then 636 * we only care about what type it is. Otherwise we can 637 * safely copy the entire event. 638 */ 639 if (next->ent.type == TRACE_GRAPH_RET) 640 data->ret = *next; 641 else 642 data->ret.ent.type = next->ent.type; 643 } 644 } 645 646 if (next->ent.type != TRACE_GRAPH_RET) 647 return NULL; 648 649 if (curr->ent.pid != next->ent.pid || 650 curr->graph_ent.func != next->ret.func) 651 return NULL; 652 653 /* this is a leaf, now advance the iterator */ 654 if (ring_iter) 655 ring_buffer_iter_advance(ring_iter); 656 657 return next; 658 } 659 660 static void print_graph_abs_time(u64 t, struct trace_seq *s) 661 { 662 unsigned long usecs_rem; 663 664 usecs_rem = do_div(t, NSEC_PER_SEC); 665 usecs_rem /= 1000; 666 667 trace_seq_printf(s, "%5lu.%06lu | ", 668 (unsigned long)t, usecs_rem); 669 } 670 671 static void 672 print_graph_rel_time(struct trace_iterator *iter, struct trace_seq *s) 673 { 674 unsigned long long usecs; 675 676 usecs = iter->ts - iter->array_buffer->time_start; 677 do_div(usecs, NSEC_PER_USEC); 678 679 trace_seq_printf(s, "%9llu us | ", usecs); 680 } 681 682 static void 683 print_graph_irq(struct trace_iterator *iter, unsigned long addr, 684 enum trace_type type, int cpu, pid_t pid, u32 flags) 685 { 686 struct trace_array *tr = iter->tr; 687 struct trace_seq *s = &iter->seq; 688 struct trace_entry *ent = iter->ent; 689 690 addr += iter->tr->text_delta; 691 692 if (addr < (unsigned long)__irqentry_text_start || 693 addr >= (unsigned long)__irqentry_text_end) 694 return; 695 696 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 697 /* Absolute time */ 698 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 699 print_graph_abs_time(iter->ts, s); 700 701 /* Relative time */ 702 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 703 print_graph_rel_time(iter, s); 704 705 /* Cpu */ 706 if (flags & TRACE_GRAPH_PRINT_CPU) 707 print_graph_cpu(s, cpu); 708 709 /* Proc */ 710 if (flags & TRACE_GRAPH_PRINT_PROC) { 711 print_graph_proc(s, pid); 712 trace_seq_puts(s, " | "); 713 } 714 715 /* Latency format */ 716 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 717 print_graph_lat_fmt(s, ent); 718 } 719 720 /* No overhead */ 721 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_START); 722 723 if (type == TRACE_GRAPH_ENT) 724 trace_seq_puts(s, "==========>"); 725 else 726 trace_seq_puts(s, "<=========="); 727 728 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_END); 729 trace_seq_putc(s, '\n'); 730 } 731 732 void 733 trace_print_graph_duration(unsigned long long duration, struct trace_seq *s) 734 { 735 unsigned long nsecs_rem = do_div(duration, 1000); 736 /* log10(ULONG_MAX) + '\0' */ 737 char usecs_str[21]; 738 char nsecs_str[5]; 739 int len; 740 int i; 741 742 sprintf(usecs_str, "%lu", (unsigned long) duration); 743 744 /* Print msecs */ 745 trace_seq_printf(s, "%s", usecs_str); 746 747 len = strlen(usecs_str); 748 749 /* Print nsecs (we don't want to exceed 7 numbers) */ 750 if (len < 7) { 751 size_t slen = min_t(size_t, sizeof(nsecs_str), 8UL - len); 752 753 snprintf(nsecs_str, slen, "%03lu", nsecs_rem); 754 trace_seq_printf(s, ".%s", nsecs_str); 755 len += strlen(nsecs_str) + 1; 756 } 757 758 trace_seq_puts(s, " us "); 759 760 /* Print remaining spaces to fit the row's width */ 761 for (i = len; i < 8; i++) 762 trace_seq_putc(s, ' '); 763 } 764 765 static void 766 print_graph_duration(struct trace_array *tr, unsigned long long duration, 767 struct trace_seq *s, u32 flags) 768 { 769 if (!(flags & TRACE_GRAPH_PRINT_DURATION) || 770 !(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 771 return; 772 773 /* No real adata, just filling the column with spaces */ 774 switch (flags & TRACE_GRAPH_PRINT_FILL_MASK) { 775 case FLAGS_FILL_FULL: 776 trace_seq_puts(s, " | "); 777 return; 778 case FLAGS_FILL_START: 779 trace_seq_puts(s, " "); 780 return; 781 case FLAGS_FILL_END: 782 trace_seq_puts(s, " |"); 783 return; 784 } 785 786 /* Signal a overhead of time execution to the output */ 787 if (flags & TRACE_GRAPH_PRINT_OVERHEAD) 788 trace_seq_printf(s, "%c ", trace_find_mark(duration)); 789 else 790 trace_seq_puts(s, " "); 791 792 trace_print_graph_duration(duration, s); 793 trace_seq_puts(s, "| "); 794 } 795 796 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 797 #define __TRACE_GRAPH_PRINT_RETVAL TRACE_GRAPH_PRINT_RETVAL 798 #else 799 #define __TRACE_GRAPH_PRINT_RETVAL 0 800 #endif 801 802 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 803 #define __TRACE_GRAPH_PRINT_RETADDR TRACE_GRAPH_PRINT_RETADDR 804 static void print_graph_retaddr(struct trace_seq *s, struct fgraph_retaddr_ent_entry *entry, 805 u32 trace_flags, bool comment) 806 { 807 if (comment) 808 trace_seq_puts(s, " /*"); 809 810 trace_seq_puts(s, " <-"); 811 seq_print_ip_sym(s, entry->graph_ent.retaddr, trace_flags | TRACE_ITER_SYM_OFFSET); 812 813 if (comment) 814 trace_seq_puts(s, " */"); 815 } 816 #else 817 #define __TRACE_GRAPH_PRINT_RETADDR 0 818 #define print_graph_retaddr(_seq, _entry, _tflags, _comment) do { } while (0) 819 #endif 820 821 #if defined(CONFIG_FUNCTION_GRAPH_RETVAL) || defined(CONFIG_FUNCTION_GRAPH_RETADDR) 822 823 static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entry *entry, 824 struct ftrace_graph_ret *graph_ret, void *func, 825 u32 opt_flags, u32 trace_flags, int args_size) 826 { 827 unsigned long err_code = 0; 828 unsigned long retval = 0; 829 bool print_retaddr = false; 830 bool print_retval = false; 831 bool hex_format = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL_HEX); 832 833 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 834 retval = graph_ret->retval; 835 print_retval = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL); 836 #endif 837 838 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 839 print_retaddr = !!(opt_flags & TRACE_GRAPH_PRINT_RETADDR); 840 #endif 841 842 if (print_retval && retval && !hex_format) { 843 /* Check if the return value matches the negative format */ 844 if (IS_ENABLED(CONFIG_64BIT) && (retval & BIT(31)) && 845 (((u64)retval) >> 32) == 0) { 846 err_code = sign_extend64(retval, 31); 847 } else { 848 err_code = retval; 849 } 850 851 if (!IS_ERR_VALUE(err_code)) 852 err_code = 0; 853 } 854 855 if (entry) { 856 if (entry->ent.type != TRACE_GRAPH_RETADDR_ENT) 857 print_retaddr = false; 858 859 trace_seq_printf(s, "%ps", func); 860 861 if (args_size >= FTRACE_REGS_MAX_ARGS * sizeof(long)) { 862 print_function_args(s, entry->args, (unsigned long)func); 863 trace_seq_putc(s, ';'); 864 } else 865 trace_seq_puts(s, "();"); 866 867 if (print_retval || print_retaddr) 868 trace_seq_puts(s, " /*"); 869 } else { 870 print_retaddr = false; 871 trace_seq_printf(s, "} /* %ps", func); 872 } 873 874 if (print_retaddr) 875 print_graph_retaddr(s, (struct fgraph_retaddr_ent_entry *)entry, 876 trace_flags, false); 877 878 if (print_retval) { 879 if (hex_format || (err_code == 0)) 880 trace_seq_printf(s, " ret=0x%lx", retval); 881 else 882 trace_seq_printf(s, " ret=%ld", err_code); 883 } 884 885 if (!entry || print_retval || print_retaddr) 886 trace_seq_puts(s, " */"); 887 } 888 889 #else 890 891 #define print_graph_retval(_seq, _ent, _ret, _func, _opt_flags, _trace_flags, args_size) \ 892 do {} while (0) 893 894 #endif 895 896 /* Case of a leaf function on its call entry */ 897 static enum print_line_t 898 print_graph_entry_leaf(struct trace_iterator *iter, 899 struct ftrace_graph_ent_entry *entry, 900 struct ftrace_graph_ret_entry *ret_entry, 901 struct trace_seq *s, u32 flags) 902 { 903 struct fgraph_data *data = iter->private; 904 struct trace_array *tr = iter->tr; 905 struct ftrace_graph_ret *graph_ret; 906 struct ftrace_graph_ent *call; 907 unsigned long long duration; 908 unsigned long ret_func; 909 int args_size; 910 int cpu = iter->cpu; 911 int i; 912 913 args_size = iter->ent_size - offsetof(struct ftrace_graph_ent_entry, args); 914 915 graph_ret = &ret_entry->ret; 916 call = &entry->graph_ent; 917 duration = ret_entry->rettime - ret_entry->calltime; 918 919 if (data) { 920 struct fgraph_cpu_data *cpu_data; 921 922 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 923 924 /* 925 * Comments display at + 1 to depth. Since 926 * this is a leaf function, keep the comments 927 * equal to this depth. 928 */ 929 cpu_data->depth = call->depth - 1; 930 931 /* No need to keep this function around for this depth */ 932 if (call->depth < FTRACE_RETFUNC_DEPTH && 933 !WARN_ON_ONCE(call->depth < 0)) 934 cpu_data->enter_funcs[call->depth] = 0; 935 } 936 937 /* Overhead and duration */ 938 print_graph_duration(tr, duration, s, flags); 939 940 /* Function */ 941 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) 942 trace_seq_putc(s, ' '); 943 944 ret_func = graph_ret->func + iter->tr->text_delta; 945 946 /* 947 * Write out the function return value or return address 948 */ 949 if (flags & (__TRACE_GRAPH_PRINT_RETVAL | __TRACE_GRAPH_PRINT_RETADDR)) { 950 print_graph_retval(s, entry, graph_ret, 951 (void *)graph_ret->func + iter->tr->text_delta, 952 flags, tr->trace_flags, args_size); 953 } else { 954 trace_seq_printf(s, "%ps", (void *)ret_func); 955 956 if (args_size >= FTRACE_REGS_MAX_ARGS * sizeof(long)) { 957 print_function_args(s, entry->args, ret_func); 958 trace_seq_putc(s, ';'); 959 } else 960 trace_seq_puts(s, "();"); 961 } 962 trace_seq_putc(s, '\n'); 963 964 print_graph_irq(iter, graph_ret->func, TRACE_GRAPH_RET, 965 cpu, iter->ent->pid, flags); 966 967 return trace_handle_return(s); 968 } 969 970 static enum print_line_t 971 print_graph_entry_nested(struct trace_iterator *iter, 972 struct ftrace_graph_ent_entry *entry, 973 struct trace_seq *s, int cpu, u32 flags) 974 { 975 struct ftrace_graph_ent *call = &entry->graph_ent; 976 struct fgraph_data *data = iter->private; 977 struct trace_array *tr = iter->tr; 978 unsigned long func; 979 int args_size; 980 int i; 981 982 if (data) { 983 struct fgraph_cpu_data *cpu_data; 984 int cpu = iter->cpu; 985 986 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 987 cpu_data->depth = call->depth; 988 989 /* Save this function pointer to see if the exit matches */ 990 if (call->depth < FTRACE_RETFUNC_DEPTH && 991 !WARN_ON_ONCE(call->depth < 0)) 992 cpu_data->enter_funcs[call->depth] = call->func; 993 } 994 995 /* No time */ 996 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_FULL); 997 998 /* Function */ 999 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) 1000 trace_seq_putc(s, ' '); 1001 1002 func = call->func + iter->tr->text_delta; 1003 1004 trace_seq_printf(s, "%ps", (void *)func); 1005 1006 args_size = iter->ent_size - offsetof(struct ftrace_graph_ent_entry, args); 1007 1008 if (args_size >= FTRACE_REGS_MAX_ARGS * sizeof(long)) 1009 print_function_args(s, entry->args, func); 1010 else 1011 trace_seq_puts(s, "()"); 1012 1013 trace_seq_puts(s, " {"); 1014 1015 if (flags & __TRACE_GRAPH_PRINT_RETADDR && 1016 entry->ent.type == TRACE_GRAPH_RETADDR_ENT) 1017 print_graph_retaddr(s, (struct fgraph_retaddr_ent_entry *)entry, 1018 tr->trace_flags, true); 1019 trace_seq_putc(s, '\n'); 1020 1021 if (trace_seq_has_overflowed(s)) 1022 return TRACE_TYPE_PARTIAL_LINE; 1023 1024 /* 1025 * we already consumed the current entry to check the next one 1026 * and see if this is a leaf. 1027 */ 1028 return TRACE_TYPE_NO_CONSUME; 1029 } 1030 1031 static void 1032 print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s, 1033 int type, unsigned long addr, u32 flags) 1034 { 1035 struct fgraph_data *data = iter->private; 1036 struct trace_entry *ent = iter->ent; 1037 struct trace_array *tr = iter->tr; 1038 int cpu = iter->cpu; 1039 1040 /* Pid */ 1041 verif_pid(s, ent->pid, cpu, data); 1042 1043 if (type) 1044 /* Interrupt */ 1045 print_graph_irq(iter, addr, type, cpu, ent->pid, flags); 1046 1047 if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 1048 return; 1049 1050 /* Absolute time */ 1051 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1052 print_graph_abs_time(iter->ts, s); 1053 1054 /* Relative time */ 1055 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1056 print_graph_rel_time(iter, s); 1057 1058 /* Cpu */ 1059 if (flags & TRACE_GRAPH_PRINT_CPU) 1060 print_graph_cpu(s, cpu); 1061 1062 /* Proc */ 1063 if (flags & TRACE_GRAPH_PRINT_PROC) { 1064 print_graph_proc(s, ent->pid); 1065 trace_seq_puts(s, " | "); 1066 } 1067 1068 /* Latency format */ 1069 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 1070 print_graph_lat_fmt(s, ent); 1071 1072 return; 1073 } 1074 1075 /* 1076 * Entry check for irq code 1077 * 1078 * returns 1 if 1079 * - we are inside irq code 1080 * - we just entered irq code 1081 * 1082 * returns 0 if 1083 * - funcgraph-interrupts option is set 1084 * - we are not inside irq code 1085 */ 1086 static int 1087 check_irq_entry(struct trace_iterator *iter, u32 flags, 1088 unsigned long addr, int depth) 1089 { 1090 int cpu = iter->cpu; 1091 int *depth_irq; 1092 struct fgraph_data *data = iter->private; 1093 1094 addr += iter->tr->text_delta; 1095 1096 /* 1097 * If we are either displaying irqs, or we got called as 1098 * a graph event and private data does not exist, 1099 * then we bypass the irq check. 1100 */ 1101 if ((flags & TRACE_GRAPH_PRINT_IRQS) || 1102 (!data)) 1103 return 0; 1104 1105 depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1106 1107 /* 1108 * We are inside the irq code 1109 */ 1110 if (*depth_irq >= 0) 1111 return 1; 1112 1113 if ((addr < (unsigned long)__irqentry_text_start) || 1114 (addr >= (unsigned long)__irqentry_text_end)) 1115 return 0; 1116 1117 /* 1118 * We are entering irq code. 1119 */ 1120 *depth_irq = depth; 1121 return 1; 1122 } 1123 1124 /* 1125 * Return check for irq code 1126 * 1127 * returns 1 if 1128 * - we are inside irq code 1129 * - we just left irq code 1130 * 1131 * returns 0 if 1132 * - funcgraph-interrupts option is set 1133 * - we are not inside irq code 1134 */ 1135 static int 1136 check_irq_return(struct trace_iterator *iter, u32 flags, int depth) 1137 { 1138 int cpu = iter->cpu; 1139 int *depth_irq; 1140 struct fgraph_data *data = iter->private; 1141 1142 /* 1143 * If we are either displaying irqs, or we got called as 1144 * a graph event and private data does not exist, 1145 * then we bypass the irq check. 1146 */ 1147 if ((flags & TRACE_GRAPH_PRINT_IRQS) || 1148 (!data)) 1149 return 0; 1150 1151 depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1152 1153 /* 1154 * We are not inside the irq code. 1155 */ 1156 if (*depth_irq == -1) 1157 return 0; 1158 1159 /* 1160 * We are inside the irq code, and this is returning entry. 1161 * Let's not trace it and clear the entry depth, since 1162 * we are out of irq code. 1163 * 1164 * This condition ensures that we 'leave the irq code' once 1165 * we are out of the entry depth. Thus protecting us from 1166 * the RETURN entry loss. 1167 */ 1168 if (*depth_irq >= depth) { 1169 *depth_irq = -1; 1170 return 1; 1171 } 1172 1173 /* 1174 * We are inside the irq code, and this is not the entry. 1175 */ 1176 return 1; 1177 } 1178 1179 static enum print_line_t 1180 print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, 1181 struct trace_iterator *iter, u32 flags) 1182 { 1183 struct fgraph_data *data = iter->private; 1184 struct ftrace_graph_ent *call; 1185 struct ftrace_graph_ret_entry *leaf_ret; 1186 static enum print_line_t ret; 1187 int cpu = iter->cpu; 1188 /* 1189 * print_graph_entry() may consume the current event, 1190 * thus @field may become invalid, so we need to save it. 1191 * sizeof(struct ftrace_graph_ent_entry) is very small, 1192 * it can be safely saved at the stack. 1193 */ 1194 struct ftrace_graph_ent_entry *entry; 1195 u8 save_buf[sizeof(*entry) + FTRACE_REGS_MAX_ARGS * sizeof(long)]; 1196 1197 /* The ent_size is expected to be as big as the entry */ 1198 if (iter->ent_size > sizeof(save_buf)) 1199 iter->ent_size = sizeof(save_buf); 1200 1201 entry = (void *)save_buf; 1202 memcpy(entry, field, iter->ent_size); 1203 1204 call = &entry->graph_ent; 1205 1206 if (check_irq_entry(iter, flags, call->func, call->depth)) 1207 return TRACE_TYPE_HANDLED; 1208 1209 print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func, flags); 1210 1211 leaf_ret = get_return_for_leaf(iter, entry); 1212 if (leaf_ret) 1213 ret = print_graph_entry_leaf(iter, entry, leaf_ret, s, flags); 1214 else 1215 ret = print_graph_entry_nested(iter, entry, s, cpu, flags); 1216 1217 if (data) { 1218 /* 1219 * If we failed to write our output, then we need to make 1220 * note of it. Because we already consumed our entry. 1221 */ 1222 if (s->full) { 1223 data->failed = 1; 1224 data->cpu = cpu; 1225 } else 1226 data->failed = 0; 1227 } 1228 1229 return ret; 1230 } 1231 1232 static enum print_line_t 1233 print_graph_return(struct ftrace_graph_ret_entry *retentry, struct trace_seq *s, 1234 struct trace_entry *ent, struct trace_iterator *iter, 1235 u32 flags) 1236 { 1237 struct ftrace_graph_ret *trace = &retentry->ret; 1238 u64 calltime = retentry->calltime; 1239 u64 rettime = retentry->rettime; 1240 unsigned long long duration = rettime - calltime; 1241 struct fgraph_data *data = iter->private; 1242 struct trace_array *tr = iter->tr; 1243 unsigned long func; 1244 pid_t pid = ent->pid; 1245 int cpu = iter->cpu; 1246 int func_match = 1; 1247 int i; 1248 1249 func = trace->func + iter->tr->text_delta; 1250 1251 if (check_irq_return(iter, flags, trace->depth)) 1252 return TRACE_TYPE_HANDLED; 1253 1254 if (data) { 1255 struct fgraph_cpu_data *cpu_data; 1256 int cpu = iter->cpu; 1257 1258 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 1259 1260 /* 1261 * Comments display at + 1 to depth. This is the 1262 * return from a function, we now want the comments 1263 * to display at the same level of the bracket. 1264 */ 1265 cpu_data->depth = trace->depth - 1; 1266 1267 if (trace->depth < FTRACE_RETFUNC_DEPTH && 1268 !WARN_ON_ONCE(trace->depth < 0)) { 1269 if (cpu_data->enter_funcs[trace->depth] != trace->func) 1270 func_match = 0; 1271 cpu_data->enter_funcs[trace->depth] = 0; 1272 } 1273 } 1274 1275 print_graph_prologue(iter, s, 0, 0, flags); 1276 1277 /* Overhead and duration */ 1278 print_graph_duration(tr, duration, s, flags); 1279 1280 /* Closing brace */ 1281 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) 1282 trace_seq_putc(s, ' '); 1283 1284 /* 1285 * Always write out the function name and its return value if the 1286 * funcgraph-retval option is enabled. 1287 */ 1288 if (flags & __TRACE_GRAPH_PRINT_RETVAL) { 1289 print_graph_retval(s, NULL, trace, (void *)func, flags, 1290 tr->trace_flags, 0); 1291 } else { 1292 /* 1293 * If the return function does not have a matching entry, 1294 * then the entry was lost. Instead of just printing 1295 * the '}' and letting the user guess what function this 1296 * belongs to, write out the function name. Always do 1297 * that if the funcgraph-tail option is enabled. 1298 */ 1299 if (func_match && !(flags & TRACE_GRAPH_PRINT_TAIL)) 1300 trace_seq_puts(s, "}"); 1301 else 1302 trace_seq_printf(s, "} /* %ps */", (void *)func); 1303 } 1304 trace_seq_putc(s, '\n'); 1305 1306 /* Overrun */ 1307 if (flags & TRACE_GRAPH_PRINT_OVERRUN) 1308 trace_seq_printf(s, " (Overruns: %u)\n", 1309 trace->overrun); 1310 1311 print_graph_irq(iter, trace->func, TRACE_GRAPH_RET, 1312 cpu, pid, flags); 1313 1314 return trace_handle_return(s); 1315 } 1316 1317 static enum print_line_t 1318 print_graph_comment(struct trace_seq *s, struct trace_entry *ent, 1319 struct trace_iterator *iter, u32 flags) 1320 { 1321 struct trace_array *tr = iter->tr; 1322 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 1323 struct fgraph_data *data = iter->private; 1324 struct trace_event *event; 1325 int depth = 0; 1326 int ret; 1327 int i; 1328 1329 if (data) 1330 depth = per_cpu_ptr(data->cpu_data, iter->cpu)->depth; 1331 1332 print_graph_prologue(iter, s, 0, 0, flags); 1333 1334 /* No time */ 1335 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_FULL); 1336 1337 /* Indentation */ 1338 if (depth > 0) 1339 for (i = 0; i < (depth + 1) * TRACE_GRAPH_INDENT; i++) 1340 trace_seq_putc(s, ' '); 1341 1342 /* The comment */ 1343 trace_seq_puts(s, "/* "); 1344 1345 switch (iter->ent->type) { 1346 case TRACE_BPUTS: 1347 ret = trace_print_bputs_msg_only(iter); 1348 if (ret != TRACE_TYPE_HANDLED) 1349 return ret; 1350 break; 1351 case TRACE_BPRINT: 1352 ret = trace_print_bprintk_msg_only(iter); 1353 if (ret != TRACE_TYPE_HANDLED) 1354 return ret; 1355 break; 1356 case TRACE_PRINT: 1357 ret = trace_print_printk_msg_only(iter); 1358 if (ret != TRACE_TYPE_HANDLED) 1359 return ret; 1360 break; 1361 default: 1362 event = ftrace_find_event(ent->type); 1363 if (!event) 1364 return TRACE_TYPE_UNHANDLED; 1365 1366 ret = event->funcs->trace(iter, sym_flags, event); 1367 if (ret != TRACE_TYPE_HANDLED) 1368 return ret; 1369 } 1370 1371 if (trace_seq_has_overflowed(s)) 1372 goto out; 1373 1374 /* Strip ending newline */ 1375 if (s->buffer[s->seq.len - 1] == '\n') { 1376 s->buffer[s->seq.len - 1] = '\0'; 1377 s->seq.len--; 1378 } 1379 1380 trace_seq_puts(s, " */\n"); 1381 out: 1382 return trace_handle_return(s); 1383 } 1384 1385 1386 enum print_line_t 1387 print_graph_function_flags(struct trace_iterator *iter, u32 flags) 1388 { 1389 struct ftrace_graph_ent_entry *field; 1390 struct fgraph_data *data = iter->private; 1391 struct trace_entry *entry = iter->ent; 1392 struct trace_seq *s = &iter->seq; 1393 int cpu = iter->cpu; 1394 int ret; 1395 1396 if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) { 1397 per_cpu_ptr(data->cpu_data, cpu)->ignore = 0; 1398 return TRACE_TYPE_HANDLED; 1399 } 1400 1401 /* 1402 * If the last output failed, there's a possibility we need 1403 * to print out the missing entry which would never go out. 1404 */ 1405 if (data && data->failed) { 1406 field = &data->ent.ent; 1407 iter->cpu = data->cpu; 1408 ret = print_graph_entry(field, s, iter, flags); 1409 if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) { 1410 per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1; 1411 ret = TRACE_TYPE_NO_CONSUME; 1412 } 1413 iter->cpu = cpu; 1414 return ret; 1415 } 1416 1417 switch (entry->type) { 1418 case TRACE_GRAPH_ENT: { 1419 trace_assign_type(field, entry); 1420 return print_graph_entry(field, s, iter, flags); 1421 } 1422 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1423 case TRACE_GRAPH_RETADDR_ENT: { 1424 struct fgraph_retaddr_ent_entry saved; 1425 struct fgraph_retaddr_ent_entry *rfield; 1426 1427 trace_assign_type(rfield, entry); 1428 saved = *rfield; 1429 return print_graph_entry((struct ftrace_graph_ent_entry *)&saved, s, iter, flags); 1430 } 1431 #endif 1432 case TRACE_GRAPH_RET: { 1433 struct ftrace_graph_ret_entry *field; 1434 trace_assign_type(field, entry); 1435 return print_graph_return(field, s, entry, iter, flags); 1436 } 1437 case TRACE_STACK: 1438 case TRACE_FN: 1439 /* dont trace stack and functions as comments */ 1440 return TRACE_TYPE_UNHANDLED; 1441 1442 default: 1443 return print_graph_comment(s, entry, iter, flags); 1444 } 1445 1446 return TRACE_TYPE_HANDLED; 1447 } 1448 1449 static enum print_line_t 1450 print_graph_function(struct trace_iterator *iter) 1451 { 1452 return print_graph_function_flags(iter, tracer_flags.val); 1453 } 1454 1455 static enum print_line_t 1456 print_graph_function_event(struct trace_iterator *iter, int flags, 1457 struct trace_event *event) 1458 { 1459 return print_graph_function(iter); 1460 } 1461 1462 static void print_lat_header(struct seq_file *s, u32 flags) 1463 { 1464 static const char spaces[] = " " /* 16 spaces */ 1465 " " /* 4 spaces */ 1466 " "; /* 17 spaces */ 1467 int size = 0; 1468 1469 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1470 size += 16; 1471 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1472 size += 16; 1473 if (flags & TRACE_GRAPH_PRINT_CPU) 1474 size += 4; 1475 if (flags & TRACE_GRAPH_PRINT_PROC) 1476 size += 17; 1477 1478 seq_printf(s, "#%.*s _-----=> irqs-off \n", size, spaces); 1479 seq_printf(s, "#%.*s / _----=> need-resched \n", size, spaces); 1480 seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces); 1481 seq_printf(s, "#%.*s|| / _--=> preempt-depth \n", size, spaces); 1482 seq_printf(s, "#%.*s||| / \n", size, spaces); 1483 } 1484 1485 static void __print_graph_headers_flags(struct trace_array *tr, 1486 struct seq_file *s, u32 flags) 1487 { 1488 int lat = tr->trace_flags & TRACE_ITER_LATENCY_FMT; 1489 1490 if (lat) 1491 print_lat_header(s, flags); 1492 1493 /* 1st line */ 1494 seq_putc(s, '#'); 1495 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1496 seq_puts(s, " TIME "); 1497 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1498 seq_puts(s, " REL TIME "); 1499 if (flags & TRACE_GRAPH_PRINT_CPU) 1500 seq_puts(s, " CPU"); 1501 if (flags & TRACE_GRAPH_PRINT_PROC) 1502 seq_puts(s, " TASK/PID "); 1503 if (lat) 1504 seq_puts(s, "|||| "); 1505 if (flags & TRACE_GRAPH_PRINT_DURATION) 1506 seq_puts(s, " DURATION "); 1507 seq_puts(s, " FUNCTION CALLS\n"); 1508 1509 /* 2nd line */ 1510 seq_putc(s, '#'); 1511 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1512 seq_puts(s, " | "); 1513 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1514 seq_puts(s, " | "); 1515 if (flags & TRACE_GRAPH_PRINT_CPU) 1516 seq_puts(s, " | "); 1517 if (flags & TRACE_GRAPH_PRINT_PROC) 1518 seq_puts(s, " | | "); 1519 if (lat) 1520 seq_puts(s, "|||| "); 1521 if (flags & TRACE_GRAPH_PRINT_DURATION) 1522 seq_puts(s, " | | "); 1523 seq_puts(s, " | | | |\n"); 1524 } 1525 1526 static void print_graph_headers(struct seq_file *s) 1527 { 1528 print_graph_headers_flags(s, tracer_flags.val); 1529 } 1530 1531 void print_graph_headers_flags(struct seq_file *s, u32 flags) 1532 { 1533 struct trace_iterator *iter = s->private; 1534 struct trace_array *tr = iter->tr; 1535 1536 if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 1537 return; 1538 1539 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) { 1540 /* print nothing if the buffers are empty */ 1541 if (trace_empty(iter)) 1542 return; 1543 1544 print_trace_header(s, iter); 1545 } 1546 1547 __print_graph_headers_flags(tr, s, flags); 1548 } 1549 1550 void graph_trace_open(struct trace_iterator *iter) 1551 { 1552 /* pid and depth on the last trace processed */ 1553 struct fgraph_data *data; 1554 gfp_t gfpflags; 1555 int cpu; 1556 1557 iter->private = NULL; 1558 1559 /* We can be called in atomic context via ftrace_dump() */ 1560 gfpflags = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL; 1561 1562 data = kzalloc(sizeof(*data), gfpflags); 1563 if (!data) 1564 goto out_err; 1565 1566 data->cpu_data = alloc_percpu_gfp(struct fgraph_cpu_data, gfpflags); 1567 if (!data->cpu_data) 1568 goto out_err_free; 1569 1570 for_each_possible_cpu(cpu) { 1571 pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); 1572 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth); 1573 int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore); 1574 int *depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1575 1576 *pid = -1; 1577 *depth = 0; 1578 *ignore = 0; 1579 *depth_irq = -1; 1580 } 1581 1582 iter->private = data; 1583 1584 return; 1585 1586 out_err_free: 1587 kfree(data); 1588 out_err: 1589 pr_warn("function graph tracer: not enough memory\n"); 1590 } 1591 1592 void graph_trace_close(struct trace_iterator *iter) 1593 { 1594 struct fgraph_data *data = iter->private; 1595 1596 if (data) { 1597 free_percpu(data->cpu_data); 1598 kfree(data); 1599 iter->private = NULL; 1600 } 1601 } 1602 1603 static int 1604 func_graph_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 1605 { 1606 if (bit == TRACE_GRAPH_PRINT_IRQS) 1607 ftrace_graph_skip_irqs = !set; 1608 1609 if (bit == TRACE_GRAPH_SLEEP_TIME) 1610 ftrace_graph_sleep_time_control(set); 1611 1612 if (bit == TRACE_GRAPH_GRAPH_TIME) 1613 ftrace_graph_graph_time_control(set); 1614 1615 if (bit == TRACE_GRAPH_ARGS) 1616 return ftrace_graph_trace_args(tr, set); 1617 1618 return 0; 1619 } 1620 1621 static struct trace_event_functions graph_functions = { 1622 .trace = print_graph_function_event, 1623 }; 1624 1625 static struct trace_event graph_trace_entry_event = { 1626 .type = TRACE_GRAPH_ENT, 1627 .funcs = &graph_functions, 1628 }; 1629 1630 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1631 static struct trace_event graph_trace_retaddr_entry_event = { 1632 .type = TRACE_GRAPH_RETADDR_ENT, 1633 .funcs = &graph_functions, 1634 }; 1635 #endif 1636 1637 static struct trace_event graph_trace_ret_event = { 1638 .type = TRACE_GRAPH_RET, 1639 .funcs = &graph_functions 1640 }; 1641 1642 static struct tracer graph_trace __tracer_data = { 1643 .name = "function_graph", 1644 .update_thresh = graph_trace_update_thresh, 1645 .open = graph_trace_open, 1646 .pipe_open = graph_trace_open, 1647 .close = graph_trace_close, 1648 .pipe_close = graph_trace_close, 1649 .init = graph_trace_init, 1650 .reset = graph_trace_reset, 1651 .print_line = print_graph_function, 1652 .print_header = print_graph_headers, 1653 .flags = &tracer_flags, 1654 .set_flag = func_graph_set_flag, 1655 .allow_instances = true, 1656 #ifdef CONFIG_FTRACE_SELFTEST 1657 .selftest = trace_selftest_startup_function_graph, 1658 #endif 1659 }; 1660 1661 1662 static ssize_t 1663 graph_depth_write(struct file *filp, const char __user *ubuf, size_t cnt, 1664 loff_t *ppos) 1665 { 1666 unsigned long val; 1667 int ret; 1668 1669 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 1670 if (ret) 1671 return ret; 1672 1673 fgraph_max_depth = val; 1674 1675 *ppos += cnt; 1676 1677 return cnt; 1678 } 1679 1680 static ssize_t 1681 graph_depth_read(struct file *filp, char __user *ubuf, size_t cnt, 1682 loff_t *ppos) 1683 { 1684 char buf[15]; /* More than enough to hold UINT_MAX + "\n"*/ 1685 int n; 1686 1687 n = sprintf(buf, "%d\n", fgraph_max_depth); 1688 1689 return simple_read_from_buffer(ubuf, cnt, ppos, buf, n); 1690 } 1691 1692 static const struct file_operations graph_depth_fops = { 1693 .open = tracing_open_generic, 1694 .write = graph_depth_write, 1695 .read = graph_depth_read, 1696 .llseek = generic_file_llseek, 1697 }; 1698 1699 static __init int init_graph_tracefs(void) 1700 { 1701 int ret; 1702 1703 ret = tracing_init_dentry(); 1704 if (ret) 1705 return 0; 1706 1707 trace_create_file("max_graph_depth", TRACE_MODE_WRITE, NULL, 1708 NULL, &graph_depth_fops); 1709 1710 return 0; 1711 } 1712 fs_initcall(init_graph_tracefs); 1713 1714 static __init int init_graph_trace(void) 1715 { 1716 max_bytes_for_cpu = snprintf(NULL, 0, "%u", nr_cpu_ids - 1); 1717 1718 if (!register_trace_event(&graph_trace_entry_event)) { 1719 pr_warn("Warning: could not register graph trace events\n"); 1720 return 1; 1721 } 1722 1723 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1724 if (!register_trace_event(&graph_trace_retaddr_entry_event)) { 1725 pr_warn("Warning: could not register graph trace retaddr events\n"); 1726 return 1; 1727 } 1728 #endif 1729 1730 if (!register_trace_event(&graph_trace_ret_event)) { 1731 pr_warn("Warning: could not register graph trace events\n"); 1732 return 1; 1733 } 1734 1735 return register_tracer(&graph_trace); 1736 } 1737 1738 core_initcall(init_graph_trace); 1739