1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * 4 * Function graph tracer. 5 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com> 6 * Mostly borrowed from function tracer which 7 * is Copyright (c) Steven Rostedt <srostedt@redhat.com> 8 * 9 */ 10 #include <linux/uaccess.h> 11 #include <linux/ftrace.h> 12 #include <linux/interrupt.h> 13 #include <linux/slab.h> 14 #include <linux/fs.h> 15 16 #include "trace.h" 17 #include "trace_output.h" 18 19 /* When set, irq functions will be ignored */ 20 static int ftrace_graph_skip_irqs; 21 22 struct fgraph_cpu_data { 23 pid_t last_pid; 24 int depth; 25 int depth_irq; 26 int ignore; 27 unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH]; 28 }; 29 30 struct fgraph_data { 31 struct fgraph_cpu_data __percpu *cpu_data; 32 33 /* Place to preserve last processed entry. */ 34 union { 35 struct ftrace_graph_ent_entry ent; 36 struct fgraph_retaddr_ent_entry rent; 37 } ent; 38 struct ftrace_graph_ret_entry ret; 39 int failed; 40 int cpu; 41 }; 42 43 #define TRACE_GRAPH_INDENT 2 44 45 unsigned int fgraph_max_depth; 46 47 static struct tracer_opt trace_opts[] = { 48 /* Display overruns? (for self-debug purpose) */ 49 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) }, 50 /* Display CPU ? */ 51 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) }, 52 /* Display Overhead ? */ 53 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) }, 54 /* Display proc name/pid */ 55 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) }, 56 /* Display duration of execution */ 57 { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) }, 58 /* Display absolute time of an entry */ 59 { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) }, 60 /* Display interrupts */ 61 { TRACER_OPT(funcgraph-irqs, TRACE_GRAPH_PRINT_IRQS) }, 62 /* Display function name after trailing } */ 63 { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) }, 64 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 65 /* Display function return value ? */ 66 { TRACER_OPT(funcgraph-retval, TRACE_GRAPH_PRINT_RETVAL) }, 67 /* Display function return value in hexadecimal format ? */ 68 { TRACER_OPT(funcgraph-retval-hex, TRACE_GRAPH_PRINT_RETVAL_HEX) }, 69 #endif 70 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 71 /* Display function return address ? */ 72 { TRACER_OPT(funcgraph-retaddr, TRACE_GRAPH_PRINT_RETADDR) }, 73 #endif 74 /* Include sleep time (scheduled out) between entry and return */ 75 { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, 76 77 #ifdef CONFIG_FUNCTION_PROFILER 78 /* Include time within nested functions */ 79 { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) }, 80 #endif 81 82 { } /* Empty entry */ 83 }; 84 85 static struct tracer_flags tracer_flags = { 86 /* Don't display overruns, proc, or tail by default */ 87 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD | 88 TRACE_GRAPH_PRINT_DURATION | TRACE_GRAPH_PRINT_IRQS | 89 TRACE_GRAPH_SLEEP_TIME | TRACE_GRAPH_GRAPH_TIME, 90 .opts = trace_opts 91 }; 92 93 static bool tracer_flags_is_set(u32 flags) 94 { 95 return (tracer_flags.val & flags) == flags; 96 } 97 98 /* 99 * DURATION column is being also used to display IRQ signs, 100 * following values are used by print_graph_irq and others 101 * to fill in space into DURATION column. 102 */ 103 enum { 104 FLAGS_FILL_FULL = 1 << TRACE_GRAPH_PRINT_FILL_SHIFT, 105 FLAGS_FILL_START = 2 << TRACE_GRAPH_PRINT_FILL_SHIFT, 106 FLAGS_FILL_END = 3 << TRACE_GRAPH_PRINT_FILL_SHIFT, 107 }; 108 109 static void 110 print_graph_duration(struct trace_array *tr, unsigned long long duration, 111 struct trace_seq *s, u32 flags); 112 113 int __trace_graph_entry(struct trace_array *tr, 114 struct ftrace_graph_ent *trace, 115 unsigned int trace_ctx) 116 { 117 struct ring_buffer_event *event; 118 struct trace_buffer *buffer = tr->array_buffer.buffer; 119 struct ftrace_graph_ent_entry *entry; 120 121 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, 122 sizeof(*entry), trace_ctx); 123 if (!event) 124 return 0; 125 entry = ring_buffer_event_data(event); 126 entry->graph_ent = *trace; 127 trace_buffer_unlock_commit_nostack(buffer, event); 128 129 return 1; 130 } 131 132 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 133 int __trace_graph_retaddr_entry(struct trace_array *tr, 134 struct ftrace_graph_ent *trace, 135 unsigned int trace_ctx, 136 unsigned long retaddr) 137 { 138 struct ring_buffer_event *event; 139 struct trace_buffer *buffer = tr->array_buffer.buffer; 140 struct fgraph_retaddr_ent_entry *entry; 141 142 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RETADDR_ENT, 143 sizeof(*entry), trace_ctx); 144 if (!event) 145 return 0; 146 entry = ring_buffer_event_data(event); 147 entry->graph_ent.func = trace->func; 148 entry->graph_ent.depth = trace->depth; 149 entry->graph_ent.retaddr = retaddr; 150 trace_buffer_unlock_commit_nostack(buffer, event); 151 152 return 1; 153 } 154 #else 155 int __trace_graph_retaddr_entry(struct trace_array *tr, 156 struct ftrace_graph_ent *trace, 157 unsigned int trace_ctx, 158 unsigned long retaddr) 159 { 160 return 1; 161 } 162 #endif 163 164 static inline int ftrace_graph_ignore_irqs(void) 165 { 166 if (!ftrace_graph_skip_irqs || trace_recursion_test(TRACE_IRQ_BIT)) 167 return 0; 168 169 return in_hardirq(); 170 } 171 172 struct fgraph_times { 173 unsigned long long calltime; 174 unsigned long long sleeptime; /* may be optional! */ 175 }; 176 177 int trace_graph_entry(struct ftrace_graph_ent *trace, 178 struct fgraph_ops *gops, 179 struct ftrace_regs *fregs) 180 { 181 unsigned long *task_var = fgraph_get_task_var(gops); 182 struct trace_array *tr = gops->private; 183 struct trace_array_cpu *data; 184 struct fgraph_times *ftimes; 185 unsigned int trace_ctx; 186 long disabled; 187 int ret = 0; 188 int cpu; 189 190 if (*task_var & TRACE_GRAPH_NOTRACE) 191 return 0; 192 193 /* 194 * Do not trace a function if it's filtered by set_graph_notrace. 195 * Make the index of ret stack negative to indicate that it should 196 * ignore further functions. But it needs its own ret stack entry 197 * to recover the original index in order to continue tracing after 198 * returning from the function. 199 */ 200 if (ftrace_graph_notrace_addr(trace->func)) { 201 *task_var |= TRACE_GRAPH_NOTRACE_BIT; 202 /* 203 * Need to return 1 to have the return called 204 * that will clear the NOTRACE bit. 205 */ 206 return 1; 207 } 208 209 if (!ftrace_trace_task(tr)) 210 return 0; 211 212 if (ftrace_graph_ignore_func(gops, trace)) 213 return 0; 214 215 if (ftrace_graph_ignore_irqs()) 216 return 0; 217 218 if (fgraph_sleep_time) { 219 /* Only need to record the calltime */ 220 ftimes = fgraph_reserve_data(gops->idx, sizeof(ftimes->calltime)); 221 } else { 222 ftimes = fgraph_reserve_data(gops->idx, sizeof(*ftimes)); 223 if (ftimes) 224 ftimes->sleeptime = current->ftrace_sleeptime; 225 } 226 if (!ftimes) 227 return 0; 228 229 ftimes->calltime = trace_clock_local(); 230 231 /* 232 * Stop here if tracing_threshold is set. We only write function return 233 * events to the ring buffer. 234 */ 235 if (tracing_thresh) 236 return 1; 237 238 preempt_disable_notrace(); 239 cpu = raw_smp_processor_id(); 240 data = per_cpu_ptr(tr->array_buffer.data, cpu); 241 disabled = atomic_read(&data->disabled); 242 if (likely(!disabled)) { 243 trace_ctx = tracing_gen_ctx(); 244 if (IS_ENABLED(CONFIG_FUNCTION_GRAPH_RETADDR) && 245 tracer_flags_is_set(TRACE_GRAPH_PRINT_RETADDR)) { 246 unsigned long retaddr = ftrace_graph_top_ret_addr(current); 247 ret = __trace_graph_retaddr_entry(tr, trace, trace_ctx, retaddr); 248 } else { 249 ret = __trace_graph_entry(tr, trace, trace_ctx); 250 } 251 } 252 preempt_enable_notrace(); 253 254 return ret; 255 } 256 257 static void 258 __trace_graph_function(struct trace_array *tr, 259 unsigned long ip, unsigned int trace_ctx) 260 { 261 u64 time = trace_clock_local(); 262 struct ftrace_graph_ent ent = { 263 .func = ip, 264 .depth = 0, 265 }; 266 struct ftrace_graph_ret ret = { 267 .func = ip, 268 .depth = 0, 269 }; 270 271 __trace_graph_entry(tr, &ent, trace_ctx); 272 __trace_graph_return(tr, &ret, trace_ctx, time, time); 273 } 274 275 void 276 trace_graph_function(struct trace_array *tr, 277 unsigned long ip, unsigned long parent_ip, 278 unsigned int trace_ctx) 279 { 280 __trace_graph_function(tr, ip, trace_ctx); 281 } 282 283 void __trace_graph_return(struct trace_array *tr, 284 struct ftrace_graph_ret *trace, 285 unsigned int trace_ctx, 286 u64 calltime, u64 rettime) 287 { 288 struct ring_buffer_event *event; 289 struct trace_buffer *buffer = tr->array_buffer.buffer; 290 struct ftrace_graph_ret_entry *entry; 291 292 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, 293 sizeof(*entry), trace_ctx); 294 if (!event) 295 return; 296 entry = ring_buffer_event_data(event); 297 entry->ret = *trace; 298 entry->calltime = calltime; 299 entry->rettime = rettime; 300 trace_buffer_unlock_commit_nostack(buffer, event); 301 } 302 303 static void handle_nosleeptime(struct ftrace_graph_ret *trace, 304 struct fgraph_times *ftimes, 305 int size) 306 { 307 if (fgraph_sleep_time || size < sizeof(*ftimes)) 308 return; 309 310 ftimes->calltime += current->ftrace_sleeptime - ftimes->sleeptime; 311 } 312 313 void trace_graph_return(struct ftrace_graph_ret *trace, 314 struct fgraph_ops *gops, struct ftrace_regs *fregs) 315 { 316 unsigned long *task_var = fgraph_get_task_var(gops); 317 struct trace_array *tr = gops->private; 318 struct trace_array_cpu *data; 319 struct fgraph_times *ftimes; 320 unsigned int trace_ctx; 321 u64 calltime, rettime; 322 long disabled; 323 int size; 324 int cpu; 325 326 rettime = trace_clock_local(); 327 328 ftrace_graph_addr_finish(gops, trace); 329 330 if (*task_var & TRACE_GRAPH_NOTRACE) { 331 *task_var &= ~TRACE_GRAPH_NOTRACE; 332 return; 333 } 334 335 ftimes = fgraph_retrieve_data(gops->idx, &size); 336 if (!ftimes) 337 return; 338 339 handle_nosleeptime(trace, ftimes, size); 340 341 calltime = ftimes->calltime; 342 343 preempt_disable_notrace(); 344 cpu = raw_smp_processor_id(); 345 data = per_cpu_ptr(tr->array_buffer.data, cpu); 346 disabled = atomic_read(&data->disabled); 347 if (likely(!disabled)) { 348 trace_ctx = tracing_gen_ctx(); 349 __trace_graph_return(tr, trace, trace_ctx, calltime, rettime); 350 } 351 preempt_enable_notrace(); 352 } 353 354 static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, 355 struct fgraph_ops *gops, 356 struct ftrace_regs *fregs) 357 { 358 struct fgraph_times *ftimes; 359 int size; 360 361 ftrace_graph_addr_finish(gops, trace); 362 363 if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { 364 trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); 365 return; 366 } 367 368 ftimes = fgraph_retrieve_data(gops->idx, &size); 369 if (!ftimes) 370 return; 371 372 handle_nosleeptime(trace, ftimes, size); 373 374 if (tracing_thresh && 375 (trace_clock_local() - ftimes->calltime < tracing_thresh)) 376 return; 377 else 378 trace_graph_return(trace, gops, fregs); 379 } 380 381 static struct fgraph_ops funcgraph_ops = { 382 .entryfunc = &trace_graph_entry, 383 .retfunc = &trace_graph_return, 384 }; 385 386 int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) 387 { 388 struct fgraph_ops *gops; 389 390 gops = kzalloc(sizeof(*gops), GFP_KERNEL); 391 if (!gops) 392 return -ENOMEM; 393 394 gops->entryfunc = &trace_graph_entry; 395 gops->retfunc = &trace_graph_return; 396 397 tr->gops = gops; 398 gops->private = tr; 399 400 fgraph_init_ops(&gops->ops, ops); 401 402 return 0; 403 } 404 405 void free_fgraph_ops(struct trace_array *tr) 406 { 407 kfree(tr->gops); 408 } 409 410 __init void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) 411 { 412 tr->gops = &funcgraph_ops; 413 funcgraph_ops.private = tr; 414 fgraph_init_ops(&tr->gops->ops, ops); 415 } 416 417 static int graph_trace_init(struct trace_array *tr) 418 { 419 int ret; 420 421 tr->gops->entryfunc = trace_graph_entry; 422 423 if (tracing_thresh) 424 tr->gops->retfunc = trace_graph_thresh_return; 425 else 426 tr->gops->retfunc = trace_graph_return; 427 428 /* Make gops functions are visible before we start tracing */ 429 smp_mb(); 430 431 ret = register_ftrace_graph(tr->gops); 432 if (ret) 433 return ret; 434 tracing_start_cmdline_record(); 435 436 return 0; 437 } 438 439 static void graph_trace_reset(struct trace_array *tr) 440 { 441 tracing_stop_cmdline_record(); 442 unregister_ftrace_graph(tr->gops); 443 } 444 445 static int graph_trace_update_thresh(struct trace_array *tr) 446 { 447 graph_trace_reset(tr); 448 return graph_trace_init(tr); 449 } 450 451 static int max_bytes_for_cpu; 452 453 static void print_graph_cpu(struct trace_seq *s, int cpu) 454 { 455 /* 456 * Start with a space character - to make it stand out 457 * to the right a bit when trace output is pasted into 458 * email: 459 */ 460 trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu); 461 } 462 463 #define TRACE_GRAPH_PROCINFO_LENGTH 14 464 465 static void print_graph_proc(struct trace_seq *s, pid_t pid) 466 { 467 char comm[TASK_COMM_LEN]; 468 /* sign + log10(MAX_INT) + '\0' */ 469 char pid_str[11]; 470 int spaces = 0; 471 int len; 472 int i; 473 474 trace_find_cmdline(pid, comm); 475 comm[7] = '\0'; 476 sprintf(pid_str, "%d", pid); 477 478 /* 1 stands for the "-" character */ 479 len = strlen(comm) + strlen(pid_str) + 1; 480 481 if (len < TRACE_GRAPH_PROCINFO_LENGTH) 482 spaces = TRACE_GRAPH_PROCINFO_LENGTH - len; 483 484 /* First spaces to align center */ 485 for (i = 0; i < spaces / 2; i++) 486 trace_seq_putc(s, ' '); 487 488 trace_seq_printf(s, "%s-%s", comm, pid_str); 489 490 /* Last spaces to align center */ 491 for (i = 0; i < spaces - (spaces / 2); i++) 492 trace_seq_putc(s, ' '); 493 } 494 495 496 static void print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry) 497 { 498 trace_seq_putc(s, ' '); 499 trace_print_lat_fmt(s, entry); 500 trace_seq_puts(s, " | "); 501 } 502 503 /* If the pid changed since the last trace, output this event */ 504 static void 505 verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data) 506 { 507 pid_t prev_pid; 508 pid_t *last_pid; 509 510 if (!data) 511 return; 512 513 last_pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); 514 515 if (*last_pid == pid) 516 return; 517 518 prev_pid = *last_pid; 519 *last_pid = pid; 520 521 if (prev_pid == -1) 522 return; 523 /* 524 * Context-switch trace line: 525 526 ------------------------------------------ 527 | 1) migration/0--1 => sshd-1755 528 ------------------------------------------ 529 530 */ 531 trace_seq_puts(s, " ------------------------------------------\n"); 532 print_graph_cpu(s, cpu); 533 print_graph_proc(s, prev_pid); 534 trace_seq_puts(s, " => "); 535 print_graph_proc(s, pid); 536 trace_seq_puts(s, "\n ------------------------------------------\n\n"); 537 } 538 539 static struct ftrace_graph_ret_entry * 540 get_return_for_leaf(struct trace_iterator *iter, 541 struct ftrace_graph_ent_entry *curr) 542 { 543 struct fgraph_data *data = iter->private; 544 struct ring_buffer_iter *ring_iter = NULL; 545 struct ring_buffer_event *event; 546 struct ftrace_graph_ret_entry *next; 547 548 /* 549 * If the previous output failed to write to the seq buffer, 550 * then we just reuse the data from before. 551 */ 552 if (data && data->failed) { 553 curr = &data->ent.ent; 554 next = &data->ret; 555 } else { 556 557 ring_iter = trace_buffer_iter(iter, iter->cpu); 558 559 /* First peek to compare current entry and the next one */ 560 if (ring_iter) 561 event = ring_buffer_iter_peek(ring_iter, NULL); 562 else { 563 /* 564 * We need to consume the current entry to see 565 * the next one. 566 */ 567 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, 568 NULL, NULL); 569 event = ring_buffer_peek(iter->array_buffer->buffer, iter->cpu, 570 NULL, NULL); 571 } 572 573 if (!event) 574 return NULL; 575 576 next = ring_buffer_event_data(event); 577 578 if (data) { 579 /* 580 * Save current and next entries for later reference 581 * if the output fails. 582 */ 583 if (unlikely(curr->ent.type == TRACE_GRAPH_RETADDR_ENT)) 584 data->ent.rent = *(struct fgraph_retaddr_ent_entry *)curr; 585 else 586 data->ent.ent = *curr; 587 /* 588 * If the next event is not a return type, then 589 * we only care about what type it is. Otherwise we can 590 * safely copy the entire event. 591 */ 592 if (next->ent.type == TRACE_GRAPH_RET) 593 data->ret = *next; 594 else 595 data->ret.ent.type = next->ent.type; 596 } 597 } 598 599 if (next->ent.type != TRACE_GRAPH_RET) 600 return NULL; 601 602 if (curr->ent.pid != next->ent.pid || 603 curr->graph_ent.func != next->ret.func) 604 return NULL; 605 606 /* this is a leaf, now advance the iterator */ 607 if (ring_iter) 608 ring_buffer_iter_advance(ring_iter); 609 610 return next; 611 } 612 613 static void print_graph_abs_time(u64 t, struct trace_seq *s) 614 { 615 unsigned long usecs_rem; 616 617 usecs_rem = do_div(t, NSEC_PER_SEC); 618 usecs_rem /= 1000; 619 620 trace_seq_printf(s, "%5lu.%06lu | ", 621 (unsigned long)t, usecs_rem); 622 } 623 624 static void 625 print_graph_rel_time(struct trace_iterator *iter, struct trace_seq *s) 626 { 627 unsigned long long usecs; 628 629 usecs = iter->ts - iter->array_buffer->time_start; 630 do_div(usecs, NSEC_PER_USEC); 631 632 trace_seq_printf(s, "%9llu us | ", usecs); 633 } 634 635 static void 636 print_graph_irq(struct trace_iterator *iter, unsigned long addr, 637 enum trace_type type, int cpu, pid_t pid, u32 flags) 638 { 639 struct trace_array *tr = iter->tr; 640 struct trace_seq *s = &iter->seq; 641 struct trace_entry *ent = iter->ent; 642 643 addr += iter->tr->text_delta; 644 645 if (addr < (unsigned long)__irqentry_text_start || 646 addr >= (unsigned long)__irqentry_text_end) 647 return; 648 649 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 650 /* Absolute time */ 651 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 652 print_graph_abs_time(iter->ts, s); 653 654 /* Relative time */ 655 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 656 print_graph_rel_time(iter, s); 657 658 /* Cpu */ 659 if (flags & TRACE_GRAPH_PRINT_CPU) 660 print_graph_cpu(s, cpu); 661 662 /* Proc */ 663 if (flags & TRACE_GRAPH_PRINT_PROC) { 664 print_graph_proc(s, pid); 665 trace_seq_puts(s, " | "); 666 } 667 668 /* Latency format */ 669 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 670 print_graph_lat_fmt(s, ent); 671 } 672 673 /* No overhead */ 674 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_START); 675 676 if (type == TRACE_GRAPH_ENT) 677 trace_seq_puts(s, "==========>"); 678 else 679 trace_seq_puts(s, "<=========="); 680 681 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_END); 682 trace_seq_putc(s, '\n'); 683 } 684 685 void 686 trace_print_graph_duration(unsigned long long duration, struct trace_seq *s) 687 { 688 unsigned long nsecs_rem = do_div(duration, 1000); 689 /* log10(ULONG_MAX) + '\0' */ 690 char usecs_str[21]; 691 char nsecs_str[5]; 692 int len; 693 int i; 694 695 sprintf(usecs_str, "%lu", (unsigned long) duration); 696 697 /* Print msecs */ 698 trace_seq_printf(s, "%s", usecs_str); 699 700 len = strlen(usecs_str); 701 702 /* Print nsecs (we don't want to exceed 7 numbers) */ 703 if (len < 7) { 704 size_t slen = min_t(size_t, sizeof(nsecs_str), 8UL - len); 705 706 snprintf(nsecs_str, slen, "%03lu", nsecs_rem); 707 trace_seq_printf(s, ".%s", nsecs_str); 708 len += strlen(nsecs_str) + 1; 709 } 710 711 trace_seq_puts(s, " us "); 712 713 /* Print remaining spaces to fit the row's width */ 714 for (i = len; i < 8; i++) 715 trace_seq_putc(s, ' '); 716 } 717 718 static void 719 print_graph_duration(struct trace_array *tr, unsigned long long duration, 720 struct trace_seq *s, u32 flags) 721 { 722 if (!(flags & TRACE_GRAPH_PRINT_DURATION) || 723 !(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 724 return; 725 726 /* No real adata, just filling the column with spaces */ 727 switch (flags & TRACE_GRAPH_PRINT_FILL_MASK) { 728 case FLAGS_FILL_FULL: 729 trace_seq_puts(s, " | "); 730 return; 731 case FLAGS_FILL_START: 732 trace_seq_puts(s, " "); 733 return; 734 case FLAGS_FILL_END: 735 trace_seq_puts(s, " |"); 736 return; 737 } 738 739 /* Signal a overhead of time execution to the output */ 740 if (flags & TRACE_GRAPH_PRINT_OVERHEAD) 741 trace_seq_printf(s, "%c ", trace_find_mark(duration)); 742 else 743 trace_seq_puts(s, " "); 744 745 trace_print_graph_duration(duration, s); 746 trace_seq_puts(s, "| "); 747 } 748 749 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 750 #define __TRACE_GRAPH_PRINT_RETVAL TRACE_GRAPH_PRINT_RETVAL 751 #else 752 #define __TRACE_GRAPH_PRINT_RETVAL 0 753 #endif 754 755 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 756 #define __TRACE_GRAPH_PRINT_RETADDR TRACE_GRAPH_PRINT_RETADDR 757 static void print_graph_retaddr(struct trace_seq *s, struct fgraph_retaddr_ent_entry *entry, 758 u32 trace_flags, bool comment) 759 { 760 if (comment) 761 trace_seq_puts(s, " /*"); 762 763 trace_seq_puts(s, " <-"); 764 seq_print_ip_sym(s, entry->graph_ent.retaddr, trace_flags | TRACE_ITER_SYM_OFFSET); 765 766 if (comment) 767 trace_seq_puts(s, " */"); 768 } 769 #else 770 #define __TRACE_GRAPH_PRINT_RETADDR 0 771 #define print_graph_retaddr(_seq, _entry, _tflags, _comment) do { } while (0) 772 #endif 773 774 #if defined(CONFIG_FUNCTION_GRAPH_RETVAL) || defined(CONFIG_FUNCTION_GRAPH_RETADDR) 775 776 static void print_graph_retval(struct trace_seq *s, struct ftrace_graph_ent_entry *entry, 777 struct ftrace_graph_ret *graph_ret, void *func, 778 u32 opt_flags, u32 trace_flags) 779 { 780 unsigned long err_code = 0; 781 unsigned long retval = 0; 782 bool print_retaddr = false; 783 bool print_retval = false; 784 bool hex_format = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL_HEX); 785 786 #ifdef CONFIG_FUNCTION_GRAPH_RETVAL 787 retval = graph_ret->retval; 788 print_retval = !!(opt_flags & TRACE_GRAPH_PRINT_RETVAL); 789 #endif 790 791 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 792 print_retaddr = !!(opt_flags & TRACE_GRAPH_PRINT_RETADDR); 793 #endif 794 795 if (print_retval && retval && !hex_format) { 796 /* Check if the return value matches the negative format */ 797 if (IS_ENABLED(CONFIG_64BIT) && (retval & BIT(31)) && 798 (((u64)retval) >> 32) == 0) { 799 err_code = sign_extend64(retval, 31); 800 } else { 801 err_code = retval; 802 } 803 804 if (!IS_ERR_VALUE(err_code)) 805 err_code = 0; 806 } 807 808 if (entry) { 809 if (entry->ent.type != TRACE_GRAPH_RETADDR_ENT) 810 print_retaddr = false; 811 812 trace_seq_printf(s, "%ps();", func); 813 if (print_retval || print_retaddr) 814 trace_seq_puts(s, " /*"); 815 else 816 trace_seq_putc(s, '\n'); 817 } else { 818 print_retaddr = false; 819 trace_seq_printf(s, "} /* %ps", func); 820 } 821 822 if (print_retaddr) 823 print_graph_retaddr(s, (struct fgraph_retaddr_ent_entry *)entry, 824 trace_flags, false); 825 826 if (print_retval) { 827 if (hex_format || (err_code == 0)) 828 trace_seq_printf(s, " ret=0x%lx", retval); 829 else 830 trace_seq_printf(s, " ret=%ld", err_code); 831 } 832 833 if (!entry || print_retval || print_retaddr) 834 trace_seq_puts(s, " */\n"); 835 } 836 837 #else 838 839 #define print_graph_retval(_seq, _ent, _ret, _func, _opt_flags, _trace_flags) do {} while (0) 840 841 #endif 842 843 /* Case of a leaf function on its call entry */ 844 static enum print_line_t 845 print_graph_entry_leaf(struct trace_iterator *iter, 846 struct ftrace_graph_ent_entry *entry, 847 struct ftrace_graph_ret_entry *ret_entry, 848 struct trace_seq *s, u32 flags) 849 { 850 struct fgraph_data *data = iter->private; 851 struct trace_array *tr = iter->tr; 852 struct ftrace_graph_ret *graph_ret; 853 struct ftrace_graph_ent *call; 854 unsigned long long duration; 855 unsigned long func; 856 int cpu = iter->cpu; 857 int i; 858 859 graph_ret = &ret_entry->ret; 860 call = &entry->graph_ent; 861 duration = ret_entry->rettime - ret_entry->calltime; 862 863 func = call->func + iter->tr->text_delta; 864 865 if (data) { 866 struct fgraph_cpu_data *cpu_data; 867 868 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 869 870 /* 871 * Comments display at + 1 to depth. Since 872 * this is a leaf function, keep the comments 873 * equal to this depth. 874 */ 875 cpu_data->depth = call->depth - 1; 876 877 /* No need to keep this function around for this depth */ 878 if (call->depth < FTRACE_RETFUNC_DEPTH && 879 !WARN_ON_ONCE(call->depth < 0)) 880 cpu_data->enter_funcs[call->depth] = 0; 881 } 882 883 /* Overhead and duration */ 884 print_graph_duration(tr, duration, s, flags); 885 886 /* Function */ 887 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) 888 trace_seq_putc(s, ' '); 889 890 /* 891 * Write out the function return value or return address 892 */ 893 if (flags & (__TRACE_GRAPH_PRINT_RETVAL | __TRACE_GRAPH_PRINT_RETADDR)) { 894 print_graph_retval(s, entry, graph_ret, 895 (void *)graph_ret->func + iter->tr->text_delta, 896 flags, tr->trace_flags); 897 } else { 898 trace_seq_printf(s, "%ps();\n", (void *)func); 899 } 900 901 print_graph_irq(iter, graph_ret->func, TRACE_GRAPH_RET, 902 cpu, iter->ent->pid, flags); 903 904 return trace_handle_return(s); 905 } 906 907 static enum print_line_t 908 print_graph_entry_nested(struct trace_iterator *iter, 909 struct ftrace_graph_ent_entry *entry, 910 struct trace_seq *s, int cpu, u32 flags) 911 { 912 struct ftrace_graph_ent *call = &entry->graph_ent; 913 struct fgraph_data *data = iter->private; 914 struct trace_array *tr = iter->tr; 915 unsigned long func; 916 int i; 917 918 if (data) { 919 struct fgraph_cpu_data *cpu_data; 920 int cpu = iter->cpu; 921 922 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 923 cpu_data->depth = call->depth; 924 925 /* Save this function pointer to see if the exit matches */ 926 if (call->depth < FTRACE_RETFUNC_DEPTH && 927 !WARN_ON_ONCE(call->depth < 0)) 928 cpu_data->enter_funcs[call->depth] = call->func; 929 } 930 931 /* No time */ 932 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_FULL); 933 934 /* Function */ 935 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) 936 trace_seq_putc(s, ' '); 937 938 func = call->func + iter->tr->text_delta; 939 940 trace_seq_printf(s, "%ps() {", (void *)func); 941 if (flags & __TRACE_GRAPH_PRINT_RETADDR && 942 entry->ent.type == TRACE_GRAPH_RETADDR_ENT) 943 print_graph_retaddr(s, (struct fgraph_retaddr_ent_entry *)entry, 944 tr->trace_flags, true); 945 trace_seq_putc(s, '\n'); 946 947 if (trace_seq_has_overflowed(s)) 948 return TRACE_TYPE_PARTIAL_LINE; 949 950 /* 951 * we already consumed the current entry to check the next one 952 * and see if this is a leaf. 953 */ 954 return TRACE_TYPE_NO_CONSUME; 955 } 956 957 static void 958 print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s, 959 int type, unsigned long addr, u32 flags) 960 { 961 struct fgraph_data *data = iter->private; 962 struct trace_entry *ent = iter->ent; 963 struct trace_array *tr = iter->tr; 964 int cpu = iter->cpu; 965 966 /* Pid */ 967 verif_pid(s, ent->pid, cpu, data); 968 969 if (type) 970 /* Interrupt */ 971 print_graph_irq(iter, addr, type, cpu, ent->pid, flags); 972 973 if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 974 return; 975 976 /* Absolute time */ 977 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 978 print_graph_abs_time(iter->ts, s); 979 980 /* Relative time */ 981 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 982 print_graph_rel_time(iter, s); 983 984 /* Cpu */ 985 if (flags & TRACE_GRAPH_PRINT_CPU) 986 print_graph_cpu(s, cpu); 987 988 /* Proc */ 989 if (flags & TRACE_GRAPH_PRINT_PROC) { 990 print_graph_proc(s, ent->pid); 991 trace_seq_puts(s, " | "); 992 } 993 994 /* Latency format */ 995 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 996 print_graph_lat_fmt(s, ent); 997 998 return; 999 } 1000 1001 /* 1002 * Entry check for irq code 1003 * 1004 * returns 1 if 1005 * - we are inside irq code 1006 * - we just entered irq code 1007 * 1008 * returns 0 if 1009 * - funcgraph-interrupts option is set 1010 * - we are not inside irq code 1011 */ 1012 static int 1013 check_irq_entry(struct trace_iterator *iter, u32 flags, 1014 unsigned long addr, int depth) 1015 { 1016 int cpu = iter->cpu; 1017 int *depth_irq; 1018 struct fgraph_data *data = iter->private; 1019 1020 addr += iter->tr->text_delta; 1021 1022 /* 1023 * If we are either displaying irqs, or we got called as 1024 * a graph event and private data does not exist, 1025 * then we bypass the irq check. 1026 */ 1027 if ((flags & TRACE_GRAPH_PRINT_IRQS) || 1028 (!data)) 1029 return 0; 1030 1031 depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1032 1033 /* 1034 * We are inside the irq code 1035 */ 1036 if (*depth_irq >= 0) 1037 return 1; 1038 1039 if ((addr < (unsigned long)__irqentry_text_start) || 1040 (addr >= (unsigned long)__irqentry_text_end)) 1041 return 0; 1042 1043 /* 1044 * We are entering irq code. 1045 */ 1046 *depth_irq = depth; 1047 return 1; 1048 } 1049 1050 /* 1051 * Return check for irq code 1052 * 1053 * returns 1 if 1054 * - we are inside irq code 1055 * - we just left irq code 1056 * 1057 * returns 0 if 1058 * - funcgraph-interrupts option is set 1059 * - we are not inside irq code 1060 */ 1061 static int 1062 check_irq_return(struct trace_iterator *iter, u32 flags, int depth) 1063 { 1064 int cpu = iter->cpu; 1065 int *depth_irq; 1066 struct fgraph_data *data = iter->private; 1067 1068 /* 1069 * If we are either displaying irqs, or we got called as 1070 * a graph event and private data does not exist, 1071 * then we bypass the irq check. 1072 */ 1073 if ((flags & TRACE_GRAPH_PRINT_IRQS) || 1074 (!data)) 1075 return 0; 1076 1077 depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1078 1079 /* 1080 * We are not inside the irq code. 1081 */ 1082 if (*depth_irq == -1) 1083 return 0; 1084 1085 /* 1086 * We are inside the irq code, and this is returning entry. 1087 * Let's not trace it and clear the entry depth, since 1088 * we are out of irq code. 1089 * 1090 * This condition ensures that we 'leave the irq code' once 1091 * we are out of the entry depth. Thus protecting us from 1092 * the RETURN entry loss. 1093 */ 1094 if (*depth_irq >= depth) { 1095 *depth_irq = -1; 1096 return 1; 1097 } 1098 1099 /* 1100 * We are inside the irq code, and this is not the entry. 1101 */ 1102 return 1; 1103 } 1104 1105 static enum print_line_t 1106 print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, 1107 struct trace_iterator *iter, u32 flags) 1108 { 1109 struct fgraph_data *data = iter->private; 1110 struct ftrace_graph_ent *call = &field->graph_ent; 1111 struct ftrace_graph_ret_entry *leaf_ret; 1112 static enum print_line_t ret; 1113 int cpu = iter->cpu; 1114 1115 if (check_irq_entry(iter, flags, call->func, call->depth)) 1116 return TRACE_TYPE_HANDLED; 1117 1118 print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func, flags); 1119 1120 leaf_ret = get_return_for_leaf(iter, field); 1121 if (leaf_ret) 1122 ret = print_graph_entry_leaf(iter, field, leaf_ret, s, flags); 1123 else 1124 ret = print_graph_entry_nested(iter, field, s, cpu, flags); 1125 1126 if (data) { 1127 /* 1128 * If we failed to write our output, then we need to make 1129 * note of it. Because we already consumed our entry. 1130 */ 1131 if (s->full) { 1132 data->failed = 1; 1133 data->cpu = cpu; 1134 } else 1135 data->failed = 0; 1136 } 1137 1138 return ret; 1139 } 1140 1141 static enum print_line_t 1142 print_graph_return(struct ftrace_graph_ret_entry *retentry, struct trace_seq *s, 1143 struct trace_entry *ent, struct trace_iterator *iter, 1144 u32 flags) 1145 { 1146 struct ftrace_graph_ret *trace = &retentry->ret; 1147 u64 calltime = retentry->calltime; 1148 u64 rettime = retentry->rettime; 1149 unsigned long long duration = rettime - calltime; 1150 struct fgraph_data *data = iter->private; 1151 struct trace_array *tr = iter->tr; 1152 unsigned long func; 1153 pid_t pid = ent->pid; 1154 int cpu = iter->cpu; 1155 int func_match = 1; 1156 int i; 1157 1158 func = trace->func + iter->tr->text_delta; 1159 1160 if (check_irq_return(iter, flags, trace->depth)) 1161 return TRACE_TYPE_HANDLED; 1162 1163 if (data) { 1164 struct fgraph_cpu_data *cpu_data; 1165 int cpu = iter->cpu; 1166 1167 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 1168 1169 /* 1170 * Comments display at + 1 to depth. This is the 1171 * return from a function, we now want the comments 1172 * to display at the same level of the bracket. 1173 */ 1174 cpu_data->depth = trace->depth - 1; 1175 1176 if (trace->depth < FTRACE_RETFUNC_DEPTH && 1177 !WARN_ON_ONCE(trace->depth < 0)) { 1178 if (cpu_data->enter_funcs[trace->depth] != trace->func) 1179 func_match = 0; 1180 cpu_data->enter_funcs[trace->depth] = 0; 1181 } 1182 } 1183 1184 print_graph_prologue(iter, s, 0, 0, flags); 1185 1186 /* Overhead and duration */ 1187 print_graph_duration(tr, duration, s, flags); 1188 1189 /* Closing brace */ 1190 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) 1191 trace_seq_putc(s, ' '); 1192 1193 /* 1194 * Always write out the function name and its return value if the 1195 * funcgraph-retval option is enabled. 1196 */ 1197 if (flags & __TRACE_GRAPH_PRINT_RETVAL) { 1198 print_graph_retval(s, NULL, trace, (void *)func, flags, tr->trace_flags); 1199 } else { 1200 /* 1201 * If the return function does not have a matching entry, 1202 * then the entry was lost. Instead of just printing 1203 * the '}' and letting the user guess what function this 1204 * belongs to, write out the function name. Always do 1205 * that if the funcgraph-tail option is enabled. 1206 */ 1207 if (func_match && !(flags & TRACE_GRAPH_PRINT_TAIL)) 1208 trace_seq_puts(s, "}\n"); 1209 else 1210 trace_seq_printf(s, "} /* %ps */\n", (void *)func); 1211 } 1212 1213 /* Overrun */ 1214 if (flags & TRACE_GRAPH_PRINT_OVERRUN) 1215 trace_seq_printf(s, " (Overruns: %u)\n", 1216 trace->overrun); 1217 1218 print_graph_irq(iter, trace->func, TRACE_GRAPH_RET, 1219 cpu, pid, flags); 1220 1221 return trace_handle_return(s); 1222 } 1223 1224 static enum print_line_t 1225 print_graph_comment(struct trace_seq *s, struct trace_entry *ent, 1226 struct trace_iterator *iter, u32 flags) 1227 { 1228 struct trace_array *tr = iter->tr; 1229 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 1230 struct fgraph_data *data = iter->private; 1231 struct trace_event *event; 1232 int depth = 0; 1233 int ret; 1234 int i; 1235 1236 if (data) 1237 depth = per_cpu_ptr(data->cpu_data, iter->cpu)->depth; 1238 1239 print_graph_prologue(iter, s, 0, 0, flags); 1240 1241 /* No time */ 1242 print_graph_duration(tr, 0, s, flags | FLAGS_FILL_FULL); 1243 1244 /* Indentation */ 1245 if (depth > 0) 1246 for (i = 0; i < (depth + 1) * TRACE_GRAPH_INDENT; i++) 1247 trace_seq_putc(s, ' '); 1248 1249 /* The comment */ 1250 trace_seq_puts(s, "/* "); 1251 1252 switch (iter->ent->type) { 1253 case TRACE_BPUTS: 1254 ret = trace_print_bputs_msg_only(iter); 1255 if (ret != TRACE_TYPE_HANDLED) 1256 return ret; 1257 break; 1258 case TRACE_BPRINT: 1259 ret = trace_print_bprintk_msg_only(iter); 1260 if (ret != TRACE_TYPE_HANDLED) 1261 return ret; 1262 break; 1263 case TRACE_PRINT: 1264 ret = trace_print_printk_msg_only(iter); 1265 if (ret != TRACE_TYPE_HANDLED) 1266 return ret; 1267 break; 1268 default: 1269 event = ftrace_find_event(ent->type); 1270 if (!event) 1271 return TRACE_TYPE_UNHANDLED; 1272 1273 ret = event->funcs->trace(iter, sym_flags, event); 1274 if (ret != TRACE_TYPE_HANDLED) 1275 return ret; 1276 } 1277 1278 if (trace_seq_has_overflowed(s)) 1279 goto out; 1280 1281 /* Strip ending newline */ 1282 if (s->buffer[s->seq.len - 1] == '\n') { 1283 s->buffer[s->seq.len - 1] = '\0'; 1284 s->seq.len--; 1285 } 1286 1287 trace_seq_puts(s, " */\n"); 1288 out: 1289 return trace_handle_return(s); 1290 } 1291 1292 1293 enum print_line_t 1294 print_graph_function_flags(struct trace_iterator *iter, u32 flags) 1295 { 1296 struct ftrace_graph_ent_entry *field; 1297 struct fgraph_data *data = iter->private; 1298 struct trace_entry *entry = iter->ent; 1299 struct trace_seq *s = &iter->seq; 1300 int cpu = iter->cpu; 1301 int ret; 1302 1303 if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) { 1304 per_cpu_ptr(data->cpu_data, cpu)->ignore = 0; 1305 return TRACE_TYPE_HANDLED; 1306 } 1307 1308 /* 1309 * If the last output failed, there's a possibility we need 1310 * to print out the missing entry which would never go out. 1311 */ 1312 if (data && data->failed) { 1313 field = &data->ent.ent; 1314 iter->cpu = data->cpu; 1315 ret = print_graph_entry(field, s, iter, flags); 1316 if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) { 1317 per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1; 1318 ret = TRACE_TYPE_NO_CONSUME; 1319 } 1320 iter->cpu = cpu; 1321 return ret; 1322 } 1323 1324 switch (entry->type) { 1325 case TRACE_GRAPH_ENT: { 1326 /* 1327 * print_graph_entry() may consume the current event, 1328 * thus @field may become invalid, so we need to save it. 1329 * sizeof(struct ftrace_graph_ent_entry) is very small, 1330 * it can be safely saved at the stack. 1331 */ 1332 struct ftrace_graph_ent_entry saved; 1333 trace_assign_type(field, entry); 1334 saved = *field; 1335 return print_graph_entry(&saved, s, iter, flags); 1336 } 1337 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1338 case TRACE_GRAPH_RETADDR_ENT: { 1339 struct fgraph_retaddr_ent_entry saved; 1340 struct fgraph_retaddr_ent_entry *rfield; 1341 1342 trace_assign_type(rfield, entry); 1343 saved = *rfield; 1344 return print_graph_entry((struct ftrace_graph_ent_entry *)&saved, s, iter, flags); 1345 } 1346 #endif 1347 case TRACE_GRAPH_RET: { 1348 struct ftrace_graph_ret_entry *field; 1349 trace_assign_type(field, entry); 1350 return print_graph_return(field, s, entry, iter, flags); 1351 } 1352 case TRACE_STACK: 1353 case TRACE_FN: 1354 /* dont trace stack and functions as comments */ 1355 return TRACE_TYPE_UNHANDLED; 1356 1357 default: 1358 return print_graph_comment(s, entry, iter, flags); 1359 } 1360 1361 return TRACE_TYPE_HANDLED; 1362 } 1363 1364 static enum print_line_t 1365 print_graph_function(struct trace_iterator *iter) 1366 { 1367 return print_graph_function_flags(iter, tracer_flags.val); 1368 } 1369 1370 static enum print_line_t 1371 print_graph_function_event(struct trace_iterator *iter, int flags, 1372 struct trace_event *event) 1373 { 1374 return print_graph_function(iter); 1375 } 1376 1377 static void print_lat_header(struct seq_file *s, u32 flags) 1378 { 1379 static const char spaces[] = " " /* 16 spaces */ 1380 " " /* 4 spaces */ 1381 " "; /* 17 spaces */ 1382 int size = 0; 1383 1384 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1385 size += 16; 1386 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1387 size += 16; 1388 if (flags & TRACE_GRAPH_PRINT_CPU) 1389 size += 4; 1390 if (flags & TRACE_GRAPH_PRINT_PROC) 1391 size += 17; 1392 1393 seq_printf(s, "#%.*s _-----=> irqs-off \n", size, spaces); 1394 seq_printf(s, "#%.*s / _----=> need-resched \n", size, spaces); 1395 seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces); 1396 seq_printf(s, "#%.*s|| / _--=> preempt-depth \n", size, spaces); 1397 seq_printf(s, "#%.*s||| / \n", size, spaces); 1398 } 1399 1400 static void __print_graph_headers_flags(struct trace_array *tr, 1401 struct seq_file *s, u32 flags) 1402 { 1403 int lat = tr->trace_flags & TRACE_ITER_LATENCY_FMT; 1404 1405 if (lat) 1406 print_lat_header(s, flags); 1407 1408 /* 1st line */ 1409 seq_putc(s, '#'); 1410 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1411 seq_puts(s, " TIME "); 1412 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1413 seq_puts(s, " REL TIME "); 1414 if (flags & TRACE_GRAPH_PRINT_CPU) 1415 seq_puts(s, " CPU"); 1416 if (flags & TRACE_GRAPH_PRINT_PROC) 1417 seq_puts(s, " TASK/PID "); 1418 if (lat) 1419 seq_puts(s, "|||| "); 1420 if (flags & TRACE_GRAPH_PRINT_DURATION) 1421 seq_puts(s, " DURATION "); 1422 seq_puts(s, " FUNCTION CALLS\n"); 1423 1424 /* 2nd line */ 1425 seq_putc(s, '#'); 1426 if (flags & TRACE_GRAPH_PRINT_ABS_TIME) 1427 seq_puts(s, " | "); 1428 if (flags & TRACE_GRAPH_PRINT_REL_TIME) 1429 seq_puts(s, " | "); 1430 if (flags & TRACE_GRAPH_PRINT_CPU) 1431 seq_puts(s, " | "); 1432 if (flags & TRACE_GRAPH_PRINT_PROC) 1433 seq_puts(s, " | | "); 1434 if (lat) 1435 seq_puts(s, "|||| "); 1436 if (flags & TRACE_GRAPH_PRINT_DURATION) 1437 seq_puts(s, " | | "); 1438 seq_puts(s, " | | | |\n"); 1439 } 1440 1441 static void print_graph_headers(struct seq_file *s) 1442 { 1443 print_graph_headers_flags(s, tracer_flags.val); 1444 } 1445 1446 void print_graph_headers_flags(struct seq_file *s, u32 flags) 1447 { 1448 struct trace_iterator *iter = s->private; 1449 struct trace_array *tr = iter->tr; 1450 1451 if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO)) 1452 return; 1453 1454 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) { 1455 /* print nothing if the buffers are empty */ 1456 if (trace_empty(iter)) 1457 return; 1458 1459 print_trace_header(s, iter); 1460 } 1461 1462 __print_graph_headers_flags(tr, s, flags); 1463 } 1464 1465 void graph_trace_open(struct trace_iterator *iter) 1466 { 1467 /* pid and depth on the last trace processed */ 1468 struct fgraph_data *data; 1469 gfp_t gfpflags; 1470 int cpu; 1471 1472 iter->private = NULL; 1473 1474 /* We can be called in atomic context via ftrace_dump() */ 1475 gfpflags = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL; 1476 1477 data = kzalloc(sizeof(*data), gfpflags); 1478 if (!data) 1479 goto out_err; 1480 1481 data->cpu_data = alloc_percpu_gfp(struct fgraph_cpu_data, gfpflags); 1482 if (!data->cpu_data) 1483 goto out_err_free; 1484 1485 for_each_possible_cpu(cpu) { 1486 pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid); 1487 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth); 1488 int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore); 1489 int *depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq); 1490 1491 *pid = -1; 1492 *depth = 0; 1493 *ignore = 0; 1494 *depth_irq = -1; 1495 } 1496 1497 iter->private = data; 1498 1499 return; 1500 1501 out_err_free: 1502 kfree(data); 1503 out_err: 1504 pr_warn("function graph tracer: not enough memory\n"); 1505 } 1506 1507 void graph_trace_close(struct trace_iterator *iter) 1508 { 1509 struct fgraph_data *data = iter->private; 1510 1511 if (data) { 1512 free_percpu(data->cpu_data); 1513 kfree(data); 1514 } 1515 } 1516 1517 static int 1518 func_graph_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 1519 { 1520 if (bit == TRACE_GRAPH_PRINT_IRQS) 1521 ftrace_graph_skip_irqs = !set; 1522 1523 if (bit == TRACE_GRAPH_SLEEP_TIME) 1524 ftrace_graph_sleep_time_control(set); 1525 1526 if (bit == TRACE_GRAPH_GRAPH_TIME) 1527 ftrace_graph_graph_time_control(set); 1528 1529 return 0; 1530 } 1531 1532 static struct trace_event_functions graph_functions = { 1533 .trace = print_graph_function_event, 1534 }; 1535 1536 static struct trace_event graph_trace_entry_event = { 1537 .type = TRACE_GRAPH_ENT, 1538 .funcs = &graph_functions, 1539 }; 1540 1541 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1542 static struct trace_event graph_trace_retaddr_entry_event = { 1543 .type = TRACE_GRAPH_RETADDR_ENT, 1544 .funcs = &graph_functions, 1545 }; 1546 #endif 1547 1548 static struct trace_event graph_trace_ret_event = { 1549 .type = TRACE_GRAPH_RET, 1550 .funcs = &graph_functions 1551 }; 1552 1553 static struct tracer graph_trace __tracer_data = { 1554 .name = "function_graph", 1555 .update_thresh = graph_trace_update_thresh, 1556 .open = graph_trace_open, 1557 .pipe_open = graph_trace_open, 1558 .close = graph_trace_close, 1559 .pipe_close = graph_trace_close, 1560 .init = graph_trace_init, 1561 .reset = graph_trace_reset, 1562 .print_line = print_graph_function, 1563 .print_header = print_graph_headers, 1564 .flags = &tracer_flags, 1565 .set_flag = func_graph_set_flag, 1566 .allow_instances = true, 1567 #ifdef CONFIG_FTRACE_SELFTEST 1568 .selftest = trace_selftest_startup_function_graph, 1569 #endif 1570 }; 1571 1572 1573 static ssize_t 1574 graph_depth_write(struct file *filp, const char __user *ubuf, size_t cnt, 1575 loff_t *ppos) 1576 { 1577 unsigned long val; 1578 int ret; 1579 1580 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 1581 if (ret) 1582 return ret; 1583 1584 fgraph_max_depth = val; 1585 1586 *ppos += cnt; 1587 1588 return cnt; 1589 } 1590 1591 static ssize_t 1592 graph_depth_read(struct file *filp, char __user *ubuf, size_t cnt, 1593 loff_t *ppos) 1594 { 1595 char buf[15]; /* More than enough to hold UINT_MAX + "\n"*/ 1596 int n; 1597 1598 n = sprintf(buf, "%d\n", fgraph_max_depth); 1599 1600 return simple_read_from_buffer(ubuf, cnt, ppos, buf, n); 1601 } 1602 1603 static const struct file_operations graph_depth_fops = { 1604 .open = tracing_open_generic, 1605 .write = graph_depth_write, 1606 .read = graph_depth_read, 1607 .llseek = generic_file_llseek, 1608 }; 1609 1610 static __init int init_graph_tracefs(void) 1611 { 1612 int ret; 1613 1614 ret = tracing_init_dentry(); 1615 if (ret) 1616 return 0; 1617 1618 trace_create_file("max_graph_depth", TRACE_MODE_WRITE, NULL, 1619 NULL, &graph_depth_fops); 1620 1621 return 0; 1622 } 1623 fs_initcall(init_graph_tracefs); 1624 1625 static __init int init_graph_trace(void) 1626 { 1627 max_bytes_for_cpu = snprintf(NULL, 0, "%u", nr_cpu_ids - 1); 1628 1629 if (!register_trace_event(&graph_trace_entry_event)) { 1630 pr_warn("Warning: could not register graph trace events\n"); 1631 return 1; 1632 } 1633 1634 #ifdef CONFIG_FUNCTION_GRAPH_RETADDR 1635 if (!register_trace_event(&graph_trace_retaddr_entry_event)) { 1636 pr_warn("Warning: could not register graph trace retaddr events\n"); 1637 return 1; 1638 } 1639 #endif 1640 1641 if (!register_trace_event(&graph_trace_ret_event)) { 1642 pr_warn("Warning: could not register graph trace events\n"); 1643 return 1; 1644 } 1645 1646 return register_tracer(&graph_trace); 1647 } 1648 1649 core_initcall(init_graph_trace); 1650