1 // SPDX-License-Identifier: GPL-2.0 2 #include <dirent.h> 3 #include <errno.h> 4 #include <inttypes.h> 5 #include <regex.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include "callchain.h" 9 #include "debug.h" 10 #include "dso.h" 11 #include "env.h" 12 #include "dwarf-regs.h" 13 #include "event.h" 14 #include "evsel.h" 15 #include "hist.h" 16 #include "machine.h" 17 #include "map.h" 18 #include "map_symbol.h" 19 #include "branch.h" 20 #include "mem-events.h" 21 #include "mem-info.h" 22 #include "path.h" 23 #include "srcline.h" 24 #include "symbol.h" 25 #include "synthetic-events.h" 26 #include "sort.h" 27 #include "strlist.h" 28 #include "target.h" 29 #include "thread.h" 30 #include "util.h" 31 #include "vdso.h" 32 #include <stdbool.h> 33 #include <sys/types.h> 34 #include <sys/stat.h> 35 #include <unistd.h> 36 #include "unwind.h" 37 #include "linux/hash.h" 38 #include "asm/bug.h" 39 #include "bpf-event.h" 40 #include <internal/lib.h> // page_size 41 #include "cgroup.h" 42 #include "arm64-frame-pointer-unwind-support.h" 43 #include <api/io_dir.h> 44 45 #include <linux/ctype.h> 46 #include <symbol/kallsyms.h> 47 #include <linux/mman.h> 48 #include <linux/string.h> 49 #include <linux/zalloc.h> 50 51 static struct dso *machine__kernel_dso(struct machine *machine) 52 { 53 return map__dso(machine->vmlinux_map); 54 } 55 56 static int machine__set_mmap_name(struct machine *machine) 57 { 58 if (machine__is_host(machine)) 59 machine->mmap_name = strdup("[kernel.kallsyms]"); 60 else if (machine__is_default_guest(machine)) 61 machine->mmap_name = strdup("[guest.kernel.kallsyms]"); 62 else if (asprintf(&machine->mmap_name, "[guest.kernel.kallsyms.%d]", 63 machine->pid) < 0) 64 machine->mmap_name = NULL; 65 66 return machine->mmap_name ? 0 : -ENOMEM; 67 } 68 69 static void thread__set_guest_comm(struct thread *thread, pid_t pid) 70 { 71 char comm[64]; 72 73 snprintf(comm, sizeof(comm), "[guest/%d]", pid); 74 thread__set_comm(thread, comm, 0); 75 } 76 77 int machine__init(struct machine *machine, const char *root_dir, pid_t pid) 78 { 79 int err = -ENOMEM; 80 81 memset(machine, 0, sizeof(*machine)); 82 RB_CLEAR_NODE(&machine->rb_node); 83 dsos__init(&machine->dsos); 84 threads__init(&machine->threads); 85 86 machine->kmaps = maps__new(machine); 87 if (machine->kmaps == NULL) 88 goto out; 89 90 machine->vdso_info = NULL; 91 machine->env = NULL; 92 93 machine->pid = pid; 94 95 machine->id_hdr_size = 0; 96 machine->kptr_restrict_warned = false; 97 machine->comm_exec = false; 98 machine->kernel_start = 0; 99 machine->vmlinux_map = NULL; 100 /* There is no initial context switch in, so we start at 1. */ 101 machine->parallelism = 1; 102 103 machine->root_dir = strdup(root_dir); 104 if (machine->root_dir == NULL) 105 goto out; 106 107 if (machine__set_mmap_name(machine)) 108 goto out; 109 110 if (pid != HOST_KERNEL_ID) { 111 struct thread *thread = machine__findnew_thread(machine, -1, 112 pid); 113 114 if (thread == NULL) 115 goto out; 116 117 thread__set_guest_comm(thread, pid); 118 thread__put(thread); 119 } 120 121 machine->current_tid = NULL; 122 err = 0; 123 124 out: 125 if (err) { 126 maps__zput(machine->kmaps); 127 zfree(&machine->root_dir); 128 zfree(&machine->mmap_name); 129 } 130 return err; 131 } 132 133 static struct machine *__machine__new_host(struct perf_env *host_env, bool kernel_maps) 134 { 135 struct machine *machine = malloc(sizeof(*machine)); 136 137 if (!machine) 138 return NULL; 139 140 if (machine__init(machine, "", HOST_KERNEL_ID) != 0) { 141 free(machine); 142 return NULL; 143 } 144 145 if (kernel_maps && machine__create_kernel_maps(machine) < 0) { 146 free(machine); 147 return NULL; 148 } 149 machine->env = host_env; 150 return machine; 151 } 152 153 struct machine *machine__new_host(struct perf_env *host_env) 154 { 155 return __machine__new_host(host_env, /*kernel_maps=*/true); 156 } 157 158 static int mmap_handler(const struct perf_tool *tool __maybe_unused, 159 union perf_event *event, 160 struct perf_sample *sample, 161 struct machine *machine) 162 { 163 return machine__process_mmap2_event(machine, event, sample); 164 } 165 166 static int machine__init_live(struct machine *machine, pid_t pid) 167 { 168 union perf_event event; 169 170 memset(&event, 0, sizeof(event)); 171 return perf_event__synthesize_mmap_events(NULL, &event, pid, pid, 172 mmap_handler, machine, true); 173 } 174 175 struct machine *machine__new_live(struct perf_env *host_env, bool kernel_maps, pid_t pid) 176 { 177 struct machine *machine = __machine__new_host(host_env, kernel_maps); 178 179 if (!machine) 180 return NULL; 181 182 if (machine__init_live(machine, pid)) { 183 machine__delete(machine); 184 return NULL; 185 } 186 return machine; 187 } 188 189 struct machine *machine__new_kallsyms(struct perf_env *host_env) 190 { 191 struct machine *machine = machine__new_host(host_env); 192 /* 193 * FIXME: 194 * 1) We should switch to machine__load_kallsyms(), i.e. not explicitly 195 * ask for not using the kcore parsing code, once this one is fixed 196 * to create a map per module. 197 */ 198 if (machine && machine__load_kallsyms(machine, "/proc/kallsyms") <= 0) { 199 machine__delete(machine); 200 machine = NULL; 201 } 202 203 return machine; 204 } 205 206 void machine__delete_threads(struct machine *machine) 207 { 208 threads__remove_all_threads(&machine->threads); 209 } 210 211 void machine__exit(struct machine *machine) 212 { 213 if (machine == NULL) 214 return; 215 216 machine__destroy_kernel_maps(machine); 217 maps__zput(machine->kmaps); 218 dsos__exit(&machine->dsos); 219 machine__exit_vdso(machine); 220 zfree(&machine->root_dir); 221 zfree(&machine->mmap_name); 222 zfree(&machine->current_tid); 223 zfree(&machine->kallsyms_filename); 224 225 threads__exit(&machine->threads); 226 } 227 228 void machine__delete(struct machine *machine) 229 { 230 if (machine) { 231 machine__exit(machine); 232 free(machine); 233 } 234 } 235 236 int machines__init(struct machines *machines) 237 { 238 int err = machine__init(&machines->host, "", HOST_KERNEL_ID); 239 240 machines->guests = RB_ROOT_CACHED; 241 return err; 242 } 243 244 void machines__exit(struct machines *machines) 245 { 246 machine__exit(&machines->host); 247 /* XXX exit guest */ 248 } 249 250 struct machine *machines__add(struct machines *machines, pid_t pid, 251 const char *root_dir) 252 { 253 struct rb_node **p = &machines->guests.rb_root.rb_node; 254 struct rb_node *parent = NULL; 255 struct machine *pos, *machine = malloc(sizeof(*machine)); 256 bool leftmost = true; 257 258 if (machine == NULL) 259 return NULL; 260 261 if (machine__init(machine, root_dir, pid) != 0) { 262 free(machine); 263 return NULL; 264 } 265 266 while (*p != NULL) { 267 parent = *p; 268 pos = rb_entry(parent, struct machine, rb_node); 269 if (pid < pos->pid) 270 p = &(*p)->rb_left; 271 else { 272 p = &(*p)->rb_right; 273 leftmost = false; 274 } 275 } 276 277 rb_link_node(&machine->rb_node, parent, p); 278 rb_insert_color_cached(&machine->rb_node, &machines->guests, leftmost); 279 280 machine->machines = machines; 281 282 return machine; 283 } 284 285 void machines__set_comm_exec(struct machines *machines, bool comm_exec) 286 { 287 struct rb_node *nd; 288 289 machines->host.comm_exec = comm_exec; 290 291 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) { 292 struct machine *machine = rb_entry(nd, struct machine, rb_node); 293 294 machine->comm_exec = comm_exec; 295 } 296 } 297 298 struct machine *machines__find(struct machines *machines, pid_t pid) 299 { 300 struct rb_node **p = &machines->guests.rb_root.rb_node; 301 struct rb_node *parent = NULL; 302 struct machine *machine; 303 struct machine *default_machine = NULL; 304 305 if (pid == HOST_KERNEL_ID) 306 return &machines->host; 307 308 while (*p != NULL) { 309 parent = *p; 310 machine = rb_entry(parent, struct machine, rb_node); 311 if (pid < machine->pid) 312 p = &(*p)->rb_left; 313 else if (pid > machine->pid) 314 p = &(*p)->rb_right; 315 else 316 return machine; 317 if (!machine->pid) 318 default_machine = machine; 319 } 320 321 return default_machine; 322 } 323 324 struct machine *machines__findnew(struct machines *machines, pid_t pid) 325 { 326 char path[PATH_MAX]; 327 const char *root_dir = ""; 328 struct machine *machine = machines__find(machines, pid); 329 330 if (machine && (machine->pid == pid)) 331 goto out; 332 333 if ((pid != HOST_KERNEL_ID) && 334 (pid != DEFAULT_GUEST_KERNEL_ID) && 335 (symbol_conf.guestmount)) { 336 snprintf(path, sizeof(path), "%s/%d", symbol_conf.guestmount, pid); 337 if (access(path, R_OK)) { 338 static struct strlist *seen; 339 340 if (!seen) 341 seen = strlist__new(NULL, NULL); 342 343 if (!strlist__has_entry(seen, path)) { 344 pr_err("Can't access file %s\n", path); 345 strlist__add(seen, path); 346 } 347 machine = NULL; 348 goto out; 349 } 350 root_dir = path; 351 } 352 353 machine = machines__add(machines, pid, root_dir); 354 out: 355 return machine; 356 } 357 358 struct machine *machines__find_guest(struct machines *machines, pid_t pid) 359 { 360 struct machine *machine = machines__find(machines, pid); 361 362 if (!machine) 363 machine = machines__findnew(machines, DEFAULT_GUEST_KERNEL_ID); 364 return machine; 365 } 366 367 /* 368 * A common case for KVM test programs is that the test program acts as the 369 * hypervisor, creating, running and destroying the virtual machine, and 370 * providing the guest object code from its own object code. In this case, 371 * the VM is not running an OS, but only the functions loaded into it by the 372 * hypervisor test program, and conveniently, loaded at the same virtual 373 * addresses. 374 * 375 * Normally to resolve addresses, MMAP events are needed to map addresses 376 * back to the object code and debug symbols for that object code. 377 * 378 * Currently, there is no way to get such mapping information from guests 379 * but, in the scenario described above, the guest has the same mappings 380 * as the hypervisor, so support for that scenario can be achieved. 381 * 382 * To support that, copy the host thread's maps to the guest thread's maps. 383 * Note, we do not discover the guest until we encounter a guest event, 384 * which works well because it is not until then that we know that the host 385 * thread's maps have been set up. 386 * 387 * This function returns the guest thread. Apart from keeping the data 388 * structures sane, using a thread belonging to the guest machine, instead 389 * of the host thread, allows it to have its own comm (refer 390 * thread__set_guest_comm()). 391 */ 392 static struct thread *findnew_guest_code(struct machine *machine, 393 struct machine *host_machine, 394 pid_t pid) 395 { 396 struct thread *host_thread; 397 struct thread *thread; 398 int err; 399 400 if (!machine) 401 return NULL; 402 403 thread = machine__findnew_thread(machine, -1, pid); 404 if (!thread) 405 return NULL; 406 407 /* Assume maps are set up if there are any */ 408 if (!maps__empty(thread__maps(thread))) 409 return thread; 410 411 host_thread = machine__find_thread(host_machine, -1, pid); 412 if (!host_thread) 413 goto out_err; 414 415 thread__set_guest_comm(thread, pid); 416 417 /* 418 * Guest code can be found in hypervisor process at the same address 419 * so copy host maps. 420 */ 421 err = maps__copy_from(thread__maps(thread), thread__maps(host_thread)); 422 thread__put(host_thread); 423 if (err) 424 goto out_err; 425 426 return thread; 427 428 out_err: 429 thread__zput(thread); 430 return NULL; 431 } 432 433 struct thread *machines__findnew_guest_code(struct machines *machines, pid_t pid) 434 { 435 struct machine *host_machine = machines__find(machines, HOST_KERNEL_ID); 436 struct machine *machine = machines__findnew(machines, pid); 437 438 return findnew_guest_code(machine, host_machine, pid); 439 } 440 441 struct thread *machine__findnew_guest_code(struct machine *machine, pid_t pid) 442 { 443 struct machines *machines = machine->machines; 444 struct machine *host_machine; 445 446 if (!machines) 447 return NULL; 448 449 host_machine = machines__find(machines, HOST_KERNEL_ID); 450 451 return findnew_guest_code(machine, host_machine, pid); 452 } 453 454 void machines__process_guests(struct machines *machines, 455 machine__process_t process, void *data) 456 { 457 struct rb_node *nd; 458 459 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) { 460 struct machine *pos = rb_entry(nd, struct machine, rb_node); 461 process(pos, data); 462 } 463 } 464 465 void machines__set_id_hdr_size(struct machines *machines, u16 id_hdr_size) 466 { 467 struct rb_node *node; 468 struct machine *machine; 469 470 machines->host.id_hdr_size = id_hdr_size; 471 472 for (node = rb_first_cached(&machines->guests); node; 473 node = rb_next(node)) { 474 machine = rb_entry(node, struct machine, rb_node); 475 machine->id_hdr_size = id_hdr_size; 476 } 477 478 return; 479 } 480 481 static void machine__update_thread_pid(struct machine *machine, 482 struct thread *th, pid_t pid) 483 { 484 struct thread *leader; 485 486 if (pid == thread__pid(th) || pid == -1 || thread__pid(th) != -1) 487 return; 488 489 thread__set_pid(th, pid); 490 491 if (thread__pid(th) == thread__tid(th)) 492 return; 493 494 leader = machine__findnew_thread(machine, thread__pid(th), thread__pid(th)); 495 if (!leader) 496 goto out_err; 497 498 if (!thread__maps(leader)) 499 thread__set_maps(leader, maps__new(machine)); 500 501 if (!thread__maps(leader)) 502 goto out_err; 503 504 if (thread__maps(th) == thread__maps(leader)) 505 goto out_put; 506 507 if (thread__maps(th)) { 508 /* 509 * Maps are created from MMAP events which provide the pid and 510 * tid. Consequently there never should be any maps on a thread 511 * with an unknown pid. Just print an error if there are. 512 */ 513 if (!maps__empty(thread__maps(th))) 514 pr_err("Discarding thread maps for %d:%d\n", 515 thread__pid(th), thread__tid(th)); 516 maps__put(thread__maps(th)); 517 } 518 519 thread__set_maps(th, maps__get(thread__maps(leader))); 520 out_put: 521 thread__put(leader); 522 return; 523 out_err: 524 pr_err("Failed to join map groups for %d:%d\n", thread__pid(th), thread__tid(th)); 525 goto out_put; 526 } 527 528 /* 529 * Caller must eventually drop thread->refcnt returned with a successful 530 * lookup/new thread inserted. 531 */ 532 static struct thread *__machine__findnew_thread(struct machine *machine, 533 pid_t pid, 534 pid_t tid, 535 bool create) 536 { 537 struct thread *th = threads__find(&machine->threads, tid); 538 bool created; 539 540 if (th) { 541 machine__update_thread_pid(machine, th, pid); 542 return th; 543 } 544 if (!create) 545 return NULL; 546 547 th = threads__findnew(&machine->threads, pid, tid, &created); 548 if (created) { 549 /* 550 * We have to initialize maps separately after rb tree is 551 * updated. 552 * 553 * The reason is that we call machine__findnew_thread within 554 * thread__init_maps to find the thread leader and that would 555 * screwed the rb tree. 556 */ 557 if (thread__init_maps(th, machine)) { 558 pr_err("Thread init failed thread %d\n", pid); 559 threads__remove(&machine->threads, th); 560 thread__put(th); 561 return NULL; 562 } 563 } else 564 machine__update_thread_pid(machine, th, pid); 565 566 return th; 567 } 568 569 struct thread *machine__findnew_thread(struct machine *machine, pid_t pid, pid_t tid) 570 { 571 return __machine__findnew_thread(machine, pid, tid, /*create=*/true); 572 } 573 574 struct thread *machine__find_thread(struct machine *machine, pid_t pid, 575 pid_t tid) 576 { 577 return __machine__findnew_thread(machine, pid, tid, /*create=*/false); 578 } 579 580 /* 581 * Threads are identified by pid and tid, and the idle task has pid == tid == 0. 582 * So here a single thread is created for that, but actually there is a separate 583 * idle task per cpu, so there should be one 'struct thread' per cpu, but there 584 * is only 1. That causes problems for some tools, requiring workarounds. For 585 * example get_idle_thread() in builtin-sched.c, or thread_stack__per_cpu(). 586 */ 587 struct thread *machine__idle_thread(struct machine *machine) 588 { 589 struct thread *thread = machine__findnew_thread(machine, 0, 0); 590 591 if (!thread || thread__set_comm(thread, "swapper", 0) || 592 thread__set_namespaces(thread, 0, NULL)) 593 pr_err("problem inserting idle task for machine pid %d\n", machine->pid); 594 595 return thread; 596 } 597 598 struct comm *machine__thread_exec_comm(struct machine *machine, 599 struct thread *thread) 600 { 601 if (machine->comm_exec) 602 return thread__exec_comm(thread); 603 else 604 return thread__comm(thread); 605 } 606 607 int machine__process_comm_event(struct machine *machine, union perf_event *event, 608 struct perf_sample *sample) 609 { 610 struct thread *thread = machine__findnew_thread(machine, 611 event->comm.pid, 612 event->comm.tid); 613 bool exec = event->header.misc & PERF_RECORD_MISC_COMM_EXEC; 614 int err = 0; 615 616 if (exec) 617 machine->comm_exec = true; 618 619 if (dump_trace) 620 perf_event__fprintf_comm(event, stdout); 621 622 if (thread == NULL || 623 __thread__set_comm(thread, event->comm.comm, sample->time, exec)) { 624 dump_printf("problem processing PERF_RECORD_COMM, skipping event.\n"); 625 err = -1; 626 } 627 628 thread__put(thread); 629 630 return err; 631 } 632 633 int machine__process_namespaces_event(struct machine *machine __maybe_unused, 634 union perf_event *event, 635 struct perf_sample *sample __maybe_unused) 636 { 637 struct thread *thread = machine__findnew_thread(machine, 638 event->namespaces.pid, 639 event->namespaces.tid); 640 int err = 0; 641 642 WARN_ONCE(event->namespaces.nr_namespaces > NR_NAMESPACES, 643 "\nWARNING: kernel seems to support more namespaces than perf" 644 " tool.\nTry updating the perf tool..\n\n"); 645 646 WARN_ONCE(event->namespaces.nr_namespaces < NR_NAMESPACES, 647 "\nWARNING: perf tool seems to support more namespaces than" 648 " the kernel.\nTry updating the kernel..\n\n"); 649 650 if (dump_trace) 651 perf_event__fprintf_namespaces(event, stdout); 652 653 if (thread == NULL || 654 thread__set_namespaces(thread, sample->time, &event->namespaces)) { 655 dump_printf("problem processing PERF_RECORD_NAMESPACES, skipping event.\n"); 656 err = -1; 657 } 658 659 thread__put(thread); 660 661 return err; 662 } 663 664 int machine__process_cgroup_event(struct machine *machine, 665 union perf_event *event, 666 struct perf_sample *sample __maybe_unused) 667 { 668 struct cgroup *cgrp; 669 670 if (dump_trace) 671 perf_event__fprintf_cgroup(event, stdout); 672 673 cgrp = cgroup__findnew(machine->env, event->cgroup.id, event->cgroup.path); 674 if (cgrp == NULL) 675 return -ENOMEM; 676 677 return 0; 678 } 679 680 int machine__process_lost_event(struct machine *machine __maybe_unused, 681 union perf_event *event, struct perf_sample *sample __maybe_unused) 682 { 683 dump_printf(": id:%" PRI_lu64 ": lost:%" PRI_lu64 "\n", 684 event->lost.id, event->lost.lost); 685 return 0; 686 } 687 688 int machine__process_lost_samples_event(struct machine *machine __maybe_unused, 689 union perf_event *event, struct perf_sample *sample) 690 { 691 dump_printf(": id:%" PRIu64 ": lost samples :%" PRI_lu64 "%s\n", 692 sample->id, event->lost_samples.lost, 693 event->header.misc & PERF_RECORD_MISC_LOST_SAMPLES_BPF ? " (BPF)" : ""); 694 return 0; 695 } 696 697 int machine__process_aux_event(struct machine *machine __maybe_unused, 698 union perf_event *event) 699 { 700 if (dump_trace) 701 perf_event__fprintf_aux(event, stdout); 702 return 0; 703 } 704 705 int machine__process_itrace_start_event(struct machine *machine __maybe_unused, 706 union perf_event *event) 707 { 708 if (dump_trace) 709 perf_event__fprintf_itrace_start(event, stdout); 710 return 0; 711 } 712 713 int machine__process_aux_output_hw_id_event(struct machine *machine __maybe_unused, 714 union perf_event *event) 715 { 716 if (dump_trace) 717 perf_event__fprintf_aux_output_hw_id(event, stdout); 718 return 0; 719 } 720 721 int machine__process_switch_event(struct machine *machine __maybe_unused, 722 union perf_event *event) 723 { 724 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; 725 726 if (dump_trace) 727 perf_event__fprintf_switch(event, stdout); 728 machine->parallelism += out ? -1 : 1; 729 return 0; 730 } 731 732 static int machine__process_ksymbol_register(struct machine *machine, 733 union perf_event *event, 734 struct perf_sample *sample __maybe_unused) 735 { 736 struct symbol *sym; 737 struct dso *dso = NULL; 738 struct map *map; 739 int err = 0; 740 741 /* Ignore mapping symbols in ksymbol events - check early before any state mutation */ 742 if (is_ignored_kernel_symbol(event->ksymbol.name)) 743 return 0; 744 745 map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); 746 747 if (!map) { 748 dso = dso__new(event->ksymbol.name); 749 750 if (!dso) { 751 err = -ENOMEM; 752 goto out; 753 } 754 dso__set_kernel(dso, DSO_SPACE__KERNEL); 755 map = map__new2(0, dso); 756 if (!map) { 757 err = -ENOMEM; 758 goto out; 759 } 760 if (event->ksymbol.ksym_type == PERF_RECORD_KSYMBOL_TYPE_OOL) { 761 dso__set_binary_type(dso, DSO_BINARY_TYPE__OOL); 762 dso__data(dso)->file_size = event->ksymbol.len; 763 dso__set_loaded(dso); 764 } 765 766 map__set_start(map, event->ksymbol.addr); 767 map__set_end(map, map__start(map) + event->ksymbol.len); 768 err = maps__fixup_overlap_and_insert(machine__kernel_maps(machine), map); 769 if (err) { 770 err = -ENOMEM; 771 goto out; 772 } 773 774 dso__set_loaded(dso); 775 776 if (is_bpf_image(event->ksymbol.name)) { 777 dso__set_binary_type(dso, DSO_BINARY_TYPE__BPF_IMAGE); 778 dso__set_long_name(dso, "", false); 779 } 780 } else { 781 dso = dso__get(map__dso(map)); 782 } 783 784 sym = symbol__new(map__map_ip(map, map__start(map)), 785 event->ksymbol.len, 786 0, 0, event->ksymbol.name); 787 if (!sym) { 788 err = -ENOMEM; 789 goto out; 790 } 791 dso__insert_symbol(dso, sym); 792 out: 793 map__put(map); 794 dso__put(dso); 795 return err; 796 } 797 798 static int machine__process_ksymbol_unregister(struct machine *machine, 799 union perf_event *event, 800 struct perf_sample *sample __maybe_unused) 801 { 802 struct symbol *sym; 803 struct map *map; 804 805 /* Ignore mapping symbols in ksymbol events */ 806 if (is_ignored_kernel_symbol(event->ksymbol.name)) 807 return 0; 808 809 map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); 810 if (!map) 811 return 0; 812 813 if (!RC_CHK_EQUAL(map, machine->vmlinux_map)) 814 maps__remove(machine__kernel_maps(machine), map); 815 else { 816 struct dso *dso = map__dso(map); 817 818 sym = dso__find_symbol(dso, map__map_ip(map, map__start(map))); 819 if (sym) 820 dso__delete_symbol(dso, sym); 821 } 822 map__put(map); 823 return 0; 824 } 825 826 int machine__process_ksymbol(struct machine *machine __maybe_unused, 827 union perf_event *event, 828 struct perf_sample *sample) 829 { 830 if (dump_trace) 831 perf_event__fprintf_ksymbol(event, stdout); 832 833 if (event->header.size < offsetof(struct perf_record_ksymbol, name) + 2 || 834 !memchr(event->ksymbol.name, '\0', 835 event->header.size - offsetof(struct perf_record_ksymbol, name))) 836 return -EINVAL; 837 838 /* no need to process non-JIT BPF as it cannot get samples */ 839 if (event->ksymbol.len == 0) 840 return 0; 841 842 if (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER) 843 return machine__process_ksymbol_unregister(machine, event, 844 sample); 845 return machine__process_ksymbol_register(machine, event, sample); 846 } 847 848 int machine__process_text_poke(struct machine *machine, union perf_event *event, 849 struct perf_sample *sample __maybe_unused) 850 { 851 struct map *map = maps__find(machine__kernel_maps(machine), event->text_poke.addr); 852 u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; 853 struct dso *dso = map ? map__dso(map) : NULL; 854 855 if (dump_trace) 856 perf_event__fprintf_text_poke(event, machine, stdout); 857 858 if (!event->text_poke.new_len) 859 goto out; 860 861 if (cpumode != PERF_RECORD_MISC_KERNEL) { 862 pr_debug("%s: unsupported cpumode - ignoring\n", __func__); 863 goto out; 864 } 865 866 if (dso) { 867 u8 *new_bytes = event->text_poke.bytes + event->text_poke.old_len; 868 int ret; 869 870 /* 871 * Kernel maps might be changed when loading symbols so loading 872 * must be done prior to using kernel maps. 873 */ 874 map__load(map); 875 ret = dso__data_write_cache_addr(dso, map, machine, 876 event->text_poke.addr, 877 new_bytes, 878 event->text_poke.new_len); 879 if (ret != event->text_poke.new_len) 880 pr_debug("Failed to write kernel text poke at %#" PRI_lx64 "\n", 881 event->text_poke.addr); 882 } else { 883 pr_debug("Failed to find kernel text poke address map for %#" PRI_lx64 "\n", 884 event->text_poke.addr); 885 } 886 out: 887 map__put(map); 888 return 0; 889 } 890 891 static struct map *machine__addnew_module_map(struct machine *machine, u64 start, 892 const char *filename) 893 { 894 struct map *map = NULL; 895 struct kmod_path m; 896 struct dso *dso; 897 int err; 898 899 if (kmod_path__parse_name(&m, filename)) 900 return NULL; 901 902 dso = dsos__findnew_module_dso(&machine->dsos, machine, &m, filename); 903 if (dso == NULL) 904 goto out; 905 906 map = map__new2(start, dso); 907 if (map == NULL) 908 goto out; 909 910 err = maps__insert(machine__kernel_maps(machine), map); 911 /* If maps__insert failed, return NULL. */ 912 if (err) { 913 map__put(map); 914 map = NULL; 915 } 916 out: 917 /* put the dso here, corresponding to machine__findnew_module_dso */ 918 dso__put(dso); 919 zfree(&m.name); 920 return map; 921 } 922 923 size_t machines__fprintf_dsos(struct machines *machines, FILE *fp) 924 { 925 struct rb_node *nd; 926 size_t ret = dsos__fprintf(&machines->host.dsos, fp); 927 928 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) { 929 struct machine *pos = rb_entry(nd, struct machine, rb_node); 930 ret += dsos__fprintf(&pos->dsos, fp); 931 } 932 933 return ret; 934 } 935 936 size_t machine__fprintf_dsos_buildid(struct machine *m, FILE *fp, 937 bool (skip)(struct dso *dso, int parm), int parm) 938 { 939 return dsos__fprintf_buildid(&m->dsos, fp, skip, parm); 940 } 941 942 size_t machines__fprintf_dsos_buildid(struct machines *machines, FILE *fp, 943 bool (skip)(struct dso *dso, int parm), int parm) 944 { 945 struct rb_node *nd; 946 size_t ret = machine__fprintf_dsos_buildid(&machines->host, fp, skip, parm); 947 948 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) { 949 struct machine *pos = rb_entry(nd, struct machine, rb_node); 950 ret += machine__fprintf_dsos_buildid(pos, fp, skip, parm); 951 } 952 return ret; 953 } 954 955 struct machine_fprintf_cb_args { 956 FILE *fp; 957 size_t printed; 958 }; 959 960 static int machine_fprintf_cb(struct thread *thread, void *data) 961 { 962 struct machine_fprintf_cb_args *args = data; 963 964 /* TODO: handle fprintf errors. */ 965 args->printed += thread__fprintf(thread, args->fp); 966 return 0; 967 } 968 969 size_t machine__fprintf(struct machine *machine, FILE *fp) 970 { 971 struct machine_fprintf_cb_args args = { 972 .fp = fp, 973 .printed = 0, 974 }; 975 size_t ret = fprintf(fp, "Threads: %zu\n", threads__nr(&machine->threads)); 976 977 machine__for_each_thread(machine, machine_fprintf_cb, &args); 978 return ret + args.printed; 979 } 980 981 static struct dso *machine__get_kernel(struct machine *machine) 982 { 983 const char *vmlinux_name = machine->mmap_name; 984 struct dso *kernel; 985 986 if (machine__is_host(machine)) { 987 if (symbol_conf.vmlinux_name) 988 vmlinux_name = symbol_conf.vmlinux_name; 989 990 kernel = machine__findnew_kernel(machine, vmlinux_name, 991 "[kernel]", DSO_SPACE__KERNEL); 992 } else { 993 if (symbol_conf.default_guest_vmlinux_name) 994 vmlinux_name = symbol_conf.default_guest_vmlinux_name; 995 996 kernel = machine__findnew_kernel(machine, vmlinux_name, 997 "[guest.kernel]", 998 DSO_SPACE__KERNEL_GUEST); 999 } 1000 1001 if (kernel != NULL && (!dso__has_build_id(kernel))) 1002 dso__read_running_kernel_build_id(kernel, machine); 1003 1004 return kernel; 1005 } 1006 1007 void machine__get_kallsyms_filename(struct machine *machine, char *buf, 1008 size_t bufsz) 1009 { 1010 if (machine__is_default_guest(machine)) 1011 scnprintf(buf, bufsz, "%s", symbol_conf.default_guest_kallsyms); 1012 else 1013 scnprintf(buf, bufsz, "%s/proc/kallsyms", machine->root_dir); 1014 } 1015 1016 const char *ref_reloc_sym_names[] = {"_text", "_stext", NULL}; 1017 1018 /* Figure out the start address of kernel map from /proc/kallsyms. 1019 * Returns the name of the start symbol in *symbol_name. Pass in NULL as 1020 * symbol_name if it's not that important. 1021 */ 1022 static int machine__get_running_kernel_start(struct machine *machine, 1023 const char **symbol_name, 1024 u64 *start, u64 *end) 1025 { 1026 char filename[PATH_MAX]; 1027 int i, err = -1; 1028 const char *name; 1029 u64 addr = 0; 1030 1031 machine__get_kallsyms_filename(machine, filename, PATH_MAX); 1032 1033 if (symbol__restricted_filename(filename, "/proc/kallsyms")) 1034 return 0; 1035 1036 for (i = 0; (name = ref_reloc_sym_names[i]) != NULL; i++) { 1037 err = kallsyms__get_function_start(filename, name, &addr); 1038 if (!err) 1039 break; 1040 } 1041 1042 if (err) 1043 return -1; 1044 1045 if (symbol_name) 1046 *symbol_name = name; 1047 1048 *start = addr; 1049 1050 err = kallsyms__get_symbol_start(filename, "_edata", &addr); 1051 if (err) 1052 err = kallsyms__get_symbol_start(filename, "_etext", &addr); 1053 if (!err) 1054 *end = addr; 1055 1056 return 0; 1057 } 1058 1059 int machine__create_extra_kernel_map(struct machine *machine, 1060 struct dso *kernel, 1061 struct extra_kernel_map *xm) 1062 { 1063 struct kmap *kmap; 1064 struct map *map; 1065 int err; 1066 1067 map = map__new2(xm->start, kernel); 1068 if (!map) 1069 return -ENOMEM; 1070 1071 map__set_end(map, xm->end); 1072 map__set_pgoff(map, xm->pgoff); 1073 1074 kmap = map__kmap(map); 1075 1076 strlcpy(kmap->name, xm->name, KMAP_NAME_LEN); 1077 1078 err = maps__insert(machine__kernel_maps(machine), map); 1079 1080 if (!err) { 1081 pr_debug2("Added extra kernel map %s %" PRIx64 "-%" PRIx64 "\n", 1082 kmap->name, map__start(map), map__end(map)); 1083 } 1084 1085 map__put(map); 1086 1087 return err; 1088 } 1089 1090 static u64 find_entry_trampoline(struct dso *dso) 1091 { 1092 /* Duplicates are removed so lookup all aliases */ 1093 const char *syms[] = { 1094 "_entry_trampoline", 1095 "__entry_trampoline_start", 1096 "entry_SYSCALL_64_trampoline", 1097 }; 1098 struct symbol *sym = dso__first_symbol(dso); 1099 unsigned int i; 1100 1101 for (; sym; sym = dso__next_symbol(sym)) { 1102 if (symbol__binding(sym) != STB_GLOBAL) 1103 continue; 1104 for (i = 0; i < ARRAY_SIZE(syms); i++) { 1105 if (!strcmp(sym->name, syms[i])) 1106 return sym->start; 1107 } 1108 } 1109 1110 return 0; 1111 } 1112 1113 /* 1114 * These values can be used for kernels that do not have symbols for the entry 1115 * trampolines in kallsyms. 1116 */ 1117 #define X86_64_CPU_ENTRY_AREA_PER_CPU 0xfffffe0000000000ULL 1118 #define X86_64_CPU_ENTRY_AREA_SIZE 0x2c000 1119 #define X86_64_ENTRY_TRAMPOLINE 0x6000 1120 1121 struct machine__map_x86_64_entry_trampolines_args { 1122 struct maps *kmaps; 1123 bool found; 1124 }; 1125 1126 static int machine__map_x86_64_entry_trampolines_cb(struct map *map, void *data) 1127 { 1128 struct machine__map_x86_64_entry_trampolines_args *args = data; 1129 struct map *dest_map; 1130 struct kmap *kmap = __map__kmap(map); 1131 1132 if (!kmap || !is_entry_trampoline(kmap->name)) 1133 return 0; 1134 1135 dest_map = maps__find(args->kmaps, map__pgoff(map)); 1136 if (RC_CHK_ACCESS(dest_map) != RC_CHK_ACCESS(map)) 1137 map__set_pgoff(map, map__map_ip(dest_map, map__pgoff(map))); 1138 1139 map__put(dest_map); 1140 args->found = true; 1141 return 0; 1142 } 1143 1144 /* Map x86_64 PTI entry trampolines */ 1145 int machine__map_x86_64_entry_trampolines(struct machine *machine, 1146 struct dso *kernel) 1147 { 1148 struct machine__map_x86_64_entry_trampolines_args args = { 1149 .kmaps = machine__kernel_maps(machine), 1150 .found = false, 1151 }; 1152 int nr_cpus_avail, cpu; 1153 u64 pgoff; 1154 1155 /* 1156 * In the vmlinux case, pgoff is a virtual address which must now be 1157 * mapped to a vmlinux offset. 1158 */ 1159 maps__for_each_map(args.kmaps, machine__map_x86_64_entry_trampolines_cb, &args); 1160 1161 if (args.found || machine->trampolines_mapped) 1162 return 0; 1163 1164 pgoff = find_entry_trampoline(kernel); 1165 if (!pgoff) 1166 return 0; 1167 1168 nr_cpus_avail = machine__nr_cpus_avail(machine); 1169 1170 /* Add a 1 page map for each CPU's entry trampoline */ 1171 for (cpu = 0; cpu < nr_cpus_avail; cpu++) { 1172 u64 va = X86_64_CPU_ENTRY_AREA_PER_CPU + 1173 cpu * X86_64_CPU_ENTRY_AREA_SIZE + 1174 X86_64_ENTRY_TRAMPOLINE; 1175 struct extra_kernel_map xm = { 1176 .start = va, 1177 .end = va + page_size, 1178 .pgoff = pgoff, 1179 }; 1180 1181 strlcpy(xm.name, ENTRY_TRAMPOLINE_NAME, KMAP_NAME_LEN); 1182 1183 if (machine__create_extra_kernel_map(machine, kernel, &xm) < 0) 1184 return -1; 1185 } 1186 1187 machine->trampolines_mapped = nr_cpus_avail; 1188 1189 return 0; 1190 } 1191 1192 int __weak machine__create_extra_kernel_maps(struct machine *machine __maybe_unused, 1193 struct dso *kernel __maybe_unused) 1194 { 1195 return 0; 1196 } 1197 1198 static int 1199 __machine__create_kernel_maps(struct machine *machine, struct dso *kernel) 1200 { 1201 /* In case of renewal the kernel map, destroy previous one */ 1202 machine__destroy_kernel_maps(machine); 1203 1204 map__put(machine->vmlinux_map); 1205 machine->vmlinux_map = map__new2(0, kernel); 1206 if (machine->vmlinux_map == NULL) 1207 return -ENOMEM; 1208 1209 map__set_mapping_type(machine->vmlinux_map, MAPPING_TYPE__IDENTITY); 1210 return maps__insert(machine__kernel_maps(machine), machine->vmlinux_map); 1211 } 1212 1213 void machine__destroy_kernel_maps(struct machine *machine) 1214 { 1215 struct kmap *kmap; 1216 struct map *map = machine__kernel_map(machine); 1217 1218 if (map == NULL) 1219 return; 1220 1221 kmap = map__kmap(map); 1222 maps__remove(machine__kernel_maps(machine), map); 1223 if (kmap && kmap->ref_reloc_sym) { 1224 zfree((char **)&kmap->ref_reloc_sym->name); 1225 zfree(&kmap->ref_reloc_sym); 1226 } 1227 1228 map__zput(machine->vmlinux_map); 1229 } 1230 1231 int machines__create_guest_kernel_maps(struct machines *machines) 1232 { 1233 int ret = 0; 1234 struct dirent **namelist = NULL; 1235 int i, items = 0; 1236 char path[PATH_MAX]; 1237 pid_t pid; 1238 char *endp; 1239 1240 if (symbol_conf.default_guest_vmlinux_name || 1241 symbol_conf.default_guest_modules || 1242 symbol_conf.default_guest_kallsyms) { 1243 machines__create_kernel_maps(machines, DEFAULT_GUEST_KERNEL_ID); 1244 } 1245 1246 if (symbol_conf.guestmount) { 1247 items = scandir(symbol_conf.guestmount, &namelist, NULL, NULL); 1248 if (items <= 0) 1249 return -ENOENT; 1250 for (i = 0; i < items; i++) { 1251 if (!isdigit(namelist[i]->d_name[0])) { 1252 /* Filter out . and .. */ 1253 continue; 1254 } 1255 pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10); 1256 if ((*endp != '\0') || 1257 (endp == namelist[i]->d_name) || 1258 (errno == ERANGE)) { 1259 pr_debug("invalid directory (%s). Skipping.\n", 1260 namelist[i]->d_name); 1261 continue; 1262 } 1263 snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", 1264 symbol_conf.guestmount, 1265 namelist[i]->d_name); 1266 ret = access(path, R_OK); 1267 if (ret) { 1268 pr_debug("Can't access file %s\n", path); 1269 goto failure; 1270 } 1271 machines__create_kernel_maps(machines, pid); 1272 } 1273 failure: 1274 free(namelist); 1275 } 1276 1277 return ret; 1278 } 1279 1280 void machines__destroy_kernel_maps(struct machines *machines) 1281 { 1282 struct rb_node *next = rb_first_cached(&machines->guests); 1283 1284 machine__destroy_kernel_maps(&machines->host); 1285 1286 while (next) { 1287 struct machine *pos = rb_entry(next, struct machine, rb_node); 1288 1289 next = rb_next(&pos->rb_node); 1290 rb_erase_cached(&pos->rb_node, &machines->guests); 1291 machine__delete(pos); 1292 } 1293 } 1294 1295 int machines__create_kernel_maps(struct machines *machines, pid_t pid) 1296 { 1297 struct machine *machine = machines__findnew(machines, pid); 1298 1299 if (machine == NULL) 1300 return -1; 1301 1302 return machine__create_kernel_maps(machine); 1303 } 1304 1305 int machine__load_kallsyms(struct machine *machine, const char *filename) 1306 { 1307 struct map *map = machine__kernel_map(machine); 1308 struct dso *dso = map__dso(map); 1309 int ret = __dso__load_kallsyms(dso, filename, map, true); 1310 1311 if (ret > 0) { 1312 dso__set_loaded(dso); 1313 /* 1314 * Since /proc/kallsyms will have multiple sessions for the 1315 * kernel, with modules between them, fixup the end of all 1316 * sections. 1317 */ 1318 maps__fixup_end(machine__kernel_maps(machine)); 1319 } 1320 1321 return ret; 1322 } 1323 1324 int machine__load_vmlinux_path(struct machine *machine) 1325 { 1326 struct map *map = machine__kernel_map(machine); 1327 struct dso *dso = map__dso(map); 1328 int ret = dso__load_vmlinux_path(dso, map); 1329 1330 if (ret > 0) 1331 dso__set_loaded(dso); 1332 1333 return ret; 1334 } 1335 1336 static char *get_kernel_version(const char *root_dir) 1337 { 1338 char version[PATH_MAX]; 1339 FILE *file; 1340 char *name, *tmp; 1341 const char *prefix = "Linux version "; 1342 1343 snprintf(version, sizeof(version), "%s/proc/version", root_dir); 1344 file = fopen(version, "r"); 1345 if (!file) 1346 return NULL; 1347 1348 tmp = fgets(version, sizeof(version), file); 1349 fclose(file); 1350 if (!tmp) 1351 return NULL; 1352 1353 name = strstr(version, prefix); 1354 if (!name) 1355 return NULL; 1356 name += strlen(prefix); 1357 tmp = strchr(name, ' '); 1358 if (tmp) 1359 *tmp = '\0'; 1360 1361 return strdup(name); 1362 } 1363 1364 static bool is_kmod_dso(struct dso *dso) 1365 { 1366 return dso__symtab_type(dso) == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE || 1367 dso__symtab_type(dso) == DSO_BINARY_TYPE__GUEST_KMODULE; 1368 } 1369 1370 static int maps__set_module_path(struct maps *maps, const char *path, struct kmod_path *m) 1371 { 1372 char *long_name; 1373 struct dso *dso; 1374 struct map *map = maps__find_by_name(maps, m->name); 1375 1376 if (map == NULL) 1377 return 0; 1378 1379 long_name = strdup(path); 1380 if (long_name == NULL) { 1381 map__put(map); 1382 return -ENOMEM; 1383 } 1384 1385 dso = map__dso(map); 1386 dso__set_long_name(dso, long_name, true); 1387 dso__kernel_module_get_build_id(dso, ""); 1388 1389 /* 1390 * Full name could reveal us kmod compression, so 1391 * we need to update the symtab_type if needed. 1392 */ 1393 if (m->comp && is_kmod_dso(dso)) { 1394 dso__set_symtab_type(dso, dso__symtab_type(dso)+1); 1395 dso__set_comp(dso, m->comp); 1396 } 1397 map__put(map); 1398 return 0; 1399 } 1400 1401 static int maps__set_modules_path_dir(struct maps *maps, char *path, size_t path_size, int depth) 1402 { 1403 struct io_dirent64 *dent; 1404 struct io_dir iod; 1405 size_t root_len = strlen(path); 1406 int ret = 0; 1407 1408 io_dir__init(&iod, open(path, O_CLOEXEC | O_DIRECTORY | O_RDONLY)); 1409 if (iod.dirfd < 0) { 1410 pr_debug("%s: cannot open %s dir\n", __func__, path); 1411 return -1; 1412 } 1413 /* Bounds check, should never happen. */ 1414 if (root_len >= path_size) 1415 return -1; 1416 path[root_len++] = '/'; 1417 while ((dent = io_dir__readdir(&iod)) != NULL) { 1418 if (io_dir__is_dir(&iod, dent)) { 1419 if (!strcmp(dent->d_name, ".") || 1420 !strcmp(dent->d_name, "..")) 1421 continue; 1422 1423 /* Do not follow top-level source and build symlinks */ 1424 if (depth == 0) { 1425 if (!strcmp(dent->d_name, "source") || 1426 !strcmp(dent->d_name, "build")) 1427 continue; 1428 } 1429 1430 /* Bounds check, should never happen. */ 1431 if (root_len + strlen(dent->d_name) >= path_size) 1432 continue; 1433 1434 strcpy(path + root_len, dent->d_name); 1435 ret = maps__set_modules_path_dir(maps, path, path_size, depth + 1); 1436 if (ret < 0) 1437 goto out; 1438 } else { 1439 struct kmod_path m; 1440 1441 ret = kmod_path__parse_name(&m, dent->d_name); 1442 if (ret) 1443 goto out; 1444 1445 if (m.kmod) { 1446 /* Bounds check, should never happen. */ 1447 if (root_len + strlen(dent->d_name) < path_size) { 1448 strcpy(path + root_len, dent->d_name); 1449 ret = maps__set_module_path(maps, path, &m); 1450 1451 } 1452 } 1453 zfree(&m.name); 1454 1455 if (ret) 1456 goto out; 1457 } 1458 } 1459 1460 out: 1461 close(iod.dirfd); 1462 return ret; 1463 } 1464 1465 static int machine__set_modules_path(struct machine *machine) 1466 { 1467 char *version; 1468 char modules_path[PATH_MAX]; 1469 1470 version = get_kernel_version(machine->root_dir); 1471 if (!version) 1472 return -1; 1473 1474 snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s", 1475 machine->root_dir, version); 1476 free(version); 1477 1478 return maps__set_modules_path_dir(machine__kernel_maps(machine), 1479 modules_path, sizeof(modules_path), 0); 1480 } 1481 int __weak arch__fix_module_text_start(u64 *start __maybe_unused, 1482 u64 *size __maybe_unused, 1483 const char *name __maybe_unused) 1484 { 1485 return 0; 1486 } 1487 1488 static int machine__create_module(void *arg, const char *name, u64 start, 1489 u64 size) 1490 { 1491 struct machine *machine = arg; 1492 struct map *map; 1493 1494 if (arch__fix_module_text_start(&start, &size, name) < 0) 1495 return -1; 1496 1497 map = machine__addnew_module_map(machine, start, name); 1498 if (map == NULL) 1499 return -1; 1500 map__set_end(map, start + size); 1501 1502 dso__kernel_module_get_build_id(map__dso(map), machine->root_dir); 1503 map__put(map); 1504 return 0; 1505 } 1506 1507 static int machine__create_modules(struct machine *machine) 1508 { 1509 const char *modules; 1510 char path[PATH_MAX]; 1511 1512 if (machine__is_default_guest(machine)) { 1513 modules = symbol_conf.default_guest_modules; 1514 } else { 1515 snprintf(path, PATH_MAX, "%s/proc/modules", machine->root_dir); 1516 modules = path; 1517 } 1518 1519 if (symbol__restricted_filename(modules, "/proc/modules")) 1520 return -1; 1521 1522 if (modules__parse(modules, machine, machine__create_module)) 1523 return -1; 1524 1525 if (!machine__set_modules_path(machine)) 1526 return 0; 1527 1528 pr_debug("Problems setting modules path maps, continuing anyway...\n"); 1529 1530 return 0; 1531 } 1532 1533 static void machine__set_kernel_mmap(struct machine *machine, 1534 u64 start, u64 end) 1535 { 1536 map__set_start(machine->vmlinux_map, start); 1537 map__set_end(machine->vmlinux_map, end); 1538 /* 1539 * Be a bit paranoid here, some perf.data file came with 1540 * a zero sized synthesized MMAP event for the kernel. 1541 */ 1542 if (start == 0 && end == 0) 1543 map__set_end(machine->vmlinux_map, ~0ULL); 1544 } 1545 1546 struct kernel_mmap_mutation_ctx { 1547 u64 start; 1548 u64 end; 1549 }; 1550 1551 static int kernel_mmap_mutate_cb(struct map *map, void *data) 1552 { 1553 struct kernel_mmap_mutation_ctx *ctx = data; 1554 1555 map__set_start(map, ctx->start); 1556 map__set_end(map, ctx->end); 1557 if (ctx->start == 0 && ctx->end == 0) 1558 map__set_end(map, ~0ULL); 1559 return 0; 1560 } 1561 1562 static int machine__update_kernel_mmap(struct machine *machine, 1563 u64 start, u64 end) 1564 { 1565 struct kernel_mmap_mutation_ctx ctx = { .start = start, .end = end }; 1566 1567 return maps__mutate_mapping(machine__kernel_maps(machine), 1568 machine->vmlinux_map, 1569 kernel_mmap_mutate_cb, &ctx); 1570 } 1571 1572 int machine__create_kernel_maps(struct machine *machine) 1573 { 1574 struct dso *kernel = machine__get_kernel(machine); 1575 const char *name = NULL; 1576 u64 start = 0, end = ~0ULL; 1577 int ret; 1578 1579 if (kernel == NULL) 1580 return -1; 1581 1582 ret = __machine__create_kernel_maps(machine, kernel); 1583 if (ret < 0) 1584 goto out_put; 1585 1586 if (symbol_conf.use_modules && machine__create_modules(machine) < 0) { 1587 if (machine__is_host(machine)) 1588 pr_debug("Problems creating module maps, " 1589 "continuing anyway...\n"); 1590 else 1591 pr_debug("Problems creating module maps for guest %d, " 1592 "continuing anyway...\n", machine->pid); 1593 } 1594 1595 if (!machine__get_running_kernel_start(machine, &name, &start, &end)) { 1596 if (name && 1597 map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, name, start)) { 1598 machine__destroy_kernel_maps(machine); 1599 ret = -1; 1600 goto out_put; 1601 } 1602 1603 /* 1604 * we have a real start address now, so re-order the kmaps 1605 * assume it's the last in the kmaps 1606 */ 1607 ret = machine__update_kernel_mmap(machine, start, end); 1608 if (ret < 0) 1609 goto out_put; 1610 } 1611 1612 if (machine__create_extra_kernel_maps(machine, kernel)) 1613 pr_debug("Problems creating extra kernel maps, continuing anyway...\n"); 1614 1615 if (end == ~0ULL) { 1616 /* update end address of the kernel map using adjacent module address */ 1617 struct map *next = maps__find_next_entry(machine__kernel_maps(machine), 1618 machine__kernel_map(machine)); 1619 1620 if (next) { 1621 machine__set_kernel_mmap(machine, start, map__start(next)); 1622 map__put(next); 1623 } 1624 } 1625 1626 maps__fixup_end(machine__kernel_maps(machine)); 1627 1628 out_put: 1629 dso__put(kernel); 1630 return ret; 1631 } 1632 1633 static int machine__uses_kcore_cb(struct dso *dso, void *data __maybe_unused) 1634 { 1635 return dso__is_kcore(dso) ? 1 : 0; 1636 } 1637 1638 static bool machine__uses_kcore(struct machine *machine) 1639 { 1640 return dsos__for_each_dso(&machine->dsos, machine__uses_kcore_cb, NULL) != 0 ? true : false; 1641 } 1642 1643 static bool machine__is(struct machine *machine, uint16_t e_machine) 1644 { 1645 if (!machine) 1646 return false; 1647 1648 if (!machine->env) { 1649 if (machine__is_host(machine)) 1650 return e_machine == EM_HOST; 1651 return false; 1652 } 1653 1654 return perf_env__e_machine(machine->env, NULL) == e_machine; 1655 } 1656 1657 static bool perf_event__is_extra_kernel_mmap(struct machine *machine, 1658 struct extra_kernel_map *xm) 1659 { 1660 return machine__is(machine, EM_X86_64) && 1661 is_entry_trampoline(xm->name); 1662 } 1663 1664 static int machine__process_extra_kernel_map(struct machine *machine, 1665 struct extra_kernel_map *xm) 1666 { 1667 struct dso *kernel = machine__kernel_dso(machine); 1668 1669 if (kernel == NULL) 1670 return -1; 1671 1672 return machine__create_extra_kernel_map(machine, kernel, xm); 1673 } 1674 1675 static int machine__process_kernel_mmap_event(struct machine *machine, 1676 struct extra_kernel_map *xm, 1677 struct build_id *bid) 1678 { 1679 enum dso_space_type dso_space; 1680 bool is_kernel_mmap; 1681 const char *mmap_name = machine->mmap_name; 1682 1683 /* If we have maps from kcore then we do not need or want any others */ 1684 if (machine__uses_kcore(machine)) 1685 return 0; 1686 1687 if (machine__is_host(machine)) 1688 dso_space = DSO_SPACE__KERNEL; 1689 else 1690 dso_space = DSO_SPACE__KERNEL_GUEST; 1691 1692 is_kernel_mmap = memcmp(xm->name, mmap_name, strlen(mmap_name) - 1) == 0; 1693 if (!is_kernel_mmap && !machine__is_host(machine)) { 1694 /* 1695 * If the event was recorded inside the guest and injected into 1696 * the host perf.data file, then it will match a host mmap_name, 1697 * so try that - see machine__set_mmap_name(). 1698 */ 1699 mmap_name = "[kernel.kallsyms]"; 1700 is_kernel_mmap = memcmp(xm->name, mmap_name, strlen(mmap_name) - 1) == 0; 1701 } 1702 if (xm->name[0] == '/' || 1703 (!is_kernel_mmap && xm->name[0] == '[')) { 1704 struct map *map = machine__addnew_module_map(machine, xm->start, xm->name); 1705 1706 if (map == NULL) 1707 goto out_problem; 1708 1709 map__set_end(map, map__start(map) + xm->end - xm->start); 1710 1711 if (build_id__is_defined(bid)) 1712 dso__set_build_id(map__dso(map), bid); 1713 1714 map__put(map); 1715 } else if (is_kernel_mmap) { 1716 const char *symbol_name = xm->name + strlen(mmap_name); 1717 /* 1718 * Should be there already, from the build-id table in 1719 * the header. 1720 */ 1721 struct dso *kernel = dsos__find_kernel_dso(&machine->dsos); 1722 1723 if (kernel == NULL) 1724 kernel = machine__findnew_dso(machine, machine->mmap_name); 1725 if (kernel == NULL) 1726 goto out_problem; 1727 1728 dso__set_kernel(kernel, dso_space); 1729 if (__machine__create_kernel_maps(machine, kernel) < 0) { 1730 dso__put(kernel); 1731 goto out_problem; 1732 } 1733 1734 if (strstr(dso__long_name(kernel), "vmlinux")) 1735 dso__set_short_name(kernel, "[kernel.vmlinux]", false); 1736 1737 if (machine__update_kernel_mmap(machine, xm->start, xm->end) < 0) { 1738 dso__put(kernel); 1739 goto out_problem; 1740 } 1741 1742 if (build_id__is_defined(bid)) 1743 dso__set_build_id(kernel, bid); 1744 1745 /* 1746 * Avoid using a zero address (kptr_restrict) for the ref reloc 1747 * symbol. Effectively having zero here means that at record 1748 * time /proc/sys/kernel/kptr_restrict was non zero. 1749 */ 1750 if (xm->pgoff != 0) { 1751 map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, 1752 symbol_name, 1753 xm->pgoff); 1754 } 1755 1756 if (machine__is_default_guest(machine)) { 1757 /* 1758 * preload dso of guest kernel and modules 1759 */ 1760 dso__load(kernel, machine__kernel_map(machine)); 1761 } 1762 dso__put(kernel); 1763 } else if (perf_event__is_extra_kernel_mmap(machine, xm)) { 1764 return machine__process_extra_kernel_map(machine, xm); 1765 } 1766 return 0; 1767 out_problem: 1768 return -1; 1769 } 1770 1771 int machine__process_mmap2_event(struct machine *machine, 1772 union perf_event *event, 1773 struct perf_sample *sample) 1774 { 1775 struct thread *thread; 1776 struct map *map; 1777 struct dso_id dso_id = dso_id_empty; 1778 int ret = 0; 1779 1780 if (dump_trace) 1781 perf_event__fprintf_mmap2(event, stdout); 1782 1783 if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) { 1784 build_id__init(&dso_id.build_id, event->mmap2.build_id, event->mmap2.build_id_size); 1785 } else { 1786 dso_id.maj = event->mmap2.maj; 1787 dso_id.min = event->mmap2.min; 1788 dso_id.ino = event->mmap2.ino; 1789 dso_id.ino_generation = event->mmap2.ino_generation; 1790 dso_id.mmap2_valid = true; 1791 dso_id.mmap2_ino_generation_valid = true; 1792 } 1793 1794 if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL || 1795 sample->cpumode == PERF_RECORD_MISC_KERNEL) { 1796 struct extra_kernel_map xm = { 1797 .start = event->mmap2.start, 1798 .end = event->mmap2.start + event->mmap2.len, 1799 .pgoff = event->mmap2.pgoff, 1800 }; 1801 1802 strlcpy(xm.name, event->mmap2.filename, KMAP_NAME_LEN); 1803 ret = machine__process_kernel_mmap_event(machine, &xm, &dso_id.build_id); 1804 if (ret < 0) 1805 goto out_problem; 1806 return 0; 1807 } 1808 1809 thread = machine__findnew_thread(machine, event->mmap2.pid, 1810 event->mmap2.tid); 1811 if (thread == NULL) 1812 goto out_problem; 1813 1814 map = map__new(machine, event->mmap2.start, 1815 event->mmap2.len, event->mmap2.pgoff, 1816 &dso_id, event->mmap2.prot, 1817 event->mmap2.flags, 1818 event->mmap2.filename, thread); 1819 1820 if (map == NULL) 1821 goto out_problem_map; 1822 1823 ret = thread__insert_map(thread, map); 1824 if (ret) 1825 goto out_problem_insert; 1826 1827 thread__put(thread); 1828 map__put(map); 1829 return 0; 1830 1831 out_problem_insert: 1832 map__put(map); 1833 out_problem_map: 1834 thread__put(thread); 1835 out_problem: 1836 dump_printf("problem processing PERF_RECORD_MMAP2, skipping event.\n"); 1837 return 0; 1838 } 1839 1840 int machine__process_mmap_event(struct machine *machine, union perf_event *event, 1841 struct perf_sample *sample) 1842 { 1843 struct thread *thread; 1844 struct map *map; 1845 u32 prot = 0; 1846 int ret = 0; 1847 1848 if (dump_trace) 1849 perf_event__fprintf_mmap(event, stdout); 1850 1851 if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL || 1852 sample->cpumode == PERF_RECORD_MISC_KERNEL) { 1853 struct extra_kernel_map xm = { 1854 .start = event->mmap.start, 1855 .end = event->mmap.start + event->mmap.len, 1856 .pgoff = event->mmap.pgoff, 1857 }; 1858 1859 strlcpy(xm.name, event->mmap.filename, KMAP_NAME_LEN); 1860 ret = machine__process_kernel_mmap_event(machine, &xm, NULL); 1861 if (ret < 0) 1862 goto out_problem; 1863 return 0; 1864 } 1865 1866 thread = machine__findnew_thread(machine, event->mmap.pid, 1867 event->mmap.tid); 1868 if (thread == NULL) 1869 goto out_problem; 1870 1871 if (!(event->header.misc & PERF_RECORD_MISC_MMAP_DATA)) 1872 prot = PROT_EXEC; 1873 1874 map = map__new(machine, event->mmap.start, 1875 event->mmap.len, event->mmap.pgoff, 1876 &dso_id_empty, prot, /*flags=*/0, event->mmap.filename, thread); 1877 1878 if (map == NULL) 1879 goto out_problem_map; 1880 1881 ret = thread__insert_map(thread, map); 1882 if (ret) 1883 goto out_problem_insert; 1884 1885 thread__put(thread); 1886 map__put(map); 1887 return 0; 1888 1889 out_problem_insert: 1890 map__put(map); 1891 out_problem_map: 1892 thread__put(thread); 1893 out_problem: 1894 dump_printf("problem processing PERF_RECORD_MMAP, skipping event.\n"); 1895 return 0; 1896 } 1897 1898 void machine__remove_thread(struct machine *machine, struct thread *th) 1899 { 1900 return threads__remove(&machine->threads, th); 1901 } 1902 1903 int machine__process_fork_event(struct machine *machine, union perf_event *event, 1904 struct perf_sample *sample) 1905 { 1906 struct thread *thread = machine__find_thread(machine, 1907 event->fork.pid, 1908 event->fork.tid); 1909 struct thread *parent = machine__findnew_thread(machine, 1910 event->fork.ppid, 1911 event->fork.ptid); 1912 bool do_maps_clone = true; 1913 int err = 0; 1914 1915 if (dump_trace) 1916 perf_event__fprintf_task(event, stdout); 1917 1918 /* 1919 * There may be an existing thread that is not actually the parent, 1920 * either because we are processing events out of order, or because the 1921 * (fork) event that would have removed the thread was lost. Assume the 1922 * latter case and continue on as best we can. 1923 */ 1924 if (thread__pid(parent) != (pid_t)event->fork.ppid) { 1925 dump_printf("removing erroneous parent thread %d/%d\n", 1926 thread__pid(parent), thread__tid(parent)); 1927 machine__remove_thread(machine, parent); 1928 thread__put(parent); 1929 parent = machine__findnew_thread(machine, event->fork.ppid, 1930 event->fork.ptid); 1931 } 1932 1933 /* if a thread currently exists for the thread id remove it */ 1934 if (thread != NULL) { 1935 machine__remove_thread(machine, thread); 1936 thread__put(thread); 1937 } 1938 1939 thread = machine__findnew_thread(machine, event->fork.pid, 1940 event->fork.tid); 1941 /* 1942 * When synthesizing FORK events, we are trying to create thread 1943 * objects for the already running tasks on the machine. 1944 * 1945 * Normally, for a kernel FORK event, we want to clone the parent's 1946 * maps because that is what the kernel just did. 1947 * 1948 * But when synthesizing, this should not be done. If we do, we end up 1949 * with overlapping maps as we process the synthesized MMAP2 events that 1950 * get delivered shortly thereafter. 1951 * 1952 * Use the FORK event misc flags in an internal way to signal this 1953 * situation, so we can elide the map clone when appropriate. 1954 */ 1955 if (event->fork.header.misc & PERF_RECORD_MISC_FORK_EXEC) 1956 do_maps_clone = false; 1957 1958 if (thread == NULL || parent == NULL || 1959 thread__fork(thread, parent, sample->time, do_maps_clone) < 0) { 1960 dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n"); 1961 err = -1; 1962 } 1963 thread__put(thread); 1964 thread__put(parent); 1965 1966 return err; 1967 } 1968 1969 int machine__process_exit_event(struct machine *machine, union perf_event *event, 1970 struct perf_sample *sample __maybe_unused) 1971 { 1972 struct thread *thread = machine__find_thread(machine, 1973 event->fork.pid, 1974 event->fork.tid); 1975 1976 if (dump_trace) 1977 perf_event__fprintf_task(event, stdout); 1978 1979 /* There is no context switch out before exit, so we decrement here. */ 1980 machine->parallelism--; 1981 if (thread != NULL) { 1982 if (symbol_conf.keep_exited_threads) 1983 thread__set_exited(thread, /*exited=*/true); 1984 else 1985 machine__remove_thread(machine, thread); 1986 } 1987 thread__put(thread); 1988 return 0; 1989 } 1990 1991 int machine__process_event(struct machine *machine, union perf_event *event, 1992 struct perf_sample *sample) 1993 { 1994 int ret; 1995 1996 switch (event->header.type) { 1997 case PERF_RECORD_COMM: 1998 ret = machine__process_comm_event(machine, event, sample); break; 1999 case PERF_RECORD_MMAP: 2000 ret = machine__process_mmap_event(machine, event, sample); break; 2001 case PERF_RECORD_NAMESPACES: 2002 ret = machine__process_namespaces_event(machine, event, sample); break; 2003 case PERF_RECORD_CGROUP: 2004 ret = machine__process_cgroup_event(machine, event, sample); break; 2005 case PERF_RECORD_MMAP2: 2006 ret = machine__process_mmap2_event(machine, event, sample); break; 2007 case PERF_RECORD_FORK: 2008 ret = machine__process_fork_event(machine, event, sample); break; 2009 case PERF_RECORD_EXIT: 2010 ret = machine__process_exit_event(machine, event, sample); break; 2011 case PERF_RECORD_LOST: 2012 ret = machine__process_lost_event(machine, event, sample); break; 2013 case PERF_RECORD_AUX: 2014 ret = machine__process_aux_event(machine, event); break; 2015 case PERF_RECORD_ITRACE_START: 2016 ret = machine__process_itrace_start_event(machine, event); break; 2017 case PERF_RECORD_LOST_SAMPLES: 2018 ret = machine__process_lost_samples_event(machine, event, sample); break; 2019 case PERF_RECORD_SWITCH: 2020 case PERF_RECORD_SWITCH_CPU_WIDE: 2021 ret = machine__process_switch_event(machine, event); break; 2022 case PERF_RECORD_KSYMBOL: 2023 ret = machine__process_ksymbol(machine, event, sample); break; 2024 case PERF_RECORD_BPF_EVENT: 2025 ret = machine__process_bpf(machine, event, sample); break; 2026 case PERF_RECORD_TEXT_POKE: 2027 ret = machine__process_text_poke(machine, event, sample); break; 2028 case PERF_RECORD_AUX_OUTPUT_HW_ID: 2029 ret = machine__process_aux_output_hw_id_event(machine, event); break; 2030 default: 2031 ret = -1; 2032 break; 2033 } 2034 2035 return ret; 2036 } 2037 2038 static bool symbol__match_regex(struct symbol *sym, regex_t *regex) 2039 { 2040 return regexec(regex, sym->name, 0, NULL, 0) == 0; 2041 } 2042 2043 static void ip__resolve_ams(struct thread *thread, 2044 struct addr_map_symbol *ams, 2045 u64 ip) 2046 { 2047 struct addr_location al; 2048 2049 addr_location__init(&al); 2050 /* 2051 * We cannot use the header.misc hint to determine whether a 2052 * branch stack address is user, kernel, guest, hypervisor. 2053 * Branches may straddle the kernel/user/hypervisor boundaries. 2054 * Thus, we have to try consecutively until we find a match 2055 * or else, the symbol is unknown 2056 */ 2057 thread__find_cpumode_addr_location(thread, ip, /*symbols=*/true, &al); 2058 2059 ams->addr = ip; 2060 ams->al_addr = al.addr; 2061 ams->al_level = al.level; 2062 ams->ms.thread = thread__get(al.thread); 2063 ams->ms.sym = al.sym; 2064 ams->ms.map = map__get(al.map); 2065 ams->phys_addr = 0; 2066 ams->data_page_size = 0; 2067 addr_location__exit(&al); 2068 } 2069 2070 static void ip__resolve_data(struct thread *thread, 2071 u8 m, struct addr_map_symbol *ams, 2072 u64 addr, u64 phys_addr, u64 daddr_page_size) 2073 { 2074 struct addr_location al; 2075 2076 addr_location__init(&al); 2077 2078 thread__find_symbol(thread, m, addr, &al); 2079 2080 ams->addr = addr; 2081 ams->al_addr = al.addr; 2082 ams->al_level = al.level; 2083 ams->ms.thread = thread__get(al.thread); 2084 ams->ms.sym = al.sym; 2085 ams->ms.map = map__get(al.map); 2086 ams->phys_addr = phys_addr; 2087 ams->data_page_size = daddr_page_size; 2088 addr_location__exit(&al); 2089 } 2090 2091 struct mem_info *sample__resolve_mem(struct perf_sample *sample, 2092 struct addr_location *al) 2093 { 2094 struct mem_info *mi = mem_info__new(); 2095 2096 if (!mi) 2097 return NULL; 2098 2099 ip__resolve_ams(al->thread, mem_info__iaddr(mi), sample->ip); 2100 ip__resolve_data(al->thread, al->cpumode, mem_info__daddr(mi), 2101 sample->addr, sample->phys_addr, 2102 sample->data_page_size); 2103 mem_info__data_src(mi)->val = sample->data_src; 2104 2105 return mi; 2106 } 2107 2108 static char *callchain_srcline(struct map_symbol *ms, u64 ip) 2109 { 2110 struct map *map = ms->map; 2111 char *srcline = NULL; 2112 struct dso *dso; 2113 2114 if (!map || callchain_param.key == CCKEY_FUNCTION) 2115 return srcline; 2116 2117 dso = map__dso(map); 2118 srcline = srcline__tree_find(dso__srclines(dso), ip); 2119 if (!srcline) { 2120 bool show_sym = false; 2121 bool show_addr = callchain_param.key == CCKEY_ADDRESS; 2122 2123 srcline = get_srcline(dso, map__rip_2objdump(map, ip), 2124 ms->sym, show_sym, show_addr, ip); 2125 srcline__tree_insert(dso__srclines(dso), ip, srcline); 2126 } 2127 2128 return srcline; 2129 } 2130 2131 struct iterations { 2132 int nr_loop_iter; 2133 u64 cycles; 2134 }; 2135 2136 static int append_inlines(struct callchain_cursor *cursor, struct map_symbol *ms, u64 ip, 2137 bool branch, struct branch_flags *flags, int nr_loop_iter, 2138 u64 iter_cycles, u64 branch_from) 2139 { 2140 struct symbol *sym = ms->sym; 2141 struct map *map = ms->map; 2142 struct inline_node *inline_node; 2143 struct inline_list *ilist; 2144 struct dso *dso; 2145 u64 addr; 2146 int ret = 1; 2147 struct map_symbol ilist_ms; 2148 bool first = true; 2149 2150 if (!symbol_conf.inline_name || !map || !sym) 2151 return ret; 2152 2153 addr = map__dso_map_ip(map, ip); 2154 addr = map__rip_2objdump(map, addr); 2155 dso = map__dso(map); 2156 2157 inline_node = inlines__tree_find(dso__inlined_nodes(dso), addr); 2158 if (!inline_node) { 2159 inline_node = dso__parse_addr_inlines(dso, addr, sym); 2160 if (!inline_node) 2161 return ret; 2162 inlines__tree_insert(dso__inlined_nodes(dso), inline_node); 2163 } 2164 2165 ilist_ms = (struct map_symbol) { 2166 .thread = thread__get(ms->thread), 2167 .map = map__get(map), 2168 }; 2169 list_for_each_entry(ilist, &inline_node->val, list) { 2170 ilist_ms.sym = ilist->symbol; 2171 if (first) { 2172 ret = callchain_cursor_append(cursor, ip, &ilist_ms, 2173 branch, flags, nr_loop_iter, 2174 iter_cycles, branch_from, ilist->srcline); 2175 } else { 2176 ret = callchain_cursor_append(cursor, ip, &ilist_ms, false, 2177 NULL, 0, 0, 0, ilist->srcline); 2178 } 2179 first = false; 2180 2181 if (ret != 0) 2182 return ret; 2183 } 2184 map_symbol__exit(&ilist_ms); 2185 2186 return ret; 2187 } 2188 2189 static int add_callchain_ip(struct thread *thread, 2190 struct callchain_cursor *cursor, 2191 struct symbol **parent, 2192 struct addr_location *root_al, 2193 u8 *cpumode, 2194 u64 ip, 2195 bool branch, 2196 struct branch_flags *flags, 2197 struct iterations *iter, 2198 u64 branch_from, 2199 bool symbols) 2200 { 2201 struct map_symbol ms = {}; 2202 struct addr_location al; 2203 int nr_loop_iter = 0, err = 0; 2204 u64 iter_cycles = 0; 2205 const char *srcline = NULL; 2206 2207 addr_location__init(&al); 2208 al.filtered = 0; 2209 al.sym = NULL; 2210 al.srcline = NULL; 2211 if (!cpumode) { 2212 thread__find_cpumode_addr_location(thread, ip, symbols, &al); 2213 } else { 2214 if (ip >= PERF_CONTEXT_MAX) { 2215 switch (ip) { 2216 case PERF_CONTEXT_HV: 2217 *cpumode = PERF_RECORD_MISC_HYPERVISOR; 2218 break; 2219 case PERF_CONTEXT_KERNEL: 2220 *cpumode = PERF_RECORD_MISC_KERNEL; 2221 break; 2222 case PERF_CONTEXT_USER: 2223 case PERF_CONTEXT_USER_DEFERRED: 2224 *cpumode = PERF_RECORD_MISC_USER; 2225 break; 2226 default: 2227 pr_debug("invalid callchain context: " 2228 "%"PRId64"\n", (s64) ip); 2229 /* 2230 * It seems the callchain is corrupted. 2231 * Discard all. 2232 */ 2233 callchain_cursor_reset(cursor); 2234 err = 1; 2235 goto out; 2236 } 2237 goto out; 2238 } 2239 if (symbols) 2240 thread__find_symbol(thread, *cpumode, ip, &al); 2241 else 2242 thread__find_map(thread, *cpumode, ip, &al); 2243 } 2244 2245 if (al.sym != NULL) { 2246 if (perf_hpp_list.parent && !*parent && 2247 symbol__match_regex(al.sym, &parent_regex)) 2248 *parent = al.sym; 2249 else if (have_ignore_callees && root_al && 2250 symbol__match_regex(al.sym, &ignore_callees_regex)) { 2251 /* Treat this symbol as the root, 2252 forgetting its callees. */ 2253 addr_location__copy(root_al, &al); 2254 callchain_cursor_reset(cursor); 2255 } 2256 } 2257 2258 if (symbol_conf.hide_unresolved && al.sym == NULL) 2259 goto out; 2260 2261 if (iter) { 2262 nr_loop_iter = iter->nr_loop_iter; 2263 iter_cycles = iter->cycles; 2264 } 2265 2266 ms.thread = thread__get(al.thread); 2267 ms.map = map__get(al.map); 2268 ms.sym = al.sym; 2269 2270 if (append_inlines(cursor, &ms, ip, branch, flags, nr_loop_iter, 2271 iter_cycles, branch_from) == 0) 2272 goto out; 2273 2274 srcline = callchain_srcline(&ms, al.addr); 2275 err = callchain_cursor_append(cursor, ip, &ms, 2276 branch, flags, nr_loop_iter, 2277 iter_cycles, branch_from, srcline); 2278 out: 2279 addr_location__exit(&al); 2280 map_symbol__exit(&ms); 2281 return err; 2282 } 2283 2284 struct branch_info *sample__resolve_bstack(struct perf_sample *sample, 2285 struct addr_location *al) 2286 { 2287 unsigned int i; 2288 const struct branch_stack *bs = sample->branch_stack; 2289 struct branch_entry *entries = perf_sample__branch_entries(sample); 2290 u64 *branch_stack_cntr = sample->branch_stack_cntr; 2291 struct branch_info *bi = calloc(bs->nr, sizeof(struct branch_info)); 2292 2293 if (!bi) 2294 return NULL; 2295 2296 for (i = 0; i < bs->nr; i++) { 2297 ip__resolve_ams(al->thread, &bi[i].to, entries[i].to); 2298 ip__resolve_ams(al->thread, &bi[i].from, entries[i].from); 2299 bi[i].flags = entries[i].flags; 2300 if (branch_stack_cntr) 2301 bi[i].branch_stack_cntr = branch_stack_cntr[i]; 2302 } 2303 return bi; 2304 } 2305 2306 static void save_iterations(struct iterations *iter, 2307 struct branch_entry *be, int nr) 2308 { 2309 int i; 2310 2311 iter->nr_loop_iter++; 2312 iter->cycles = 0; 2313 2314 for (i = 0; i < nr; i++) 2315 iter->cycles += be[i].flags.cycles; 2316 } 2317 2318 #define CHASHSZ 127 2319 #define CHASHBITS 7 2320 #define NO_ENTRY 0xff 2321 2322 #define PERF_MAX_BRANCH_DEPTH 127 2323 2324 /* Remove loops. */ 2325 static int remove_loops(struct branch_entry *l, int nr, 2326 struct iterations *iter) 2327 { 2328 int i, j, off; 2329 unsigned char chash[CHASHSZ]; 2330 2331 memset(chash, NO_ENTRY, sizeof(chash)); 2332 2333 BUG_ON(PERF_MAX_BRANCH_DEPTH > 255); 2334 2335 for (i = 0; i < nr; i++) { 2336 int h = hash_64(l[i].from, CHASHBITS) % CHASHSZ; 2337 2338 /* no collision handling for now */ 2339 if (chash[h] == NO_ENTRY) { 2340 chash[h] = i; 2341 } else if (l[chash[h]].from == l[i].from) { 2342 bool is_loop = true; 2343 /* check if it is a real loop */ 2344 off = 0; 2345 for (j = chash[h]; j < i && i + off < nr; j++, off++) 2346 if (l[j].from != l[i + off].from) { 2347 is_loop = false; 2348 break; 2349 } 2350 if (is_loop) { 2351 j = nr - (i + off); 2352 if (j > 0) { 2353 save_iterations(iter + i + off, 2354 l + i, off); 2355 2356 memmove(iter + i, iter + i + off, 2357 j * sizeof(*iter)); 2358 2359 memmove(l + i, l + i + off, 2360 j * sizeof(*l)); 2361 } 2362 2363 nr -= off; 2364 } 2365 } 2366 } 2367 return nr; 2368 } 2369 2370 static int lbr_callchain_add_kernel_ip(struct thread *thread, 2371 struct callchain_cursor *cursor, 2372 struct perf_sample *sample, 2373 struct symbol **parent, 2374 struct addr_location *root_al, 2375 u64 branch_from, 2376 bool callee, int end, 2377 bool symbols) 2378 { 2379 struct ip_callchain *chain = sample->callchain; 2380 u8 cpumode = PERF_RECORD_MISC_USER; 2381 int err, i; 2382 2383 if (callee) { 2384 for (i = 0; i < end + 1; i++) { 2385 err = add_callchain_ip(thread, cursor, parent, 2386 root_al, &cpumode, chain->ips[i], 2387 false, NULL, NULL, branch_from, 2388 symbols); 2389 if (err) 2390 return err; 2391 } 2392 return 0; 2393 } 2394 2395 for (i = end; i >= 0; i--) { 2396 err = add_callchain_ip(thread, cursor, parent, 2397 root_al, &cpumode, chain->ips[i], 2398 false, NULL, NULL, branch_from, 2399 symbols); 2400 if (err) 2401 return err; 2402 } 2403 2404 return 0; 2405 } 2406 2407 static void save_lbr_cursor_node(struct thread *thread, 2408 struct callchain_cursor *cursor, 2409 int idx) 2410 { 2411 struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread); 2412 2413 if (!lbr_stitch) 2414 return; 2415 2416 if (cursor->pos == cursor->nr) { 2417 lbr_stitch->prev_lbr_cursor[idx].valid = false; 2418 return; 2419 } 2420 2421 if (!cursor->curr) 2422 cursor->curr = cursor->first; 2423 else 2424 cursor->curr = cursor->curr->next; 2425 2426 map_symbol__exit(&lbr_stitch->prev_lbr_cursor[idx].ms); 2427 memcpy(&lbr_stitch->prev_lbr_cursor[idx], cursor->curr, 2428 sizeof(struct callchain_cursor_node)); 2429 lbr_stitch->prev_lbr_cursor[idx].ms.thread = thread__get(cursor->curr->ms.thread); 2430 lbr_stitch->prev_lbr_cursor[idx].ms.map = map__get(cursor->curr->ms.map); 2431 2432 lbr_stitch->prev_lbr_cursor[idx].valid = true; 2433 cursor->pos++; 2434 } 2435 2436 static int lbr_callchain_add_lbr_ip(struct thread *thread, 2437 struct callchain_cursor *cursor, 2438 struct perf_sample *sample, 2439 struct symbol **parent, 2440 struct addr_location *root_al, 2441 u64 *branch_from, 2442 bool callee, 2443 bool symbols) 2444 { 2445 struct branch_stack *lbr_stack = sample->branch_stack; 2446 struct branch_entry *entries = perf_sample__branch_entries(sample); 2447 u8 cpumode = PERF_RECORD_MISC_USER; 2448 int lbr_nr = lbr_stack->nr; 2449 struct branch_flags *flags; 2450 int err, i; 2451 u64 ip; 2452 2453 /* 2454 * The curr and pos are not used in writing session. They are cleared 2455 * in callchain_cursor_commit() when the writing session is closed. 2456 * Using curr and pos to track the current cursor node. 2457 */ 2458 if (thread__lbr_stitch(thread)) { 2459 cursor->curr = NULL; 2460 cursor->pos = cursor->nr; 2461 if (cursor->nr) { 2462 cursor->curr = cursor->first; 2463 for (i = 0; i < (int)(cursor->nr - 1); i++) 2464 cursor->curr = cursor->curr->next; 2465 } 2466 } 2467 2468 if (callee) { 2469 /* 2470 * Set the (first) leaf function's IP to sample->ip (the 2471 * location of the sample) but if not recorded use entries.to 2472 */ 2473 if (sample->ip) 2474 ip = sample->ip; 2475 else 2476 ip = entries[0].to; 2477 flags = &entries[0].flags; 2478 *branch_from = entries[0].from; 2479 err = add_callchain_ip(thread, cursor, parent, 2480 root_al, &cpumode, ip, 2481 true, flags, NULL, 2482 *branch_from, symbols); 2483 if (err) 2484 return err; 2485 2486 /* 2487 * The number of cursor node increases. 2488 * Move the current cursor node. 2489 * But does not need to save current cursor node for entry 0. 2490 * It's impossible to stitch the whole LBRs of previous sample. 2491 */ 2492 if (thread__lbr_stitch(thread) && (cursor->pos != cursor->nr)) { 2493 if (!cursor->curr) 2494 cursor->curr = cursor->first; 2495 else 2496 cursor->curr = cursor->curr->next; 2497 cursor->pos++; 2498 } 2499 2500 /* Add LBR ip from entries.from one by one. */ 2501 for (i = 0; i < lbr_nr; i++) { 2502 ip = entries[i].from; 2503 flags = &entries[i].flags; 2504 err = add_callchain_ip(thread, cursor, parent, 2505 root_al, &cpumode, ip, 2506 true, flags, NULL, 2507 *branch_from, symbols); 2508 if (err) 2509 return err; 2510 save_lbr_cursor_node(thread, cursor, i); 2511 } 2512 return 0; 2513 } 2514 2515 /* Add LBR ip from entries.from one by one. */ 2516 for (i = lbr_nr - 1; i >= 0; i--) { 2517 ip = entries[i].from; 2518 flags = &entries[i].flags; 2519 err = add_callchain_ip(thread, cursor, parent, 2520 root_al, &cpumode, ip, 2521 true, flags, NULL, 2522 *branch_from, symbols); 2523 if (err) 2524 return err; 2525 save_lbr_cursor_node(thread, cursor, i); 2526 } 2527 2528 if (lbr_nr > 0) { 2529 /* 2530 * Set the (first) leaf function's IP to sample->ip (the 2531 * location of the sample) but if not recorded use entries.to 2532 */ 2533 if (sample->ip) 2534 ip = sample->ip; 2535 else 2536 ip = entries[0].to; 2537 flags = &entries[0].flags; 2538 *branch_from = entries[0].from; 2539 err = add_callchain_ip(thread, cursor, parent, 2540 root_al, &cpumode, ip, 2541 true, flags, NULL, 2542 *branch_from, symbols); 2543 if (err) 2544 return err; 2545 } 2546 2547 return 0; 2548 } 2549 2550 static int lbr_callchain_add_stitched_lbr_ip(struct thread *thread, 2551 struct callchain_cursor *cursor) 2552 { 2553 struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread); 2554 struct callchain_cursor_node *cnode; 2555 struct stitch_list *stitch_node; 2556 int err; 2557 2558 list_for_each_entry(stitch_node, &lbr_stitch->lists, node) { 2559 cnode = &stitch_node->cursor; 2560 2561 err = callchain_cursor_append(cursor, cnode->ip, 2562 &cnode->ms, 2563 cnode->branch, 2564 &cnode->branch_flags, 2565 cnode->nr_loop_iter, 2566 cnode->iter_cycles, 2567 cnode->branch_from, 2568 cnode->srcline); 2569 if (err) 2570 return err; 2571 } 2572 return 0; 2573 } 2574 2575 static struct stitch_list *get_stitch_node(struct thread *thread) 2576 { 2577 struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread); 2578 struct stitch_list *stitch_node; 2579 2580 if (!list_empty(&lbr_stitch->free_lists)) { 2581 stitch_node = list_first_entry(&lbr_stitch->free_lists, 2582 struct stitch_list, node); 2583 list_del(&stitch_node->node); 2584 2585 return stitch_node; 2586 } 2587 2588 return malloc(sizeof(struct stitch_list)); 2589 } 2590 2591 static bool has_stitched_lbr(struct thread *thread, 2592 struct perf_sample *cur, 2593 struct perf_sample *prev, 2594 unsigned int max_lbr, 2595 bool callee) 2596 { 2597 struct branch_stack *cur_stack = cur->branch_stack; 2598 struct branch_entry *cur_entries = perf_sample__branch_entries(cur); 2599 struct branch_stack *prev_stack = prev->branch_stack; 2600 struct branch_entry *prev_entries = perf_sample__branch_entries(prev); 2601 struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread); 2602 int i, j, nr_identical_branches = 0; 2603 struct stitch_list *stitch_node; 2604 u64 cur_base, distance; 2605 2606 if (!cur_stack || !prev_stack) 2607 return false; 2608 2609 /* Find the physical index of the base-of-stack for current sample. */ 2610 cur_base = max_lbr - cur_stack->nr + cur_stack->hw_idx + 1; 2611 2612 distance = (prev_stack->hw_idx > cur_base) ? (prev_stack->hw_idx - cur_base) : 2613 (max_lbr + prev_stack->hw_idx - cur_base); 2614 /* Previous sample has shorter stack. Nothing can be stitched. */ 2615 if (distance + 1 > prev_stack->nr) 2616 return false; 2617 2618 /* 2619 * Check if there are identical LBRs between two samples. 2620 * Identical LBRs must have same from, to and flags values. Also, 2621 * they have to be saved in the same LBR registers (same physical 2622 * index). 2623 * 2624 * Starts from the base-of-stack of current sample. 2625 */ 2626 for (i = distance, j = cur_stack->nr - 1; (i >= 0) && (j >= 0); i--, j--) { 2627 if ((prev_entries[i].from != cur_entries[j].from) || 2628 (prev_entries[i].to != cur_entries[j].to) || 2629 (prev_entries[i].flags.value != cur_entries[j].flags.value)) 2630 break; 2631 nr_identical_branches++; 2632 } 2633 2634 if (!nr_identical_branches) 2635 return false; 2636 2637 /* 2638 * Save the LBRs between the base-of-stack of previous sample 2639 * and the base-of-stack of current sample into lbr_stitch->lists. 2640 * These LBRs will be stitched later. 2641 */ 2642 for (i = prev_stack->nr - 1; i > (int)distance; i--) { 2643 2644 if (!lbr_stitch->prev_lbr_cursor[i].valid) 2645 continue; 2646 2647 stitch_node = get_stitch_node(thread); 2648 if (!stitch_node) 2649 return false; 2650 2651 memcpy(&stitch_node->cursor, &lbr_stitch->prev_lbr_cursor[i], 2652 sizeof(struct callchain_cursor_node)); 2653 2654 stitch_node->cursor.ms.thread = 2655 thread__get(lbr_stitch->prev_lbr_cursor[i].ms.thread); 2656 stitch_node->cursor.ms.map = map__get(lbr_stitch->prev_lbr_cursor[i].ms.map); 2657 2658 if (callee) 2659 list_add(&stitch_node->node, &lbr_stitch->lists); 2660 else 2661 list_add_tail(&stitch_node->node, &lbr_stitch->lists); 2662 } 2663 2664 return true; 2665 } 2666 2667 static bool alloc_lbr_stitch(struct thread *thread, unsigned int max_lbr) 2668 { 2669 if (thread__lbr_stitch(thread)) 2670 return true; 2671 2672 thread__set_lbr_stitch(thread, zalloc(sizeof(struct lbr_stitch))); 2673 if (!thread__lbr_stitch(thread)) 2674 goto err; 2675 2676 thread__lbr_stitch(thread)->prev_lbr_cursor = 2677 calloc(max_lbr + 1, sizeof(struct callchain_cursor_node)); 2678 if (!thread__lbr_stitch(thread)->prev_lbr_cursor) 2679 goto free_lbr_stitch; 2680 2681 thread__lbr_stitch(thread)->prev_lbr_cursor_size = max_lbr + 1; 2682 2683 INIT_LIST_HEAD(&thread__lbr_stitch(thread)->lists); 2684 INIT_LIST_HEAD(&thread__lbr_stitch(thread)->free_lists); 2685 2686 return true; 2687 2688 free_lbr_stitch: 2689 free(thread__lbr_stitch(thread)); 2690 thread__set_lbr_stitch(thread, NULL); 2691 err: 2692 pr_warning("Failed to allocate space for stitched LBRs. Disable LBR stitch\n"); 2693 thread__set_lbr_stitch_enable(thread, false); 2694 return false; 2695 } 2696 2697 /* 2698 * Resolve LBR callstack chain sample 2699 * Return: 2700 * 1 on success get LBR callchain information 2701 * 0 no available LBR callchain information, should try fp 2702 * negative error code on other errors. 2703 */ 2704 static int resolve_lbr_callchain_sample(struct thread *thread, 2705 struct callchain_cursor *cursor, 2706 struct perf_sample *sample, 2707 struct symbol **parent, 2708 struct addr_location *root_al, 2709 int max_stack, 2710 unsigned int max_lbr, 2711 bool symbols) 2712 { 2713 bool callee = (callchain_param.order == ORDER_CALLEE); 2714 struct ip_callchain *chain = sample->callchain; 2715 int chain_nr = min(max_stack, (int)chain->nr), i; 2716 struct lbr_stitch *lbr_stitch; 2717 bool stitched_lbr = false; 2718 u64 branch_from = 0; 2719 int err; 2720 2721 for (i = 0; i < chain_nr; i++) { 2722 if (chain->ips[i] == PERF_CONTEXT_USER) 2723 break; 2724 } 2725 2726 /* LBR only affects the user callchain */ 2727 if (i == chain_nr) 2728 return 0; 2729 2730 if (thread__lbr_stitch_enable(thread) && !sample->no_hw_idx && 2731 (max_lbr > 0) && alloc_lbr_stitch(thread, max_lbr)) { 2732 lbr_stitch = thread__lbr_stitch(thread); 2733 2734 stitched_lbr = has_stitched_lbr(thread, sample, 2735 &lbr_stitch->prev_sample, 2736 max_lbr, callee); 2737 2738 if (!stitched_lbr && !list_empty(&lbr_stitch->lists)) { 2739 struct stitch_list *stitch_node; 2740 2741 list_for_each_entry(stitch_node, &lbr_stitch->lists, node) 2742 map_symbol__exit(&stitch_node->cursor.ms); 2743 2744 list_splice_init(&lbr_stitch->lists, &lbr_stitch->free_lists); 2745 } 2746 memcpy(&lbr_stitch->prev_sample, sample, sizeof(*sample)); 2747 } 2748 2749 if (callee) { 2750 /* Add kernel ip */ 2751 err = lbr_callchain_add_kernel_ip(thread, cursor, sample, 2752 parent, root_al, branch_from, 2753 true, i, symbols); 2754 if (err) 2755 goto error; 2756 2757 err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent, 2758 root_al, &branch_from, true, symbols); 2759 if (err) 2760 goto error; 2761 2762 if (stitched_lbr) { 2763 err = lbr_callchain_add_stitched_lbr_ip(thread, cursor); 2764 if (err) 2765 goto error; 2766 } 2767 2768 } else { 2769 if (stitched_lbr) { 2770 err = lbr_callchain_add_stitched_lbr_ip(thread, cursor); 2771 if (err) 2772 goto error; 2773 } 2774 err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent, 2775 root_al, &branch_from, false, symbols); 2776 if (err) 2777 goto error; 2778 2779 /* Add kernel ip */ 2780 err = lbr_callchain_add_kernel_ip(thread, cursor, sample, 2781 parent, root_al, branch_from, 2782 false, i, symbols); 2783 if (err) 2784 goto error; 2785 } 2786 return 1; 2787 2788 error: 2789 return (err < 0) ? err : 0; 2790 } 2791 2792 static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread, 2793 struct callchain_cursor *cursor, 2794 struct symbol **parent, 2795 struct addr_location *root_al, 2796 u8 *cpumode, int ent, bool symbols) 2797 { 2798 int err = 0; 2799 2800 while (--ent >= 0) { 2801 u64 ip = chain->ips[ent]; 2802 2803 if (ip >= PERF_CONTEXT_MAX) { 2804 err = add_callchain_ip(thread, cursor, parent, 2805 root_al, cpumode, ip, 2806 false, NULL, NULL, 0, symbols); 2807 break; 2808 } 2809 } 2810 return err; 2811 } 2812 2813 static u64 get_leaf_frame_caller(struct perf_sample *sample, 2814 struct thread *thread, int usr_idx) 2815 { 2816 if (thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL) == EM_AARCH64) 2817 return get_leaf_frame_caller_aarch64(sample, thread, usr_idx); 2818 else 2819 return 0; 2820 } 2821 2822 static int thread__resolve_callchain_sample(struct thread *thread, 2823 struct callchain_cursor *cursor, 2824 struct perf_sample *sample, 2825 struct symbol **parent, 2826 struct addr_location *root_al, 2827 int max_stack, 2828 bool symbols) 2829 { 2830 struct evsel *evsel = sample->evsel; 2831 struct branch_stack *branch = sample->branch_stack; 2832 struct branch_entry *entries = perf_sample__branch_entries(sample); 2833 struct ip_callchain *chain = sample->callchain; 2834 int chain_nr = 0; 2835 u8 cpumode = PERF_RECORD_MISC_USER; 2836 int i, j, err, nr_entries, usr_idx; 2837 int skip_idx = -1; 2838 int first_call = 0; 2839 u64 leaf_frame_caller; 2840 2841 if (chain) 2842 chain_nr = chain->nr; 2843 2844 if (evsel__has_branch_callstack(evsel)) { 2845 struct perf_env *env = evsel__env(evsel); 2846 2847 err = resolve_lbr_callchain_sample(thread, cursor, sample, parent, 2848 root_al, max_stack, 2849 !env ? 0 : env->max_branches, 2850 symbols); 2851 if (err) 2852 return (err < 0) ? err : 0; 2853 } 2854 2855 /* 2856 * Based on DWARF debug information, some architectures skip 2857 * a callchain entry saved by the kernel. 2858 */ 2859 skip_idx = arch_skip_callchain_idx(thread, chain); 2860 2861 /* 2862 * Add branches to call stack for easier browsing. This gives 2863 * more context for a sample than just the callers. 2864 * 2865 * This uses individual histograms of paths compared to the 2866 * aggregated histograms the normal LBR mode uses. 2867 * 2868 * Limitations for now: 2869 * - No extra filters 2870 * - No annotations (should annotate somehow) 2871 */ 2872 2873 if (branch && callchain_param.branch_callstack) { 2874 int nr = min(max_stack, (int)branch->nr); 2875 struct branch_entry be[nr]; 2876 struct iterations iter[nr]; 2877 2878 if (branch->nr > PERF_MAX_BRANCH_DEPTH) { 2879 pr_warning("corrupted branch chain. skipping...\n"); 2880 goto check_calls; 2881 } 2882 2883 for (i = 0; i < nr; i++) { 2884 if (callchain_param.order == ORDER_CALLEE) { 2885 be[i] = entries[i]; 2886 2887 if (chain == NULL) 2888 continue; 2889 2890 /* 2891 * Check for overlap into the callchain. 2892 * The return address is one off compared to 2893 * the branch entry. To adjust for this 2894 * assume the calling instruction is not longer 2895 * than 8 bytes. 2896 */ 2897 if (i == skip_idx || 2898 chain->ips[first_call] >= PERF_CONTEXT_MAX) 2899 first_call++; 2900 else if (be[i].from < chain->ips[first_call] && 2901 be[i].from >= chain->ips[first_call] - 8) 2902 first_call++; 2903 } else 2904 be[i] = entries[branch->nr - i - 1]; 2905 } 2906 2907 memset(iter, 0, sizeof(struct iterations) * nr); 2908 nr = remove_loops(be, nr, iter); 2909 2910 for (i = 0; i < nr; i++) { 2911 err = add_callchain_ip(thread, cursor, parent, 2912 root_al, 2913 NULL, be[i].to, 2914 true, &be[i].flags, 2915 NULL, be[i].from, symbols); 2916 2917 if (!err) { 2918 err = add_callchain_ip(thread, cursor, parent, root_al, 2919 NULL, be[i].from, 2920 true, &be[i].flags, 2921 &iter[i], 0, symbols); 2922 } 2923 if (err == -EINVAL) 2924 break; 2925 if (err) 2926 return err; 2927 } 2928 2929 if (chain_nr == 0) 2930 return 0; 2931 2932 chain_nr -= nr; 2933 } 2934 2935 check_calls: 2936 if (chain && callchain_param.order != ORDER_CALLEE) { 2937 err = find_prev_cpumode(chain, thread, cursor, parent, root_al, 2938 &cpumode, chain->nr - first_call, symbols); 2939 if (err) 2940 return (err < 0) ? err : 0; 2941 } 2942 for (i = first_call, nr_entries = 0; 2943 i < chain_nr && nr_entries < max_stack; i++) { 2944 u64 ip; 2945 2946 if (callchain_param.order == ORDER_CALLEE) 2947 j = i; 2948 else 2949 j = chain->nr - i - 1; 2950 2951 #ifdef HAVE_SKIP_CALLCHAIN_IDX 2952 if (j == skip_idx) 2953 continue; 2954 #endif 2955 ip = chain->ips[j]; 2956 if (ip < PERF_CONTEXT_MAX) 2957 ++nr_entries; 2958 else if (callchain_param.order != ORDER_CALLEE) { 2959 err = find_prev_cpumode(chain, thread, cursor, parent, 2960 root_al, &cpumode, j, symbols); 2961 if (err) 2962 return (err < 0) ? err : 0; 2963 continue; 2964 } 2965 2966 /* 2967 * PERF_CONTEXT_USER allows us to locate where the user stack ends. 2968 * Depending on callchain_param.order and the position of PERF_CONTEXT_USER, 2969 * the index will be different in order to add the missing frame 2970 * at the right place. 2971 */ 2972 2973 usr_idx = callchain_param.order == ORDER_CALLEE ? j-2 : j-1; 2974 2975 if (usr_idx >= 0 && chain->ips[usr_idx] == PERF_CONTEXT_USER) { 2976 2977 leaf_frame_caller = get_leaf_frame_caller(sample, thread, usr_idx); 2978 2979 /* 2980 * check if leaf_frame_Caller != ip to not add the same 2981 * value twice. 2982 */ 2983 2984 if (leaf_frame_caller && leaf_frame_caller != ip) { 2985 2986 err = add_callchain_ip(thread, cursor, parent, 2987 root_al, &cpumode, leaf_frame_caller, 2988 false, NULL, NULL, 0, symbols); 2989 if (err) 2990 return (err < 0) ? err : 0; 2991 } 2992 } 2993 2994 err = add_callchain_ip(thread, cursor, parent, 2995 root_al, &cpumode, ip, 2996 false, NULL, NULL, 0, symbols); 2997 2998 if (err) 2999 return (err < 0) ? err : 0; 3000 } 3001 3002 return 0; 3003 } 3004 3005 static int unwind_entry(struct unwind_entry *entry, void *arg) 3006 { 3007 struct callchain_cursor *cursor = arg; 3008 const char *srcline = NULL; 3009 u64 addr = entry->ip; 3010 3011 if (symbol_conf.hide_unresolved && entry->ms.sym == NULL) 3012 return 0; 3013 3014 if (append_inlines(cursor, &entry->ms, entry->ip, /*branch=*/false, /*branch_flags=*/NULL, 3015 /*nr_loop_iter=*/0, /*iter_cycles=*/0, /*branch_from=*/0) == 0) 3016 return 0; 3017 3018 /* 3019 * Convert entry->ip from a virtual address to an offset in 3020 * its corresponding binary. 3021 */ 3022 if (entry->ms.map) 3023 addr = map__dso_map_ip(entry->ms.map, entry->ip); 3024 3025 srcline = callchain_srcline(&entry->ms, addr); 3026 return callchain_cursor_append(cursor, entry->ip, &entry->ms, 3027 false, NULL, 0, 0, 0, srcline); 3028 } 3029 3030 static int thread__resolve_callchain_unwind(struct thread *thread, 3031 struct callchain_cursor *cursor, 3032 struct perf_sample *sample, 3033 int max_stack, bool symbols) 3034 { 3035 struct evsel *evsel = sample->evsel; 3036 3037 /* Can we do dwarf post unwind? */ 3038 if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) && 3039 (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER))) 3040 return 0; 3041 3042 /* Bail out if nothing was captured. */ 3043 if (!sample->user_regs || !sample->user_regs->regs || 3044 !sample->user_stack.size) 3045 return 0; 3046 3047 if (!symbols) 3048 pr_debug("Not resolving symbols with an unwinder isn't currently supported\n"); 3049 3050 return unwind__get_entries(unwind_entry, cursor, 3051 thread, sample, max_stack, false); 3052 } 3053 3054 int __thread__resolve_callchain(struct thread *thread, 3055 struct callchain_cursor *cursor, 3056 struct perf_sample *sample, 3057 struct symbol **parent, 3058 struct addr_location *root_al, 3059 int max_stack, 3060 bool symbols) 3061 { 3062 int ret = 0; 3063 3064 if (cursor == NULL) 3065 return -ENOMEM; 3066 3067 callchain_cursor_reset(cursor); 3068 3069 if (callchain_param.order == ORDER_CALLEE) { 3070 ret = thread__resolve_callchain_sample(thread, cursor, 3071 sample, 3072 parent, root_al, 3073 max_stack, symbols); 3074 if (ret) 3075 return ret; 3076 ret = thread__resolve_callchain_unwind(thread, cursor, 3077 sample, 3078 max_stack, symbols); 3079 } else { 3080 ret = thread__resolve_callchain_unwind(thread, cursor, 3081 sample, 3082 max_stack, symbols); 3083 if (ret) 3084 return ret; 3085 ret = thread__resolve_callchain_sample(thread, cursor, 3086 sample, 3087 parent, root_al, 3088 max_stack, symbols); 3089 } 3090 3091 return ret; 3092 } 3093 3094 int machine__for_each_thread(struct machine *machine, 3095 int (*fn)(struct thread *thread, void *p), 3096 void *priv) 3097 { 3098 return threads__for_each_thread(&machine->threads, fn, priv); 3099 } 3100 3101 int machines__for_each_thread(struct machines *machines, 3102 int (*fn)(struct thread *thread, void *p), 3103 void *priv) 3104 { 3105 struct rb_node *nd; 3106 int rc = 0; 3107 3108 rc = machine__for_each_thread(&machines->host, fn, priv); 3109 if (rc != 0) 3110 return rc; 3111 3112 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) { 3113 struct machine *machine = rb_entry(nd, struct machine, rb_node); 3114 3115 rc = machine__for_each_thread(machine, fn, priv); 3116 if (rc != 0) 3117 return rc; 3118 } 3119 return rc; 3120 } 3121 3122 3123 static int thread_list_cb(struct thread *thread, void *data) 3124 { 3125 struct list_head *list = data; 3126 struct thread_list *entry = malloc(sizeof(*entry)); 3127 3128 if (!entry) 3129 return -ENOMEM; 3130 3131 entry->thread = thread__get(thread); 3132 list_add_tail(&entry->list, list); 3133 return 0; 3134 } 3135 3136 int machine__thread_list(struct machine *machine, struct list_head *list) 3137 { 3138 return machine__for_each_thread(machine, thread_list_cb, list); 3139 } 3140 3141 void thread_list__delete(struct list_head *list) 3142 { 3143 struct thread_list *pos, *next; 3144 3145 list_for_each_entry_safe(pos, next, list, list) { 3146 thread__zput(pos->thread); 3147 list_del(&pos->list); 3148 free(pos); 3149 } 3150 } 3151 3152 pid_t machine__get_current_tid(struct machine *machine, int cpu) 3153 { 3154 if (cpu < 0 || (size_t)cpu >= machine->current_tid_sz) 3155 return -1; 3156 3157 return machine->current_tid[cpu]; 3158 } 3159 3160 int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid, 3161 pid_t tid) 3162 { 3163 struct thread *thread; 3164 const pid_t init_val = -1; 3165 3166 if (cpu < 0) 3167 return -EINVAL; 3168 3169 if (realloc_array_as_needed(machine->current_tid, 3170 machine->current_tid_sz, 3171 (unsigned int)cpu, 3172 &init_val)) 3173 return -ENOMEM; 3174 3175 machine->current_tid[cpu] = tid; 3176 3177 thread = machine__findnew_thread(machine, pid, tid); 3178 if (!thread) 3179 return -ENOMEM; 3180 3181 thread__set_cpu(thread, cpu); 3182 thread__put(thread); 3183 3184 return 0; 3185 } 3186 3187 int machine__nr_cpus_avail(struct machine *machine) 3188 { 3189 return machine ? perf_env__nr_cpus_avail(machine->env) : 0; 3190 } 3191 3192 int machine__get_kernel_start(struct machine *machine) 3193 { 3194 struct map *map = machine__kernel_map(machine); 3195 int err = 0; 3196 3197 /* 3198 * The only addresses above 2^63 are kernel addresses of a 64-bit 3199 * kernel. Note that addresses are unsigned so that on a 32-bit system 3200 * all addresses including kernel addresses are less than 2^32. In 3201 * that case (32-bit system), if the kernel mapping is unknown, all 3202 * addresses will be assumed to be in user space - see 3203 * machine__kernel_ip(). 3204 */ 3205 machine->kernel_start = 1ULL << 63; 3206 if (map) { 3207 err = map__load(map); 3208 /* 3209 * On x86_64, PTI entry trampolines are less than the 3210 * start of kernel text, but still above 2^63. So leave 3211 * kernel_start = 1ULL << 63 for x86_64. 3212 */ 3213 if (!err && !machine__is(machine, EM_X86_64)) 3214 machine->kernel_start = map__start(map); 3215 } 3216 return err; 3217 } 3218 3219 u8 machine__addr_cpumode(struct machine *machine, u8 cpumode, u64 addr) 3220 { 3221 u8 addr_cpumode = cpumode; 3222 bool kernel_ip; 3223 3224 if (!machine->single_address_space) 3225 goto out; 3226 3227 kernel_ip = machine__kernel_ip(machine, addr); 3228 switch (cpumode) { 3229 case PERF_RECORD_MISC_KERNEL: 3230 case PERF_RECORD_MISC_USER: 3231 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_KERNEL : 3232 PERF_RECORD_MISC_USER; 3233 break; 3234 case PERF_RECORD_MISC_GUEST_KERNEL: 3235 case PERF_RECORD_MISC_GUEST_USER: 3236 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_GUEST_KERNEL : 3237 PERF_RECORD_MISC_GUEST_USER; 3238 break; 3239 default: 3240 break; 3241 } 3242 out: 3243 return addr_cpumode; 3244 } 3245 3246 struct dso *machine__findnew_dso_id(struct machine *machine, const char *filename, 3247 const struct dso_id *id) 3248 { 3249 return dsos__findnew_id(&machine->dsos, filename, id); 3250 } 3251 3252 struct dso *machine__findnew_dso(struct machine *machine, const char *filename) 3253 { 3254 return machine__findnew_dso_id(machine, filename, &dso_id_empty); 3255 } 3256 3257 char *machine__resolve_kernel_addr(void *vmachine, unsigned long long *addrp, char **modp) 3258 { 3259 struct machine *machine = vmachine; 3260 struct map *map; 3261 struct symbol *sym = machine__find_kernel_symbol(machine, *addrp, &map); 3262 3263 if (sym == NULL) 3264 return NULL; 3265 3266 *modp = __map__is_kmodule(map) ? (char *)dso__short_name(map__dso(map)) : NULL; 3267 *addrp = map__unmap_ip(map, sym->start); 3268 return sym->name; 3269 } 3270 3271 struct machine__for_each_dso_cb_args { 3272 struct machine *machine; 3273 machine__dso_t fn; 3274 void *priv; 3275 }; 3276 3277 static int machine__for_each_dso_cb(struct dso *dso, void *data) 3278 { 3279 struct machine__for_each_dso_cb_args *args = data; 3280 3281 return args->fn(dso, args->machine, args->priv); 3282 } 3283 3284 int machine__for_each_dso(struct machine *machine, machine__dso_t fn, void *priv) 3285 { 3286 struct machine__for_each_dso_cb_args args = { 3287 .machine = machine, 3288 .fn = fn, 3289 .priv = priv, 3290 }; 3291 3292 return dsos__for_each_dso(&machine->dsos, machine__for_each_dso_cb, &args); 3293 } 3294 3295 int machine__for_each_kernel_map(struct machine *machine, machine__map_t fn, void *priv) 3296 { 3297 struct maps *maps = machine__kernel_maps(machine); 3298 3299 return maps__for_each_map(maps, fn, priv); 3300 } 3301 3302 bool machine__is_lock_function(struct machine *machine, u64 addr) 3303 { 3304 if (!machine->sched.text_start) { 3305 struct map *kmap; 3306 struct symbol *sym = machine__find_kernel_symbol_by_name(machine, "__sched_text_start", &kmap); 3307 3308 if (!sym) { 3309 /* to avoid retry */ 3310 machine->sched.text_start = 1; 3311 return false; 3312 } 3313 3314 machine->sched.text_start = map__unmap_ip(kmap, sym->start); 3315 3316 /* should not fail from here */ 3317 sym = machine__find_kernel_symbol_by_name(machine, "__sched_text_end", &kmap); 3318 machine->sched.text_end = map__unmap_ip(kmap, sym->start); 3319 3320 sym = machine__find_kernel_symbol_by_name(machine, "__lock_text_start", &kmap); 3321 machine->lock.text_start = map__unmap_ip(kmap, sym->start); 3322 3323 sym = machine__find_kernel_symbol_by_name(machine, "__lock_text_end", &kmap); 3324 machine->lock.text_end = map__unmap_ip(kmap, sym->start); 3325 3326 sym = machine__find_kernel_symbol_by_name(machine, "__traceiter_contention_begin", &kmap); 3327 if (sym) { 3328 machine->traceiter.text_start = map__unmap_ip(kmap, sym->start); 3329 machine->traceiter.text_end = map__unmap_ip(kmap, sym->end); 3330 } 3331 sym = machine__find_kernel_symbol_by_name(machine, "trace_contention_begin", &kmap); 3332 if (sym) { 3333 machine->trace.text_start = map__unmap_ip(kmap, sym->start); 3334 machine->trace.text_end = map__unmap_ip(kmap, sym->end); 3335 } 3336 } 3337 3338 /* failed to get kernel symbols */ 3339 if (machine->sched.text_start == 1) 3340 return false; 3341 3342 /* mutex and rwsem functions are in sched text section */ 3343 if (machine->sched.text_start <= addr && addr < machine->sched.text_end) 3344 return true; 3345 3346 /* spinlock functions are in lock text section */ 3347 if (machine->lock.text_start <= addr && addr < machine->lock.text_end) 3348 return true; 3349 3350 /* traceiter functions currently don't have their own section 3351 * but we consider them lock functions 3352 */ 3353 if (machine->traceiter.text_start != 0) { 3354 if (machine->traceiter.text_start <= addr && addr < machine->traceiter.text_end) 3355 return true; 3356 } 3357 3358 if (machine->trace.text_start != 0) { 3359 if (machine->trace.text_start <= addr && addr < machine->trace.text_end) 3360 return true; 3361 } 3362 3363 return false; 3364 } 3365 3366 int machine__hit_all_dsos(struct machine *machine) 3367 { 3368 return dsos__hit_all(&machine->dsos); 3369 } 3370