1 // SPDX-License-Identifier: GPL-2.0 2 #include "callchain.h" 3 #include "debug.h" 4 #include "dso.h" 5 #include "env.h" 6 #include "map.h" 7 #include "perf_regs.h" 8 #include "session.h" 9 #include "symbol.h" 10 #include "thread.h" 11 #include "unwind.h" 12 #include "libunwind-arch/libunwind-arch.h" 13 #include <dwarf-regs.h> 14 #include <elf.h> 15 #include <fcntl.h> 16 #include <gelf.h> 17 #include <inttypes.h> 18 19 #define DW_EH_PE_FORMAT_MASK 0x0f /* format of the encoded value */ 20 #define DW_EH_PE_APPL_MASK 0x70 /* how the value is to be applied */ 21 22 /* Pointer-encoding formats: */ 23 #define DW_EH_PE_omit 0xff 24 #define DW_EH_PE_ptr 0x00 /* pointer-sized unsigned value */ 25 #define DW_EH_PE_udata4 0x03 /* unsigned 32-bit value */ 26 #define DW_EH_PE_udata8 0x04 /* unsigned 64-bit value */ 27 #define DW_EH_PE_sdata4 0x0b /* signed 32-bit value */ 28 #define DW_EH_PE_sdata8 0x0c /* signed 64-bit value */ 29 30 /* Pointer-encoding application: */ 31 #define DW_EH_PE_absptr 0x00 /* absolute value */ 32 #define DW_EH_PE_pcrel 0x10 /* rel. to addr. of encoded value */ 33 34 /* 35 * The following are not documented by LSB v1.3, yet they are used by 36 * GCC, presumably they aren't documented by LSB since they aren't 37 * used on Linux: 38 */ 39 #define DW_EH_PE_funcrel 0x40 /* start-of-procedure-relative */ 40 #define DW_EH_PE_aligned 0x50 /* aligned pointer */ 41 42 /* Flags intentionally not handled, since they're not needed: 43 * #define DW_EH_PE_indirect 0x80 44 * #define DW_EH_PE_uleb128 0x01 45 * #define DW_EH_PE_udata2 0x02 46 * #define DW_EH_PE_sleb128 0x09 47 * #define DW_EH_PE_sdata2 0x0a 48 * #define DW_EH_PE_textrel 0x20 49 * #define DW_EH_PE_datarel 0x30 50 */ 51 52 #define dw_read(ptr, type, end) ({ \ 53 type *__p = (type *) ptr; \ 54 type __v; \ 55 if ((__p + 1) > (type *) end) \ 56 return -EINVAL; \ 57 __v = *__p++; \ 58 ptr = (typeof(ptr)) __p; \ 59 __v; \ 60 }) 61 62 static int __dw_read_encoded_value(u8 **p, u8 *end, u64 *val, 63 u8 encoding) 64 { 65 u8 *cur = *p; 66 *val = 0; 67 68 switch (encoding) { 69 case DW_EH_PE_omit: 70 *val = 0; 71 goto out; 72 case DW_EH_PE_ptr: 73 *val = dw_read(cur, unsigned long, end); 74 goto out; 75 default: 76 break; 77 } 78 79 switch (encoding & DW_EH_PE_APPL_MASK) { 80 case DW_EH_PE_absptr: 81 break; 82 case DW_EH_PE_pcrel: 83 *val = (unsigned long) cur; 84 break; 85 default: 86 return -EINVAL; 87 } 88 89 if ((encoding & 0x07) == 0x00) 90 encoding |= DW_EH_PE_udata4; 91 92 switch (encoding & DW_EH_PE_FORMAT_MASK) { 93 case DW_EH_PE_sdata4: 94 *val += dw_read(cur, s32, end); 95 break; 96 case DW_EH_PE_udata4: 97 *val += dw_read(cur, u32, end); 98 break; 99 case DW_EH_PE_sdata8: 100 *val += dw_read(cur, s64, end); 101 break; 102 case DW_EH_PE_udata8: 103 *val += dw_read(cur, u64, end); 104 break; 105 default: 106 return -EINVAL; 107 } 108 109 out: 110 *p = cur; 111 return 0; 112 } 113 114 #define dw_read_encoded_value(ptr, end, enc) ({ \ 115 u64 __v; \ 116 if (__dw_read_encoded_value(&ptr, end, &__v, enc)) { \ 117 return -EINVAL; \ 118 } \ 119 __v; \ 120 }) 121 122 static u64 elf_base_address(int fd) 123 { 124 Elf *elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); 125 GElf_Phdr phdr; 126 u64 retval = 0; 127 size_t i, phdrnum = 0; 128 129 if (elf == NULL) 130 return 0; 131 (void)elf_getphdrnum(elf, &phdrnum); 132 /* PT_LOAD segments are sorted by p_vaddr, so the first has the minimum p_vaddr. */ 133 for (i = 0; i < phdrnum; i++) { 134 if (gelf_getphdr(elf, i, &phdr) && phdr.p_type == PT_LOAD) { 135 retval = phdr.p_vaddr & -getpagesize(); 136 break; 137 } 138 } 139 140 elf_end(elf); 141 return retval; 142 } 143 144 static int unwind_spec_ehframe(struct dso *dso, struct machine *machine, 145 u64 offset, u64 *table_data_offset, u64 *fde_count) 146 { 147 struct eh_frame_hdr { 148 unsigned char version; 149 unsigned char eh_frame_ptr_enc; 150 unsigned char fde_count_enc; 151 unsigned char table_enc; 152 153 /* 154 * The rest of the header is variable-length and consists of the 155 * following members: 156 * 157 * encoded_t eh_frame_ptr; 158 * encoded_t fde_count; 159 */ 160 161 /* A single encoded pointer should not be more than 8 bytes. */ 162 u64 enc[2]; 163 164 /* 165 * struct { 166 * encoded_t start_ip; 167 * encoded_t fde_addr; 168 * } binary_search_table[fde_count]; 169 */ 170 char data[]; 171 } __packed hdr; 172 u8 *enc = (u8 *) &hdr.enc; 173 u8 *end = (u8 *) &hdr.data; 174 ssize_t r; 175 176 r = dso__data_read_offset(dso, machine, offset, (u8 *) &hdr, sizeof(hdr)); 177 if (r != sizeof(hdr)) 178 return -EINVAL; 179 180 /* We dont need eh_frame_ptr, just skip it. */ 181 dw_read_encoded_value(enc, end, hdr.eh_frame_ptr_enc); 182 183 *fde_count = dw_read_encoded_value(enc, end, hdr.fde_count_enc); 184 *table_data_offset = enc - (u8 *) &hdr; 185 return 0; 186 } 187 188 struct read_unwind_spec_eh_frame_maps_cb_args { 189 struct dso *dso; 190 u64 base_addr; 191 }; 192 193 static int read_unwind_spec_eh_frame_maps_cb(struct map *map, void *data) 194 { 195 196 struct read_unwind_spec_eh_frame_maps_cb_args *args = data; 197 198 if (map__dso(map) == args->dso && map__start(map) - map__pgoff(map) < args->base_addr) 199 args->base_addr = map__start(map) - map__pgoff(map); 200 201 return 0; 202 } 203 204 static int elf_section_address_and_offset(int fd, const char *name, u64 *address, u64 *offset) 205 { 206 Elf *elf; 207 GElf_Ehdr ehdr; 208 GElf_Shdr shdr; 209 int ret = -1; 210 211 elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); 212 if (elf == NULL) 213 return -1; 214 215 if (gelf_getehdr(elf, &ehdr) == NULL) 216 goto out_err; 217 218 if (!elf_section_by_name(elf, &ehdr, &shdr, name, NULL)) 219 goto out_err; 220 221 *address = shdr.sh_addr; 222 *offset = shdr.sh_offset; 223 ret = 0; 224 out_err: 225 elf_end(elf); 226 return ret; 227 } 228 229 static int read_unwind_spec_eh_frame(struct dso *dso, struct unwind_info *ui, 230 u64 *table_data, u64 *segbase, 231 u64 *fde_count) 232 { 233 struct read_unwind_spec_eh_frame_maps_cb_args args = { 234 .dso = dso, 235 .base_addr = UINT64_MAX, 236 }; 237 int ret, fd; 238 239 if (dso__data(dso)->eh_frame_hdr_offset == 0) { 240 if (!dso__data_get_fd(dso, ui->machine, &fd)) 241 return -EINVAL; 242 243 /* Check the .eh_frame section for unwinding info */ 244 ret = elf_section_address_and_offset(fd, ".eh_frame_hdr", 245 &dso__data(dso)->eh_frame_hdr_addr, 246 &dso__data(dso)->eh_frame_hdr_offset); 247 dso__data(dso)->elf_base_addr = elf_base_address(fd); 248 dso__data_put_fd(dso); 249 if (ret || dso__data(dso)->eh_frame_hdr_offset == 0) 250 return -EINVAL; 251 } 252 253 maps__for_each_map(thread__maps(ui->thread), read_unwind_spec_eh_frame_maps_cb, &args); 254 255 args.base_addr -= dso__data(dso)->elf_base_addr; 256 /* Address of .eh_frame_hdr */ 257 *segbase = args.base_addr + dso__data(dso)->eh_frame_hdr_addr; 258 ret = unwind_spec_ehframe(dso, ui->machine, dso__data(dso)->eh_frame_hdr_offset, 259 table_data, fde_count); 260 if (ret) 261 return ret; 262 /* binary_search_table offset plus .eh_frame_hdr address */ 263 *table_data += *segbase; 264 return 0; 265 } 266 267 static u64 elf_section_offset(int fd, const char *name) 268 { 269 u64 address, offset = 0; 270 271 if (elf_section_address_and_offset(fd, name, &address, &offset)) 272 return 0; 273 274 return offset; 275 } 276 277 static int read_unwind_spec_debug_frame(struct dso *dso, 278 struct machine *machine, u64 *offset) 279 { 280 int fd; 281 u64 ofs = dso__data(dso)->debug_frame_offset; 282 283 /* debug_frame can reside in: 284 * - dso 285 * - debug pointed by symsrc_filename 286 * - gnu_debuglink, which doesn't necessary 287 * has to be pointed by symsrc_filename 288 */ 289 if (ofs == 0) { 290 if (dso__data_get_fd(dso, machine, &fd)) { 291 ofs = elf_section_offset(fd, ".debug_frame"); 292 dso__data_put_fd(dso); 293 } 294 295 if (ofs <= 0) { 296 fd = open(dso__symsrc_filename(dso), O_RDONLY); 297 if (fd >= 0) { 298 ofs = elf_section_offset(fd, ".debug_frame"); 299 close(fd); 300 } 301 } 302 303 if (ofs <= 0) { 304 char *debuglink = malloc(PATH_MAX); 305 int ret = 0; 306 307 if (debuglink == NULL) { 308 pr_err("unwind: Can't read unwind spec debug frame.\n"); 309 return -ENOMEM; 310 } 311 312 ret = dso__read_binary_type_filename( 313 dso, DSO_BINARY_TYPE__DEBUGLINK, 314 machine->root_dir, debuglink, PATH_MAX); 315 if (!ret) { 316 fd = open(debuglink, O_RDONLY); 317 if (fd >= 0) { 318 ofs = elf_section_offset(fd, 319 ".debug_frame"); 320 close(fd); 321 } 322 } 323 if (ofs > 0) { 324 if (dso__symsrc_filename(dso) != NULL) { 325 pr_warning( 326 "%s: overwrite symsrc(%s,%s)\n", 327 __func__, 328 dso__symsrc_filename(dso), 329 debuglink); 330 dso__free_symsrc_filename(dso); 331 } 332 dso__set_symsrc_filename(dso, debuglink); 333 } else { 334 free(debuglink); 335 } 336 } 337 338 dso__data(dso)->debug_frame_offset = ofs; 339 } 340 341 *offset = ofs; 342 if (*offset) 343 return 0; 344 345 return -EINVAL; 346 } 347 348 static struct map *find_map(uint64_t ip, struct unwind_info *ui) 349 { 350 struct addr_location al; 351 struct map *ret; 352 353 addr_location__init(&al); 354 thread__find_map(ui->thread, PERF_RECORD_MISC_USER, ip, &al); 355 ret = map__get(al.map); 356 addr_location__exit(&al); 357 return ret; 358 } 359 360 static int elf_is_exec(int fd, const char *name) 361 { 362 Elf *elf; 363 GElf_Ehdr ehdr; 364 int retval = 0; 365 366 elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); 367 if (elf == NULL) 368 return 0; 369 if (gelf_getehdr(elf, &ehdr) == NULL) 370 goto out; 371 372 retval = (ehdr.e_type == ET_EXEC); 373 374 out: 375 elf_end(elf); 376 pr_debug3("unwind: elf_is_exec(%s): %d\n", name, retval); 377 return retval; 378 } 379 380 int __libunwind__find_proc_info(void *as, uint64_t ip, void *pi, int need_unwind_info, void *arg) 381 { 382 struct unwind_info *ui = arg; 383 struct map *map; 384 struct dso *dso; 385 u64 table_data, segbase, fde_count; 386 int ret = -EINVAL; 387 388 map = find_map(ip, ui); 389 if (!map) 390 return -EINVAL; 391 392 dso = map__dso(map); 393 if (!dso) { 394 map__put(map); 395 return -EINVAL; 396 } 397 398 pr_debug3("unwind: find_proc_info dso %s\n", dso__name(dso)); 399 400 /* Check the .eh_frame section for unwinding info */ 401 if (!read_unwind_spec_eh_frame(dso, ui, &table_data, &segbase, &fde_count)) { 402 struct table_entry { 403 u32 start_ip_offset; 404 u32 fde_offset; 405 }; 406 struct libarch_unwind__dyn_info di = { 407 .start_ip = map__start(map), 408 .end_ip = map__end(map), 409 .segbase = segbase, 410 .table_data = table_data, 411 .table_len = fde_count * sizeof(struct table_entry) / ui->unw_word_t_size, 412 }; 413 414 ret = libunwind_arch__dwarf_search_unwind_table(ui->e_machine, as, ip, &di, pi, 415 need_unwind_info, arg); 416 } 417 418 /* Check the .debug_frame section for unwinding info */ 419 if (ret < 0 && !read_unwind_spec_debug_frame(dso, ui->machine, &segbase)) { 420 int fd; 421 u64 start = map__start(map); 422 u64 base = start; 423 const char *symfile; 424 struct libarch_unwind__dyn_info di = {}; 425 426 if (dso__data_get_fd(dso, ui->machine, &fd)) { 427 if (elf_is_exec(fd, dso__name(dso))) 428 base = 0; 429 dso__data_put_fd(dso); 430 } 431 432 symfile = dso__symsrc_filename(dso) ?: dso__name(dso); 433 434 if (libunwind_arch__dwarf_find_debug_frame(ui->e_machine, /*found=*/0, &di, ip, 435 base, symfile, start, map__end(map))) { 436 ret = libunwind_arch__dwarf_search_unwind_table(ui->e_machine, as, ip, &di, pi, 437 need_unwind_info, arg); 438 } 439 } 440 map__put(map); 441 return ret; 442 } 443 444 static int access_dso_mem(struct unwind_info *ui, uint64_t addr, void *data_word) 445 { 446 struct map *map; 447 struct dso *dso; 448 ssize_t size; 449 450 map = find_map(addr, ui); 451 if (!map) { 452 pr_debug("unwind: no map for %lx\n", (unsigned long)addr); 453 return -1; 454 } 455 456 dso = map__dso(map); 457 458 if (!dso) { 459 map__put(map); 460 return -1; 461 } 462 463 size = dso__data_read_addr(dso, map, ui->machine, 464 addr, 465 (u8 *) data_word, 466 ui->unw_word_t_size); 467 map__put(map); 468 return !((size_t)size == ui->unw_word_t_size); 469 } 470 471 int __libunwind__access_mem(void *as __maybe_unused, uint64_t addr, void *valp_word, 472 int __write, void *arg) 473 { 474 struct unwind_info *ui = arg; 475 struct stack_dump *stack = &ui->sample->user_stack; 476 u64 start, end; 477 int offset; 478 int ret; 479 480 /* Don't support write, probably not needed. */ 481 if (__write || !stack || !ui->sample->user_regs || !ui->sample->user_regs->regs) { 482 uint64_t zero = 0; 483 484 memcpy(valp_word, &zero, ui->unw_word_t_size); 485 return 0; 486 } 487 488 ret = perf_reg_value(&start, perf_sample__user_regs(ui->sample), 489 perf_arch_reg_sp(ui->e_machine)); 490 if (ret) 491 return ret; 492 493 end = start + stack->size; 494 495 /* Check overflow. */ 496 if (addr + ui->unw_word_t_size < addr) 497 return -EINVAL; 498 499 if (addr < start || addr + ui->unw_word_t_size >= end) { 500 ret = access_dso_mem(ui, addr, valp_word); 501 if (ret) { 502 pr_debug3("unwind: access_mem %p not inside range" 503 " 0x%" PRIx64 "-0x%" PRIx64 "\n", 504 (void *) (uintptr_t) addr, start, end); 505 memset(valp_word, 0, ui->unw_word_t_size); 506 return ret; 507 } 508 return 0; 509 } 510 511 offset = addr - start; 512 memcpy(valp_word, &stack->data[offset], ui->unw_word_t_size); 513 pr_debug3("unwind: access_mem addr %p val %lx, offset %d\n", 514 (void *) (uintptr_t) addr, *((unsigned long *)valp_word), offset); 515 return 0; 516 } 517 518 int __libunwind__access_reg(void *as __maybe_unused, int regnum, void *valp_word, int __write, 519 void *arg) 520 { 521 struct unwind_info *ui = arg; 522 int id, ret; 523 u64 val; 524 525 /* Don't support write, I suspect we don't need it. */ 526 if (__write) { 527 pr_err("unwind: access_reg w %d\n", regnum); 528 return 0; 529 } 530 531 if (!ui->sample->user_regs || !ui->sample->user_regs->regs) { 532 memset(valp_word, 0, ui->unw_word_t_size); 533 return 0; 534 } 535 536 id = get_perf_regnum_for_unw_regnum(ui->e_machine, regnum); 537 if (id < 0) 538 return -EINVAL; 539 540 ret = perf_reg_value(&val, perf_sample__user_regs(ui->sample), id); 541 if (ret) { 542 if (!ui->best_effort) 543 pr_err("unwind: can't read reg %d\n", regnum); 544 return ret; 545 } 546 547 if (ui->unw_word_t_size == 8) 548 *(uint64_t *)valp_word = val; 549 else 550 *(uint32_t *)valp_word = (uint32_t)val; 551 pr_debug3("unwind: reg %d, val %lx\n", regnum, val); 552 return 0; 553 } 554 555 int unwind__prepare_access(struct maps *maps, uint16_t e_machine) 556 { 557 void *addr_space; 558 559 if (!dwarf_callchain_users) 560 return 0; 561 562 if (maps__addr_space(maps)) { 563 pr_debug3("unwind: thread map already set\n"); 564 return 0; 565 } 566 567 if (e_machine == EM_NONE) 568 return 0; 569 570 maps__set_e_machine(maps, e_machine); 571 addr_space = libunwind_arch__create_addr_space(e_machine); 572 573 maps__set_addr_space(maps, addr_space); 574 if (!addr_space) { 575 pr_err("unwind: Can't create unwind address space.\n"); 576 return -ENOMEM; 577 } 578 579 return 0; 580 } 581 582 void unwind__flush_access(struct maps *maps) 583 { 584 libunwind_arch__flush_access(maps); 585 } 586 587 void unwind__finish_access(struct maps *maps) 588 { 589 libunwind_arch__finish_access(maps); 590 } 591 592 static int entry(uint64_t ip, struct thread *thread, unwind_entry_cb_t cb, void *arg) 593 { 594 struct unwind_entry e; 595 struct addr_location al; 596 int ret; 597 598 addr_location__init(&al); 599 e.ms.sym = thread__find_symbol(thread, PERF_RECORD_MISC_USER, ip, &al); 600 e.ip = ip; 601 e.ms.map = al.map; 602 e.ms.thread = thread__get(al.thread); 603 604 pr_debug("unwind: %s:ip = 0x%" PRIx64 " (0x%" PRIx64 ")\n", 605 al.sym ? al.sym->name : "''", 606 ip, 607 al.map ? map__map_ip(al.map, ip) : (u64) 0); 608 609 ret = cb(&e, arg); 610 addr_location__exit(&al); 611 return ret; 612 } 613 614 int libunwind__get_entries(unwind_entry_cb_t cb, void *arg, 615 struct thread *thread, 616 struct perf_sample *sample, int max_stack, 617 bool best_effort) 618 { 619 struct unwind_info *ui; 620 uint64_t first_ip; 621 int ret, i = 0, entries = 0; 622 uint16_t e_machine; 623 624 if (!sample->user_regs || !sample->user_regs->regs) 625 return 0; 626 627 if (max_stack <= 0) 628 return 0; 629 630 if (!thread) { 631 pr_warning_once("WARNING: thread is NULL"); 632 return 0; 633 } 634 635 e_machine = thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL); 636 ret = perf_reg_value(&first_ip, perf_sample__user_regs(sample), 637 perf_arch_reg_ip(e_machine)); 638 if (ret) 639 return 0; 640 641 if (max_stack == 1) { 642 /* Special case for a single entry. */ 643 ret = entry(first_ip, thread, cb, arg); 644 return ret ? (ret == -ENOMEM ? -ENOMEM : 0) : 1; 645 } 646 647 ui = libunwind_arch_unwind_info__new(thread, sample, max_stack, best_effort, e_machine, first_ip); 648 if (!ui) 649 return -ENOMEM; 650 651 do { 652 ret = libunwind_arch__unwind_step(ui); 653 if (ret < 0) 654 goto out; 655 656 } while (ret); 657 658 /* 659 * Display what we got based on the order setup. 660 */ 661 for (i = 0; i < ui->cur_ip; i++) { 662 int j = callchain_param.order == ORDER_CALLEE ? i : ui->cur_ip - i - 1; 663 664 if (ui->ips[j]) { 665 ret = entry(ui->ips[j], thread, cb, arg); 666 if (ret) 667 break; 668 entries++; 669 } 670 } 671 out: 672 libunwind_arch_unwind_info__delete(ui); 673 /* 674 * Unwinder return contract: 675 * > 0 : unwinding succeeded (stops fallback). 676 * 0 : unwinding failed without yielding frames. Ignore non-fatal errors 677 * (e.g. stepping failure) to allow fallback unwinder or kernel callchains. 678 * < 0 : fatal error (e.g. -ENOMEM). Aborts unwinding entirely. 679 */ 680 if (ret == -ENOMEM) 681 return -ENOMEM; 682 return (entries > 0 || ret == 0) ? entries : 0; 683 } 684