1 // SPDX-License-Identifier: GPL-2.0 2 #include <asm/bug.h> 3 #include <linux/kernel.h> 4 #include <linux/string.h> 5 #include <linux/zalloc.h> 6 #include <sys/time.h> 7 #include <sys/resource.h> 8 #include <sys/types.h> 9 #include <sys/stat.h> 10 #include <unistd.h> 11 #include <errno.h> 12 #include <fcntl.h> 13 #include <stdlib.h> 14 #ifdef HAVE_LIBBPF_SUPPORT 15 #include <bpf/libbpf.h> 16 #include "bpf-event.h" 17 #include "bpf-utils.h" 18 #endif 19 #include "compress.h" 20 #include "env.h" 21 #include "namespaces.h" 22 #include "path.h" 23 #include "map.h" 24 #include "symbol.h" 25 #include "srcline.h" 26 #include "dso.h" 27 #include "dsos.h" 28 #include "machine.h" 29 #include "auxtrace.h" 30 #include "util.h" /* O_CLOEXEC for older systems */ 31 #include "debug.h" 32 #include "string2.h" 33 #include "vdso.h" 34 #include "annotate-data.h" 35 #include "libdw.h" 36 37 static const char * const debuglink_paths[] = { 38 "%.0s%s", 39 "%s/%s", 40 "%s/.debug/%s", 41 "/usr/lib/debug%s/%s" 42 }; 43 44 void dso__set_nsinfo(struct dso *dso, struct nsinfo *nsi) 45 { 46 nsinfo__put(RC_CHK_ACCESS(dso)->nsinfo); 47 RC_CHK_ACCESS(dso)->nsinfo = nsi; 48 } 49 50 char dso__symtab_origin(const struct dso *dso) 51 { 52 static const char origin[] = { 53 [DSO_BINARY_TYPE__KALLSYMS] = 'k', 54 [DSO_BINARY_TYPE__VMLINUX] = 'v', 55 [DSO_BINARY_TYPE__JAVA_JIT] = 'j', 56 [DSO_BINARY_TYPE__DEBUGLINK] = 'l', 57 [DSO_BINARY_TYPE__BUILD_ID_CACHE] = 'B', 58 [DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO] = 'D', 59 [DSO_BINARY_TYPE__FEDORA_DEBUGINFO] = 'f', 60 [DSO_BINARY_TYPE__UBUNTU_DEBUGINFO] = 'u', 61 [DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO] = 'x', 62 [DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO] = 'o', 63 [DSO_BINARY_TYPE__BUILDID_DEBUGINFO] = 'b', 64 [DSO_BINARY_TYPE__SYSTEM_PATH_DSO] = 'd', 65 [DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE] = 'K', 66 [DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP] = 'm', 67 [DSO_BINARY_TYPE__GUEST_KALLSYMS] = 'g', 68 [DSO_BINARY_TYPE__GUEST_KMODULE] = 'G', 69 [DSO_BINARY_TYPE__GUEST_KMODULE_COMP] = 'M', 70 [DSO_BINARY_TYPE__GUEST_VMLINUX] = 'V', 71 [DSO_BINARY_TYPE__GNU_DEBUGDATA] = 'n', 72 }; 73 74 if (dso == NULL || dso__symtab_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) 75 return '!'; 76 return origin[dso__symtab_type(dso)]; 77 } 78 79 bool dso__is_object_file(const struct dso *dso) 80 { 81 switch (dso__binary_type(dso)) { 82 case DSO_BINARY_TYPE__KALLSYMS: 83 case DSO_BINARY_TYPE__GUEST_KALLSYMS: 84 case DSO_BINARY_TYPE__JAVA_JIT: 85 case DSO_BINARY_TYPE__BPF_PROG_INFO: 86 case DSO_BINARY_TYPE__BPF_IMAGE: 87 case DSO_BINARY_TYPE__OOL: 88 return false; 89 case DSO_BINARY_TYPE__VMLINUX: 90 case DSO_BINARY_TYPE__GUEST_VMLINUX: 91 case DSO_BINARY_TYPE__DEBUGLINK: 92 case DSO_BINARY_TYPE__BUILD_ID_CACHE: 93 case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: 94 case DSO_BINARY_TYPE__FEDORA_DEBUGINFO: 95 case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: 96 case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: 97 case DSO_BINARY_TYPE__BUILDID_DEBUGINFO: 98 case DSO_BINARY_TYPE__GNU_DEBUGDATA: 99 case DSO_BINARY_TYPE__SYSTEM_PATH_DSO: 100 case DSO_BINARY_TYPE__GUEST_KMODULE: 101 case DSO_BINARY_TYPE__GUEST_KMODULE_COMP: 102 case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: 103 case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: 104 case DSO_BINARY_TYPE__KCORE: 105 case DSO_BINARY_TYPE__GUEST_KCORE: 106 case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: 107 case DSO_BINARY_TYPE__NOT_FOUND: 108 default: 109 return true; 110 } 111 } 112 113 int dso__read_binary_type_filename(const struct dso *dso, 114 enum dso_binary_type type, 115 const char *root_dir, char *filename, size_t size) 116 { 117 char build_id_hex[SBUILD_ID_SIZE]; 118 int ret = 0; 119 size_t len; 120 121 switch (type) { 122 case DSO_BINARY_TYPE__DEBUGLINK: 123 { 124 const char *last_slash; 125 char dso_dir[PATH_MAX]; 126 char symfile[PATH_MAX]; 127 unsigned int i; 128 129 len = __symbol__join_symfs(filename, size, dso__long_name(dso)); 130 last_slash = filename + len; 131 while (last_slash != filename && *last_slash != '/') 132 last_slash--; 133 134 strncpy(dso_dir, filename, last_slash - filename); 135 dso_dir[last_slash-filename] = '\0'; 136 137 if (!is_regular_file(filename)) { 138 ret = -1; 139 break; 140 } 141 142 ret = filename__read_debuglink(filename, symfile, PATH_MAX); 143 if (ret) 144 break; 145 146 /* Check predefined locations where debug file might reside */ 147 ret = -1; 148 for (i = 0; i < ARRAY_SIZE(debuglink_paths); i++) { 149 snprintf(filename, size, 150 debuglink_paths[i], dso_dir, symfile); 151 if (is_regular_file(filename)) { 152 ret = 0; 153 break; 154 } 155 } 156 157 break; 158 } 159 case DSO_BINARY_TYPE__BUILD_ID_CACHE: 160 if (dso__build_id_filename(dso, filename, size, false) == NULL) 161 ret = -1; 162 break; 163 164 case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: 165 if (dso__build_id_filename(dso, filename, size, true) == NULL) 166 ret = -1; 167 break; 168 169 case DSO_BINARY_TYPE__FEDORA_DEBUGINFO: 170 len = __symbol__join_symfs(filename, size, "/usr/lib/debug"); 171 snprintf(filename + len, size - len, "%s.debug", dso__long_name(dso)); 172 break; 173 174 case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: 175 len = __symbol__join_symfs(filename, size, "/usr/lib/debug"); 176 snprintf(filename + len, size - len, "%s", dso__long_name(dso)); 177 break; 178 179 case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: 180 /* 181 * Ubuntu can mixup /usr/lib with /lib, putting debuginfo in 182 * /usr/lib/debug/lib when it is expected to be in 183 * /usr/lib/debug/usr/lib 184 */ 185 if (strlen(dso__long_name(dso)) < 9 || 186 strncmp(dso__long_name(dso), "/usr/lib/", 9)) { 187 ret = -1; 188 break; 189 } 190 len = __symbol__join_symfs(filename, size, "/usr/lib/debug"); 191 snprintf(filename + len, size - len, "%s", dso__long_name(dso) + 4); 192 break; 193 194 case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: 195 { 196 const char *last_slash; 197 size_t dir_size; 198 199 last_slash = dso__long_name(dso) + dso__long_name_len(dso); 200 while (last_slash != dso__long_name(dso) && *last_slash != '/') 201 last_slash--; 202 203 len = __symbol__join_symfs(filename, size, ""); 204 dir_size = last_slash - dso__long_name(dso) + 2; 205 if (dir_size > (size - len)) { 206 ret = -1; 207 break; 208 } 209 len += scnprintf(filename + len, dir_size, "%s", dso__long_name(dso)); 210 len += scnprintf(filename + len , size - len, ".debug%s", 211 last_slash); 212 break; 213 } 214 215 case DSO_BINARY_TYPE__BUILDID_DEBUGINFO: 216 if (!dso__has_build_id(dso)) { 217 ret = -1; 218 break; 219 } 220 221 build_id__snprintf(dso__bid(dso), build_id_hex, sizeof(build_id_hex)); 222 len = __symbol__join_symfs(filename, size, "/usr/lib/debug/.build-id/"); 223 snprintf(filename + len, size - len, "%.2s/%s.debug", 224 build_id_hex, build_id_hex + 2); 225 break; 226 227 case DSO_BINARY_TYPE__VMLINUX: 228 case DSO_BINARY_TYPE__GUEST_VMLINUX: 229 case DSO_BINARY_TYPE__SYSTEM_PATH_DSO: 230 case DSO_BINARY_TYPE__GNU_DEBUGDATA: 231 __symbol__join_symfs(filename, size, dso__long_name(dso)); 232 break; 233 234 case DSO_BINARY_TYPE__GUEST_KMODULE: 235 case DSO_BINARY_TYPE__GUEST_KMODULE_COMP: 236 path__join3(filename, size, symbol_conf.symfs, 237 root_dir, dso__long_name(dso)); 238 break; 239 240 case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: 241 case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: 242 __symbol__join_symfs(filename, size, dso__long_name(dso)); 243 break; 244 245 case DSO_BINARY_TYPE__KCORE: 246 case DSO_BINARY_TYPE__GUEST_KCORE: 247 snprintf(filename, size, "%s", dso__long_name(dso)); 248 break; 249 250 default: 251 case DSO_BINARY_TYPE__KALLSYMS: 252 case DSO_BINARY_TYPE__GUEST_KALLSYMS: 253 case DSO_BINARY_TYPE__JAVA_JIT: 254 case DSO_BINARY_TYPE__BPF_PROG_INFO: 255 case DSO_BINARY_TYPE__BPF_IMAGE: 256 case DSO_BINARY_TYPE__OOL: 257 case DSO_BINARY_TYPE__NOT_FOUND: 258 ret = -1; 259 break; 260 } 261 262 return ret; 263 } 264 265 enum { 266 COMP_ID__NONE = 0, 267 }; 268 269 static const struct { 270 const char *fmt; 271 int (*decompress)(const char *input, int output); 272 bool (*is_compressed)(const char *input); 273 } compressions[] = { 274 [COMP_ID__NONE] = { .fmt = NULL, }, 275 #ifdef HAVE_ZLIB_SUPPORT 276 { "gz", gzip_decompress_to_file, gzip_is_compressed }, 277 #endif 278 #ifdef HAVE_LZMA_SUPPORT 279 { "xz", lzma_decompress_to_file, lzma_is_compressed }, 280 #endif 281 { NULL, NULL, NULL }, 282 }; 283 284 static int is_supported_compression(const char *ext) 285 { 286 unsigned i; 287 288 for (i = 1; compressions[i].fmt; i++) { 289 if (!strcmp(ext, compressions[i].fmt)) 290 return i; 291 } 292 return COMP_ID__NONE; 293 } 294 295 bool is_kernel_module(const char *pathname, int cpumode) 296 { 297 struct kmod_path m; 298 int mode = cpumode & PERF_RECORD_MISC_CPUMODE_MASK; 299 300 WARN_ONCE(mode != cpumode, 301 "Internal error: passing unmasked cpumode (%x) to is_kernel_module", 302 cpumode); 303 304 switch (mode) { 305 case PERF_RECORD_MISC_USER: 306 case PERF_RECORD_MISC_HYPERVISOR: 307 case PERF_RECORD_MISC_GUEST_USER: 308 return false; 309 /* Treat PERF_RECORD_MISC_CPUMODE_UNKNOWN as kernel */ 310 default: 311 if (kmod_path__parse(&m, pathname)) { 312 pr_err("Failed to check whether %s is a kernel module or not. Assume it is.", 313 pathname); 314 return true; 315 } 316 } 317 318 return m.kmod; 319 } 320 321 bool dso__needs_decompress(struct dso *dso) 322 { 323 return dso__symtab_type(dso) == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP || 324 dso__symtab_type(dso) == DSO_BINARY_TYPE__GUEST_KMODULE_COMP; 325 } 326 327 int filename__decompress(const char *name, char *pathname, 328 size_t len, int comp, int *err) 329 { 330 char tmpbuf[] = KMOD_DECOMP_NAME; 331 int fd = -1; 332 333 /* 334 * We have proper compression id for DSO and yet the file 335 * behind the 'name' can still be plain uncompressed object. 336 * 337 * The reason is behind the logic we open the DSO object files, 338 * when we try all possible 'debug' objects until we find the 339 * data. So even if the DSO is represented by 'krava.xz' module, 340 * we can end up here opening ~/.debug/....23432432/debug' file 341 * which is not compressed. 342 * 343 * To keep this transparent, we detect this and return the file 344 * descriptor to the uncompressed file. 345 */ 346 if (!compressions[comp].is_compressed(name)) { 347 if (pathname && len > 0) 348 pathname[0] = '\0'; 349 return open(name, O_RDONLY | O_CLOEXEC); 350 } 351 352 fd = mkostemp(tmpbuf, O_CLOEXEC); 353 if (fd < 0) { 354 *err = errno; 355 return -1; 356 } 357 358 if (compressions[comp].decompress(name, fd)) { 359 *err = DSO_LOAD_ERRNO__DECOMPRESSION_FAILURE; 360 close(fd); 361 fd = -1; 362 } 363 364 if (!pathname || (fd < 0)) 365 unlink(tmpbuf); 366 367 if (pathname && (fd >= 0)) 368 strlcpy(pathname, tmpbuf, len); 369 370 return fd; 371 } 372 373 static int decompress_kmodule(struct dso *dso, const char *name, 374 char *pathname, size_t len) 375 { 376 if (!dso__needs_decompress(dso)) 377 return -1; 378 379 if (dso__comp(dso) == COMP_ID__NONE) 380 return -1; 381 382 return filename__decompress(name, pathname, len, dso__comp(dso), dso__load_errno(dso)); 383 } 384 385 int dso__decompress_kmodule_fd(struct dso *dso, const char *name) 386 { 387 return decompress_kmodule(dso, name, NULL, 0); 388 } 389 390 int dso__decompress_kmodule_path(struct dso *dso, const char *name, 391 char *pathname, size_t len) 392 { 393 int fd = decompress_kmodule(dso, name, pathname, len); 394 395 close(fd); 396 return fd >= 0 ? 0 : -1; 397 } 398 399 /* 400 * Parses kernel module specified in @path and updates 401 * @m argument like: 402 * 403 * @comp - true if @path contains supported compression suffix, 404 * false otherwise 405 * @kmod - true if @path contains '.ko' suffix in right position, 406 * false otherwise 407 * @name - if (@alloc_name && @kmod) is true, it contains strdup-ed base name 408 * of the kernel module without suffixes, otherwise strudup-ed 409 * base name of @path 410 * @ext - if (@alloc_ext && @comp) is true, it contains strdup-ed string 411 * the compression suffix 412 * 413 * Returns 0 if there's no strdup error, -ENOMEM otherwise. 414 */ 415 int __kmod_path__parse(struct kmod_path *m, const char *path, 416 bool alloc_name) 417 { 418 const char *name = strrchr(path, '/'); 419 const char *ext = strrchr(path, '.'); 420 bool is_simple_name = false; 421 422 memset(m, 0x0, sizeof(*m)); 423 name = name ? name + 1 : path; 424 425 /* 426 * '.' is also a valid character for module name. For example: 427 * [aaa.bbb] is a valid module name. '[' should have higher 428 * priority than '.ko' suffix. 429 * 430 * The kernel names are from machine__mmap_name. Such 431 * name should belong to kernel itself, not kernel module. 432 */ 433 if (name[0] == '[') { 434 is_simple_name = true; 435 if ((strncmp(name, "[kernel.kallsyms]", 17) == 0) || 436 (strncmp(name, "[guest.kernel.kallsyms", 22) == 0) || 437 (strncmp(name, "[vdso]", 6) == 0) || 438 (strncmp(name, "[vdso32]", 8) == 0) || 439 (strncmp(name, "[vdsox32]", 9) == 0) || 440 (strncmp(name, "[vsyscall]", 10) == 0)) { 441 m->kmod = false; 442 443 } else 444 m->kmod = true; 445 } 446 447 /* No extension, just return name. */ 448 if ((ext == NULL) || is_simple_name) { 449 if (alloc_name) { 450 m->name = strdup(name); 451 return m->name ? 0 : -ENOMEM; 452 } 453 return 0; 454 } 455 456 m->comp = is_supported_compression(ext + 1); 457 if (m->comp > COMP_ID__NONE) 458 ext -= 3; 459 460 /* Check .ko extension only if there's enough name left. */ 461 if (ext > name) 462 m->kmod = !strncmp(ext, ".ko", 3); 463 464 if (alloc_name) { 465 if (m->kmod) { 466 if (asprintf(&m->name, "[%.*s]", (int) (ext - name), name) == -1) 467 return -ENOMEM; 468 } else { 469 if (asprintf(&m->name, "%s", name) == -1) 470 return -ENOMEM; 471 } 472 473 strreplace(m->name, '-', '_'); 474 } 475 476 return 0; 477 } 478 479 void dso__set_module_info(struct dso *dso, struct kmod_path *m, 480 struct machine *machine) 481 { 482 if (machine__is_host(machine)) 483 dso__set_symtab_type(dso, DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE); 484 else 485 dso__set_symtab_type(dso, DSO_BINARY_TYPE__GUEST_KMODULE); 486 487 /* _KMODULE_COMP should be next to _KMODULE */ 488 if (m->kmod && m->comp) { 489 dso__set_symtab_type(dso, dso__symtab_type(dso) + 1); 490 dso__set_comp(dso, m->comp); 491 } 492 493 dso__set_is_kmod(dso); 494 dso__set_short_name(dso, strdup(m->name), true); 495 } 496 497 /* 498 * Global list of open DSOs and the counter. 499 */ 500 struct mutex _dso__data_open_lock; 501 static LIST_HEAD(dso__data_open); 502 static long dso__data_open_cnt GUARDED_BY(_dso__data_open_lock); 503 504 static void dso__data_open_lock_init(void) 505 { 506 mutex_init(&_dso__data_open_lock); 507 } 508 509 static struct mutex *dso__data_open_lock(void) LOCK_RETURNED(_dso__data_open_lock) 510 { 511 static pthread_once_t data_open_lock_once = PTHREAD_ONCE_INIT; 512 513 pthread_once(&data_open_lock_once, dso__data_open_lock_init); 514 515 return &_dso__data_open_lock; 516 } 517 518 static void dso__list_add(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 519 { 520 list_add_tail(&dso__data(dso)->open_entry, &dso__data_open); 521 #ifdef REFCNT_CHECKING 522 dso__data(dso)->dso = dso__get(dso); 523 #endif 524 /* Assume the dso is part of dsos, hence the optional reference count above. */ 525 assert(dso__dsos(dso)); 526 dso__data_open_cnt++; 527 } 528 529 static void dso__list_del(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 530 { 531 list_del_init(&dso__data(dso)->open_entry); 532 #ifdef REFCNT_CHECKING 533 mutex_unlock(dso__data_open_lock()); 534 dso__put(dso__data(dso)->dso); 535 mutex_lock(dso__data_open_lock()); 536 #endif 537 WARN_ONCE(dso__data_open_cnt <= 0, 538 "DSO data fd counter out of bounds."); 539 dso__data_open_cnt--; 540 } 541 542 static void close_first_dso(void); 543 544 static int do_open(char *name) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 545 { 546 do { 547 int fd = open(name, O_RDONLY|O_CLOEXEC); 548 549 if (fd >= 0) 550 return fd; 551 552 pr_debug("dso open failed: %m\n"); 553 if (!dso__data_open_cnt || errno != EMFILE) 554 break; 555 556 close_first_dso(); 557 } while (1); 558 559 return -1; 560 } 561 562 char *dso__filename_with_chroot(const struct dso *dso, const char *filename) 563 { 564 return filename_with_chroot(nsinfo__pid(dso__nsinfo_const(dso)), filename); 565 } 566 567 static char *dso__get_filename(struct dso *dso, const char *root_dir, 568 bool *decomp) 569 { 570 char *name = malloc(PATH_MAX); 571 572 *decomp = false; 573 574 if (name == NULL) 575 return NULL; 576 577 if (dso__read_binary_type_filename(dso, dso__binary_type(dso), 578 root_dir, name, PATH_MAX)) 579 goto out; 580 581 if (!is_regular_file(name)) { 582 char *new_name; 583 584 if (errno != ENOENT || dso__nsinfo(dso) == NULL) 585 goto out; 586 587 new_name = dso__filename_with_chroot(dso, name); 588 if (!new_name) 589 goto out; 590 591 free(name); 592 name = new_name; 593 } 594 595 if (dso__needs_decompress(dso)) { 596 char newpath[KMOD_DECOMP_LEN]; 597 size_t len = sizeof(newpath); 598 599 if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) { 600 errno = *dso__load_errno(dso); 601 goto out; 602 } 603 604 /* empty pathname means file wasn't actually compressed */ 605 if (newpath[0] != '\0') { 606 *decomp = true; 607 strcpy(name, newpath); 608 } 609 } 610 return name; 611 612 out: 613 free(name); 614 return NULL; 615 } 616 617 static int __open_dso(struct dso *dso, struct machine *machine) 618 EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 619 { 620 int fd = -EINVAL; 621 char *name; 622 bool decomp = false; 623 624 mutex_lock(dso__lock(dso)); 625 626 name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp); 627 if (name) 628 fd = do_open(name); 629 else 630 fd = -errno; 631 632 if (decomp) 633 unlink(name); 634 635 mutex_unlock(dso__lock(dso)); 636 free(name); 637 return fd; 638 } 639 640 static void check_data_close(void); 641 642 /** 643 * dso_close - Open DSO data file 644 * @dso: dso object 645 * 646 * Open @dso's data file descriptor and updates 647 * list/count of open DSO objects. 648 */ 649 static int open_dso(struct dso *dso, struct machine *machine) 650 EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 651 { 652 int fd; 653 struct nscookie nsc; 654 655 if (dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE) { 656 mutex_lock(dso__lock(dso)); 657 nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); 658 mutex_unlock(dso__lock(dso)); 659 } 660 fd = __open_dso(dso, machine); 661 if (dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE) 662 nsinfo__mountns_exit(&nsc); 663 664 if (fd >= 0) { 665 dso__list_add(dso); 666 /* 667 * Check if we crossed the allowed number 668 * of opened DSOs and close one if needed. 669 */ 670 check_data_close(); 671 } 672 673 return fd; 674 } 675 676 static void close_data_fd(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 677 { 678 if (dso__data(dso)->fd >= 0) { 679 close(dso__data(dso)->fd); 680 dso__data(dso)->fd = -1; 681 dso__data(dso)->file_size = 0; 682 dso__list_del(dso); 683 } 684 } 685 686 /** 687 * dso_close - Close DSO data file 688 * @dso: dso object 689 * 690 * Close @dso's data file descriptor and updates 691 * list/count of open DSO objects. 692 */ 693 static void close_dso(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 694 { 695 close_data_fd(dso); 696 } 697 698 static void close_first_dso(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 699 { 700 struct dso_data *dso_data; 701 struct dso *dso; 702 703 dso_data = list_first_entry(&dso__data_open, struct dso_data, open_entry); 704 #ifdef REFCNT_CHECKING 705 dso = dso_data->dso; 706 #else 707 dso = container_of(dso_data, struct dso, data); 708 #endif 709 close_dso(dso); 710 } 711 712 static rlim_t get_fd_limit(void) 713 { 714 struct rlimit l; 715 rlim_t limit = 0; 716 717 /* Allow half of the current open fd limit. */ 718 if (getrlimit(RLIMIT_NOFILE, &l) == 0) { 719 if (l.rlim_cur == RLIM_INFINITY) 720 limit = l.rlim_cur; 721 else 722 limit = l.rlim_cur / 2; 723 } else { 724 pr_err("failed to get fd limit\n"); 725 limit = 1; 726 } 727 728 return limit; 729 } 730 731 static rlim_t fd_limit; 732 733 /* 734 * Used only by tests/dso-data.c to reset the environment 735 * for tests. I dont expect we should change this during 736 * standard runtime. 737 */ 738 void reset_fd_limit(void) 739 { 740 fd_limit = 0; 741 } 742 743 static bool may_cache_fd(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 744 { 745 if (!fd_limit) 746 fd_limit = get_fd_limit(); 747 748 if (fd_limit == RLIM_INFINITY) 749 return true; 750 751 return fd_limit > (rlim_t) dso__data_open_cnt; 752 } 753 754 /* 755 * Check and close LRU dso if we crossed allowed limit 756 * for opened dso file descriptors. The limit is half 757 * of the RLIMIT_NOFILE files opened. 758 */ 759 static void check_data_close(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 760 { 761 bool cache_fd = may_cache_fd(); 762 763 if (!cache_fd) 764 close_first_dso(); 765 } 766 767 /** 768 * dso__data_close - Close DSO data file 769 * @dso: dso object 770 * 771 * External interface to close @dso's data file descriptor. 772 */ 773 void dso__data_close(struct dso *dso) 774 { 775 mutex_lock(dso__data_open_lock()); 776 close_dso(dso); 777 mutex_unlock(dso__data_open_lock()); 778 } 779 780 static void try_to_open_dso(struct dso *dso, struct machine *machine) 781 EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) 782 { 783 enum dso_binary_type binary_type_data[] = { 784 DSO_BINARY_TYPE__BUILD_ID_CACHE, 785 DSO_BINARY_TYPE__SYSTEM_PATH_DSO, 786 DSO_BINARY_TYPE__NOT_FOUND, 787 }; 788 int i = 0; 789 struct dso_data *dso_data = dso__data(dso); 790 791 if (dso_data->fd >= 0) 792 return; 793 794 if (dso__binary_type(dso) != DSO_BINARY_TYPE__NOT_FOUND) { 795 dso_data->fd = open_dso(dso, machine); 796 goto out; 797 } 798 799 do { 800 dso__set_binary_type(dso, binary_type_data[i++]); 801 802 dso_data->fd = open_dso(dso, machine); 803 if (dso_data->fd >= 0) 804 goto out; 805 806 } while (dso__binary_type(dso) != DSO_BINARY_TYPE__NOT_FOUND); 807 out: 808 if (dso_data->fd >= 0) 809 dso_data->status = DSO_DATA_STATUS_OK; 810 else 811 dso_data->status = DSO_DATA_STATUS_ERROR; 812 } 813 814 /** 815 * dso__data_get_fd - Get dso's data file descriptor 816 * @dso: dso object 817 * @machine: machine object 818 * 819 * External interface to find dso's file, open it and 820 * returns file descriptor. It should be paired with 821 * dso__data_put_fd() if it returns non-negative value. 822 */ 823 bool dso__data_get_fd(struct dso *dso, struct machine *machine, int *fd) 824 { 825 *fd = -1; 826 if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) 827 return false; 828 829 mutex_lock(dso__data_open_lock()); 830 831 try_to_open_dso(dso, machine); 832 833 *fd = dso__data(dso)->fd; 834 if (*fd >= 0) 835 return true; 836 837 mutex_unlock(dso__data_open_lock()); 838 return false; 839 } 840 841 void dso__data_put_fd(struct dso *dso __maybe_unused) 842 { 843 mutex_unlock(dso__data_open_lock()); 844 } 845 846 bool dso__data_status_seen(struct dso *dso, enum dso_data_status_seen by) 847 { 848 u32 flag = 1 << by; 849 850 if (dso__data(dso)->status_seen & flag) 851 return true; 852 853 dso__data(dso)->status_seen |= flag; 854 855 return false; 856 } 857 858 #ifdef HAVE_LIBBPF_SUPPORT 859 static ssize_t bpf_read(struct dso *dso, u64 offset, char *data) 860 { 861 struct bpf_prog_info_node *node; 862 ssize_t size = DSO__DATA_CACHE_SIZE; 863 struct dso_bpf_prog *dso_bpf_prog = dso__bpf_prog(dso); 864 u64 len; 865 u8 *buf; 866 867 node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id); 868 if (!node || !node->info_linear) { 869 dso__data(dso)->status = DSO_DATA_STATUS_ERROR; 870 return -1; 871 } 872 873 len = node->info_linear->info.jited_prog_len; 874 buf = (u8 *)(uintptr_t)node->info_linear->info.jited_prog_insns; 875 876 if (offset >= len) 877 return -1; 878 879 size = (ssize_t)min(len - offset, (u64)size); 880 memcpy(data, buf + offset, size); 881 return size; 882 } 883 884 static int bpf_size(struct dso *dso) 885 { 886 struct bpf_prog_info_node *node; 887 struct dso_bpf_prog *dso_bpf_prog = dso__bpf_prog(dso); 888 889 node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id); 890 if (!node || !node->info_linear) { 891 dso__data(dso)->status = DSO_DATA_STATUS_ERROR; 892 return -1; 893 } 894 895 dso__data(dso)->file_size = node->info_linear->info.jited_prog_len; 896 return 0; 897 } 898 #endif // HAVE_LIBBPF_SUPPORT 899 900 static void 901 dso_cache__free(struct dso *dso) 902 { 903 struct rb_root *root = &dso__data(dso)->cache; 904 struct rb_node *next = rb_first(root); 905 906 mutex_lock(dso__lock(dso)); 907 while (next) { 908 struct dso_cache *cache; 909 910 cache = rb_entry(next, struct dso_cache, rb_node); 911 next = rb_next(&cache->rb_node); 912 rb_erase(&cache->rb_node, root); 913 free(cache); 914 } 915 mutex_unlock(dso__lock(dso)); 916 } 917 918 static struct dso_cache *__dso_cache__find(struct dso *dso, u64 offset) 919 { 920 const struct rb_root *root = &dso__data(dso)->cache; 921 struct rb_node * const *p = &root->rb_node; 922 const struct rb_node *parent = NULL; 923 struct dso_cache *cache; 924 925 while (*p != NULL) { 926 u64 end; 927 928 parent = *p; 929 cache = rb_entry(parent, struct dso_cache, rb_node); 930 end = cache->offset + DSO__DATA_CACHE_SIZE; 931 932 if (offset < cache->offset) 933 p = &(*p)->rb_left; 934 else if (offset >= end) 935 p = &(*p)->rb_right; 936 else 937 return cache; 938 } 939 940 return NULL; 941 } 942 943 static struct dso_cache * 944 dso_cache__insert(struct dso *dso, struct dso_cache *new) 945 { 946 struct rb_root *root = &dso__data(dso)->cache; 947 struct rb_node **p = &root->rb_node; 948 struct rb_node *parent = NULL; 949 struct dso_cache *cache; 950 u64 offset = new->offset; 951 952 mutex_lock(dso__lock(dso)); 953 while (*p != NULL) { 954 u64 end; 955 956 parent = *p; 957 cache = rb_entry(parent, struct dso_cache, rb_node); 958 end = cache->offset + DSO__DATA_CACHE_SIZE; 959 960 if (offset < cache->offset) 961 p = &(*p)->rb_left; 962 else if (offset >= end) 963 p = &(*p)->rb_right; 964 else 965 goto out; 966 } 967 968 rb_link_node(&new->rb_node, parent, p); 969 rb_insert_color(&new->rb_node, root); 970 971 cache = NULL; 972 out: 973 mutex_unlock(dso__lock(dso)); 974 return cache; 975 } 976 977 static ssize_t dso_cache__memcpy(struct dso_cache *cache, u64 offset, u8 *data, 978 u64 size, bool out) 979 { 980 u64 cache_offset = offset - cache->offset; 981 u64 cache_size = min(cache->size - cache_offset, size); 982 983 if (out) 984 memcpy(data, cache->data + cache_offset, cache_size); 985 else 986 memcpy(cache->data + cache_offset, data, cache_size); 987 return cache_size; 988 } 989 990 static ssize_t file_read(struct dso *dso, struct machine *machine, 991 u64 offset, char *data) 992 { 993 ssize_t ret; 994 995 mutex_lock(dso__data_open_lock()); 996 997 /* 998 * dso__data(dso)->fd might be closed if other thread opened another 999 * file (dso) due to open file limit (RLIMIT_NOFILE). 1000 */ 1001 try_to_open_dso(dso, machine); 1002 1003 if (dso__data(dso)->fd < 0) { 1004 dso__data(dso)->status = DSO_DATA_STATUS_ERROR; 1005 ret = -errno; 1006 goto out; 1007 } 1008 1009 ret = pread(dso__data(dso)->fd, data, DSO__DATA_CACHE_SIZE, offset); 1010 out: 1011 mutex_unlock(dso__data_open_lock()); 1012 return ret; 1013 } 1014 1015 static struct dso_cache *dso_cache__populate(struct dso *dso, 1016 struct machine *machine, 1017 u64 offset, ssize_t *ret) 1018 { 1019 u64 cache_offset = offset & DSO__DATA_CACHE_MASK; 1020 struct dso_cache *cache; 1021 struct dso_cache *old; 1022 1023 cache = zalloc(sizeof(*cache) + DSO__DATA_CACHE_SIZE); 1024 if (!cache) { 1025 *ret = -ENOMEM; 1026 return NULL; 1027 } 1028 #ifdef HAVE_LIBBPF_SUPPORT 1029 if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) 1030 *ret = bpf_read(dso, cache_offset, cache->data); 1031 else 1032 #endif 1033 if (dso__binary_type(dso) == DSO_BINARY_TYPE__OOL) 1034 *ret = DSO__DATA_CACHE_SIZE; 1035 else 1036 *ret = file_read(dso, machine, cache_offset, cache->data); 1037 1038 if (*ret <= 0) { 1039 free(cache); 1040 return NULL; 1041 } 1042 1043 cache->offset = cache_offset; 1044 cache->size = *ret; 1045 1046 old = dso_cache__insert(dso, cache); 1047 if (old) { 1048 /* we lose the race */ 1049 free(cache); 1050 cache = old; 1051 } 1052 1053 return cache; 1054 } 1055 1056 static struct dso_cache *dso_cache__find(struct dso *dso, 1057 struct machine *machine, 1058 u64 offset, 1059 ssize_t *ret) 1060 { 1061 struct dso_cache *cache = __dso_cache__find(dso, offset); 1062 1063 return cache ? cache : dso_cache__populate(dso, machine, offset, ret); 1064 } 1065 1066 static ssize_t dso_cache_io(struct dso *dso, struct machine *machine, 1067 u64 offset, u8 *data, ssize_t size, bool out) 1068 { 1069 struct dso_cache *cache; 1070 ssize_t ret = 0; 1071 1072 cache = dso_cache__find(dso, machine, offset, &ret); 1073 if (!cache) 1074 return ret; 1075 1076 return dso_cache__memcpy(cache, offset, data, size, out); 1077 } 1078 1079 /* 1080 * Reads and caches dso data DSO__DATA_CACHE_SIZE size chunks 1081 * in the rb_tree. Any read to already cached data is served 1082 * by cached data. Writes update the cache only, not the backing file. 1083 */ 1084 static ssize_t cached_io(struct dso *dso, struct machine *machine, 1085 u64 offset, u8 *data, ssize_t size, bool out) 1086 { 1087 ssize_t r = 0; 1088 u8 *p = data; 1089 1090 do { 1091 ssize_t ret; 1092 1093 ret = dso_cache_io(dso, machine, offset, p, size, out); 1094 if (ret < 0) 1095 return ret; 1096 1097 /* Reached EOF, return what we have. */ 1098 if (!ret) 1099 break; 1100 1101 BUG_ON(ret > size); 1102 1103 r += ret; 1104 p += ret; 1105 offset += ret; 1106 size -= ret; 1107 1108 } while (size); 1109 1110 return r; 1111 } 1112 1113 static int file_size(struct dso *dso, struct machine *machine) 1114 { 1115 int ret = 0; 1116 struct stat st; 1117 1118 mutex_lock(dso__data_open_lock()); 1119 1120 /* 1121 * dso__data(dso)->fd might be closed if other thread opened another 1122 * file (dso) due to open file limit (RLIMIT_NOFILE). 1123 */ 1124 try_to_open_dso(dso, machine); 1125 1126 if (dso__data(dso)->fd < 0) { 1127 ret = -errno; 1128 dso__data(dso)->status = DSO_DATA_STATUS_ERROR; 1129 goto out; 1130 } 1131 1132 if (fstat(dso__data(dso)->fd, &st) < 0) { 1133 ret = -errno; 1134 pr_err("dso cache fstat failed: %m\n"); 1135 dso__data(dso)->status = DSO_DATA_STATUS_ERROR; 1136 goto out; 1137 } 1138 dso__data(dso)->file_size = st.st_size; 1139 1140 out: 1141 mutex_unlock(dso__data_open_lock()); 1142 return ret; 1143 } 1144 1145 int dso__data_file_size(struct dso *dso, struct machine *machine) 1146 { 1147 if (dso__data(dso)->file_size) 1148 return 0; 1149 1150 if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) 1151 return -1; 1152 #ifdef HAVE_LIBBPF_SUPPORT 1153 if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) 1154 return bpf_size(dso); 1155 #endif 1156 return file_size(dso, machine); 1157 } 1158 1159 /** 1160 * dso__data_size - Return dso data size 1161 * @dso: dso object 1162 * @machine: machine object 1163 * 1164 * Return: dso data size 1165 */ 1166 off_t dso__data_size(struct dso *dso, struct machine *machine) 1167 { 1168 if (dso__data_file_size(dso, machine)) 1169 return -1; 1170 1171 /* For now just estimate dso data size is close to file size */ 1172 return dso__data(dso)->file_size; 1173 } 1174 1175 static ssize_t data_read_write_offset(struct dso *dso, struct machine *machine, 1176 u64 offset, u8 *data, ssize_t size, 1177 bool out) 1178 { 1179 if (dso__data_file_size(dso, machine)) 1180 return -1; 1181 1182 /* Check the offset sanity. */ 1183 if (offset > dso__data(dso)->file_size) 1184 return -1; 1185 1186 if (offset + size < offset) 1187 return -1; 1188 1189 return cached_io(dso, machine, offset, data, size, out); 1190 } 1191 1192 /** 1193 * dso__data_read_offset - Read data from dso file offset 1194 * @dso: dso object 1195 * @machine: machine object 1196 * @offset: file offset 1197 * @data: buffer to store data 1198 * @size: size of the @data buffer 1199 * 1200 * External interface to read data from dso file offset. Open 1201 * dso data file and use cached_read to get the data. 1202 */ 1203 ssize_t dso__data_read_offset(struct dso *dso, struct machine *machine, 1204 u64 offset, u8 *data, ssize_t size) 1205 { 1206 if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) 1207 return -1; 1208 1209 return data_read_write_offset(dso, machine, offset, data, size, true); 1210 } 1211 1212 static enum dso_swap_type dso_swap_type__from_elf_data(unsigned char eidata) 1213 { 1214 static const unsigned int endian = 1; 1215 1216 switch (eidata) { 1217 case ELFDATA2LSB: 1218 /* We are big endian, DSO is little endian. */ 1219 return (*(unsigned char const *)&endian != 1) ? DSO_SWAP__YES : DSO_SWAP__NO; 1220 case ELFDATA2MSB: 1221 /* We are little endian, DSO is big endian. */ 1222 return (*(unsigned char const *)&endian != 0) ? DSO_SWAP__YES : DSO_SWAP__NO; 1223 default: 1224 return DSO_SWAP__UNSET; 1225 } 1226 } 1227 1228 /* Reads e_machine from fd, optionally caching data in dso. */ 1229 uint16_t dso__read_e_machine_endian(struct dso *optional_dso, int fd, uint32_t *e_flags, 1230 bool *is_big_endian) 1231 { 1232 uint16_t e_machine = EM_NONE; 1233 unsigned char e_ident[EI_NIDENT]; 1234 enum dso_swap_type swap_type; 1235 bool need_e_flags; 1236 1237 if (e_flags) 1238 *e_flags = 0; 1239 1240 { 1241 _Static_assert(offsetof(Elf32_Ehdr, e_ident) == 0, "Unexpected offset"); 1242 _Static_assert(offsetof(Elf64_Ehdr, e_ident) == 0, "Unexpected offset"); 1243 } 1244 if (pread(fd, &e_ident, sizeof(e_ident), 0) != sizeof(e_ident)) 1245 return EM_NONE; // Read failed. 1246 1247 if (memcmp(e_ident, ELFMAG, SELFMAG) != 0) 1248 return EM_NONE; // Not an ELF file. 1249 1250 if (e_ident[EI_CLASS] == ELFCLASSNONE || e_ident[EI_CLASS] >= ELFCLASSNUM) 1251 return EM_NONE; // Bad ELF class (32 or 64-bit objects). 1252 1253 if (e_ident[EI_VERSION] != EV_CURRENT) 1254 return EM_NONE; // Bad ELF version. 1255 1256 swap_type = dso_swap_type__from_elf_data(e_ident[EI_DATA]); 1257 if (swap_type == DSO_SWAP__UNSET) 1258 return EM_NONE; // Bad ELF data encoding. 1259 1260 if (is_big_endian) 1261 *is_big_endian = (e_ident[EI_DATA] == ELFDATA2MSB); 1262 1263 /* Cache the need for swapping. */ 1264 if (optional_dso) { 1265 assert(dso__needs_swap(optional_dso) == DSO_SWAP__UNSET || 1266 dso__needs_swap(optional_dso) == swap_type); 1267 dso__set_needs_swap(optional_dso, swap_type); 1268 } 1269 1270 { 1271 _Static_assert(offsetof(Elf32_Ehdr, e_machine) == 18, "Unexpected offset"); 1272 _Static_assert(offsetof(Elf64_Ehdr, e_machine) == 18, "Unexpected offset"); 1273 } 1274 if (pread(fd, &e_machine, sizeof(e_machine), 18) != sizeof(e_machine)) 1275 return EM_NONE; // e_machine read failed. 1276 1277 e_machine = DSO_SWAP_TYPE__SWAP(swap_type, uint16_t, e_machine); 1278 if (e_machine >= EM_NUM) 1279 return EM_NONE; // Bad ELF machine number. 1280 1281 #ifdef NDEBUG 1282 /* In production code the e_flags are only needed on CSKY. */ 1283 need_e_flags = e_flags && e_machine == EM_CSKY; 1284 #else 1285 /* Debug code will always read the e_flags. */ 1286 need_e_flags = e_flags != NULL; 1287 #endif 1288 if (need_e_flags) { 1289 off_t offset = e_ident[EI_CLASS] == ELFCLASS32 1290 ? offsetof(Elf32_Ehdr, e_flags) 1291 : offsetof(Elf64_Ehdr, e_flags); 1292 1293 if (pread(fd, e_flags, sizeof(*e_flags), offset) != sizeof(*e_flags)) { 1294 *e_flags = 0; 1295 return EM_NONE; // e_flags read failed. 1296 } 1297 } 1298 return e_machine; 1299 } 1300 1301 uint16_t dso__e_machine_endian(struct dso *dso, struct machine *machine, uint32_t *e_flags, 1302 bool *is_big_endian) 1303 { 1304 uint16_t e_machine = EM_NONE; 1305 int fd; 1306 1307 switch (dso__binary_type(dso)) { 1308 case DSO_BINARY_TYPE__KALLSYMS: 1309 case DSO_BINARY_TYPE__GUEST_KALLSYMS: 1310 case DSO_BINARY_TYPE__VMLINUX: 1311 case DSO_BINARY_TYPE__GUEST_VMLINUX: 1312 case DSO_BINARY_TYPE__GUEST_KMODULE: 1313 case DSO_BINARY_TYPE__GUEST_KMODULE_COMP: 1314 case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: 1315 case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: 1316 case DSO_BINARY_TYPE__KCORE: 1317 case DSO_BINARY_TYPE__GUEST_KCORE: 1318 case DSO_BINARY_TYPE__BPF_PROG_INFO: 1319 case DSO_BINARY_TYPE__BPF_IMAGE: 1320 case DSO_BINARY_TYPE__OOL: 1321 case DSO_BINARY_TYPE__JAVA_JIT: 1322 if (is_big_endian) { 1323 *is_big_endian = perf_arch_is_big_endian( 1324 machine && machine->env ? perf_env__arch(machine->env) : NULL); 1325 } 1326 return perf_env__e_machine(machine ? machine->env : NULL, e_flags); 1327 case DSO_BINARY_TYPE__DEBUGLINK: 1328 case DSO_BINARY_TYPE__BUILD_ID_CACHE: 1329 case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: 1330 case DSO_BINARY_TYPE__GNU_DEBUGDATA: 1331 case DSO_BINARY_TYPE__SYSTEM_PATH_DSO: 1332 case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: 1333 case DSO_BINARY_TYPE__FEDORA_DEBUGINFO: 1334 case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: 1335 case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: 1336 case DSO_BINARY_TYPE__BUILDID_DEBUGINFO: 1337 break; 1338 case DSO_BINARY_TYPE__NOT_FOUND: 1339 default: 1340 if (e_flags) 1341 *e_flags = 0; 1342 return EM_NONE; 1343 } 1344 1345 mutex_lock(dso__data_open_lock()); 1346 1347 /* 1348 * dso__data(dso)->fd might be closed if other thread opened another 1349 * file (dso) due to open file limit (RLIMIT_NOFILE). 1350 */ 1351 try_to_open_dso(dso, machine); 1352 fd = dso__data(dso)->fd; 1353 if (fd >= 0) 1354 e_machine = dso__read_e_machine_endian(dso, fd, e_flags, is_big_endian); 1355 else if (e_flags) 1356 *e_flags = 0; 1357 1358 mutex_unlock(dso__data_open_lock()); 1359 return e_machine; 1360 } 1361 1362 /** 1363 * dso__data_read_addr - Read data from dso address 1364 * @dso: dso object 1365 * @machine: machine object 1366 * @add: virtual memory address 1367 * @data: buffer to store data 1368 * @size: size of the @data buffer 1369 * 1370 * External interface to read data from dso address. 1371 */ 1372 ssize_t dso__data_read_addr(struct dso *dso, struct map *map, 1373 struct machine *machine, u64 addr, 1374 u8 *data, ssize_t size) 1375 { 1376 u64 offset = map__map_ip(map, addr); 1377 1378 return dso__data_read_offset(dso, machine, offset, data, size); 1379 } 1380 1381 /** 1382 * dso__data_write_cache_offs - Write data to dso data cache at file offset 1383 * @dso: dso object 1384 * @machine: machine object 1385 * @offset: file offset 1386 * @data: buffer to write 1387 * @size: size of the @data buffer 1388 * 1389 * Write into the dso file data cache, but do not change the file itself. 1390 */ 1391 ssize_t dso__data_write_cache_offs(struct dso *dso, struct machine *machine, 1392 u64 offset, const u8 *data_in, ssize_t size) 1393 { 1394 u8 *data = (u8 *)data_in; /* cast away const to use same fns for r/w */ 1395 1396 if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) 1397 return -1; 1398 1399 return data_read_write_offset(dso, machine, offset, data, size, false); 1400 } 1401 1402 /** 1403 * dso__data_write_cache_addr - Write data to dso data cache at dso address 1404 * @dso: dso object 1405 * @machine: machine object 1406 * @add: virtual memory address 1407 * @data: buffer to write 1408 * @size: size of the @data buffer 1409 * 1410 * External interface to write into the dso file data cache, but do not change 1411 * the file itself. 1412 */ 1413 ssize_t dso__data_write_cache_addr(struct dso *dso, struct map *map, 1414 struct machine *machine, u64 addr, 1415 const u8 *data, ssize_t size) 1416 { 1417 u64 offset = map__map_ip(map, addr); 1418 1419 return dso__data_write_cache_offs(dso, machine, offset, data, size); 1420 } 1421 1422 struct map *dso__new_map(const char *name) 1423 { 1424 struct map *map = NULL; 1425 struct dso *dso = dso__new(name); 1426 1427 if (dso) { 1428 map = map__new2(0, dso); 1429 dso__put(dso); 1430 } 1431 1432 return map; 1433 } 1434 1435 struct dso *machine__findnew_kernel(struct machine *machine, const char *name, 1436 const char *short_name, int dso_type) 1437 { 1438 /* 1439 * The kernel dso could be created by build_id processing. 1440 */ 1441 struct dso *dso = machine__findnew_dso(machine, name); 1442 1443 /* 1444 * We need to run this in all cases, since during the build_id 1445 * processing we had no idea this was the kernel dso. 1446 */ 1447 if (dso != NULL) { 1448 dso__set_short_name(dso, short_name, false); 1449 dso__set_kernel(dso, dso_type); 1450 } 1451 1452 return dso; 1453 } 1454 1455 static void __dso__set_long_name_id(struct dso *dso, const char *name, bool name_allocated) 1456 { 1457 if (dso__long_name_allocated(dso)) 1458 free((char *)dso__long_name(dso)); 1459 1460 RC_CHK_ACCESS(dso)->long_name = name; 1461 RC_CHK_ACCESS(dso)->long_name_len = strlen(name); 1462 dso__set_long_name_allocated(dso, name_allocated); 1463 } 1464 1465 static void dso__set_long_name_id(struct dso *dso, const char *name, bool name_allocated) 1466 { 1467 struct dsos *dsos = dso__dsos(dso); 1468 1469 if (name == NULL) 1470 return; 1471 1472 if (dsos) { 1473 /* 1474 * Need to avoid re-sorting the dsos breaking by non-atomically 1475 * renaming the dso. 1476 */ 1477 down_write(&dsos->lock); 1478 __dso__set_long_name_id(dso, name, name_allocated); 1479 dsos->sorted = false; 1480 up_write(&dsos->lock); 1481 } else { 1482 __dso__set_long_name_id(dso, name, name_allocated); 1483 } 1484 } 1485 1486 static int __dso_id__cmp(const struct dso_id *a, const struct dso_id *b) 1487 { 1488 if (a->mmap2_valid && b->mmap2_valid) { 1489 if (a->maj > b->maj) return -1; 1490 if (a->maj < b->maj) return 1; 1491 1492 if (a->min > b->min) return -1; 1493 if (a->min < b->min) return 1; 1494 1495 if (a->ino > b->ino) return -1; 1496 if (a->ino < b->ino) return 1; 1497 } 1498 if (a->mmap2_ino_generation_valid && b->mmap2_ino_generation_valid) { 1499 if (a->ino_generation > b->ino_generation) return -1; 1500 if (a->ino_generation < b->ino_generation) return 1; 1501 } 1502 if (build_id__is_defined(&a->build_id) && build_id__is_defined(&b->build_id)) { 1503 if (a->build_id.size != b->build_id.size) 1504 return a->build_id.size < b->build_id.size ? -1 : 1; 1505 return memcmp(a->build_id.data, b->build_id.data, a->build_id.size); 1506 } 1507 return 0; 1508 } 1509 1510 const struct dso_id dso_id_empty = { 1511 { 1512 .maj = 0, 1513 .min = 0, 1514 .ino = 0, 1515 .ino_generation = 0, 1516 }, 1517 .mmap2_valid = false, 1518 .mmap2_ino_generation_valid = false, 1519 { 1520 .size = 0, 1521 } 1522 }; 1523 1524 void __dso__improve_id(struct dso *dso, const struct dso_id *id) 1525 { 1526 struct dsos *dsos = dso__dsos(dso); 1527 struct dso_id *dso_id = dso__id(dso); 1528 bool changed = false; 1529 1530 /* dsos write lock held by caller. */ 1531 1532 if (id->mmap2_valid && !dso_id->mmap2_valid) { 1533 dso_id->maj = id->maj; 1534 dso_id->min = id->min; 1535 dso_id->ino = id->ino; 1536 dso_id->mmap2_valid = true; 1537 changed = true; 1538 } 1539 if (id->mmap2_ino_generation_valid && !dso_id->mmap2_ino_generation_valid) { 1540 dso_id->ino_generation = id->ino_generation; 1541 dso_id->mmap2_ino_generation_valid = true; 1542 changed = true; 1543 } 1544 if (build_id__is_defined(&id->build_id) && !build_id__is_defined(&dso_id->build_id)) { 1545 dso_id->build_id = id->build_id; 1546 changed = true; 1547 } 1548 if (changed && dsos) 1549 dsos->sorted = false; 1550 } 1551 1552 int dso_id__cmp(const struct dso_id *a, const struct dso_id *b) 1553 { 1554 if (a == &dso_id_empty || b == &dso_id_empty) { 1555 /* There is no valid data to compare so the comparison always returns identical. */ 1556 return 0; 1557 } 1558 1559 return __dso_id__cmp(a, b); 1560 } 1561 1562 int dso__cmp_id(struct dso *a, struct dso *b) 1563 { 1564 return __dso_id__cmp(dso__id(a), dso__id(b)); 1565 } 1566 1567 void dso__set_long_name(struct dso *dso, const char *name, bool name_allocated) 1568 { 1569 dso__set_long_name_id(dso, name, name_allocated); 1570 } 1571 1572 static void __dso__set_short_name(struct dso *dso, const char *name, bool name_allocated) 1573 { 1574 if (dso__short_name_allocated(dso)) 1575 free((char *)dso__short_name(dso)); 1576 1577 RC_CHK_ACCESS(dso)->short_name = name; 1578 RC_CHK_ACCESS(dso)->short_name_len = strlen(name); 1579 dso__set_short_name_allocated(dso, name_allocated); 1580 } 1581 1582 void dso__set_short_name(struct dso *dso, const char *name, bool name_allocated) 1583 { 1584 struct dsos *dsos = dso__dsos(dso); 1585 1586 if (name == NULL) 1587 return; 1588 1589 if (dsos) { 1590 /* 1591 * Need to avoid re-sorting the dsos breaking by non-atomically 1592 * renaming the dso. 1593 */ 1594 down_write(&dsos->lock); 1595 __dso__set_short_name(dso, name, name_allocated); 1596 dsos->sorted = false; 1597 up_write(&dsos->lock); 1598 } else { 1599 __dso__set_short_name(dso, name, name_allocated); 1600 } 1601 } 1602 1603 int dso__name_len(const struct dso *dso) 1604 { 1605 if (!dso) 1606 return strlen("[unknown]"); 1607 if (verbose > 0) 1608 return dso__long_name_len(dso); 1609 1610 return dso__short_name_len(dso); 1611 } 1612 1613 bool dso__loaded(const struct dso *dso) 1614 { 1615 return RC_CHK_ACCESS(dso)->loaded; 1616 } 1617 1618 bool dso__sorted_by_name(const struct dso *dso) 1619 { 1620 return RC_CHK_ACCESS(dso)->sorted_by_name; 1621 } 1622 1623 void dso__set_sorted_by_name(struct dso *dso) 1624 { 1625 RC_CHK_ACCESS(dso)->sorted_by_name = true; 1626 } 1627 1628 struct dso *dso__new_id(const char *name, const struct dso_id *id) 1629 { 1630 RC_STRUCT(dso) *dso = zalloc(sizeof(*dso) + strlen(name) + 1); 1631 struct dso *res; 1632 struct dso_data *data; 1633 1634 if (!dso) 1635 return NULL; 1636 1637 if (ADD_RC_CHK(res, dso)) { 1638 strcpy(dso->name, name); 1639 if (id) 1640 dso->id = *id; 1641 dso__set_long_name_id(res, dso->name, false); 1642 dso__set_short_name(res, dso->name, false); 1643 dso->symbols = RB_ROOT_CACHED; 1644 dso->symbol_names = NULL; 1645 dso->symbol_names_len = 0; 1646 dso->inlined_nodes = RB_ROOT_CACHED; 1647 dso->srclines = RB_ROOT_CACHED; 1648 dso->data_types = RB_ROOT; 1649 dso->global_vars = RB_ROOT; 1650 dso->data.fd = -1; 1651 dso->data.status = DSO_DATA_STATUS_UNKNOWN; 1652 dso->symtab_type = DSO_BINARY_TYPE__NOT_FOUND; 1653 dso->binary_type = DSO_BINARY_TYPE__NOT_FOUND; 1654 dso->is_64_bit = (sizeof(void *) == 8); 1655 dso->loaded = 0; 1656 dso->rel = 0; 1657 dso->sorted_by_name = 0; 1658 dso->has_srcline = 1; 1659 dso->a2l_fails = 1; 1660 dso->kernel = DSO_SPACE__USER; 1661 dso->is_kmod = 0; 1662 dso->needs_swap = DSO_SWAP__UNSET; 1663 dso->comp = COMP_ID__NONE; 1664 mutex_init(&dso->lock); 1665 refcount_set(&dso->refcnt, 1); 1666 data = &dso->data; 1667 data->cache = RB_ROOT; 1668 data->fd = -1; 1669 data->status = DSO_DATA_STATUS_UNKNOWN; 1670 INIT_LIST_HEAD(&data->open_entry); 1671 #ifdef REFCNT_CHECKING 1672 data->dso = NULL; /* Set when on the open_entry list. */ 1673 #endif 1674 } 1675 return res; 1676 } 1677 1678 struct dso *dso__new(const char *name) 1679 { 1680 return dso__new_id(name, NULL); 1681 } 1682 1683 void dso__delete(struct dso *dso) 1684 { 1685 if (dso__dsos(dso)) 1686 pr_err("DSO %s is still in rbtree when being deleted!\n", dso__long_name(dso)); 1687 1688 /* free inlines first, as they reference symbols */ 1689 inlines__tree_delete(&RC_CHK_ACCESS(dso)->inlined_nodes); 1690 srcline__tree_delete(&RC_CHK_ACCESS(dso)->srclines); 1691 symbols__delete(&RC_CHK_ACCESS(dso)->symbols); 1692 RC_CHK_ACCESS(dso)->symbol_names_len = 0; 1693 zfree(&RC_CHK_ACCESS(dso)->symbol_names); 1694 annotated_data_type__tree_delete(dso__data_types(dso)); 1695 global_var_type__tree_delete(dso__global_vars(dso)); 1696 1697 if (RC_CHK_ACCESS(dso)->short_name_allocated) { 1698 zfree((char **)&RC_CHK_ACCESS(dso)->short_name); 1699 RC_CHK_ACCESS(dso)->short_name_allocated = false; 1700 } 1701 1702 if (RC_CHK_ACCESS(dso)->long_name_allocated) { 1703 zfree((char **)&RC_CHK_ACCESS(dso)->long_name); 1704 RC_CHK_ACCESS(dso)->long_name_allocated = false; 1705 } 1706 1707 dso__data_close(dso); 1708 auxtrace_cache__free(RC_CHK_ACCESS(dso)->auxtrace_cache); 1709 dso_cache__free(dso); 1710 dso__free_a2l(dso); 1711 dso__free_libdw(dso); 1712 dso__free_symsrc_filename(dso); 1713 nsinfo__zput(RC_CHK_ACCESS(dso)->nsinfo); 1714 mutex_destroy(dso__lock(dso)); 1715 RC_CHK_FREE(dso); 1716 } 1717 1718 struct dso *dso__get(struct dso *dso) 1719 { 1720 struct dso *result; 1721 1722 if (RC_CHK_GET(result, dso)) 1723 refcount_inc(&RC_CHK_ACCESS(dso)->refcnt); 1724 1725 return result; 1726 } 1727 1728 void dso__put(struct dso *dso) 1729 { 1730 #ifdef REFCNT_CHECKING 1731 if (dso && dso__data(dso) && refcount_read(&RC_CHK_ACCESS(dso)->refcnt) == 2) 1732 dso__data_close(dso); 1733 #endif 1734 if (dso && refcount_dec_and_test(&RC_CHK_ACCESS(dso)->refcnt)) 1735 dso__delete(dso); 1736 else 1737 RC_CHK_PUT(dso); 1738 } 1739 1740 int dso__swap_init(struct dso *dso, unsigned char eidata) 1741 { 1742 enum dso_swap_type type = dso_swap_type__from_elf_data(eidata); 1743 1744 dso__set_needs_swap(dso, type); 1745 if (type == DSO_SWAP__UNSET) { 1746 pr_err("unrecognized DSO data encoding %d\n", eidata); 1747 return -EINVAL; 1748 } 1749 return 0; 1750 } 1751 1752 void dso__set_build_id(struct dso *dso, const struct build_id *bid) 1753 { 1754 dso__id(dso)->build_id = *bid; 1755 } 1756 1757 bool dso__build_id_equal(const struct dso *dso, const struct build_id *bid) 1758 { 1759 const struct build_id *dso_bid = dso__bid(dso); 1760 1761 if (dso_bid->size > bid->size && dso_bid->size == BUILD_ID_SIZE) { 1762 /* 1763 * For the backward compatibility, it allows a build-id has 1764 * trailing zeros. 1765 */ 1766 return !memcmp(dso_bid->data, bid->data, bid->size) && 1767 !memchr_inv(&dso_bid->data[bid->size], 0, 1768 dso_bid->size - bid->size); 1769 } 1770 1771 return dso_bid->size == bid->size && 1772 memcmp(dso_bid->data, bid->data, dso_bid->size) == 0; 1773 } 1774 1775 void dso__read_running_kernel_build_id(struct dso *dso, struct machine *machine) 1776 { 1777 char path[PATH_MAX]; 1778 struct build_id bid = { .size = 0, }; 1779 1780 if (machine__is_default_guest(machine)) 1781 return; 1782 snprintf(path, sizeof(path), "%s/sys/kernel/notes", machine->root_dir); 1783 sysfs__read_build_id(path, &bid); 1784 dso__set_build_id(dso, &bid); 1785 } 1786 1787 int dso__kernel_module_get_build_id(struct dso *dso, 1788 const char *root_dir) 1789 { 1790 char filename[PATH_MAX]; 1791 struct build_id bid = { .size = 0, }; 1792 /* 1793 * kernel module short names are of the form "[module]" and 1794 * we need just "module" here. 1795 */ 1796 const char *name = dso__short_name(dso) + 1; 1797 1798 snprintf(filename, sizeof(filename), 1799 "%s/sys/module/%.*s/notes/.note.gnu.build-id", 1800 root_dir, (int)strlen(name) - 1, name); 1801 1802 sysfs__read_build_id(filename, &bid); 1803 dso__set_build_id(dso, &bid); 1804 return 0; 1805 } 1806 1807 static size_t dso__fprintf_buildid(struct dso *dso, FILE *fp) 1808 { 1809 char sbuild_id[SBUILD_ID_SIZE]; 1810 1811 build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id)); 1812 return fprintf(fp, "%s", sbuild_id); 1813 } 1814 1815 size_t dso__fprintf(struct dso *dso, FILE *fp) 1816 { 1817 struct rb_node *nd; 1818 size_t ret = fprintf(fp, "dso: %s (", dso__short_name(dso)); 1819 1820 if (dso__short_name(dso) != dso__long_name(dso)) 1821 ret += fprintf(fp, "%s, ", dso__long_name(dso)); 1822 ret += fprintf(fp, "%sloaded, ", dso__loaded(dso) ? "" : "NOT "); 1823 ret += dso__fprintf_buildid(dso, fp); 1824 ret += fprintf(fp, ")\n"); 1825 for (nd = rb_first_cached(dso__symbols(dso)); nd; nd = rb_next(nd)) { 1826 struct symbol *pos = rb_entry(nd, struct symbol, rb_node); 1827 ret += symbol__fprintf(pos, fp); 1828 } 1829 1830 return ret; 1831 } 1832 1833 enum dso_type dso__type(struct dso *dso, struct machine *machine) 1834 { 1835 int fd = -1; 1836 enum dso_type type = DSO__TYPE_UNKNOWN; 1837 1838 if (dso__data_get_fd(dso, machine, &fd)) { 1839 type = dso__type_fd(fd); 1840 dso__data_put_fd(dso); 1841 } 1842 1843 return type; 1844 } 1845 1846 int dso__strerror_load(struct dso *dso, char *buf, size_t buflen) 1847 { 1848 int idx, errnum = *dso__load_errno(dso); 1849 /* 1850 * This must have a same ordering as the enum dso_load_errno. 1851 */ 1852 static const char *dso_load__error_str[] = { 1853 "Internal tools/perf/ library error", 1854 "Invalid ELF file", 1855 "Can not read build id", 1856 "Mismatching build id", 1857 "Decompression failure", 1858 }; 1859 1860 BUG_ON(buflen == 0); 1861 1862 if (errnum >= 0) { 1863 errno = errnum; 1864 scnprintf(buf, buflen, "%m"); 1865 1866 return 0; 1867 } 1868 1869 if (errnum < __DSO_LOAD_ERRNO__START || errnum >= __DSO_LOAD_ERRNO__END) 1870 return -1; 1871 1872 idx = errnum - __DSO_LOAD_ERRNO__START; 1873 scnprintf(buf, buflen, "%s", dso_load__error_str[idx]); 1874 return 0; 1875 } 1876 1877 bool perf_pid_map_tid(const char *dso_name, int *tid) 1878 { 1879 return sscanf(dso_name, "/tmp/perf-%d.map", tid) == 1; 1880 } 1881 1882 bool is_perf_pid_map_name(const char *dso_name) 1883 { 1884 int tid; 1885 1886 return perf_pid_map_tid(dso_name, &tid); 1887 } 1888 1889 struct find_file_offset_data { 1890 u64 ip; 1891 u64 offset; 1892 }; 1893 1894 /* This will be called for each PHDR in an ELF binary */ 1895 static int find_file_offset(u64 start, u64 len, u64 pgoff, void *arg) 1896 { 1897 struct find_file_offset_data *data = arg; 1898 1899 if (start <= data->ip && data->ip < start + len) { 1900 data->offset = pgoff + data->ip - start; 1901 return 1; 1902 } 1903 return 0; 1904 } 1905 1906 static const u8 *__dso__read_symbol(struct dso *dso, const char *symfs_filename, 1907 u64 start, size_t len, 1908 u8 **out_buf, u64 *out_buf_len, bool *is_64bit) 1909 { 1910 struct nscookie nsc; 1911 int fd; 1912 ssize_t count; 1913 struct find_file_offset_data data = { 1914 .ip = start, 1915 }; 1916 u8 *code_buf = NULL; 1917 int saved_errno; 1918 1919 nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); 1920 fd = open(symfs_filename, O_RDONLY | O_CLOEXEC); 1921 saved_errno = errno; 1922 nsinfo__mountns_exit(&nsc); 1923 if (fd < 0) { 1924 errno = saved_errno; 1925 return NULL; 1926 } 1927 if (file__read_maps(fd, /*exe=*/true, find_file_offset, &data, is_64bit) <= 0) { 1928 close(fd); 1929 errno = ENOENT; 1930 return NULL; 1931 } 1932 code_buf = malloc(len); 1933 if (code_buf == NULL) { 1934 close(fd); 1935 errno = ENOMEM; 1936 return NULL; 1937 } 1938 count = pread(fd, code_buf, len, data.offset); 1939 saved_errno = errno; 1940 close(fd); 1941 if ((u64)count != len) { 1942 free(code_buf); 1943 errno = saved_errno; 1944 return NULL; 1945 } 1946 *out_buf = code_buf; 1947 *out_buf_len = len; 1948 return code_buf; 1949 } 1950 1951 /* 1952 * Read a symbol into memory for disassembly by a library like capstone of 1953 * libLLVM. If memory is allocated out_buf holds it. 1954 */ 1955 const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, 1956 const struct map *map, const struct symbol *sym, 1957 u8 **out_buf, u64 *out_buf_len, bool *is_64bit) 1958 { 1959 u64 start = map__rip_2objdump(map, sym->start); 1960 u64 end = map__rip_2objdump(map, sym->end); 1961 size_t len = end - start; 1962 1963 *out_buf = NULL; 1964 *out_buf_len = 0; 1965 *is_64bit = false; 1966 1967 if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_IMAGE) { 1968 /* 1969 * Note, there is fallback BPF image disassembly in the objdump 1970 * version but it currently does nothing. 1971 */ 1972 errno = EOPNOTSUPP; 1973 return NULL; 1974 } 1975 if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) { 1976 #ifdef HAVE_LIBBPF_SUPPORT 1977 struct bpf_prog_info_node *info_node; 1978 struct perf_bpil *info_linear; 1979 1980 *is_64bit = sizeof(void *) == sizeof(u64); 1981 info_node = perf_env__find_bpf_prog_info(dso__bpf_prog(dso)->env, 1982 dso__bpf_prog(dso)->id); 1983 if (!info_node) { 1984 errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; 1985 return NULL; 1986 } 1987 info_linear = info_node->info_linear; 1988 assert(len <= info_linear->info.jited_prog_len); 1989 *out_buf_len = len; 1990 return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns); 1991 #else 1992 pr_debug("No BPF program disassembly support\n"); 1993 errno = EOPNOTSUPP; 1994 return NULL; 1995 #endif 1996 } 1997 return __dso__read_symbol(dso, symfs_filename, start, len, 1998 out_buf, out_buf_len, is_64bit); 1999 } 2000 2001 struct debuginfo *dso__debuginfo(struct dso *dso) 2002 { 2003 char *name; 2004 bool decomp = false; 2005 struct debuginfo *dinfo = NULL; 2006 2007 mutex_lock(dso__lock(dso)); 2008 2009 name = dso__get_filename(dso, "", &decomp); 2010 if (name) 2011 dinfo = debuginfo__new(name); 2012 2013 if (decomp) 2014 unlink(name); 2015 2016 mutex_unlock(dso__lock(dso)); 2017 free(name); 2018 return dinfo; 2019 } 2020