1 #include <sys/types.h> 2 #include <stdio.h> 3 #include <stdlib.h> 4 #include <string.h> 5 #include <fcntl.h> 6 #include <unistd.h> 7 #include <inttypes.h> 8 #include <byteswap.h> 9 #include <sys/stat.h> 10 #include <sys/mman.h> 11 12 #include "util.h" 13 #include "event.h" 14 #include "debug.h" 15 #include "evlist.h" 16 #include "symbol.h" 17 #include "strlist.h" 18 #include <elf.h> 19 20 #include "session.h" 21 #include "jit.h" 22 #include "jitdump.h" 23 #include "genelf.h" 24 #include "../builtin.h" 25 26 struct jit_buf_desc { 27 struct perf_data_file *output; 28 struct perf_session *session; 29 struct machine *machine; 30 union jr_entry *entry; 31 void *buf; 32 uint64_t sample_type; 33 size_t bufsize; 34 FILE *in; 35 bool needs_bswap; /* handles cross-endianess */ 36 void *debug_data; 37 size_t nr_debug_entries; 38 uint32_t code_load_count; 39 u64 bytes_written; 40 struct rb_root code_root; 41 char dir[PATH_MAX]; 42 }; 43 44 struct debug_line_info { 45 unsigned long vma; 46 unsigned int lineno; 47 /* The filename format is unspecified, absolute path, relative etc. */ 48 char const filename[0]; 49 }; 50 51 struct jit_tool { 52 struct perf_tool tool; 53 struct perf_data_file output; 54 struct perf_data_file input; 55 u64 bytes_written; 56 }; 57 58 #define hmax(a, b) ((a) > (b) ? (a) : (b)) 59 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool)) 60 61 static int 62 jit_emit_elf(char *filename, 63 const char *sym, 64 uint64_t code_addr, 65 const void *code, 66 int csize, 67 void *debug, 68 int nr_debug_entries) 69 { 70 int ret, fd; 71 72 if (verbose > 0) 73 fprintf(stderr, "write ELF image %s\n", filename); 74 75 fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); 76 if (fd == -1) { 77 pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno)); 78 return -1; 79 } 80 81 ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries); 82 83 close(fd); 84 85 if (ret) 86 unlink(filename); 87 88 return ret; 89 } 90 91 static void 92 jit_close(struct jit_buf_desc *jd) 93 { 94 if (!(jd && jd->in)) 95 return; 96 funlockfile(jd->in); 97 fclose(jd->in); 98 jd->in = NULL; 99 } 100 101 static int 102 jit_validate_events(struct perf_session *session) 103 { 104 struct perf_evsel *evsel; 105 106 /* 107 * check that all events use CLOCK_MONOTONIC 108 */ 109 evlist__for_each(session->evlist, evsel) { 110 if (evsel->attr.use_clockid == 0 || evsel->attr.clockid != CLOCK_MONOTONIC) 111 return -1; 112 } 113 return 0; 114 } 115 116 static int 117 jit_open(struct jit_buf_desc *jd, const char *name) 118 { 119 struct jitheader header; 120 struct jr_prefix *prefix; 121 ssize_t bs, bsz = 0; 122 void *n, *buf = NULL; 123 int ret, retval = -1; 124 125 jd->in = fopen(name, "r"); 126 if (!jd->in) 127 return -1; 128 129 bsz = hmax(sizeof(header), sizeof(*prefix)); 130 131 buf = malloc(bsz); 132 if (!buf) 133 goto error; 134 135 /* 136 * protect from writer modifying the file while we are reading it 137 */ 138 flockfile(jd->in); 139 140 ret = fread(buf, sizeof(header), 1, jd->in); 141 if (ret != 1) 142 goto error; 143 144 memcpy(&header, buf, sizeof(header)); 145 146 if (header.magic != JITHEADER_MAGIC) { 147 if (header.magic != JITHEADER_MAGIC_SW) 148 goto error; 149 jd->needs_bswap = true; 150 } 151 152 if (jd->needs_bswap) { 153 header.version = bswap_32(header.version); 154 header.total_size = bswap_32(header.total_size); 155 header.pid = bswap_32(header.pid); 156 header.elf_mach = bswap_32(header.elf_mach); 157 header.timestamp = bswap_64(header.timestamp); 158 header.flags = bswap_64(header.flags); 159 } 160 161 if (verbose > 2) 162 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\n", 163 header.version, 164 header.total_size, 165 (unsigned long long)header.timestamp, 166 header.pid, 167 header.elf_mach); 168 169 if (header.flags & JITDUMP_FLAGS_RESERVED) { 170 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n", 171 (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED); 172 goto error; 173 } 174 175 /* 176 * validate event is using the correct clockid 177 */ 178 if (jit_validate_events(jd->session)) { 179 pr_err("error, jitted code must be sampled with perf record -k 1\n"); 180 goto error; 181 } 182 183 bs = header.total_size - sizeof(header); 184 185 if (bs > bsz) { 186 n = realloc(buf, bs); 187 if (!n) 188 goto error; 189 bsz = bs; 190 buf = n; 191 /* read extra we do not know about */ 192 ret = fread(buf, bs - bsz, 1, jd->in); 193 if (ret != 1) 194 goto error; 195 } 196 /* 197 * keep dirname for generating files and mmap records 198 */ 199 strcpy(jd->dir, name); 200 dirname(jd->dir); 201 202 return 0; 203 error: 204 funlockfile(jd->in); 205 fclose(jd->in); 206 return retval; 207 } 208 209 static union jr_entry * 210 jit_get_next_entry(struct jit_buf_desc *jd) 211 { 212 struct jr_prefix *prefix; 213 union jr_entry *jr; 214 void *addr; 215 size_t bs, size; 216 int id, ret; 217 218 if (!(jd && jd->in)) 219 return NULL; 220 221 if (jd->buf == NULL) { 222 size_t sz = getpagesize(); 223 if (sz < sizeof(*prefix)) 224 sz = sizeof(*prefix); 225 226 jd->buf = malloc(sz); 227 if (jd->buf == NULL) 228 return NULL; 229 230 jd->bufsize = sz; 231 } 232 233 prefix = jd->buf; 234 235 /* 236 * file is still locked at this point 237 */ 238 ret = fread(prefix, sizeof(*prefix), 1, jd->in); 239 if (ret != 1) 240 return NULL; 241 242 if (jd->needs_bswap) { 243 prefix->id = bswap_32(prefix->id); 244 prefix->total_size = bswap_32(prefix->total_size); 245 prefix->timestamp = bswap_64(prefix->timestamp); 246 } 247 id = prefix->id; 248 size = prefix->total_size; 249 250 bs = (size_t)size; 251 if (bs < sizeof(*prefix)) 252 return NULL; 253 254 if (id >= JIT_CODE_MAX) { 255 pr_warning("next_entry: unknown prefix %d, skipping\n", id); 256 return NULL; 257 } 258 if (bs > jd->bufsize) { 259 void *n; 260 n = realloc(jd->buf, bs); 261 if (!n) 262 return NULL; 263 jd->buf = n; 264 jd->bufsize = bs; 265 } 266 267 addr = ((void *)jd->buf) + sizeof(*prefix); 268 269 ret = fread(addr, bs - sizeof(*prefix), 1, jd->in); 270 if (ret != 1) 271 return NULL; 272 273 jr = (union jr_entry *)jd->buf; 274 275 switch(id) { 276 case JIT_CODE_DEBUG_INFO: 277 if (jd->needs_bswap) { 278 uint64_t n; 279 jr->info.code_addr = bswap_64(jr->info.code_addr); 280 jr->info.nr_entry = bswap_64(jr->info.nr_entry); 281 for (n = 0 ; n < jr->info.nr_entry; n++) { 282 jr->info.entries[n].addr = bswap_64(jr->info.entries[n].addr); 283 jr->info.entries[n].lineno = bswap_32(jr->info.entries[n].lineno); 284 jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim); 285 } 286 } 287 break; 288 case JIT_CODE_CLOSE: 289 break; 290 case JIT_CODE_LOAD: 291 if (jd->needs_bswap) { 292 jr->load.pid = bswap_32(jr->load.pid); 293 jr->load.tid = bswap_32(jr->load.tid); 294 jr->load.vma = bswap_64(jr->load.vma); 295 jr->load.code_addr = bswap_64(jr->load.code_addr); 296 jr->load.code_size = bswap_64(jr->load.code_size); 297 jr->load.code_index= bswap_64(jr->load.code_index); 298 } 299 jd->code_load_count++; 300 break; 301 case JIT_CODE_MOVE: 302 if (jd->needs_bswap) { 303 jr->move.pid = bswap_32(jr->move.pid); 304 jr->move.tid = bswap_32(jr->move.tid); 305 jr->move.vma = bswap_64(jr->move.vma); 306 jr->move.old_code_addr = bswap_64(jr->move.old_code_addr); 307 jr->move.new_code_addr = bswap_64(jr->move.new_code_addr); 308 jr->move.code_size = bswap_64(jr->move.code_size); 309 jr->move.code_index = bswap_64(jr->move.code_index); 310 } 311 break; 312 case JIT_CODE_MAX: 313 default: 314 return NULL; 315 } 316 return jr; 317 } 318 319 static int 320 jit_inject_event(struct jit_buf_desc *jd, union perf_event *event) 321 { 322 ssize_t size; 323 324 size = perf_data_file__write(jd->output, event, event->header.size); 325 if (size < 0) 326 return -1; 327 328 jd->bytes_written += size; 329 return 0; 330 } 331 332 static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr) 333 { 334 struct perf_sample sample; 335 union perf_event *event; 336 struct perf_tool *tool = jd->session->tool; 337 uint64_t code, addr; 338 uintptr_t uaddr; 339 char *filename; 340 struct stat st; 341 size_t size; 342 u16 idr_size; 343 const char *sym; 344 uint32_t count; 345 int ret, csize; 346 pid_t pid, tid; 347 struct { 348 u32 pid, tid; 349 u64 time; 350 } *id; 351 352 pid = jr->load.pid; 353 tid = jr->load.tid; 354 csize = jr->load.code_size; 355 addr = jr->load.code_addr; 356 sym = (void *)((unsigned long)jr + sizeof(jr->load)); 357 code = (unsigned long)jr + jr->load.p.total_size - csize; 358 count = jr->load.code_index; 359 idr_size = jd->machine->id_hdr_size; 360 361 event = calloc(1, sizeof(*event) + idr_size); 362 if (!event) 363 return -1; 364 365 filename = event->mmap2.filename; 366 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so", 367 jd->dir, 368 pid, 369 count); 370 371 size++; /* for \0 */ 372 373 size = PERF_ALIGN(size, sizeof(u64)); 374 uaddr = (uintptr_t)code; 375 ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries); 376 377 if (jd->debug_data && jd->nr_debug_entries) { 378 free(jd->debug_data); 379 jd->debug_data = NULL; 380 jd->nr_debug_entries = 0; 381 } 382 383 if (ret) { 384 free(event); 385 return -1; 386 } 387 if (stat(filename, &st)) 388 memset(&st, 0, sizeof(stat)); 389 390 event->mmap2.header.type = PERF_RECORD_MMAP2; 391 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 392 event->mmap2.header.size = (sizeof(event->mmap2) - 393 (sizeof(event->mmap2.filename) - size) + idr_size); 394 395 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 396 event->mmap2.start = addr; 397 event->mmap2.len = csize; 398 event->mmap2.pid = pid; 399 event->mmap2.tid = tid; 400 event->mmap2.ino = st.st_ino; 401 event->mmap2.maj = major(st.st_dev); 402 event->mmap2.min = minor(st.st_dev); 403 event->mmap2.prot = st.st_mode; 404 event->mmap2.flags = MAP_SHARED; 405 event->mmap2.ino_generation = 1; 406 407 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 408 if (jd->sample_type & PERF_SAMPLE_TID) { 409 id->pid = pid; 410 id->tid = tid; 411 } 412 if (jd->sample_type & PERF_SAMPLE_TIME) 413 id->time = jr->load.p.timestamp; 414 415 /* 416 * create pseudo sample to induce dso hit increment 417 * use first address as sample address 418 */ 419 memset(&sample, 0, sizeof(sample)); 420 sample.pid = pid; 421 sample.tid = tid; 422 sample.time = id->time; 423 sample.ip = addr; 424 425 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 426 if (ret) 427 return ret; 428 429 ret = jit_inject_event(jd, event); 430 /* 431 * mark dso as use to generate buildid in the header 432 */ 433 if (!ret) 434 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 435 436 return ret; 437 } 438 439 static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr) 440 { 441 struct perf_sample sample; 442 union perf_event *event; 443 struct perf_tool *tool = jd->session->tool; 444 char *filename; 445 size_t size; 446 struct stat st; 447 u16 idr_size; 448 int ret; 449 pid_t pid, tid; 450 struct { 451 u32 pid, tid; 452 u64 time; 453 } *id; 454 455 pid = jr->move.pid; 456 tid = jr->move.tid; 457 idr_size = jd->machine->id_hdr_size; 458 459 /* 460 * +16 to account for sample_id_all (hack) 461 */ 462 event = calloc(1, sizeof(*event) + 16); 463 if (!event) 464 return -1; 465 466 filename = event->mmap2.filename; 467 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64, 468 jd->dir, 469 pid, 470 jr->move.code_index); 471 472 size++; /* for \0 */ 473 474 if (stat(filename, &st)) 475 memset(&st, 0, sizeof(stat)); 476 477 size = PERF_ALIGN(size, sizeof(u64)); 478 479 event->mmap2.header.type = PERF_RECORD_MMAP2; 480 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 481 event->mmap2.header.size = (sizeof(event->mmap2) - 482 (sizeof(event->mmap2.filename) - size) + idr_size); 483 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 484 event->mmap2.start = jr->move.new_code_addr; 485 event->mmap2.len = jr->move.code_size; 486 event->mmap2.pid = pid; 487 event->mmap2.tid = tid; 488 event->mmap2.ino = st.st_ino; 489 event->mmap2.maj = major(st.st_dev); 490 event->mmap2.min = minor(st.st_dev); 491 event->mmap2.prot = st.st_mode; 492 event->mmap2.flags = MAP_SHARED; 493 event->mmap2.ino_generation = 1; 494 495 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 496 if (jd->sample_type & PERF_SAMPLE_TID) { 497 id->pid = pid; 498 id->tid = tid; 499 } 500 if (jd->sample_type & PERF_SAMPLE_TIME) 501 id->time = jr->load.p.timestamp; 502 503 /* 504 * create pseudo sample to induce dso hit increment 505 * use first address as sample address 506 */ 507 memset(&sample, 0, sizeof(sample)); 508 sample.pid = pid; 509 sample.tid = tid; 510 sample.time = id->time; 511 sample.ip = jr->move.new_code_addr; 512 513 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 514 if (ret) 515 return ret; 516 517 ret = jit_inject_event(jd, event); 518 if (!ret) 519 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 520 521 return ret; 522 } 523 524 static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr) 525 { 526 void *data; 527 size_t sz; 528 529 if (!(jd && jr)) 530 return -1; 531 532 sz = jr->prefix.total_size - sizeof(jr->info); 533 data = malloc(sz); 534 if (!data) 535 return -1; 536 537 memcpy(data, &jr->info.entries, sz); 538 539 jd->debug_data = data; 540 541 /* 542 * we must use nr_entry instead of size here because 543 * we cannot distinguish actual entry from padding otherwise 544 */ 545 jd->nr_debug_entries = jr->info.nr_entry; 546 547 return 0; 548 } 549 550 static int 551 jit_process_dump(struct jit_buf_desc *jd) 552 { 553 union jr_entry *jr; 554 int ret; 555 556 while ((jr = jit_get_next_entry(jd))) { 557 switch(jr->prefix.id) { 558 case JIT_CODE_LOAD: 559 ret = jit_repipe_code_load(jd, jr); 560 break; 561 case JIT_CODE_MOVE: 562 ret = jit_repipe_code_move(jd, jr); 563 break; 564 case JIT_CODE_DEBUG_INFO: 565 ret = jit_repipe_debug_info(jd, jr); 566 break; 567 default: 568 ret = 0; 569 continue; 570 } 571 } 572 return ret; 573 } 574 575 static int 576 jit_inject(struct jit_buf_desc *jd, char *path) 577 { 578 int ret; 579 580 if (verbose > 0) 581 fprintf(stderr, "injecting: %s\n", path); 582 583 ret = jit_open(jd, path); 584 if (ret) 585 return -1; 586 587 ret = jit_process_dump(jd); 588 589 jit_close(jd); 590 591 if (verbose > 0) 592 fprintf(stderr, "injected: %s (%d)\n", path, ret); 593 594 return 0; 595 } 596 597 /* 598 * File must be with pattern .../jit-XXXX.dump 599 * where XXXX is the PID of the process which did the mmap() 600 * as captured in the RECORD_MMAP record 601 */ 602 static int 603 jit_detect(char *mmap_name, pid_t pid) 604 { 605 char *p; 606 char *end = NULL; 607 pid_t pid2; 608 609 if (verbose > 2) 610 fprintf(stderr, "jit marker trying : %s\n", mmap_name); 611 /* 612 * get file name 613 */ 614 p = strrchr(mmap_name, '/'); 615 if (!p) 616 return -1; 617 618 /* 619 * match prefix 620 */ 621 if (strncmp(p, "/jit-", 5)) 622 return -1; 623 624 /* 625 * skip prefix 626 */ 627 p += 5; 628 629 /* 630 * must be followed by a pid 631 */ 632 if (!isdigit(*p)) 633 return -1; 634 635 pid2 = (int)strtol(p, &end, 10); 636 if (!end) 637 return -1; 638 639 /* 640 * pid does not match mmap pid 641 * pid==0 in system-wide mode (synthesized) 642 */ 643 if (pid && pid2 != pid) 644 return -1; 645 /* 646 * validate suffix 647 */ 648 if (strcmp(end, ".dump")) 649 return -1; 650 651 if (verbose > 0) 652 fprintf(stderr, "jit marker found: %s\n", mmap_name); 653 654 return 0; 655 } 656 657 int 658 jit_process(struct perf_session *session, 659 struct perf_data_file *output, 660 struct machine *machine, 661 char *filename, 662 pid_t pid, 663 u64 *nbytes) 664 { 665 struct perf_evsel *first; 666 struct jit_buf_desc jd; 667 int ret; 668 669 /* 670 * first, detect marker mmap (i.e., the jitdump mmap) 671 */ 672 if (jit_detect(filename, pid)) 673 return 0; 674 675 memset(&jd, 0, sizeof(jd)); 676 677 jd.session = session; 678 jd.output = output; 679 jd.machine = machine; 680 681 /* 682 * track sample_type to compute id_all layout 683 * perf sets the same sample type to all events as of now 684 */ 685 first = perf_evlist__first(session->evlist); 686 jd.sample_type = first->attr.sample_type; 687 688 *nbytes = 0; 689 690 ret = jit_inject(&jd, filename); 691 if (!ret) { 692 *nbytes = jd.bytes_written; 693 ret = 1; 694 } 695 696 return ret; 697 } 698