1 #include <sys/types.h> 2 #include <stdio.h> 3 #include <stdlib.h> 4 #include <string.h> 5 #include <fcntl.h> 6 #include <unistd.h> 7 #include <inttypes.h> 8 #include <byteswap.h> 9 #include <sys/stat.h> 10 #include <sys/mman.h> 11 12 #include "util.h" 13 #include "event.h" 14 #include "debug.h" 15 #include "evlist.h" 16 #include "symbol.h" 17 #include "strlist.h" 18 #include <elf.h> 19 20 #include "session.h" 21 #include "jit.h" 22 #include "jitdump.h" 23 #include "genelf.h" 24 #include "../builtin.h" 25 26 struct jit_buf_desc { 27 struct perf_data_file *output; 28 struct perf_session *session; 29 struct machine *machine; 30 union jr_entry *entry; 31 void *buf; 32 uint64_t sample_type; 33 size_t bufsize; 34 FILE *in; 35 bool needs_bswap; /* handles cross-endianess */ 36 void *debug_data; 37 size_t nr_debug_entries; 38 uint32_t code_load_count; 39 u64 bytes_written; 40 struct rb_root code_root; 41 char dir[PATH_MAX]; 42 }; 43 44 struct debug_line_info { 45 unsigned long vma; 46 unsigned int lineno; 47 /* The filename format is unspecified, absolute path, relative etc. */ 48 char const filename[0]; 49 }; 50 51 struct jit_tool { 52 struct perf_tool tool; 53 struct perf_data_file output; 54 struct perf_data_file input; 55 u64 bytes_written; 56 }; 57 58 #define hmax(a, b) ((a) > (b) ? (a) : (b)) 59 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool)) 60 61 static int 62 jit_emit_elf(char *filename, 63 const char *sym, 64 uint64_t code_addr, 65 const void *code, 66 int csize, 67 void *debug, 68 int nr_debug_entries) 69 { 70 int ret, fd; 71 72 if (verbose > 0) 73 fprintf(stderr, "write ELF image %s\n", filename); 74 75 fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); 76 if (fd == -1) { 77 pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno)); 78 return -1; 79 } 80 81 ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries); 82 83 close(fd); 84 85 if (ret) 86 unlink(filename); 87 88 return ret; 89 } 90 91 static void 92 jit_close(struct jit_buf_desc *jd) 93 { 94 if (!(jd && jd->in)) 95 return; 96 funlockfile(jd->in); 97 fclose(jd->in); 98 jd->in = NULL; 99 } 100 101 static int 102 jit_open(struct jit_buf_desc *jd, const char *name) 103 { 104 struct jitheader header; 105 struct jr_prefix *prefix; 106 ssize_t bs, bsz = 0; 107 void *n, *buf = NULL; 108 int ret, retval = -1; 109 110 jd->in = fopen(name, "r"); 111 if (!jd->in) 112 return -1; 113 114 bsz = hmax(sizeof(header), sizeof(*prefix)); 115 116 buf = malloc(bsz); 117 if (!buf) 118 goto error; 119 120 /* 121 * protect from writer modifying the file while we are reading it 122 */ 123 flockfile(jd->in); 124 125 ret = fread(buf, sizeof(header), 1, jd->in); 126 if (ret != 1) 127 goto error; 128 129 memcpy(&header, buf, sizeof(header)); 130 131 if (header.magic != JITHEADER_MAGIC) { 132 if (header.magic != JITHEADER_MAGIC_SW) 133 goto error; 134 jd->needs_bswap = true; 135 } 136 137 if (jd->needs_bswap) { 138 header.version = bswap_32(header.version); 139 header.total_size = bswap_32(header.total_size); 140 header.pid = bswap_32(header.pid); 141 header.elf_mach = bswap_32(header.elf_mach); 142 header.timestamp = bswap_64(header.timestamp); 143 header.flags = bswap_64(header.flags); 144 } 145 146 if (verbose > 2) 147 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\n", 148 header.version, 149 header.total_size, 150 (unsigned long long)header.timestamp, 151 header.pid, 152 header.elf_mach); 153 154 if (header.flags & JITDUMP_FLAGS_RESERVED) { 155 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n", 156 (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED); 157 goto error; 158 } 159 160 bs = header.total_size - sizeof(header); 161 162 if (bs > bsz) { 163 n = realloc(buf, bs); 164 if (!n) 165 goto error; 166 bsz = bs; 167 buf = n; 168 /* read extra we do not know about */ 169 ret = fread(buf, bs - bsz, 1, jd->in); 170 if (ret != 1) 171 goto error; 172 } 173 /* 174 * keep dirname for generating files and mmap records 175 */ 176 strcpy(jd->dir, name); 177 dirname(jd->dir); 178 179 return 0; 180 error: 181 funlockfile(jd->in); 182 fclose(jd->in); 183 return retval; 184 } 185 186 static union jr_entry * 187 jit_get_next_entry(struct jit_buf_desc *jd) 188 { 189 struct jr_prefix *prefix; 190 union jr_entry *jr; 191 void *addr; 192 size_t bs, size; 193 int id, ret; 194 195 if (!(jd && jd->in)) 196 return NULL; 197 198 if (jd->buf == NULL) { 199 size_t sz = getpagesize(); 200 if (sz < sizeof(*prefix)) 201 sz = sizeof(*prefix); 202 203 jd->buf = malloc(sz); 204 if (jd->buf == NULL) 205 return NULL; 206 207 jd->bufsize = sz; 208 } 209 210 prefix = jd->buf; 211 212 /* 213 * file is still locked at this point 214 */ 215 ret = fread(prefix, sizeof(*prefix), 1, jd->in); 216 if (ret != 1) 217 return NULL; 218 219 if (jd->needs_bswap) { 220 prefix->id = bswap_32(prefix->id); 221 prefix->total_size = bswap_32(prefix->total_size); 222 prefix->timestamp = bswap_64(prefix->timestamp); 223 } 224 id = prefix->id; 225 size = prefix->total_size; 226 227 bs = (size_t)size; 228 if (bs < sizeof(*prefix)) 229 return NULL; 230 231 if (id >= JIT_CODE_MAX) { 232 pr_warning("next_entry: unknown prefix %d, skipping\n", id); 233 return NULL; 234 } 235 if (bs > jd->bufsize) { 236 void *n; 237 n = realloc(jd->buf, bs); 238 if (!n) 239 return NULL; 240 jd->buf = n; 241 jd->bufsize = bs; 242 } 243 244 addr = ((void *)jd->buf) + sizeof(*prefix); 245 246 ret = fread(addr, bs - sizeof(*prefix), 1, jd->in); 247 if (ret != 1) 248 return NULL; 249 250 jr = (union jr_entry *)jd->buf; 251 252 switch(id) { 253 case JIT_CODE_DEBUG_INFO: 254 if (jd->needs_bswap) { 255 uint64_t n; 256 jr->info.code_addr = bswap_64(jr->info.code_addr); 257 jr->info.nr_entry = bswap_64(jr->info.nr_entry); 258 for (n = 0 ; n < jr->info.nr_entry; n++) { 259 jr->info.entries[n].addr = bswap_64(jr->info.entries[n].addr); 260 jr->info.entries[n].lineno = bswap_32(jr->info.entries[n].lineno); 261 jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim); 262 } 263 } 264 break; 265 case JIT_CODE_CLOSE: 266 break; 267 case JIT_CODE_LOAD: 268 if (jd->needs_bswap) { 269 jr->load.pid = bswap_32(jr->load.pid); 270 jr->load.tid = bswap_32(jr->load.tid); 271 jr->load.vma = bswap_64(jr->load.vma); 272 jr->load.code_addr = bswap_64(jr->load.code_addr); 273 jr->load.code_size = bswap_64(jr->load.code_size); 274 jr->load.code_index= bswap_64(jr->load.code_index); 275 } 276 jd->code_load_count++; 277 break; 278 case JIT_CODE_MOVE: 279 if (jd->needs_bswap) { 280 jr->move.pid = bswap_32(jr->move.pid); 281 jr->move.tid = bswap_32(jr->move.tid); 282 jr->move.vma = bswap_64(jr->move.vma); 283 jr->move.old_code_addr = bswap_64(jr->move.old_code_addr); 284 jr->move.new_code_addr = bswap_64(jr->move.new_code_addr); 285 jr->move.code_size = bswap_64(jr->move.code_size); 286 jr->move.code_index = bswap_64(jr->move.code_index); 287 } 288 break; 289 case JIT_CODE_MAX: 290 default: 291 return NULL; 292 } 293 return jr; 294 } 295 296 static int 297 jit_inject_event(struct jit_buf_desc *jd, union perf_event *event) 298 { 299 ssize_t size; 300 301 size = perf_data_file__write(jd->output, event, event->header.size); 302 if (size < 0) 303 return -1; 304 305 jd->bytes_written += size; 306 return 0; 307 } 308 309 static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr) 310 { 311 struct perf_sample sample; 312 union perf_event *event; 313 struct perf_tool *tool = jd->session->tool; 314 uint64_t code, addr; 315 uintptr_t uaddr; 316 char *filename; 317 struct stat st; 318 size_t size; 319 u16 idr_size; 320 const char *sym; 321 uint32_t count; 322 int ret, csize; 323 pid_t pid, tid; 324 struct { 325 u32 pid, tid; 326 u64 time; 327 } *id; 328 329 pid = jr->load.pid; 330 tid = jr->load.tid; 331 csize = jr->load.code_size; 332 addr = jr->load.code_addr; 333 sym = (void *)((unsigned long)jr + sizeof(jr->load)); 334 code = (unsigned long)jr + jr->load.p.total_size - csize; 335 count = jr->load.code_index; 336 idr_size = jd->machine->id_hdr_size; 337 338 event = calloc(1, sizeof(*event) + idr_size); 339 if (!event) 340 return -1; 341 342 filename = event->mmap2.filename; 343 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so", 344 jd->dir, 345 pid, 346 count); 347 348 size++; /* for \0 */ 349 350 size = PERF_ALIGN(size, sizeof(u64)); 351 uaddr = (uintptr_t)code; 352 ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries); 353 354 if (jd->debug_data && jd->nr_debug_entries) { 355 free(jd->debug_data); 356 jd->debug_data = NULL; 357 jd->nr_debug_entries = 0; 358 } 359 360 if (ret) { 361 free(event); 362 return -1; 363 } 364 if (stat(filename, &st)) 365 memset(&st, 0, sizeof(stat)); 366 367 event->mmap2.header.type = PERF_RECORD_MMAP2; 368 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 369 event->mmap2.header.size = (sizeof(event->mmap2) - 370 (sizeof(event->mmap2.filename) - size) + idr_size); 371 372 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 373 event->mmap2.start = addr; 374 event->mmap2.len = csize; 375 event->mmap2.pid = pid; 376 event->mmap2.tid = tid; 377 event->mmap2.ino = st.st_ino; 378 event->mmap2.maj = major(st.st_dev); 379 event->mmap2.min = minor(st.st_dev); 380 event->mmap2.prot = st.st_mode; 381 event->mmap2.flags = MAP_SHARED; 382 event->mmap2.ino_generation = 1; 383 384 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 385 if (jd->sample_type & PERF_SAMPLE_TID) { 386 id->pid = pid; 387 id->tid = tid; 388 } 389 if (jd->sample_type & PERF_SAMPLE_TIME) 390 id->time = jr->load.p.timestamp; 391 392 /* 393 * create pseudo sample to induce dso hit increment 394 * use first address as sample address 395 */ 396 memset(&sample, 0, sizeof(sample)); 397 sample.pid = pid; 398 sample.tid = tid; 399 sample.time = id->time; 400 sample.ip = addr; 401 402 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 403 if (ret) 404 return ret; 405 406 ret = jit_inject_event(jd, event); 407 /* 408 * mark dso as use to generate buildid in the header 409 */ 410 if (!ret) 411 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 412 413 return ret; 414 } 415 416 static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr) 417 { 418 struct perf_sample sample; 419 union perf_event *event; 420 struct perf_tool *tool = jd->session->tool; 421 char *filename; 422 size_t size; 423 struct stat st; 424 u16 idr_size; 425 int ret; 426 pid_t pid, tid; 427 struct { 428 u32 pid, tid; 429 u64 time; 430 } *id; 431 432 pid = jr->move.pid; 433 tid = jr->move.tid; 434 idr_size = jd->machine->id_hdr_size; 435 436 /* 437 * +16 to account for sample_id_all (hack) 438 */ 439 event = calloc(1, sizeof(*event) + 16); 440 if (!event) 441 return -1; 442 443 filename = event->mmap2.filename; 444 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64, 445 jd->dir, 446 pid, 447 jr->move.code_index); 448 449 size++; /* for \0 */ 450 451 if (stat(filename, &st)) 452 memset(&st, 0, sizeof(stat)); 453 454 size = PERF_ALIGN(size, sizeof(u64)); 455 456 event->mmap2.header.type = PERF_RECORD_MMAP2; 457 event->mmap2.header.misc = PERF_RECORD_MISC_USER; 458 event->mmap2.header.size = (sizeof(event->mmap2) - 459 (sizeof(event->mmap2.filename) - size) + idr_size); 460 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET; 461 event->mmap2.start = jr->move.new_code_addr; 462 event->mmap2.len = jr->move.code_size; 463 event->mmap2.pid = pid; 464 event->mmap2.tid = tid; 465 event->mmap2.ino = st.st_ino; 466 event->mmap2.maj = major(st.st_dev); 467 event->mmap2.min = minor(st.st_dev); 468 event->mmap2.prot = st.st_mode; 469 event->mmap2.flags = MAP_SHARED; 470 event->mmap2.ino_generation = 1; 471 472 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size); 473 if (jd->sample_type & PERF_SAMPLE_TID) { 474 id->pid = pid; 475 id->tid = tid; 476 } 477 if (jd->sample_type & PERF_SAMPLE_TIME) 478 id->time = jr->load.p.timestamp; 479 480 /* 481 * create pseudo sample to induce dso hit increment 482 * use first address as sample address 483 */ 484 memset(&sample, 0, sizeof(sample)); 485 sample.pid = pid; 486 sample.tid = tid; 487 sample.time = id->time; 488 sample.ip = jr->move.new_code_addr; 489 490 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine); 491 if (ret) 492 return ret; 493 494 ret = jit_inject_event(jd, event); 495 if (!ret) 496 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); 497 498 return ret; 499 } 500 501 static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr) 502 { 503 void *data; 504 size_t sz; 505 506 if (!(jd && jr)) 507 return -1; 508 509 sz = jr->prefix.total_size - sizeof(jr->info); 510 data = malloc(sz); 511 if (!data) 512 return -1; 513 514 memcpy(data, &jr->info.entries, sz); 515 516 jd->debug_data = data; 517 518 /* 519 * we must use nr_entry instead of size here because 520 * we cannot distinguish actual entry from padding otherwise 521 */ 522 jd->nr_debug_entries = jr->info.nr_entry; 523 524 return 0; 525 } 526 527 static int 528 jit_process_dump(struct jit_buf_desc *jd) 529 { 530 union jr_entry *jr; 531 int ret; 532 533 while ((jr = jit_get_next_entry(jd))) { 534 switch(jr->prefix.id) { 535 case JIT_CODE_LOAD: 536 ret = jit_repipe_code_load(jd, jr); 537 break; 538 case JIT_CODE_MOVE: 539 ret = jit_repipe_code_move(jd, jr); 540 break; 541 case JIT_CODE_DEBUG_INFO: 542 ret = jit_repipe_debug_info(jd, jr); 543 break; 544 default: 545 ret = 0; 546 continue; 547 } 548 } 549 return ret; 550 } 551 552 static int 553 jit_inject(struct jit_buf_desc *jd, char *path) 554 { 555 int ret; 556 557 if (verbose > 0) 558 fprintf(stderr, "injecting: %s\n", path); 559 560 ret = jit_open(jd, path); 561 if (ret) 562 return -1; 563 564 ret = jit_process_dump(jd); 565 566 jit_close(jd); 567 568 if (verbose > 0) 569 fprintf(stderr, "injected: %s (%d)\n", path, ret); 570 571 return 0; 572 } 573 574 /* 575 * File must be with pattern .../jit-XXXX.dump 576 * where XXXX is the PID of the process which did the mmap() 577 * as captured in the RECORD_MMAP record 578 */ 579 static int 580 jit_detect(char *mmap_name, pid_t pid) 581 { 582 char *p; 583 char *end = NULL; 584 pid_t pid2; 585 586 if (verbose > 2) 587 fprintf(stderr, "jit marker trying : %s\n", mmap_name); 588 /* 589 * get file name 590 */ 591 p = strrchr(mmap_name, '/'); 592 if (!p) 593 return -1; 594 595 /* 596 * match prefix 597 */ 598 if (strncmp(p, "/jit-", 5)) 599 return -1; 600 601 /* 602 * skip prefix 603 */ 604 p += 5; 605 606 /* 607 * must be followed by a pid 608 */ 609 if (!isdigit(*p)) 610 return -1; 611 612 pid2 = (int)strtol(p, &end, 10); 613 if (!end) 614 return -1; 615 616 /* 617 * pid does not match mmap pid 618 * pid==0 in system-wide mode (synthesized) 619 */ 620 if (pid && pid2 != pid) 621 return -1; 622 /* 623 * validate suffix 624 */ 625 if (strcmp(end, ".dump")) 626 return -1; 627 628 if (verbose > 0) 629 fprintf(stderr, "jit marker found: %s\n", mmap_name); 630 631 return 0; 632 } 633 634 int 635 jit_process(struct perf_session *session, 636 struct perf_data_file *output, 637 struct machine *machine, 638 char *filename, 639 pid_t pid, 640 u64 *nbytes) 641 { 642 struct perf_evsel *first; 643 struct jit_buf_desc jd; 644 int ret; 645 646 /* 647 * first, detect marker mmap (i.e., the jitdump mmap) 648 */ 649 if (jit_detect(filename, pid)) 650 return -1; 651 652 memset(&jd, 0, sizeof(jd)); 653 654 jd.session = session; 655 jd.output = output; 656 jd.machine = machine; 657 658 /* 659 * track sample_type to compute id_all layout 660 * perf sets the same sample type to all events as of now 661 */ 662 first = perf_evlist__first(session->evlist); 663 jd.sample_type = first->attr.sample_type; 664 665 *nbytes = 0; 666 667 ret = jit_inject(&jd, filename); 668 if (!ret) 669 *nbytes = jd.bytes_written; 670 671 return ret; 672 } 673