1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * builtin-inject.c 4 * 5 * Builtin inject command: Examine the live mode (stdin) event stream 6 * and repipe it to stdout while optionally injecting additional 7 * events into it. 8 */ 9 #include "builtin.h" 10 11 #include "util/color.h" 12 #include "util/dso.h" 13 #include "util/vdso.h" 14 #include "util/evlist.h" 15 #include "util/evsel.h" 16 #include "util/map.h" 17 #include "util/session.h" 18 #include "util/tool.h" 19 #include "util/debug.h" 20 #include "util/build-id.h" 21 #include "util/data.h" 22 #include "util/auxtrace.h" 23 #include "util/jit.h" 24 #include "util/string2.h" 25 #include "util/symbol.h" 26 #include "util/synthetic-events.h" 27 #include "util/thread.h" 28 #include "util/namespaces.h" 29 #include "util/unwind.h" 30 #include "util/util.h" 31 #include "util/tsc.h" 32 33 #include <internal/lib.h> 34 35 #include <linux/err.h> 36 #include <subcmd/parse-options.h> 37 #include <uapi/linux/mman.h> /* To get things like MAP_HUGETLB even on older libc headers */ 38 39 #include <linux/list.h> 40 #include <linux/string.h> 41 #include <linux/zalloc.h> 42 #include <linux/hash.h> 43 #include <ctype.h> 44 #include <errno.h> 45 #include <signal.h> 46 #include <inttypes.h> 47 48 struct guest_event { 49 struct perf_sample sample; 50 union perf_event *event; 51 char *event_buf; 52 }; 53 54 struct guest_id { 55 /* hlist_node must be first, see free_hlist() */ 56 struct hlist_node node; 57 u64 id; 58 u64 host_id; 59 u32 vcpu; 60 }; 61 62 struct guest_tid { 63 /* hlist_node must be first, see free_hlist() */ 64 struct hlist_node node; 65 /* Thread ID of QEMU thread */ 66 u32 tid; 67 u32 vcpu; 68 }; 69 70 struct guest_vcpu { 71 /* Current host CPU */ 72 u32 cpu; 73 /* Thread ID of QEMU thread */ 74 u32 tid; 75 }; 76 77 struct guest_session { 78 char *perf_data_file; 79 u32 machine_pid; 80 u64 time_offset; 81 double time_scale; 82 struct perf_tool tool; 83 struct perf_data data; 84 struct perf_session *session; 85 char *tmp_file_name; 86 int tmp_fd; 87 struct perf_tsc_conversion host_tc; 88 struct perf_tsc_conversion guest_tc; 89 bool copy_kcore_dir; 90 bool have_tc; 91 bool fetched; 92 bool ready; 93 u16 dflt_id_hdr_size; 94 u64 dflt_id; 95 u64 highest_id; 96 /* Array of guest_vcpu */ 97 struct guest_vcpu *vcpu; 98 size_t vcpu_cnt; 99 /* Hash table for guest_id */ 100 struct hlist_head heads[PERF_EVLIST__HLIST_SIZE]; 101 /* Hash table for guest_tid */ 102 struct hlist_head tids[PERF_EVLIST__HLIST_SIZE]; 103 /* Place to stash next guest event */ 104 struct guest_event ev; 105 }; 106 107 enum build_id_rewrite_style { 108 BID_RWS__NONE = 0, 109 BID_RWS__INJECT_HEADER_LAZY, 110 BID_RWS__INJECT_HEADER_ALL, 111 BID_RWS__MMAP2_BUILDID_ALL, 112 BID_RWS__MMAP2_BUILDID_LAZY, 113 }; 114 115 struct perf_inject { 116 struct perf_tool tool; 117 struct perf_session *session; 118 enum build_id_rewrite_style build_id_style; 119 bool sched_stat; 120 bool have_auxtrace; 121 bool strip; 122 bool jit_mode; 123 bool in_place_update; 124 bool in_place_update_dry_run; 125 bool copy_kcore_dir; 126 bool convert_callchain; 127 const char *input_name; 128 struct perf_data output; 129 u64 bytes_written; 130 u64 aux_id; 131 struct list_head samples; 132 struct itrace_synth_opts itrace_synth_opts; 133 char *event_copy; 134 struct perf_file_section secs[HEADER_FEAT_BITS]; 135 struct guest_session guest_session; 136 struct strlist *known_build_ids; 137 struct evsel *mmap_evsel; 138 struct ip_callchain *raw_callchain; 139 }; 140 141 struct event_entry { 142 struct list_head node; 143 u32 tid; 144 union perf_event event[]; 145 }; 146 147 static int tool__inject_build_id(const struct perf_tool *tool, 148 struct perf_sample *sample, 149 struct machine *machine, 150 __u16 misc, 151 const char *filename, 152 struct dso *dso, u32 flags); 153 static int tool__inject_mmap2_build_id(const struct perf_tool *tool, 154 struct perf_sample *sample, 155 struct machine *machine, 156 __u16 misc, 157 __u32 pid, __u32 tid, 158 __u64 start, __u64 len, __u64 pgoff, 159 struct dso *dso, 160 __u32 prot, __u32 flags, 161 const char *filename); 162 163 static int output_bytes(struct perf_inject *inject, void *buf, size_t sz) 164 { 165 ssize_t size; 166 167 size = perf_data__write(&inject->output, buf, sz); 168 if (size < 0) 169 return -errno; 170 171 inject->bytes_written += size; 172 return 0; 173 } 174 175 static int perf_event__repipe_synth(const struct perf_tool *tool, 176 union perf_event *event) 177 178 { 179 struct perf_inject *inject = container_of(tool, struct perf_inject, 180 tool); 181 182 return output_bytes(inject, event, event->header.size); 183 } 184 185 static int perf_event__repipe_oe_synth(const struct perf_tool *tool, 186 union perf_event *event, 187 struct ordered_events *oe __maybe_unused) 188 { 189 return perf_event__repipe_synth(tool, event); 190 } 191 192 #ifdef HAVE_JITDUMP 193 static int perf_event__drop_oe(const struct perf_tool *tool __maybe_unused, 194 union perf_event *event __maybe_unused, 195 struct ordered_events *oe __maybe_unused) 196 { 197 return 0; 198 } 199 #endif 200 201 static int perf_event__repipe_op2_synth(const struct perf_tool *tool, 202 struct perf_session *session __maybe_unused, 203 union perf_event *event) 204 { 205 return perf_event__repipe_synth(tool, event); 206 } 207 208 static int perf_event__repipe_op4_synth(const struct perf_tool *tool, 209 struct perf_session *session __maybe_unused, 210 union perf_event *event, 211 u64 data __maybe_unused, 212 const char *str __maybe_unused) 213 { 214 return perf_event__repipe_synth(tool, event); 215 } 216 217 static int perf_event__repipe_synth_cb(const struct perf_tool *tool, 218 union perf_event *event, 219 struct perf_sample *sample __maybe_unused, 220 struct machine *machine __maybe_unused) 221 { 222 return perf_event__repipe_synth(tool, event); 223 } 224 225 static int perf_event__repipe_attr(const struct perf_tool *tool, 226 union perf_event *event, 227 struct evlist **pevlist) 228 { 229 struct perf_inject *inject = container_of(tool, struct perf_inject, 230 tool); 231 struct perf_event_attr attr; 232 u32 raw_attr_size, attr_size; 233 size_t n_ids; 234 u64 *ids; 235 int ret; 236 237 ret = perf_event__process_attr(tool, event, pevlist); 238 if (ret) 239 return ret; 240 241 /* If the output isn't a pipe then the attributes will be written as part of the header. */ 242 if (!inject->output.is_pipe) 243 return 0; 244 245 if (!inject->itrace_synth_opts.set) 246 return perf_event__repipe_synth(tool, event); 247 248 if (event->header.size < sizeof(struct perf_event_header) + PERF_ATTR_SIZE_VER0) { 249 pr_err("Attribute event size %u is too small\n", event->header.size); 250 return -EINVAL; 251 } 252 253 /* 254 * ABI0 pipe/inject events have attr.size == 0; default to 255 * PERF_ATTR_SIZE_VER0 (the ABI0 footprint) for the bounded 256 * copy and ID array position. Same pattern as 257 * perf_event__process_attr() in header.c. 258 */ 259 raw_attr_size = event->attr.attr.size; 260 attr_size = raw_attr_size ?: PERF_ATTR_SIZE_VER0; 261 262 if (raw_attr_size && (raw_attr_size < PERF_ATTR_SIZE_VER0 || 263 raw_attr_size > event->header.size - sizeof(event->header))) { 264 pr_err("Attribute event size %u is too small for attr.size %u\n", 265 event->header.size, raw_attr_size); 266 return -EINVAL; 267 } 268 269 memset(&attr, 0, sizeof(attr)); 270 memcpy(&attr, &event->attr.attr, 271 min_t(size_t, sizeof(attr), attr_size)); 272 273 n_ids = event->header.size - sizeof(event->header) - attr_size; 274 n_ids /= sizeof(u64); 275 ids = (void *)&event->attr.attr + attr_size; 276 277 attr.size = sizeof(struct perf_event_attr); 278 attr.sample_type &= ~PERF_SAMPLE_AUX; 279 280 if (inject->itrace_synth_opts.add_last_branch) { 281 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; 282 attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; 283 } 284 return perf_event__synthesize_attr(tool, &attr, (u32)n_ids, ids, 285 perf_event__repipe_synth_cb); 286 } 287 288 static int perf_event__repipe_event_update(const struct perf_tool *tool, 289 union perf_event *event, 290 struct evlist **pevlist __maybe_unused) 291 { 292 return perf_event__repipe_synth(tool, event); 293 } 294 295 static int copy_bytes(struct perf_inject *inject, struct perf_data *data, off_t size) 296 { 297 char buf[4096]; 298 ssize_t ssz; 299 int ret; 300 301 while (size > 0) { 302 ssz = perf_data__read(data, buf, min(size, (off_t)sizeof(buf))); 303 if (ssz < 0) 304 return -errno; 305 ret = output_bytes(inject, buf, ssz); 306 if (ret) 307 return ret; 308 size -= ssz; 309 } 310 311 return 0; 312 } 313 314 static s64 perf_event__repipe_auxtrace(const struct perf_tool *tool, 315 struct perf_session *session, 316 union perf_event *event) 317 { 318 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 319 int ret; 320 321 inject->have_auxtrace = true; 322 323 if (!inject->output.is_pipe) { 324 off_t offset = perf_data__seek(&inject->output, 0, SEEK_CUR); 325 326 if (offset == -1) 327 return -errno; 328 ret = auxtrace_index__auxtrace_event(&session->auxtrace_index, 329 event, offset); 330 if (ret < 0) 331 return ret; 332 } 333 334 if (perf_data__is_pipe(session->data) || !session->one_mmap) { 335 ret = output_bytes(inject, event, event->header.size); 336 if (ret < 0) 337 return ret; 338 ret = copy_bytes(inject, session->data, 339 event->auxtrace.size); 340 } else { 341 ret = output_bytes(inject, event, 342 event->header.size + event->auxtrace.size); 343 } 344 if (ret < 0) 345 return ret; 346 347 return event->auxtrace.size; 348 } 349 350 static int perf_event__repipe(const struct perf_tool *tool, 351 union perf_event *event, 352 struct perf_sample *sample __maybe_unused, 353 struct machine *machine __maybe_unused) 354 { 355 return perf_event__repipe_synth(tool, event); 356 } 357 358 static int perf_event__drop(const struct perf_tool *tool __maybe_unused, 359 union perf_event *event __maybe_unused, 360 struct perf_sample *sample __maybe_unused, 361 struct machine *machine __maybe_unused) 362 { 363 return 0; 364 } 365 366 static int perf_event__drop_aux(const struct perf_tool *tool, 367 union perf_event *event __maybe_unused, 368 struct perf_sample *sample, 369 struct machine *machine __maybe_unused) 370 { 371 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 372 373 if (!inject->aux_id) 374 inject->aux_id = sample->id; 375 376 return 0; 377 } 378 379 static union perf_event * 380 perf_inject__cut_auxtrace_sample(struct perf_inject *inject, 381 union perf_event *event, 382 struct perf_sample *sample) 383 { 384 size_t sz1 = sample->aux_sample.data - (void *)event - sizeof(u64); 385 size_t sz2 = event->header.size - sample->aux_sample.size - (sz1 + sizeof(u64)); 386 union perf_event *ev; 387 388 if (inject->event_copy == NULL) { 389 inject->event_copy = malloc(PERF_SAMPLE_MAX_SIZE); 390 if (!inject->event_copy) 391 return ERR_PTR(-ENOMEM); 392 } 393 ev = (union perf_event *)inject->event_copy; 394 if (sz1 > event->header.size || sz2 > event->header.size || 395 sz1 + sz2 > event->header.size || 396 sz1 < sizeof(struct perf_event_header)) 397 return event; 398 399 memcpy(ev, event, sz1); 400 memcpy((void *)ev + sz1, (void *)event + event->header.size - sz2, sz2); 401 ev->header.size = sz1 + sz2; 402 403 return ev; 404 } 405 406 typedef int (*inject_handler)(const struct perf_tool *tool, 407 union perf_event *event, 408 struct perf_sample *sample, 409 struct machine *machine); 410 411 static int perf_event__repipe_sample(const struct perf_tool *tool, 412 union perf_event *event, 413 struct perf_sample *sample, 414 struct machine *machine) 415 { 416 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 417 struct evsel *evsel = sample->evsel; 418 419 if (evsel == NULL) 420 return perf_event__repipe_synth(tool, event); 421 422 if (evsel->handler) { 423 inject_handler f = evsel->handler; 424 return f(tool, event, sample, machine); 425 } 426 427 build_id__mark_dso_hit(tool, event, sample, machine); 428 429 if (inject->itrace_synth_opts.set && 430 (inject->itrace_synth_opts.last_branch || 431 inject->itrace_synth_opts.add_last_branch)) { 432 union perf_event *event_copy = (void *)inject->event_copy; 433 struct branch_stack dummy_bs = { .nr = 0, .hw_idx = 0 }; 434 int err; 435 size_t sz; 436 u64 orig_type = evsel->core.attr.sample_type; 437 u64 orig_branch_type = evsel->core.attr.branch_sample_type; 438 439 struct branch_stack *orig_bs = sample->branch_stack; 440 441 if (event_copy == NULL) { 442 inject->event_copy = malloc(PERF_SAMPLE_MAX_SIZE); 443 if (!inject->event_copy) 444 return -ENOMEM; 445 446 event_copy = (void *)inject->event_copy; 447 } 448 449 if (!sample->branch_stack) 450 sample->branch_stack = &dummy_bs; 451 452 if (inject->itrace_synth_opts.add_last_branch) { 453 /* Temporarily add in type bits for synthesis. */ 454 evsel->core.attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; 455 evsel->core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; 456 } 457 evsel->core.attr.sample_type &= ~PERF_SAMPLE_AUX; 458 459 sz = perf_event__sample_event_size(sample, evsel->core.attr.sample_type, 460 evsel->core.attr.read_format, 461 evsel->core.attr.branch_sample_type); 462 463 if (sz >= PERF_SAMPLE_MAX_SIZE) { 464 pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); 465 evsel->core.attr.sample_type = orig_type; 466 evsel->core.attr.branch_sample_type = orig_branch_type; 467 sample->branch_stack = orig_bs; 468 return -EFAULT; 469 } 470 471 event_copy->header.type = PERF_RECORD_SAMPLE; 472 event_copy->header.misc = event->header.misc; 473 event_copy->header.size = sz; 474 475 err = perf_event__synthesize_sample(event_copy, evsel->core.attr.sample_type, 476 evsel->core.attr.read_format, 477 evsel->core.attr.branch_sample_type, sample); 478 479 evsel->core.attr.sample_type = orig_type; 480 evsel->core.attr.branch_sample_type = orig_branch_type; 481 sample->branch_stack = orig_bs; 482 483 if (err) { 484 pr_err("Failed to synthesize sample\n"); 485 return err; 486 } 487 event = event_copy; 488 } else if (inject->itrace_synth_opts.set && 489 (evsel->core.attr.sample_type & PERF_SAMPLE_AUX)) { 490 event = perf_inject__cut_auxtrace_sample(inject, event, sample); 491 if (IS_ERR(event)) 492 return PTR_ERR(event); 493 } 494 495 return perf_event__repipe_synth(tool, event); 496 } 497 498 static int perf_event__convert_sample_callchain(const struct perf_tool *tool, 499 union perf_event *event, 500 struct perf_sample *sample, 501 struct machine *machine) 502 { 503 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 504 struct evsel *evsel = sample->evsel; 505 struct callchain_cursor *cursor = get_tls_callchain_cursor(); 506 union perf_event *event_copy = (void *)inject->event_copy; 507 struct callchain_cursor_node *node; 508 struct thread *thread; 509 u64 sample_type = evsel->core.attr.sample_type; 510 size_t sz; 511 u64 i, k; 512 int ret; 513 514 if (event_copy == NULL) { 515 inject->event_copy = malloc(PERF_SAMPLE_MAX_SIZE); 516 if (!inject->event_copy) 517 return -ENOMEM; 518 519 event_copy = (void *)inject->event_copy; 520 } 521 522 if (cursor == NULL) 523 return -ENOMEM; 524 525 callchain_cursor_reset(cursor); 526 527 thread = machine__find_thread(machine, sample->tid, sample->pid); 528 if (thread == NULL) 529 goto out; 530 531 /* this will parse DWARF using stack and register data */ 532 ret = thread__resolve_callchain(thread, cursor, sample, 533 /*parent=*/NULL, /*root_al=*/NULL, 534 PERF_MAX_STACK_DEPTH); 535 thread__put(thread); 536 if (ret != 0) 537 goto out; 538 539 /* copy kernel callchain and context entries */ 540 for (i = 0; i < sample->callchain->nr; i++) { 541 inject->raw_callchain->ips[i] = sample->callchain->ips[i]; 542 if (sample->callchain->ips[i] == PERF_CONTEXT_USER) { 543 i++; 544 break; 545 } 546 } 547 if (i == 0 || inject->raw_callchain->ips[i - 1] != PERF_CONTEXT_USER) 548 inject->raw_callchain->ips[i++] = PERF_CONTEXT_USER; 549 550 node = cursor->first; 551 for (k = 0; k < cursor->nr && i < PERF_MAX_STACK_DEPTH; k++) { 552 if (!(machine->single_address_space && 553 machine__kernel_ip(machine, node->ip)) && 554 !(node->ms.sym && symbol__inlined(node->ms.sym))) { 555 inject->raw_callchain->ips[i++] = node->ip; 556 } 557 558 node = node->next; 559 } 560 561 inject->raw_callchain->nr = i; 562 sample->callchain = inject->raw_callchain; 563 564 out: 565 memcpy(event_copy, event, sizeof(event->header)); 566 567 /* remove sample_type {STACK,REGS}_USER for synthesize */ 568 sample_type &= ~(PERF_SAMPLE_STACK_USER | PERF_SAMPLE_REGS_USER); 569 570 sz = perf_event__sample_event_size(sample, sample_type, 571 evsel->core.attr.read_format, 572 evsel->core.attr.branch_sample_type); 573 if (sz >= PERF_SAMPLE_MAX_SIZE) { 574 pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); 575 return -EFAULT; 576 } 577 event_copy->header.size = sz; 578 579 ret = perf_event__synthesize_sample(event_copy, sample_type, 580 evsel->core.attr.read_format, 581 evsel->core.attr.branch_sample_type, sample); 582 if (ret) { 583 pr_err("Failed to synthesize sample\n"); 584 return ret; 585 } 586 return perf_event__repipe_synth(tool, event_copy); 587 } 588 589 static struct dso *findnew_dso(int pid, int tid, const char *filename, 590 const struct dso_id *id, struct machine *machine) 591 { 592 struct thread *thread; 593 struct nsinfo *nsi = NULL; 594 struct nsinfo *nnsi; 595 struct dso *dso; 596 bool vdso; 597 598 thread = machine__findnew_thread(machine, pid, tid); 599 if (thread == NULL) { 600 pr_err("cannot find or create a task %d/%d.\n", tid, pid); 601 return NULL; 602 } 603 604 vdso = is_vdso_map(filename); 605 nsi = nsinfo__get(thread__nsinfo(thread)); 606 607 if (vdso) { 608 /* The vdso maps are always on the host and not the 609 * container. Ensure that we don't use setns to look 610 * them up. 611 */ 612 nnsi = nsinfo__copy(nsi); 613 if (nnsi) { 614 nsinfo__put(nsi); 615 nsinfo__clear_need_setns(nnsi); 616 nsi = nnsi; 617 } 618 dso = machine__findnew_vdso(machine, thread); 619 } else { 620 dso = machine__findnew_dso_id(machine, filename, id); 621 } 622 623 if (dso) { 624 mutex_lock(dso__lock(dso)); 625 dso__set_nsinfo(dso, nsi); 626 mutex_unlock(dso__lock(dso)); 627 } else 628 nsinfo__put(nsi); 629 630 thread__put(thread); 631 return dso; 632 } 633 634 /* 635 * The evsel used for the sample ID for mmap events. Typically stashed when 636 * processing mmap events. If not stashed, search the evlist for the first mmap 637 * gathering event. 638 */ 639 static struct evsel *inject__mmap_evsel(struct perf_inject *inject) 640 { 641 struct evsel *pos; 642 643 if (inject->mmap_evsel) 644 return inject->mmap_evsel; 645 646 evlist__for_each_entry(inject->session->evlist, pos) { 647 if (pos->core.attr.mmap) { 648 inject->mmap_evsel = pos; 649 return pos; 650 } 651 } 652 pr_err("No mmap events found\n"); 653 return NULL; 654 } 655 656 static int perf_event__repipe_common_mmap(const struct perf_tool *tool, 657 union perf_event *event, 658 struct perf_sample *sample, 659 struct machine *machine, 660 __u32 pid, __u32 tid, 661 __u64 start, __u64 len, __u64 pgoff, 662 __u32 flags, __u32 prot, 663 const char *filename, 664 const struct dso_id *dso_id, 665 int (*perf_event_process)(const struct perf_tool *tool, 666 union perf_event *event, 667 struct perf_sample *sample, 668 struct machine *machine)) 669 { 670 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 671 struct dso *dso = NULL; 672 bool dso_sought = false; 673 674 #ifdef HAVE_JITDUMP 675 if (inject->jit_mode) { 676 u64 n = 0; 677 int ret; 678 679 /* If jit marker, then inject jit mmaps and generate ELF images. */ 680 ret = jit_process(inject->session, &inject->output, machine, 681 filename, pid, tid, &n); 682 if (ret < 0) 683 return ret; 684 if (ret) { 685 inject->bytes_written += n; 686 return 0; 687 } 688 } 689 #endif 690 if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) { 691 dso = findnew_dso(pid, tid, filename, dso_id, machine); 692 dso_sought = true; 693 if (dso) { 694 /* mark it not to inject build-id */ 695 dso__set_hit(dso); 696 } 697 } 698 if (inject->build_id_style == BID_RWS__INJECT_HEADER_ALL) { 699 if (!dso_sought) { 700 dso = findnew_dso(pid, tid, filename, dso_id, machine); 701 dso_sought = true; 702 } 703 704 if (dso && !dso__hit(dso)) { 705 if (!sample->evsel) 706 sample->evsel = evlist__event2evsel(inject->session->evlist, event); 707 708 if (sample->evsel) { 709 dso__set_hit(dso); 710 tool__inject_build_id(tool, sample, machine, 711 /*misc=*/sample->cpumode, 712 filename, dso, flags); 713 } 714 } 715 } else { 716 int err; 717 718 /* 719 * Remember the evsel for lazy build id generation. It is used 720 * for the sample id header type. 721 */ 722 if ((inject->build_id_style == BID_RWS__INJECT_HEADER_LAZY || 723 inject->build_id_style == BID_RWS__MMAP2_BUILDID_LAZY) && 724 !inject->mmap_evsel) 725 inject->mmap_evsel = evlist__event2evsel(inject->session->evlist, event); 726 727 /* Create the thread, map, etc. Not done for the unordered inject all case. */ 728 err = perf_event_process(tool, event, sample, machine); 729 730 if (err) { 731 dso__put(dso); 732 return err; 733 } 734 } 735 if ((inject->build_id_style == BID_RWS__MMAP2_BUILDID_ALL) && 736 !(event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID)) { 737 struct evsel *saved_evsel = sample->evsel; 738 739 sample->evsel = evlist__event2evsel(inject->session->evlist, event); 740 if (sample->evsel && !dso_sought) { 741 dso = findnew_dso(pid, tid, filename, dso_id, machine); 742 dso_sought = true; 743 } 744 if (sample->evsel && dso && 745 !tool__inject_mmap2_build_id(tool, sample, machine, 746 sample->cpumode | PERF_RECORD_MISC_MMAP_BUILD_ID, 747 pid, tid, start, len, pgoff, 748 dso, 749 prot, flags, 750 filename)) { 751 /* Injected mmap2 so no need to repipe. */ 752 sample->evsel = saved_evsel; 753 dso__put(dso); 754 return 0; 755 } 756 sample->evsel = saved_evsel; 757 } 758 dso__put(dso); 759 if (inject->build_id_style == BID_RWS__MMAP2_BUILDID_LAZY) 760 return 0; 761 762 return perf_event__repipe(tool, event, sample, machine); 763 } 764 765 static int perf_event__repipe_mmap(const struct perf_tool *tool, 766 union perf_event *event, 767 struct perf_sample *sample, 768 struct machine *machine) 769 { 770 return perf_event__repipe_common_mmap( 771 tool, event, sample, machine, 772 event->mmap.pid, event->mmap.tid, 773 event->mmap.start, event->mmap.len, event->mmap.pgoff, 774 /*flags=*/0, PROT_EXEC, 775 event->mmap.filename, /*dso_id=*/NULL, 776 perf_event__process_mmap); 777 } 778 779 static int perf_event__repipe_mmap2(const struct perf_tool *tool, 780 union perf_event *event, 781 struct perf_sample *sample, 782 struct machine *machine) 783 { 784 struct dso_id id = dso_id_empty; 785 786 if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) { 787 build_id__init(&id.build_id, event->mmap2.build_id, event->mmap2.build_id_size); 788 } else { 789 id.maj = event->mmap2.maj; 790 id.min = event->mmap2.min; 791 id.ino = event->mmap2.ino; 792 id.ino_generation = event->mmap2.ino_generation; 793 id.mmap2_valid = true; 794 id.mmap2_ino_generation_valid = true; 795 } 796 797 return perf_event__repipe_common_mmap( 798 tool, event, sample, machine, 799 event->mmap2.pid, event->mmap2.tid, 800 event->mmap2.start, event->mmap2.len, event->mmap2.pgoff, 801 event->mmap2.flags, event->mmap2.prot, 802 event->mmap2.filename, &id, 803 perf_event__process_mmap2); 804 } 805 806 static int perf_event__repipe_fork(const struct perf_tool *tool, 807 union perf_event *event, 808 struct perf_sample *sample, 809 struct machine *machine) 810 { 811 int err; 812 813 err = perf_event__process_fork(tool, event, sample, machine); 814 perf_event__repipe(tool, event, sample, machine); 815 816 return err; 817 } 818 819 static int perf_event__repipe_comm(const struct perf_tool *tool, 820 union perf_event *event, 821 struct perf_sample *sample, 822 struct machine *machine) 823 { 824 int err; 825 826 err = perf_event__process_comm(tool, event, sample, machine); 827 perf_event__repipe(tool, event, sample, machine); 828 829 return err; 830 } 831 832 static int perf_event__repipe_namespaces(const struct perf_tool *tool, 833 union perf_event *event, 834 struct perf_sample *sample, 835 struct machine *machine) 836 { 837 int err = perf_event__process_namespaces(tool, event, sample, machine); 838 839 perf_event__repipe(tool, event, sample, machine); 840 841 return err; 842 } 843 844 static int perf_event__repipe_exit(const struct perf_tool *tool, 845 union perf_event *event, 846 struct perf_sample *sample, 847 struct machine *machine) 848 { 849 int err; 850 851 err = perf_event__process_exit(tool, event, sample, machine); 852 perf_event__repipe(tool, event, sample, machine); 853 854 return err; 855 } 856 857 #ifdef HAVE_LIBTRACEEVENT 858 static int perf_event__repipe_tracing_data(const struct perf_tool *tool, 859 struct perf_session *session, 860 union perf_event *event) 861 { 862 perf_event__repipe_synth(tool, event); 863 864 return perf_event__process_tracing_data(tool, session, event); 865 } 866 #endif 867 868 static int dso__read_build_id(struct dso *dso) 869 { 870 struct nscookie nsc; 871 struct build_id bid = { .size = 0, }; 872 873 if (dso__has_build_id(dso)) 874 return 0; 875 876 mutex_lock(dso__lock(dso)); 877 nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); 878 if (filename__read_build_id(dso__long_name(dso), &bid) > 0) 879 dso__set_build_id(dso, &bid); 880 else if (dso__nsinfo(dso)) { 881 char *new_name = dso__filename_with_chroot(dso, dso__long_name(dso)); 882 883 if (new_name && filename__read_build_id(new_name, &bid) > 0) 884 dso__set_build_id(dso, &bid); 885 free(new_name); 886 } 887 nsinfo__mountns_exit(&nsc); 888 mutex_unlock(dso__lock(dso)); 889 890 return dso__has_build_id(dso) ? 0 : -1; 891 } 892 893 static struct strlist *perf_inject__parse_known_build_ids( 894 const char *known_build_ids_string) 895 { 896 struct str_node *pos, *tmp; 897 struct strlist *known_build_ids; 898 int bid_len; 899 900 known_build_ids = strlist__new(known_build_ids_string, NULL); 901 if (known_build_ids == NULL) 902 return NULL; 903 strlist__for_each_entry_safe(pos, tmp, known_build_ids) { 904 const char *build_id, *dso_name; 905 906 build_id = skip_spaces(pos->s); 907 dso_name = strchr(build_id, ' '); 908 if (dso_name == NULL) { 909 strlist__remove(known_build_ids, pos); 910 continue; 911 } 912 bid_len = dso_name - pos->s; 913 dso_name = skip_spaces(dso_name); 914 if (bid_len % 2 != 0 || bid_len >= SBUILD_ID_SIZE) { 915 strlist__remove(known_build_ids, pos); 916 continue; 917 } 918 for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) { 919 if (!isxdigit(build_id[2 * ix]) || 920 !isxdigit(build_id[2 * ix + 1])) { 921 strlist__remove(known_build_ids, pos); 922 break; 923 } 924 } 925 } 926 return known_build_ids; 927 } 928 929 static bool perf_inject__lookup_known_build_id(struct perf_inject *inject, 930 struct dso *dso) 931 { 932 struct str_node *pos; 933 934 strlist__for_each_entry(pos, inject->known_build_ids) { 935 struct build_id bid; 936 const char *build_id, *dso_name; 937 size_t bid_len; 938 939 build_id = skip_spaces(pos->s); 940 dso_name = strchr(build_id, ' '); 941 bid_len = dso_name - pos->s; 942 if (bid_len > sizeof(bid.data)) 943 bid_len = sizeof(bid.data); 944 dso_name = skip_spaces(dso_name); 945 if (strcmp(dso__long_name(dso), dso_name)) 946 continue; 947 for (size_t ix = 0; 2 * ix + 1 < bid_len; ++ix) { 948 bid.data[ix] = (hex(build_id[2 * ix]) << 4 | 949 hex(build_id[2 * ix + 1])); 950 } 951 bid.size = bid_len / 2; 952 dso__set_build_id(dso, &bid); 953 return true; 954 } 955 return false; 956 } 957 958 static int tool__inject_build_id(const struct perf_tool *tool, 959 struct perf_sample *sample, 960 struct machine *machine, 961 __u16 misc, 962 const char *filename, 963 struct dso *dso, u32 flags) 964 { 965 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 966 int err; 967 968 if (is_anon_memory(filename) || flags & MAP_HUGETLB) 969 return 0; 970 if (is_no_dso_memory(filename)) 971 return 0; 972 973 if (inject->known_build_ids != NULL && 974 perf_inject__lookup_known_build_id(inject, dso)) 975 return 1; 976 977 if (dso__read_build_id(dso) < 0) { 978 pr_debug("no build_id found for %s\n", filename); 979 return -1; 980 } 981 982 err = perf_event__synthesize_build_id(tool, sample, machine, 983 perf_event__repipe, 984 misc, dso__bid(dso), 985 filename); 986 if (err) { 987 pr_err("Can't synthesize build_id event for %s\n", filename); 988 return -1; 989 } 990 991 return 0; 992 } 993 994 static int tool__inject_mmap2_build_id(const struct perf_tool *tool, 995 struct perf_sample *sample, 996 struct machine *machine, 997 __u16 misc, 998 __u32 pid, __u32 tid, 999 __u64 start, __u64 len, __u64 pgoff, 1000 struct dso *dso, 1001 __u32 prot, __u32 flags, 1002 const char *filename) 1003 { 1004 int err; 1005 1006 /* Return to repipe anonymous maps. */ 1007 if (is_anon_memory(filename) || flags & MAP_HUGETLB) 1008 return 1; 1009 if (is_no_dso_memory(filename)) 1010 return 1; 1011 1012 if (dso__read_build_id(dso)) { 1013 pr_debug("no build_id found for %s\n", filename); 1014 return -1; 1015 } 1016 1017 err = perf_event__synthesize_mmap2_build_id(tool, sample, machine, 1018 perf_event__repipe, 1019 misc, pid, tid, 1020 start, len, pgoff, 1021 dso__bid(dso), 1022 prot, flags, 1023 filename); 1024 if (err) { 1025 pr_err("Can't synthesize build_id event for %s\n", filename); 1026 return -1; 1027 } 1028 return 0; 1029 } 1030 1031 static int mark_dso_hit(const struct perf_inject *inject, 1032 const struct perf_tool *tool, 1033 struct perf_sample *sample, 1034 struct machine *machine, 1035 struct evsel *mmap_evsel, 1036 struct map *map, bool sample_in_dso) 1037 { 1038 struct dso *dso; 1039 u16 misc = sample->cpumode; 1040 1041 if (!map) 1042 return 0; 1043 1044 if (!sample_in_dso) { 1045 u16 guest_mask = PERF_RECORD_MISC_GUEST_KERNEL | 1046 PERF_RECORD_MISC_GUEST_USER; 1047 1048 if ((misc & guest_mask) != 0) { 1049 misc &= PERF_RECORD_MISC_HYPERVISOR; 1050 misc |= __map__is_kernel(map) 1051 ? PERF_RECORD_MISC_GUEST_KERNEL 1052 : PERF_RECORD_MISC_GUEST_USER; 1053 } else { 1054 misc &= PERF_RECORD_MISC_HYPERVISOR; 1055 misc |= __map__is_kernel(map) 1056 ? PERF_RECORD_MISC_KERNEL 1057 : PERF_RECORD_MISC_USER; 1058 } 1059 } 1060 dso = map__dso(map); 1061 if (inject->build_id_style == BID_RWS__INJECT_HEADER_LAZY) { 1062 if (dso && !dso__hit(dso)) { 1063 /* 1064 * The sample is just read for identifiers which we want 1065 * to match the for the event of the sample. 1066 */ 1067 dso__set_hit(dso); 1068 tool__inject_build_id(tool, sample, machine, 1069 misc, dso__long_name(dso), dso, 1070 map__flags(map)); 1071 } 1072 } else if (inject->build_id_style == BID_RWS__MMAP2_BUILDID_LAZY) { 1073 if (!map__hit(map)) { 1074 const struct build_id null_bid = { .size = 0 }; 1075 const struct build_id *bid = dso ? dso__bid(dso) : &null_bid; 1076 const char *filename = dso ? dso__long_name(dso) : ""; 1077 struct evsel *saved_evsel = sample->evsel; 1078 1079 map__set_hit(map); 1080 /* Creating a new mmap2 event which has an evsel for the mmap event. */ 1081 sample->evsel = mmap_evsel; 1082 perf_event__synthesize_mmap2_build_id(tool, sample, machine, 1083 perf_event__repipe, 1084 misc, 1085 sample->pid, sample->tid, 1086 map__start(map), 1087 map__end(map) - map__start(map), 1088 map__pgoff(map), 1089 bid, 1090 map__prot(map), 1091 map__flags(map), 1092 filename); 1093 sample->evsel = saved_evsel; 1094 } 1095 } 1096 return 0; 1097 } 1098 1099 struct mark_dso_hit_args { 1100 const struct perf_inject *inject; 1101 const struct perf_tool *tool; 1102 struct perf_sample *sample; 1103 struct machine *machine; 1104 struct evsel *mmap_evsel; 1105 }; 1106 1107 static int mark_dso_hit_callback(struct callchain_cursor_node *node, void *data) 1108 { 1109 struct mark_dso_hit_args *args = data; 1110 struct map *map = node->ms.map; 1111 1112 return mark_dso_hit(args->inject, args->tool, args->sample, args->machine, 1113 args->mmap_evsel, map, /*sample_in_dso=*/false); 1114 } 1115 1116 static int perf_event__inject_buildid(const struct perf_tool *tool, union perf_event *event, 1117 struct perf_sample *sample, struct machine *machine) 1118 { 1119 struct addr_location al; 1120 struct thread *thread; 1121 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 1122 struct mark_dso_hit_args args = { 1123 .inject = inject, 1124 .tool = tool, 1125 /* 1126 * Use the parsed sample data of the sample event, which will 1127 * have a later timestamp than the mmap event. 1128 */ 1129 .sample = sample, 1130 .machine = machine, 1131 .mmap_evsel = inject__mmap_evsel(inject), 1132 }; 1133 1134 addr_location__init(&al); 1135 thread = machine__findnew_thread(machine, sample->pid, sample->tid); 1136 if (thread == NULL) { 1137 pr_err("problem processing %d event, skipping it.\n", 1138 event->header.type); 1139 goto repipe; 1140 } 1141 1142 if (thread__find_map(thread, sample->cpumode, sample->ip, &al)) { 1143 mark_dso_hit(inject, tool, sample, machine, args.mmap_evsel, al.map, 1144 /*sample_in_dso=*/true); 1145 } 1146 1147 sample__for_each_callchain_node(thread, sample, PERF_MAX_STACK_DEPTH, 1148 /*symbols=*/false, mark_dso_hit_callback, &args); 1149 thread__put(thread); 1150 repipe: 1151 perf_event__repipe(tool, event, sample, machine); 1152 addr_location__exit(&al); 1153 return 0; 1154 } 1155 1156 static int perf_inject__sched_process_exit(const struct perf_tool *tool, 1157 union perf_event *event __maybe_unused, 1158 struct perf_sample *sample, 1159 struct machine *machine __maybe_unused) 1160 { 1161 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 1162 struct event_entry *ent; 1163 1164 list_for_each_entry(ent, &inject->samples, node) { 1165 if (sample->tid == ent->tid) { 1166 list_del_init(&ent->node); 1167 free(ent); 1168 break; 1169 } 1170 } 1171 1172 return 0; 1173 } 1174 1175 static int perf_inject__sched_switch(const struct perf_tool *tool, 1176 union perf_event *event, 1177 struct perf_sample *sample, 1178 struct machine *machine) 1179 { 1180 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 1181 struct event_entry *ent; 1182 1183 perf_inject__sched_process_exit(tool, event, sample, machine); 1184 1185 ent = malloc(event->header.size + sizeof(struct event_entry)); 1186 if (ent == NULL) { 1187 color_fprintf(stderr, PERF_COLOR_RED, 1188 "Not enough memory to process sched switch event!"); 1189 return -1; 1190 } 1191 1192 ent->tid = sample->tid; 1193 memcpy(&ent->event, event, event->header.size); 1194 list_add(&ent->node, &inject->samples); 1195 return 0; 1196 } 1197 1198 #ifdef HAVE_LIBTRACEEVENT 1199 static int perf_inject__sched_stat(const struct perf_tool *tool, 1200 union perf_event *event __maybe_unused, 1201 struct perf_sample *sample, 1202 struct machine *machine) 1203 { 1204 struct event_entry *ent; 1205 union perf_event *event_sw; 1206 struct perf_sample sample_sw; 1207 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 1208 struct evsel *evsel = sample->evsel; 1209 u32 pid = perf_sample__intval(sample, "pid"); 1210 int ret; 1211 1212 list_for_each_entry(ent, &inject->samples, node) { 1213 if (pid == ent->tid) 1214 goto found; 1215 } 1216 1217 return 0; 1218 found: 1219 event_sw = &ent->event[0]; 1220 evsel__parse_sample(evsel, event_sw, &sample_sw); 1221 1222 sample_sw.period = sample->period; 1223 sample_sw.time = sample->time; 1224 perf_event__synthesize_sample(event_sw, evsel->core.attr.sample_type, 1225 evsel->core.attr.read_format, 1226 evsel->core.attr.branch_sample_type, &sample_sw); 1227 build_id__mark_dso_hit(tool, event_sw, &sample_sw, machine); 1228 ret = perf_event__repipe(tool, event_sw, &sample_sw, machine); 1229 perf_sample__exit(&sample_sw); 1230 return ret; 1231 } 1232 #endif 1233 1234 static struct guest_vcpu *guest_session__vcpu(struct guest_session *gs, u32 vcpu) 1235 { 1236 if (realloc_array_as_needed(gs->vcpu, gs->vcpu_cnt, vcpu, NULL)) 1237 return NULL; 1238 return &gs->vcpu[vcpu]; 1239 } 1240 1241 static int guest_session__output_bytes(struct guest_session *gs, void *buf, size_t sz) 1242 { 1243 ssize_t ret = writen(gs->tmp_fd, buf, sz); 1244 1245 return ret < 0 ? ret : 0; 1246 } 1247 1248 static int guest_session__repipe(const struct perf_tool *tool, 1249 union perf_event *event, 1250 struct perf_sample *sample __maybe_unused, 1251 struct machine *machine __maybe_unused) 1252 { 1253 struct guest_session *gs = container_of(tool, struct guest_session, tool); 1254 1255 return guest_session__output_bytes(gs, event, event->header.size); 1256 } 1257 1258 static int guest_session__map_tid(struct guest_session *gs, u32 tid, u32 vcpu) 1259 { 1260 struct guest_tid *guest_tid = zalloc(sizeof(*guest_tid)); 1261 int hash; 1262 1263 if (!guest_tid) 1264 return -ENOMEM; 1265 1266 guest_tid->tid = tid; 1267 guest_tid->vcpu = vcpu; 1268 hash = hash_32(guest_tid->tid, PERF_EVLIST__HLIST_BITS); 1269 hlist_add_head(&guest_tid->node, &gs->tids[hash]); 1270 1271 return 0; 1272 } 1273 1274 static int host_peek_vm_comms_cb(struct perf_session *session __maybe_unused, 1275 union perf_event *event, 1276 u64 offset __maybe_unused, void *data) 1277 { 1278 struct guest_session *gs = data; 1279 unsigned int vcpu; 1280 struct guest_vcpu *guest_vcpu; 1281 int ret; 1282 1283 if (event->header.type != PERF_RECORD_COMM || 1284 event->comm.pid != gs->machine_pid) 1285 return 0; 1286 1287 /* 1288 * QEMU option -name debug-threads=on, causes thread names formatted as 1289 * below, although it is not an ABI. Also libvirt seems to use this by 1290 * default. Here we rely on it to tell us which thread is which VCPU. 1291 */ 1292 ret = sscanf(event->comm.comm, "CPU %u/KVM", &vcpu); 1293 if (ret <= 0) 1294 return ret; 1295 pr_debug("Found VCPU: tid %u comm %s vcpu %u\n", 1296 event->comm.tid, event->comm.comm, vcpu); 1297 if (vcpu > INT_MAX) { 1298 pr_err("Invalid VCPU %u\n", vcpu); 1299 return -EINVAL; 1300 } 1301 guest_vcpu = guest_session__vcpu(gs, vcpu); 1302 if (!guest_vcpu) 1303 return -ENOMEM; 1304 if (guest_vcpu->tid && guest_vcpu->tid != event->comm.tid) { 1305 pr_err("Fatal error: Two threads found with the same VCPU\n"); 1306 return -EINVAL; 1307 } 1308 guest_vcpu->tid = event->comm.tid; 1309 1310 return guest_session__map_tid(gs, event->comm.tid, vcpu); 1311 } 1312 1313 static int host_peek_vm_comms(struct perf_session *session, struct guest_session *gs) 1314 { 1315 return perf_session__peek_events(session, session->header.data_offset, 1316 session->header.data_size, 1317 host_peek_vm_comms_cb, gs); 1318 } 1319 1320 static bool evlist__is_id_used(struct evlist *evlist, u64 id) 1321 { 1322 return evlist__id2sid(evlist, id); 1323 } 1324 1325 static u64 guest_session__allocate_new_id(struct guest_session *gs, struct evlist *host_evlist) 1326 { 1327 do { 1328 gs->highest_id += 1; 1329 } while (!gs->highest_id || evlist__is_id_used(host_evlist, gs->highest_id)); 1330 1331 return gs->highest_id; 1332 } 1333 1334 static int guest_session__map_id(struct guest_session *gs, u64 id, u64 host_id, u32 vcpu) 1335 { 1336 struct guest_id *guest_id = zalloc(sizeof(*guest_id)); 1337 int hash; 1338 1339 if (!guest_id) 1340 return -ENOMEM; 1341 1342 guest_id->id = id; 1343 guest_id->host_id = host_id; 1344 guest_id->vcpu = vcpu; 1345 hash = hash_64(guest_id->id, PERF_EVLIST__HLIST_BITS); 1346 hlist_add_head(&guest_id->node, &gs->heads[hash]); 1347 1348 return 0; 1349 } 1350 1351 static u64 evlist__find_highest_id(struct evlist *evlist) 1352 { 1353 struct evsel *evsel; 1354 u64 highest_id = 1; 1355 1356 evlist__for_each_entry(evlist, evsel) { 1357 u32 j; 1358 1359 for (j = 0; j < evsel->core.ids; j++) { 1360 u64 id = evsel->core.id[j]; 1361 1362 if (id > highest_id) 1363 highest_id = id; 1364 } 1365 } 1366 1367 return highest_id; 1368 } 1369 1370 static int guest_session__map_ids(struct guest_session *gs, struct evlist *host_evlist) 1371 { 1372 struct evlist *evlist = gs->session->evlist; 1373 struct evsel *evsel; 1374 int ret; 1375 1376 evlist__for_each_entry(evlist, evsel) { 1377 u32 j; 1378 1379 for (j = 0; j < evsel->core.ids; j++) { 1380 struct perf_sample_id *sid; 1381 u64 host_id; 1382 u64 id; 1383 1384 id = evsel->core.id[j]; 1385 sid = evlist__id2sid(evlist, id); 1386 if (!sid || sid->cpu.cpu == -1) 1387 continue; 1388 host_id = guest_session__allocate_new_id(gs, host_evlist); 1389 ret = guest_session__map_id(gs, id, host_id, sid->cpu.cpu); 1390 if (ret) 1391 return ret; 1392 } 1393 } 1394 1395 return 0; 1396 } 1397 1398 static struct guest_id *guest_session__lookup_id(struct guest_session *gs, u64 id) 1399 { 1400 struct hlist_head *head; 1401 struct guest_id *guest_id; 1402 int hash; 1403 1404 hash = hash_64(id, PERF_EVLIST__HLIST_BITS); 1405 head = &gs->heads[hash]; 1406 1407 hlist_for_each_entry(guest_id, head, node) 1408 if (guest_id->id == id) 1409 return guest_id; 1410 1411 return NULL; 1412 } 1413 1414 static int process_attr(const struct perf_tool *tool, union perf_event *event, 1415 struct perf_sample *sample __maybe_unused, 1416 struct machine *machine __maybe_unused) 1417 { 1418 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 1419 1420 return perf_event__process_attr(tool, event, &inject->session->evlist); 1421 } 1422 1423 static int guest_session__add_attr(struct guest_session *gs, struct evsel *evsel) 1424 { 1425 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session); 1426 struct perf_event_attr attr = evsel->core.attr; 1427 u64 *id_array; 1428 u32 *vcpu_array; 1429 int ret = -ENOMEM; 1430 u32 i; 1431 1432 id_array = calloc(evsel->core.ids, sizeof(*id_array)); 1433 if (!id_array) 1434 return -ENOMEM; 1435 1436 vcpu_array = calloc(evsel->core.ids, sizeof(*vcpu_array)); 1437 if (!vcpu_array) 1438 goto out; 1439 1440 for (i = 0; i < evsel->core.ids; i++) { 1441 u64 id = evsel->core.id[i]; 1442 struct guest_id *guest_id = guest_session__lookup_id(gs, id); 1443 1444 if (!guest_id) { 1445 pr_err("Failed to find guest id %"PRIu64"\n", id); 1446 ret = -EINVAL; 1447 goto out; 1448 } 1449 id_array[i] = guest_id->host_id; 1450 vcpu_array[i] = guest_id->vcpu; 1451 } 1452 1453 attr.sample_type |= PERF_SAMPLE_IDENTIFIER; 1454 attr.exclude_host = 1; 1455 attr.exclude_guest = 0; 1456 1457 ret = perf_event__synthesize_attr(&inject->tool, &attr, evsel->core.ids, 1458 id_array, process_attr); 1459 if (ret) 1460 pr_err("Failed to add guest attr.\n"); 1461 1462 for (i = 0; i < evsel->core.ids; i++) { 1463 struct perf_sample_id *sid; 1464 u32 vcpu = vcpu_array[i]; 1465 1466 sid = evlist__id2sid(inject->session->evlist, id_array[i]); 1467 /* Guest event is per-thread from the host point of view */ 1468 sid->cpu.cpu = -1; 1469 sid->tid = gs->vcpu[vcpu].tid; 1470 sid->machine_pid = gs->machine_pid; 1471 sid->vcpu.cpu = vcpu; 1472 } 1473 out: 1474 free(vcpu_array); 1475 free(id_array); 1476 return ret; 1477 } 1478 1479 static int guest_session__add_attrs(struct guest_session *gs) 1480 { 1481 struct evlist *evlist = gs->session->evlist; 1482 struct evsel *evsel; 1483 int ret; 1484 1485 evlist__for_each_entry(evlist, evsel) { 1486 ret = guest_session__add_attr(gs, evsel); 1487 if (ret) 1488 return ret; 1489 } 1490 1491 return 0; 1492 } 1493 1494 static int synthesize_id_index(struct perf_inject *inject, size_t new_cnt) 1495 { 1496 struct perf_session *session = inject->session; 1497 struct evlist *evlist = session->evlist; 1498 struct machine *machine = &session->machines.host; 1499 size_t from = evlist->core.nr_entries - new_cnt; 1500 1501 return __perf_event__synthesize_id_index(&inject->tool, perf_event__repipe, 1502 evlist, machine, from); 1503 } 1504 1505 static struct guest_tid *guest_session__lookup_tid(struct guest_session *gs, u32 tid) 1506 { 1507 struct hlist_head *head; 1508 struct guest_tid *guest_tid; 1509 int hash; 1510 1511 hash = hash_32(tid, PERF_EVLIST__HLIST_BITS); 1512 head = &gs->tids[hash]; 1513 1514 hlist_for_each_entry(guest_tid, head, node) 1515 if (guest_tid->tid == tid) 1516 return guest_tid; 1517 1518 return NULL; 1519 } 1520 1521 static bool dso__is_in_kernel_space(struct dso *dso) 1522 { 1523 if (dso__is_vdso(dso)) 1524 return false; 1525 1526 return dso__is_kcore(dso) || 1527 dso__kernel(dso) || 1528 is_kernel_module(dso__long_name(dso), PERF_RECORD_MISC_CPUMODE_UNKNOWN); 1529 } 1530 1531 static u64 evlist__first_id(struct evlist *evlist) 1532 { 1533 struct evsel *evsel; 1534 1535 evlist__for_each_entry(evlist, evsel) { 1536 if (evsel->core.ids) 1537 return evsel->core.id[0]; 1538 } 1539 return 0; 1540 } 1541 1542 static int process_build_id(const struct perf_tool *tool, 1543 union perf_event *event, 1544 struct perf_sample *sample __maybe_unused, 1545 struct machine *machine __maybe_unused) 1546 { 1547 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 1548 1549 return perf_event__process_build_id(tool, inject->session, event); 1550 } 1551 1552 static int synthesize_build_id(struct perf_inject *inject, struct dso *dso, pid_t machine_pid) 1553 { 1554 struct machine *machine = perf_session__findnew_machine(inject->session, machine_pid); 1555 struct perf_sample synth_sample = { 1556 .evsel = inject__mmap_evsel(inject), 1557 .pid = -1, 1558 .tid = -1, 1559 .time = -1, 1560 .stream_id = -1, 1561 .cpu = -1, 1562 .period = 1, 1563 .cpumode = dso__is_in_kernel_space(dso) 1564 ? PERF_RECORD_MISC_GUEST_KERNEL 1565 : PERF_RECORD_MISC_GUEST_USER, 1566 }; 1567 1568 if (!machine) 1569 return -ENOMEM; 1570 1571 dso__set_hit(dso); 1572 1573 return perf_event__synthesize_build_id(&inject->tool, &synth_sample, machine, 1574 process_build_id, 1575 /*misc=*/synth_sample.cpumode, 1576 dso__bid(dso), dso__long_name(dso)); 1577 } 1578 1579 static int guest_session__add_build_ids_cb(struct dso *dso, void *data) 1580 { 1581 struct guest_session *gs = data; 1582 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session); 1583 1584 if (!dso__has_build_id(dso)) 1585 return 0; 1586 1587 return synthesize_build_id(inject, dso, gs->machine_pid); 1588 1589 } 1590 1591 static int guest_session__add_build_ids(struct guest_session *gs) 1592 { 1593 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session); 1594 1595 /* Build IDs will be put in the Build ID feature section */ 1596 perf_header__set_feat(&inject->session->header, HEADER_BUILD_ID); 1597 1598 return dsos__for_each_dso(&gs->session->machines.host.dsos, 1599 guest_session__add_build_ids_cb, 1600 gs); 1601 } 1602 1603 static int guest_session__ksymbol_event(const struct perf_tool *tool, 1604 union perf_event *event, 1605 struct perf_sample *sample __maybe_unused, 1606 struct machine *machine __maybe_unused) 1607 { 1608 struct guest_session *gs = container_of(tool, struct guest_session, tool); 1609 1610 /* Only support out-of-line i.e. no BPF support */ 1611 if (event->ksymbol.ksym_type != PERF_RECORD_KSYMBOL_TYPE_OOL) 1612 return 0; 1613 1614 return guest_session__output_bytes(gs, event, event->header.size); 1615 } 1616 1617 static int guest_session__start(struct guest_session *gs, const char *name, bool force) 1618 { 1619 char tmp_file_name[] = "/tmp/perf-inject-guest_session-XXXXXX"; 1620 struct perf_session *session; 1621 int ret; 1622 1623 /* Only these events will be injected */ 1624 gs->tool.mmap = guest_session__repipe; 1625 gs->tool.mmap2 = guest_session__repipe; 1626 gs->tool.comm = guest_session__repipe; 1627 gs->tool.fork = guest_session__repipe; 1628 gs->tool.exit = guest_session__repipe; 1629 gs->tool.lost = guest_session__repipe; 1630 gs->tool.context_switch = guest_session__repipe; 1631 gs->tool.ksymbol = guest_session__ksymbol_event; 1632 gs->tool.text_poke = guest_session__repipe; 1633 /* 1634 * Processing a build ID creates a struct dso with that build ID. Later, 1635 * all guest dsos are iterated and the build IDs processed into the host 1636 * session where they will be output to the Build ID feature section 1637 * when the perf.data file header is written. 1638 */ 1639 gs->tool.build_id = perf_event__process_build_id; 1640 /* Process the id index to know what VCPU an ID belongs to */ 1641 gs->tool.id_index = perf_event__process_id_index; 1642 1643 gs->tool.ordered_events = true; 1644 gs->tool.ordering_requires_timestamps = true; 1645 1646 gs->data.path = name; 1647 gs->data.force = force; 1648 gs->data.mode = PERF_DATA_MODE_READ; 1649 1650 session = perf_session__new(&gs->data, &gs->tool); 1651 if (IS_ERR(session)) 1652 return PTR_ERR(session); 1653 gs->session = session; 1654 1655 /* 1656 * Initial events have zero'd ID samples. Get default ID sample size 1657 * used for removing them. 1658 */ 1659 gs->dflt_id_hdr_size = session->machines.host.id_hdr_size; 1660 /* And default ID for adding back a host-compatible ID sample */ 1661 gs->dflt_id = evlist__first_id(session->evlist); 1662 if (!gs->dflt_id) { 1663 pr_err("Guest data has no sample IDs"); 1664 return -EINVAL; 1665 } 1666 1667 /* Temporary file for guest events */ 1668 gs->tmp_file_name = strdup(tmp_file_name); 1669 if (!gs->tmp_file_name) 1670 return -ENOMEM; 1671 gs->tmp_fd = mkstemp(gs->tmp_file_name); 1672 if (gs->tmp_fd < 0) 1673 return -errno; 1674 1675 if (zstd_init(&gs->session->zstd_data, 0) < 0) 1676 pr_warning("Guest session decompression initialization failed.\n"); 1677 1678 /* 1679 * perf does not support processing 2 sessions simultaneously, so output 1680 * guest events to a temporary file. 1681 */ 1682 ret = perf_session__process_events(gs->session); 1683 if (ret) 1684 return ret; 1685 1686 if (lseek(gs->tmp_fd, 0, SEEK_SET)) 1687 return -errno; 1688 1689 return 0; 1690 } 1691 1692 /* Free hlist nodes assuming hlist_node is the first member of hlist entries */ 1693 static void free_hlist(struct hlist_head *heads, size_t hlist_sz) 1694 { 1695 struct hlist_node *pos, *n; 1696 size_t i; 1697 1698 for (i = 0; i < hlist_sz; ++i) { 1699 hlist_for_each_safe(pos, n, &heads[i]) { 1700 hlist_del(pos); 1701 free(pos); 1702 } 1703 } 1704 } 1705 1706 static void guest_session__exit(struct guest_session *gs) 1707 { 1708 if (gs->session) { 1709 perf_session__delete(gs->session); 1710 free_hlist(gs->heads, PERF_EVLIST__HLIST_SIZE); 1711 free_hlist(gs->tids, PERF_EVLIST__HLIST_SIZE); 1712 } 1713 if (gs->tmp_file_name) { 1714 if (gs->tmp_fd >= 0) 1715 close(gs->tmp_fd); 1716 unlink(gs->tmp_file_name); 1717 zfree(&gs->tmp_file_name); 1718 } 1719 zfree(&gs->vcpu); 1720 zfree(&gs->perf_data_file); 1721 } 1722 1723 static void get_tsc_conv(struct perf_tsc_conversion *tc, struct perf_record_time_conv *time_conv) 1724 { 1725 tc->time_shift = time_conv->time_shift; 1726 tc->time_mult = time_conv->time_mult; 1727 tc->time_zero = time_conv->time_zero; 1728 tc->time_cycles = time_conv->time_cycles; 1729 tc->time_mask = time_conv->time_mask; 1730 tc->cap_user_time_zero = time_conv->cap_user_time_zero; 1731 tc->cap_user_time_short = time_conv->cap_user_time_short; 1732 } 1733 1734 static void guest_session__get_tc(struct guest_session *gs) 1735 { 1736 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session); 1737 1738 get_tsc_conv(&gs->host_tc, &inject->session->time_conv); 1739 get_tsc_conv(&gs->guest_tc, &gs->session->time_conv); 1740 } 1741 1742 static void guest_session__convert_time(struct guest_session *gs, u64 guest_time, u64 *host_time) 1743 { 1744 u64 tsc; 1745 1746 if (!guest_time) { 1747 *host_time = 0; 1748 return; 1749 } 1750 1751 if (gs->guest_tc.cap_user_time_zero) 1752 tsc = perf_time_to_tsc(guest_time, &gs->guest_tc); 1753 else 1754 tsc = guest_time; 1755 1756 /* 1757 * This is the correct order of operations for x86 if the TSC Offset and 1758 * Multiplier values are used. 1759 */ 1760 tsc -= gs->time_offset; 1761 tsc /= gs->time_scale; 1762 1763 if (gs->host_tc.cap_user_time_zero) 1764 *host_time = tsc_to_perf_time(tsc, &gs->host_tc); 1765 else 1766 *host_time = tsc; 1767 } 1768 1769 static int guest_session__fetch(struct guest_session *gs) 1770 { 1771 void *buf; 1772 struct perf_event_header *hdr; 1773 size_t hdr_sz = sizeof(*hdr); 1774 ssize_t ret; 1775 1776 perf_sample__init(&gs->ev.sample, /*all=*/false); 1777 buf = gs->ev.event_buf; 1778 if (!buf) { 1779 buf = malloc(PERF_SAMPLE_MAX_SIZE); 1780 if (!buf) 1781 return -ENOMEM; 1782 gs->ev.event_buf = buf; 1783 } 1784 hdr = buf; 1785 ret = readn(gs->tmp_fd, buf, hdr_sz); 1786 if (ret < 0) 1787 return ret; 1788 1789 if (!ret) { 1790 /* Zero size means EOF */ 1791 hdr->size = 0; 1792 return 0; 1793 } 1794 1795 buf += hdr_sz; 1796 1797 ret = readn(gs->tmp_fd, buf, hdr->size - hdr_sz); 1798 if (ret < 0) 1799 return ret; 1800 1801 gs->ev.event = (union perf_event *)gs->ev.event_buf; 1802 gs->ev.sample.time = 0; 1803 1804 if (hdr->type >= PERF_RECORD_USER_TYPE_START) { 1805 pr_err("Unexpected type fetching guest event"); 1806 return 0; 1807 } 1808 1809 ret = evlist__parse_sample(gs->session->evlist, gs->ev.event, &gs->ev.sample); 1810 if (ret) { 1811 pr_err("Parse failed fetching guest event"); 1812 return ret; 1813 } 1814 1815 if (!gs->have_tc) { 1816 guest_session__get_tc(gs); 1817 gs->have_tc = true; 1818 } 1819 1820 guest_session__convert_time(gs, gs->ev.sample.time, &gs->ev.sample.time); 1821 1822 return 0; 1823 } 1824 1825 static int evlist__append_id_sample(struct evlist *evlist, union perf_event *ev, 1826 const struct perf_sample *sample) 1827 { 1828 struct evsel *evsel; 1829 void *array; 1830 int ret; 1831 1832 evsel = evlist__id2evsel(evlist, sample->id); 1833 array = ev; 1834 1835 if (!evsel) { 1836 pr_err("No evsel for id %"PRIu64"\n", sample->id); 1837 return -EINVAL; 1838 } 1839 1840 array += ev->header.size; 1841 ret = perf_event__synthesize_id_sample(array, evsel->core.attr.sample_type, sample); 1842 if (ret < 0) 1843 return ret; 1844 1845 if (ret & 7) { 1846 pr_err("Bad id sample size %d\n", ret); 1847 return -EINVAL; 1848 } 1849 1850 ev->header.size += ret; 1851 1852 return 0; 1853 } 1854 1855 static int guest_session__inject_events(struct guest_session *gs, u64 timestamp) 1856 { 1857 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session); 1858 int ret; 1859 1860 if (!gs->ready) 1861 return 0; 1862 1863 while (1) { 1864 struct perf_sample *sample; 1865 struct guest_id *guest_id; 1866 union perf_event *ev; 1867 u16 id_hdr_size; 1868 u8 cpumode; 1869 u64 id; 1870 1871 if (!gs->fetched) { 1872 ret = guest_session__fetch(gs); 1873 if (ret) 1874 break; 1875 gs->fetched = true; 1876 } 1877 1878 ev = gs->ev.event; 1879 sample = &gs->ev.sample; 1880 1881 if (!ev->header.size) { 1882 /* EOF */ 1883 perf_sample__exit(&gs->ev.sample); 1884 gs->fetched = false; 1885 ret = 0; 1886 break; 1887 } 1888 if (sample->time > timestamp) { 1889 ret = 0; 1890 break; 1891 } 1892 1893 /* Change cpumode to guest */ 1894 cpumode = ev->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; 1895 if (cpumode & PERF_RECORD_MISC_USER) 1896 cpumode = PERF_RECORD_MISC_GUEST_USER; 1897 else 1898 cpumode = PERF_RECORD_MISC_GUEST_KERNEL; 1899 ev->header.misc &= ~PERF_RECORD_MISC_CPUMODE_MASK; 1900 ev->header.misc |= cpumode; 1901 1902 id = sample->id; 1903 if (!id) { 1904 id = gs->dflt_id; 1905 id_hdr_size = gs->dflt_id_hdr_size; 1906 } else { 1907 struct evsel *evsel = evlist__id2evsel(gs->session->evlist, id); 1908 1909 id_hdr_size = evsel__id_hdr_size(evsel); 1910 } 1911 1912 if (id_hdr_size & 7) { 1913 pr_err("Bad id_hdr_size %u\n", id_hdr_size); 1914 ret = -EINVAL; 1915 break; 1916 } 1917 1918 if (ev->header.size & 7) { 1919 pr_err("Bad event size %u\n", ev->header.size); 1920 ret = -EINVAL; 1921 break; 1922 } 1923 1924 /* Remove guest id sample */ 1925 ev->header.size -= id_hdr_size; 1926 1927 if (ev->header.size & 7) { 1928 pr_err("Bad raw event size %u\n", ev->header.size); 1929 ret = -EINVAL; 1930 break; 1931 } 1932 1933 guest_id = guest_session__lookup_id(gs, id); 1934 if (!guest_id) { 1935 pr_err("Guest event with unknown id %llu\n", 1936 (unsigned long long)id); 1937 ret = -EINVAL; 1938 break; 1939 } 1940 1941 /* Change to host ID to avoid conflicting ID values */ 1942 sample->id = guest_id->host_id; 1943 sample->stream_id = guest_id->host_id; 1944 1945 if (sample->cpu != (u32)-1) { 1946 if (sample->cpu >= gs->vcpu_cnt) { 1947 pr_err("Guest event with unknown VCPU %u\n", 1948 sample->cpu); 1949 return -EINVAL; 1950 } 1951 /* Change to host CPU instead of guest VCPU */ 1952 sample->cpu = gs->vcpu[sample->cpu].cpu; 1953 } 1954 1955 /* New id sample with new ID and CPU */ 1956 ret = evlist__append_id_sample(inject->session->evlist, ev, sample); 1957 if (ret) 1958 break; 1959 1960 if (ev->header.size & 7) { 1961 pr_err("Bad new event size %u\n", ev->header.size); 1962 ret = -EINVAL; 1963 break; 1964 } 1965 1966 ret = output_bytes(inject, ev, ev->header.size); 1967 if (ret) 1968 break; 1969 1970 /* Reset for next guest session event fetch. */ 1971 perf_sample__exit(sample); 1972 gs->fetched = false; 1973 } 1974 if (ret && gs->fetched) { 1975 /* Clear saved sample state on error. */ 1976 perf_sample__exit(&gs->ev.sample); 1977 gs->fetched = false; 1978 } 1979 return ret; 1980 } 1981 1982 static int guest_session__flush_events(struct guest_session *gs) 1983 { 1984 return guest_session__inject_events(gs, -1); 1985 } 1986 1987 static int host__repipe(const struct perf_tool *tool, 1988 union perf_event *event, 1989 struct perf_sample *sample, 1990 struct machine *machine) 1991 { 1992 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 1993 int ret; 1994 1995 ret = guest_session__inject_events(&inject->guest_session, sample->time); 1996 if (ret) 1997 return ret; 1998 1999 return perf_event__repipe(tool, event, sample, machine); 2000 } 2001 2002 static int host__finished_init(const struct perf_tool *tool, struct perf_session *session, 2003 union perf_event *event) 2004 { 2005 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 2006 struct guest_session *gs = &inject->guest_session; 2007 int ret; 2008 2009 /* 2010 * Peek through host COMM events to find QEMU threads and the VCPU they 2011 * are running. 2012 */ 2013 ret = host_peek_vm_comms(session, gs); 2014 if (ret) 2015 return ret; 2016 2017 if (!gs->vcpu_cnt) { 2018 pr_err("No VCPU threads found for pid %u\n", gs->machine_pid); 2019 return -EINVAL; 2020 } 2021 2022 /* 2023 * Allocate new (unused) host sample IDs and map them to the guest IDs. 2024 */ 2025 gs->highest_id = evlist__find_highest_id(session->evlist); 2026 ret = guest_session__map_ids(gs, session->evlist); 2027 if (ret) 2028 return ret; 2029 2030 ret = guest_session__add_attrs(gs); 2031 if (ret) 2032 return ret; 2033 2034 ret = synthesize_id_index(inject, gs->session->evlist->core.nr_entries); 2035 if (ret) { 2036 pr_err("Failed to synthesize id_index\n"); 2037 return ret; 2038 } 2039 2040 ret = guest_session__add_build_ids(gs); 2041 if (ret) { 2042 pr_err("Failed to add guest build IDs\n"); 2043 return ret; 2044 } 2045 2046 gs->ready = true; 2047 2048 ret = guest_session__inject_events(gs, 0); 2049 if (ret) 2050 return ret; 2051 2052 return perf_event__repipe_op2_synth(tool, session, event); 2053 } 2054 2055 /* 2056 * Obey finished-round ordering. The FINISHED_ROUND event is first processed 2057 * which flushes host events to file up until the last flush time. Then inject 2058 * guest events up to the same time. Finally write out the FINISHED_ROUND event 2059 * itself. 2060 */ 2061 static int host__finished_round(const struct perf_tool *tool, 2062 union perf_event *event, 2063 struct ordered_events *oe) 2064 { 2065 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 2066 int ret = perf_event__process_finished_round(tool, event, oe); 2067 u64 timestamp = ordered_events__last_flush_time(oe); 2068 2069 if (ret) 2070 return ret; 2071 2072 ret = guest_session__inject_events(&inject->guest_session, timestamp); 2073 if (ret) 2074 return ret; 2075 2076 return perf_event__repipe_oe_synth(tool, event, oe); 2077 } 2078 2079 static int host__context_switch(const struct perf_tool *tool, 2080 union perf_event *event, 2081 struct perf_sample *sample, 2082 struct machine *machine) 2083 { 2084 struct perf_inject *inject = container_of(tool, struct perf_inject, tool); 2085 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; 2086 struct guest_session *gs = &inject->guest_session; 2087 u32 pid = event->context_switch.next_prev_pid; 2088 u32 tid = event->context_switch.next_prev_tid; 2089 struct guest_tid *guest_tid; 2090 u32 vcpu; 2091 2092 if (out || pid != gs->machine_pid) 2093 goto out; 2094 2095 guest_tid = guest_session__lookup_tid(gs, tid); 2096 if (!guest_tid) 2097 goto out; 2098 2099 if (sample->cpu == (u32)-1) { 2100 pr_err("Switch event does not have CPU\n"); 2101 return -EINVAL; 2102 } 2103 2104 vcpu = guest_tid->vcpu; 2105 if (vcpu >= gs->vcpu_cnt) 2106 return -EINVAL; 2107 2108 /* Guest is switching in, record which CPU the VCPU is now running on */ 2109 gs->vcpu[vcpu].cpu = sample->cpu; 2110 out: 2111 return host__repipe(tool, event, sample, machine); 2112 } 2113 2114 static void sig_handler(int sig __maybe_unused) 2115 { 2116 session_done = 1; 2117 } 2118 2119 static int evsel__check_stype(struct evsel *evsel, u64 sample_type, const char *sample_msg) 2120 { 2121 struct perf_event_attr *attr = &evsel->core.attr; 2122 const char *name = evsel__name(evsel); 2123 2124 if (!(attr->sample_type & sample_type)) { 2125 pr_err("Samples for %s event do not have %s attribute set.", 2126 name, sample_msg); 2127 return -EINVAL; 2128 } 2129 2130 return 0; 2131 } 2132 2133 static int drop_sample(const struct perf_tool *tool __maybe_unused, 2134 union perf_event *event __maybe_unused, 2135 struct perf_sample *sample __maybe_unused, 2136 struct machine *machine __maybe_unused) 2137 { 2138 return 0; 2139 } 2140 2141 static void strip_init(struct perf_inject *inject) 2142 { 2143 struct evlist *evlist = inject->session->evlist; 2144 struct evsel *evsel; 2145 2146 inject->tool.context_switch = perf_event__drop; 2147 2148 evlist__for_each_entry(evlist, evsel) 2149 evsel->handler = drop_sample; 2150 } 2151 2152 static int parse_vm_time_correlation(const struct option *opt, const char *str, int unset) 2153 { 2154 struct perf_inject *inject = opt->value; 2155 const char *args; 2156 char *dry_run; 2157 2158 if (unset) 2159 return 0; 2160 2161 inject->itrace_synth_opts.set = true; 2162 inject->itrace_synth_opts.vm_time_correlation = true; 2163 inject->in_place_update = true; 2164 2165 if (!str) 2166 return 0; 2167 2168 dry_run = skip_spaces(str); 2169 if (!strncmp(dry_run, "dry-run", strlen("dry-run"))) { 2170 inject->itrace_synth_opts.vm_tm_corr_dry_run = true; 2171 inject->in_place_update_dry_run = true; 2172 args = dry_run + strlen("dry-run"); 2173 } else { 2174 args = str; 2175 } 2176 2177 inject->itrace_synth_opts.vm_tm_corr_args = strdup(args); 2178 2179 return inject->itrace_synth_opts.vm_tm_corr_args ? 0 : -ENOMEM; 2180 } 2181 2182 static int parse_guest_data(const struct option *opt, const char *str, int unset) 2183 { 2184 struct perf_inject *inject = opt->value; 2185 struct guest_session *gs = &inject->guest_session; 2186 char *tok; 2187 char *s; 2188 2189 if (unset) 2190 return 0; 2191 2192 if (!str) 2193 goto bad_args; 2194 2195 s = strdup(str); 2196 if (!s) 2197 return -ENOMEM; 2198 2199 gs->perf_data_file = strsep(&s, ","); 2200 if (!gs->perf_data_file) 2201 goto bad_args; 2202 2203 gs->copy_kcore_dir = has_kcore_dir(gs->perf_data_file); 2204 if (gs->copy_kcore_dir) 2205 inject->output.is_dir = true; 2206 2207 tok = strsep(&s, ","); 2208 if (!tok) 2209 goto bad_args; 2210 gs->machine_pid = strtoul(tok, NULL, 0); 2211 if (!inject->guest_session.machine_pid) 2212 goto bad_args; 2213 2214 gs->time_scale = 1; 2215 2216 tok = strsep(&s, ","); 2217 if (!tok) 2218 goto out; 2219 gs->time_offset = strtoull(tok, NULL, 0); 2220 2221 tok = strsep(&s, ","); 2222 if (!tok) 2223 goto out; 2224 gs->time_scale = strtod(tok, NULL); 2225 if (!gs->time_scale) 2226 goto bad_args; 2227 out: 2228 return 0; 2229 2230 bad_args: 2231 pr_err("--guest-data option requires guest perf.data file name, " 2232 "guest machine PID, and optionally guest timestamp offset, " 2233 "and guest timestamp scale factor, separated by commas.\n"); 2234 return -1; 2235 } 2236 2237 static int save_section_info_cb(struct perf_file_section *section, 2238 struct perf_header *ph __maybe_unused, 2239 int feat, int fd __maybe_unused, void *data) 2240 { 2241 struct perf_inject *inject = data; 2242 2243 inject->secs[feat] = *section; 2244 return 0; 2245 } 2246 2247 static int save_section_info(struct perf_inject *inject) 2248 { 2249 struct perf_header *header = &inject->session->header; 2250 int fd = perf_data__fd(inject->session->data); 2251 2252 return perf_header__process_sections(header, fd, inject, save_section_info_cb); 2253 } 2254 2255 static bool keep_feat(struct perf_inject *inject, int feat) 2256 { 2257 switch (feat) { 2258 /* Keep original information that describes the machine or software */ 2259 case HEADER_TRACING_DATA: 2260 case HEADER_HOSTNAME: 2261 case HEADER_OSRELEASE: 2262 case HEADER_VERSION: 2263 case HEADER_ARCH: 2264 case HEADER_NRCPUS: 2265 case HEADER_CPUDESC: 2266 case HEADER_CPUID: 2267 case HEADER_TOTAL_MEM: 2268 case HEADER_CPU_TOPOLOGY: 2269 case HEADER_NUMA_TOPOLOGY: 2270 case HEADER_PMU_MAPPINGS: 2271 case HEADER_CACHE: 2272 case HEADER_MEM_TOPOLOGY: 2273 case HEADER_CLOCKID: 2274 case HEADER_BPF_PROG_INFO: 2275 case HEADER_BPF_BTF: 2276 case HEADER_CPU_PMU_CAPS: 2277 case HEADER_CLOCK_DATA: 2278 case HEADER_HYBRID_TOPOLOGY: 2279 case HEADER_PMU_CAPS: 2280 case HEADER_CPU_DOMAIN_INFO: 2281 case HEADER_CLN_SIZE: 2282 return true; 2283 /* Information that can be updated */ 2284 case HEADER_BUILD_ID: 2285 return inject->build_id_style == BID_RWS__NONE; 2286 case HEADER_CMDLINE: 2287 case HEADER_EVENT_DESC: 2288 case HEADER_BRANCH_STACK: 2289 case HEADER_GROUP_DESC: 2290 case HEADER_AUXTRACE: 2291 case HEADER_STAT: 2292 case HEADER_SAMPLE_TIME: 2293 case HEADER_DIR_FORMAT: 2294 case HEADER_COMPRESSED: 2295 default: 2296 return false; 2297 }; 2298 } 2299 2300 static int read_file(int fd, u64 offs, void *buf, size_t sz) 2301 { 2302 ssize_t ret = preadn(fd, buf, sz, offs); 2303 2304 if (ret < 0) 2305 return -errno; 2306 if ((size_t)ret != sz) 2307 return -EINVAL; 2308 return 0; 2309 } 2310 2311 static int feat_copy(struct perf_inject *inject, int feat, struct feat_writer *fw) 2312 { 2313 int fd = perf_data__fd(inject->session->data); 2314 u64 offs = inject->secs[feat].offset; 2315 size_t sz = inject->secs[feat].size; 2316 void *buf = malloc(sz); 2317 int ret; 2318 2319 if (!buf) 2320 return -ENOMEM; 2321 2322 ret = read_file(fd, offs, buf, sz); 2323 if (ret) 2324 goto out_free; 2325 2326 ret = fw->write(fw, buf, sz); 2327 out_free: 2328 free(buf); 2329 return ret; 2330 } 2331 2332 struct inject_fc { 2333 struct feat_copier fc; 2334 struct perf_inject *inject; 2335 }; 2336 2337 static int feat_copy_cb(struct feat_copier *fc, int feat, struct feat_writer *fw) 2338 { 2339 struct inject_fc *inj_fc = container_of(fc, struct inject_fc, fc); 2340 struct perf_inject *inject = inj_fc->inject; 2341 int ret; 2342 2343 if (!inject->secs[feat].offset || 2344 !keep_feat(inject, feat)) 2345 return 0; 2346 2347 ret = feat_copy(inject, feat, fw); 2348 if (ret < 0) 2349 return ret; 2350 2351 return 1; /* Feature section copied */ 2352 } 2353 2354 static int copy_kcore_dir(struct perf_inject *inject) 2355 { 2356 char *cmd; 2357 int ret; 2358 2359 ret = asprintf(&cmd, "cp -r -n %s/kcore_dir* %s >/dev/null 2>&1", 2360 inject->input_name, inject->output.path); 2361 if (ret < 0) 2362 return ret; 2363 pr_debug("%s\n", cmd); 2364 ret = system(cmd); 2365 free(cmd); 2366 return ret; 2367 } 2368 2369 static int guest_session__copy_kcore_dir(struct guest_session *gs) 2370 { 2371 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session); 2372 char *cmd; 2373 int ret; 2374 2375 ret = asprintf(&cmd, "cp -r -n %s/kcore_dir %s/kcore_dir__%u >/dev/null 2>&1", 2376 gs->perf_data_file, inject->output.path, gs->machine_pid); 2377 if (ret < 0) 2378 return ret; 2379 pr_debug("%s\n", cmd); 2380 ret = system(cmd); 2381 free(cmd); 2382 return ret; 2383 } 2384 2385 static int output_fd(struct perf_inject *inject) 2386 { 2387 return inject->in_place_update ? -1 : perf_data__fd(&inject->output); 2388 } 2389 2390 static int __cmd_inject(struct perf_inject *inject) 2391 { 2392 int ret = -EINVAL; 2393 struct guest_session *gs = &inject->guest_session; 2394 struct perf_session *session = inject->session; 2395 int fd = output_fd(inject); 2396 u64 output_data_offset = perf_session__data_offset(session->evlist); 2397 /* 2398 * Pipe input hasn't loaded the attributes and will handle them as 2399 * events. So that the attributes don't overlap the data, write the 2400 * attributes after the data. 2401 */ 2402 bool write_attrs_after_data = !inject->output.is_pipe && inject->session->data->is_pipe; 2403 2404 signal(SIGINT, sig_handler); 2405 2406 if (inject->build_id_style != BID_RWS__NONE || inject->sched_stat || 2407 inject->itrace_synth_opts.set) { 2408 inject->tool.mmap = perf_event__repipe_mmap; 2409 inject->tool.mmap2 = perf_event__repipe_mmap2; 2410 inject->tool.fork = perf_event__repipe_fork; 2411 #ifdef HAVE_LIBTRACEEVENT 2412 inject->tool.tracing_data = perf_event__repipe_tracing_data; 2413 #endif 2414 } 2415 2416 if (inject->build_id_style == BID_RWS__INJECT_HEADER_LAZY || 2417 inject->build_id_style == BID_RWS__MMAP2_BUILDID_LAZY) { 2418 inject->tool.sample = perf_event__inject_buildid; 2419 } else if (inject->sched_stat) { 2420 struct evsel *evsel; 2421 2422 evlist__for_each_entry(session->evlist, evsel) { 2423 const char *name = evsel__name(evsel); 2424 2425 if (!strcmp(name, "sched:sched_switch")) { 2426 if (evsel__check_stype(evsel, PERF_SAMPLE_TID, "TID")) 2427 return -EINVAL; 2428 2429 evsel->handler = perf_inject__sched_switch; 2430 } else if (!strcmp(name, "sched:sched_process_exit")) 2431 evsel->handler = perf_inject__sched_process_exit; 2432 #ifdef HAVE_LIBTRACEEVENT 2433 else if (!strncmp(name, "sched:sched_stat_", 17)) 2434 evsel->handler = perf_inject__sched_stat; 2435 #endif 2436 } 2437 } else if (inject->itrace_synth_opts.vm_time_correlation) { 2438 session->itrace_synth_opts = &inject->itrace_synth_opts; 2439 memset(&inject->tool, 0, sizeof(inject->tool)); 2440 inject->tool.id_index = perf_event__process_id_index; 2441 inject->tool.auxtrace_info = perf_event__process_auxtrace_info; 2442 inject->tool.auxtrace = perf_event__process_auxtrace; 2443 inject->tool.auxtrace_error = perf_event__process_auxtrace_error; 2444 inject->tool.ordered_events = true; 2445 inject->tool.ordering_requires_timestamps = true; 2446 } else if (inject->itrace_synth_opts.set) { 2447 session->itrace_synth_opts = &inject->itrace_synth_opts; 2448 inject->itrace_synth_opts.inject = true; 2449 inject->tool.comm = perf_event__repipe_comm; 2450 inject->tool.namespaces = perf_event__repipe_namespaces; 2451 inject->tool.exit = perf_event__repipe_exit; 2452 inject->tool.id_index = perf_event__process_id_index; 2453 inject->tool.auxtrace_info = perf_event__process_auxtrace_info; 2454 inject->tool.auxtrace = perf_event__process_auxtrace; 2455 inject->tool.aux = perf_event__drop_aux; 2456 inject->tool.itrace_start = perf_event__drop_aux; 2457 inject->tool.aux_output_hw_id = perf_event__drop_aux; 2458 inject->tool.ordered_events = true; 2459 inject->tool.ordering_requires_timestamps = true; 2460 /* Allow space in the header for new attributes */ 2461 output_data_offset = roundup(8192 + session->header.data_offset, 4096); 2462 if (inject->strip) 2463 strip_init(inject); 2464 } else if (gs->perf_data_file) { 2465 char *name = gs->perf_data_file; 2466 2467 /* 2468 * Not strictly necessary, but keep these events in order wrt 2469 * guest events. 2470 */ 2471 inject->tool.mmap = host__repipe; 2472 inject->tool.mmap2 = host__repipe; 2473 inject->tool.comm = host__repipe; 2474 inject->tool.fork = host__repipe; 2475 inject->tool.exit = host__repipe; 2476 inject->tool.lost = host__repipe; 2477 inject->tool.context_switch = host__repipe; 2478 inject->tool.ksymbol = host__repipe; 2479 inject->tool.text_poke = host__repipe; 2480 /* 2481 * Once the host session has initialized, set up sample ID 2482 * mapping and feed in guest attrs, build IDs and initial 2483 * events. 2484 */ 2485 inject->tool.finished_init = host__finished_init; 2486 /* Obey finished round ordering */ 2487 inject->tool.finished_round = host__finished_round; 2488 /* Keep track of which CPU a VCPU is runnng on */ 2489 inject->tool.context_switch = host__context_switch; 2490 /* 2491 * Must order events to be able to obey finished round 2492 * ordering. 2493 */ 2494 inject->tool.ordered_events = true; 2495 inject->tool.ordering_requires_timestamps = true; 2496 /* Set up a separate session to process guest perf.data file */ 2497 ret = guest_session__start(gs, name, session->data->force); 2498 if (ret) { 2499 pr_err("Failed to process %s, error %d\n", name, ret); 2500 return ret; 2501 } 2502 /* Allow space in the header for guest attributes */ 2503 output_data_offset += gs->session->header.data_offset; 2504 output_data_offset = roundup(output_data_offset, 4096); 2505 } else if (inject->convert_callchain) { 2506 inject->tool.sample = perf_event__convert_sample_callchain; 2507 inject->tool.fork = perf_event__repipe_fork; 2508 inject->tool.comm = perf_event__repipe_comm; 2509 inject->tool.exit = perf_event__repipe_exit; 2510 inject->tool.mmap = perf_event__repipe_mmap; 2511 inject->tool.mmap2 = perf_event__repipe_mmap2; 2512 inject->tool.ordered_events = true; 2513 inject->tool.ordering_requires_timestamps = true; 2514 } 2515 2516 if (!inject->itrace_synth_opts.set) 2517 auxtrace_index__free(&session->auxtrace_index); 2518 2519 if (!inject->output.is_pipe && !inject->in_place_update) 2520 lseek(fd, output_data_offset, SEEK_SET); 2521 2522 ret = perf_session__process_events(session); 2523 if (ret) 2524 return ret; 2525 2526 if (gs->session) { 2527 /* 2528 * Remaining guest events have later timestamps. Flush them 2529 * out to file. 2530 */ 2531 ret = guest_session__flush_events(gs); 2532 if (ret) { 2533 pr_err("Failed to flush guest events\n"); 2534 return ret; 2535 } 2536 } 2537 2538 if (!inject->output.is_pipe && !inject->in_place_update) { 2539 struct inject_fc inj_fc = { 2540 .fc.copy = feat_copy_cb, 2541 .inject = inject, 2542 }; 2543 2544 if (inject->build_id_style == BID_RWS__INJECT_HEADER_LAZY || 2545 inject->build_id_style == BID_RWS__INJECT_HEADER_ALL) 2546 perf_header__set_feat(&session->header, HEADER_BUILD_ID); 2547 /* 2548 * Keep all buildids when there is unprocessed AUX data because 2549 * it is not known which ones the AUX trace hits. 2550 */ 2551 if (perf_header__has_feat(&session->header, HEADER_BUILD_ID) && 2552 inject->have_auxtrace && !inject->itrace_synth_opts.set) 2553 perf_session__dsos_hit_all(session); 2554 /* 2555 * The AUX areas have been removed and replaced with 2556 * synthesized hardware events, so clear the feature flag. 2557 */ 2558 if (inject->itrace_synth_opts.set) { 2559 struct evsel *evsel; 2560 2561 perf_header__clear_feat(&session->header, 2562 HEADER_AUXTRACE); 2563 2564 evlist__for_each_entry(session->evlist, evsel) { 2565 evsel->core.attr.sample_type &= ~PERF_SAMPLE_AUX; 2566 } 2567 2568 if (inject->itrace_synth_opts.add_last_branch) { 2569 perf_header__set_feat(&session->header, 2570 HEADER_BRANCH_STACK); 2571 2572 evlist__for_each_entry(session->evlist, evsel) { 2573 evsel->core.attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; 2574 if (evsel->core.attr.size < PERF_ATTR_SIZE_VER2) 2575 evsel->core.attr.size = PERF_ATTR_SIZE_VER2; 2576 evsel->core.attr.branch_sample_type |= 2577 PERF_SAMPLE_BRANCH_HW_INDEX; 2578 } 2579 } 2580 } 2581 2582 /* 2583 * The converted data file won't have stack and registers. 2584 * Update the perf_event_attr to remove them before writing. 2585 */ 2586 if (inject->convert_callchain) { 2587 struct evsel *evsel; 2588 2589 evlist__for_each_entry(session->evlist, evsel) { 2590 evsel__reset_sample_bit(evsel, REGS_USER); 2591 evsel__reset_sample_bit(evsel, STACK_USER); 2592 evsel->core.attr.sample_regs_user = 0; 2593 evsel->core.attr.sample_stack_user = 0; 2594 evsel->core.attr.exclude_callchain_user = 0; 2595 } 2596 } 2597 2598 session->header.data_offset = output_data_offset; 2599 session->header.data_size = inject->bytes_written; 2600 perf_session__inject_header(session, session->evlist, fd, &inj_fc.fc, 2601 write_attrs_after_data); 2602 2603 if (inject->copy_kcore_dir) { 2604 ret = copy_kcore_dir(inject); 2605 if (ret) { 2606 pr_err("Failed to copy kcore\n"); 2607 return ret; 2608 } 2609 } 2610 if (gs->copy_kcore_dir) { 2611 ret = guest_session__copy_kcore_dir(gs); 2612 if (ret) { 2613 pr_err("Failed to copy guest kcore\n"); 2614 return ret; 2615 } 2616 } 2617 } 2618 2619 return ret; 2620 } 2621 2622 static bool evsel__has_dwarf_callchain(struct evsel *evsel) 2623 { 2624 struct perf_event_attr *attr = &evsel->core.attr; 2625 const u64 dwarf_callchain_flags = 2626 PERF_SAMPLE_STACK_USER | PERF_SAMPLE_REGS_USER | PERF_SAMPLE_CALLCHAIN; 2627 2628 if (!attr->exclude_callchain_user) 2629 return false; 2630 2631 return (attr->sample_type & dwarf_callchain_flags) == dwarf_callchain_flags; 2632 } 2633 2634 int cmd_inject(int argc, const char **argv) 2635 { 2636 struct perf_inject inject = { 2637 .input_name = "-", 2638 .samples = LIST_HEAD_INIT(inject.samples), 2639 .output = { 2640 .path = "-", 2641 .mode = PERF_DATA_MODE_WRITE, 2642 .file.use_stdio = true, 2643 }, 2644 }; 2645 struct perf_data data = { 2646 .mode = PERF_DATA_MODE_READ, 2647 .file.use_stdio = true, 2648 }; 2649 int ret; 2650 const char *known_build_ids = NULL; 2651 bool build_ids = false; 2652 bool build_id_all = false; 2653 bool mmap2_build_ids = false; 2654 bool mmap2_build_id_all = false; 2655 2656 struct option options[] = { 2657 OPT_BOOLEAN('b', "build-ids", &build_ids, 2658 "Inject build-ids into the output stream"), 2659 OPT_BOOLEAN(0, "buildid-all", &build_id_all, 2660 "Inject build-ids of all DSOs into the output stream"), 2661 OPT_BOOLEAN('B', "mmap2-buildids", &mmap2_build_ids, 2662 "Drop unused mmap events, make others mmap2 with build IDs"), 2663 OPT_BOOLEAN(0, "mmap2-buildid-all", &mmap2_build_id_all, 2664 "Rewrite all mmap events as mmap2 events with build IDs"), 2665 OPT_STRING(0, "known-build-ids", &known_build_ids, 2666 "buildid path [,buildid path...]", 2667 "build-ids to use for given paths"), 2668 OPT_STRING('i', "input", &inject.input_name, "file", 2669 "input file name"), 2670 OPT_STRING('o', "output", &inject.output.path, "file", 2671 "output file name"), 2672 OPT_BOOLEAN('s', "sched-stat", &inject.sched_stat, 2673 "Merge sched-stat and sched-switch for getting events " 2674 "where and how long tasks slept"), 2675 #ifdef HAVE_JITDUMP 2676 OPT_BOOLEAN('j', "jit", &inject.jit_mode, "merge jitdump files into perf.data file"), 2677 #endif 2678 OPT_INCR('v', "verbose", &verbose, 2679 "be more verbose (show build ids, etc)"), 2680 OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name, 2681 "file", "vmlinux pathname"), 2682 OPT_BOOLEAN(0, "ignore-vmlinux", &symbol_conf.ignore_vmlinux, 2683 "don't load vmlinux even if found"), 2684 OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name, "file", 2685 "kallsyms pathname"), 2686 OPT_BOOLEAN('f', "force", &data.force, "don't complain, do it"), 2687 OPT_CALLBACK_OPTARG(0, "itrace", &inject.itrace_synth_opts, 2688 NULL, "opts", "Instruction Tracing options\n" 2689 ITRACE_HELP, 2690 itrace_parse_synth_opts), 2691 OPT_BOOLEAN(0, "strip", &inject.strip, 2692 "strip non-synthesized events (use with --itrace)"), 2693 OPT_CALLBACK_OPTARG(0, "vm-time-correlation", &inject, NULL, "opts", 2694 "correlate time between VM guests and the host", 2695 parse_vm_time_correlation), 2696 OPT_CALLBACK_OPTARG(0, "guest-data", &inject, NULL, "opts", 2697 "inject events from a guest perf.data file", 2698 parse_guest_data), 2699 OPT_STRING(0, "guestmount", &symbol_conf.guestmount, "directory", 2700 "guest mount directory under which every guest os" 2701 " instance has a subdir"), 2702 OPT_CALLBACK(0, "unwind-style", NULL, "unwind style", 2703 "unwind styles (libdw,libunwind)", 2704 unwind__option), 2705 OPT_BOOLEAN(0, "convert-callchain", &inject.convert_callchain, 2706 "Generate callchains using DWARF and drop register/stack data"), 2707 OPT_END() 2708 }; 2709 const char * const inject_usage[] = { 2710 "perf inject [<options>]", 2711 NULL 2712 }; 2713 bool ordered_events; 2714 2715 if (!inject.itrace_synth_opts.set) { 2716 /* Disable eager loading of kernel symbols that adds overhead to perf inject. */ 2717 symbol_conf.lazy_load_kernel_maps = true; 2718 } 2719 2720 #ifndef HAVE_JITDUMP 2721 set_option_nobuild(options, 'j', "jit", "NO_LIBELF=1", true); 2722 #endif 2723 #ifndef HAVE_LIBDW_SUPPORT 2724 set_option_nobuild(options, 0, "convert-callchain", "NO_LIBDW=1", true); 2725 #endif 2726 argc = parse_options(argc, argv, options, inject_usage, 0); 2727 2728 /* 2729 * Any (unrecognized) arguments left? 2730 */ 2731 if (argc) 2732 usage_with_options(inject_usage, options); 2733 2734 if (inject.strip && !inject.itrace_synth_opts.set) { 2735 pr_err("--strip option requires --itrace option\n"); 2736 return -1; 2737 } 2738 2739 if (symbol__validate_sym_arguments()) 2740 return -1; 2741 2742 if (inject.in_place_update) { 2743 if (!strcmp(inject.input_name, "-")) { 2744 pr_err("Input file name required for in-place updating\n"); 2745 return -1; 2746 } 2747 if (strcmp(inject.output.path, "-")) { 2748 pr_err("Output file name must not be specified for in-place updating\n"); 2749 return -1; 2750 } 2751 if (!data.force && !inject.in_place_update_dry_run) { 2752 pr_err("The input file would be updated in place, " 2753 "the --force option is required.\n"); 2754 return -1; 2755 } 2756 if (!inject.in_place_update_dry_run) 2757 data.in_place_update = true; 2758 } else { 2759 if (strcmp(inject.output.path, "-") && !inject.strip && 2760 has_kcore_dir(inject.input_name)) { 2761 inject.output.is_dir = true; 2762 inject.copy_kcore_dir = true; 2763 } 2764 if (perf_data__open(&inject.output)) { 2765 perror("failed to create output file"); 2766 return -1; 2767 } 2768 } 2769 if (mmap2_build_ids) 2770 inject.build_id_style = BID_RWS__MMAP2_BUILDID_LAZY; 2771 if (mmap2_build_id_all) 2772 inject.build_id_style = BID_RWS__MMAP2_BUILDID_ALL; 2773 if (build_ids) 2774 inject.build_id_style = BID_RWS__INJECT_HEADER_LAZY; 2775 if (build_id_all) 2776 inject.build_id_style = BID_RWS__INJECT_HEADER_ALL; 2777 2778 data.path = inject.input_name; 2779 2780 ordered_events = inject.jit_mode || inject.sched_stat || 2781 inject.build_id_style == BID_RWS__INJECT_HEADER_LAZY || 2782 inject.build_id_style == BID_RWS__MMAP2_BUILDID_LAZY; 2783 perf_tool__init(&inject.tool, ordered_events); 2784 inject.tool.sample = perf_event__repipe_sample; 2785 inject.tool.read = perf_event__repipe_sample; 2786 inject.tool.mmap = perf_event__repipe; 2787 inject.tool.mmap2 = perf_event__repipe; 2788 inject.tool.comm = perf_event__repipe; 2789 inject.tool.namespaces = perf_event__repipe; 2790 inject.tool.cgroup = perf_event__repipe; 2791 inject.tool.fork = perf_event__repipe; 2792 inject.tool.exit = perf_event__repipe; 2793 inject.tool.lost = perf_event__repipe; 2794 inject.tool.lost_samples = perf_event__repipe; 2795 inject.tool.aux = perf_event__repipe; 2796 inject.tool.itrace_start = perf_event__repipe; 2797 inject.tool.aux_output_hw_id = perf_event__repipe; 2798 inject.tool.context_switch = perf_event__repipe; 2799 inject.tool.throttle = perf_event__repipe; 2800 inject.tool.unthrottle = perf_event__repipe; 2801 inject.tool.ksymbol = perf_event__repipe; 2802 inject.tool.bpf = perf_event__repipe; 2803 inject.tool.text_poke = perf_event__repipe; 2804 inject.tool.attr = perf_event__repipe_attr; 2805 inject.tool.event_update = perf_event__repipe_event_update; 2806 inject.tool.tracing_data = perf_event__repipe_op2_synth; 2807 inject.tool.finished_round = perf_event__repipe_oe_synth; 2808 inject.tool.build_id = perf_event__repipe_op2_synth; 2809 inject.tool.id_index = perf_event__repipe_op2_synth; 2810 inject.tool.auxtrace_info = perf_event__repipe_op2_synth; 2811 inject.tool.auxtrace_error = perf_event__repipe_op2_synth; 2812 inject.tool.time_conv = perf_event__repipe_op2_synth; 2813 inject.tool.thread_map = perf_event__repipe_op2_synth; 2814 inject.tool.cpu_map = perf_event__repipe_op2_synth; 2815 inject.tool.stat_config = perf_event__repipe_op2_synth; 2816 inject.tool.stat = perf_event__repipe_op2_synth; 2817 inject.tool.stat_round = perf_event__repipe_op2_synth; 2818 inject.tool.feature = perf_event__repipe_op2_synth; 2819 inject.tool.finished_init = perf_event__repipe_op2_synth; 2820 inject.tool.compressed = perf_event__repipe_op4_synth; 2821 inject.tool.auxtrace = perf_event__repipe_auxtrace; 2822 inject.tool.bpf_metadata = perf_event__repipe_op2_synth; 2823 inject.tool.schedstat_cpu = perf_event__repipe_op2_synth; 2824 inject.tool.schedstat_domain = perf_event__repipe_op2_synth; 2825 inject.tool.dont_split_sample_group = true; 2826 inject.tool.merge_deferred_callchains = false; 2827 inject.session = __perf_session__new(&data, &inject.tool, 2828 /*trace_event_repipe=*/inject.output.is_pipe, 2829 /*host_env=*/NULL); 2830 2831 if (IS_ERR(inject.session)) { 2832 ret = PTR_ERR(inject.session); 2833 goto out_close_output; 2834 } 2835 2836 if (zstd_init(&(inject.session->zstd_data), 0) < 0) 2837 pr_warning("Decompression initialization failed.\n"); 2838 2839 /* Save original section info before feature bits change */ 2840 ret = save_section_info(&inject); 2841 if (ret) 2842 goto out_delete; 2843 2844 if (inject.output.is_pipe) { 2845 ret = perf_header__write_pipe(perf_data__fd(&inject.output)); 2846 if (ret < 0) { 2847 pr_err("Couldn't write a new pipe header.\n"); 2848 goto out_delete; 2849 } 2850 2851 /* 2852 * If the input is already a pipe then the features and 2853 * attributes don't need synthesizing, they will be present in 2854 * the input. 2855 */ 2856 if (!data.is_pipe) { 2857 ret = perf_event__synthesize_for_pipe(&inject.tool, 2858 inject.session, 2859 &inject.output, 2860 perf_event__repipe); 2861 if (ret < 0) 2862 goto out_delete; 2863 } 2864 } 2865 2866 if (inject.build_id_style == BID_RWS__INJECT_HEADER_LAZY || 2867 inject.build_id_style == BID_RWS__MMAP2_BUILDID_LAZY) { 2868 /* 2869 * to make sure the mmap records are ordered correctly 2870 * and so that the correct especially due to jitted code 2871 * mmaps. We cannot generate the buildid hit list and 2872 * inject the jit mmaps at the same time for now. 2873 */ 2874 inject.tool.ordering_requires_timestamps = true; 2875 } 2876 if (inject.build_id_style != BID_RWS__NONE && known_build_ids != NULL) { 2877 inject.known_build_ids = 2878 perf_inject__parse_known_build_ids(known_build_ids); 2879 2880 if (inject.known_build_ids == NULL) { 2881 pr_err("Couldn't parse known build ids.\n"); 2882 goto out_delete; 2883 } 2884 } 2885 2886 if (inject.convert_callchain) { 2887 struct evsel *evsel; 2888 2889 if (inject.output.is_pipe || inject.session->data->is_pipe) { 2890 pr_err("--convert-callchain cannot work with pipe\n"); 2891 goto out_delete; 2892 } 2893 2894 evlist__for_each_entry(inject.session->evlist, evsel) { 2895 if (!evsel__has_dwarf_callchain(evsel) && !evsel__is_dummy_event(evsel)) { 2896 pr_err("--convert-callchain requires DWARF call graph.\n"); 2897 goto out_delete; 2898 } 2899 } 2900 2901 inject.raw_callchain = calloc(PERF_MAX_STACK_DEPTH, sizeof(u64)); 2902 if (inject.raw_callchain == NULL) { 2903 pr_err("callchain allocation failed\n"); 2904 goto out_delete; 2905 } 2906 } 2907 2908 #ifdef HAVE_JITDUMP 2909 if (inject.jit_mode) { 2910 inject.tool.mmap2 = perf_event__repipe_mmap2; 2911 inject.tool.mmap = perf_event__repipe_mmap; 2912 inject.tool.ordering_requires_timestamps = true; 2913 /* 2914 * JIT MMAP injection injects all MMAP events in one go, so it 2915 * does not obey finished_round semantics. 2916 */ 2917 inject.tool.finished_round = perf_event__drop_oe; 2918 } 2919 #endif 2920 ret = symbol__init(perf_session__env(inject.session)); 2921 if (ret < 0) 2922 goto out_delete; 2923 2924 ret = __cmd_inject(&inject); 2925 2926 guest_session__exit(&inject.guest_session); 2927 2928 out_delete: 2929 strlist__delete(inject.known_build_ids); 2930 zstd_fini(&(inject.session->zstd_data)); 2931 perf_session__delete(inject.session); 2932 out_close_output: 2933 if (!inject.in_place_update) 2934 perf_data__close(&inject.output); 2935 free(inject.itrace_synth_opts.vm_tm_corr_args); 2936 free(inject.event_copy); 2937 free(inject.guest_session.ev.event_buf); 2938 free(inject.raw_callchain); 2939 return ret; 2940 } 2941