1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Arm Statistical Profiling Extensions (SPE) support 4 * Copyright (c) 2017-2018, Arm Ltd. 5 */ 6 7 #include <byteswap.h> 8 #include <endian.h> 9 #include <errno.h> 10 #include <inttypes.h> 11 #include <linux/bitops.h> 12 #include <linux/kernel.h> 13 #include <linux/log2.h> 14 #include <linux/types.h> 15 #include <linux/zalloc.h> 16 #include <stdlib.h> 17 #include <unistd.h> 18 19 #include "auxtrace.h" 20 #include "color.h" 21 #include "debug.h" 22 #include "evlist.h" 23 #include "evsel.h" 24 #include "machine.h" 25 #include "session.h" 26 #include "symbol.h" 27 #include "thread.h" 28 #include "thread-stack.h" 29 #include "tsc.h" 30 #include "tool.h" 31 #include "util/synthetic-events.h" 32 33 #include "arm-spe.h" 34 #include "arm-spe-decoder/arm-spe-decoder.h" 35 #include "arm-spe-decoder/arm-spe-pkt-decoder.h" 36 37 #define MAX_TIMESTAMP (~0ULL) 38 39 struct arm_spe { 40 struct auxtrace auxtrace; 41 struct auxtrace_queues queues; 42 struct auxtrace_heap heap; 43 struct itrace_synth_opts synth_opts; 44 u32 auxtrace_type; 45 struct perf_session *session; 46 struct machine *machine; 47 u32 pmu_type; 48 49 struct perf_tsc_conversion tc; 50 51 u8 timeless_decoding; 52 u8 data_queued; 53 54 u64 sample_type; 55 u8 sample_flc; 56 u8 sample_llc; 57 u8 sample_tlb; 58 u8 sample_branch; 59 u8 sample_remote_access; 60 u8 sample_memory; 61 u8 sample_instructions; 62 u64 instructions_sample_period; 63 64 u64 l1d_miss_id; 65 u64 l1d_access_id; 66 u64 llc_miss_id; 67 u64 llc_access_id; 68 u64 tlb_miss_id; 69 u64 tlb_access_id; 70 u64 branch_miss_id; 71 u64 remote_access_id; 72 u64 memory_id; 73 u64 instructions_id; 74 75 u64 kernel_start; 76 77 unsigned long num_events; 78 u8 use_ctx_pkt_for_pid; 79 }; 80 81 struct arm_spe_queue { 82 struct arm_spe *spe; 83 unsigned int queue_nr; 84 struct auxtrace_buffer *buffer; 85 struct auxtrace_buffer *old_buffer; 86 union perf_event *event_buf; 87 bool on_heap; 88 bool done; 89 pid_t pid; 90 pid_t tid; 91 int cpu; 92 struct arm_spe_decoder *decoder; 93 u64 time; 94 u64 timestamp; 95 struct thread *thread; 96 u64 period_instructions; 97 }; 98 99 static void arm_spe_dump(struct arm_spe *spe __maybe_unused, 100 unsigned char *buf, size_t len) 101 { 102 struct arm_spe_pkt packet; 103 size_t pos = 0; 104 int ret, pkt_len, i; 105 char desc[ARM_SPE_PKT_DESC_MAX]; 106 const char *color = PERF_COLOR_BLUE; 107 108 color_fprintf(stdout, color, 109 ". ... ARM SPE data: size %#zx bytes\n", 110 len); 111 112 while (len) { 113 ret = arm_spe_get_packet(buf, len, &packet); 114 if (ret > 0) 115 pkt_len = ret; 116 else 117 pkt_len = 1; 118 printf("."); 119 color_fprintf(stdout, color, " %08x: ", pos); 120 for (i = 0; i < pkt_len; i++) 121 color_fprintf(stdout, color, " %02x", buf[i]); 122 for (; i < 16; i++) 123 color_fprintf(stdout, color, " "); 124 if (ret > 0) { 125 ret = arm_spe_pkt_desc(&packet, desc, 126 ARM_SPE_PKT_DESC_MAX); 127 if (!ret) 128 color_fprintf(stdout, color, " %s\n", desc); 129 } else { 130 color_fprintf(stdout, color, " Bad packet!\n"); 131 } 132 pos += pkt_len; 133 buf += pkt_len; 134 len -= pkt_len; 135 } 136 } 137 138 static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf, 139 size_t len) 140 { 141 printf(".\n"); 142 arm_spe_dump(spe, buf, len); 143 } 144 145 static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data) 146 { 147 struct arm_spe_queue *speq = data; 148 struct auxtrace_buffer *buffer = speq->buffer; 149 struct auxtrace_buffer *old_buffer = speq->old_buffer; 150 struct auxtrace_queue *queue; 151 152 queue = &speq->spe->queues.queue_array[speq->queue_nr]; 153 154 buffer = auxtrace_buffer__next(queue, buffer); 155 /* If no more data, drop the previous auxtrace_buffer and return */ 156 if (!buffer) { 157 if (old_buffer) 158 auxtrace_buffer__drop_data(old_buffer); 159 b->len = 0; 160 return 0; 161 } 162 163 speq->buffer = buffer; 164 165 /* If the aux_buffer doesn't have data associated, try to load it */ 166 if (!buffer->data) { 167 /* get the file desc associated with the perf data file */ 168 int fd = perf_data__fd(speq->spe->session->data); 169 170 buffer->data = auxtrace_buffer__get_data(buffer, fd); 171 if (!buffer->data) 172 return -ENOMEM; 173 } 174 175 b->len = buffer->size; 176 b->buf = buffer->data; 177 178 if (b->len) { 179 if (old_buffer) 180 auxtrace_buffer__drop_data(old_buffer); 181 speq->old_buffer = buffer; 182 } else { 183 auxtrace_buffer__drop_data(buffer); 184 return arm_spe_get_trace(b, data); 185 } 186 187 return 0; 188 } 189 190 static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe, 191 unsigned int queue_nr) 192 { 193 struct arm_spe_params params = { .get_trace = 0, }; 194 struct arm_spe_queue *speq; 195 196 speq = zalloc(sizeof(*speq)); 197 if (!speq) 198 return NULL; 199 200 speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); 201 if (!speq->event_buf) 202 goto out_free; 203 204 speq->spe = spe; 205 speq->queue_nr = queue_nr; 206 speq->pid = -1; 207 speq->tid = -1; 208 speq->cpu = -1; 209 speq->period_instructions = 0; 210 211 /* params set */ 212 params.get_trace = arm_spe_get_trace; 213 params.data = speq; 214 215 /* create new decoder */ 216 speq->decoder = arm_spe_decoder_new(¶ms); 217 if (!speq->decoder) 218 goto out_free; 219 220 return speq; 221 222 out_free: 223 zfree(&speq->event_buf); 224 free(speq); 225 226 return NULL; 227 } 228 229 static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip) 230 { 231 return ip >= spe->kernel_start ? 232 PERF_RECORD_MISC_KERNEL : 233 PERF_RECORD_MISC_USER; 234 } 235 236 static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe, 237 struct auxtrace_queue *queue) 238 { 239 struct arm_spe_queue *speq = queue->priv; 240 pid_t tid; 241 242 tid = machine__get_current_tid(spe->machine, speq->cpu); 243 if (tid != -1) { 244 speq->tid = tid; 245 thread__zput(speq->thread); 246 } else 247 speq->tid = queue->tid; 248 249 if ((!speq->thread) && (speq->tid != -1)) { 250 speq->thread = machine__find_thread(spe->machine, -1, 251 speq->tid); 252 } 253 254 if (speq->thread) { 255 speq->pid = speq->thread->pid_; 256 if (queue->cpu == -1) 257 speq->cpu = speq->thread->cpu; 258 } 259 } 260 261 static int arm_spe_set_tid(struct arm_spe_queue *speq, pid_t tid) 262 { 263 struct arm_spe *spe = speq->spe; 264 int err = machine__set_current_tid(spe->machine, speq->cpu, -1, tid); 265 266 if (err) 267 return err; 268 269 arm_spe_set_pid_tid_cpu(spe, &spe->queues.queue_array[speq->queue_nr]); 270 271 return 0; 272 } 273 274 static void arm_spe_prep_sample(struct arm_spe *spe, 275 struct arm_spe_queue *speq, 276 union perf_event *event, 277 struct perf_sample *sample) 278 { 279 struct arm_spe_record *record = &speq->decoder->record; 280 281 if (!spe->timeless_decoding) 282 sample->time = tsc_to_perf_time(record->timestamp, &spe->tc); 283 284 sample->ip = record->from_ip; 285 sample->cpumode = arm_spe_cpumode(spe, sample->ip); 286 sample->pid = speq->pid; 287 sample->tid = speq->tid; 288 sample->period = 1; 289 sample->cpu = speq->cpu; 290 291 event->sample.header.type = PERF_RECORD_SAMPLE; 292 event->sample.header.misc = sample->cpumode; 293 event->sample.header.size = sizeof(struct perf_event_header); 294 } 295 296 static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type) 297 { 298 event->header.size = perf_event__sample_event_size(sample, type, 0); 299 return perf_event__synthesize_sample(event, type, 0, sample); 300 } 301 302 static inline int 303 arm_spe_deliver_synth_event(struct arm_spe *spe, 304 struct arm_spe_queue *speq __maybe_unused, 305 union perf_event *event, 306 struct perf_sample *sample) 307 { 308 int ret; 309 310 if (spe->synth_opts.inject) { 311 ret = arm_spe__inject_event(event, sample, spe->sample_type); 312 if (ret) 313 return ret; 314 } 315 316 ret = perf_session__deliver_synth_event(spe->session, event, sample); 317 if (ret) 318 pr_err("ARM SPE: failed to deliver event, error %d\n", ret); 319 320 return ret; 321 } 322 323 static int arm_spe__synth_mem_sample(struct arm_spe_queue *speq, 324 u64 spe_events_id, u64 data_src) 325 { 326 struct arm_spe *spe = speq->spe; 327 struct arm_spe_record *record = &speq->decoder->record; 328 union perf_event *event = speq->event_buf; 329 struct perf_sample sample = { .ip = 0, }; 330 331 arm_spe_prep_sample(spe, speq, event, &sample); 332 333 sample.id = spe_events_id; 334 sample.stream_id = spe_events_id; 335 sample.addr = record->virt_addr; 336 sample.phys_addr = record->phys_addr; 337 sample.data_src = data_src; 338 sample.weight = record->latency; 339 340 return arm_spe_deliver_synth_event(spe, speq, event, &sample); 341 } 342 343 static int arm_spe__synth_branch_sample(struct arm_spe_queue *speq, 344 u64 spe_events_id) 345 { 346 struct arm_spe *spe = speq->spe; 347 struct arm_spe_record *record = &speq->decoder->record; 348 union perf_event *event = speq->event_buf; 349 struct perf_sample sample = { .ip = 0, }; 350 351 arm_spe_prep_sample(spe, speq, event, &sample); 352 353 sample.id = spe_events_id; 354 sample.stream_id = spe_events_id; 355 sample.addr = record->to_ip; 356 sample.weight = record->latency; 357 358 return arm_spe_deliver_synth_event(spe, speq, event, &sample); 359 } 360 361 static int arm_spe__synth_instruction_sample(struct arm_spe_queue *speq, 362 u64 spe_events_id, u64 data_src) 363 { 364 struct arm_spe *spe = speq->spe; 365 struct arm_spe_record *record = &speq->decoder->record; 366 union perf_event *event = speq->event_buf; 367 struct perf_sample sample = { .ip = 0, }; 368 369 /* 370 * Handles perf instruction sampling period. 371 */ 372 speq->period_instructions++; 373 if (speq->period_instructions < spe->instructions_sample_period) 374 return 0; 375 speq->period_instructions = 0; 376 377 arm_spe_prep_sample(spe, speq, event, &sample); 378 379 sample.id = spe_events_id; 380 sample.stream_id = spe_events_id; 381 sample.addr = record->virt_addr; 382 sample.phys_addr = record->phys_addr; 383 sample.data_src = data_src; 384 sample.period = spe->instructions_sample_period; 385 sample.weight = record->latency; 386 387 return arm_spe_deliver_synth_event(spe, speq, event, &sample); 388 } 389 390 #define SPE_MEM_TYPE (ARM_SPE_L1D_ACCESS | ARM_SPE_L1D_MISS | \ 391 ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS | \ 392 ARM_SPE_REMOTE_ACCESS) 393 394 static bool arm_spe__is_memory_event(enum arm_spe_sample_type type) 395 { 396 if (type & SPE_MEM_TYPE) 397 return true; 398 399 return false; 400 } 401 402 static u64 arm_spe__synth_data_source(const struct arm_spe_record *record) 403 { 404 union perf_mem_data_src data_src = { 0 }; 405 406 if (record->op == ARM_SPE_LD) 407 data_src.mem_op = PERF_MEM_OP_LOAD; 408 else 409 data_src.mem_op = PERF_MEM_OP_STORE; 410 411 if (record->type & (ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS)) { 412 data_src.mem_lvl = PERF_MEM_LVL_L3; 413 414 if (record->type & ARM_SPE_LLC_MISS) 415 data_src.mem_lvl |= PERF_MEM_LVL_MISS; 416 else 417 data_src.mem_lvl |= PERF_MEM_LVL_HIT; 418 } else if (record->type & (ARM_SPE_L1D_ACCESS | ARM_SPE_L1D_MISS)) { 419 data_src.mem_lvl = PERF_MEM_LVL_L1; 420 421 if (record->type & ARM_SPE_L1D_MISS) 422 data_src.mem_lvl |= PERF_MEM_LVL_MISS; 423 else 424 data_src.mem_lvl |= PERF_MEM_LVL_HIT; 425 } 426 427 if (record->type & ARM_SPE_REMOTE_ACCESS) 428 data_src.mem_lvl |= PERF_MEM_LVL_REM_CCE1; 429 430 if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) { 431 data_src.mem_dtlb = PERF_MEM_TLB_WK; 432 433 if (record->type & ARM_SPE_TLB_MISS) 434 data_src.mem_dtlb |= PERF_MEM_TLB_MISS; 435 else 436 data_src.mem_dtlb |= PERF_MEM_TLB_HIT; 437 } 438 439 return data_src.val; 440 } 441 442 static int arm_spe_sample(struct arm_spe_queue *speq) 443 { 444 const struct arm_spe_record *record = &speq->decoder->record; 445 struct arm_spe *spe = speq->spe; 446 u64 data_src; 447 int err; 448 449 data_src = arm_spe__synth_data_source(record); 450 451 if (spe->sample_flc) { 452 if (record->type & ARM_SPE_L1D_MISS) { 453 err = arm_spe__synth_mem_sample(speq, spe->l1d_miss_id, 454 data_src); 455 if (err) 456 return err; 457 } 458 459 if (record->type & ARM_SPE_L1D_ACCESS) { 460 err = arm_spe__synth_mem_sample(speq, spe->l1d_access_id, 461 data_src); 462 if (err) 463 return err; 464 } 465 } 466 467 if (spe->sample_llc) { 468 if (record->type & ARM_SPE_LLC_MISS) { 469 err = arm_spe__synth_mem_sample(speq, spe->llc_miss_id, 470 data_src); 471 if (err) 472 return err; 473 } 474 475 if (record->type & ARM_SPE_LLC_ACCESS) { 476 err = arm_spe__synth_mem_sample(speq, spe->llc_access_id, 477 data_src); 478 if (err) 479 return err; 480 } 481 } 482 483 if (spe->sample_tlb) { 484 if (record->type & ARM_SPE_TLB_MISS) { 485 err = arm_spe__synth_mem_sample(speq, spe->tlb_miss_id, 486 data_src); 487 if (err) 488 return err; 489 } 490 491 if (record->type & ARM_SPE_TLB_ACCESS) { 492 err = arm_spe__synth_mem_sample(speq, spe->tlb_access_id, 493 data_src); 494 if (err) 495 return err; 496 } 497 } 498 499 if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) { 500 err = arm_spe__synth_branch_sample(speq, spe->branch_miss_id); 501 if (err) 502 return err; 503 } 504 505 if (spe->sample_remote_access && 506 (record->type & ARM_SPE_REMOTE_ACCESS)) { 507 err = arm_spe__synth_mem_sample(speq, spe->remote_access_id, 508 data_src); 509 if (err) 510 return err; 511 } 512 513 if (spe->sample_memory && arm_spe__is_memory_event(record->type)) { 514 err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src); 515 if (err) 516 return err; 517 } 518 519 if (spe->sample_instructions) { 520 err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src); 521 if (err) 522 return err; 523 } 524 525 return 0; 526 } 527 528 static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp) 529 { 530 struct arm_spe *spe = speq->spe; 531 struct arm_spe_record *record; 532 int ret; 533 534 if (!spe->kernel_start) 535 spe->kernel_start = machine__kernel_start(spe->machine); 536 537 while (1) { 538 /* 539 * The usual logic is firstly to decode the packets, and then 540 * based the record to synthesize sample; but here the flow is 541 * reversed: it calls arm_spe_sample() for synthesizing samples 542 * prior to arm_spe_decode(). 543 * 544 * Two reasons for this code logic: 545 * 1. Firstly, when setup queue in arm_spe__setup_queue(), it 546 * has decoded trace data and generated a record, but the record 547 * is left to generate sample until run to here, so it's correct 548 * to synthesize sample for the left record. 549 * 2. After decoding trace data, it needs to compare the record 550 * timestamp with the coming perf event, if the record timestamp 551 * is later than the perf event, it needs bail out and pushs the 552 * record into auxtrace heap, thus the record can be deferred to 553 * synthesize sample until run to here at the next time; so this 554 * can correlate samples between Arm SPE trace data and other 555 * perf events with correct time ordering. 556 */ 557 558 /* 559 * Update pid/tid info. 560 */ 561 record = &speq->decoder->record; 562 if (!spe->timeless_decoding && record->context_id != (u64)-1) { 563 ret = arm_spe_set_tid(speq, record->context_id); 564 if (ret) 565 return ret; 566 567 spe->use_ctx_pkt_for_pid = true; 568 } 569 570 ret = arm_spe_sample(speq); 571 if (ret) 572 return ret; 573 574 ret = arm_spe_decode(speq->decoder); 575 if (!ret) { 576 pr_debug("No data or all data has been processed.\n"); 577 return 1; 578 } 579 580 /* 581 * Error is detected when decode SPE trace data, continue to 582 * the next trace data and find out more records. 583 */ 584 if (ret < 0) 585 continue; 586 587 record = &speq->decoder->record; 588 589 /* Update timestamp for the last record */ 590 if (record->timestamp > speq->timestamp) 591 speq->timestamp = record->timestamp; 592 593 /* 594 * If the timestamp of the queue is later than timestamp of the 595 * coming perf event, bail out so can allow the perf event to 596 * be processed ahead. 597 */ 598 if (!spe->timeless_decoding && speq->timestamp >= *timestamp) { 599 *timestamp = speq->timestamp; 600 return 0; 601 } 602 } 603 604 return 0; 605 } 606 607 static int arm_spe__setup_queue(struct arm_spe *spe, 608 struct auxtrace_queue *queue, 609 unsigned int queue_nr) 610 { 611 struct arm_spe_queue *speq = queue->priv; 612 struct arm_spe_record *record; 613 614 if (list_empty(&queue->head) || speq) 615 return 0; 616 617 speq = arm_spe__alloc_queue(spe, queue_nr); 618 619 if (!speq) 620 return -ENOMEM; 621 622 queue->priv = speq; 623 624 if (queue->cpu != -1) 625 speq->cpu = queue->cpu; 626 627 if (!speq->on_heap) { 628 int ret; 629 630 if (spe->timeless_decoding) 631 return 0; 632 633 retry: 634 ret = arm_spe_decode(speq->decoder); 635 636 if (!ret) 637 return 0; 638 639 if (ret < 0) 640 goto retry; 641 642 record = &speq->decoder->record; 643 644 speq->timestamp = record->timestamp; 645 ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp); 646 if (ret) 647 return ret; 648 speq->on_heap = true; 649 } 650 651 return 0; 652 } 653 654 static int arm_spe__setup_queues(struct arm_spe *spe) 655 { 656 unsigned int i; 657 int ret; 658 659 for (i = 0; i < spe->queues.nr_queues; i++) { 660 ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i); 661 if (ret) 662 return ret; 663 } 664 665 return 0; 666 } 667 668 static int arm_spe__update_queues(struct arm_spe *spe) 669 { 670 if (spe->queues.new_data) { 671 spe->queues.new_data = false; 672 return arm_spe__setup_queues(spe); 673 } 674 675 return 0; 676 } 677 678 static bool arm_spe__is_timeless_decoding(struct arm_spe *spe) 679 { 680 struct evsel *evsel; 681 struct evlist *evlist = spe->session->evlist; 682 bool timeless_decoding = true; 683 684 /* 685 * Circle through the list of event and complain if we find one 686 * with the time bit set. 687 */ 688 evlist__for_each_entry(evlist, evsel) { 689 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) 690 timeless_decoding = false; 691 } 692 693 return timeless_decoding; 694 } 695 696 static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp) 697 { 698 unsigned int queue_nr; 699 u64 ts; 700 int ret; 701 702 while (1) { 703 struct auxtrace_queue *queue; 704 struct arm_spe_queue *speq; 705 706 if (!spe->heap.heap_cnt) 707 return 0; 708 709 if (spe->heap.heap_array[0].ordinal >= timestamp) 710 return 0; 711 712 queue_nr = spe->heap.heap_array[0].queue_nr; 713 queue = &spe->queues.queue_array[queue_nr]; 714 speq = queue->priv; 715 716 auxtrace_heap__pop(&spe->heap); 717 718 if (spe->heap.heap_cnt) { 719 ts = spe->heap.heap_array[0].ordinal + 1; 720 if (ts > timestamp) 721 ts = timestamp; 722 } else { 723 ts = timestamp; 724 } 725 726 /* 727 * A previous context-switch event has set pid/tid in the machine's context, so 728 * here we need to update the pid/tid in the thread and SPE queue. 729 */ 730 if (!spe->use_ctx_pkt_for_pid) 731 arm_spe_set_pid_tid_cpu(spe, queue); 732 733 ret = arm_spe_run_decoder(speq, &ts); 734 if (ret < 0) { 735 auxtrace_heap__add(&spe->heap, queue_nr, ts); 736 return ret; 737 } 738 739 if (!ret) { 740 ret = auxtrace_heap__add(&spe->heap, queue_nr, ts); 741 if (ret < 0) 742 return ret; 743 } else { 744 speq->on_heap = false; 745 } 746 } 747 748 return 0; 749 } 750 751 static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid, 752 u64 time_) 753 { 754 struct auxtrace_queues *queues = &spe->queues; 755 unsigned int i; 756 u64 ts = 0; 757 758 for (i = 0; i < queues->nr_queues; i++) { 759 struct auxtrace_queue *queue = &spe->queues.queue_array[i]; 760 struct arm_spe_queue *speq = queue->priv; 761 762 if (speq && (tid == -1 || speq->tid == tid)) { 763 speq->time = time_; 764 arm_spe_set_pid_tid_cpu(spe, queue); 765 arm_spe_run_decoder(speq, &ts); 766 } 767 } 768 return 0; 769 } 770 771 static int arm_spe_context_switch(struct arm_spe *spe, union perf_event *event, 772 struct perf_sample *sample) 773 { 774 pid_t pid, tid; 775 int cpu; 776 777 if (!(event->header.misc & PERF_RECORD_MISC_SWITCH_OUT)) 778 return 0; 779 780 pid = event->context_switch.next_prev_pid; 781 tid = event->context_switch.next_prev_tid; 782 cpu = sample->cpu; 783 784 if (tid == -1) 785 pr_warning("context_switch event has no tid\n"); 786 787 return machine__set_current_tid(spe->machine, cpu, pid, tid); 788 } 789 790 static int arm_spe_process_event(struct perf_session *session, 791 union perf_event *event, 792 struct perf_sample *sample, 793 struct perf_tool *tool) 794 { 795 int err = 0; 796 u64 timestamp; 797 struct arm_spe *spe = container_of(session->auxtrace, 798 struct arm_spe, auxtrace); 799 800 if (dump_trace) 801 return 0; 802 803 if (!tool->ordered_events) { 804 pr_err("SPE trace requires ordered events\n"); 805 return -EINVAL; 806 } 807 808 if (sample->time && (sample->time != (u64) -1)) 809 timestamp = perf_time_to_tsc(sample->time, &spe->tc); 810 else 811 timestamp = 0; 812 813 if (timestamp || spe->timeless_decoding) { 814 err = arm_spe__update_queues(spe); 815 if (err) 816 return err; 817 } 818 819 if (spe->timeless_decoding) { 820 if (event->header.type == PERF_RECORD_EXIT) { 821 err = arm_spe_process_timeless_queues(spe, 822 event->fork.tid, 823 sample->time); 824 } 825 } else if (timestamp) { 826 err = arm_spe_process_queues(spe, timestamp); 827 if (err) 828 return err; 829 830 if (!spe->use_ctx_pkt_for_pid && 831 (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE || 832 event->header.type == PERF_RECORD_SWITCH)) 833 err = arm_spe_context_switch(spe, event, sample); 834 } 835 836 return err; 837 } 838 839 static int arm_spe_process_auxtrace_event(struct perf_session *session, 840 union perf_event *event, 841 struct perf_tool *tool __maybe_unused) 842 { 843 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, 844 auxtrace); 845 846 if (!spe->data_queued) { 847 struct auxtrace_buffer *buffer; 848 off_t data_offset; 849 int fd = perf_data__fd(session->data); 850 int err; 851 852 if (perf_data__is_pipe(session->data)) { 853 data_offset = 0; 854 } else { 855 data_offset = lseek(fd, 0, SEEK_CUR); 856 if (data_offset == -1) 857 return -errno; 858 } 859 860 err = auxtrace_queues__add_event(&spe->queues, session, event, 861 data_offset, &buffer); 862 if (err) 863 return err; 864 865 /* Dump here now we have copied a piped trace out of the pipe */ 866 if (dump_trace) { 867 if (auxtrace_buffer__get_data(buffer, fd)) { 868 arm_spe_dump_event(spe, buffer->data, 869 buffer->size); 870 auxtrace_buffer__put_data(buffer); 871 } 872 } 873 } 874 875 return 0; 876 } 877 878 static int arm_spe_flush(struct perf_session *session __maybe_unused, 879 struct perf_tool *tool __maybe_unused) 880 { 881 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, 882 auxtrace); 883 int ret; 884 885 if (dump_trace) 886 return 0; 887 888 if (!tool->ordered_events) 889 return -EINVAL; 890 891 ret = arm_spe__update_queues(spe); 892 if (ret < 0) 893 return ret; 894 895 if (spe->timeless_decoding) 896 return arm_spe_process_timeless_queues(spe, -1, 897 MAX_TIMESTAMP - 1); 898 899 ret = arm_spe_process_queues(spe, MAX_TIMESTAMP); 900 if (ret) 901 return ret; 902 903 if (!spe->use_ctx_pkt_for_pid) 904 ui__warning("Arm SPE CONTEXT packets not found in the traces.\n" 905 "Matching of TIDs to SPE events could be inaccurate.\n"); 906 907 return 0; 908 } 909 910 static void arm_spe_free_queue(void *priv) 911 { 912 struct arm_spe_queue *speq = priv; 913 914 if (!speq) 915 return; 916 thread__zput(speq->thread); 917 arm_spe_decoder_free(speq->decoder); 918 zfree(&speq->event_buf); 919 free(speq); 920 } 921 922 static void arm_spe_free_events(struct perf_session *session) 923 { 924 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, 925 auxtrace); 926 struct auxtrace_queues *queues = &spe->queues; 927 unsigned int i; 928 929 for (i = 0; i < queues->nr_queues; i++) { 930 arm_spe_free_queue(queues->queue_array[i].priv); 931 queues->queue_array[i].priv = NULL; 932 } 933 auxtrace_queues__free(queues); 934 } 935 936 static void arm_spe_free(struct perf_session *session) 937 { 938 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, 939 auxtrace); 940 941 auxtrace_heap__free(&spe->heap); 942 arm_spe_free_events(session); 943 session->auxtrace = NULL; 944 free(spe); 945 } 946 947 static bool arm_spe_evsel_is_auxtrace(struct perf_session *session, 948 struct evsel *evsel) 949 { 950 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace); 951 952 return evsel->core.attr.type == spe->pmu_type; 953 } 954 955 static const char * const arm_spe_info_fmts[] = { 956 [ARM_SPE_PMU_TYPE] = " PMU Type %"PRId64"\n", 957 }; 958 959 static void arm_spe_print_info(__u64 *arr) 960 { 961 if (!dump_trace) 962 return; 963 964 fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]); 965 } 966 967 struct arm_spe_synth { 968 struct perf_tool dummy_tool; 969 struct perf_session *session; 970 }; 971 972 static int arm_spe_event_synth(struct perf_tool *tool, 973 union perf_event *event, 974 struct perf_sample *sample __maybe_unused, 975 struct machine *machine __maybe_unused) 976 { 977 struct arm_spe_synth *arm_spe_synth = 978 container_of(tool, struct arm_spe_synth, dummy_tool); 979 980 return perf_session__deliver_synth_event(arm_spe_synth->session, 981 event, NULL); 982 } 983 984 static int arm_spe_synth_event(struct perf_session *session, 985 struct perf_event_attr *attr, u64 id) 986 { 987 struct arm_spe_synth arm_spe_synth; 988 989 memset(&arm_spe_synth, 0, sizeof(struct arm_spe_synth)); 990 arm_spe_synth.session = session; 991 992 return perf_event__synthesize_attr(&arm_spe_synth.dummy_tool, attr, 1, 993 &id, arm_spe_event_synth); 994 } 995 996 static void arm_spe_set_event_name(struct evlist *evlist, u64 id, 997 const char *name) 998 { 999 struct evsel *evsel; 1000 1001 evlist__for_each_entry(evlist, evsel) { 1002 if (evsel->core.id && evsel->core.id[0] == id) { 1003 if (evsel->name) 1004 zfree(&evsel->name); 1005 evsel->name = strdup(name); 1006 break; 1007 } 1008 } 1009 } 1010 1011 static int 1012 arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session) 1013 { 1014 struct evlist *evlist = session->evlist; 1015 struct evsel *evsel; 1016 struct perf_event_attr attr; 1017 bool found = false; 1018 u64 id; 1019 int err; 1020 1021 evlist__for_each_entry(evlist, evsel) { 1022 if (evsel->core.attr.type == spe->pmu_type) { 1023 found = true; 1024 break; 1025 } 1026 } 1027 1028 if (!found) { 1029 pr_debug("No selected events with SPE trace data\n"); 1030 return 0; 1031 } 1032 1033 memset(&attr, 0, sizeof(struct perf_event_attr)); 1034 attr.size = sizeof(struct perf_event_attr); 1035 attr.type = PERF_TYPE_HARDWARE; 1036 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK; 1037 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | 1038 PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC | 1039 PERF_SAMPLE_WEIGHT; 1040 if (spe->timeless_decoding) 1041 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; 1042 else 1043 attr.sample_type |= PERF_SAMPLE_TIME; 1044 1045 spe->sample_type = attr.sample_type; 1046 1047 attr.exclude_user = evsel->core.attr.exclude_user; 1048 attr.exclude_kernel = evsel->core.attr.exclude_kernel; 1049 attr.exclude_hv = evsel->core.attr.exclude_hv; 1050 attr.exclude_host = evsel->core.attr.exclude_host; 1051 attr.exclude_guest = evsel->core.attr.exclude_guest; 1052 attr.sample_id_all = evsel->core.attr.sample_id_all; 1053 attr.read_format = evsel->core.attr.read_format; 1054 1055 /* create new id val to be a fixed offset from evsel id */ 1056 id = evsel->core.id[0] + 1000000000; 1057 1058 if (!id) 1059 id = 1; 1060 1061 if (spe->synth_opts.flc) { 1062 spe->sample_flc = true; 1063 1064 /* Level 1 data cache miss */ 1065 err = arm_spe_synth_event(session, &attr, id); 1066 if (err) 1067 return err; 1068 spe->l1d_miss_id = id; 1069 arm_spe_set_event_name(evlist, id, "l1d-miss"); 1070 id += 1; 1071 1072 /* Level 1 data cache access */ 1073 err = arm_spe_synth_event(session, &attr, id); 1074 if (err) 1075 return err; 1076 spe->l1d_access_id = id; 1077 arm_spe_set_event_name(evlist, id, "l1d-access"); 1078 id += 1; 1079 } 1080 1081 if (spe->synth_opts.llc) { 1082 spe->sample_llc = true; 1083 1084 /* Last level cache miss */ 1085 err = arm_spe_synth_event(session, &attr, id); 1086 if (err) 1087 return err; 1088 spe->llc_miss_id = id; 1089 arm_spe_set_event_name(evlist, id, "llc-miss"); 1090 id += 1; 1091 1092 /* Last level cache access */ 1093 err = arm_spe_synth_event(session, &attr, id); 1094 if (err) 1095 return err; 1096 spe->llc_access_id = id; 1097 arm_spe_set_event_name(evlist, id, "llc-access"); 1098 id += 1; 1099 } 1100 1101 if (spe->synth_opts.tlb) { 1102 spe->sample_tlb = true; 1103 1104 /* TLB miss */ 1105 err = arm_spe_synth_event(session, &attr, id); 1106 if (err) 1107 return err; 1108 spe->tlb_miss_id = id; 1109 arm_spe_set_event_name(evlist, id, "tlb-miss"); 1110 id += 1; 1111 1112 /* TLB access */ 1113 err = arm_spe_synth_event(session, &attr, id); 1114 if (err) 1115 return err; 1116 spe->tlb_access_id = id; 1117 arm_spe_set_event_name(evlist, id, "tlb-access"); 1118 id += 1; 1119 } 1120 1121 if (spe->synth_opts.branches) { 1122 spe->sample_branch = true; 1123 1124 /* Branch miss */ 1125 err = arm_spe_synth_event(session, &attr, id); 1126 if (err) 1127 return err; 1128 spe->branch_miss_id = id; 1129 arm_spe_set_event_name(evlist, id, "branch-miss"); 1130 id += 1; 1131 } 1132 1133 if (spe->synth_opts.remote_access) { 1134 spe->sample_remote_access = true; 1135 1136 /* Remote access */ 1137 err = arm_spe_synth_event(session, &attr, id); 1138 if (err) 1139 return err; 1140 spe->remote_access_id = id; 1141 arm_spe_set_event_name(evlist, id, "remote-access"); 1142 id += 1; 1143 } 1144 1145 if (spe->synth_opts.mem) { 1146 spe->sample_memory = true; 1147 1148 err = arm_spe_synth_event(session, &attr, id); 1149 if (err) 1150 return err; 1151 spe->memory_id = id; 1152 arm_spe_set_event_name(evlist, id, "memory"); 1153 id += 1; 1154 } 1155 1156 if (spe->synth_opts.instructions) { 1157 if (spe->synth_opts.period_type != PERF_ITRACE_PERIOD_INSTRUCTIONS) { 1158 pr_warning("Only instruction-based sampling period is currently supported by Arm SPE.\n"); 1159 goto synth_instructions_out; 1160 } 1161 if (spe->synth_opts.period > 1) 1162 pr_warning("Arm SPE has a hardware-based sample period.\n" 1163 "Additional instruction events will be discarded by --itrace\n"); 1164 1165 spe->sample_instructions = true; 1166 attr.config = PERF_COUNT_HW_INSTRUCTIONS; 1167 attr.sample_period = spe->synth_opts.period; 1168 spe->instructions_sample_period = attr.sample_period; 1169 err = arm_spe_synth_event(session, &attr, id); 1170 if (err) 1171 return err; 1172 spe->instructions_id = id; 1173 arm_spe_set_event_name(evlist, id, "instructions"); 1174 } 1175 synth_instructions_out: 1176 1177 return 0; 1178 } 1179 1180 int arm_spe_process_auxtrace_info(union perf_event *event, 1181 struct perf_session *session) 1182 { 1183 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; 1184 size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX; 1185 struct perf_record_time_conv *tc = &session->time_conv; 1186 struct arm_spe *spe; 1187 int err; 1188 1189 if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) + 1190 min_sz) 1191 return -EINVAL; 1192 1193 spe = zalloc(sizeof(struct arm_spe)); 1194 if (!spe) 1195 return -ENOMEM; 1196 1197 err = auxtrace_queues__init(&spe->queues); 1198 if (err) 1199 goto err_free; 1200 1201 spe->session = session; 1202 spe->machine = &session->machines.host; /* No kvm support */ 1203 spe->auxtrace_type = auxtrace_info->type; 1204 spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE]; 1205 1206 spe->timeless_decoding = arm_spe__is_timeless_decoding(spe); 1207 1208 /* 1209 * The synthesized event PERF_RECORD_TIME_CONV has been handled ahead 1210 * and the parameters for hardware clock are stored in the session 1211 * context. Passes these parameters to the struct perf_tsc_conversion 1212 * in "spe->tc", which is used for later conversion between clock 1213 * counter and timestamp. 1214 * 1215 * For backward compatibility, copies the fields starting from 1216 * "time_cycles" only if they are contained in the event. 1217 */ 1218 spe->tc.time_shift = tc->time_shift; 1219 spe->tc.time_mult = tc->time_mult; 1220 spe->tc.time_zero = tc->time_zero; 1221 1222 if (event_contains(*tc, time_cycles)) { 1223 spe->tc.time_cycles = tc->time_cycles; 1224 spe->tc.time_mask = tc->time_mask; 1225 spe->tc.cap_user_time_zero = tc->cap_user_time_zero; 1226 spe->tc.cap_user_time_short = tc->cap_user_time_short; 1227 } 1228 1229 spe->auxtrace.process_event = arm_spe_process_event; 1230 spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event; 1231 spe->auxtrace.flush_events = arm_spe_flush; 1232 spe->auxtrace.free_events = arm_spe_free_events; 1233 spe->auxtrace.free = arm_spe_free; 1234 spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace; 1235 session->auxtrace = &spe->auxtrace; 1236 1237 arm_spe_print_info(&auxtrace_info->priv[0]); 1238 1239 if (dump_trace) 1240 return 0; 1241 1242 if (session->itrace_synth_opts && session->itrace_synth_opts->set) 1243 spe->synth_opts = *session->itrace_synth_opts; 1244 else 1245 itrace_synth_opts__set_default(&spe->synth_opts, false); 1246 1247 err = arm_spe_synth_events(spe, session); 1248 if (err) 1249 goto err_free_queues; 1250 1251 err = auxtrace_queues__process_index(&spe->queues, session); 1252 if (err) 1253 goto err_free_queues; 1254 1255 if (spe->queues.populated) 1256 spe->data_queued = true; 1257 1258 return 0; 1259 1260 err_free_queues: 1261 auxtrace_queues__free(&spe->queues); 1262 session->auxtrace = NULL; 1263 err_free: 1264 free(spe); 1265 return err; 1266 } 1267