1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Arm Statistical Profiling Extensions (SPE) support 4 * Copyright (c) 2017-2018, Arm Ltd. 5 */ 6 7 #include <byteswap.h> 8 #include <endian.h> 9 #include <errno.h> 10 #include <inttypes.h> 11 #include <linux/bitops.h> 12 #include <linux/kernel.h> 13 #include <linux/log2.h> 14 #include <linux/types.h> 15 #include <linux/zalloc.h> 16 #include <stdlib.h> 17 #include <unistd.h> 18 19 #include "auxtrace.h" 20 #include "color.h" 21 #include "debug.h" 22 #include "evlist.h" 23 #include "evsel.h" 24 #include "machine.h" 25 #include "session.h" 26 #include "symbol.h" 27 #include "thread.h" 28 #include "thread-stack.h" 29 #include "tsc.h" 30 #include "tool.h" 31 #include "util/synthetic-events.h" 32 33 #include "arm-spe.h" 34 #include "arm-spe-decoder/arm-spe-decoder.h" 35 #include "arm-spe-decoder/arm-spe-pkt-decoder.h" 36 37 #include "../../arch/arm64/include/asm/cputype.h" 38 #define MAX_TIMESTAMP (~0ULL) 39 40 struct arm_spe { 41 struct auxtrace auxtrace; 42 struct auxtrace_queues queues; 43 struct auxtrace_heap heap; 44 struct itrace_synth_opts synth_opts; 45 u32 auxtrace_type; 46 struct perf_session *session; 47 struct machine *machine; 48 u32 pmu_type; 49 u64 midr; 50 51 struct perf_tsc_conversion tc; 52 53 u8 timeless_decoding; 54 u8 data_queued; 55 56 u64 sample_type; 57 u8 sample_flc; 58 u8 sample_llc; 59 u8 sample_tlb; 60 u8 sample_branch; 61 u8 sample_remote_access; 62 u8 sample_memory; 63 u8 sample_instructions; 64 u64 instructions_sample_period; 65 66 u64 l1d_miss_id; 67 u64 l1d_access_id; 68 u64 llc_miss_id; 69 u64 llc_access_id; 70 u64 tlb_miss_id; 71 u64 tlb_access_id; 72 u64 branch_miss_id; 73 u64 remote_access_id; 74 u64 memory_id; 75 u64 instructions_id; 76 77 u64 kernel_start; 78 79 unsigned long num_events; 80 u8 use_ctx_pkt_for_pid; 81 }; 82 83 struct arm_spe_queue { 84 struct arm_spe *spe; 85 unsigned int queue_nr; 86 struct auxtrace_buffer *buffer; 87 struct auxtrace_buffer *old_buffer; 88 union perf_event *event_buf; 89 bool on_heap; 90 bool done; 91 pid_t pid; 92 pid_t tid; 93 int cpu; 94 struct arm_spe_decoder *decoder; 95 u64 time; 96 u64 timestamp; 97 struct thread *thread; 98 u64 period_instructions; 99 }; 100 101 static void arm_spe_dump(struct arm_spe *spe __maybe_unused, 102 unsigned char *buf, size_t len) 103 { 104 struct arm_spe_pkt packet; 105 size_t pos = 0; 106 int ret, pkt_len, i; 107 char desc[ARM_SPE_PKT_DESC_MAX]; 108 const char *color = PERF_COLOR_BLUE; 109 110 color_fprintf(stdout, color, 111 ". ... ARM SPE data: size %#zx bytes\n", 112 len); 113 114 while (len) { 115 ret = arm_spe_get_packet(buf, len, &packet); 116 if (ret > 0) 117 pkt_len = ret; 118 else 119 pkt_len = 1; 120 printf("."); 121 color_fprintf(stdout, color, " %08x: ", pos); 122 for (i = 0; i < pkt_len; i++) 123 color_fprintf(stdout, color, " %02x", buf[i]); 124 for (; i < 16; i++) 125 color_fprintf(stdout, color, " "); 126 if (ret > 0) { 127 ret = arm_spe_pkt_desc(&packet, desc, 128 ARM_SPE_PKT_DESC_MAX); 129 if (!ret) 130 color_fprintf(stdout, color, " %s\n", desc); 131 } else { 132 color_fprintf(stdout, color, " Bad packet!\n"); 133 } 134 pos += pkt_len; 135 buf += pkt_len; 136 len -= pkt_len; 137 } 138 } 139 140 static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf, 141 size_t len) 142 { 143 printf(".\n"); 144 arm_spe_dump(spe, buf, len); 145 } 146 147 static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data) 148 { 149 struct arm_spe_queue *speq = data; 150 struct auxtrace_buffer *buffer = speq->buffer; 151 struct auxtrace_buffer *old_buffer = speq->old_buffer; 152 struct auxtrace_queue *queue; 153 154 queue = &speq->spe->queues.queue_array[speq->queue_nr]; 155 156 buffer = auxtrace_buffer__next(queue, buffer); 157 /* If no more data, drop the previous auxtrace_buffer and return */ 158 if (!buffer) { 159 if (old_buffer) 160 auxtrace_buffer__drop_data(old_buffer); 161 b->len = 0; 162 return 0; 163 } 164 165 speq->buffer = buffer; 166 167 /* If the aux_buffer doesn't have data associated, try to load it */ 168 if (!buffer->data) { 169 /* get the file desc associated with the perf data file */ 170 int fd = perf_data__fd(speq->spe->session->data); 171 172 buffer->data = auxtrace_buffer__get_data(buffer, fd); 173 if (!buffer->data) 174 return -ENOMEM; 175 } 176 177 b->len = buffer->size; 178 b->buf = buffer->data; 179 180 if (b->len) { 181 if (old_buffer) 182 auxtrace_buffer__drop_data(old_buffer); 183 speq->old_buffer = buffer; 184 } else { 185 auxtrace_buffer__drop_data(buffer); 186 return arm_spe_get_trace(b, data); 187 } 188 189 return 0; 190 } 191 192 static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe, 193 unsigned int queue_nr) 194 { 195 struct arm_spe_params params = { .get_trace = 0, }; 196 struct arm_spe_queue *speq; 197 198 speq = zalloc(sizeof(*speq)); 199 if (!speq) 200 return NULL; 201 202 speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); 203 if (!speq->event_buf) 204 goto out_free; 205 206 speq->spe = spe; 207 speq->queue_nr = queue_nr; 208 speq->pid = -1; 209 speq->tid = -1; 210 speq->cpu = -1; 211 speq->period_instructions = 0; 212 213 /* params set */ 214 params.get_trace = arm_spe_get_trace; 215 params.data = speq; 216 217 /* create new decoder */ 218 speq->decoder = arm_spe_decoder_new(¶ms); 219 if (!speq->decoder) 220 goto out_free; 221 222 return speq; 223 224 out_free: 225 zfree(&speq->event_buf); 226 free(speq); 227 228 return NULL; 229 } 230 231 static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip) 232 { 233 return ip >= spe->kernel_start ? 234 PERF_RECORD_MISC_KERNEL : 235 PERF_RECORD_MISC_USER; 236 } 237 238 static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe, 239 struct auxtrace_queue *queue) 240 { 241 struct arm_spe_queue *speq = queue->priv; 242 pid_t tid; 243 244 tid = machine__get_current_tid(spe->machine, speq->cpu); 245 if (tid != -1) { 246 speq->tid = tid; 247 thread__zput(speq->thread); 248 } else 249 speq->tid = queue->tid; 250 251 if ((!speq->thread) && (speq->tid != -1)) { 252 speq->thread = machine__find_thread(spe->machine, -1, 253 speq->tid); 254 } 255 256 if (speq->thread) { 257 speq->pid = speq->thread->pid_; 258 if (queue->cpu == -1) 259 speq->cpu = speq->thread->cpu; 260 } 261 } 262 263 static int arm_spe_set_tid(struct arm_spe_queue *speq, pid_t tid) 264 { 265 struct arm_spe *spe = speq->spe; 266 int err = machine__set_current_tid(spe->machine, speq->cpu, -1, tid); 267 268 if (err) 269 return err; 270 271 arm_spe_set_pid_tid_cpu(spe, &spe->queues.queue_array[speq->queue_nr]); 272 273 return 0; 274 } 275 276 static void arm_spe_prep_sample(struct arm_spe *spe, 277 struct arm_spe_queue *speq, 278 union perf_event *event, 279 struct perf_sample *sample) 280 { 281 struct arm_spe_record *record = &speq->decoder->record; 282 283 if (!spe->timeless_decoding) 284 sample->time = tsc_to_perf_time(record->timestamp, &spe->tc); 285 286 sample->ip = record->from_ip; 287 sample->cpumode = arm_spe_cpumode(spe, sample->ip); 288 sample->pid = speq->pid; 289 sample->tid = speq->tid; 290 sample->period = 1; 291 sample->cpu = speq->cpu; 292 293 event->sample.header.type = PERF_RECORD_SAMPLE; 294 event->sample.header.misc = sample->cpumode; 295 event->sample.header.size = sizeof(struct perf_event_header); 296 } 297 298 static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type) 299 { 300 event->header.size = perf_event__sample_event_size(sample, type, 0); 301 return perf_event__synthesize_sample(event, type, 0, sample); 302 } 303 304 static inline int 305 arm_spe_deliver_synth_event(struct arm_spe *spe, 306 struct arm_spe_queue *speq __maybe_unused, 307 union perf_event *event, 308 struct perf_sample *sample) 309 { 310 int ret; 311 312 if (spe->synth_opts.inject) { 313 ret = arm_spe__inject_event(event, sample, spe->sample_type); 314 if (ret) 315 return ret; 316 } 317 318 ret = perf_session__deliver_synth_event(spe->session, event, sample); 319 if (ret) 320 pr_err("ARM SPE: failed to deliver event, error %d\n", ret); 321 322 return ret; 323 } 324 325 static int arm_spe__synth_mem_sample(struct arm_spe_queue *speq, 326 u64 spe_events_id, u64 data_src) 327 { 328 struct arm_spe *spe = speq->spe; 329 struct arm_spe_record *record = &speq->decoder->record; 330 union perf_event *event = speq->event_buf; 331 struct perf_sample sample = { .ip = 0, }; 332 333 arm_spe_prep_sample(spe, speq, event, &sample); 334 335 sample.id = spe_events_id; 336 sample.stream_id = spe_events_id; 337 sample.addr = record->virt_addr; 338 sample.phys_addr = record->phys_addr; 339 sample.data_src = data_src; 340 sample.weight = record->latency; 341 342 return arm_spe_deliver_synth_event(spe, speq, event, &sample); 343 } 344 345 static int arm_spe__synth_branch_sample(struct arm_spe_queue *speq, 346 u64 spe_events_id) 347 { 348 struct arm_spe *spe = speq->spe; 349 struct arm_spe_record *record = &speq->decoder->record; 350 union perf_event *event = speq->event_buf; 351 struct perf_sample sample = { .ip = 0, }; 352 353 arm_spe_prep_sample(spe, speq, event, &sample); 354 355 sample.id = spe_events_id; 356 sample.stream_id = spe_events_id; 357 sample.addr = record->to_ip; 358 sample.weight = record->latency; 359 360 return arm_spe_deliver_synth_event(spe, speq, event, &sample); 361 } 362 363 static int arm_spe__synth_instruction_sample(struct arm_spe_queue *speq, 364 u64 spe_events_id, u64 data_src) 365 { 366 struct arm_spe *spe = speq->spe; 367 struct arm_spe_record *record = &speq->decoder->record; 368 union perf_event *event = speq->event_buf; 369 struct perf_sample sample = { .ip = 0, }; 370 371 /* 372 * Handles perf instruction sampling period. 373 */ 374 speq->period_instructions++; 375 if (speq->period_instructions < spe->instructions_sample_period) 376 return 0; 377 speq->period_instructions = 0; 378 379 arm_spe_prep_sample(spe, speq, event, &sample); 380 381 sample.id = spe_events_id; 382 sample.stream_id = spe_events_id; 383 sample.addr = record->virt_addr; 384 sample.phys_addr = record->phys_addr; 385 sample.data_src = data_src; 386 sample.period = spe->instructions_sample_period; 387 sample.weight = record->latency; 388 389 return arm_spe_deliver_synth_event(spe, speq, event, &sample); 390 } 391 392 static const struct midr_range neoverse_spe[] = { 393 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), 394 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), 395 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), 396 {}, 397 }; 398 399 static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *record, 400 union perf_mem_data_src *data_src) 401 { 402 /* 403 * Even though four levels of cache hierarchy are possible, no known 404 * production Neoverse systems currently include more than three levels 405 * so for the time being we assume three exist. If a production system 406 * is built with four the this function would have to be changed to 407 * detect the number of levels for reporting. 408 */ 409 410 /* 411 * We have no data on the hit level or data source for stores in the 412 * Neoverse SPE records. 413 */ 414 if (record->op & ARM_SPE_ST) { 415 data_src->mem_lvl = PERF_MEM_LVL_NA; 416 data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA; 417 data_src->mem_snoop = PERF_MEM_SNOOP_NA; 418 return; 419 } 420 421 switch (record->source) { 422 case ARM_SPE_NV_L1D: 423 data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; 424 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; 425 data_src->mem_snoop = PERF_MEM_SNOOP_NONE; 426 break; 427 case ARM_SPE_NV_L2: 428 data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; 429 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; 430 data_src->mem_snoop = PERF_MEM_SNOOP_NONE; 431 break; 432 case ARM_SPE_NV_PEER_CORE: 433 data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; 434 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; 435 data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; 436 break; 437 /* 438 * We don't know if this is L1, L2 but we do know it was a cache-2-cache 439 * transfer, so set SNOOPX_PEER 440 */ 441 case ARM_SPE_NV_LOCAL_CLUSTER: 442 case ARM_SPE_NV_PEER_CLUSTER: 443 data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; 444 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; 445 data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; 446 break; 447 /* 448 * System cache is assumed to be L3 449 */ 450 case ARM_SPE_NV_SYS_CACHE: 451 data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; 452 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; 453 data_src->mem_snoop = PERF_MEM_SNOOP_HIT; 454 break; 455 /* 456 * We don't know what level it hit in, except it came from the other 457 * socket 458 */ 459 case ARM_SPE_NV_REMOTE: 460 data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1; 461 data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE; 462 data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; 463 data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; 464 break; 465 case ARM_SPE_NV_DRAM: 466 data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT; 467 data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM; 468 data_src->mem_snoop = PERF_MEM_SNOOP_NONE; 469 break; 470 default: 471 break; 472 } 473 } 474 475 static void arm_spe__synth_data_source_generic(const struct arm_spe_record *record, 476 union perf_mem_data_src *data_src) 477 { 478 if (record->type & (ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS)) { 479 data_src->mem_lvl = PERF_MEM_LVL_L3; 480 481 if (record->type & ARM_SPE_LLC_MISS) 482 data_src->mem_lvl |= PERF_MEM_LVL_MISS; 483 else 484 data_src->mem_lvl |= PERF_MEM_LVL_HIT; 485 } else if (record->type & (ARM_SPE_L1D_ACCESS | ARM_SPE_L1D_MISS)) { 486 data_src->mem_lvl = PERF_MEM_LVL_L1; 487 488 if (record->type & ARM_SPE_L1D_MISS) 489 data_src->mem_lvl |= PERF_MEM_LVL_MISS; 490 else 491 data_src->mem_lvl |= PERF_MEM_LVL_HIT; 492 } 493 494 if (record->type & ARM_SPE_REMOTE_ACCESS) 495 data_src->mem_lvl |= PERF_MEM_LVL_REM_CCE1; 496 } 497 498 static u64 arm_spe__synth_data_source(const struct arm_spe_record *record, u64 midr) 499 { 500 union perf_mem_data_src data_src = { 0 }; 501 bool is_neoverse = is_midr_in_range(midr, neoverse_spe); 502 503 if (record->op == ARM_SPE_LD) 504 data_src.mem_op = PERF_MEM_OP_LOAD; 505 else if (record->op == ARM_SPE_ST) 506 data_src.mem_op = PERF_MEM_OP_STORE; 507 else 508 return 0; 509 510 if (is_neoverse) 511 arm_spe__synth_data_source_neoverse(record, &data_src); 512 else 513 arm_spe__synth_data_source_generic(record, &data_src); 514 515 if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) { 516 data_src.mem_dtlb = PERF_MEM_TLB_WK; 517 518 if (record->type & ARM_SPE_TLB_MISS) 519 data_src.mem_dtlb |= PERF_MEM_TLB_MISS; 520 else 521 data_src.mem_dtlb |= PERF_MEM_TLB_HIT; 522 } 523 524 return data_src.val; 525 } 526 527 static int arm_spe_sample(struct arm_spe_queue *speq) 528 { 529 const struct arm_spe_record *record = &speq->decoder->record; 530 struct arm_spe *spe = speq->spe; 531 u64 data_src; 532 int err; 533 534 data_src = arm_spe__synth_data_source(record, spe->midr); 535 536 if (spe->sample_flc) { 537 if (record->type & ARM_SPE_L1D_MISS) { 538 err = arm_spe__synth_mem_sample(speq, spe->l1d_miss_id, 539 data_src); 540 if (err) 541 return err; 542 } 543 544 if (record->type & ARM_SPE_L1D_ACCESS) { 545 err = arm_spe__synth_mem_sample(speq, spe->l1d_access_id, 546 data_src); 547 if (err) 548 return err; 549 } 550 } 551 552 if (spe->sample_llc) { 553 if (record->type & ARM_SPE_LLC_MISS) { 554 err = arm_spe__synth_mem_sample(speq, spe->llc_miss_id, 555 data_src); 556 if (err) 557 return err; 558 } 559 560 if (record->type & ARM_SPE_LLC_ACCESS) { 561 err = arm_spe__synth_mem_sample(speq, spe->llc_access_id, 562 data_src); 563 if (err) 564 return err; 565 } 566 } 567 568 if (spe->sample_tlb) { 569 if (record->type & ARM_SPE_TLB_MISS) { 570 err = arm_spe__synth_mem_sample(speq, spe->tlb_miss_id, 571 data_src); 572 if (err) 573 return err; 574 } 575 576 if (record->type & ARM_SPE_TLB_ACCESS) { 577 err = arm_spe__synth_mem_sample(speq, spe->tlb_access_id, 578 data_src); 579 if (err) 580 return err; 581 } 582 } 583 584 if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) { 585 err = arm_spe__synth_branch_sample(speq, spe->branch_miss_id); 586 if (err) 587 return err; 588 } 589 590 if (spe->sample_remote_access && 591 (record->type & ARM_SPE_REMOTE_ACCESS)) { 592 err = arm_spe__synth_mem_sample(speq, spe->remote_access_id, 593 data_src); 594 if (err) 595 return err; 596 } 597 598 /* 599 * When data_src is zero it means the record is not a memory operation, 600 * skip to synthesize memory sample for this case. 601 */ 602 if (spe->sample_memory && data_src) { 603 err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src); 604 if (err) 605 return err; 606 } 607 608 if (spe->sample_instructions) { 609 err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src); 610 if (err) 611 return err; 612 } 613 614 return 0; 615 } 616 617 static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp) 618 { 619 struct arm_spe *spe = speq->spe; 620 struct arm_spe_record *record; 621 int ret; 622 623 if (!spe->kernel_start) 624 spe->kernel_start = machine__kernel_start(spe->machine); 625 626 while (1) { 627 /* 628 * The usual logic is firstly to decode the packets, and then 629 * based the record to synthesize sample; but here the flow is 630 * reversed: it calls arm_spe_sample() for synthesizing samples 631 * prior to arm_spe_decode(). 632 * 633 * Two reasons for this code logic: 634 * 1. Firstly, when setup queue in arm_spe__setup_queue(), it 635 * has decoded trace data and generated a record, but the record 636 * is left to generate sample until run to here, so it's correct 637 * to synthesize sample for the left record. 638 * 2. After decoding trace data, it needs to compare the record 639 * timestamp with the coming perf event, if the record timestamp 640 * is later than the perf event, it needs bail out and pushs the 641 * record into auxtrace heap, thus the record can be deferred to 642 * synthesize sample until run to here at the next time; so this 643 * can correlate samples between Arm SPE trace data and other 644 * perf events with correct time ordering. 645 */ 646 647 /* 648 * Update pid/tid info. 649 */ 650 record = &speq->decoder->record; 651 if (!spe->timeless_decoding && record->context_id != (u64)-1) { 652 ret = arm_spe_set_tid(speq, record->context_id); 653 if (ret) 654 return ret; 655 656 spe->use_ctx_pkt_for_pid = true; 657 } 658 659 ret = arm_spe_sample(speq); 660 if (ret) 661 return ret; 662 663 ret = arm_spe_decode(speq->decoder); 664 if (!ret) { 665 pr_debug("No data or all data has been processed.\n"); 666 return 1; 667 } 668 669 /* 670 * Error is detected when decode SPE trace data, continue to 671 * the next trace data and find out more records. 672 */ 673 if (ret < 0) 674 continue; 675 676 record = &speq->decoder->record; 677 678 /* Update timestamp for the last record */ 679 if (record->timestamp > speq->timestamp) 680 speq->timestamp = record->timestamp; 681 682 /* 683 * If the timestamp of the queue is later than timestamp of the 684 * coming perf event, bail out so can allow the perf event to 685 * be processed ahead. 686 */ 687 if (!spe->timeless_decoding && speq->timestamp >= *timestamp) { 688 *timestamp = speq->timestamp; 689 return 0; 690 } 691 } 692 693 return 0; 694 } 695 696 static int arm_spe__setup_queue(struct arm_spe *spe, 697 struct auxtrace_queue *queue, 698 unsigned int queue_nr) 699 { 700 struct arm_spe_queue *speq = queue->priv; 701 struct arm_spe_record *record; 702 703 if (list_empty(&queue->head) || speq) 704 return 0; 705 706 speq = arm_spe__alloc_queue(spe, queue_nr); 707 708 if (!speq) 709 return -ENOMEM; 710 711 queue->priv = speq; 712 713 if (queue->cpu != -1) 714 speq->cpu = queue->cpu; 715 716 if (!speq->on_heap) { 717 int ret; 718 719 if (spe->timeless_decoding) 720 return 0; 721 722 retry: 723 ret = arm_spe_decode(speq->decoder); 724 725 if (!ret) 726 return 0; 727 728 if (ret < 0) 729 goto retry; 730 731 record = &speq->decoder->record; 732 733 speq->timestamp = record->timestamp; 734 ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp); 735 if (ret) 736 return ret; 737 speq->on_heap = true; 738 } 739 740 return 0; 741 } 742 743 static int arm_spe__setup_queues(struct arm_spe *spe) 744 { 745 unsigned int i; 746 int ret; 747 748 for (i = 0; i < spe->queues.nr_queues; i++) { 749 ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i); 750 if (ret) 751 return ret; 752 } 753 754 return 0; 755 } 756 757 static int arm_spe__update_queues(struct arm_spe *spe) 758 { 759 if (spe->queues.new_data) { 760 spe->queues.new_data = false; 761 return arm_spe__setup_queues(spe); 762 } 763 764 return 0; 765 } 766 767 static bool arm_spe__is_timeless_decoding(struct arm_spe *spe) 768 { 769 struct evsel *evsel; 770 struct evlist *evlist = spe->session->evlist; 771 bool timeless_decoding = true; 772 773 /* 774 * Circle through the list of event and complain if we find one 775 * with the time bit set. 776 */ 777 evlist__for_each_entry(evlist, evsel) { 778 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) 779 timeless_decoding = false; 780 } 781 782 return timeless_decoding; 783 } 784 785 static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp) 786 { 787 unsigned int queue_nr; 788 u64 ts; 789 int ret; 790 791 while (1) { 792 struct auxtrace_queue *queue; 793 struct arm_spe_queue *speq; 794 795 if (!spe->heap.heap_cnt) 796 return 0; 797 798 if (spe->heap.heap_array[0].ordinal >= timestamp) 799 return 0; 800 801 queue_nr = spe->heap.heap_array[0].queue_nr; 802 queue = &spe->queues.queue_array[queue_nr]; 803 speq = queue->priv; 804 805 auxtrace_heap__pop(&spe->heap); 806 807 if (spe->heap.heap_cnt) { 808 ts = spe->heap.heap_array[0].ordinal + 1; 809 if (ts > timestamp) 810 ts = timestamp; 811 } else { 812 ts = timestamp; 813 } 814 815 /* 816 * A previous context-switch event has set pid/tid in the machine's context, so 817 * here we need to update the pid/tid in the thread and SPE queue. 818 */ 819 if (!spe->use_ctx_pkt_for_pid) 820 arm_spe_set_pid_tid_cpu(spe, queue); 821 822 ret = arm_spe_run_decoder(speq, &ts); 823 if (ret < 0) { 824 auxtrace_heap__add(&spe->heap, queue_nr, ts); 825 return ret; 826 } 827 828 if (!ret) { 829 ret = auxtrace_heap__add(&spe->heap, queue_nr, ts); 830 if (ret < 0) 831 return ret; 832 } else { 833 speq->on_heap = false; 834 } 835 } 836 837 return 0; 838 } 839 840 static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid, 841 u64 time_) 842 { 843 struct auxtrace_queues *queues = &spe->queues; 844 unsigned int i; 845 u64 ts = 0; 846 847 for (i = 0; i < queues->nr_queues; i++) { 848 struct auxtrace_queue *queue = &spe->queues.queue_array[i]; 849 struct arm_spe_queue *speq = queue->priv; 850 851 if (speq && (tid == -1 || speq->tid == tid)) { 852 speq->time = time_; 853 arm_spe_set_pid_tid_cpu(spe, queue); 854 arm_spe_run_decoder(speq, &ts); 855 } 856 } 857 return 0; 858 } 859 860 static int arm_spe_context_switch(struct arm_spe *spe, union perf_event *event, 861 struct perf_sample *sample) 862 { 863 pid_t pid, tid; 864 int cpu; 865 866 if (!(event->header.misc & PERF_RECORD_MISC_SWITCH_OUT)) 867 return 0; 868 869 pid = event->context_switch.next_prev_pid; 870 tid = event->context_switch.next_prev_tid; 871 cpu = sample->cpu; 872 873 if (tid == -1) 874 pr_warning("context_switch event has no tid\n"); 875 876 return machine__set_current_tid(spe->machine, cpu, pid, tid); 877 } 878 879 static int arm_spe_process_event(struct perf_session *session, 880 union perf_event *event, 881 struct perf_sample *sample, 882 struct perf_tool *tool) 883 { 884 int err = 0; 885 u64 timestamp; 886 struct arm_spe *spe = container_of(session->auxtrace, 887 struct arm_spe, auxtrace); 888 889 if (dump_trace) 890 return 0; 891 892 if (!tool->ordered_events) { 893 pr_err("SPE trace requires ordered events\n"); 894 return -EINVAL; 895 } 896 897 if (sample->time && (sample->time != (u64) -1)) 898 timestamp = perf_time_to_tsc(sample->time, &spe->tc); 899 else 900 timestamp = 0; 901 902 if (timestamp || spe->timeless_decoding) { 903 err = arm_spe__update_queues(spe); 904 if (err) 905 return err; 906 } 907 908 if (spe->timeless_decoding) { 909 if (event->header.type == PERF_RECORD_EXIT) { 910 err = arm_spe_process_timeless_queues(spe, 911 event->fork.tid, 912 sample->time); 913 } 914 } else if (timestamp) { 915 err = arm_spe_process_queues(spe, timestamp); 916 if (err) 917 return err; 918 919 if (!spe->use_ctx_pkt_for_pid && 920 (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE || 921 event->header.type == PERF_RECORD_SWITCH)) 922 err = arm_spe_context_switch(spe, event, sample); 923 } 924 925 return err; 926 } 927 928 static int arm_spe_process_auxtrace_event(struct perf_session *session, 929 union perf_event *event, 930 struct perf_tool *tool __maybe_unused) 931 { 932 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, 933 auxtrace); 934 935 if (!spe->data_queued) { 936 struct auxtrace_buffer *buffer; 937 off_t data_offset; 938 int fd = perf_data__fd(session->data); 939 int err; 940 941 if (perf_data__is_pipe(session->data)) { 942 data_offset = 0; 943 } else { 944 data_offset = lseek(fd, 0, SEEK_CUR); 945 if (data_offset == -1) 946 return -errno; 947 } 948 949 err = auxtrace_queues__add_event(&spe->queues, session, event, 950 data_offset, &buffer); 951 if (err) 952 return err; 953 954 /* Dump here now we have copied a piped trace out of the pipe */ 955 if (dump_trace) { 956 if (auxtrace_buffer__get_data(buffer, fd)) { 957 arm_spe_dump_event(spe, buffer->data, 958 buffer->size); 959 auxtrace_buffer__put_data(buffer); 960 } 961 } 962 } 963 964 return 0; 965 } 966 967 static int arm_spe_flush(struct perf_session *session __maybe_unused, 968 struct perf_tool *tool __maybe_unused) 969 { 970 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, 971 auxtrace); 972 int ret; 973 974 if (dump_trace) 975 return 0; 976 977 if (!tool->ordered_events) 978 return -EINVAL; 979 980 ret = arm_spe__update_queues(spe); 981 if (ret < 0) 982 return ret; 983 984 if (spe->timeless_decoding) 985 return arm_spe_process_timeless_queues(spe, -1, 986 MAX_TIMESTAMP - 1); 987 988 ret = arm_spe_process_queues(spe, MAX_TIMESTAMP); 989 if (ret) 990 return ret; 991 992 if (!spe->use_ctx_pkt_for_pid) 993 ui__warning("Arm SPE CONTEXT packets not found in the traces.\n" 994 "Matching of TIDs to SPE events could be inaccurate.\n"); 995 996 return 0; 997 } 998 999 static void arm_spe_free_queue(void *priv) 1000 { 1001 struct arm_spe_queue *speq = priv; 1002 1003 if (!speq) 1004 return; 1005 thread__zput(speq->thread); 1006 arm_spe_decoder_free(speq->decoder); 1007 zfree(&speq->event_buf); 1008 free(speq); 1009 } 1010 1011 static void arm_spe_free_events(struct perf_session *session) 1012 { 1013 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, 1014 auxtrace); 1015 struct auxtrace_queues *queues = &spe->queues; 1016 unsigned int i; 1017 1018 for (i = 0; i < queues->nr_queues; i++) { 1019 arm_spe_free_queue(queues->queue_array[i].priv); 1020 queues->queue_array[i].priv = NULL; 1021 } 1022 auxtrace_queues__free(queues); 1023 } 1024 1025 static void arm_spe_free(struct perf_session *session) 1026 { 1027 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, 1028 auxtrace); 1029 1030 auxtrace_heap__free(&spe->heap); 1031 arm_spe_free_events(session); 1032 session->auxtrace = NULL; 1033 free(spe); 1034 } 1035 1036 static bool arm_spe_evsel_is_auxtrace(struct perf_session *session, 1037 struct evsel *evsel) 1038 { 1039 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace); 1040 1041 return evsel->core.attr.type == spe->pmu_type; 1042 } 1043 1044 static const char * const arm_spe_info_fmts[] = { 1045 [ARM_SPE_PMU_TYPE] = " PMU Type %"PRId64"\n", 1046 }; 1047 1048 static void arm_spe_print_info(__u64 *arr) 1049 { 1050 if (!dump_trace) 1051 return; 1052 1053 fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]); 1054 } 1055 1056 struct arm_spe_synth { 1057 struct perf_tool dummy_tool; 1058 struct perf_session *session; 1059 }; 1060 1061 static int arm_spe_event_synth(struct perf_tool *tool, 1062 union perf_event *event, 1063 struct perf_sample *sample __maybe_unused, 1064 struct machine *machine __maybe_unused) 1065 { 1066 struct arm_spe_synth *arm_spe_synth = 1067 container_of(tool, struct arm_spe_synth, dummy_tool); 1068 1069 return perf_session__deliver_synth_event(arm_spe_synth->session, 1070 event, NULL); 1071 } 1072 1073 static int arm_spe_synth_event(struct perf_session *session, 1074 struct perf_event_attr *attr, u64 id) 1075 { 1076 struct arm_spe_synth arm_spe_synth; 1077 1078 memset(&arm_spe_synth, 0, sizeof(struct arm_spe_synth)); 1079 arm_spe_synth.session = session; 1080 1081 return perf_event__synthesize_attr(&arm_spe_synth.dummy_tool, attr, 1, 1082 &id, arm_spe_event_synth); 1083 } 1084 1085 static void arm_spe_set_event_name(struct evlist *evlist, u64 id, 1086 const char *name) 1087 { 1088 struct evsel *evsel; 1089 1090 evlist__for_each_entry(evlist, evsel) { 1091 if (evsel->core.id && evsel->core.id[0] == id) { 1092 if (evsel->name) 1093 zfree(&evsel->name); 1094 evsel->name = strdup(name); 1095 break; 1096 } 1097 } 1098 } 1099 1100 static int 1101 arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session) 1102 { 1103 struct evlist *evlist = session->evlist; 1104 struct evsel *evsel; 1105 struct perf_event_attr attr; 1106 bool found = false; 1107 u64 id; 1108 int err; 1109 1110 evlist__for_each_entry(evlist, evsel) { 1111 if (evsel->core.attr.type == spe->pmu_type) { 1112 found = true; 1113 break; 1114 } 1115 } 1116 1117 if (!found) { 1118 pr_debug("No selected events with SPE trace data\n"); 1119 return 0; 1120 } 1121 1122 memset(&attr, 0, sizeof(struct perf_event_attr)); 1123 attr.size = sizeof(struct perf_event_attr); 1124 attr.type = PERF_TYPE_HARDWARE; 1125 attr.sample_type = evsel->core.attr.sample_type & 1126 (PERF_SAMPLE_MASK | PERF_SAMPLE_PHYS_ADDR); 1127 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | 1128 PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC | 1129 PERF_SAMPLE_WEIGHT | PERF_SAMPLE_ADDR; 1130 if (spe->timeless_decoding) 1131 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; 1132 else 1133 attr.sample_type |= PERF_SAMPLE_TIME; 1134 1135 spe->sample_type = attr.sample_type; 1136 1137 attr.exclude_user = evsel->core.attr.exclude_user; 1138 attr.exclude_kernel = evsel->core.attr.exclude_kernel; 1139 attr.exclude_hv = evsel->core.attr.exclude_hv; 1140 attr.exclude_host = evsel->core.attr.exclude_host; 1141 attr.exclude_guest = evsel->core.attr.exclude_guest; 1142 attr.sample_id_all = evsel->core.attr.sample_id_all; 1143 attr.read_format = evsel->core.attr.read_format; 1144 1145 /* create new id val to be a fixed offset from evsel id */ 1146 id = evsel->core.id[0] + 1000000000; 1147 1148 if (!id) 1149 id = 1; 1150 1151 if (spe->synth_opts.flc) { 1152 spe->sample_flc = true; 1153 1154 /* Level 1 data cache miss */ 1155 err = arm_spe_synth_event(session, &attr, id); 1156 if (err) 1157 return err; 1158 spe->l1d_miss_id = id; 1159 arm_spe_set_event_name(evlist, id, "l1d-miss"); 1160 id += 1; 1161 1162 /* Level 1 data cache access */ 1163 err = arm_spe_synth_event(session, &attr, id); 1164 if (err) 1165 return err; 1166 spe->l1d_access_id = id; 1167 arm_spe_set_event_name(evlist, id, "l1d-access"); 1168 id += 1; 1169 } 1170 1171 if (spe->synth_opts.llc) { 1172 spe->sample_llc = true; 1173 1174 /* Last level cache miss */ 1175 err = arm_spe_synth_event(session, &attr, id); 1176 if (err) 1177 return err; 1178 spe->llc_miss_id = id; 1179 arm_spe_set_event_name(evlist, id, "llc-miss"); 1180 id += 1; 1181 1182 /* Last level cache access */ 1183 err = arm_spe_synth_event(session, &attr, id); 1184 if (err) 1185 return err; 1186 spe->llc_access_id = id; 1187 arm_spe_set_event_name(evlist, id, "llc-access"); 1188 id += 1; 1189 } 1190 1191 if (spe->synth_opts.tlb) { 1192 spe->sample_tlb = true; 1193 1194 /* TLB miss */ 1195 err = arm_spe_synth_event(session, &attr, id); 1196 if (err) 1197 return err; 1198 spe->tlb_miss_id = id; 1199 arm_spe_set_event_name(evlist, id, "tlb-miss"); 1200 id += 1; 1201 1202 /* TLB access */ 1203 err = arm_spe_synth_event(session, &attr, id); 1204 if (err) 1205 return err; 1206 spe->tlb_access_id = id; 1207 arm_spe_set_event_name(evlist, id, "tlb-access"); 1208 id += 1; 1209 } 1210 1211 if (spe->synth_opts.branches) { 1212 spe->sample_branch = true; 1213 1214 /* Branch miss */ 1215 err = arm_spe_synth_event(session, &attr, id); 1216 if (err) 1217 return err; 1218 spe->branch_miss_id = id; 1219 arm_spe_set_event_name(evlist, id, "branch-miss"); 1220 id += 1; 1221 } 1222 1223 if (spe->synth_opts.remote_access) { 1224 spe->sample_remote_access = true; 1225 1226 /* Remote access */ 1227 err = arm_spe_synth_event(session, &attr, id); 1228 if (err) 1229 return err; 1230 spe->remote_access_id = id; 1231 arm_spe_set_event_name(evlist, id, "remote-access"); 1232 id += 1; 1233 } 1234 1235 if (spe->synth_opts.mem) { 1236 spe->sample_memory = true; 1237 1238 err = arm_spe_synth_event(session, &attr, id); 1239 if (err) 1240 return err; 1241 spe->memory_id = id; 1242 arm_spe_set_event_name(evlist, id, "memory"); 1243 id += 1; 1244 } 1245 1246 if (spe->synth_opts.instructions) { 1247 if (spe->synth_opts.period_type != PERF_ITRACE_PERIOD_INSTRUCTIONS) { 1248 pr_warning("Only instruction-based sampling period is currently supported by Arm SPE.\n"); 1249 goto synth_instructions_out; 1250 } 1251 if (spe->synth_opts.period > 1) 1252 pr_warning("Arm SPE has a hardware-based sample period.\n" 1253 "Additional instruction events will be discarded by --itrace\n"); 1254 1255 spe->sample_instructions = true; 1256 attr.config = PERF_COUNT_HW_INSTRUCTIONS; 1257 attr.sample_period = spe->synth_opts.period; 1258 spe->instructions_sample_period = attr.sample_period; 1259 err = arm_spe_synth_event(session, &attr, id); 1260 if (err) 1261 return err; 1262 spe->instructions_id = id; 1263 arm_spe_set_event_name(evlist, id, "instructions"); 1264 } 1265 synth_instructions_out: 1266 1267 return 0; 1268 } 1269 1270 int arm_spe_process_auxtrace_info(union perf_event *event, 1271 struct perf_session *session) 1272 { 1273 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; 1274 size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX; 1275 struct perf_record_time_conv *tc = &session->time_conv; 1276 const char *cpuid = perf_env__cpuid(session->evlist->env); 1277 u64 midr = strtol(cpuid, NULL, 16); 1278 struct arm_spe *spe; 1279 int err; 1280 1281 if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) + 1282 min_sz) 1283 return -EINVAL; 1284 1285 spe = zalloc(sizeof(struct arm_spe)); 1286 if (!spe) 1287 return -ENOMEM; 1288 1289 err = auxtrace_queues__init(&spe->queues); 1290 if (err) 1291 goto err_free; 1292 1293 spe->session = session; 1294 spe->machine = &session->machines.host; /* No kvm support */ 1295 spe->auxtrace_type = auxtrace_info->type; 1296 spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE]; 1297 spe->midr = midr; 1298 1299 spe->timeless_decoding = arm_spe__is_timeless_decoding(spe); 1300 1301 /* 1302 * The synthesized event PERF_RECORD_TIME_CONV has been handled ahead 1303 * and the parameters for hardware clock are stored in the session 1304 * context. Passes these parameters to the struct perf_tsc_conversion 1305 * in "spe->tc", which is used for later conversion between clock 1306 * counter and timestamp. 1307 * 1308 * For backward compatibility, copies the fields starting from 1309 * "time_cycles" only if they are contained in the event. 1310 */ 1311 spe->tc.time_shift = tc->time_shift; 1312 spe->tc.time_mult = tc->time_mult; 1313 spe->tc.time_zero = tc->time_zero; 1314 1315 if (event_contains(*tc, time_cycles)) { 1316 spe->tc.time_cycles = tc->time_cycles; 1317 spe->tc.time_mask = tc->time_mask; 1318 spe->tc.cap_user_time_zero = tc->cap_user_time_zero; 1319 spe->tc.cap_user_time_short = tc->cap_user_time_short; 1320 } 1321 1322 spe->auxtrace.process_event = arm_spe_process_event; 1323 spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event; 1324 spe->auxtrace.flush_events = arm_spe_flush; 1325 spe->auxtrace.free_events = arm_spe_free_events; 1326 spe->auxtrace.free = arm_spe_free; 1327 spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace; 1328 session->auxtrace = &spe->auxtrace; 1329 1330 arm_spe_print_info(&auxtrace_info->priv[0]); 1331 1332 if (dump_trace) 1333 return 0; 1334 1335 if (session->itrace_synth_opts && session->itrace_synth_opts->set) 1336 spe->synth_opts = *session->itrace_synth_opts; 1337 else 1338 itrace_synth_opts__set_default(&spe->synth_opts, false); 1339 1340 err = arm_spe_synth_events(spe, session); 1341 if (err) 1342 goto err_free_queues; 1343 1344 err = auxtrace_queues__process_index(&spe->queues, session); 1345 if (err) 1346 goto err_free_queues; 1347 1348 if (spe->queues.populated) 1349 spe->data_queued = true; 1350 1351 return 0; 1352 1353 err_free_queues: 1354 auxtrace_queues__free(&spe->queues); 1355 session->auxtrace = NULL; 1356 err_free: 1357 free(spe); 1358 return err; 1359 } 1360