1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright(C) 2015-2018 Linaro Limited. 4 * 5 * Author: Tor Jeremiassen <tor@ti.com> 6 * Author: Mathieu Poirier <mathieu.poirier@linaro.org> 7 */ 8 9 #include <limits.h> 10 #include <linux/bitfield.h> 11 #include <linux/bitops.h> 12 #include <linux/coresight-pmu.h> 13 #include <linux/err.h> 14 #include <linux/log2.h> 15 #include <linux/types.h> 16 #include <linux/zalloc.h> 17 18 #include <stdlib.h> 19 20 #include "auxtrace.h" 21 #include "callchain.h" 22 #include "color.h" 23 #include "cs-etm.h" 24 #include "cs-etm-decoder/cs-etm-decoder.h" 25 #include "debug.h" 26 #include "dso.h" 27 #include "evlist.h" 28 #include "intlist.h" 29 #include "machine.h" 30 #include "map.h" 31 #include "perf.h" 32 #include "session.h" 33 #include "map_symbol.h" 34 #include "branch.h" 35 #include "symbol.h" 36 #include "tool.h" 37 #include "thread.h" 38 #include "thread-stack.h" 39 #include "tsc.h" 40 #include <tools/libc_compat.h> 41 #include "util/synthetic-events.h" 42 #include "util/util.h" 43 44 struct cs_etm_auxtrace { 45 struct auxtrace auxtrace; 46 struct auxtrace_queues queues; 47 struct auxtrace_heap heap; 48 struct itrace_synth_opts synth_opts; 49 struct perf_session *session; 50 struct perf_tsc_conversion tc; 51 52 /* 53 * Timeless has no timestamps in the trace so overlapping mmap lookups 54 * are less accurate but produces smaller trace data. We use context IDs 55 * in the trace instead of matching timestamps with fork records so 56 * they're not really needed in the general case. Overlapping mmaps 57 * happen in cases like between a fork and an exec. 58 */ 59 bool timeless_decoding; 60 61 /* 62 * Per-thread ignores the trace channel ID and instead assumes that 63 * everything in a buffer comes from the same process regardless of 64 * which CPU it ran on. It also implies no context IDs so the TID is 65 * taken from the auxtrace buffer. 66 */ 67 bool per_thread_decoding; 68 bool snapshot_mode; 69 bool data_queued; 70 bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */ 71 bool use_thread_stack; 72 bool use_callchain; 73 74 int num_cpu; 75 u64 latest_kernel_timestamp; 76 u32 auxtrace_type; 77 u32 branches_filter; 78 u64 branches_sample_type; 79 u64 branches_id; 80 u64 instructions_sample_type; 81 u64 instructions_sample_period; 82 u64 instructions_id; 83 u64 **metadata; 84 unsigned int pmu_type; 85 enum cs_etm_pid_fmt pid_fmt; 86 }; 87 88 struct cs_etm_traceid_queue { 89 u8 trace_chan_id; 90 u64 period_instructions; 91 u64 kernel_start; 92 union perf_event *event_buf; 93 unsigned int br_stack_sz; 94 struct branch_stack *last_branch; 95 struct ip_callchain *callchain; 96 struct cs_etm_packet *prev_packet; 97 struct cs_etm_packet *packet; 98 struct cs_etm_packet_queue packet_queue; 99 100 struct thread *decode_thread; 101 ocsd_ex_level decode_el; 102 103 /* 104 * The frontend accesses the EL from '[prev_]packet' because it needs 105 * previous EL for branch and current EL for instruction samples. It's 106 * not possible to change thread in a single branch sample so no need to 107 * store or access the thread through the packet. 108 */ 109 struct thread *frontend_thread; 110 }; 111 112 enum cs_etm_format { 113 UNSET, 114 FORMATTED, 115 UNFORMATTED 116 }; 117 118 struct cs_etm_queue { 119 struct cs_etm_auxtrace *etm; 120 struct cs_etm_decoder *decoder; 121 struct auxtrace_buffer *buffer; 122 unsigned int queue_nr; 123 u8 pending_timestamp_chan_id; 124 enum cs_etm_format format; 125 u64 offset; 126 const unsigned char *buf; 127 size_t buf_len, buf_used; 128 /* Conversion between traceID and index in traceid_queues array */ 129 struct intlist *traceid_queues_list; 130 struct cs_etm_traceid_queue **traceid_queues; 131 /* Conversion between traceID and metadata pointers */ 132 struct intlist *traceid_list; 133 /* 134 * Same as traceid_list, but traceid_list may be a reference to another 135 * queue's which has a matching sink ID. 136 */ 137 struct intlist *own_traceid_list; 138 u32 sink_id; 139 }; 140 141 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm); 142 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, 143 pid_t tid); 144 static int cs_etm__get_data_block(struct cs_etm_queue *etmq); 145 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq); 146 static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata); 147 static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu); 148 static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata); 149 150 /* PTMs ETMIDR [11:8] set to b0011 */ 151 #define ETMIDR_PTM_VERSION 0x00000300 152 153 /* 154 * A struct auxtrace_heap_item only has a queue_nr and a timestamp to 155 * work with. One option is to modify to auxtrace_heap_XYZ() API or simply 156 * encode the etm queue number as the upper 16 bit and the channel as 157 * the lower 16 bit. 158 */ 159 #define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \ 160 (queue_nr << 16 | trace_chan_id) 161 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16) 162 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff) 163 #define SINK_UNSET ((u32) -1) 164 165 static u32 cs_etm__get_v7_protocol_version(u32 etmidr) 166 { 167 etmidr &= ETMIDR_PTM_VERSION; 168 169 if (etmidr == ETMIDR_PTM_VERSION) 170 return CS_ETM_PROTO_PTM; 171 172 return CS_ETM_PROTO_ETMV3; 173 } 174 175 static int cs_etm__get_magic(struct cs_etm_queue *etmq, u8 trace_chan_id, u64 *magic) 176 { 177 struct int_node *inode; 178 u64 *metadata; 179 180 inode = intlist__find(etmq->traceid_list, trace_chan_id); 181 if (!inode) 182 return -EINVAL; 183 184 metadata = inode->priv; 185 *magic = metadata[CS_ETM_MAGIC]; 186 return 0; 187 } 188 189 int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu) 190 { 191 struct int_node *inode; 192 u64 *metadata; 193 194 inode = intlist__find(etmq->traceid_list, trace_chan_id); 195 if (!inode) 196 return -EINVAL; 197 198 metadata = inode->priv; 199 *cpu = (int)metadata[CS_ETM_CPU]; 200 return 0; 201 } 202 203 /* 204 * The returned PID format is presented as an enum: 205 * 206 * CS_ETM_PIDFMT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced. 207 * CS_ETM_PIDFMT_CTXTID2: CONTEXTIDR_EL2 is traced. 208 * CS_ETM_PIDFMT_NONE: No context IDs 209 * 210 * It's possible that the two format attributes 'contextid1' and 'contextid2' 211 * are enabled at the same time when the session runs on an EL2 kernel. 212 * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be 213 * recorded in the trace data, the tool will selectively use 214 * CONTEXTIDR_EL2 as PID. 215 * 216 * The result is cached in etm->pid_fmt so this function only needs to be called 217 * when processing the aux info. 218 */ 219 static enum cs_etm_pid_fmt cs_etm__init_pid_fmt(u64 *metadata) 220 { 221 u64 val; 222 223 if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) { 224 val = metadata[CS_ETM_ETMCR]; 225 /* CONTEXTIDR is traced */ 226 if (val & ETMCR_CTXTID) 227 return CS_ETM_PIDFMT_CTXTID; 228 } else { 229 val = metadata[CS_ETMV4_TRCCONFIGR]; 230 /* CONTEXTIDR_EL2 is traced */ 231 if (val & (TRCCONFIGR_VMID | TRCCONFIGR_VMIDOPT)) 232 return CS_ETM_PIDFMT_CTXTID2; 233 /* CONTEXTIDR_EL1 is traced */ 234 else if (val & TRCCONFIGR_CID) 235 return CS_ETM_PIDFMT_CTXTID; 236 } 237 238 return CS_ETM_PIDFMT_NONE; 239 } 240 241 enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq) 242 { 243 return etmq->etm->pid_fmt; 244 } 245 246 static int cs_etm__insert_trace_id_node(struct cs_etm_queue *etmq, 247 u8 trace_chan_id, u64 *cpu_metadata) 248 { 249 /* Get an RB node for this CPU */ 250 struct int_node *inode = intlist__findnew(etmq->traceid_list, trace_chan_id); 251 252 /* Something went wrong, no need to continue */ 253 if (!inode) 254 return -ENOMEM; 255 256 /* Disallow re-mapping a different traceID to metadata pair. */ 257 if (inode->priv) { 258 u64 *curr_cpu_data = inode->priv; 259 u8 curr_chan_id; 260 int err; 261 262 if (curr_cpu_data[CS_ETM_CPU] != cpu_metadata[CS_ETM_CPU]) { 263 /* 264 * With > CORESIGHT_TRACE_IDS_MAX ETMs, overlapping IDs 265 * are expected (but not supported) in per-thread mode, 266 * rather than signifying an error. 267 */ 268 if (etmq->etm->per_thread_decoding) 269 pr_err("CS_ETM: overlapping Trace IDs aren't currently supported in per-thread mode\n"); 270 else 271 pr_err("CS_ETM: map mismatch between HW_ID packet CPU and Trace ID\n"); 272 273 return -EINVAL; 274 } 275 276 /* check that the mapped ID matches */ 277 err = cs_etm__metadata_get_trace_id(&curr_chan_id, curr_cpu_data); 278 if (err) 279 return err; 280 281 if (curr_chan_id != trace_chan_id) { 282 pr_err("CS_ETM: mismatch between CPU trace ID and HW_ID packet ID\n"); 283 return -EINVAL; 284 } 285 286 /* Skip re-adding the same mappings if everything matched */ 287 return 0; 288 } 289 290 /* Not one we've seen before, associate the traceID with the metadata pointer */ 291 inode->priv = cpu_metadata; 292 293 return 0; 294 } 295 296 static struct cs_etm_queue *cs_etm__get_queue(struct cs_etm_auxtrace *etm, int cpu) 297 { 298 if (etm->per_thread_decoding) 299 return etm->queues.queue_array[0].priv; 300 301 if (cpu < 0 || cpu >= (int)etm->queues.nr_queues) 302 return NULL; 303 304 return etm->queues.queue_array[cpu].priv; 305 } 306 307 static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id, 308 u64 *cpu_metadata) 309 { 310 struct cs_etm_queue *etmq; 311 312 /* 313 * If the queue is unformatted then only save one mapping in the 314 * queue associated with that CPU so only one decoder is made. 315 */ 316 etmq = cs_etm__get_queue(etm, cpu_metadata[CS_ETM_CPU]); 317 if (!etmq) 318 return -EINVAL; 319 320 if (etmq->format == UNFORMATTED) 321 return cs_etm__insert_trace_id_node(etmq, trace_chan_id, 322 cpu_metadata); 323 324 /* 325 * Otherwise, version 0 trace IDs are global so save them into every 326 * queue. 327 */ 328 for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) { 329 int ret; 330 331 etmq = etm->queues.queue_array[i].priv; 332 if (!etmq) 333 continue; 334 335 ret = cs_etm__insert_trace_id_node(etmq, trace_chan_id, 336 cpu_metadata); 337 if (ret) 338 return ret; 339 } 340 341 return 0; 342 } 343 344 static int cs_etm__process_trace_id_v0(struct cs_etm_auxtrace *etm, int cpu, 345 u64 hw_id) 346 { 347 int err; 348 u64 *cpu_data; 349 u8 trace_chan_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id); 350 351 cpu_data = get_cpu_data(etm, cpu); 352 if (cpu_data == NULL) 353 return -EINVAL; 354 355 err = cs_etm__map_trace_id_v0(etm, trace_chan_id, cpu_data); 356 if (err) 357 return err; 358 359 /* 360 * if we are picking up the association from the packet, need to plug 361 * the correct trace ID into the metadata for setting up decoders later. 362 */ 363 return cs_etm__metadata_set_trace_id(trace_chan_id, cpu_data); 364 } 365 366 static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, 367 u64 hw_id) 368 { 369 struct cs_etm_queue *etmq = cs_etm__get_queue(etm, cpu); 370 int ret; 371 u64 *cpu_data; 372 u32 sink_id = FIELD_GET(CS_AUX_HW_ID_SINK_ID_MASK, hw_id); 373 u8 trace_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id); 374 375 if (!etmq) 376 return -EINVAL; 377 378 /* 379 * Check sink id hasn't changed in per-cpu mode. In per-thread mode, 380 * let it pass for now until an actual overlapping trace ID is hit. In 381 * most cases IDs won't overlap even if the sink changes. 382 */ 383 if (!etmq->etm->per_thread_decoding && etmq->sink_id != SINK_UNSET && 384 etmq->sink_id != sink_id) { 385 pr_err("CS_ETM: mismatch between sink IDs\n"); 386 return -EINVAL; 387 } 388 389 etmq->sink_id = sink_id; 390 391 /* Find which other queues use this sink and link their ID maps */ 392 for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) { 393 struct cs_etm_queue *other_etmq = etm->queues.queue_array[i].priv; 394 395 if (!other_etmq) 396 continue; 397 398 /* Different sinks, skip */ 399 if (other_etmq->sink_id != etmq->sink_id) 400 continue; 401 402 /* Already linked, skip */ 403 if (other_etmq->traceid_list == etmq->traceid_list) 404 continue; 405 406 /* At the point of first linking, this one should be empty */ 407 if (!intlist__empty(etmq->traceid_list)) { 408 pr_err("CS_ETM: Can't link populated trace ID lists\n"); 409 return -EINVAL; 410 } 411 412 etmq->own_traceid_list = NULL; 413 intlist__delete(etmq->traceid_list); 414 etmq->traceid_list = other_etmq->traceid_list; 415 break; 416 } 417 418 cpu_data = get_cpu_data(etm, cpu); 419 if (!cpu_data) 420 return -EINVAL; 421 422 ret = cs_etm__insert_trace_id_node(etmq, trace_id, cpu_data); 423 if (ret) 424 return ret; 425 426 ret = cs_etm__metadata_set_trace_id(trace_id, cpu_data); 427 if (ret) 428 return ret; 429 430 return 0; 431 } 432 433 static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata) 434 { 435 u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC]; 436 437 switch (cs_etm_magic) { 438 case __perf_cs_etmv3_magic: 439 *trace_chan_id = (u8)(cpu_metadata[CS_ETM_ETMTRACEIDR] & 440 CORESIGHT_TRACE_ID_VAL_MASK); 441 break; 442 case __perf_cs_etmv4_magic: 443 case __perf_cs_ete_magic: 444 *trace_chan_id = (u8)(cpu_metadata[CS_ETMV4_TRCTRACEIDR] & 445 CORESIGHT_TRACE_ID_VAL_MASK); 446 break; 447 default: 448 return -EINVAL; 449 } 450 return 0; 451 } 452 453 /* 454 * update metadata trace ID from the value found in the AUX_HW_INFO packet. 455 */ 456 static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata) 457 { 458 u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC]; 459 460 switch (cs_etm_magic) { 461 case __perf_cs_etmv3_magic: 462 cpu_metadata[CS_ETM_ETMTRACEIDR] = trace_chan_id; 463 break; 464 case __perf_cs_etmv4_magic: 465 case __perf_cs_ete_magic: 466 cpu_metadata[CS_ETMV4_TRCTRACEIDR] = trace_chan_id; 467 break; 468 469 default: 470 return -EINVAL; 471 } 472 return 0; 473 } 474 475 /* 476 * Get a metadata index for a specific cpu from an array. 477 * 478 */ 479 static int get_cpu_data_idx(struct cs_etm_auxtrace *etm, int cpu) 480 { 481 int i; 482 483 for (i = 0; i < etm->num_cpu; i++) { 484 if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) { 485 return i; 486 } 487 } 488 489 return -1; 490 } 491 492 /* 493 * Get a metadata for a specific cpu from an array. 494 * 495 */ 496 static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu) 497 { 498 int idx = get_cpu_data_idx(etm, cpu); 499 500 return (idx != -1) ? etm->metadata[idx] : NULL; 501 } 502 503 /* 504 * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event. 505 * 506 * The payload associates the Trace ID and the CPU. 507 * The routine is tolerant of seeing multiple packets with the same association, 508 * but a CPU / Trace ID association changing during a session is an error. 509 */ 510 static int cs_etm__process_aux_output_hw_id(struct perf_session *session, 511 union perf_event *event) 512 { 513 struct cs_etm_auxtrace *etm; 514 struct perf_sample sample; 515 struct evsel *evsel; 516 u64 hw_id; 517 int cpu, version, err; 518 519 /* extract and parse the HW ID */ 520 hw_id = event->aux_output_hw_id.hw_id; 521 version = FIELD_GET(CS_AUX_HW_ID_MAJOR_VERSION_MASK, hw_id); 522 523 /* check that we can handle this version */ 524 if (version > CS_AUX_HW_ID_MAJOR_VERSION) { 525 pr_err("CS ETM Trace: PERF_RECORD_AUX_OUTPUT_HW_ID version %d not supported. Please update Perf.\n", 526 version); 527 return -EINVAL; 528 } 529 530 /* get access to the etm metadata */ 531 etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace); 532 if (!etm || !etm->metadata) 533 return -EINVAL; 534 535 /* parse the sample to get the CPU */ 536 evsel = evlist__event2evsel(session->evlist, event); 537 if (!evsel) 538 return -EINVAL; 539 perf_sample__init(&sample, /*all=*/false); 540 err = evsel__parse_sample(evsel, event, &sample); 541 if (err) 542 goto out; 543 cpu = sample.cpu; 544 if (cpu == -1) { 545 /* no CPU in the sample - possibly recorded with an old version of perf */ 546 pr_err("CS_ETM: no CPU AUX_OUTPUT_HW_ID sample. Use compatible perf to record."); 547 err = -EINVAL; 548 goto out; 549 } 550 551 if (FIELD_GET(CS_AUX_HW_ID_MINOR_VERSION_MASK, hw_id) == 0) { 552 err = cs_etm__process_trace_id_v0(etm, cpu, hw_id); 553 goto out; 554 } 555 556 err = cs_etm__process_trace_id_v0_1(etm, cpu, hw_id); 557 out: 558 perf_sample__exit(&sample); 559 return err; 560 } 561 562 void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, 563 u8 trace_chan_id) 564 { 565 /* 566 * When a timestamp packet is encountered the backend code 567 * is stopped so that the front end has time to process packets 568 * that were accumulated in the traceID queue. Since there can 569 * be more than one channel per cs_etm_queue, we need to specify 570 * what traceID queue needs servicing. 571 */ 572 etmq->pending_timestamp_chan_id = trace_chan_id; 573 } 574 575 static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq, 576 u8 *trace_chan_id) 577 { 578 struct cs_etm_packet_queue *packet_queue; 579 580 if (!etmq->pending_timestamp_chan_id) 581 return 0; 582 583 if (trace_chan_id) 584 *trace_chan_id = etmq->pending_timestamp_chan_id; 585 586 packet_queue = cs_etm__etmq_get_packet_queue(etmq, 587 etmq->pending_timestamp_chan_id); 588 if (!packet_queue) 589 return 0; 590 591 /* Acknowledge pending status */ 592 etmq->pending_timestamp_chan_id = 0; 593 594 /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */ 595 return packet_queue->cs_timestamp; 596 } 597 598 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue) 599 { 600 int i; 601 602 queue->head = 0; 603 queue->tail = 0; 604 queue->packet_count = 0; 605 for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) { 606 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN; 607 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR; 608 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR; 609 queue->packet_buffer[i].instr_count = 0; 610 queue->packet_buffer[i].last_instr_taken_branch = false; 611 queue->packet_buffer[i].last_instr_size = 0; 612 queue->packet_buffer[i].last_instr_type = 0; 613 queue->packet_buffer[i].last_instr_subtype = 0; 614 queue->packet_buffer[i].last_instr_cond = 0; 615 queue->packet_buffer[i].flags = 0; 616 queue->packet_buffer[i].exception_number = UINT32_MAX; 617 queue->packet_buffer[i].trace_chan_id = UINT8_MAX; 618 queue->packet_buffer[i].cpu = INT_MIN; 619 } 620 } 621 622 static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq) 623 { 624 int idx; 625 struct int_node *inode; 626 struct cs_etm_traceid_queue *tidq; 627 struct intlist *traceid_queues_list = etmq->traceid_queues_list; 628 629 intlist__for_each_entry(inode, traceid_queues_list) { 630 idx = (int)(intptr_t)inode->priv; 631 tidq = etmq->traceid_queues[idx]; 632 cs_etm__clear_packet_queue(&tidq->packet_queue); 633 } 634 } 635 636 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, 637 struct cs_etm_traceid_queue *tidq, 638 u8 trace_chan_id) 639 { 640 int rc = -ENOMEM; 641 struct auxtrace_queue *queue; 642 struct cs_etm_auxtrace *etm = etmq->etm; 643 644 cs_etm__clear_packet_queue(&tidq->packet_queue); 645 646 queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; 647 tidq->trace_chan_id = trace_chan_id; 648 tidq->decode_el = ocsd_EL_unknown; 649 tidq->frontend_thread = machine__findnew_thread(&etm->session->machines.host, -1, 650 queue->tid); 651 tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1, 652 queue->tid); 653 if (!tidq->frontend_thread || !tidq->decode_thread) 654 goto out; 655 656 tidq->packet = zalloc(sizeof(struct cs_etm_packet)); 657 if (!tidq->packet) 658 goto out; 659 660 tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet)); 661 if (!tidq->prev_packet) 662 goto out_free; 663 664 if (etm->use_thread_stack) { 665 size_t sz = sizeof(struct branch_stack); 666 667 sz += etm->synth_opts.last_branch_sz * 668 sizeof(struct branch_entry); 669 tidq->last_branch = zalloc(sz); 670 if (!tidq->last_branch) 671 goto out_free; 672 673 tidq->br_stack_sz = etm->synth_opts.last_branch_sz; 674 } 675 676 if (etm->synth_opts.callchain) { 677 /* Add 1 to callchain_sz for callchain context */ 678 tidq->callchain = 679 zalloc(struct_size(tidq->callchain, ips, 680 etm->synth_opts.callchain_sz + 1)); 681 if (!tidq->callchain) 682 goto out_free; 683 } 684 685 tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); 686 if (!tidq->event_buf) 687 goto out_free; 688 689 return 0; 690 691 out_free: 692 zfree(&tidq->callchain); 693 zfree(&tidq->last_branch); 694 zfree(&tidq->prev_packet); 695 zfree(&tidq->packet); 696 out: 697 thread__zput(tidq->frontend_thread); 698 thread__zput(tidq->decode_thread); 699 return rc; 700 } 701 702 static struct cs_etm_traceid_queue 703 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) 704 { 705 int idx; 706 struct int_node *inode; 707 struct intlist *traceid_queues_list; 708 struct cs_etm_traceid_queue *tidq, **traceid_queues; 709 struct cs_etm_auxtrace *etm = etmq->etm; 710 711 if (etm->per_thread_decoding) 712 trace_chan_id = CS_ETM_PER_THREAD_TRACEID; 713 714 traceid_queues_list = etmq->traceid_queues_list; 715 716 /* 717 * Check if the traceid_queue exist for this traceID by looking 718 * in the queue list. 719 */ 720 inode = intlist__find(traceid_queues_list, trace_chan_id); 721 if (inode) { 722 idx = (int)(intptr_t)inode->priv; 723 return etmq->traceid_queues[idx]; 724 } 725 726 /* We couldn't find a traceid_queue for this traceID, allocate one */ 727 tidq = malloc(sizeof(*tidq)); 728 if (!tidq) 729 return NULL; 730 731 memset(tidq, 0, sizeof(*tidq)); 732 733 /* Get a valid index for the new traceid_queue */ 734 idx = intlist__nr_entries(traceid_queues_list); 735 /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */ 736 inode = intlist__findnew(traceid_queues_list, trace_chan_id); 737 if (!inode) 738 goto out_free; 739 740 /* Associate this traceID with this index */ 741 inode->priv = (void *)(intptr_t)idx; 742 743 if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id)) 744 goto out_free; 745 746 /* Grow the traceid_queues array by one unit */ 747 traceid_queues = etmq->traceid_queues; 748 traceid_queues = reallocarray(traceid_queues, 749 idx + 1, 750 sizeof(*traceid_queues)); 751 752 /* 753 * On failure reallocarray() returns NULL and the original block of 754 * memory is left untouched. 755 */ 756 if (!traceid_queues) 757 goto out_free; 758 759 traceid_queues[idx] = tidq; 760 etmq->traceid_queues = traceid_queues; 761 762 return etmq->traceid_queues[idx]; 763 764 out_free: 765 /* 766 * Function intlist__remove() removes the inode from the list 767 * and delete the memory associated to it. 768 */ 769 intlist__remove(traceid_queues_list, inode); 770 free(tidq); 771 772 return NULL; 773 } 774 775 struct cs_etm_packet_queue 776 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) 777 { 778 struct cs_etm_traceid_queue *tidq; 779 780 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 781 if (tidq) 782 return &tidq->packet_queue; 783 784 return NULL; 785 } 786 787 static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm, 788 struct cs_etm_traceid_queue *tidq) 789 { 790 struct cs_etm_packet *tmp; 791 792 if (etm->synth_opts.branches || etm->synth_opts.last_branch || 793 etm->synth_opts.instructions) { 794 /* 795 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for 796 * the next incoming packet. 797 */ 798 tmp = tidq->packet; 799 tidq->packet = tidq->prev_packet; 800 tidq->prev_packet = tmp; 801 } 802 } 803 804 static void cs_etm__packet_dump(const char *pkt_string, void *data) 805 { 806 const char *color = PERF_COLOR_BLUE; 807 int len = strlen(pkt_string); 808 struct cs_etm_queue *etmq = data; 809 char queue_nr[64]; 810 811 if (verbose) 812 snprintf(queue_nr, sizeof(queue_nr), "Qnr:%u; ", etmq->queue_nr); 813 else 814 queue_nr[0] = '\0'; 815 816 if (len && (pkt_string[len-1] == '\n')) 817 color_fprintf(stdout, color, " %s%s", queue_nr, pkt_string); 818 else 819 color_fprintf(stdout, color, " %s%s\n", queue_nr, pkt_string); 820 821 fflush(stdout); 822 } 823 824 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params, 825 u64 *metadata, u32 etmidr) 826 { 827 t_params->protocol = cs_etm__get_v7_protocol_version(etmidr); 828 t_params->etmv3.reg_ctrl = metadata[CS_ETM_ETMCR]; 829 t_params->etmv3.reg_trc_id = metadata[CS_ETM_ETMTRACEIDR]; 830 } 831 832 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params, 833 u64 *metadata) 834 { 835 t_params->protocol = CS_ETM_PROTO_ETMV4i; 836 t_params->etmv4.reg_idr0 = metadata[CS_ETMV4_TRCIDR0]; 837 t_params->etmv4.reg_idr1 = metadata[CS_ETMV4_TRCIDR1]; 838 t_params->etmv4.reg_idr2 = metadata[CS_ETMV4_TRCIDR2]; 839 t_params->etmv4.reg_idr8 = metadata[CS_ETMV4_TRCIDR8]; 840 t_params->etmv4.reg_configr = metadata[CS_ETMV4_TRCCONFIGR]; 841 t_params->etmv4.reg_traceidr = metadata[CS_ETMV4_TRCTRACEIDR]; 842 } 843 844 static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params, 845 u64 *metadata) 846 { 847 t_params->protocol = CS_ETM_PROTO_ETE; 848 t_params->ete.reg_idr0 = metadata[CS_ETE_TRCIDR0]; 849 t_params->ete.reg_idr1 = metadata[CS_ETE_TRCIDR1]; 850 t_params->ete.reg_idr2 = metadata[CS_ETE_TRCIDR2]; 851 t_params->ete.reg_idr8 = metadata[CS_ETE_TRCIDR8]; 852 t_params->ete.reg_configr = metadata[CS_ETE_TRCCONFIGR]; 853 t_params->ete.reg_traceidr = metadata[CS_ETE_TRCTRACEIDR]; 854 t_params->ete.reg_devarch = metadata[CS_ETE_TRCDEVARCH]; 855 } 856 857 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params, 858 struct cs_etm_queue *etmq) 859 { 860 struct int_node *inode; 861 862 intlist__for_each_entry(inode, etmq->traceid_list) { 863 u64 *metadata = inode->priv; 864 u64 architecture = metadata[CS_ETM_MAGIC]; 865 u32 etmidr; 866 867 switch (architecture) { 868 case __perf_cs_etmv3_magic: 869 etmidr = metadata[CS_ETM_ETMIDR]; 870 cs_etm__set_trace_param_etmv3(t_params++, metadata, etmidr); 871 break; 872 case __perf_cs_etmv4_magic: 873 cs_etm__set_trace_param_etmv4(t_params++, metadata); 874 break; 875 case __perf_cs_ete_magic: 876 cs_etm__set_trace_param_ete(t_params++, metadata); 877 break; 878 default: 879 return -EINVAL; 880 } 881 } 882 883 return 0; 884 } 885 886 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params, 887 struct cs_etm_queue *etmq, 888 enum cs_etm_decoder_operation mode) 889 { 890 int ret = -EINVAL; 891 892 if (!(mode < CS_ETM_OPERATION_MAX)) 893 goto out; 894 895 d_params->packet_printer = cs_etm__packet_dump; 896 d_params->operation = mode; 897 d_params->data = etmq; 898 d_params->formatted = etmq->format == FORMATTED; 899 d_params->fsyncs = false; 900 d_params->hsyncs = false; 901 d_params->frame_aligned = true; 902 903 ret = 0; 904 out: 905 return ret; 906 } 907 908 static void cs_etm__dump_event(struct cs_etm_queue *etmq, 909 struct auxtrace_buffer *buffer) 910 { 911 int ret; 912 const char *color = PERF_COLOR_BLUE; 913 size_t buffer_used = 0; 914 915 fprintf(stdout, "\n"); 916 color_fprintf(stdout, color, 917 ". ... CoreSight %s Trace data: size %#zx bytes\n", 918 cs_etm_decoder__get_name(etmq->decoder), buffer->size); 919 920 do { 921 size_t consumed; 922 923 ret = cs_etm_decoder__process_data_block( 924 etmq->decoder, buffer->offset, 925 &((u8 *)buffer->data)[buffer_used], 926 buffer->size - buffer_used, &consumed); 927 if (ret) 928 break; 929 930 buffer_used += consumed; 931 } while (buffer_used < buffer->size); 932 933 cs_etm_decoder__reset(etmq->decoder); 934 } 935 936 static int cs_etm__flush_events(struct perf_session *session, 937 const struct perf_tool *tool) 938 { 939 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 940 struct cs_etm_auxtrace, 941 auxtrace); 942 if (dump_trace) 943 return 0; 944 945 if (!tool->ordered_events) 946 return -EINVAL; 947 948 if (etm->timeless_decoding) { 949 /* 950 * Pass tid = -1 to process all queues. But likely they will have 951 * already been processed on PERF_RECORD_EXIT anyway. 952 */ 953 return cs_etm__process_timeless_queues(etm, -1); 954 } 955 956 return cs_etm__process_timestamped_queues(etm); 957 } 958 959 static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq) 960 { 961 int idx; 962 uintptr_t priv; 963 struct int_node *inode, *tmp; 964 struct cs_etm_traceid_queue *tidq; 965 struct intlist *traceid_queues_list = etmq->traceid_queues_list; 966 967 intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) { 968 priv = (uintptr_t)inode->priv; 969 idx = priv; 970 971 /* Free this traceid_queue from the array */ 972 tidq = etmq->traceid_queues[idx]; 973 thread__zput(tidq->frontend_thread); 974 thread__zput(tidq->decode_thread); 975 zfree(&tidq->event_buf); 976 zfree(&tidq->callchain); 977 zfree(&tidq->last_branch); 978 zfree(&tidq->prev_packet); 979 zfree(&tidq->packet); 980 zfree(&tidq); 981 982 /* 983 * Function intlist__remove() removes the inode from the list 984 * and delete the memory associated to it. 985 */ 986 intlist__remove(traceid_queues_list, inode); 987 } 988 989 /* Then the RB tree itself */ 990 intlist__delete(traceid_queues_list); 991 etmq->traceid_queues_list = NULL; 992 993 /* finally free the traceid_queues array */ 994 zfree(&etmq->traceid_queues); 995 } 996 997 static void cs_etm__free_queue(void *priv) 998 { 999 struct int_node *inode, *tmp; 1000 struct cs_etm_queue *etmq = priv; 1001 1002 if (!etmq) 1003 return; 1004 1005 cs_etm_decoder__free(etmq->decoder); 1006 cs_etm__free_traceid_queues(etmq); 1007 1008 if (etmq->own_traceid_list) { 1009 /* First remove all traceID/metadata nodes for the RB tree */ 1010 intlist__for_each_entry_safe(inode, tmp, etmq->own_traceid_list) 1011 intlist__remove(etmq->own_traceid_list, inode); 1012 1013 /* Then the RB tree itself */ 1014 intlist__delete(etmq->own_traceid_list); 1015 } 1016 1017 free(etmq); 1018 } 1019 1020 static void cs_etm__free_events(struct perf_session *session) 1021 { 1022 unsigned int i; 1023 struct cs_etm_auxtrace *aux = container_of(session->auxtrace, 1024 struct cs_etm_auxtrace, 1025 auxtrace); 1026 struct auxtrace_queues *queues = &aux->queues; 1027 1028 for (i = 0; i < queues->nr_queues; i++) { 1029 cs_etm__free_queue(queues->queue_array[i].priv); 1030 queues->queue_array[i].priv = NULL; 1031 } 1032 1033 auxtrace_queues__free(queues); 1034 } 1035 1036 static void cs_etm__free(struct perf_session *session) 1037 { 1038 int i; 1039 struct cs_etm_auxtrace *aux = container_of(session->auxtrace, 1040 struct cs_etm_auxtrace, 1041 auxtrace); 1042 cs_etm__free_events(session); 1043 session->auxtrace = NULL; 1044 1045 for (i = 0; i < aux->num_cpu; i++) 1046 zfree(&aux->metadata[i]); 1047 1048 zfree(&aux->metadata); 1049 zfree(&aux); 1050 } 1051 1052 static bool cs_etm__evsel_is_auxtrace(struct perf_session *session, 1053 struct evsel *evsel) 1054 { 1055 struct cs_etm_auxtrace *aux = container_of(session->auxtrace, 1056 struct cs_etm_auxtrace, 1057 auxtrace); 1058 1059 return evsel->core.attr.type == aux->pmu_type; 1060 } 1061 1062 static struct machine *cs_etm__get_machine(struct cs_etm_queue *etmq, 1063 ocsd_ex_level el) 1064 { 1065 enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq); 1066 1067 /* 1068 * For any virtualisation based on nVHE (e.g. pKVM), or host kernels 1069 * running at EL1 assume everything is the host. 1070 */ 1071 if (pid_fmt == CS_ETM_PIDFMT_CTXTID) 1072 return &etmq->etm->session->machines.host; 1073 1074 /* 1075 * Not perfect, but otherwise assume anything in EL1 is the default 1076 * guest, and everything else is the host. Distinguishing between guest 1077 * and host userspaces isn't currently supported either. Neither is 1078 * multiple guest support. All this does is reduce the likeliness of 1079 * decode errors where we look into the host kernel maps when it should 1080 * have been the guest maps. 1081 */ 1082 switch (el) { 1083 case ocsd_EL1: 1084 return machines__find_guest(&etmq->etm->session->machines, 1085 DEFAULT_GUEST_KERNEL_ID); 1086 case ocsd_EL3: 1087 case ocsd_EL2: 1088 case ocsd_EL0: 1089 case ocsd_EL_unknown: 1090 default: 1091 return &etmq->etm->session->machines.host; 1092 } 1093 } 1094 1095 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address, 1096 ocsd_ex_level el) 1097 { 1098 struct machine *machine = cs_etm__get_machine(etmq, el); 1099 1100 if (address >= machine__kernel_start(machine)) { 1101 if (machine__is_host(machine)) 1102 return PERF_RECORD_MISC_KERNEL; 1103 else 1104 return PERF_RECORD_MISC_GUEST_KERNEL; 1105 } else { 1106 if (machine__is_host(machine)) 1107 return PERF_RECORD_MISC_USER; 1108 else { 1109 /* 1110 * Can't really happen at the moment because 1111 * cs_etm__get_machine() will always return 1112 * machines.host for any non EL1 trace. 1113 */ 1114 return PERF_RECORD_MISC_GUEST_USER; 1115 } 1116 } 1117 } 1118 1119 static u32 __cs_etm__mem_access(struct cs_etm_queue *etmq, 1120 u64 address, size_t size, u8 *buffer, 1121 const ocsd_mem_space_acc_t mem_space, 1122 ocsd_ex_level el, struct thread *thread) 1123 { 1124 u8 cpumode; 1125 u64 offset; 1126 int len; 1127 struct addr_location al; 1128 struct dso *dso; 1129 int ret = 0; 1130 1131 if (!etmq) 1132 return 0; 1133 1134 addr_location__init(&al); 1135 1136 /* 1137 * We track EL for the frontend and the backend when receiving context 1138 * and range packets. OpenCSD doesn't distinguish between EL0 and EL1 1139 * for this mem access callback so we had to do the extra tracking. Skip 1140 * validation if it's any of the 'any' values. 1141 */ 1142 if (!(mem_space == OCSD_MEM_SPACE_ANY || 1143 mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) { 1144 if (mem_space & OCSD_MEM_SPACE_EL1N) { 1145 /* Includes both non secure EL1 and EL0 */ 1146 assert(el == ocsd_EL1 || el == ocsd_EL0); 1147 } else if (mem_space & OCSD_MEM_SPACE_EL2) 1148 assert(el == ocsd_EL2); 1149 else if (mem_space & OCSD_MEM_SPACE_EL3) 1150 assert(el == ocsd_EL3); 1151 } 1152 1153 cpumode = cs_etm__cpu_mode(etmq, address, el); 1154 1155 if (!thread__find_map(thread, cpumode, address, &al)) 1156 goto out; 1157 1158 dso = map__dso(al.map); 1159 if (!dso) 1160 goto out; 1161 1162 if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR && 1163 dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE)) 1164 goto out; 1165 1166 offset = map__map_ip(al.map, address); 1167 1168 map__load(al.map); 1169 1170 len = dso__data_read_offset(dso, maps__machine(thread__maps(thread)), 1171 offset, buffer, size); 1172 1173 if (len <= 0) { 1174 ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n" 1175 " Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n"); 1176 if (!dso__auxtrace_warned(dso)) { 1177 pr_err("CS ETM Trace: Debug data not found for address %#"PRIx64" in %s\n", 1178 address, 1179 dso__long_name(dso) ? dso__long_name(dso) : "Unknown"); 1180 dso__set_auxtrace_warned(dso); 1181 } 1182 goto out; 1183 } 1184 ret = len; 1185 out: 1186 addr_location__exit(&al); 1187 return ret; 1188 } 1189 1190 static u32 cs_etm__frontend_mem_access(struct cs_etm_queue *etmq, 1191 struct cs_etm_traceid_queue *tidq, 1192 struct cs_etm_packet *packet, 1193 u64 address, size_t size, u8 *buffer) 1194 { 1195 return __cs_etm__mem_access(etmq, address, size, buffer, 0, packet->el, 1196 tidq->frontend_thread); 1197 } 1198 1199 static u32 cs_etm__decoder_mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, 1200 u64 address, size_t size, u8 *buffer, 1201 const ocsd_mem_space_acc_t mem_space) 1202 { 1203 struct cs_etm_traceid_queue *tidq; 1204 1205 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 1206 if (!tidq) 1207 return 0; 1208 1209 return __cs_etm__mem_access(etmq, address, size, buffer, 1210 mem_space, tidq->decode_el, 1211 tidq->decode_thread); 1212 } 1213 1214 static struct cs_etm_queue *cs_etm__alloc_queue(void) 1215 { 1216 struct cs_etm_queue *etmq = zalloc(sizeof(*etmq)); 1217 if (!etmq) 1218 return NULL; 1219 1220 etmq->traceid_queues_list = intlist__new(NULL); 1221 if (!etmq->traceid_queues_list) 1222 goto out_free; 1223 1224 /* 1225 * Create an RB tree for traceID-metadata tuple. Since the conversion 1226 * has to be made for each packet that gets decoded, optimizing access 1227 * in anything other than a sequential array is worth doing. 1228 */ 1229 etmq->traceid_list = etmq->own_traceid_list = intlist__new(NULL); 1230 if (!etmq->traceid_list) 1231 goto out_free; 1232 1233 return etmq; 1234 1235 out_free: 1236 intlist__delete(etmq->traceid_queues_list); 1237 free(etmq); 1238 1239 return NULL; 1240 } 1241 1242 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, 1243 struct auxtrace_queue *queue, 1244 unsigned int queue_nr) 1245 { 1246 struct cs_etm_queue *etmq = queue->priv; 1247 1248 if (etmq) 1249 return 0; 1250 1251 etmq = cs_etm__alloc_queue(); 1252 1253 if (!etmq) 1254 return -ENOMEM; 1255 1256 queue->priv = etmq; 1257 etmq->etm = etm; 1258 etmq->queue_nr = queue_nr; 1259 queue->cpu = queue_nr; /* Placeholder, may be reset to -1 in per-thread mode */ 1260 etmq->offset = 0; 1261 etmq->sink_id = SINK_UNSET; 1262 1263 return 0; 1264 } 1265 1266 static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm, 1267 struct cs_etm_queue *etmq, 1268 unsigned int queue_nr) 1269 { 1270 int ret = 0; 1271 unsigned int cs_queue_nr; 1272 u8 trace_chan_id; 1273 u64 cs_timestamp; 1274 1275 /* 1276 * We are under a CPU-wide trace scenario. As such we need to know 1277 * when the code that generated the traces started to execute so that 1278 * it can be correlated with execution on other CPUs. So we get a 1279 * handle on the beginning of traces and decode until we find a 1280 * timestamp. The timestamp is then added to the auxtrace min heap 1281 * in order to know what nibble (of all the etmqs) to decode first. 1282 */ 1283 while (1) { 1284 /* 1285 * Fetch an aux_buffer from this etmq. Bail if no more 1286 * blocks or an error has been encountered. 1287 */ 1288 ret = cs_etm__get_data_block(etmq); 1289 if (ret <= 0) 1290 goto out; 1291 1292 /* 1293 * Run decoder on the trace block. The decoder will stop when 1294 * encountering a CS timestamp, a full packet queue or the end of 1295 * trace for that block. 1296 */ 1297 ret = cs_etm__decode_data_block(etmq); 1298 if (ret) 1299 goto out; 1300 1301 /* 1302 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all 1303 * the timestamp calculation for us. 1304 */ 1305 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); 1306 1307 /* We found a timestamp, no need to continue. */ 1308 if (cs_timestamp) 1309 break; 1310 1311 /* 1312 * We didn't find a timestamp so empty all the traceid packet 1313 * queues before looking for another timestamp packet, either 1314 * in the current data block or a new one. Packets that were 1315 * just decoded are useless since no timestamp has been 1316 * associated with them. As such simply discard them. 1317 */ 1318 cs_etm__clear_all_packet_queues(etmq); 1319 } 1320 1321 /* 1322 * We have a timestamp. Add it to the min heap to reflect when 1323 * instructions conveyed by the range packets of this traceID queue 1324 * started to execute. Once the same has been done for all the traceID 1325 * queues of each etmq, redenring and decoding can start in 1326 * chronological order. 1327 * 1328 * Note that packets decoded above are still in the traceID's packet 1329 * queue and will be processed in cs_etm__process_timestamped_queues(). 1330 */ 1331 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); 1332 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); 1333 out: 1334 return ret; 1335 } 1336 1337 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq, 1338 struct cs_etm_traceid_queue *tidq, 1339 struct cs_etm_packet *packet, u64 addr) 1340 { 1341 u8 instrBytes[2]; 1342 1343 cs_etm__frontend_mem_access(etmq, tidq, packet, addr, 1344 ARRAY_SIZE(instrBytes), instrBytes); 1345 /* 1346 * T32 instruction size is indicated by bits[15:11] of the first 1347 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111 1348 * denote a 32-bit instruction. 1349 */ 1350 return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2; 1351 } 1352 1353 static inline int cs_etm__instr_size(struct cs_etm_queue *etmq, 1354 struct cs_etm_traceid_queue *tidq, 1355 struct cs_etm_packet *packet, 1356 u64 addr) 1357 { 1358 if (packet->isa == CS_ETM_ISA_T32) 1359 return cs_etm__t32_instr_size(etmq, tidq, packet, addr); 1360 1361 /* Otherwise, 4-byte instruction size for A32/A64 */ 1362 return 4; 1363 } 1364 1365 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet) 1366 { 1367 /* 1368 * Return 0 for packets that have no addresses so that CS_ETM_INVAL_ADDR doesn't 1369 * appear in samples. 1370 */ 1371 if (packet->sample_type == CS_ETM_DISCONTINUITY || 1372 packet->sample_type == CS_ETM_EXCEPTION) 1373 return 0; 1374 1375 return packet->start_addr; 1376 } 1377 1378 static inline 1379 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet) 1380 { 1381 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */ 1382 if (packet->sample_type == CS_ETM_DISCONTINUITY) 1383 return 0; 1384 1385 return packet->end_addr - packet->last_instr_size; 1386 } 1387 1388 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq, 1389 struct cs_etm_traceid_queue *tidq, 1390 struct cs_etm_packet *packet, 1391 u64 offset) 1392 { 1393 u64 addr = packet->start_addr; 1394 1395 /* 4-byte instruction size for A32/A64 */ 1396 if (packet->isa == CS_ETM_ISA_A64 || packet->isa == CS_ETM_ISA_A32) 1397 return addr + offset * 4; 1398 1399 while (offset) { 1400 addr += cs_etm__instr_size(etmq, tidq, packet, addr); 1401 offset--; 1402 } 1403 return addr; 1404 } 1405 1406 static int cs_etm__inject_event(struct cs_etm_auxtrace *etm, union perf_event *event, 1407 struct perf_sample *sample, u64 type) 1408 { 1409 struct evsel *evsel = sample->evsel; 1410 u64 branch_sample_type = 0; 1411 size_t sz; 1412 1413 if (!evsel && etm->session && etm->session->evlist) 1414 evsel = evlist__id2evsel(etm->session->evlist, sample->id); 1415 1416 if (evsel) 1417 branch_sample_type = evsel->core.attr.branch_sample_type; 1418 1419 sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, 1420 branch_sample_type); 1421 if (sz >= PERF_SAMPLE_MAX_SIZE) { 1422 pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); 1423 return -EFAULT; 1424 } 1425 event->header.size = sz; 1426 1427 return perf_event__synthesize_sample(event, type, /*read_format=*/0, 1428 branch_sample_type, sample); 1429 } 1430 1431 1432 static int 1433 cs_etm__get_trace(struct cs_etm_queue *etmq) 1434 { 1435 struct auxtrace_buffer *aux_buffer = etmq->buffer; 1436 struct auxtrace_buffer *old_buffer = aux_buffer; 1437 struct auxtrace_queue *queue; 1438 1439 queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; 1440 1441 aux_buffer = auxtrace_buffer__next(queue, aux_buffer); 1442 1443 /* If no more data, drop the previous auxtrace_buffer and return */ 1444 if (!aux_buffer) { 1445 if (old_buffer) 1446 auxtrace_buffer__drop_data(old_buffer); 1447 etmq->buf_len = 0; 1448 return 0; 1449 } 1450 1451 etmq->buffer = aux_buffer; 1452 1453 /* If the aux_buffer doesn't have data associated, try to load it */ 1454 if (!aux_buffer->data) { 1455 /* get the file desc associated with the perf data file */ 1456 int fd = perf_data__fd(etmq->etm->session->data); 1457 1458 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd); 1459 if (!aux_buffer->data) 1460 return -ENOMEM; 1461 } 1462 1463 /* If valid, drop the previous buffer */ 1464 if (old_buffer) 1465 auxtrace_buffer__drop_data(old_buffer); 1466 1467 etmq->buf_used = 0; 1468 etmq->buf_len = aux_buffer->size; 1469 etmq->buf = aux_buffer->data; 1470 return 0; 1471 } 1472 1473 /* 1474 * Convert a raw thread number to a thread struct and assign it to **thread. 1475 */ 1476 static int cs_etm__etmq_update_thread(struct cs_etm_queue *etmq, 1477 ocsd_ex_level el, pid_t tid, 1478 struct thread **thread) 1479 { 1480 struct machine *machine = cs_etm__get_machine(etmq, el); 1481 1482 if (!machine || !*thread) 1483 return -EINVAL; 1484 1485 if (tid != -1) { 1486 thread__zput(*thread); 1487 *thread = machine__find_thread(machine, -1, tid); 1488 } 1489 1490 /* Couldn't find a known thread */ 1491 if (!*thread) 1492 *thread = machine__idle_thread(machine); 1493 1494 return 0; 1495 } 1496 1497 /* 1498 * Set the thread and EL of the decode context which is ahead in time of the 1499 * frontend context. 1500 */ 1501 int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, 1502 u8 trace_chan_id, 1503 ocsd_ex_level el, pid_t tid) 1504 { 1505 struct cs_etm_traceid_queue *tidq; 1506 int ret; 1507 1508 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 1509 if (!tidq) 1510 return -EINVAL; 1511 1512 ret = cs_etm__etmq_update_thread(etmq, el, tid, 1513 &tidq->decode_thread); 1514 if (ret) 1515 return ret; 1516 1517 tidq->decode_el = el; 1518 return 0; 1519 } 1520 1521 bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq) 1522 { 1523 return !!etmq->etm->timeless_decoding; 1524 } 1525 1526 static void cs_etm__copy_insn(struct cs_etm_queue *etmq, 1527 struct cs_etm_traceid_queue *tidq, 1528 struct cs_etm_packet *packet, 1529 struct perf_sample *sample) 1530 { 1531 /* 1532 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY 1533 * packet, so directly bail out with 'insn_len' = 0. 1534 */ 1535 if (packet->sample_type == CS_ETM_DISCONTINUITY) { 1536 sample->insn_len = 0; 1537 return; 1538 } 1539 1540 sample->insn_len = cs_etm__instr_size(etmq, tidq, packet, sample->ip); 1541 1542 cs_etm__frontend_mem_access(etmq, tidq, packet, sample->ip, 1543 sample->insn_len, (void *)sample->insn); 1544 } 1545 1546 u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp) 1547 { 1548 struct cs_etm_auxtrace *etm = etmq->etm; 1549 1550 if (etm->has_virtual_ts) 1551 return tsc_to_perf_time(cs_timestamp, &etm->tc); 1552 else 1553 return cs_timestamp; 1554 } 1555 1556 static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq, 1557 struct cs_etm_traceid_queue *tidq) 1558 { 1559 struct cs_etm_auxtrace *etm = etmq->etm; 1560 struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue; 1561 1562 if (!etm->timeless_decoding && etm->has_virtual_ts) 1563 return packet_queue->cs_timestamp; 1564 else 1565 return etm->latest_kernel_timestamp; 1566 } 1567 1568 static bool cs_etm__packet_has_taken_branch(struct cs_etm_packet *packet) 1569 { 1570 if (packet->sample_type == CS_ETM_RANGE && 1571 packet->last_instr_taken_branch) 1572 return true; 1573 1574 return false; 1575 } 1576 1577 static void cs_etm__add_stack_event(struct cs_etm_queue *etmq, 1578 struct cs_etm_traceid_queue *tidq) 1579 { 1580 struct cs_etm_auxtrace *etm = etmq->etm; 1581 u64 from, to; 1582 int size; 1583 1584 if (!etm->synth_opts.branches && !etm->synth_opts.instructions) 1585 return; 1586 1587 if (!cs_etm__packet_has_taken_branch(tidq->prev_packet)) 1588 return; 1589 1590 if (etmq->etm->use_thread_stack) { 1591 from = cs_etm__last_executed_instr(tidq->prev_packet); 1592 to = cs_etm__first_executed_instr(tidq->packet); 1593 1594 size = cs_etm__instr_size(etmq, tidq, tidq->prev_packet, from); 1595 1596 /* Enable callchain so thread stack entry can be allocated */ 1597 thread_stack__event(tidq->frontend_thread, tidq->prev_packet->cpu, 1598 tidq->prev_packet->flags, from, to, size, 1599 etmq->buffer->buffer_nr + 1, 1600 etmq->etm->use_callchain, 1601 tidq->br_stack_sz, 0); 1602 } else { 1603 thread_stack__set_trace_nr(tidq->frontend_thread, 1604 tidq->prev_packet->cpu, 1605 etmq->buffer->buffer_nr + 1); 1606 } 1607 } 1608 1609 static void cs_etm__sample_branch_stack(struct cs_etm_auxtrace *etm, 1610 struct cs_etm_traceid_queue *tidq, 1611 struct perf_sample *sample) 1612 { 1613 if (etm->synth_opts.last_branch) { 1614 thread_stack__br_sample(tidq->frontend_thread, tidq->packet->cpu, 1615 tidq->last_branch, tidq->br_stack_sz); 1616 sample->branch_stack = tidq->last_branch; 1617 } 1618 1619 if (etm->synth_opts.callchain) { 1620 if (tidq->kernel_start) 1621 thread_stack__sample(tidq->frontend_thread, 1622 tidq->packet->cpu, 1623 tidq->callchain, 1624 etm->synth_opts.callchain_sz + 1, 1625 sample->ip, tidq->kernel_start); 1626 else 1627 /* 1628 * Clear the callchain when the kernel start address is 1629 * not available yet. The empty callchain can then be 1630 * consumed by cs_etm__inject_event(). 1631 */ 1632 memset(tidq->callchain, 0, 1633 struct_size(tidq->callchain, ips, 1634 etm->synth_opts.callchain_sz + 1)); 1635 1636 sample->callchain = tidq->callchain; 1637 } 1638 } 1639 1640 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, 1641 struct cs_etm_traceid_queue *tidq, 1642 struct cs_etm_packet *packet, 1643 u64 addr, u64 period) 1644 { 1645 int ret = 0; 1646 struct cs_etm_auxtrace *etm = etmq->etm; 1647 union perf_event *event = tidq->event_buf; 1648 struct perf_sample sample; 1649 1650 perf_sample__init(&sample, /*all=*/true); 1651 event->sample.header.type = PERF_RECORD_SAMPLE; 1652 event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, packet->el); 1653 event->sample.header.size = sizeof(struct perf_event_header); 1654 1655 /* Set time field based on etm auxtrace config. */ 1656 sample.time = cs_etm__resolve_sample_time(etmq, tidq); 1657 1658 sample.ip = addr; 1659 sample.pid = thread__pid(tidq->frontend_thread); 1660 sample.tid = thread__tid(tidq->frontend_thread); 1661 sample.id = etmq->etm->instructions_id; 1662 sample.stream_id = etmq->etm->instructions_id; 1663 sample.period = period; 1664 sample.cpu = packet->cpu; 1665 sample.flags = tidq->prev_packet->flags; 1666 sample.cpumode = event->sample.header.misc; 1667 1668 cs_etm__copy_insn(etmq, tidq, packet, &sample); 1669 cs_etm__sample_branch_stack(etm, tidq, &sample); 1670 1671 if (etm->synth_opts.inject) { 1672 ret = cs_etm__inject_event(etm, event, &sample, 1673 etm->instructions_sample_type); 1674 if (ret) 1675 return ret; 1676 } 1677 1678 ret = perf_session__deliver_synth_event(etm->session, event, &sample); 1679 1680 if (ret) 1681 pr_err( 1682 "CS ETM Trace: failed to deliver instruction event, error %d\n", 1683 ret); 1684 1685 perf_sample__exit(&sample); 1686 return ret; 1687 } 1688 1689 /* 1690 * The cs etm packet encodes an instruction range between a branch target 1691 * and the next taken branch. Generate sample accordingly. 1692 */ 1693 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, 1694 struct cs_etm_traceid_queue *tidq) 1695 { 1696 int ret = 0; 1697 struct cs_etm_auxtrace *etm = etmq->etm; 1698 struct perf_sample sample; 1699 union perf_event *event = tidq->event_buf; 1700 1701 struct dummy_branch_stack { 1702 u64 nr; 1703 u64 hw_idx; 1704 struct branch_entry entries; 1705 } dummy_bs; 1706 u64 ip; 1707 1708 if (etm->branches_filter && 1709 !(etm->branches_filter & tidq->prev_packet->flags)) 1710 return 0; 1711 1712 perf_sample__init(&sample, /*all=*/true); 1713 ip = cs_etm__last_executed_instr(tidq->prev_packet); 1714 1715 event->sample.header.type = PERF_RECORD_SAMPLE; 1716 event->sample.header.misc = cs_etm__cpu_mode(etmq, ip, 1717 tidq->prev_packet->el); 1718 event->sample.header.size = sizeof(struct perf_event_header); 1719 1720 /* Set time field based on etm auxtrace config. */ 1721 sample.time = cs_etm__resolve_sample_time(etmq, tidq); 1722 1723 sample.ip = ip; 1724 sample.pid = thread__pid(tidq->frontend_thread); 1725 sample.tid = thread__tid(tidq->frontend_thread); 1726 sample.addr = cs_etm__first_executed_instr(tidq->packet); 1727 sample.id = etmq->etm->branches_id; 1728 sample.stream_id = etmq->etm->branches_id; 1729 sample.period = 1; 1730 sample.cpu = tidq->packet->cpu; 1731 sample.flags = tidq->prev_packet->flags; 1732 sample.cpumode = event->sample.header.misc; 1733 1734 cs_etm__copy_insn(etmq, tidq, tidq->prev_packet, &sample); 1735 1736 /* 1737 * perf report cannot handle events without a branch stack 1738 */ 1739 if (etm->synth_opts.last_branch) { 1740 dummy_bs = (struct dummy_branch_stack){ 1741 .nr = 1, 1742 .hw_idx = -1ULL, 1743 .entries = { 1744 .from = sample.ip, 1745 .to = sample.addr, 1746 }, 1747 }; 1748 sample.branch_stack = (struct branch_stack *)&dummy_bs; 1749 } 1750 1751 if (etm->synth_opts.inject) { 1752 ret = cs_etm__inject_event(etm, event, &sample, 1753 etm->branches_sample_type); 1754 if (ret) 1755 return ret; 1756 } 1757 1758 ret = perf_session__deliver_synth_event(etm->session, event, &sample); 1759 1760 if (ret) 1761 pr_err( 1762 "CS ETM Trace: failed to deliver instruction event, error %d\n", 1763 ret); 1764 1765 perf_sample__exit(&sample); 1766 return ret; 1767 } 1768 1769 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm, 1770 struct perf_session *session) 1771 { 1772 struct evlist *evlist = session->evlist; 1773 struct evsel *evsel; 1774 struct perf_event_attr attr; 1775 bool found = false; 1776 u64 id; 1777 int err; 1778 1779 evlist__for_each_entry(evlist, evsel) { 1780 if (evsel->core.attr.type == etm->pmu_type) { 1781 found = true; 1782 break; 1783 } 1784 } 1785 1786 if (!found) { 1787 pr_debug("No selected events with CoreSight Trace data\n"); 1788 return 0; 1789 } 1790 1791 memset(&attr, 0, sizeof(struct perf_event_attr)); 1792 attr.size = sizeof(struct perf_event_attr); 1793 attr.type = PERF_TYPE_HARDWARE; 1794 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK; 1795 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | 1796 PERF_SAMPLE_PERIOD; 1797 if (etm->timeless_decoding) 1798 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; 1799 else 1800 attr.sample_type |= PERF_SAMPLE_TIME; 1801 1802 attr.exclude_user = evsel->core.attr.exclude_user; 1803 attr.exclude_kernel = evsel->core.attr.exclude_kernel; 1804 attr.exclude_hv = evsel->core.attr.exclude_hv; 1805 attr.exclude_host = evsel->core.attr.exclude_host; 1806 attr.exclude_guest = evsel->core.attr.exclude_guest; 1807 attr.sample_id_all = evsel->core.attr.sample_id_all; 1808 attr.read_format = evsel->core.attr.read_format; 1809 1810 /* create new id val to be a fixed offset from evsel id */ 1811 id = auxtrace_synth_id_range_start(evsel); 1812 1813 if (etm->synth_opts.branches) { 1814 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS; 1815 attr.sample_period = 1; 1816 attr.sample_type |= PERF_SAMPLE_ADDR; 1817 err = perf_session__deliver_synth_attr_event(session, &attr, id); 1818 if (err) 1819 return err; 1820 etm->branches_sample_type = attr.sample_type; 1821 etm->branches_id = id; 1822 id += 1; 1823 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR; 1824 } 1825 1826 if (etm->synth_opts.last_branch) { 1827 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; 1828 /* 1829 * We don't use the hardware index, but the sample generation 1830 * code uses the new format branch_stack with this field, 1831 * so the event attributes must indicate that it's present. 1832 */ 1833 attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; 1834 } 1835 1836 if (etm->synth_opts.callchain) 1837 attr.sample_type |= PERF_SAMPLE_CALLCHAIN; 1838 1839 if (etm->synth_opts.instructions) { 1840 attr.config = PERF_COUNT_HW_INSTRUCTIONS; 1841 attr.sample_period = etm->synth_opts.period; 1842 etm->instructions_sample_period = attr.sample_period; 1843 err = perf_session__deliver_synth_attr_event(session, &attr, id); 1844 if (err) 1845 return err; 1846 etm->instructions_sample_type = attr.sample_type; 1847 etm->instructions_id = id; 1848 id += 1; 1849 } 1850 1851 return 0; 1852 } 1853 1854 static int cs_etm__sample(struct cs_etm_queue *etmq, 1855 struct cs_etm_traceid_queue *tidq) 1856 { 1857 struct cs_etm_auxtrace *etm = etmq->etm; 1858 int ret; 1859 u64 instrs_prev; 1860 1861 /* Get instructions remainder from previous packet */ 1862 instrs_prev = tidq->period_instructions; 1863 1864 tidq->period_instructions += tidq->packet->instr_count; 1865 1866 cs_etm__add_stack_event(etmq, tidq); 1867 1868 if (etm->synth_opts.instructions && 1869 tidq->period_instructions >= etm->instructions_sample_period) { 1870 /* 1871 * Emit instruction sample periodically 1872 * TODO: allow period to be defined in cycles and clock time 1873 */ 1874 1875 /* 1876 * Below diagram demonstrates the instruction samples 1877 * generation flows: 1878 * 1879 * Instrs Instrs Instrs Instrs 1880 * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3) 1881 * | | | | 1882 * V V V V 1883 * -------------------------------------------------- 1884 * ^ ^ 1885 * | | 1886 * Period Period 1887 * instructions(Pi) instructions(Pi') 1888 * 1889 * | | 1890 * \---------------- -----------------/ 1891 * V 1892 * tidq->packet->instr_count 1893 * 1894 * Instrs Sample(n...) are the synthesised samples occurring 1895 * every etm->instructions_sample_period instructions - as 1896 * defined on the perf command line. Sample(n) is being the 1897 * last sample before the current etm packet, n+1 to n+3 1898 * samples are generated from the current etm packet. 1899 * 1900 * tidq->packet->instr_count represents the number of 1901 * instructions in the current etm packet. 1902 * 1903 * Period instructions (Pi) contains the number of 1904 * instructions executed after the sample point(n) from the 1905 * previous etm packet. This will always be less than 1906 * etm->instructions_sample_period. 1907 * 1908 * When generate new samples, it combines with two parts 1909 * instructions, one is the tail of the old packet and another 1910 * is the head of the new coming packet, to generate 1911 * sample(n+1); sample(n+2) and sample(n+3) consume the 1912 * instructions with sample period. After sample(n+3), the rest 1913 * instructions will be used by later packet and it is assigned 1914 * to tidq->period_instructions for next round calculation. 1915 */ 1916 1917 /* 1918 * Get the initial offset into the current packet instructions; 1919 * entry conditions ensure that instrs_prev is less than 1920 * etm->instructions_sample_period. 1921 */ 1922 u64 offset = etm->instructions_sample_period - instrs_prev; 1923 u64 addr; 1924 1925 while (tidq->period_instructions >= 1926 etm->instructions_sample_period) { 1927 /* 1928 * Calculate the address of the sampled instruction (-1 1929 * as sample is reported as though instruction has just 1930 * been executed, but PC has not advanced to next 1931 * instruction) 1932 */ 1933 addr = cs_etm__instr_addr(etmq, tidq, tidq->packet, 1934 offset - 1); 1935 ret = cs_etm__synth_instruction_sample( 1936 etmq, tidq, tidq->packet, addr, 1937 etm->instructions_sample_period); 1938 if (ret) 1939 return ret; 1940 1941 offset += etm->instructions_sample_period; 1942 tidq->period_instructions -= 1943 etm->instructions_sample_period; 1944 } 1945 } 1946 1947 if (etm->synth_opts.branches) { 1948 bool generate_sample = false; 1949 1950 /* Generate sample for tracing on packet */ 1951 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY) 1952 generate_sample = true; 1953 1954 /* Generate sample for branch taken packet */ 1955 if (cs_etm__packet_has_taken_branch(tidq->prev_packet)) 1956 generate_sample = true; 1957 1958 if (generate_sample) { 1959 ret = cs_etm__synth_branch_sample(etmq, tidq); 1960 if (ret) 1961 return ret; 1962 } 1963 } 1964 1965 cs_etm__packet_swap(etm, tidq); 1966 1967 return 0; 1968 } 1969 1970 static int cs_etm__context(struct cs_etm_queue *etmq, 1971 struct cs_etm_traceid_queue *tidq) 1972 { 1973 ocsd_ex_level el = tidq->packet->el; 1974 struct machine *machine; 1975 int ret; 1976 1977 machine = cs_etm__get_machine(etmq, el); 1978 if (!machine) { 1979 ret = -EINVAL; 1980 goto err; 1981 } 1982 1983 tidq->kernel_start = machine__kernel_start(machine); 1984 1985 ret = cs_etm__etmq_update_thread(etmq, el, tidq->packet->tid, 1986 &tidq->frontend_thread); 1987 if (ret) 1988 goto err; 1989 1990 return 0; 1991 1992 err: 1993 thread__zput(tidq->frontend_thread); 1994 tidq->kernel_start = 0; 1995 return ret; 1996 } 1997 1998 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq) 1999 { 2000 /* 2001 * When the exception packet is inserted, whether the last instruction 2002 * in previous range packet is taken branch or not, we need to force 2003 * to set 'prev_packet->last_instr_taken_branch' to true. This ensures 2004 * to generate branch sample for the instruction range before the 2005 * exception is trapped to kernel or before the exception returning. 2006 * 2007 * The exception packet includes the dummy address values, so don't 2008 * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful 2009 * for generating instruction and branch samples. 2010 */ 2011 if (tidq->prev_packet->sample_type == CS_ETM_RANGE) 2012 tidq->prev_packet->last_instr_taken_branch = true; 2013 2014 return 0; 2015 } 2016 2017 static int cs_etm__flush(struct cs_etm_queue *etmq, 2018 struct cs_etm_traceid_queue *tidq) 2019 { 2020 int err = 0; 2021 struct cs_etm_auxtrace *etm = etmq->etm; 2022 2023 /* Handle start tracing packet */ 2024 if (tidq->prev_packet->sample_type == CS_ETM_EMPTY) 2025 goto swap_packet; 2026 2027 if (etmq->etm->synth_opts.last_branch && 2028 etmq->etm->synth_opts.instructions && 2029 tidq->prev_packet->sample_type == CS_ETM_RANGE) { 2030 u64 addr; 2031 /* 2032 * Generate a last branch event for the branches left in the 2033 * circular buffer at the end of the trace. 2034 * 2035 * Use the address of the end of the last reported execution 2036 * range 2037 */ 2038 addr = cs_etm__last_executed_instr(tidq->prev_packet); 2039 2040 err = cs_etm__synth_instruction_sample( 2041 etmq, tidq, tidq->prev_packet, addr, 2042 tidq->period_instructions); 2043 if (err) 2044 return err; 2045 2046 tidq->period_instructions = 0; 2047 2048 } 2049 2050 if (etm->synth_opts.branches && 2051 tidq->prev_packet->sample_type == CS_ETM_RANGE) { 2052 err = cs_etm__synth_branch_sample(etmq, tidq); 2053 if (err) 2054 return err; 2055 } 2056 2057 swap_packet: 2058 cs_etm__packet_swap(etm, tidq); 2059 2060 /* Reset last branches after flush the trace */ 2061 if (etm->use_thread_stack) 2062 thread_stack__flush(tidq->frontend_thread); 2063 2064 return err; 2065 } 2066 2067 static int cs_etm__end_block(struct cs_etm_queue *etmq, 2068 struct cs_etm_traceid_queue *tidq) 2069 { 2070 int err; 2071 2072 /* 2073 * It has no new packet coming and 'etmq->packet' contains the stale 2074 * packet which was set at the previous time with packets swapping; 2075 * so skip to generate branch sample to avoid stale packet. 2076 * 2077 * For this case only flush branch stack and generate a last branch 2078 * event for the branches left in the circular buffer at the end of 2079 * the trace. 2080 */ 2081 if (etmq->etm->synth_opts.last_branch && 2082 etmq->etm->synth_opts.instructions && 2083 tidq->prev_packet->sample_type == CS_ETM_RANGE) { 2084 u64 addr; 2085 2086 /* 2087 * Use the address of the end of the last reported execution 2088 * range. 2089 */ 2090 addr = cs_etm__last_executed_instr(tidq->prev_packet); 2091 2092 err = cs_etm__synth_instruction_sample( 2093 etmq, tidq, tidq->prev_packet, addr, 2094 tidq->period_instructions); 2095 if (err) 2096 return err; 2097 2098 tidq->period_instructions = 0; 2099 } 2100 2101 return 0; 2102 } 2103 2104 static int cs_etm__flush_stack_cb(struct thread *thread, 2105 void *data __maybe_unused) 2106 { 2107 thread_stack__flush(thread); 2108 return 0; 2109 } 2110 2111 static void cs_etm__flush_machine_stack(struct cs_etm_queue *etmq, pid_t pid) 2112 { 2113 struct machine *machine; 2114 2115 machine = machines__find(&etmq->etm->session->machines, pid); 2116 if (machine) 2117 machine__for_each_thread(machine, cs_etm__flush_stack_cb, NULL); 2118 } 2119 2120 static void cs_etm__flush_all_stack(struct cs_etm_queue *etmq) 2121 { 2122 enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq); 2123 2124 if (!etmq->etm->use_thread_stack) 2125 return; 2126 2127 switch (pid_fmt) { 2128 case CS_ETM_PIDFMT_CTXTID2: 2129 /* Clear the guest stack if virtualization is supported */ 2130 cs_etm__flush_machine_stack(etmq, DEFAULT_GUEST_KERNEL_ID); 2131 fallthrough; 2132 case CS_ETM_PIDFMT_CTXTID: 2133 cs_etm__flush_machine_stack(etmq, HOST_KERNEL_ID); 2134 break; 2135 case CS_ETM_PIDFMT_NONE: 2136 default: 2137 break; 2138 2139 } 2140 } 2141 2142 /* 2143 * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue 2144 * if need be. 2145 * Returns: < 0 if error 2146 * = 0 if no more auxtrace_buffer to read 2147 * > 0 if the current buffer isn't empty yet 2148 */ 2149 static int cs_etm__get_data_block(struct cs_etm_queue *etmq) 2150 { 2151 int ret; 2152 2153 /* The current block is not finished */ 2154 if (etmq->buf_len) 2155 return 1; 2156 2157 ret = cs_etm__get_trace(etmq); 2158 if (ret < 0) 2159 return ret; 2160 2161 /* No more buffer to read */ 2162 if (!etmq->buf_len) 2163 return 0; 2164 2165 /* 2166 * We cannot assume consecutive blocks in the data file 2167 * are contiguous, reset the decoder to force re-sync. 2168 */ 2169 ret = cs_etm_decoder__reset(etmq->decoder); 2170 if (ret) 2171 return ret; 2172 2173 /* 2174 * Since the decoder is reset, this causes a global trace 2175 * discontinuity. Flush all thread stacks. 2176 */ 2177 cs_etm__flush_all_stack(etmq); 2178 2179 return 1; 2180 } 2181 2182 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, 2183 struct cs_etm_traceid_queue *tidq, 2184 struct cs_etm_packet *packet, u64 end_addr) 2185 { 2186 /* Initialise to keep compiler happy */ 2187 u16 instr16 = 0; 2188 u32 instr32 = 0; 2189 u64 addr; 2190 2191 switch (packet->isa) { 2192 case CS_ETM_ISA_T32: 2193 /* 2194 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247: 2195 * 2196 * b'15 b'8 2197 * +-----------------+--------+ 2198 * | 1 1 0 1 1 1 1 1 | imm8 | 2199 * +-----------------+--------+ 2200 * 2201 * According to the specification, it only defines SVC for T32 2202 * with 16 bits instruction and has no definition for 32bits; 2203 * so below only read 2 bytes as instruction size for T32. 2204 */ 2205 addr = end_addr - 2; 2206 cs_etm__frontend_mem_access(etmq, tidq, packet, addr, 2207 sizeof(instr16), (u8 *)&instr16); 2208 if ((instr16 & 0xFF00) == 0xDF00) 2209 return true; 2210 2211 break; 2212 case CS_ETM_ISA_A32: 2213 /* 2214 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247: 2215 * 2216 * b'31 b'28 b'27 b'24 2217 * +---------+---------+-------------------------+ 2218 * | !1111 | 1 1 1 1 | imm24 | 2219 * +---------+---------+-------------------------+ 2220 */ 2221 addr = end_addr - 4; 2222 cs_etm__frontend_mem_access(etmq, tidq, packet, addr, 2223 sizeof(instr32), (u8 *)&instr32); 2224 if ((instr32 & 0x0F000000) == 0x0F000000 && 2225 (instr32 & 0xF0000000) != 0xF0000000) 2226 return true; 2227 2228 break; 2229 case CS_ETM_ISA_A64: 2230 /* 2231 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294: 2232 * 2233 * b'31 b'21 b'4 b'0 2234 * +-----------------------+---------+-----------+ 2235 * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 | 2236 * +-----------------------+---------+-----------+ 2237 */ 2238 addr = end_addr - 4; 2239 cs_etm__frontend_mem_access(etmq, tidq, packet, addr, 2240 sizeof(instr32), (u8 *)&instr32); 2241 if ((instr32 & 0xFFE0001F) == 0xd4000001) 2242 return true; 2243 2244 break; 2245 case CS_ETM_ISA_UNKNOWN: 2246 default: 2247 break; 2248 } 2249 2250 return false; 2251 } 2252 2253 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, 2254 struct cs_etm_traceid_queue *tidq, u64 magic) 2255 { 2256 struct cs_etm_packet *packet = tidq->packet; 2257 struct cs_etm_packet *prev_packet = tidq->prev_packet; 2258 2259 if (magic == __perf_cs_etmv3_magic) 2260 if (packet->exception_number == CS_ETMV3_EXC_SVC) 2261 return true; 2262 2263 /* 2264 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and 2265 * HVC cases; need to check if it's SVC instruction based on 2266 * packet address. 2267 */ 2268 if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) { 2269 if (packet->exception_number == CS_ETMV4_EXC_CALL && 2270 cs_etm__is_svc_instr(etmq, tidq, prev_packet, 2271 prev_packet->end_addr)) 2272 return true; 2273 } 2274 2275 return false; 2276 } 2277 2278 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq, 2279 u64 magic) 2280 { 2281 struct cs_etm_packet *packet = tidq->packet; 2282 2283 if (magic == __perf_cs_etmv3_magic) 2284 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT || 2285 packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT || 2286 packet->exception_number == CS_ETMV3_EXC_PE_RESET || 2287 packet->exception_number == CS_ETMV3_EXC_IRQ || 2288 packet->exception_number == CS_ETMV3_EXC_FIQ) 2289 return true; 2290 2291 if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) 2292 if (packet->exception_number == CS_ETMV4_EXC_RESET || 2293 packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT || 2294 packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR || 2295 packet->exception_number == CS_ETMV4_EXC_INST_DEBUG || 2296 packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG || 2297 packet->exception_number == CS_ETMV4_EXC_IRQ || 2298 packet->exception_number == CS_ETMV4_EXC_FIQ) 2299 return true; 2300 2301 return false; 2302 } 2303 2304 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, 2305 struct cs_etm_traceid_queue *tidq, 2306 u64 magic) 2307 { 2308 struct cs_etm_packet *packet = tidq->packet; 2309 struct cs_etm_packet *prev_packet = tidq->prev_packet; 2310 2311 if (magic == __perf_cs_etmv3_magic) 2312 if (packet->exception_number == CS_ETMV3_EXC_SMC || 2313 packet->exception_number == CS_ETMV3_EXC_HYP || 2314 packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE || 2315 packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR || 2316 packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT || 2317 packet->exception_number == CS_ETMV3_EXC_DATA_FAULT || 2318 packet->exception_number == CS_ETMV3_EXC_GENERIC) 2319 return true; 2320 2321 if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) { 2322 if (packet->exception_number == CS_ETMV4_EXC_TRAP || 2323 packet->exception_number == CS_ETMV4_EXC_ALIGNMENT || 2324 packet->exception_number == CS_ETMV4_EXC_INST_FAULT || 2325 packet->exception_number == CS_ETMV4_EXC_DATA_FAULT) 2326 return true; 2327 2328 /* 2329 * For CS_ETMV4_EXC_CALL, except SVC other instructions 2330 * (SMC, HVC) are taken as sync exceptions. 2331 */ 2332 if (packet->exception_number == CS_ETMV4_EXC_CALL && 2333 !cs_etm__is_svc_instr(etmq, tidq, prev_packet, 2334 prev_packet->end_addr)) 2335 return true; 2336 2337 /* 2338 * ETMv4 has 5 bits for exception number; if the numbers 2339 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ] 2340 * they are implementation defined exceptions. 2341 * 2342 * For this case, simply take it as sync exception. 2343 */ 2344 if (packet->exception_number > CS_ETMV4_EXC_FIQ && 2345 packet->exception_number <= CS_ETMV4_EXC_END) 2346 return true; 2347 } 2348 2349 return false; 2350 } 2351 2352 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, 2353 struct cs_etm_traceid_queue *tidq) 2354 { 2355 struct cs_etm_packet *packet = tidq->packet; 2356 struct cs_etm_packet *prev_packet = tidq->prev_packet; 2357 u64 magic; 2358 int ret; 2359 2360 switch (packet->sample_type) { 2361 case CS_ETM_RANGE: 2362 /* 2363 * Immediate branch instruction without neither link nor 2364 * return flag, it's normal branch instruction within 2365 * the function. 2366 */ 2367 if (packet->last_instr_type == OCSD_INSTR_BR && 2368 packet->last_instr_subtype == OCSD_S_INSTR_NONE) { 2369 packet->flags = PERF_IP_FLAG_BRANCH; 2370 2371 if (packet->last_instr_cond) 2372 packet->flags |= PERF_IP_FLAG_CONDITIONAL; 2373 } 2374 2375 /* 2376 * Immediate branch instruction with link (e.g. BL), this is 2377 * branch instruction for function call. 2378 */ 2379 if (packet->last_instr_type == OCSD_INSTR_BR && 2380 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) 2381 packet->flags = PERF_IP_FLAG_BRANCH | 2382 PERF_IP_FLAG_CALL; 2383 2384 /* 2385 * Indirect branch instruction with link (e.g. BLR), this is 2386 * branch instruction for function call. 2387 */ 2388 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 2389 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) 2390 packet->flags = PERF_IP_FLAG_BRANCH | 2391 PERF_IP_FLAG_CALL; 2392 2393 /* 2394 * Indirect branch instruction with subtype of 2395 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for 2396 * function return for A32/T32. 2397 */ 2398 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 2399 packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET) 2400 packet->flags = PERF_IP_FLAG_BRANCH | 2401 PERF_IP_FLAG_RETURN; 2402 2403 /* 2404 * Indirect branch instruction without link (e.g. BR), usually 2405 * this is used for function return, especially for functions 2406 * within dynamic link lib. 2407 */ 2408 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 2409 packet->last_instr_subtype == OCSD_S_INSTR_NONE) 2410 packet->flags = PERF_IP_FLAG_BRANCH | 2411 PERF_IP_FLAG_RETURN; 2412 2413 /* Return instruction for function return. */ 2414 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && 2415 packet->last_instr_subtype == OCSD_S_INSTR_V8_RET) 2416 packet->flags = PERF_IP_FLAG_BRANCH | 2417 PERF_IP_FLAG_RETURN; 2418 2419 /* 2420 * Decoder might insert a discontinuity in the middle of 2421 * instruction packets, fixup prev_packet with flag 2422 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace. 2423 */ 2424 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY) 2425 prev_packet->flags |= PERF_IP_FLAG_BRANCH | 2426 PERF_IP_FLAG_TRACE_BEGIN; 2427 2428 /* 2429 * If the previous packet is an exception return packet 2430 * and the return address just follows SVC instruction, 2431 * it needs to calibrate the previous packet sample flags 2432 * as PERF_IP_FLAG_SYSCALLRET. 2433 */ 2434 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH | 2435 PERF_IP_FLAG_RETURN | 2436 PERF_IP_FLAG_INTERRUPT) && 2437 cs_etm__is_svc_instr(etmq, tidq, packet, packet->start_addr)) { 2438 prev_packet->flags = PERF_IP_FLAG_BRANCH | 2439 PERF_IP_FLAG_RETURN | 2440 PERF_IP_FLAG_SYSCALLRET; 2441 } 2442 break; 2443 case CS_ETM_DISCONTINUITY: 2444 /* 2445 * The trace is discontinuous, if the previous packet is 2446 * instruction packet, set flag PERF_IP_FLAG_TRACE_END 2447 * for previous packet. 2448 */ 2449 if (prev_packet->sample_type == CS_ETM_RANGE) 2450 prev_packet->flags |= PERF_IP_FLAG_BRANCH | 2451 PERF_IP_FLAG_TRACE_END; 2452 break; 2453 case CS_ETM_EXCEPTION: 2454 ret = cs_etm__get_magic(etmq, packet->trace_chan_id, &magic); 2455 if (ret) 2456 return ret; 2457 2458 /* The exception is for system call. */ 2459 if (cs_etm__is_syscall(etmq, tidq, magic)) 2460 packet->flags = PERF_IP_FLAG_BRANCH | 2461 PERF_IP_FLAG_CALL | 2462 PERF_IP_FLAG_SYSCALLRET; 2463 /* 2464 * The exceptions are triggered by external signals from bus, 2465 * interrupt controller, debug module, PE reset or halt. 2466 */ 2467 else if (cs_etm__is_async_exception(tidq, magic)) 2468 packet->flags = PERF_IP_FLAG_BRANCH | 2469 PERF_IP_FLAG_CALL | 2470 PERF_IP_FLAG_ASYNC | 2471 PERF_IP_FLAG_INTERRUPT; 2472 /* 2473 * Otherwise, exception is caused by trap, instruction & 2474 * data fault, or alignment errors. 2475 */ 2476 else if (cs_etm__is_sync_exception(etmq, tidq, magic)) 2477 packet->flags = PERF_IP_FLAG_BRANCH | 2478 PERF_IP_FLAG_CALL | 2479 PERF_IP_FLAG_INTERRUPT; 2480 2481 /* 2482 * When the exception packet is inserted, since exception 2483 * packet is not used standalone for generating samples 2484 * and it's affiliation to the previous instruction range 2485 * packet; so set previous range packet flags to tell perf 2486 * it is an exception taken branch. 2487 */ 2488 if (prev_packet->sample_type == CS_ETM_RANGE) 2489 prev_packet->flags = packet->flags; 2490 break; 2491 case CS_ETM_EXCEPTION_RET: 2492 /* 2493 * When the exception return packet is inserted, since 2494 * exception return packet is not used standalone for 2495 * generating samples and it's affiliation to the previous 2496 * instruction range packet; so set previous range packet 2497 * flags to tell perf it is an exception return branch. 2498 * 2499 * The exception return can be for either system call or 2500 * other exception types; unfortunately the packet doesn't 2501 * contain exception type related info so we cannot decide 2502 * the exception type purely based on exception return packet. 2503 * If we record the exception number from exception packet and 2504 * reuse it for exception return packet, this is not reliable 2505 * due the trace can be discontinuity or the interrupt can 2506 * be nested, thus the recorded exception number cannot be 2507 * used for exception return packet for these two cases. 2508 * 2509 * For exception return packet, we only need to distinguish the 2510 * packet is for system call or for other types. Thus the 2511 * decision can be deferred when receive the next packet which 2512 * contains the return address, based on the return address we 2513 * can read out the previous instruction and check if it's a 2514 * system call instruction and then calibrate the sample flag 2515 * as needed. 2516 */ 2517 if (prev_packet->sample_type == CS_ETM_RANGE) 2518 prev_packet->flags = PERF_IP_FLAG_BRANCH | 2519 PERF_IP_FLAG_RETURN | 2520 PERF_IP_FLAG_INTERRUPT; 2521 break; 2522 case CS_ETM_CONTEXT: 2523 case CS_ETM_EMPTY: 2524 default: 2525 break; 2526 } 2527 2528 return 0; 2529 } 2530 2531 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq) 2532 { 2533 int ret = 0; 2534 size_t processed = 0; 2535 2536 /* 2537 * Packets are decoded and added to the decoder's packet queue 2538 * until the decoder packet processing callback has requested that 2539 * processing stops or there is nothing left in the buffer. Normal 2540 * operations that stop processing are a timestamp packet or a full 2541 * decoder buffer queue. 2542 */ 2543 ret = cs_etm_decoder__process_data_block(etmq->decoder, 2544 etmq->offset, 2545 &etmq->buf[etmq->buf_used], 2546 etmq->buf_len, 2547 &processed); 2548 if (ret) 2549 goto out; 2550 2551 etmq->offset += processed; 2552 etmq->buf_used += processed; 2553 etmq->buf_len -= processed; 2554 2555 out: 2556 return ret; 2557 } 2558 2559 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, 2560 struct cs_etm_traceid_queue *tidq) 2561 { 2562 int ret; 2563 struct cs_etm_packet_queue *packet_queue; 2564 2565 packet_queue = &tidq->packet_queue; 2566 2567 /* Process each packet in this chunk */ 2568 while (1) { 2569 ret = cs_etm_decoder__get_packet(packet_queue, 2570 tidq->packet); 2571 if (ret <= 0) 2572 /* 2573 * Stop processing this chunk on 2574 * end of data or error 2575 */ 2576 break; 2577 2578 /* 2579 * Since packet addresses are swapped in packet 2580 * handling within below switch() statements, 2581 * thus setting sample flags must be called 2582 * prior to switch() statement to use address 2583 * information before packets swapping. 2584 */ 2585 ret = cs_etm__set_sample_flags(etmq, tidq); 2586 if (ret < 0) 2587 break; 2588 2589 switch (tidq->packet->sample_type) { 2590 case CS_ETM_RANGE: 2591 /* 2592 * If the packet contains an instruction 2593 * range, generate instruction sequence 2594 * events. 2595 */ 2596 cs_etm__sample(etmq, tidq); 2597 break; 2598 case CS_ETM_CONTEXT: 2599 /* 2600 * Update context but don't swap packet. Keep the 2601 * previous one for branch source address info, if 2602 * tracing the kernel the context packet will be emitted 2603 * between two ranges. 2604 */ 2605 ret = cs_etm__context(etmq, tidq); 2606 if (ret) 2607 goto out; 2608 break; 2609 case CS_ETM_EXCEPTION: 2610 case CS_ETM_EXCEPTION_RET: 2611 /* 2612 * If the exception packet is coming, 2613 * make sure the previous instruction 2614 * range packet to be handled properly. 2615 */ 2616 cs_etm__exception(tidq); 2617 break; 2618 case CS_ETM_DISCONTINUITY: 2619 /* 2620 * Discontinuity in trace, flush 2621 * previous branch stack 2622 */ 2623 cs_etm__flush(etmq, tidq); 2624 break; 2625 case CS_ETM_EMPTY: 2626 /* 2627 * Should not receive empty packet, 2628 * report error. 2629 */ 2630 pr_err("CS ETM Trace: empty packet\n"); 2631 return -EINVAL; 2632 default: 2633 break; 2634 } 2635 } 2636 2637 out: 2638 return ret; 2639 } 2640 2641 static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq) 2642 { 2643 int idx; 2644 struct int_node *inode; 2645 struct cs_etm_traceid_queue *tidq; 2646 struct intlist *traceid_queues_list = etmq->traceid_queues_list; 2647 2648 intlist__for_each_entry(inode, traceid_queues_list) { 2649 idx = (int)(intptr_t)inode->priv; 2650 tidq = etmq->traceid_queues[idx]; 2651 2652 /* Ignore return value */ 2653 cs_etm__process_traceid_queue(etmq, tidq); 2654 } 2655 } 2656 2657 static int cs_etm__run_per_thread_timeless_decoder(struct cs_etm_queue *etmq) 2658 { 2659 int err = 0; 2660 struct cs_etm_traceid_queue *tidq; 2661 2662 tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID); 2663 if (!tidq) 2664 return -EINVAL; 2665 2666 /* Go through each buffer in the queue and decode them one by one */ 2667 while (1) { 2668 err = cs_etm__get_data_block(etmq); 2669 if (err <= 0) 2670 return err; 2671 2672 /* Run trace decoder until buffer consumed or end of trace */ 2673 do { 2674 err = cs_etm__decode_data_block(etmq); 2675 if (err) 2676 return err; 2677 2678 /* 2679 * Process each packet in this chunk, nothing to do if 2680 * an error occurs other than hoping the next one will 2681 * be better. 2682 */ 2683 err = cs_etm__process_traceid_queue(etmq, tidq); 2684 2685 } while (etmq->buf_len); 2686 2687 if (err == 0) 2688 /* Flush any remaining branch stack entries */ 2689 err = cs_etm__end_block(etmq, tidq); 2690 } 2691 2692 return err; 2693 } 2694 2695 static int cs_etm__run_per_cpu_timeless_decoder(struct cs_etm_queue *etmq) 2696 { 2697 int idx, err = 0; 2698 struct cs_etm_traceid_queue *tidq; 2699 struct int_node *inode; 2700 2701 /* Go through each buffer in the queue and decode them one by one */ 2702 while (1) { 2703 err = cs_etm__get_data_block(etmq); 2704 if (err <= 0) 2705 return err; 2706 2707 /* Run trace decoder until buffer consumed or end of trace */ 2708 do { 2709 err = cs_etm__decode_data_block(etmq); 2710 if (err) 2711 return err; 2712 2713 /* 2714 * cs_etm__run_per_thread_timeless_decoder() runs on a 2715 * single traceID queue because each TID has a separate 2716 * buffer. But here in per-cpu mode we need to iterate 2717 * over each channel instead. 2718 */ 2719 intlist__for_each_entry(inode, 2720 etmq->traceid_queues_list) { 2721 idx = (int)(intptr_t)inode->priv; 2722 tidq = etmq->traceid_queues[idx]; 2723 cs_etm__process_traceid_queue(etmq, tidq); 2724 } 2725 } while (etmq->buf_len); 2726 2727 intlist__for_each_entry(inode, etmq->traceid_queues_list) { 2728 idx = (int)(intptr_t)inode->priv; 2729 tidq = etmq->traceid_queues[idx]; 2730 /* Flush any remaining branch stack entries */ 2731 err = cs_etm__end_block(etmq, tidq); 2732 if (err) 2733 return err; 2734 } 2735 } 2736 2737 return err; 2738 } 2739 2740 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, 2741 pid_t tid) 2742 { 2743 unsigned int i; 2744 struct auxtrace_queues *queues = &etm->queues; 2745 2746 for (i = 0; i < queues->nr_queues; i++) { 2747 struct auxtrace_queue *queue = &etm->queues.queue_array[i]; 2748 struct cs_etm_queue *etmq = queue->priv; 2749 struct cs_etm_traceid_queue *tidq; 2750 2751 if (!etmq) 2752 continue; 2753 2754 if (etm->per_thread_decoding) { 2755 tidq = cs_etm__etmq_get_traceid_queue( 2756 etmq, CS_ETM_PER_THREAD_TRACEID); 2757 2758 if (!tidq) 2759 continue; 2760 2761 if (tid == -1 || thread__tid(tidq->frontend_thread) == tid) 2762 cs_etm__run_per_thread_timeless_decoder(etmq); 2763 } else 2764 cs_etm__run_per_cpu_timeless_decoder(etmq); 2765 } 2766 2767 return 0; 2768 } 2769 2770 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm) 2771 { 2772 int ret = 0; 2773 unsigned int cs_queue_nr, queue_nr, i; 2774 u8 trace_chan_id; 2775 u64 cs_timestamp; 2776 struct auxtrace_queue *queue; 2777 struct cs_etm_queue *etmq; 2778 struct cs_etm_traceid_queue *tidq; 2779 2780 /* 2781 * Pre-populate the heap with one entry from each queue so that we can 2782 * start processing in time order across all queues. 2783 */ 2784 for (i = 0; i < etm->queues.nr_queues; i++) { 2785 etmq = etm->queues.queue_array[i].priv; 2786 if (!etmq) 2787 continue; 2788 2789 ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i); 2790 if (ret) 2791 return ret; 2792 } 2793 2794 while (1) { 2795 if (!etm->heap.heap_cnt) 2796 break; 2797 2798 /* Take the entry at the top of the min heap */ 2799 cs_queue_nr = etm->heap.heap_array[0].queue_nr; 2800 queue_nr = TO_QUEUE_NR(cs_queue_nr); 2801 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr); 2802 queue = &etm->queues.queue_array[queue_nr]; 2803 etmq = queue->priv; 2804 2805 /* 2806 * Remove the top entry from the heap since we are about 2807 * to process it. 2808 */ 2809 auxtrace_heap__pop(&etm->heap); 2810 2811 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); 2812 if (!tidq) { 2813 /* 2814 * No traceID queue has been allocated for this traceID, 2815 * which means something somewhere went very wrong. No 2816 * other choice than simply exit. 2817 */ 2818 ret = -EINVAL; 2819 goto out; 2820 } 2821 2822 /* 2823 * Packets associated with this timestamp are already in 2824 * the etmq's traceID queue, so process them. 2825 */ 2826 ret = cs_etm__process_traceid_queue(etmq, tidq); 2827 if (ret < 0) 2828 goto out; 2829 2830 /* 2831 * Packets for this timestamp have been processed, time to 2832 * move on to the next timestamp, fetching a new auxtrace_buffer 2833 * if need be. 2834 */ 2835 refetch: 2836 ret = cs_etm__get_data_block(etmq); 2837 if (ret < 0) 2838 goto out; 2839 2840 /* 2841 * No more auxtrace_buffers to process in this etmq, simply 2842 * move on to another entry in the auxtrace_heap. 2843 */ 2844 if (!ret) 2845 continue; 2846 2847 ret = cs_etm__decode_data_block(etmq); 2848 if (ret) 2849 goto out; 2850 2851 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); 2852 2853 if (!cs_timestamp) { 2854 /* 2855 * Function cs_etm__decode_data_block() returns when 2856 * there is no more traces to decode in the current 2857 * auxtrace_buffer OR when a timestamp has been 2858 * encountered on any of the traceID queues. Since we 2859 * did not get a timestamp, there is no more traces to 2860 * process in this auxtrace_buffer. As such empty and 2861 * flush all traceID queues. 2862 */ 2863 cs_etm__clear_all_traceid_queues(etmq); 2864 2865 /* Fetch another auxtrace_buffer for this etmq */ 2866 goto refetch; 2867 } 2868 2869 /* 2870 * Add to the min heap the timestamp for packets that have 2871 * just been decoded. They will be processed and synthesized 2872 * during the next call to cs_etm__process_traceid_queue() for 2873 * this queue/traceID. 2874 */ 2875 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); 2876 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); 2877 } 2878 2879 for (i = 0; i < etm->queues.nr_queues; i++) { 2880 struct int_node *inode; 2881 2882 etmq = etm->queues.queue_array[i].priv; 2883 if (!etmq) 2884 continue; 2885 2886 intlist__for_each_entry(inode, etmq->traceid_queues_list) { 2887 int idx = (int)(intptr_t)inode->priv; 2888 2889 /* Flush any remaining branch stack entries */ 2890 tidq = etmq->traceid_queues[idx]; 2891 ret = cs_etm__end_block(etmq, tidq); 2892 if (ret) 2893 return ret; 2894 } 2895 } 2896 out: 2897 return ret; 2898 } 2899 2900 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm, 2901 union perf_event *event) 2902 { 2903 struct thread *th; 2904 2905 if (etm->timeless_decoding) 2906 return 0; 2907 2908 /* 2909 * Add the tid/pid to the log so that we can get a match when we get a 2910 * contextID from the decoder. Only track for the host: only kernel 2911 * trace is supported for guests which wouldn't need pids so this should 2912 * be fine. 2913 */ 2914 th = machine__findnew_thread(&etm->session->machines.host, 2915 event->itrace_start.pid, 2916 event->itrace_start.tid); 2917 if (!th) 2918 return -ENOMEM; 2919 2920 thread__put(th); 2921 2922 return 0; 2923 } 2924 2925 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm, 2926 union perf_event *event) 2927 { 2928 struct thread *th; 2929 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; 2930 2931 /* 2932 * Context switch in per-thread mode are irrelevant since perf 2933 * will start/stop tracing as the process is scheduled. 2934 */ 2935 if (etm->timeless_decoding) 2936 return 0; 2937 2938 /* 2939 * SWITCH_IN events carry the next process to be switched out while 2940 * SWITCH_OUT events carry the process to be switched in. As such 2941 * we don't care about IN events. 2942 */ 2943 if (!out) 2944 return 0; 2945 2946 /* 2947 * Add the tid/pid to the log so that we can get a match when we get a 2948 * contextID from the decoder. Only track for the host: only kernel 2949 * trace is supported for guests which wouldn't need pids so this should 2950 * be fine. 2951 */ 2952 th = machine__findnew_thread(&etm->session->machines.host, 2953 event->context_switch.next_prev_pid, 2954 event->context_switch.next_prev_tid); 2955 if (!th) 2956 return -ENOMEM; 2957 2958 thread__put(th); 2959 2960 return 0; 2961 } 2962 2963 static int cs_etm__process_event(struct perf_session *session, 2964 union perf_event *event, 2965 struct perf_sample *sample, 2966 const struct perf_tool *tool) 2967 { 2968 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 2969 struct cs_etm_auxtrace, 2970 auxtrace); 2971 2972 if (dump_trace) 2973 return 0; 2974 2975 if (!tool->ordered_events) { 2976 pr_err("CoreSight ETM Trace requires ordered events\n"); 2977 return -EINVAL; 2978 } 2979 2980 switch (event->header.type) { 2981 case PERF_RECORD_EXIT: 2982 /* 2983 * Don't need to wait for cs_etm__flush_events() in per-thread mode to 2984 * start the decode because we know there will be no more trace from 2985 * this thread. All this does is emit samples earlier than waiting for 2986 * the flush in other modes, but with timestamps it makes sense to wait 2987 * for flush so that events from different threads are interleaved 2988 * properly. 2989 */ 2990 if (etm->per_thread_decoding && etm->timeless_decoding) 2991 return cs_etm__process_timeless_queues(etm, 2992 event->fork.tid); 2993 break; 2994 2995 case PERF_RECORD_ITRACE_START: 2996 return cs_etm__process_itrace_start(etm, event); 2997 2998 case PERF_RECORD_SWITCH_CPU_WIDE: 2999 return cs_etm__process_switch_cpu_wide(etm, event); 3000 3001 case PERF_RECORD_AUX: 3002 /* 3003 * Record the latest kernel timestamp available in the header 3004 * for samples so that synthesised samples occur from this point 3005 * onwards. 3006 */ 3007 if (sample->time && (sample->time != (u64)-1)) 3008 etm->latest_kernel_timestamp = sample->time; 3009 break; 3010 3011 default: 3012 break; 3013 } 3014 3015 return 0; 3016 } 3017 3018 static void dump_queued_data(struct cs_etm_auxtrace *etm, 3019 struct perf_record_auxtrace *event) 3020 { 3021 struct auxtrace_buffer *buf; 3022 unsigned int i; 3023 /* 3024 * Find all buffers with same reference in the queues and dump them. 3025 * This is because the queues can contain multiple entries of the same 3026 * buffer that were split on aux records. 3027 */ 3028 for (i = 0; i < etm->queues.nr_queues; ++i) 3029 list_for_each_entry(buf, &etm->queues.queue_array[i].head, list) 3030 if (buf->reference == event->reference) 3031 cs_etm__dump_event(etm->queues.queue_array[i].priv, buf); 3032 } 3033 3034 static int cs_etm__process_auxtrace_event(struct perf_session *session, 3035 union perf_event *event, 3036 const struct perf_tool *tool __maybe_unused) 3037 { 3038 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 3039 struct cs_etm_auxtrace, 3040 auxtrace); 3041 if (!etm->data_queued) { 3042 struct auxtrace_buffer *buffer; 3043 off_t data_offset; 3044 int fd = perf_data__fd(session->data); 3045 bool is_pipe = perf_data__is_pipe(session->data); 3046 int err; 3047 int idx = event->auxtrace.idx; 3048 3049 if (is_pipe) 3050 data_offset = 0; 3051 else { 3052 data_offset = lseek(fd, 0, SEEK_CUR); 3053 if (data_offset == -1) 3054 return -errno; 3055 } 3056 3057 err = auxtrace_queues__add_event(&etm->queues, session, 3058 event, data_offset, &buffer); 3059 if (err) 3060 return err; 3061 3062 if (dump_trace) 3063 if (auxtrace_buffer__get_data(buffer, fd)) { 3064 cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer); 3065 auxtrace_buffer__put_data(buffer); 3066 } 3067 } else if (dump_trace) 3068 dump_queued_data(etm, &event->auxtrace); 3069 3070 return 0; 3071 } 3072 3073 static void cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm) 3074 { 3075 /* Take first ETM as all options will be the same for all ETMs */ 3076 u64 *metadata = etm->metadata[0]; 3077 3078 /* Override timeless mode with user input from --itrace=Z */ 3079 if (etm->synth_opts.timeless_decoding) { 3080 etm->timeless_decoding = true; 3081 return; 3082 } 3083 3084 if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) 3085 etm->timeless_decoding = !(metadata[CS_ETM_ETMCR] & ETMCR_TIMESTAMP_EN); 3086 else 3087 etm->timeless_decoding = !(metadata[CS_ETMV4_TRCCONFIGR] & TRCCONFIGR_TS); 3088 } 3089 3090 /* 3091 * Read a single cpu parameter block from the auxtrace_info priv block. 3092 * 3093 * For version 1 there is a per cpu nr_params entry. If we are handling 3094 * version 1 file, then there may be less, the same, or more params 3095 * indicated by this value than the compile time number we understand. 3096 * 3097 * For a version 0 info block, there are a fixed number, and we need to 3098 * fill out the nr_param value in the metadata we create. 3099 */ 3100 static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset, 3101 int out_blk_size, int nr_params_v0) 3102 { 3103 u64 *metadata = NULL; 3104 int hdr_version; 3105 int nr_in_params, nr_out_params, nr_cmn_params; 3106 int i, k; 3107 3108 metadata = zalloc(sizeof(*metadata) * out_blk_size); 3109 if (!metadata) 3110 return NULL; 3111 3112 /* read block current index & version */ 3113 i = *buff_in_offset; 3114 hdr_version = buff_in[CS_HEADER_VERSION]; 3115 3116 if (!hdr_version) { 3117 /* read version 0 info block into a version 1 metadata block */ 3118 nr_in_params = nr_params_v0; 3119 metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC]; 3120 metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU]; 3121 metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params; 3122 /* remaining block params at offset +1 from source */ 3123 for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++) 3124 metadata[k + 1] = buff_in[i + k]; 3125 /* version 0 has 2 common params */ 3126 nr_cmn_params = 2; 3127 } else { 3128 /* read version 1 info block - input and output nr_params may differ */ 3129 /* version 1 has 3 common params */ 3130 nr_cmn_params = 3; 3131 nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS]; 3132 3133 /* if input has more params than output - skip excess */ 3134 nr_out_params = nr_in_params + nr_cmn_params; 3135 if (nr_out_params > out_blk_size) 3136 nr_out_params = out_blk_size; 3137 3138 for (k = CS_ETM_MAGIC; k < nr_out_params; k++) 3139 metadata[k] = buff_in[i + k]; 3140 3141 /* record the actual nr params we copied */ 3142 metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params; 3143 } 3144 3145 /* adjust in offset by number of in params used */ 3146 i += nr_in_params + nr_cmn_params; 3147 *buff_in_offset = i; 3148 return metadata; 3149 } 3150 3151 /** 3152 * Puts a fragment of an auxtrace buffer into the auxtrace queues based 3153 * on the bounds of aux_event, if it matches with the buffer that's at 3154 * file_offset. 3155 * 3156 * Normally, whole auxtrace buffers would be added to the queue. But we 3157 * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder 3158 * is reset across each buffer, so splitting the buffers up in advance has 3159 * the same effect. 3160 */ 3161 static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz, 3162 struct perf_record_aux *aux_event, struct perf_sample *sample) 3163 { 3164 int err; 3165 char buf[PERF_SAMPLE_MAX_SIZE]; 3166 union perf_event *auxtrace_event_union; 3167 struct perf_record_auxtrace *auxtrace_event; 3168 union perf_event auxtrace_fragment; 3169 __u64 aux_offset, aux_size; 3170 enum cs_etm_format format; 3171 3172 struct cs_etm_auxtrace *etm = container_of(session->auxtrace, 3173 struct cs_etm_auxtrace, 3174 auxtrace); 3175 3176 /* 3177 * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got 3178 * from looping through the auxtrace index. 3179 */ 3180 err = perf_session__peek_event(session, file_offset, buf, 3181 PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL); 3182 if (err) 3183 return err; 3184 auxtrace_event = &auxtrace_event_union->auxtrace; 3185 if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE) 3186 return -EINVAL; 3187 3188 if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) || 3189 auxtrace_event->header.size != sz) { 3190 return -EINVAL; 3191 } 3192 3193 /* 3194 * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See 3195 * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a 3196 * CPU as we set this always for the AUX_OUTPUT_HW_ID event. 3197 * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1. 3198 * Return 'not found' if mismatch. 3199 */ 3200 if (auxtrace_event->cpu == (__u32) -1) { 3201 etm->per_thread_decoding = true; 3202 if (auxtrace_event->tid != sample->tid) 3203 return 1; 3204 } else if (auxtrace_event->cpu != sample->cpu) { 3205 if (etm->per_thread_decoding) { 3206 /* 3207 * Found a per-cpu buffer after a per-thread one was 3208 * already found 3209 */ 3210 pr_err("CS ETM: Inconsistent per-thread/per-cpu mode.\n"); 3211 return -EINVAL; 3212 } 3213 return 1; 3214 } 3215 3216 if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) { 3217 /* 3218 * Clamp size in snapshot mode. The buffer size is clamped in 3219 * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect 3220 * the buffer size. 3221 */ 3222 aux_size = min(aux_event->aux_size, auxtrace_event->size); 3223 3224 /* 3225 * In this mode, the head also points to the end of the buffer so aux_offset 3226 * needs to have the size subtracted so it points to the beginning as in normal mode 3227 */ 3228 aux_offset = aux_event->aux_offset - aux_size; 3229 } else { 3230 aux_size = aux_event->aux_size; 3231 aux_offset = aux_event->aux_offset; 3232 } 3233 3234 if (aux_offset >= auxtrace_event->offset && 3235 aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { 3236 struct cs_etm_queue *etmq = cs_etm__get_queue(etm, auxtrace_event->cpu); 3237 3238 if (!etmq) 3239 return -EINVAL; 3240 3241 /* 3242 * If this AUX event was inside this buffer somewhere, create a new auxtrace event 3243 * based on the sizes of the aux event, and queue that fragment. 3244 */ 3245 auxtrace_fragment.auxtrace = *auxtrace_event; 3246 auxtrace_fragment.auxtrace.size = aux_size; 3247 auxtrace_fragment.auxtrace.offset = aux_offset; 3248 auxtrace_fragment.auxtrace.idx = etmq->queue_nr; 3249 file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size; 3250 3251 pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64 3252 " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu); 3253 err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment, 3254 file_offset, NULL); 3255 if (err) 3256 return err; 3257 3258 format = (aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW) ? 3259 UNFORMATTED : FORMATTED; 3260 if (etmq->format != UNSET && format != etmq->format) { 3261 pr_err("CS_ETM: mixed formatted and unformatted trace not supported\n"); 3262 return -EINVAL; 3263 } 3264 etmq->format = format; 3265 return 0; 3266 } 3267 3268 /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */ 3269 return 1; 3270 } 3271 3272 static int cs_etm__process_aux_hw_id_cb(struct perf_session *session, union perf_event *event, 3273 u64 offset __maybe_unused, void *data __maybe_unused) 3274 { 3275 /* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */ 3276 if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID) { 3277 (*(int *)data)++; /* increment found count */ 3278 return cs_etm__process_aux_output_hw_id(session, event); 3279 } 3280 return 0; 3281 } 3282 3283 static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event, 3284 u64 offset __maybe_unused, void *data __maybe_unused) 3285 { 3286 struct perf_sample sample; 3287 int ret; 3288 struct auxtrace_index_entry *ent; 3289 struct auxtrace_index *auxtrace_index; 3290 struct evsel *evsel; 3291 size_t i; 3292 3293 /* Don't care about any other events, we're only queuing buffers for AUX events */ 3294 if (event->header.type != PERF_RECORD_AUX) 3295 return 0; 3296 3297 if (event->header.size < sizeof(struct perf_record_aux)) 3298 return -EINVAL; 3299 3300 /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */ 3301 if (!event->aux.aux_size) 3302 return 0; 3303 3304 /* 3305 * Parse the sample, we need the sample_id_all data that comes after the event so that the 3306 * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID. 3307 */ 3308 evsel = evlist__event2evsel(session->evlist, event); 3309 if (!evsel) 3310 return -EINVAL; 3311 perf_sample__init(&sample, /*all=*/false); 3312 ret = evsel__parse_sample(evsel, event, &sample); 3313 if (ret) 3314 goto out; 3315 3316 /* 3317 * Loop through the auxtrace index to find the buffer that matches up with this aux event. 3318 */ 3319 list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) { 3320 for (i = 0; i < auxtrace_index->nr; i++) { 3321 ent = &auxtrace_index->entries[i]; 3322 ret = cs_etm__queue_aux_fragment(session, ent->file_offset, 3323 ent->sz, &event->aux, &sample); 3324 /* 3325 * Stop search on error or successful values. Continue search on 3326 * 1 ('not found') 3327 */ 3328 if (ret != 1) 3329 goto out; 3330 } 3331 } 3332 3333 /* 3334 * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but 3335 * don't exit with an error because it will still be possible to decode other aux records. 3336 */ 3337 pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64 3338 " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu); 3339 ret = 0; 3340 out: 3341 perf_sample__exit(&sample); 3342 return ret; 3343 } 3344 3345 static int cs_etm__queue_aux_records(struct perf_session *session) 3346 { 3347 struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index, 3348 struct auxtrace_index, list); 3349 if (index && index->nr > 0) 3350 return perf_session__peek_events(session, session->header.data_offset, 3351 session->header.data_size, 3352 cs_etm__queue_aux_records_cb, NULL); 3353 3354 /* 3355 * We would get here if there are no entries in the index (either no auxtrace 3356 * buffers or no index at all). Fail silently as there is the possibility of 3357 * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still 3358 * false. 3359 * 3360 * In that scenario, buffers will not be split by AUX records. 3361 */ 3362 return 0; 3363 } 3364 3365 #define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \ 3366 (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1)) 3367 3368 /* 3369 * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual 3370 * timestamps). 3371 */ 3372 static bool cs_etm__has_virtual_ts(u64 **metadata, int num_cpu) 3373 { 3374 int j; 3375 3376 for (j = 0; j < num_cpu; j++) { 3377 switch (metadata[j][CS_ETM_MAGIC]) { 3378 case __perf_cs_etmv4_magic: 3379 if (HAS_PARAM(j, ETMV4, TS_SOURCE) || metadata[j][CS_ETMV4_TS_SOURCE] != 1) 3380 return false; 3381 break; 3382 case __perf_cs_ete_magic: 3383 if (HAS_PARAM(j, ETE, TS_SOURCE) || metadata[j][CS_ETE_TS_SOURCE] != 1) 3384 return false; 3385 break; 3386 default: 3387 /* Unknown / unsupported magic number. */ 3388 return false; 3389 } 3390 } 3391 return true; 3392 } 3393 3394 /* map trace ids to correct metadata block, from information in metadata */ 3395 static int cs_etm__map_trace_ids_metadata(struct cs_etm_auxtrace *etm, int num_cpu, 3396 u64 **metadata) 3397 { 3398 u64 cs_etm_magic; 3399 u8 trace_chan_id; 3400 int i, err; 3401 3402 for (i = 0; i < num_cpu; i++) { 3403 cs_etm_magic = metadata[i][CS_ETM_MAGIC]; 3404 switch (cs_etm_magic) { 3405 case __perf_cs_etmv3_magic: 3406 metadata[i][CS_ETM_ETMTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK; 3407 trace_chan_id = (u8)(metadata[i][CS_ETM_ETMTRACEIDR]); 3408 break; 3409 case __perf_cs_etmv4_magic: 3410 case __perf_cs_ete_magic: 3411 metadata[i][CS_ETMV4_TRCTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK; 3412 trace_chan_id = (u8)(metadata[i][CS_ETMV4_TRCTRACEIDR]); 3413 break; 3414 default: 3415 /* unknown magic number */ 3416 return -EINVAL; 3417 } 3418 err = cs_etm__map_trace_id_v0(etm, trace_chan_id, metadata[i]); 3419 if (err) 3420 return err; 3421 } 3422 return 0; 3423 } 3424 3425 /* 3426 * Use the data gathered by the peeks for HW_ID (trace ID mappings) and AUX 3427 * (formatted or not) packets to create the decoders. 3428 */ 3429 static int cs_etm__create_queue_decoders(struct cs_etm_queue *etmq) 3430 { 3431 struct cs_etm_decoder_params d_params; 3432 struct cs_etm_trace_params *t_params; 3433 int decoders = intlist__nr_entries(etmq->traceid_list); 3434 3435 if (decoders == 0) 3436 return 0; 3437 3438 /* 3439 * Each queue can only contain data from one CPU when unformatted, so only one decoder is 3440 * needed. 3441 */ 3442 if (etmq->format == UNFORMATTED) 3443 assert(decoders == 1); 3444 3445 /* Use metadata to fill in trace parameters for trace decoder */ 3446 t_params = zalloc(sizeof(*t_params) * decoders); 3447 3448 if (!t_params) 3449 goto out_free; 3450 3451 if (cs_etm__init_trace_params(t_params, etmq)) 3452 goto out_free; 3453 3454 /* Set decoder parameters to decode trace packets */ 3455 if (cs_etm__init_decoder_params(&d_params, etmq, 3456 dump_trace ? CS_ETM_OPERATION_PRINT : 3457 CS_ETM_OPERATION_DECODE)) 3458 goto out_free; 3459 3460 etmq->decoder = cs_etm_decoder__new(decoders, &d_params, 3461 t_params); 3462 3463 if (!etmq->decoder) 3464 goto out_free; 3465 3466 /* 3467 * Register a function to handle all memory accesses required by 3468 * the trace decoder library. 3469 */ 3470 if (cs_etm_decoder__add_mem_access_cb(etmq->decoder, 3471 0x0L, ((u64) -1L), 3472 cs_etm__decoder_mem_access)) 3473 goto out_free_decoder; 3474 3475 zfree(&t_params); 3476 return 0; 3477 3478 out_free_decoder: 3479 cs_etm_decoder__free(etmq->decoder); 3480 out_free: 3481 zfree(&t_params); 3482 return -EINVAL; 3483 } 3484 3485 static int cs_etm__create_decoders(struct cs_etm_auxtrace *etm) 3486 { 3487 struct auxtrace_queues *queues = &etm->queues; 3488 3489 for (unsigned int i = 0; i < queues->nr_queues; i++) { 3490 bool empty = list_empty(&queues->queue_array[i].head); 3491 struct cs_etm_queue *etmq = queues->queue_array[i].priv; 3492 int ret; 3493 3494 /* 3495 * Don't create decoders for empty queues, mainly because 3496 * etmq->format is unknown for empty queues. 3497 */ 3498 assert(empty || etmq->format != UNSET); 3499 if (empty) 3500 continue; 3501 3502 ret = cs_etm__create_queue_decoders(etmq); 3503 if (ret) 3504 return ret; 3505 } 3506 return 0; 3507 } 3508 3509 int cs_etm__process_auxtrace_info_full(union perf_event *event, 3510 struct perf_session *session) 3511 { 3512 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; 3513 struct cs_etm_auxtrace *etm = NULL; 3514 struct perf_record_time_conv *tc = &session->time_conv; 3515 int event_header_size = sizeof(struct perf_event_header); 3516 int total_size = auxtrace_info->header.size; 3517 int priv_size = 0; 3518 int num_cpu, max_cpu = 0; 3519 int err = 0; 3520 int aux_hw_id_found; 3521 int i; 3522 u64 *ptr = NULL; 3523 u64 **metadata = NULL; 3524 3525 /* First the global part */ 3526 ptr = (u64 *) auxtrace_info->priv; 3527 num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff; 3528 3529 /* 3530 * Bound num_cpu by the event size: the global header consumes 3531 * CS_ETM_HEADER_SIZE bytes, and each CPU needs at least one u64 3532 * metadata entry after that. 3533 */ 3534 priv_size = total_size - event_header_size - INFO_HEADER_SIZE - 3535 CS_ETM_HEADER_SIZE; 3536 if (num_cpu <= 0 || priv_size <= 0 || 3537 num_cpu > priv_size / (int)sizeof(u64)) 3538 return -EINVAL; 3539 3540 metadata = zalloc(sizeof(*metadata) * num_cpu); 3541 if (!metadata) 3542 return -ENOMEM; 3543 3544 /* Start parsing after the common part of the header */ 3545 i = CS_HEADER_VERSION_MAX; 3546 3547 /* 3548 * The metadata is stored in the auxtrace_info section and encodes 3549 * the configuration of the ARM embedded trace macrocell which is 3550 * required by the trace decoder to properly decode the trace due 3551 * to its highly compressed nature. 3552 */ 3553 for (int j = 0; j < num_cpu; j++) { 3554 if (ptr[i] == __perf_cs_etmv3_magic) { 3555 metadata[j] = 3556 cs_etm__create_meta_blk(ptr, &i, 3557 CS_ETM_PRIV_MAX, 3558 CS_ETM_NR_TRC_PARAMS_V0); 3559 } else if (ptr[i] == __perf_cs_etmv4_magic) { 3560 metadata[j] = 3561 cs_etm__create_meta_blk(ptr, &i, 3562 CS_ETMV4_PRIV_MAX, 3563 CS_ETMV4_NR_TRC_PARAMS_V0); 3564 } else if (ptr[i] == __perf_cs_ete_magic) { 3565 metadata[j] = cs_etm__create_meta_blk(ptr, &i, CS_ETE_PRIV_MAX, -1); 3566 } else { 3567 ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n", 3568 ptr[i]); 3569 err = -EINVAL; 3570 goto err_free_metadata; 3571 } 3572 3573 if (!metadata[j]) { 3574 err = -ENOMEM; 3575 goto err_free_metadata; 3576 } 3577 3578 /* CPU id comes from perf.data and must fit max_cpu + 1 without overflow */ 3579 if (metadata[j][CS_ETM_CPU] >= INT_MAX) { 3580 err = -EINVAL; 3581 goto err_free_metadata; 3582 } 3583 3584 if ((int)metadata[j][CS_ETM_CPU] > max_cpu) 3585 max_cpu = metadata[j][CS_ETM_CPU]; 3586 } 3587 3588 /* 3589 * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and 3590 * CS_ETMV4_PRIV_MAX mark how many double words are in the 3591 * global metadata, and each cpu's metadata respectively. 3592 * The following tests if the correct number of double words was 3593 * present in the auxtrace info section. 3594 */ 3595 priv_size = total_size - event_header_size - INFO_HEADER_SIZE; 3596 if (i * 8 != priv_size) { 3597 err = -EINVAL; 3598 goto err_free_metadata; 3599 } 3600 3601 etm = zalloc(sizeof(*etm)); 3602 3603 if (!etm) { 3604 err = -ENOMEM; 3605 goto err_free_metadata; 3606 } 3607 3608 /* 3609 * As all the ETMs run at the same exception level, the system should 3610 * have the same PID format crossing CPUs. So cache the PID format 3611 * and reuse it for sequential decoding. 3612 */ 3613 etm->pid_fmt = cs_etm__init_pid_fmt(metadata[0]); 3614 3615 err = auxtrace_queues__init_nr(&etm->queues, max_cpu + 1); 3616 if (err) 3617 goto err_free_etm; 3618 3619 for (unsigned int j = 0; j < etm->queues.nr_queues; ++j) { 3620 err = cs_etm__setup_queue(etm, &etm->queues.queue_array[j], j); 3621 if (err) 3622 goto err_free_queues; 3623 } 3624 3625 if (session->itrace_synth_opts->set) { 3626 etm->synth_opts = *session->itrace_synth_opts; 3627 } else { 3628 itrace_synth_opts__set_default(&etm->synth_opts, 3629 session->itrace_synth_opts->default_no_sample); 3630 etm->synth_opts.callchain = false; 3631 etm->synth_opts.thread_stack = session->itrace_synth_opts->thread_stack; 3632 } 3633 3634 if (etm->synth_opts.calls) 3635 etm->branches_filter |= PERF_IP_FLAG_CALL | 3636 PERF_IP_FLAG_TRACE_BEGIN | 3637 PERF_IP_FLAG_TRACE_END; 3638 3639 if (etm->synth_opts.returns) 3640 etm->branches_filter |= PERF_IP_FLAG_RETURN | 3641 PERF_IP_FLAG_TRACE_BEGIN | 3642 PERF_IP_FLAG_TRACE_END; 3643 3644 if (etm->synth_opts.callchain && !symbol_conf.use_callchain) { 3645 symbol_conf.use_callchain = true; 3646 if (callchain_register_param(&callchain_param) < 0) { 3647 symbol_conf.use_callchain = false; 3648 etm->synth_opts.callchain = false; 3649 } 3650 } 3651 3652 etm->session = session; 3653 3654 etm->num_cpu = num_cpu; 3655 etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff); 3656 etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0); 3657 etm->metadata = metadata; 3658 etm->auxtrace_type = auxtrace_info->type; 3659 3660 if (etm->synth_opts.use_timestamp) 3661 /* 3662 * Prior to Armv8.4, Arm CPUs don't support FEAT_TRF feature, 3663 * therefore the decoder cannot know if the timestamp trace is 3664 * same with the kernel time. 3665 * 3666 * If a user has knowledge for the working platform and can 3667 * specify itrace option 'T' to tell decoder to forcely use the 3668 * traced timestamp as the kernel time. 3669 */ 3670 etm->has_virtual_ts = true; 3671 else 3672 /* Use virtual timestamps if all ETMs report ts_source = 1 */ 3673 etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu); 3674 3675 if (!etm->has_virtual_ts) 3676 ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n" 3677 "The time field of the samples will not be set accurately.\n" 3678 "For Arm CPUs prior to Armv8.4 or without support FEAT_TRF,\n" 3679 "you can specify the itrace option 'T' for timestamp decoding\n" 3680 "if the Coresight timestamp on the platform is same with the kernel time.\n\n"); 3681 3682 etm->auxtrace.process_event = cs_etm__process_event; 3683 etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event; 3684 etm->auxtrace.flush_events = cs_etm__flush_events; 3685 etm->auxtrace.free_events = cs_etm__free_events; 3686 etm->auxtrace.free = cs_etm__free; 3687 etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace; 3688 session->auxtrace = &etm->auxtrace; 3689 3690 cs_etm__setup_timeless_decoding(etm); 3691 3692 etm->tc.time_shift = tc->time_shift; 3693 etm->tc.time_mult = tc->time_mult; 3694 etm->tc.time_zero = tc->time_zero; 3695 if (event_contains(*tc, cap_user_time_short)) { 3696 etm->tc.time_cycles = tc->time_cycles; 3697 etm->tc.time_mask = tc->time_mask; 3698 etm->tc.cap_user_time_zero = tc->cap_user_time_zero; 3699 etm->tc.cap_user_time_short = tc->cap_user_time_short; 3700 } 3701 3702 etm->use_thread_stack = etm->synth_opts.thread_stack || 3703 etm->synth_opts.last_branch || 3704 etm->synth_opts.callchain; 3705 3706 etm->use_callchain = etm->synth_opts.thread_stack || 3707 etm->synth_opts.callchain; 3708 3709 err = cs_etm__synth_events(etm, session); 3710 if (err) 3711 goto err_free_queues; 3712 3713 err = cs_etm__queue_aux_records(session); 3714 if (err) 3715 goto err_free_queues; 3716 3717 /* 3718 * Map Trace ID values to CPU metadata. 3719 * 3720 * Trace metadata will always contain Trace ID values from the legacy algorithm 3721 * in case it's read by a version of Perf that doesn't know about HW_ID packets 3722 * or the kernel doesn't emit them. 3723 * 3724 * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use 3725 * the same IDs as the old algorithm as far as is possible, unless there are clashes 3726 * in which case a different value will be used. This means an older perf may still 3727 * be able to record and read files generate on a newer system. 3728 * 3729 * For a perf able to interpret AUX_HW_ID packets we first check for the presence of 3730 * those packets. If they are there then the values will be mapped and plugged into 3731 * the metadata and decoders are only created for each mapping received. 3732 * 3733 * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel 3734 * then we map Trace ID values to CPU directly from the metadata and create decoders 3735 * for all mappings. 3736 */ 3737 3738 /* Scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */ 3739 aux_hw_id_found = 0; 3740 err = perf_session__peek_events(session, session->header.data_offset, 3741 session->header.data_size, 3742 cs_etm__process_aux_hw_id_cb, &aux_hw_id_found); 3743 if (err) 3744 goto err_free_queues; 3745 3746 /* if no HW ID found this is a file with metadata values only, map from metadata */ 3747 if (!aux_hw_id_found) { 3748 err = cs_etm__map_trace_ids_metadata(etm, num_cpu, metadata); 3749 if (err) 3750 goto err_free_queues; 3751 } 3752 3753 err = cs_etm__create_decoders(etm); 3754 if (err) 3755 goto err_free_queues; 3756 3757 etm->data_queued = etm->queues.populated; 3758 return 0; 3759 3760 err_free_queues: 3761 auxtrace_queues__free(&etm->queues); 3762 session->auxtrace = NULL; 3763 err_free_etm: 3764 zfree(&etm); 3765 err_free_metadata: 3766 /* No need to check @metadata[j], free(NULL) is supported */ 3767 for (int j = 0; j < num_cpu; j++) 3768 zfree(&metadata[j]); 3769 zfree(&metadata); 3770 return err; 3771 } 3772