1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define BITULL(x) (1ULL << (x)) 28 29 /* IA32_RTIT_CTL MSR bits */ 30 #define INTEL_PT_CYC_ENABLE BITULL(1) 31 #define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32 #define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define INTEL_PT_RETURN 1 39 40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 41 #define INTEL_PT_MAX_LOOPS 10000 42 43 struct intel_pt_blk { 44 struct intel_pt_blk *prev; 45 uint64_t ip[INTEL_PT_BLK_SIZE]; 46 }; 47 48 struct intel_pt_stack { 49 struct intel_pt_blk *blk; 50 struct intel_pt_blk *spare; 51 int pos; 52 }; 53 54 enum intel_pt_pkt_state { 55 INTEL_PT_STATE_NO_PSB, 56 INTEL_PT_STATE_NO_IP, 57 INTEL_PT_STATE_ERR_RESYNC, 58 INTEL_PT_STATE_IN_SYNC, 59 INTEL_PT_STATE_TNT_CONT, 60 INTEL_PT_STATE_TNT, 61 INTEL_PT_STATE_TIP, 62 INTEL_PT_STATE_TIP_PGD, 63 INTEL_PT_STATE_FUP, 64 INTEL_PT_STATE_FUP_NO_TIP, 65 INTEL_PT_STATE_FUP_IN_PSB, 66 INTEL_PT_STATE_RESAMPLE, 67 }; 68 69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 70 { 71 switch (pkt_state) { 72 case INTEL_PT_STATE_NO_PSB: 73 case INTEL_PT_STATE_NO_IP: 74 case INTEL_PT_STATE_ERR_RESYNC: 75 case INTEL_PT_STATE_IN_SYNC: 76 case INTEL_PT_STATE_TNT_CONT: 77 case INTEL_PT_STATE_RESAMPLE: 78 return true; 79 case INTEL_PT_STATE_TNT: 80 case INTEL_PT_STATE_TIP: 81 case INTEL_PT_STATE_TIP_PGD: 82 case INTEL_PT_STATE_FUP: 83 case INTEL_PT_STATE_FUP_NO_TIP: 84 case INTEL_PT_STATE_FUP_IN_PSB: 85 return false; 86 default: 87 return true; 88 }; 89 } 90 91 #ifdef INTEL_PT_STRICT 92 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 93 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 94 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 95 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 96 #else 97 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 98 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 99 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 100 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 101 #endif 102 103 struct intel_pt_decoder { 104 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 105 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 106 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 107 uint64_t max_insn_cnt, void *data); 108 bool (*pgd_ip)(uint64_t ip, void *data); 109 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 110 void *data; 111 struct intel_pt_state state; 112 const unsigned char *buf; 113 size_t len; 114 bool return_compression; 115 bool branch_enable; 116 bool mtc_insn; 117 bool pge; 118 bool have_tma; 119 bool have_cyc; 120 bool fixup_last_mtc; 121 bool have_last_ip; 122 bool in_psb; 123 bool hop; 124 bool leap; 125 bool nr; 126 bool next_nr; 127 enum intel_pt_param_flags flags; 128 uint64_t pos; 129 uint64_t last_ip; 130 uint64_t ip; 131 uint64_t pip_payload; 132 uint64_t timestamp; 133 uint64_t tsc_timestamp; 134 uint64_t ref_timestamp; 135 uint64_t buf_timestamp; 136 uint64_t sample_timestamp; 137 uint64_t ret_addr; 138 uint64_t ctc_timestamp; 139 uint64_t ctc_delta; 140 uint64_t cycle_cnt; 141 uint64_t cyc_ref_timestamp; 142 uint32_t last_mtc; 143 uint32_t tsc_ctc_ratio_n; 144 uint32_t tsc_ctc_ratio_d; 145 uint32_t tsc_ctc_mult; 146 uint32_t tsc_slip; 147 uint32_t ctc_rem_mask; 148 int mtc_shift; 149 struct intel_pt_stack stack; 150 enum intel_pt_pkt_state pkt_state; 151 enum intel_pt_pkt_ctx pkt_ctx; 152 enum intel_pt_pkt_ctx prev_pkt_ctx; 153 enum intel_pt_blk_type blk_type; 154 int blk_type_pos; 155 struct intel_pt_pkt packet; 156 struct intel_pt_pkt tnt; 157 int pkt_step; 158 int pkt_len; 159 int last_packet_type; 160 unsigned int cbr; 161 unsigned int cbr_seen; 162 unsigned int max_non_turbo_ratio; 163 double max_non_turbo_ratio_fp; 164 double cbr_cyc_to_tsc; 165 double calc_cyc_to_tsc; 166 bool have_calc_cyc_to_tsc; 167 int exec_mode; 168 unsigned int insn_bytes; 169 uint64_t period; 170 enum intel_pt_period_type period_type; 171 uint64_t tot_insn_cnt; 172 uint64_t period_insn_cnt; 173 uint64_t period_mask; 174 uint64_t period_ticks; 175 uint64_t last_masked_timestamp; 176 uint64_t tot_cyc_cnt; 177 uint64_t sample_tot_cyc_cnt; 178 uint64_t base_cyc_cnt; 179 uint64_t cyc_cnt_timestamp; 180 uint64_t ctl; 181 uint64_t cyc_threshold; 182 double tsc_to_cyc; 183 bool continuous_period; 184 bool overflow; 185 bool set_fup_tx_flags; 186 bool set_fup_ptw; 187 bool set_fup_mwait; 188 bool set_fup_pwre; 189 bool set_fup_exstop; 190 bool set_fup_bep; 191 bool sample_cyc; 192 unsigned int fup_tx_flags; 193 unsigned int tx_flags; 194 uint64_t fup_ptw_payload; 195 uint64_t fup_mwait_payload; 196 uint64_t fup_pwre_payload; 197 uint64_t cbr_payload; 198 uint64_t timestamp_insn_cnt; 199 uint64_t sample_insn_cnt; 200 uint64_t stuck_ip; 201 int no_progress; 202 int stuck_ip_prd; 203 int stuck_ip_cnt; 204 uint64_t psb_ip; 205 const unsigned char *next_buf; 206 size_t next_len; 207 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 208 }; 209 210 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 211 { 212 int i; 213 214 for (i = 0; x != 1; i++) 215 x >>= 1; 216 217 return x << i; 218 } 219 220 static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 221 { 222 if (!(ctl & INTEL_PT_CYC_ENABLE)) 223 return 0; 224 225 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 226 } 227 228 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 229 { 230 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 231 uint64_t period; 232 233 period = intel_pt_lower_power_of_2(decoder->period); 234 decoder->period_mask = ~(period - 1); 235 decoder->period_ticks = period; 236 } 237 } 238 239 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 240 { 241 if (!d) 242 return 0; 243 return (t / d) * n + ((t % d) * n) / d; 244 } 245 246 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 247 { 248 struct intel_pt_decoder *decoder; 249 250 if (!params->get_trace || !params->walk_insn) 251 return NULL; 252 253 decoder = zalloc(sizeof(struct intel_pt_decoder)); 254 if (!decoder) 255 return NULL; 256 257 decoder->get_trace = params->get_trace; 258 decoder->walk_insn = params->walk_insn; 259 decoder->pgd_ip = params->pgd_ip; 260 decoder->lookahead = params->lookahead; 261 decoder->data = params->data; 262 decoder->return_compression = params->return_compression; 263 decoder->branch_enable = params->branch_enable; 264 decoder->hop = params->quick >= 1; 265 decoder->leap = params->quick >= 2; 266 267 decoder->flags = params->flags; 268 269 decoder->ctl = params->ctl; 270 decoder->period = params->period; 271 decoder->period_type = params->period_type; 272 273 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 274 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 275 276 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 277 278 intel_pt_setup_period(decoder); 279 280 decoder->mtc_shift = params->mtc_period; 281 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 282 283 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 284 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 285 286 if (!decoder->tsc_ctc_ratio_n) 287 decoder->tsc_ctc_ratio_d = 0; 288 289 if (decoder->tsc_ctc_ratio_d) { 290 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 291 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 292 decoder->tsc_ctc_ratio_d; 293 } 294 295 /* 296 * A TSC packet can slip past MTC packets so that the timestamp appears 297 * to go backwards. One estimate is that can be up to about 40 CPU 298 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 299 * slippage an order of magnitude more to be on the safe side. 300 */ 301 decoder->tsc_slip = 0x10000; 302 303 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 304 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 305 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 306 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 307 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 308 309 if (decoder->hop) 310 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 311 312 return decoder; 313 } 314 315 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 316 { 317 struct intel_pt_blk *blk = stack->blk; 318 319 stack->blk = blk->prev; 320 if (!stack->spare) 321 stack->spare = blk; 322 else 323 free(blk); 324 } 325 326 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 327 { 328 if (!stack->pos) { 329 if (!stack->blk) 330 return 0; 331 intel_pt_pop_blk(stack); 332 if (!stack->blk) 333 return 0; 334 stack->pos = INTEL_PT_BLK_SIZE; 335 } 336 return stack->blk->ip[--stack->pos]; 337 } 338 339 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 340 { 341 struct intel_pt_blk *blk; 342 343 if (stack->spare) { 344 blk = stack->spare; 345 stack->spare = NULL; 346 } else { 347 blk = malloc(sizeof(struct intel_pt_blk)); 348 if (!blk) 349 return -ENOMEM; 350 } 351 352 blk->prev = stack->blk; 353 stack->blk = blk; 354 stack->pos = 0; 355 return 0; 356 } 357 358 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 359 { 360 int err; 361 362 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 363 err = intel_pt_alloc_blk(stack); 364 if (err) 365 return err; 366 } 367 368 stack->blk->ip[stack->pos++] = ip; 369 return 0; 370 } 371 372 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 373 { 374 while (stack->blk) 375 intel_pt_pop_blk(stack); 376 stack->pos = 0; 377 } 378 379 static void intel_pt_free_stack(struct intel_pt_stack *stack) 380 { 381 intel_pt_clear_stack(stack); 382 zfree(&stack->blk); 383 zfree(&stack->spare); 384 } 385 386 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 387 { 388 intel_pt_free_stack(&decoder->stack); 389 free(decoder); 390 } 391 392 static int intel_pt_ext_err(int code) 393 { 394 switch (code) { 395 case -ENOMEM: 396 return INTEL_PT_ERR_NOMEM; 397 case -ENOSYS: 398 return INTEL_PT_ERR_INTERN; 399 case -EBADMSG: 400 return INTEL_PT_ERR_BADPKT; 401 case -ENODATA: 402 return INTEL_PT_ERR_NODATA; 403 case -EILSEQ: 404 return INTEL_PT_ERR_NOINSN; 405 case -ENOENT: 406 return INTEL_PT_ERR_MISMAT; 407 case -EOVERFLOW: 408 return INTEL_PT_ERR_OVR; 409 case -ENOSPC: 410 return INTEL_PT_ERR_LOST; 411 case -ELOOP: 412 return INTEL_PT_ERR_NELOOP; 413 default: 414 return INTEL_PT_ERR_UNK; 415 } 416 } 417 418 static const char *intel_pt_err_msgs[] = { 419 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 420 [INTEL_PT_ERR_INTERN] = "Internal error", 421 [INTEL_PT_ERR_BADPKT] = "Bad packet", 422 [INTEL_PT_ERR_NODATA] = "No more data", 423 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 424 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 425 [INTEL_PT_ERR_OVR] = "Overflow packet", 426 [INTEL_PT_ERR_LOST] = "Lost trace data", 427 [INTEL_PT_ERR_UNK] = "Unknown error!", 428 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 429 }; 430 431 int intel_pt__strerror(int code, char *buf, size_t buflen) 432 { 433 if (code < 1 || code >= INTEL_PT_ERR_MAX) 434 code = INTEL_PT_ERR_UNK; 435 strlcpy(buf, intel_pt_err_msgs[code], buflen); 436 return 0; 437 } 438 439 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 440 uint64_t last_ip) 441 { 442 uint64_t ip; 443 444 switch (packet->count) { 445 case 1: 446 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 447 packet->payload; 448 break; 449 case 2: 450 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 451 packet->payload; 452 break; 453 case 3: 454 ip = packet->payload; 455 /* Sign-extend 6-byte ip */ 456 if (ip & (uint64_t)0x800000000000ULL) 457 ip |= (uint64_t)0xffff000000000000ULL; 458 break; 459 case 4: 460 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 461 packet->payload; 462 break; 463 case 6: 464 ip = packet->payload; 465 break; 466 default: 467 return 0; 468 } 469 470 return ip; 471 } 472 473 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 474 { 475 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 476 decoder->have_last_ip = true; 477 } 478 479 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 480 { 481 intel_pt_set_last_ip(decoder); 482 decoder->ip = decoder->last_ip; 483 } 484 485 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 486 { 487 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 488 decoder->buf); 489 } 490 491 static int intel_pt_bug(struct intel_pt_decoder *decoder) 492 { 493 intel_pt_log("ERROR: Internal error\n"); 494 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 495 return -ENOSYS; 496 } 497 498 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 499 { 500 decoder->tx_flags = 0; 501 } 502 503 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 504 { 505 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 506 } 507 508 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder) 509 { 510 decoder->pip_payload = decoder->packet.payload; 511 } 512 513 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder) 514 { 515 decoder->next_nr = decoder->pip_payload & 1; 516 } 517 518 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder) 519 { 520 decoder->nr = decoder->pip_payload & 1; 521 decoder->next_nr = decoder->nr; 522 } 523 524 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder) 525 { 526 intel_pt_update_pip(decoder); 527 intel_pt_set_nr(decoder); 528 } 529 530 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 531 { 532 intel_pt_clear_tx_flags(decoder); 533 decoder->have_tma = false; 534 decoder->pkt_len = 1; 535 decoder->pkt_step = 1; 536 intel_pt_decoder_log_packet(decoder); 537 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 538 intel_pt_log("ERROR: Bad packet\n"); 539 decoder->pkt_state = INTEL_PT_STATE_ERR1; 540 } 541 return -EBADMSG; 542 } 543 544 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 545 { 546 decoder->sample_timestamp = decoder->timestamp; 547 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 548 } 549 550 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 551 { 552 decoder->ip = 0; 553 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 554 decoder->timestamp = 0; 555 decoder->have_tma = false; 556 } 557 558 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 559 { 560 struct intel_pt_buffer buffer = { .buf = 0, }; 561 int ret; 562 563 decoder->pkt_step = 0; 564 565 intel_pt_log("Getting more data\n"); 566 ret = decoder->get_trace(&buffer, decoder->data); 567 if (ret) 568 return ret; 569 decoder->buf = buffer.buf; 570 decoder->len = buffer.len; 571 if (!decoder->len) { 572 intel_pt_log("No more data\n"); 573 return -ENODATA; 574 } 575 decoder->buf_timestamp = buffer.ref_timestamp; 576 if (!buffer.consecutive || reposition) { 577 intel_pt_reposition(decoder); 578 decoder->ref_timestamp = buffer.ref_timestamp; 579 decoder->state.trace_nr = buffer.trace_nr; 580 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 581 decoder->ref_timestamp); 582 return -ENOLINK; 583 } 584 585 return 0; 586 } 587 588 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 589 bool reposition) 590 { 591 if (!decoder->next_buf) 592 return intel_pt_get_data(decoder, reposition); 593 594 decoder->buf = decoder->next_buf; 595 decoder->len = decoder->next_len; 596 decoder->next_buf = 0; 597 decoder->next_len = 0; 598 return 0; 599 } 600 601 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 602 { 603 unsigned char *buf = decoder->temp_buf; 604 size_t old_len, len, n; 605 int ret; 606 607 old_len = decoder->len; 608 len = decoder->len; 609 memcpy(buf, decoder->buf, len); 610 611 ret = intel_pt_get_data(decoder, false); 612 if (ret) { 613 decoder->pos += old_len; 614 return ret < 0 ? ret : -EINVAL; 615 } 616 617 n = INTEL_PT_PKT_MAX_SZ - len; 618 if (n > decoder->len) 619 n = decoder->len; 620 memcpy(buf + len, decoder->buf, n); 621 len += n; 622 623 decoder->prev_pkt_ctx = decoder->pkt_ctx; 624 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 625 if (ret < (int)old_len) { 626 decoder->next_buf = decoder->buf; 627 decoder->next_len = decoder->len; 628 decoder->buf = buf; 629 decoder->len = old_len; 630 return intel_pt_bad_packet(decoder); 631 } 632 633 decoder->next_buf = decoder->buf + (ret - old_len); 634 decoder->next_len = decoder->len - (ret - old_len); 635 636 decoder->buf = buf; 637 decoder->len = ret; 638 639 return ret; 640 } 641 642 struct intel_pt_pkt_info { 643 struct intel_pt_decoder *decoder; 644 struct intel_pt_pkt packet; 645 uint64_t pos; 646 int pkt_len; 647 int last_packet_type; 648 void *data; 649 }; 650 651 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 652 653 /* Lookahead packets in current buffer */ 654 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 655 intel_pt_pkt_cb_t cb, void *data) 656 { 657 struct intel_pt_pkt_info pkt_info; 658 const unsigned char *buf = decoder->buf; 659 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 660 size_t len = decoder->len; 661 int ret; 662 663 pkt_info.decoder = decoder; 664 pkt_info.pos = decoder->pos; 665 pkt_info.pkt_len = decoder->pkt_step; 666 pkt_info.last_packet_type = decoder->last_packet_type; 667 pkt_info.data = data; 668 669 while (1) { 670 do { 671 pkt_info.pos += pkt_info.pkt_len; 672 buf += pkt_info.pkt_len; 673 len -= pkt_info.pkt_len; 674 675 if (!len) 676 return INTEL_PT_NEED_MORE_BYTES; 677 678 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 679 &pkt_ctx); 680 if (!ret) 681 return INTEL_PT_NEED_MORE_BYTES; 682 if (ret < 0) 683 return ret; 684 685 pkt_info.pkt_len = ret; 686 } while (pkt_info.packet.type == INTEL_PT_PAD); 687 688 ret = cb(&pkt_info); 689 if (ret) 690 return 0; 691 692 pkt_info.last_packet_type = pkt_info.packet.type; 693 } 694 } 695 696 struct intel_pt_calc_cyc_to_tsc_info { 697 uint64_t cycle_cnt; 698 unsigned int cbr; 699 uint32_t last_mtc; 700 uint64_t ctc_timestamp; 701 uint64_t ctc_delta; 702 uint64_t tsc_timestamp; 703 uint64_t timestamp; 704 bool have_tma; 705 bool fixup_last_mtc; 706 bool from_mtc; 707 double cbr_cyc_to_tsc; 708 }; 709 710 /* 711 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 712 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 713 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 714 * packet by copying the missing bits from the current MTC assuming the least 715 * difference between the two, and that the current MTC comes after last_mtc. 716 */ 717 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 718 uint32_t *last_mtc) 719 { 720 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 721 uint32_t mask = ~(first_missing_bit - 1); 722 723 *last_mtc |= mtc & mask; 724 if (*last_mtc >= mtc) { 725 *last_mtc -= first_missing_bit; 726 *last_mtc &= 0xff; 727 } 728 } 729 730 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 731 { 732 struct intel_pt_decoder *decoder = pkt_info->decoder; 733 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 734 uint64_t timestamp; 735 double cyc_to_tsc; 736 unsigned int cbr; 737 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 738 739 switch (pkt_info->packet.type) { 740 case INTEL_PT_TNT: 741 case INTEL_PT_TIP_PGE: 742 case INTEL_PT_TIP: 743 case INTEL_PT_FUP: 744 case INTEL_PT_PSB: 745 case INTEL_PT_PIP: 746 case INTEL_PT_MODE_EXEC: 747 case INTEL_PT_MODE_TSX: 748 case INTEL_PT_PSBEND: 749 case INTEL_PT_PAD: 750 case INTEL_PT_VMCS: 751 case INTEL_PT_MNT: 752 case INTEL_PT_PTWRITE: 753 case INTEL_PT_PTWRITE_IP: 754 case INTEL_PT_BBP: 755 case INTEL_PT_BIP: 756 case INTEL_PT_BEP: 757 case INTEL_PT_BEP_IP: 758 return 0; 759 760 case INTEL_PT_MTC: 761 if (!data->have_tma) 762 return 0; 763 764 mtc = pkt_info->packet.payload; 765 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 766 data->fixup_last_mtc = false; 767 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 768 &data->last_mtc); 769 } 770 if (mtc > data->last_mtc) 771 mtc_delta = mtc - data->last_mtc; 772 else 773 mtc_delta = mtc + 256 - data->last_mtc; 774 data->ctc_delta += mtc_delta << decoder->mtc_shift; 775 data->last_mtc = mtc; 776 777 if (decoder->tsc_ctc_mult) { 778 timestamp = data->ctc_timestamp + 779 data->ctc_delta * decoder->tsc_ctc_mult; 780 } else { 781 timestamp = data->ctc_timestamp + 782 multdiv(data->ctc_delta, 783 decoder->tsc_ctc_ratio_n, 784 decoder->tsc_ctc_ratio_d); 785 } 786 787 if (timestamp < data->timestamp) 788 return 1; 789 790 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 791 data->timestamp = timestamp; 792 return 0; 793 } 794 795 break; 796 797 case INTEL_PT_TSC: 798 /* 799 * For now, do not support using TSC packets - refer 800 * intel_pt_calc_cyc_to_tsc(). 801 */ 802 if (data->from_mtc) 803 return 1; 804 timestamp = pkt_info->packet.payload | 805 (data->timestamp & (0xffULL << 56)); 806 if (data->from_mtc && timestamp < data->timestamp && 807 data->timestamp - timestamp < decoder->tsc_slip) 808 return 1; 809 if (timestamp < data->timestamp) 810 timestamp += (1ULL << 56); 811 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 812 if (data->from_mtc) 813 return 1; 814 data->tsc_timestamp = timestamp; 815 data->timestamp = timestamp; 816 return 0; 817 } 818 break; 819 820 case INTEL_PT_TMA: 821 if (data->from_mtc) 822 return 1; 823 824 if (!decoder->tsc_ctc_ratio_d) 825 return 0; 826 827 ctc = pkt_info->packet.payload; 828 fc = pkt_info->packet.count; 829 ctc_rem = ctc & decoder->ctc_rem_mask; 830 831 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 832 833 data->ctc_timestamp = data->tsc_timestamp - fc; 834 if (decoder->tsc_ctc_mult) { 835 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 836 } else { 837 data->ctc_timestamp -= 838 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 839 decoder->tsc_ctc_ratio_d); 840 } 841 842 data->ctc_delta = 0; 843 data->have_tma = true; 844 data->fixup_last_mtc = true; 845 846 return 0; 847 848 case INTEL_PT_CYC: 849 data->cycle_cnt += pkt_info->packet.payload; 850 return 0; 851 852 case INTEL_PT_CBR: 853 cbr = pkt_info->packet.payload; 854 if (data->cbr && data->cbr != cbr) 855 return 1; 856 data->cbr = cbr; 857 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 858 return 0; 859 860 case INTEL_PT_TIP_PGD: 861 case INTEL_PT_TRACESTOP: 862 case INTEL_PT_EXSTOP: 863 case INTEL_PT_EXSTOP_IP: 864 case INTEL_PT_MWAIT: 865 case INTEL_PT_PWRE: 866 case INTEL_PT_PWRX: 867 case INTEL_PT_OVF: 868 case INTEL_PT_BAD: /* Does not happen */ 869 default: 870 return 1; 871 } 872 873 if (!data->cbr && decoder->cbr) { 874 data->cbr = decoder->cbr; 875 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 876 } 877 878 if (!data->cycle_cnt) 879 return 1; 880 881 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 882 883 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 884 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 885 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 886 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 887 return 1; 888 } 889 890 decoder->calc_cyc_to_tsc = cyc_to_tsc; 891 decoder->have_calc_cyc_to_tsc = true; 892 893 if (data->cbr) { 894 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 895 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 896 } else { 897 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 898 cyc_to_tsc, pkt_info->pos); 899 } 900 901 return 1; 902 } 903 904 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 905 bool from_mtc) 906 { 907 struct intel_pt_calc_cyc_to_tsc_info data = { 908 .cycle_cnt = 0, 909 .cbr = 0, 910 .last_mtc = decoder->last_mtc, 911 .ctc_timestamp = decoder->ctc_timestamp, 912 .ctc_delta = decoder->ctc_delta, 913 .tsc_timestamp = decoder->tsc_timestamp, 914 .timestamp = decoder->timestamp, 915 .have_tma = decoder->have_tma, 916 .fixup_last_mtc = decoder->fixup_last_mtc, 917 .from_mtc = from_mtc, 918 .cbr_cyc_to_tsc = 0, 919 }; 920 921 /* 922 * For now, do not support using TSC packets for at least the reasons: 923 * 1) timing might have stopped 924 * 2) TSC packets within PSB+ can slip against CYC packets 925 */ 926 if (!from_mtc) 927 return; 928 929 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 930 } 931 932 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 933 { 934 int ret; 935 936 decoder->last_packet_type = decoder->packet.type; 937 938 do { 939 decoder->pos += decoder->pkt_step; 940 decoder->buf += decoder->pkt_step; 941 decoder->len -= decoder->pkt_step; 942 943 if (!decoder->len) { 944 ret = intel_pt_get_next_data(decoder, false); 945 if (ret) 946 return ret; 947 } 948 949 decoder->prev_pkt_ctx = decoder->pkt_ctx; 950 ret = intel_pt_get_packet(decoder->buf, decoder->len, 951 &decoder->packet, &decoder->pkt_ctx); 952 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 953 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 954 ret = intel_pt_get_split_packet(decoder); 955 if (ret < 0) 956 return ret; 957 } 958 if (ret <= 0) 959 return intel_pt_bad_packet(decoder); 960 961 decoder->pkt_len = ret; 962 decoder->pkt_step = ret; 963 intel_pt_decoder_log_packet(decoder); 964 } while (decoder->packet.type == INTEL_PT_PAD); 965 966 return 0; 967 } 968 969 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 970 { 971 uint64_t timestamp, masked_timestamp; 972 973 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 974 masked_timestamp = timestamp & decoder->period_mask; 975 if (decoder->continuous_period) { 976 if (masked_timestamp > decoder->last_masked_timestamp) 977 return 1; 978 } else { 979 timestamp += 1; 980 masked_timestamp = timestamp & decoder->period_mask; 981 if (masked_timestamp > decoder->last_masked_timestamp) { 982 decoder->last_masked_timestamp = masked_timestamp; 983 decoder->continuous_period = true; 984 } 985 } 986 987 if (masked_timestamp < decoder->last_masked_timestamp) 988 return decoder->period_ticks; 989 990 return decoder->period_ticks - (timestamp - masked_timestamp); 991 } 992 993 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 994 { 995 switch (decoder->period_type) { 996 case INTEL_PT_PERIOD_INSTRUCTIONS: 997 return decoder->period - decoder->period_insn_cnt; 998 case INTEL_PT_PERIOD_TICKS: 999 return intel_pt_next_period(decoder); 1000 case INTEL_PT_PERIOD_NONE: 1001 case INTEL_PT_PERIOD_MTC: 1002 default: 1003 return 0; 1004 } 1005 } 1006 1007 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 1008 { 1009 uint64_t timestamp, masked_timestamp; 1010 1011 switch (decoder->period_type) { 1012 case INTEL_PT_PERIOD_INSTRUCTIONS: 1013 decoder->period_insn_cnt = 0; 1014 break; 1015 case INTEL_PT_PERIOD_TICKS: 1016 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1017 masked_timestamp = timestamp & decoder->period_mask; 1018 if (masked_timestamp > decoder->last_masked_timestamp) 1019 decoder->last_masked_timestamp = masked_timestamp; 1020 else 1021 decoder->last_masked_timestamp += decoder->period_ticks; 1022 break; 1023 case INTEL_PT_PERIOD_NONE: 1024 case INTEL_PT_PERIOD_MTC: 1025 default: 1026 break; 1027 } 1028 1029 decoder->state.type |= INTEL_PT_INSTRUCTION; 1030 } 1031 1032 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1033 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1034 { 1035 uint64_t max_insn_cnt, insn_cnt = 0; 1036 int err; 1037 1038 if (!decoder->mtc_insn) 1039 decoder->mtc_insn = true; 1040 1041 max_insn_cnt = intel_pt_next_sample(decoder); 1042 1043 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1044 max_insn_cnt, decoder->data); 1045 1046 decoder->tot_insn_cnt += insn_cnt; 1047 decoder->timestamp_insn_cnt += insn_cnt; 1048 decoder->sample_insn_cnt += insn_cnt; 1049 decoder->period_insn_cnt += insn_cnt; 1050 1051 if (err) { 1052 decoder->no_progress = 0; 1053 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1054 intel_pt_log_at("ERROR: Failed to get instruction", 1055 decoder->ip); 1056 if (err == -ENOENT) 1057 return -ENOLINK; 1058 return -EILSEQ; 1059 } 1060 1061 if (ip && decoder->ip == ip) { 1062 err = -EAGAIN; 1063 goto out; 1064 } 1065 1066 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1067 intel_pt_sample_insn(decoder); 1068 1069 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1070 decoder->state.type = INTEL_PT_INSTRUCTION; 1071 decoder->state.from_ip = decoder->ip; 1072 decoder->state.to_ip = 0; 1073 decoder->ip += intel_pt_insn->length; 1074 err = INTEL_PT_RETURN; 1075 goto out; 1076 } 1077 1078 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1079 /* Zero-length calls are excluded */ 1080 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1081 intel_pt_insn->rel) { 1082 err = intel_pt_push(&decoder->stack, decoder->ip + 1083 intel_pt_insn->length); 1084 if (err) 1085 goto out; 1086 } 1087 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1088 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1089 } 1090 1091 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1092 int cnt = decoder->no_progress++; 1093 1094 decoder->state.from_ip = decoder->ip; 1095 decoder->ip += intel_pt_insn->length + 1096 intel_pt_insn->rel; 1097 decoder->state.to_ip = decoder->ip; 1098 err = INTEL_PT_RETURN; 1099 1100 /* 1101 * Check for being stuck in a loop. This can happen if a 1102 * decoder error results in the decoder erroneously setting the 1103 * ip to an address that is itself in an infinite loop that 1104 * consumes no packets. When that happens, there must be an 1105 * unconditional branch. 1106 */ 1107 if (cnt) { 1108 if (cnt == 1) { 1109 decoder->stuck_ip = decoder->state.to_ip; 1110 decoder->stuck_ip_prd = 1; 1111 decoder->stuck_ip_cnt = 1; 1112 } else if (cnt > INTEL_PT_MAX_LOOPS || 1113 decoder->state.to_ip == decoder->stuck_ip) { 1114 intel_pt_log_at("ERROR: Never-ending loop", 1115 decoder->state.to_ip); 1116 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1117 err = -ELOOP; 1118 goto out; 1119 } else if (!--decoder->stuck_ip_cnt) { 1120 decoder->stuck_ip_prd += 1; 1121 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1122 decoder->stuck_ip = decoder->state.to_ip; 1123 } 1124 } 1125 goto out_no_progress; 1126 } 1127 out: 1128 decoder->no_progress = 0; 1129 out_no_progress: 1130 decoder->state.insn_op = intel_pt_insn->op; 1131 decoder->state.insn_len = intel_pt_insn->length; 1132 memcpy(decoder->state.insn, intel_pt_insn->buf, 1133 INTEL_PT_INSN_BUF_SZ); 1134 1135 if (decoder->tx_flags & INTEL_PT_IN_TX) 1136 decoder->state.flags |= INTEL_PT_IN_TX; 1137 1138 return err; 1139 } 1140 1141 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1142 { 1143 bool ret = false; 1144 1145 if (decoder->set_fup_tx_flags) { 1146 decoder->set_fup_tx_flags = false; 1147 decoder->tx_flags = decoder->fup_tx_flags; 1148 decoder->state.type = INTEL_PT_TRANSACTION; 1149 decoder->state.from_ip = decoder->ip; 1150 decoder->state.to_ip = 0; 1151 decoder->state.flags = decoder->fup_tx_flags; 1152 return true; 1153 } 1154 if (decoder->set_fup_ptw) { 1155 decoder->set_fup_ptw = false; 1156 decoder->state.type = INTEL_PT_PTW; 1157 decoder->state.flags |= INTEL_PT_FUP_IP; 1158 decoder->state.from_ip = decoder->ip; 1159 decoder->state.to_ip = 0; 1160 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1161 return true; 1162 } 1163 if (decoder->set_fup_mwait) { 1164 decoder->set_fup_mwait = false; 1165 decoder->state.type = INTEL_PT_MWAIT_OP; 1166 decoder->state.from_ip = decoder->ip; 1167 decoder->state.to_ip = 0; 1168 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1169 ret = true; 1170 } 1171 if (decoder->set_fup_pwre) { 1172 decoder->set_fup_pwre = false; 1173 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1174 decoder->state.type &= ~INTEL_PT_BRANCH; 1175 decoder->state.from_ip = decoder->ip; 1176 decoder->state.to_ip = 0; 1177 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1178 ret = true; 1179 } 1180 if (decoder->set_fup_exstop) { 1181 decoder->set_fup_exstop = false; 1182 decoder->state.type |= INTEL_PT_EX_STOP; 1183 decoder->state.type &= ~INTEL_PT_BRANCH; 1184 decoder->state.flags |= INTEL_PT_FUP_IP; 1185 decoder->state.from_ip = decoder->ip; 1186 decoder->state.to_ip = 0; 1187 ret = true; 1188 } 1189 if (decoder->set_fup_bep) { 1190 decoder->set_fup_bep = false; 1191 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1192 decoder->state.type &= ~INTEL_PT_BRANCH; 1193 decoder->state.from_ip = decoder->ip; 1194 decoder->state.to_ip = 0; 1195 ret = true; 1196 } 1197 return ret; 1198 } 1199 1200 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1201 struct intel_pt_insn *intel_pt_insn, 1202 uint64_t ip, int err) 1203 { 1204 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1205 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1206 ip == decoder->ip + intel_pt_insn->length; 1207 } 1208 1209 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1210 { 1211 struct intel_pt_insn intel_pt_insn; 1212 uint64_t ip; 1213 int err; 1214 1215 ip = decoder->last_ip; 1216 1217 while (1) { 1218 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1219 if (err == INTEL_PT_RETURN) 1220 return 0; 1221 if (err == -EAGAIN || 1222 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1223 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1224 if (intel_pt_fup_event(decoder)) 1225 return 0; 1226 return -EAGAIN; 1227 } 1228 decoder->set_fup_tx_flags = false; 1229 if (err) 1230 return err; 1231 1232 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1233 intel_pt_log_at("ERROR: Unexpected indirect branch", 1234 decoder->ip); 1235 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1236 return -ENOENT; 1237 } 1238 1239 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1240 intel_pt_log_at("ERROR: Unexpected conditional branch", 1241 decoder->ip); 1242 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1243 return -ENOENT; 1244 } 1245 1246 intel_pt_bug(decoder); 1247 } 1248 } 1249 1250 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1251 { 1252 struct intel_pt_insn intel_pt_insn; 1253 int err; 1254 1255 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1256 if (err == INTEL_PT_RETURN && 1257 decoder->pgd_ip && 1258 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1259 (decoder->state.type & INTEL_PT_BRANCH) && 1260 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1261 /* Unconditional branch leaving filter region */ 1262 decoder->no_progress = 0; 1263 decoder->pge = false; 1264 decoder->continuous_period = false; 1265 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1266 decoder->state.type |= INTEL_PT_TRACE_END; 1267 intel_pt_update_nr(decoder); 1268 return 0; 1269 } 1270 if (err == INTEL_PT_RETURN) 1271 return 0; 1272 if (err) 1273 return err; 1274 1275 intel_pt_update_nr(decoder); 1276 1277 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1278 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1279 decoder->pge = false; 1280 decoder->continuous_period = false; 1281 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1282 decoder->state.from_ip = decoder->ip; 1283 if (decoder->packet.count == 0) { 1284 decoder->state.to_ip = 0; 1285 } else { 1286 decoder->state.to_ip = decoder->last_ip; 1287 decoder->ip = decoder->last_ip; 1288 } 1289 decoder->state.type |= INTEL_PT_TRACE_END; 1290 } else { 1291 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1292 decoder->state.from_ip = decoder->ip; 1293 if (decoder->packet.count == 0) { 1294 decoder->state.to_ip = 0; 1295 } else { 1296 decoder->state.to_ip = decoder->last_ip; 1297 decoder->ip = decoder->last_ip; 1298 } 1299 } 1300 return 0; 1301 } 1302 1303 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1304 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1305 intel_pt_insn.rel; 1306 1307 if (decoder->pgd_ip && 1308 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1309 decoder->pgd_ip(to_ip, decoder->data)) { 1310 /* Conditional branch leaving filter region */ 1311 decoder->pge = false; 1312 decoder->continuous_period = false; 1313 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1314 decoder->ip = to_ip; 1315 decoder->state.from_ip = decoder->ip; 1316 decoder->state.to_ip = to_ip; 1317 decoder->state.type |= INTEL_PT_TRACE_END; 1318 return 0; 1319 } 1320 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1321 decoder->ip); 1322 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1323 return -ENOENT; 1324 } 1325 1326 return intel_pt_bug(decoder); 1327 } 1328 1329 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1330 { 1331 struct intel_pt_insn intel_pt_insn; 1332 int err; 1333 1334 while (1) { 1335 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1336 if (err == INTEL_PT_RETURN) 1337 return 0; 1338 if (err) 1339 return err; 1340 1341 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1342 if (!decoder->return_compression) { 1343 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1344 decoder->ip); 1345 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1346 return -ENOENT; 1347 } 1348 if (!decoder->ret_addr) { 1349 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1350 decoder->ip); 1351 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1352 return -ENOENT; 1353 } 1354 if (!(decoder->tnt.payload & BIT63)) { 1355 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1356 decoder->ip); 1357 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1358 return -ENOENT; 1359 } 1360 decoder->tnt.count -= 1; 1361 if (decoder->tnt.count) 1362 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1363 else 1364 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1365 decoder->tnt.payload <<= 1; 1366 decoder->state.from_ip = decoder->ip; 1367 decoder->ip = decoder->ret_addr; 1368 decoder->state.to_ip = decoder->ip; 1369 return 0; 1370 } 1371 1372 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1373 /* Handle deferred TIPs */ 1374 err = intel_pt_get_next_packet(decoder); 1375 if (err) 1376 return err; 1377 if (decoder->packet.type != INTEL_PT_TIP || 1378 decoder->packet.count == 0) { 1379 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1380 decoder->ip); 1381 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1382 decoder->pkt_step = 0; 1383 return -ENOENT; 1384 } 1385 intel_pt_set_last_ip(decoder); 1386 decoder->state.from_ip = decoder->ip; 1387 decoder->state.to_ip = decoder->last_ip; 1388 decoder->ip = decoder->last_ip; 1389 intel_pt_update_nr(decoder); 1390 return 0; 1391 } 1392 1393 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1394 decoder->tnt.count -= 1; 1395 if (decoder->tnt.count) 1396 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1397 else 1398 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1399 if (decoder->tnt.payload & BIT63) { 1400 decoder->tnt.payload <<= 1; 1401 decoder->state.from_ip = decoder->ip; 1402 decoder->ip += intel_pt_insn.length + 1403 intel_pt_insn.rel; 1404 decoder->state.to_ip = decoder->ip; 1405 return 0; 1406 } 1407 /* Instruction sample for a non-taken branch */ 1408 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1409 decoder->tnt.payload <<= 1; 1410 decoder->state.type = INTEL_PT_INSTRUCTION; 1411 decoder->state.from_ip = decoder->ip; 1412 decoder->state.to_ip = 0; 1413 decoder->ip += intel_pt_insn.length; 1414 return 0; 1415 } 1416 decoder->sample_cyc = false; 1417 decoder->ip += intel_pt_insn.length; 1418 if (!decoder->tnt.count) { 1419 intel_pt_update_sample_time(decoder); 1420 return -EAGAIN; 1421 } 1422 decoder->tnt.payload <<= 1; 1423 continue; 1424 } 1425 1426 return intel_pt_bug(decoder); 1427 } 1428 } 1429 1430 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1431 { 1432 unsigned int fup_tx_flags; 1433 int err; 1434 1435 fup_tx_flags = decoder->packet.payload & 1436 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1437 err = intel_pt_get_next_packet(decoder); 1438 if (err) 1439 return err; 1440 if (decoder->packet.type == INTEL_PT_FUP) { 1441 decoder->fup_tx_flags = fup_tx_flags; 1442 decoder->set_fup_tx_flags = true; 1443 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1444 *no_tip = true; 1445 } else { 1446 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1447 decoder->pos); 1448 intel_pt_update_in_tx(decoder); 1449 } 1450 return 0; 1451 } 1452 1453 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1454 { 1455 timestamp |= (ref_timestamp & (0xffULL << 56)); 1456 1457 if (timestamp < ref_timestamp) { 1458 if (ref_timestamp - timestamp > (1ULL << 55)) 1459 timestamp += (1ULL << 56); 1460 } else { 1461 if (timestamp - ref_timestamp > (1ULL << 55)) 1462 timestamp -= (1ULL << 56); 1463 } 1464 1465 return timestamp; 1466 } 1467 1468 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1469 { 1470 uint64_t timestamp; 1471 1472 decoder->have_tma = false; 1473 1474 if (decoder->ref_timestamp) { 1475 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1476 decoder->ref_timestamp); 1477 decoder->tsc_timestamp = timestamp; 1478 decoder->timestamp = timestamp; 1479 decoder->ref_timestamp = 0; 1480 decoder->timestamp_insn_cnt = 0; 1481 } else if (decoder->timestamp) { 1482 timestamp = decoder->packet.payload | 1483 (decoder->timestamp & (0xffULL << 56)); 1484 decoder->tsc_timestamp = timestamp; 1485 if (timestamp < decoder->timestamp && 1486 decoder->timestamp - timestamp < decoder->tsc_slip) { 1487 intel_pt_log_to("Suppressing backwards timestamp", 1488 timestamp); 1489 timestamp = decoder->timestamp; 1490 } 1491 if (timestamp < decoder->timestamp) { 1492 intel_pt_log_to("Wraparound timestamp", timestamp); 1493 timestamp += (1ULL << 56); 1494 decoder->tsc_timestamp = timestamp; 1495 } 1496 decoder->timestamp = timestamp; 1497 decoder->timestamp_insn_cnt = 0; 1498 } 1499 1500 if (decoder->last_packet_type == INTEL_PT_CYC) { 1501 decoder->cyc_ref_timestamp = decoder->timestamp; 1502 decoder->cycle_cnt = 0; 1503 decoder->have_calc_cyc_to_tsc = false; 1504 intel_pt_calc_cyc_to_tsc(decoder, false); 1505 } 1506 1507 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1508 } 1509 1510 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1511 { 1512 intel_pt_log("ERROR: Buffer overflow\n"); 1513 intel_pt_clear_tx_flags(decoder); 1514 intel_pt_set_nr(decoder); 1515 decoder->timestamp_insn_cnt = 0; 1516 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1517 decoder->overflow = true; 1518 return -EOVERFLOW; 1519 } 1520 1521 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1522 { 1523 if (decoder->have_cyc) 1524 return; 1525 1526 decoder->cyc_cnt_timestamp = decoder->timestamp; 1527 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1528 } 1529 1530 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1531 { 1532 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1533 1534 if (decoder->pge) 1535 intel_pt_mtc_cyc_cnt_pge(decoder); 1536 } 1537 1538 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1539 { 1540 uint64_t tot_cyc_cnt, tsc_delta; 1541 1542 if (decoder->have_cyc) 1543 return; 1544 1545 decoder->sample_cyc = true; 1546 1547 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1548 return; 1549 1550 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1551 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1552 1553 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1554 decoder->tot_cyc_cnt = tot_cyc_cnt; 1555 } 1556 1557 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1558 { 1559 uint32_t ctc = decoder->packet.payload; 1560 uint32_t fc = decoder->packet.count; 1561 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1562 1563 if (!decoder->tsc_ctc_ratio_d) 1564 return; 1565 1566 if (decoder->pge && !decoder->in_psb) 1567 intel_pt_mtc_cyc_cnt_pge(decoder); 1568 else 1569 intel_pt_mtc_cyc_cnt_upd(decoder); 1570 1571 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1572 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1573 if (decoder->tsc_ctc_mult) { 1574 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1575 } else { 1576 decoder->ctc_timestamp -= multdiv(ctc_rem, 1577 decoder->tsc_ctc_ratio_n, 1578 decoder->tsc_ctc_ratio_d); 1579 } 1580 decoder->ctc_delta = 0; 1581 decoder->have_tma = true; 1582 decoder->fixup_last_mtc = true; 1583 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1584 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1585 } 1586 1587 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1588 { 1589 uint64_t timestamp; 1590 uint32_t mtc, mtc_delta; 1591 1592 if (!decoder->have_tma) 1593 return; 1594 1595 mtc = decoder->packet.payload; 1596 1597 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1598 decoder->fixup_last_mtc = false; 1599 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1600 &decoder->last_mtc); 1601 } 1602 1603 if (mtc > decoder->last_mtc) 1604 mtc_delta = mtc - decoder->last_mtc; 1605 else 1606 mtc_delta = mtc + 256 - decoder->last_mtc; 1607 1608 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1609 1610 if (decoder->tsc_ctc_mult) { 1611 timestamp = decoder->ctc_timestamp + 1612 decoder->ctc_delta * decoder->tsc_ctc_mult; 1613 } else { 1614 timestamp = decoder->ctc_timestamp + 1615 multdiv(decoder->ctc_delta, 1616 decoder->tsc_ctc_ratio_n, 1617 decoder->tsc_ctc_ratio_d); 1618 } 1619 1620 if (timestamp < decoder->timestamp) 1621 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1622 timestamp, decoder->timestamp); 1623 else 1624 decoder->timestamp = timestamp; 1625 1626 intel_pt_mtc_cyc_cnt_upd(decoder); 1627 1628 decoder->timestamp_insn_cnt = 0; 1629 decoder->last_mtc = mtc; 1630 1631 if (decoder->last_packet_type == INTEL_PT_CYC) { 1632 decoder->cyc_ref_timestamp = decoder->timestamp; 1633 decoder->cycle_cnt = 0; 1634 decoder->have_calc_cyc_to_tsc = false; 1635 intel_pt_calc_cyc_to_tsc(decoder, true); 1636 } 1637 1638 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1639 } 1640 1641 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1642 { 1643 unsigned int cbr = decoder->packet.payload & 0xff; 1644 1645 decoder->cbr_payload = decoder->packet.payload; 1646 1647 if (decoder->cbr == cbr) 1648 return; 1649 1650 decoder->cbr = cbr; 1651 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1652 1653 intel_pt_mtc_cyc_cnt_cbr(decoder); 1654 } 1655 1656 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1657 { 1658 uint64_t timestamp = decoder->cyc_ref_timestamp; 1659 1660 decoder->have_cyc = true; 1661 1662 decoder->cycle_cnt += decoder->packet.payload; 1663 if (decoder->pge) 1664 decoder->tot_cyc_cnt += decoder->packet.payload; 1665 decoder->sample_cyc = true; 1666 1667 if (!decoder->cyc_ref_timestamp) 1668 return; 1669 1670 if (decoder->have_calc_cyc_to_tsc) 1671 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1672 else if (decoder->cbr) 1673 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1674 else 1675 return; 1676 1677 if (timestamp < decoder->timestamp) 1678 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1679 timestamp, decoder->timestamp); 1680 else 1681 decoder->timestamp = timestamp; 1682 1683 decoder->timestamp_insn_cnt = 0; 1684 1685 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1686 } 1687 1688 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1689 { 1690 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1691 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1692 decoder->state.items.is_32_bit = false; 1693 } 1694 decoder->blk_type = decoder->packet.payload; 1695 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1696 if (decoder->blk_type == INTEL_PT_GP_REGS) 1697 decoder->state.items.is_32_bit = decoder->packet.count; 1698 if (decoder->blk_type_pos < 0) { 1699 intel_pt_log("WARNING: Unknown block type %u\n", 1700 decoder->blk_type); 1701 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1702 intel_pt_log("WARNING: Duplicate block type %u\n", 1703 decoder->blk_type); 1704 } 1705 } 1706 1707 static void intel_pt_bip(struct intel_pt_decoder *decoder) 1708 { 1709 uint32_t id = decoder->packet.count; 1710 uint32_t bit = 1 << id; 1711 int pos = decoder->blk_type_pos; 1712 1713 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1714 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1715 id, decoder->blk_type); 1716 return; 1717 } 1718 1719 if (decoder->state.items.mask[pos] & bit) { 1720 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1721 id, decoder->blk_type); 1722 } 1723 1724 decoder->state.items.mask[pos] |= bit; 1725 decoder->state.items.val[pos][id] = decoder->packet.payload; 1726 } 1727 1728 /* Walk PSB+ packets when already in sync. */ 1729 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1730 { 1731 int err; 1732 1733 decoder->in_psb = true; 1734 1735 while (1) { 1736 err = intel_pt_get_next_packet(decoder); 1737 if (err) 1738 goto out; 1739 1740 switch (decoder->packet.type) { 1741 case INTEL_PT_PSBEND: 1742 err = 0; 1743 goto out; 1744 1745 case INTEL_PT_TIP_PGD: 1746 case INTEL_PT_TIP_PGE: 1747 case INTEL_PT_TIP: 1748 case INTEL_PT_TNT: 1749 case INTEL_PT_TRACESTOP: 1750 case INTEL_PT_BAD: 1751 case INTEL_PT_PSB: 1752 case INTEL_PT_PTWRITE: 1753 case INTEL_PT_PTWRITE_IP: 1754 case INTEL_PT_EXSTOP: 1755 case INTEL_PT_EXSTOP_IP: 1756 case INTEL_PT_MWAIT: 1757 case INTEL_PT_PWRE: 1758 case INTEL_PT_PWRX: 1759 case INTEL_PT_BBP: 1760 case INTEL_PT_BIP: 1761 case INTEL_PT_BEP: 1762 case INTEL_PT_BEP_IP: 1763 decoder->have_tma = false; 1764 intel_pt_log("ERROR: Unexpected packet\n"); 1765 err = -EAGAIN; 1766 goto out; 1767 1768 case INTEL_PT_OVF: 1769 err = intel_pt_overflow(decoder); 1770 goto out; 1771 1772 case INTEL_PT_TSC: 1773 intel_pt_calc_tsc_timestamp(decoder); 1774 break; 1775 1776 case INTEL_PT_TMA: 1777 intel_pt_calc_tma(decoder); 1778 break; 1779 1780 case INTEL_PT_CBR: 1781 intel_pt_calc_cbr(decoder); 1782 break; 1783 1784 case INTEL_PT_MODE_EXEC: 1785 decoder->exec_mode = decoder->packet.payload; 1786 break; 1787 1788 case INTEL_PT_PIP: 1789 intel_pt_set_pip(decoder); 1790 break; 1791 1792 case INTEL_PT_FUP: 1793 decoder->pge = true; 1794 if (decoder->packet.count) { 1795 intel_pt_set_last_ip(decoder); 1796 decoder->psb_ip = decoder->last_ip; 1797 } 1798 break; 1799 1800 case INTEL_PT_MODE_TSX: 1801 intel_pt_update_in_tx(decoder); 1802 break; 1803 1804 case INTEL_PT_MTC: 1805 intel_pt_calc_mtc_timestamp(decoder); 1806 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1807 decoder->state.type |= INTEL_PT_INSTRUCTION; 1808 break; 1809 1810 case INTEL_PT_CYC: 1811 intel_pt_calc_cyc_timestamp(decoder); 1812 break; 1813 1814 case INTEL_PT_VMCS: 1815 case INTEL_PT_MNT: 1816 case INTEL_PT_PAD: 1817 default: 1818 break; 1819 } 1820 } 1821 out: 1822 decoder->in_psb = false; 1823 1824 return err; 1825 } 1826 1827 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1828 { 1829 int err; 1830 1831 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1832 decoder->tx_flags = 0; 1833 decoder->state.flags &= ~INTEL_PT_IN_TX; 1834 decoder->state.flags |= INTEL_PT_ABORT_TX; 1835 } else { 1836 decoder->state.flags |= INTEL_PT_ASYNC; 1837 } 1838 1839 while (1) { 1840 err = intel_pt_get_next_packet(decoder); 1841 if (err) 1842 return err; 1843 1844 switch (decoder->packet.type) { 1845 case INTEL_PT_TNT: 1846 case INTEL_PT_FUP: 1847 case INTEL_PT_TRACESTOP: 1848 case INTEL_PT_PSB: 1849 case INTEL_PT_TSC: 1850 case INTEL_PT_TMA: 1851 case INTEL_PT_MODE_TSX: 1852 case INTEL_PT_BAD: 1853 case INTEL_PT_PSBEND: 1854 case INTEL_PT_PTWRITE: 1855 case INTEL_PT_PTWRITE_IP: 1856 case INTEL_PT_EXSTOP: 1857 case INTEL_PT_EXSTOP_IP: 1858 case INTEL_PT_MWAIT: 1859 case INTEL_PT_PWRE: 1860 case INTEL_PT_PWRX: 1861 case INTEL_PT_BBP: 1862 case INTEL_PT_BIP: 1863 case INTEL_PT_BEP: 1864 case INTEL_PT_BEP_IP: 1865 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1866 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1867 decoder->pkt_step = 0; 1868 return -ENOENT; 1869 1870 case INTEL_PT_CBR: 1871 intel_pt_calc_cbr(decoder); 1872 break; 1873 1874 case INTEL_PT_OVF: 1875 return intel_pt_overflow(decoder); 1876 1877 case INTEL_PT_TIP_PGD: 1878 decoder->state.from_ip = decoder->ip; 1879 if (decoder->packet.count == 0) { 1880 decoder->state.to_ip = 0; 1881 } else { 1882 intel_pt_set_ip(decoder); 1883 decoder->state.to_ip = decoder->ip; 1884 } 1885 decoder->pge = false; 1886 decoder->continuous_period = false; 1887 decoder->state.type |= INTEL_PT_TRACE_END; 1888 intel_pt_update_nr(decoder); 1889 return 0; 1890 1891 case INTEL_PT_TIP_PGE: 1892 decoder->pge = true; 1893 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1894 decoder->ip); 1895 decoder->state.from_ip = 0; 1896 if (decoder->packet.count == 0) { 1897 decoder->state.to_ip = 0; 1898 } else { 1899 intel_pt_set_ip(decoder); 1900 decoder->state.to_ip = decoder->ip; 1901 } 1902 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 1903 intel_pt_mtc_cyc_cnt_pge(decoder); 1904 intel_pt_set_nr(decoder); 1905 return 0; 1906 1907 case INTEL_PT_TIP: 1908 decoder->state.from_ip = decoder->ip; 1909 if (decoder->packet.count == 0) { 1910 decoder->state.to_ip = 0; 1911 } else { 1912 intel_pt_set_ip(decoder); 1913 decoder->state.to_ip = decoder->ip; 1914 } 1915 intel_pt_update_nr(decoder); 1916 return 0; 1917 1918 case INTEL_PT_PIP: 1919 intel_pt_update_pip(decoder); 1920 break; 1921 1922 case INTEL_PT_MTC: 1923 intel_pt_calc_mtc_timestamp(decoder); 1924 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1925 decoder->state.type |= INTEL_PT_INSTRUCTION; 1926 break; 1927 1928 case INTEL_PT_CYC: 1929 intel_pt_calc_cyc_timestamp(decoder); 1930 break; 1931 1932 case INTEL_PT_MODE_EXEC: 1933 decoder->exec_mode = decoder->packet.payload; 1934 break; 1935 1936 case INTEL_PT_VMCS: 1937 case INTEL_PT_MNT: 1938 case INTEL_PT_PAD: 1939 break; 1940 1941 default: 1942 return intel_pt_bug(decoder); 1943 } 1944 } 1945 } 1946 1947 static int intel_pt_resample(struct intel_pt_decoder *decoder) 1948 { 1949 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1950 decoder->state.type = INTEL_PT_INSTRUCTION; 1951 decoder->state.from_ip = decoder->ip; 1952 decoder->state.to_ip = 0; 1953 return 0; 1954 } 1955 1956 #define HOP_PROCESS 0 1957 #define HOP_IGNORE 1 1958 #define HOP_RETURN 2 1959 #define HOP_AGAIN 3 1960 1961 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 1962 1963 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 1964 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 1965 { 1966 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 1967 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 1968 *err = intel_pt_scan_for_psb(decoder); 1969 if (*err) 1970 return HOP_RETURN; 1971 } 1972 1973 switch (decoder->packet.type) { 1974 case INTEL_PT_TNT: 1975 return HOP_IGNORE; 1976 1977 case INTEL_PT_TIP_PGD: 1978 if (!decoder->packet.count) { 1979 intel_pt_set_nr(decoder); 1980 return HOP_IGNORE; 1981 } 1982 intel_pt_set_ip(decoder); 1983 decoder->state.type |= INTEL_PT_TRACE_END; 1984 decoder->state.from_ip = 0; 1985 decoder->state.to_ip = decoder->ip; 1986 intel_pt_update_nr(decoder); 1987 return HOP_RETURN; 1988 1989 case INTEL_PT_TIP: 1990 if (!decoder->packet.count) { 1991 intel_pt_set_nr(decoder); 1992 return HOP_IGNORE; 1993 } 1994 intel_pt_set_ip(decoder); 1995 decoder->state.type = INTEL_PT_INSTRUCTION; 1996 decoder->state.from_ip = decoder->ip; 1997 decoder->state.to_ip = 0; 1998 intel_pt_update_nr(decoder); 1999 return HOP_RETURN; 2000 2001 case INTEL_PT_FUP: 2002 if (!decoder->packet.count) 2003 return HOP_IGNORE; 2004 intel_pt_set_ip(decoder); 2005 if (intel_pt_fup_event(decoder)) 2006 return HOP_RETURN; 2007 if (!decoder->branch_enable) 2008 *no_tip = true; 2009 if (*no_tip) { 2010 decoder->state.type = INTEL_PT_INSTRUCTION; 2011 decoder->state.from_ip = decoder->ip; 2012 decoder->state.to_ip = 0; 2013 return HOP_RETURN; 2014 } 2015 *err = intel_pt_walk_fup_tip(decoder); 2016 if (!*err) 2017 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2018 return HOP_RETURN; 2019 2020 case INTEL_PT_PSB: 2021 decoder->state.psb_offset = decoder->pos; 2022 decoder->psb_ip = 0; 2023 decoder->last_ip = 0; 2024 decoder->have_last_ip = true; 2025 *err = intel_pt_walk_psbend(decoder); 2026 if (*err == -EAGAIN) 2027 return HOP_AGAIN; 2028 if (*err) 2029 return HOP_RETURN; 2030 decoder->state.type = INTEL_PT_PSB_EVT; 2031 if (decoder->psb_ip) { 2032 decoder->state.type |= INTEL_PT_INSTRUCTION; 2033 decoder->ip = decoder->psb_ip; 2034 } 2035 decoder->state.from_ip = decoder->psb_ip; 2036 decoder->state.to_ip = 0; 2037 return HOP_RETURN; 2038 2039 case INTEL_PT_BAD: 2040 case INTEL_PT_PAD: 2041 case INTEL_PT_TIP_PGE: 2042 case INTEL_PT_TSC: 2043 case INTEL_PT_TMA: 2044 case INTEL_PT_MODE_EXEC: 2045 case INTEL_PT_MODE_TSX: 2046 case INTEL_PT_MTC: 2047 case INTEL_PT_CYC: 2048 case INTEL_PT_VMCS: 2049 case INTEL_PT_PSBEND: 2050 case INTEL_PT_CBR: 2051 case INTEL_PT_TRACESTOP: 2052 case INTEL_PT_PIP: 2053 case INTEL_PT_OVF: 2054 case INTEL_PT_MNT: 2055 case INTEL_PT_PTWRITE: 2056 case INTEL_PT_PTWRITE_IP: 2057 case INTEL_PT_EXSTOP: 2058 case INTEL_PT_EXSTOP_IP: 2059 case INTEL_PT_MWAIT: 2060 case INTEL_PT_PWRE: 2061 case INTEL_PT_PWRX: 2062 case INTEL_PT_BBP: 2063 case INTEL_PT_BIP: 2064 case INTEL_PT_BEP: 2065 case INTEL_PT_BEP_IP: 2066 default: 2067 return HOP_PROCESS; 2068 } 2069 } 2070 2071 struct intel_pt_psb_info { 2072 struct intel_pt_pkt fup_packet; 2073 bool fup; 2074 int after_psbend; 2075 }; 2076 2077 /* Lookahead and get the FUP packet from PSB+ */ 2078 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2079 { 2080 struct intel_pt_psb_info *data = pkt_info->data; 2081 2082 switch (pkt_info->packet.type) { 2083 case INTEL_PT_PAD: 2084 case INTEL_PT_MNT: 2085 case INTEL_PT_TSC: 2086 case INTEL_PT_TMA: 2087 case INTEL_PT_MODE_EXEC: 2088 case INTEL_PT_MODE_TSX: 2089 case INTEL_PT_MTC: 2090 case INTEL_PT_CYC: 2091 case INTEL_PT_VMCS: 2092 case INTEL_PT_CBR: 2093 case INTEL_PT_PIP: 2094 if (data->after_psbend) { 2095 data->after_psbend -= 1; 2096 if (!data->after_psbend) 2097 return 1; 2098 } 2099 break; 2100 2101 case INTEL_PT_FUP: 2102 if (data->after_psbend) 2103 return 1; 2104 if (data->fup || pkt_info->packet.count == 0) 2105 return 1; 2106 data->fup_packet = pkt_info->packet; 2107 data->fup = true; 2108 break; 2109 2110 case INTEL_PT_PSBEND: 2111 if (!data->fup) 2112 return 1; 2113 /* Keep going to check for a TIP.PGE */ 2114 data->after_psbend = 6; 2115 break; 2116 2117 case INTEL_PT_TIP_PGE: 2118 /* Ignore FUP in PSB+ if followed by TIP.PGE */ 2119 if (data->after_psbend) 2120 data->fup = false; 2121 return 1; 2122 2123 case INTEL_PT_PTWRITE: 2124 case INTEL_PT_PTWRITE_IP: 2125 case INTEL_PT_EXSTOP: 2126 case INTEL_PT_EXSTOP_IP: 2127 case INTEL_PT_MWAIT: 2128 case INTEL_PT_PWRE: 2129 case INTEL_PT_PWRX: 2130 case INTEL_PT_BBP: 2131 case INTEL_PT_BIP: 2132 case INTEL_PT_BEP: 2133 case INTEL_PT_BEP_IP: 2134 if (data->after_psbend) { 2135 data->after_psbend -= 1; 2136 if (!data->after_psbend) 2137 return 1; 2138 break; 2139 } 2140 return 1; 2141 2142 case INTEL_PT_OVF: 2143 case INTEL_PT_BAD: 2144 case INTEL_PT_TNT: 2145 case INTEL_PT_TIP_PGD: 2146 case INTEL_PT_TIP: 2147 case INTEL_PT_PSB: 2148 case INTEL_PT_TRACESTOP: 2149 default: 2150 return 1; 2151 } 2152 2153 return 0; 2154 } 2155 2156 static int intel_pt_psb(struct intel_pt_decoder *decoder) 2157 { 2158 int err; 2159 2160 decoder->last_ip = 0; 2161 decoder->psb_ip = 0; 2162 decoder->have_last_ip = true; 2163 intel_pt_clear_stack(&decoder->stack); 2164 err = intel_pt_walk_psbend(decoder); 2165 if (err) 2166 return err; 2167 decoder->state.type = INTEL_PT_PSB_EVT; 2168 decoder->state.from_ip = decoder->psb_ip; 2169 decoder->state.to_ip = 0; 2170 return 0; 2171 } 2172 2173 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder) 2174 { 2175 int err; 2176 2177 if (decoder->ip != decoder->last_ip) { 2178 err = intel_pt_walk_fup(decoder); 2179 if (!err || err != -EAGAIN) 2180 return err; 2181 } 2182 2183 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2184 err = intel_pt_psb(decoder); 2185 if (err) { 2186 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2187 return -ENOENT; 2188 } 2189 2190 return 0; 2191 } 2192 2193 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err) 2194 { 2195 struct intel_pt_psb_info data = { .fup = false }; 2196 2197 if (!decoder->branch_enable || !decoder->pge) 2198 return false; 2199 2200 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); 2201 if (!data.fup) 2202 return false; 2203 2204 decoder->packet = data.fup_packet; 2205 intel_pt_set_last_ip(decoder); 2206 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; 2207 2208 *err = intel_pt_fup_in_psb(decoder); 2209 2210 return true; 2211 } 2212 2213 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 2214 { 2215 int last_packet_type = INTEL_PT_PAD; 2216 bool no_tip = false; 2217 int err; 2218 2219 while (1) { 2220 err = intel_pt_get_next_packet(decoder); 2221 if (err) 2222 return err; 2223 next: 2224 if (decoder->cyc_threshold) { 2225 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 2226 decoder->sample_cyc = false; 2227 last_packet_type = decoder->packet.type; 2228 } 2229 2230 if (decoder->hop) { 2231 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 2232 case HOP_IGNORE: 2233 continue; 2234 case HOP_RETURN: 2235 return err; 2236 case HOP_AGAIN: 2237 goto next; 2238 default: 2239 break; 2240 } 2241 } 2242 2243 switch (decoder->packet.type) { 2244 case INTEL_PT_TNT: 2245 if (!decoder->packet.count) 2246 break; 2247 decoder->tnt = decoder->packet; 2248 decoder->pkt_state = INTEL_PT_STATE_TNT; 2249 err = intel_pt_walk_tnt(decoder); 2250 if (err == -EAGAIN) 2251 break; 2252 return err; 2253 2254 case INTEL_PT_TIP_PGD: 2255 if (decoder->packet.count != 0) 2256 intel_pt_set_last_ip(decoder); 2257 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 2258 return intel_pt_walk_tip(decoder); 2259 2260 case INTEL_PT_TIP_PGE: { 2261 decoder->pge = true; 2262 intel_pt_mtc_cyc_cnt_pge(decoder); 2263 intel_pt_set_nr(decoder); 2264 if (decoder->packet.count == 0) { 2265 intel_pt_log_at("Skipping zero TIP.PGE", 2266 decoder->pos); 2267 break; 2268 } 2269 intel_pt_set_ip(decoder); 2270 decoder->state.from_ip = 0; 2271 decoder->state.to_ip = decoder->ip; 2272 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2273 /* 2274 * In hop mode, resample to get the to_ip as an 2275 * "instruction" sample. 2276 */ 2277 if (decoder->hop) 2278 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2279 return 0; 2280 } 2281 2282 case INTEL_PT_OVF: 2283 return intel_pt_overflow(decoder); 2284 2285 case INTEL_PT_TIP: 2286 if (decoder->packet.count != 0) 2287 intel_pt_set_last_ip(decoder); 2288 decoder->pkt_state = INTEL_PT_STATE_TIP; 2289 return intel_pt_walk_tip(decoder); 2290 2291 case INTEL_PT_FUP: 2292 if (decoder->packet.count == 0) { 2293 intel_pt_log_at("Skipping zero FUP", 2294 decoder->pos); 2295 no_tip = false; 2296 break; 2297 } 2298 intel_pt_set_last_ip(decoder); 2299 if (!decoder->branch_enable) { 2300 decoder->ip = decoder->last_ip; 2301 if (intel_pt_fup_event(decoder)) 2302 return 0; 2303 no_tip = false; 2304 break; 2305 } 2306 if (decoder->set_fup_mwait) 2307 no_tip = true; 2308 if (no_tip) 2309 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 2310 else 2311 decoder->pkt_state = INTEL_PT_STATE_FUP; 2312 err = intel_pt_walk_fup(decoder); 2313 if (err != -EAGAIN) 2314 return err; 2315 if (no_tip) { 2316 no_tip = false; 2317 break; 2318 } 2319 return intel_pt_walk_fup_tip(decoder); 2320 2321 case INTEL_PT_TRACESTOP: 2322 decoder->pge = false; 2323 decoder->continuous_period = false; 2324 intel_pt_clear_tx_flags(decoder); 2325 decoder->have_tma = false; 2326 break; 2327 2328 case INTEL_PT_PSB: 2329 decoder->state.psb_offset = decoder->pos; 2330 decoder->psb_ip = 0; 2331 if (intel_pt_psb_with_fup(decoder, &err)) 2332 return err; 2333 err = intel_pt_psb(decoder); 2334 if (err == -EAGAIN) 2335 goto next; 2336 return err; 2337 2338 case INTEL_PT_PIP: 2339 intel_pt_update_pip(decoder); 2340 break; 2341 2342 case INTEL_PT_MTC: 2343 intel_pt_calc_mtc_timestamp(decoder); 2344 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 2345 break; 2346 /* 2347 * Ensure that there has been an instruction since the 2348 * last MTC. 2349 */ 2350 if (!decoder->mtc_insn) 2351 break; 2352 decoder->mtc_insn = false; 2353 /* Ensure that there is a timestamp */ 2354 if (!decoder->timestamp) 2355 break; 2356 decoder->state.type = INTEL_PT_INSTRUCTION; 2357 decoder->state.from_ip = decoder->ip; 2358 decoder->state.to_ip = 0; 2359 decoder->mtc_insn = false; 2360 return 0; 2361 2362 case INTEL_PT_TSC: 2363 intel_pt_calc_tsc_timestamp(decoder); 2364 break; 2365 2366 case INTEL_PT_TMA: 2367 intel_pt_calc_tma(decoder); 2368 break; 2369 2370 case INTEL_PT_CYC: 2371 intel_pt_calc_cyc_timestamp(decoder); 2372 break; 2373 2374 case INTEL_PT_CBR: 2375 intel_pt_calc_cbr(decoder); 2376 if (decoder->cbr != decoder->cbr_seen) { 2377 decoder->state.type = 0; 2378 return 0; 2379 } 2380 break; 2381 2382 case INTEL_PT_MODE_EXEC: 2383 decoder->exec_mode = decoder->packet.payload; 2384 break; 2385 2386 case INTEL_PT_MODE_TSX: 2387 /* MODE_TSX need not be followed by FUP */ 2388 if (!decoder->pge || decoder->in_psb) { 2389 intel_pt_update_in_tx(decoder); 2390 break; 2391 } 2392 err = intel_pt_mode_tsx(decoder, &no_tip); 2393 if (err) 2394 return err; 2395 goto next; 2396 2397 case INTEL_PT_BAD: /* Does not happen */ 2398 return intel_pt_bug(decoder); 2399 2400 case INTEL_PT_PSBEND: 2401 case INTEL_PT_VMCS: 2402 case INTEL_PT_MNT: 2403 case INTEL_PT_PAD: 2404 break; 2405 2406 case INTEL_PT_PTWRITE_IP: 2407 decoder->fup_ptw_payload = decoder->packet.payload; 2408 err = intel_pt_get_next_packet(decoder); 2409 if (err) 2410 return err; 2411 if (decoder->packet.type == INTEL_PT_FUP) { 2412 decoder->set_fup_ptw = true; 2413 no_tip = true; 2414 } else { 2415 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 2416 decoder->pos); 2417 } 2418 goto next; 2419 2420 case INTEL_PT_PTWRITE: 2421 decoder->state.type = INTEL_PT_PTW; 2422 decoder->state.from_ip = decoder->ip; 2423 decoder->state.to_ip = 0; 2424 decoder->state.ptw_payload = decoder->packet.payload; 2425 return 0; 2426 2427 case INTEL_PT_MWAIT: 2428 decoder->fup_mwait_payload = decoder->packet.payload; 2429 decoder->set_fup_mwait = true; 2430 break; 2431 2432 case INTEL_PT_PWRE: 2433 if (decoder->set_fup_mwait) { 2434 decoder->fup_pwre_payload = 2435 decoder->packet.payload; 2436 decoder->set_fup_pwre = true; 2437 break; 2438 } 2439 decoder->state.type = INTEL_PT_PWR_ENTRY; 2440 decoder->state.from_ip = decoder->ip; 2441 decoder->state.to_ip = 0; 2442 decoder->state.pwrx_payload = decoder->packet.payload; 2443 return 0; 2444 2445 case INTEL_PT_EXSTOP_IP: 2446 err = intel_pt_get_next_packet(decoder); 2447 if (err) 2448 return err; 2449 if (decoder->packet.type == INTEL_PT_FUP) { 2450 decoder->set_fup_exstop = true; 2451 no_tip = true; 2452 } else { 2453 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 2454 decoder->pos); 2455 } 2456 goto next; 2457 2458 case INTEL_PT_EXSTOP: 2459 decoder->state.type = INTEL_PT_EX_STOP; 2460 decoder->state.from_ip = decoder->ip; 2461 decoder->state.to_ip = 0; 2462 return 0; 2463 2464 case INTEL_PT_PWRX: 2465 decoder->state.type = INTEL_PT_PWR_EXIT; 2466 decoder->state.from_ip = decoder->ip; 2467 decoder->state.to_ip = 0; 2468 decoder->state.pwrx_payload = decoder->packet.payload; 2469 return 0; 2470 2471 case INTEL_PT_BBP: 2472 intel_pt_bbp(decoder); 2473 break; 2474 2475 case INTEL_PT_BIP: 2476 intel_pt_bip(decoder); 2477 break; 2478 2479 case INTEL_PT_BEP: 2480 decoder->state.type = INTEL_PT_BLK_ITEMS; 2481 decoder->state.from_ip = decoder->ip; 2482 decoder->state.to_ip = 0; 2483 return 0; 2484 2485 case INTEL_PT_BEP_IP: 2486 err = intel_pt_get_next_packet(decoder); 2487 if (err) 2488 return err; 2489 if (decoder->packet.type == INTEL_PT_FUP) { 2490 decoder->set_fup_bep = true; 2491 no_tip = true; 2492 } else { 2493 intel_pt_log_at("ERROR: Missing FUP after BEP", 2494 decoder->pos); 2495 } 2496 goto next; 2497 2498 default: 2499 return intel_pt_bug(decoder); 2500 } 2501 } 2502 } 2503 2504 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 2505 { 2506 return decoder->packet.count && 2507 (decoder->have_last_ip || decoder->packet.count == 3 || 2508 decoder->packet.count == 6); 2509 } 2510 2511 /* Walk PSB+ packets to get in sync. */ 2512 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 2513 { 2514 int err; 2515 2516 decoder->in_psb = true; 2517 2518 while (1) { 2519 err = intel_pt_get_next_packet(decoder); 2520 if (err) 2521 goto out; 2522 2523 switch (decoder->packet.type) { 2524 case INTEL_PT_TIP_PGD: 2525 decoder->continuous_period = false; 2526 __fallthrough; 2527 case INTEL_PT_TIP_PGE: 2528 case INTEL_PT_TIP: 2529 case INTEL_PT_PTWRITE: 2530 case INTEL_PT_PTWRITE_IP: 2531 case INTEL_PT_EXSTOP: 2532 case INTEL_PT_EXSTOP_IP: 2533 case INTEL_PT_MWAIT: 2534 case INTEL_PT_PWRE: 2535 case INTEL_PT_PWRX: 2536 case INTEL_PT_BBP: 2537 case INTEL_PT_BIP: 2538 case INTEL_PT_BEP: 2539 case INTEL_PT_BEP_IP: 2540 intel_pt_log("ERROR: Unexpected packet\n"); 2541 err = -ENOENT; 2542 goto out; 2543 2544 case INTEL_PT_FUP: 2545 decoder->pge = true; 2546 if (intel_pt_have_ip(decoder)) { 2547 uint64_t current_ip = decoder->ip; 2548 2549 intel_pt_set_ip(decoder); 2550 decoder->psb_ip = decoder->ip; 2551 if (current_ip) 2552 intel_pt_log_to("Setting IP", 2553 decoder->ip); 2554 } 2555 break; 2556 2557 case INTEL_PT_MTC: 2558 intel_pt_calc_mtc_timestamp(decoder); 2559 break; 2560 2561 case INTEL_PT_TSC: 2562 intel_pt_calc_tsc_timestamp(decoder); 2563 break; 2564 2565 case INTEL_PT_TMA: 2566 intel_pt_calc_tma(decoder); 2567 break; 2568 2569 case INTEL_PT_CYC: 2570 intel_pt_calc_cyc_timestamp(decoder); 2571 break; 2572 2573 case INTEL_PT_CBR: 2574 intel_pt_calc_cbr(decoder); 2575 break; 2576 2577 case INTEL_PT_PIP: 2578 intel_pt_set_pip(decoder); 2579 break; 2580 2581 case INTEL_PT_MODE_EXEC: 2582 decoder->exec_mode = decoder->packet.payload; 2583 break; 2584 2585 case INTEL_PT_MODE_TSX: 2586 intel_pt_update_in_tx(decoder); 2587 break; 2588 2589 case INTEL_PT_TRACESTOP: 2590 decoder->pge = false; 2591 decoder->continuous_period = false; 2592 intel_pt_clear_tx_flags(decoder); 2593 __fallthrough; 2594 2595 case INTEL_PT_TNT: 2596 decoder->have_tma = false; 2597 intel_pt_log("ERROR: Unexpected packet\n"); 2598 if (decoder->ip) 2599 decoder->pkt_state = INTEL_PT_STATE_ERR4; 2600 else 2601 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2602 err = -ENOENT; 2603 goto out; 2604 2605 case INTEL_PT_BAD: /* Does not happen */ 2606 err = intel_pt_bug(decoder); 2607 goto out; 2608 2609 case INTEL_PT_OVF: 2610 err = intel_pt_overflow(decoder); 2611 goto out; 2612 2613 case INTEL_PT_PSBEND: 2614 err = 0; 2615 goto out; 2616 2617 case INTEL_PT_PSB: 2618 case INTEL_PT_VMCS: 2619 case INTEL_PT_MNT: 2620 case INTEL_PT_PAD: 2621 default: 2622 break; 2623 } 2624 } 2625 out: 2626 decoder->in_psb = false; 2627 2628 return err; 2629 } 2630 2631 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 2632 { 2633 int err; 2634 2635 while (1) { 2636 err = intel_pt_get_next_packet(decoder); 2637 if (err) 2638 return err; 2639 2640 switch (decoder->packet.type) { 2641 case INTEL_PT_TIP_PGD: 2642 decoder->continuous_period = false; 2643 decoder->pge = false; 2644 if (intel_pt_have_ip(decoder)) 2645 intel_pt_set_ip(decoder); 2646 if (!decoder->ip) 2647 break; 2648 decoder->state.type |= INTEL_PT_TRACE_END; 2649 return 0; 2650 2651 case INTEL_PT_TIP_PGE: 2652 decoder->pge = true; 2653 intel_pt_mtc_cyc_cnt_pge(decoder); 2654 if (intel_pt_have_ip(decoder)) 2655 intel_pt_set_ip(decoder); 2656 if (!decoder->ip) 2657 break; 2658 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2659 return 0; 2660 2661 case INTEL_PT_TIP: 2662 decoder->pge = true; 2663 if (intel_pt_have_ip(decoder)) 2664 intel_pt_set_ip(decoder); 2665 if (!decoder->ip) 2666 break; 2667 return 0; 2668 2669 case INTEL_PT_FUP: 2670 if (intel_pt_have_ip(decoder)) 2671 intel_pt_set_ip(decoder); 2672 if (decoder->ip) 2673 return 0; 2674 break; 2675 2676 case INTEL_PT_MTC: 2677 intel_pt_calc_mtc_timestamp(decoder); 2678 break; 2679 2680 case INTEL_PT_TSC: 2681 intel_pt_calc_tsc_timestamp(decoder); 2682 break; 2683 2684 case INTEL_PT_TMA: 2685 intel_pt_calc_tma(decoder); 2686 break; 2687 2688 case INTEL_PT_CYC: 2689 intel_pt_calc_cyc_timestamp(decoder); 2690 break; 2691 2692 case INTEL_PT_CBR: 2693 intel_pt_calc_cbr(decoder); 2694 break; 2695 2696 case INTEL_PT_PIP: 2697 intel_pt_set_pip(decoder); 2698 break; 2699 2700 case INTEL_PT_MODE_EXEC: 2701 decoder->exec_mode = decoder->packet.payload; 2702 break; 2703 2704 case INTEL_PT_MODE_TSX: 2705 intel_pt_update_in_tx(decoder); 2706 break; 2707 2708 case INTEL_PT_OVF: 2709 return intel_pt_overflow(decoder); 2710 2711 case INTEL_PT_BAD: /* Does not happen */ 2712 return intel_pt_bug(decoder); 2713 2714 case INTEL_PT_TRACESTOP: 2715 decoder->pge = false; 2716 decoder->continuous_period = false; 2717 intel_pt_clear_tx_flags(decoder); 2718 decoder->have_tma = false; 2719 break; 2720 2721 case INTEL_PT_PSB: 2722 decoder->state.psb_offset = decoder->pos; 2723 decoder->psb_ip = 0; 2724 decoder->last_ip = 0; 2725 decoder->have_last_ip = true; 2726 intel_pt_clear_stack(&decoder->stack); 2727 err = intel_pt_walk_psb(decoder); 2728 if (err) 2729 return err; 2730 decoder->state.type = INTEL_PT_PSB_EVT; 2731 decoder->state.from_ip = decoder->psb_ip; 2732 decoder->state.to_ip = 0; 2733 return 0; 2734 2735 case INTEL_PT_TNT: 2736 case INTEL_PT_PSBEND: 2737 case INTEL_PT_VMCS: 2738 case INTEL_PT_MNT: 2739 case INTEL_PT_PAD: 2740 case INTEL_PT_PTWRITE: 2741 case INTEL_PT_PTWRITE_IP: 2742 case INTEL_PT_EXSTOP: 2743 case INTEL_PT_EXSTOP_IP: 2744 case INTEL_PT_MWAIT: 2745 case INTEL_PT_PWRE: 2746 case INTEL_PT_PWRX: 2747 case INTEL_PT_BBP: 2748 case INTEL_PT_BIP: 2749 case INTEL_PT_BEP: 2750 case INTEL_PT_BEP_IP: 2751 default: 2752 break; 2753 } 2754 } 2755 } 2756 2757 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 2758 { 2759 int err; 2760 2761 decoder->set_fup_tx_flags = false; 2762 decoder->set_fup_ptw = false; 2763 decoder->set_fup_mwait = false; 2764 decoder->set_fup_pwre = false; 2765 decoder->set_fup_exstop = false; 2766 decoder->set_fup_bep = false; 2767 2768 if (!decoder->branch_enable) { 2769 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2770 decoder->overflow = false; 2771 decoder->state.type = 0; /* Do not have a sample */ 2772 return 0; 2773 } 2774 2775 intel_pt_log("Scanning for full IP\n"); 2776 err = intel_pt_walk_to_ip(decoder); 2777 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) 2778 return err; 2779 2780 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 2781 if (decoder->hop) 2782 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2783 else 2784 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2785 decoder->overflow = false; 2786 2787 decoder->state.from_ip = 0; 2788 decoder->state.to_ip = decoder->ip; 2789 intel_pt_log_to("Setting IP", decoder->ip); 2790 2791 return 0; 2792 } 2793 2794 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 2795 { 2796 const unsigned char *end = decoder->buf + decoder->len; 2797 size_t i; 2798 2799 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 2800 if (i > decoder->len) 2801 continue; 2802 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 2803 return i; 2804 } 2805 return 0; 2806 } 2807 2808 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 2809 { 2810 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 2811 const char *psb = INTEL_PT_PSB_STR; 2812 2813 if (rest_psb > decoder->len || 2814 memcmp(decoder->buf, psb + part_psb, rest_psb)) 2815 return 0; 2816 2817 return rest_psb; 2818 } 2819 2820 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 2821 int part_psb) 2822 { 2823 int rest_psb, ret; 2824 2825 decoder->pos += decoder->len; 2826 decoder->len = 0; 2827 2828 ret = intel_pt_get_next_data(decoder, false); 2829 if (ret) 2830 return ret; 2831 2832 rest_psb = intel_pt_rest_psb(decoder, part_psb); 2833 if (!rest_psb) 2834 return 0; 2835 2836 decoder->pos -= part_psb; 2837 decoder->next_buf = decoder->buf + rest_psb; 2838 decoder->next_len = decoder->len - rest_psb; 2839 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 2840 decoder->buf = decoder->temp_buf; 2841 decoder->len = INTEL_PT_PSB_LEN; 2842 2843 return 0; 2844 } 2845 2846 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 2847 { 2848 unsigned char *next; 2849 int ret; 2850 2851 intel_pt_log("Scanning for PSB\n"); 2852 while (1) { 2853 if (!decoder->len) { 2854 ret = intel_pt_get_next_data(decoder, false); 2855 if (ret) 2856 return ret; 2857 } 2858 2859 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 2860 INTEL_PT_PSB_LEN); 2861 if (!next) { 2862 int part_psb; 2863 2864 part_psb = intel_pt_part_psb(decoder); 2865 if (part_psb) { 2866 ret = intel_pt_get_split_psb(decoder, part_psb); 2867 if (ret) 2868 return ret; 2869 } else { 2870 decoder->pos += decoder->len; 2871 decoder->len = 0; 2872 } 2873 continue; 2874 } 2875 2876 decoder->pkt_step = next - decoder->buf; 2877 return intel_pt_get_next_packet(decoder); 2878 } 2879 } 2880 2881 static int intel_pt_sync(struct intel_pt_decoder *decoder) 2882 { 2883 int err; 2884 2885 decoder->pge = false; 2886 decoder->continuous_period = false; 2887 decoder->have_last_ip = false; 2888 decoder->last_ip = 0; 2889 decoder->psb_ip = 0; 2890 decoder->ip = 0; 2891 intel_pt_clear_stack(&decoder->stack); 2892 2893 err = intel_pt_scan_for_psb(decoder); 2894 if (err) 2895 return err; 2896 2897 decoder->have_last_ip = true; 2898 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 2899 2900 err = intel_pt_walk_psb(decoder); 2901 if (err) 2902 return err; 2903 2904 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ 2905 decoder->state.from_ip = decoder->psb_ip; 2906 decoder->state.to_ip = 0; 2907 2908 if (decoder->ip) { 2909 /* 2910 * In hop mode, resample to get the PSB FUP ip as an 2911 * "instruction" sample. 2912 */ 2913 if (decoder->hop) 2914 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2915 else 2916 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2917 } 2918 2919 return 0; 2920 } 2921 2922 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 2923 { 2924 uint64_t est = decoder->sample_insn_cnt << 1; 2925 2926 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 2927 goto out; 2928 2929 est *= decoder->max_non_turbo_ratio; 2930 est /= decoder->cbr; 2931 out: 2932 return decoder->sample_timestamp + est; 2933 } 2934 2935 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 2936 { 2937 int err; 2938 2939 do { 2940 decoder->state.type = INTEL_PT_BRANCH; 2941 decoder->state.flags = 0; 2942 2943 switch (decoder->pkt_state) { 2944 case INTEL_PT_STATE_NO_PSB: 2945 err = intel_pt_sync(decoder); 2946 break; 2947 case INTEL_PT_STATE_NO_IP: 2948 decoder->have_last_ip = false; 2949 decoder->last_ip = 0; 2950 decoder->ip = 0; 2951 __fallthrough; 2952 case INTEL_PT_STATE_ERR_RESYNC: 2953 err = intel_pt_sync_ip(decoder); 2954 break; 2955 case INTEL_PT_STATE_IN_SYNC: 2956 err = intel_pt_walk_trace(decoder); 2957 break; 2958 case INTEL_PT_STATE_TNT: 2959 case INTEL_PT_STATE_TNT_CONT: 2960 err = intel_pt_walk_tnt(decoder); 2961 if (err == -EAGAIN) 2962 err = intel_pt_walk_trace(decoder); 2963 break; 2964 case INTEL_PT_STATE_TIP: 2965 case INTEL_PT_STATE_TIP_PGD: 2966 err = intel_pt_walk_tip(decoder); 2967 break; 2968 case INTEL_PT_STATE_FUP: 2969 err = intel_pt_walk_fup(decoder); 2970 if (err == -EAGAIN) 2971 err = intel_pt_walk_fup_tip(decoder); 2972 break; 2973 case INTEL_PT_STATE_FUP_NO_TIP: 2974 err = intel_pt_walk_fup(decoder); 2975 if (err == -EAGAIN) 2976 err = intel_pt_walk_trace(decoder); 2977 break; 2978 case INTEL_PT_STATE_FUP_IN_PSB: 2979 err = intel_pt_fup_in_psb(decoder); 2980 break; 2981 case INTEL_PT_STATE_RESAMPLE: 2982 err = intel_pt_resample(decoder); 2983 break; 2984 default: 2985 err = intel_pt_bug(decoder); 2986 break; 2987 } 2988 } while (err == -ENOLINK); 2989 2990 if (err) { 2991 decoder->state.err = intel_pt_ext_err(err); 2992 decoder->state.from_ip = decoder->ip; 2993 intel_pt_update_sample_time(decoder); 2994 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 2995 intel_pt_set_nr(decoder); 2996 } else { 2997 decoder->state.err = 0; 2998 if (decoder->cbr != decoder->cbr_seen) { 2999 decoder->cbr_seen = decoder->cbr; 3000 if (!decoder->state.type) { 3001 decoder->state.from_ip = decoder->ip; 3002 decoder->state.to_ip = 0; 3003 } 3004 decoder->state.type |= INTEL_PT_CBR_CHG; 3005 decoder->state.cbr_payload = decoder->cbr_payload; 3006 decoder->state.cbr = decoder->cbr; 3007 } 3008 if (intel_pt_sample_time(decoder->pkt_state)) { 3009 intel_pt_update_sample_time(decoder); 3010 if (decoder->sample_cyc) { 3011 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3012 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3013 decoder->sample_cyc = false; 3014 } 3015 } 3016 /* 3017 * When using only TSC/MTC to compute cycles, IPC can be 3018 * sampled as soon as the cycle count changes. 3019 */ 3020 if (!decoder->have_cyc) 3021 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3022 } 3023 3024 /* Let PSB event always have TSC timestamp */ 3025 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) 3026 decoder->sample_timestamp = decoder->tsc_timestamp; 3027 3028 decoder->state.from_nr = decoder->nr; 3029 decoder->state.to_nr = decoder->next_nr; 3030 decoder->nr = decoder->next_nr; 3031 3032 decoder->state.timestamp = decoder->sample_timestamp; 3033 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 3034 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 3035 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 3036 3037 return &decoder->state; 3038 } 3039 3040 /** 3041 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 3042 * @buf: pointer to buffer pointer 3043 * @len: size of buffer 3044 * 3045 * Updates the buffer pointer to point to the start of the next PSB packet if 3046 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 3047 * @len is adjusted accordingly. 3048 * 3049 * Return: %true if a PSB packet is found, %false otherwise. 3050 */ 3051 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 3052 { 3053 unsigned char *next; 3054 3055 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3056 if (next) { 3057 *len -= next - *buf; 3058 *buf = next; 3059 return true; 3060 } 3061 return false; 3062 } 3063 3064 /** 3065 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 3066 * packet. 3067 * @buf: pointer to buffer pointer 3068 * @len: size of buffer 3069 * 3070 * Updates the buffer pointer to point to the start of the following PSB packet 3071 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 3072 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 3073 * 3074 * Return: %true if a PSB packet is found, %false otherwise. 3075 */ 3076 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 3077 { 3078 unsigned char *next; 3079 3080 if (!*len) 3081 return false; 3082 3083 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3084 if (next) { 3085 *len -= next - *buf; 3086 *buf = next; 3087 return true; 3088 } 3089 return false; 3090 } 3091 3092 /** 3093 * intel_pt_last_psb - find the last PSB packet in a buffer. 3094 * @buf: buffer 3095 * @len: size of buffer 3096 * 3097 * This function finds the last PSB in a buffer. 3098 * 3099 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 3100 */ 3101 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 3102 { 3103 const char *n = INTEL_PT_PSB_STR; 3104 unsigned char *p; 3105 size_t k; 3106 3107 if (len < INTEL_PT_PSB_LEN) 3108 return NULL; 3109 3110 k = len - INTEL_PT_PSB_LEN + 1; 3111 while (1) { 3112 p = memrchr(buf, n[0], k); 3113 if (!p) 3114 return NULL; 3115 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 3116 return p; 3117 k = p - buf; 3118 if (!k) 3119 return NULL; 3120 } 3121 } 3122 3123 /** 3124 * intel_pt_next_tsc - find and return next TSC. 3125 * @buf: buffer 3126 * @len: size of buffer 3127 * @tsc: TSC value returned 3128 * @rem: returns remaining size when TSC is found 3129 * 3130 * Find a TSC packet in @buf and return the TSC value. This function assumes 3131 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 3132 * PSBEND packet is found. 3133 * 3134 * Return: %true if TSC is found, false otherwise. 3135 */ 3136 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 3137 size_t *rem) 3138 { 3139 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 3140 struct intel_pt_pkt packet; 3141 int ret; 3142 3143 while (len) { 3144 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 3145 if (ret <= 0) 3146 return false; 3147 if (packet.type == INTEL_PT_TSC) { 3148 *tsc = packet.payload; 3149 *rem = len; 3150 return true; 3151 } 3152 if (packet.type == INTEL_PT_PSBEND) 3153 return false; 3154 buf += ret; 3155 len -= ret; 3156 } 3157 return false; 3158 } 3159 3160 /** 3161 * intel_pt_tsc_cmp - compare 7-byte TSCs. 3162 * @tsc1: first TSC to compare 3163 * @tsc2: second TSC to compare 3164 * 3165 * This function compares 7-byte TSC values allowing for the possibility that 3166 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 3167 * around so for that purpose this function assumes the absolute difference is 3168 * less than half the maximum difference. 3169 * 3170 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 3171 * after @tsc2. 3172 */ 3173 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 3174 { 3175 const uint64_t halfway = (1ULL << 55); 3176 3177 if (tsc1 == tsc2) 3178 return 0; 3179 3180 if (tsc1 < tsc2) { 3181 if (tsc2 - tsc1 < halfway) 3182 return -1; 3183 else 3184 return 1; 3185 } else { 3186 if (tsc1 - tsc2 < halfway) 3187 return 1; 3188 else 3189 return -1; 3190 } 3191 } 3192 3193 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 3194 3195 /** 3196 * adj_for_padding - adjust overlap to account for padding. 3197 * @buf_b: second buffer 3198 * @buf_a: first buffer 3199 * @len_a: size of first buffer 3200 * 3201 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 3202 * accordingly. 3203 * 3204 * Return: A pointer into @buf_b from where non-overlapped data starts 3205 */ 3206 static unsigned char *adj_for_padding(unsigned char *buf_b, 3207 unsigned char *buf_a, size_t len_a) 3208 { 3209 unsigned char *p = buf_b - MAX_PADDING; 3210 unsigned char *q = buf_a + len_a - MAX_PADDING; 3211 int i; 3212 3213 for (i = MAX_PADDING; i; i--, p++, q++) { 3214 if (*p != *q) 3215 break; 3216 } 3217 3218 return p; 3219 } 3220 3221 /** 3222 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 3223 * using TSC. 3224 * @buf_a: first buffer 3225 * @len_a: size of first buffer 3226 * @buf_b: second buffer 3227 * @len_b: size of second buffer 3228 * @consecutive: returns true if there is data in buf_b that is consecutive 3229 * to buf_a 3230 * 3231 * If the trace contains TSC we can look at the last TSC of @buf_a and the 3232 * first TSC of @buf_b in order to determine if the buffers overlap, and then 3233 * walk forward in @buf_b until a later TSC is found. A precondition is that 3234 * @buf_a and @buf_b are positioned at a PSB. 3235 * 3236 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3237 * @buf_b + @len_b if there is no non-overlapped data. 3238 */ 3239 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 3240 size_t len_a, 3241 unsigned char *buf_b, 3242 size_t len_b, bool *consecutive) 3243 { 3244 uint64_t tsc_a, tsc_b; 3245 unsigned char *p; 3246 size_t len, rem_a, rem_b; 3247 3248 p = intel_pt_last_psb(buf_a, len_a); 3249 if (!p) 3250 return buf_b; /* No PSB in buf_a => no overlap */ 3251 3252 len = len_a - (p - buf_a); 3253 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 3254 /* The last PSB+ in buf_a is incomplete, so go back one more */ 3255 len_a -= len; 3256 p = intel_pt_last_psb(buf_a, len_a); 3257 if (!p) 3258 return buf_b; /* No full PSB+ => assume no overlap */ 3259 len = len_a - (p - buf_a); 3260 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 3261 return buf_b; /* No TSC in buf_a => assume no overlap */ 3262 } 3263 3264 while (1) { 3265 /* Ignore PSB+ with no TSC */ 3266 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 3267 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 3268 3269 /* Same TSC, so buffers are consecutive */ 3270 if (!cmp && rem_b >= rem_a) { 3271 unsigned char *start; 3272 3273 *consecutive = true; 3274 start = buf_b + len_b - (rem_b - rem_a); 3275 return adj_for_padding(start, buf_a, len_a); 3276 } 3277 if (cmp < 0) 3278 return buf_b; /* tsc_a < tsc_b => no overlap */ 3279 } 3280 3281 if (!intel_pt_step_psb(&buf_b, &len_b)) 3282 return buf_b + len_b; /* No PSB in buf_b => no data */ 3283 } 3284 } 3285 3286 /** 3287 * intel_pt_find_overlap - determine start of non-overlapped trace data. 3288 * @buf_a: first buffer 3289 * @len_a: size of first buffer 3290 * @buf_b: second buffer 3291 * @len_b: size of second buffer 3292 * @have_tsc: can use TSC packets to detect overlap 3293 * @consecutive: returns true if there is data in buf_b that is consecutive 3294 * to buf_a 3295 * 3296 * When trace samples or snapshots are recorded there is the possibility that 3297 * the data overlaps. Note that, for the purposes of decoding, data is only 3298 * useful if it begins with a PSB packet. 3299 * 3300 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3301 * @buf_b + @len_b if there is no non-overlapped data. 3302 */ 3303 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 3304 unsigned char *buf_b, size_t len_b, 3305 bool have_tsc, bool *consecutive) 3306 { 3307 unsigned char *found; 3308 3309 /* Buffer 'b' must start at PSB so throw away everything before that */ 3310 if (!intel_pt_next_psb(&buf_b, &len_b)) 3311 return buf_b + len_b; /* No PSB */ 3312 3313 if (!intel_pt_next_psb(&buf_a, &len_a)) 3314 return buf_b; /* No overlap */ 3315 3316 if (have_tsc) { 3317 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 3318 consecutive); 3319 if (found) 3320 return found; 3321 } 3322 3323 /* 3324 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 3325 * we can ignore the first part of buffer 'a'. 3326 */ 3327 while (len_b < len_a) { 3328 if (!intel_pt_step_psb(&buf_a, &len_a)) 3329 return buf_b; /* No overlap */ 3330 } 3331 3332 /* Now len_b >= len_a */ 3333 while (1) { 3334 /* Potential overlap so check the bytes */ 3335 found = memmem(buf_a, len_a, buf_b, len_a); 3336 if (found) { 3337 *consecutive = true; 3338 return adj_for_padding(buf_b + len_a, buf_a, len_a); 3339 } 3340 3341 /* Try again at next PSB in buffer 'a' */ 3342 if (!intel_pt_step_psb(&buf_a, &len_a)) 3343 return buf_b; /* No overlap */ 3344 } 3345 } 3346 3347 /** 3348 * struct fast_forward_data - data used by intel_pt_ff_cb(). 3349 * @timestamp: timestamp to fast forward towards 3350 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 3351 * the fast forward timestamp. 3352 */ 3353 struct fast_forward_data { 3354 uint64_t timestamp; 3355 uint64_t buf_timestamp; 3356 }; 3357 3358 /** 3359 * intel_pt_ff_cb - fast forward lookahead callback. 3360 * @buffer: Intel PT trace buffer 3361 * @data: opaque pointer to fast forward data (struct fast_forward_data) 3362 * 3363 * Determine if @buffer trace is past the fast forward timestamp. 3364 * 3365 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 3366 * timestamp, and 0 otherwise. 3367 */ 3368 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 3369 { 3370 struct fast_forward_data *d = data; 3371 unsigned char *buf; 3372 uint64_t tsc; 3373 size_t rem; 3374 size_t len; 3375 3376 buf = (unsigned char *)buffer->buf; 3377 len = buffer->len; 3378 3379 if (!intel_pt_next_psb(&buf, &len) || 3380 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 3381 return 0; 3382 3383 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 3384 3385 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 3386 tsc, buffer->ref_timestamp); 3387 3388 /* 3389 * If the buffer contains a timestamp earlier that the fast forward 3390 * timestamp, then record it, else stop. 3391 */ 3392 if (tsc < d->timestamp) 3393 d->buf_timestamp = buffer->ref_timestamp; 3394 else 3395 return 1; 3396 3397 return 0; 3398 } 3399 3400 /** 3401 * intel_pt_fast_forward - reposition decoder forwards. 3402 * @decoder: Intel PT decoder 3403 * @timestamp: timestamp to fast forward towards 3404 * 3405 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 3406 * 3407 * Return: 0 on success or negative error code on failure. 3408 */ 3409 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 3410 { 3411 struct fast_forward_data d = { .timestamp = timestamp }; 3412 unsigned char *buf; 3413 size_t len; 3414 int err; 3415 3416 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 3417 3418 /* Find buffer timestamp of buffer to fast forward to */ 3419 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 3420 if (err < 0) 3421 return err; 3422 3423 /* Walk to buffer with same buffer timestamp */ 3424 if (d.buf_timestamp) { 3425 do { 3426 decoder->pos += decoder->len; 3427 decoder->len = 0; 3428 err = intel_pt_get_next_data(decoder, true); 3429 /* -ENOLINK means non-consecutive trace */ 3430 if (err && err != -ENOLINK) 3431 return err; 3432 } while (decoder->buf_timestamp != d.buf_timestamp); 3433 } 3434 3435 if (!decoder->buf) 3436 return 0; 3437 3438 buf = (unsigned char *)decoder->buf; 3439 len = decoder->len; 3440 3441 if (!intel_pt_next_psb(&buf, &len)) 3442 return 0; 3443 3444 /* 3445 * Walk PSBs while the PSB timestamp is less than the fast forward 3446 * timestamp. 3447 */ 3448 do { 3449 uint64_t tsc; 3450 size_t rem; 3451 3452 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 3453 break; 3454 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 3455 /* 3456 * A TSC packet can slip past MTC packets but, after fast 3457 * forward, decoding starts at the TSC timestamp. That means 3458 * the timestamps may not be exactly the same as the timestamps 3459 * that would have been decoded without fast forward. 3460 */ 3461 if (tsc < timestamp) { 3462 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 3463 decoder->pos += decoder->len - len; 3464 decoder->buf = buf; 3465 decoder->len = len; 3466 intel_pt_reposition(decoder); 3467 } else { 3468 break; 3469 } 3470 } while (intel_pt_step_psb(&buf, &len)); 3471 3472 return 0; 3473 } 3474