1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define BITULL(x) (1ULL << (x)) 28 29 /* IA32_RTIT_CTL MSR bits */ 30 #define INTEL_PT_CYC_ENABLE BITULL(1) 31 #define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32 #define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define SEVEN_BYTES 0xffffffffffffffULL 39 40 #define NO_VMCS 0xffffffffffULL 41 42 #define INTEL_PT_RETURN 1 43 44 /* 45 * Default maximum number of loops with no packets consumed i.e. stuck in a 46 * loop. 47 */ 48 #define INTEL_PT_MAX_LOOPS 100000 49 50 struct intel_pt_blk { 51 struct intel_pt_blk *prev; 52 uint64_t ip[INTEL_PT_BLK_SIZE]; 53 }; 54 55 struct intel_pt_stack { 56 struct intel_pt_blk *blk; 57 struct intel_pt_blk *spare; 58 int pos; 59 }; 60 61 enum intel_pt_p_once { 62 INTEL_PT_PRT_ONCE_UNK_VMCS, 63 INTEL_PT_PRT_ONCE_ERANGE, 64 }; 65 66 enum intel_pt_pkt_state { 67 INTEL_PT_STATE_NO_PSB, 68 INTEL_PT_STATE_NO_IP, 69 INTEL_PT_STATE_ERR_RESYNC, 70 INTEL_PT_STATE_IN_SYNC, 71 INTEL_PT_STATE_TNT_CONT, 72 INTEL_PT_STATE_TNT, 73 INTEL_PT_STATE_TIP, 74 INTEL_PT_STATE_TIP_PGD, 75 INTEL_PT_STATE_FUP, 76 INTEL_PT_STATE_FUP_NO_TIP, 77 INTEL_PT_STATE_FUP_IN_PSB, 78 INTEL_PT_STATE_RESAMPLE, 79 INTEL_PT_STATE_VM_TIME_CORRELATION, 80 }; 81 82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 83 { 84 switch (pkt_state) { 85 case INTEL_PT_STATE_NO_PSB: 86 case INTEL_PT_STATE_NO_IP: 87 case INTEL_PT_STATE_ERR_RESYNC: 88 case INTEL_PT_STATE_IN_SYNC: 89 case INTEL_PT_STATE_TNT_CONT: 90 case INTEL_PT_STATE_RESAMPLE: 91 case INTEL_PT_STATE_VM_TIME_CORRELATION: 92 return true; 93 case INTEL_PT_STATE_TNT: 94 case INTEL_PT_STATE_TIP: 95 case INTEL_PT_STATE_TIP_PGD: 96 case INTEL_PT_STATE_FUP: 97 case INTEL_PT_STATE_FUP_NO_TIP: 98 case INTEL_PT_STATE_FUP_IN_PSB: 99 return false; 100 default: 101 return true; 102 }; 103 } 104 105 #ifdef INTEL_PT_STRICT 106 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 107 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 108 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 109 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 110 #else 111 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 112 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 113 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 114 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 115 #endif 116 117 struct intel_pt_decoder { 118 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 119 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 120 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 121 uint64_t max_insn_cnt, void *data); 122 bool (*pgd_ip)(uint64_t ip, void *data); 123 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 124 struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); 125 void *data; 126 struct intel_pt_state state; 127 const unsigned char *buf; 128 size_t len; 129 bool return_compression; 130 bool branch_enable; 131 bool mtc_insn; 132 bool pge; 133 bool have_tma; 134 bool have_cyc; 135 bool fixup_last_mtc; 136 bool have_last_ip; 137 bool in_psb; 138 bool hop; 139 bool leap; 140 bool emulated_ptwrite; 141 bool vm_time_correlation; 142 bool vm_tm_corr_dry_run; 143 bool vm_tm_corr_reliable; 144 bool vm_tm_corr_same_buf; 145 bool vm_tm_corr_continuous; 146 bool nr; 147 bool next_nr; 148 bool iflag; 149 bool next_iflag; 150 enum intel_pt_param_flags flags; 151 uint64_t pos; 152 uint64_t last_ip; 153 uint64_t ip; 154 uint64_t pip_payload; 155 uint64_t timestamp; 156 uint64_t tsc_timestamp; 157 uint64_t ref_timestamp; 158 uint64_t buf_timestamp; 159 uint64_t sample_timestamp; 160 uint64_t ret_addr; 161 uint64_t ctc_timestamp; 162 uint64_t ctc_delta; 163 uint64_t cycle_cnt; 164 uint64_t cyc_ref_timestamp; 165 uint64_t first_timestamp; 166 uint64_t last_reliable_timestamp; 167 uint64_t vmcs; 168 uint64_t print_once; 169 uint64_t last_ctc; 170 uint32_t last_mtc; 171 uint32_t tsc_ctc_ratio_n; 172 uint32_t tsc_ctc_ratio_d; 173 uint32_t tsc_ctc_mult; 174 uint32_t tsc_slip; 175 uint32_t ctc_rem_mask; 176 int mtc_shift; 177 struct intel_pt_stack stack; 178 enum intel_pt_pkt_state pkt_state; 179 enum intel_pt_pkt_ctx pkt_ctx; 180 enum intel_pt_pkt_ctx prev_pkt_ctx; 181 enum intel_pt_blk_type blk_type; 182 int blk_type_pos; 183 struct intel_pt_pkt packet; 184 struct intel_pt_pkt tnt; 185 int pkt_step; 186 int pkt_len; 187 int last_packet_type; 188 unsigned int cbr; 189 unsigned int cbr_seen; 190 unsigned int max_non_turbo_ratio; 191 double max_non_turbo_ratio_fp; 192 double cbr_cyc_to_tsc; 193 double calc_cyc_to_tsc; 194 bool have_calc_cyc_to_tsc; 195 int exec_mode; 196 unsigned int insn_bytes; 197 uint64_t period; 198 enum intel_pt_period_type period_type; 199 uint64_t tot_insn_cnt; 200 uint64_t period_insn_cnt; 201 uint64_t period_mask; 202 uint64_t period_ticks; 203 uint64_t last_masked_timestamp; 204 uint64_t tot_cyc_cnt; 205 uint64_t sample_tot_cyc_cnt; 206 uint64_t base_cyc_cnt; 207 uint64_t cyc_cnt_timestamp; 208 uint64_t ctl; 209 uint64_t cyc_threshold; 210 double tsc_to_cyc; 211 bool continuous_period; 212 bool overflow; 213 bool set_fup_tx_flags; 214 bool set_fup_ptw; 215 bool set_fup_mwait; 216 bool set_fup_pwre; 217 bool set_fup_exstop; 218 bool set_fup_bep; 219 bool set_fup_cfe_ip; 220 bool set_fup_cfe; 221 bool set_fup_mode_exec; 222 bool sample_cyc; 223 unsigned int fup_tx_flags; 224 unsigned int tx_flags; 225 uint64_t fup_ptw_payload; 226 uint64_t fup_mwait_payload; 227 uint64_t fup_pwre_payload; 228 uint64_t cbr_payload; 229 uint64_t timestamp_insn_cnt; 230 uint64_t sample_insn_cnt; 231 uint64_t stuck_ip; 232 struct intel_pt_pkt fup_cfe_pkt; 233 int max_loops; 234 int no_progress; 235 int stuck_ip_prd; 236 int stuck_ip_cnt; 237 uint64_t psb_ip; 238 const unsigned char *next_buf; 239 size_t next_len; 240 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 241 int evd_cnt; 242 struct intel_pt_evd evd[INTEL_PT_MAX_EVDS]; 243 }; 244 245 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 246 { 247 int i; 248 249 for (i = 0; x != 1; i++) 250 x >>= 1; 251 252 return x << i; 253 } 254 255 __printf(1, 2) 256 static void p_log(const char *fmt, ...) 257 { 258 char buf[512]; 259 va_list args; 260 261 va_start(args, fmt); 262 vsnprintf(buf, sizeof(buf), fmt, args); 263 va_end(args); 264 265 fprintf(stderr, "%s\n", buf); 266 intel_pt_log("%s\n", buf); 267 } 268 269 static bool intel_pt_print_once(struct intel_pt_decoder *decoder, 270 enum intel_pt_p_once id) 271 { 272 uint64_t bit = 1ULL << id; 273 274 if (decoder->print_once & bit) 275 return false; 276 decoder->print_once |= bit; 277 return true; 278 } 279 280 static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 281 { 282 if (!(ctl & INTEL_PT_CYC_ENABLE)) 283 return 0; 284 285 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 286 } 287 288 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 289 { 290 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 291 uint64_t period; 292 293 period = intel_pt_lower_power_of_2(decoder->period); 294 decoder->period_mask = ~(period - 1); 295 decoder->period_ticks = period; 296 } 297 } 298 299 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 300 { 301 if (!d) 302 return 0; 303 return (t / d) * n + ((t % d) * n) / d; 304 } 305 306 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 307 { 308 struct intel_pt_decoder *decoder; 309 310 if (!params->get_trace || !params->walk_insn) 311 return NULL; 312 313 decoder = zalloc(sizeof(struct intel_pt_decoder)); 314 if (!decoder) 315 return NULL; 316 317 decoder->get_trace = params->get_trace; 318 decoder->walk_insn = params->walk_insn; 319 decoder->pgd_ip = params->pgd_ip; 320 decoder->lookahead = params->lookahead; 321 decoder->findnew_vmcs_info = params->findnew_vmcs_info; 322 decoder->data = params->data; 323 decoder->return_compression = params->return_compression; 324 decoder->branch_enable = params->branch_enable; 325 decoder->hop = params->quick >= 1; 326 decoder->leap = params->quick >= 2; 327 decoder->vm_time_correlation = params->vm_time_correlation; 328 decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; 329 decoder->first_timestamp = params->first_timestamp; 330 decoder->last_reliable_timestamp = params->first_timestamp; 331 decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS; 332 333 decoder->flags = params->flags; 334 335 decoder->ctl = params->ctl; 336 decoder->period = params->period; 337 decoder->period_type = params->period_type; 338 339 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 340 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 341 342 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 343 344 intel_pt_setup_period(decoder); 345 346 decoder->mtc_shift = params->mtc_period; 347 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 348 349 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 350 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 351 352 if (!decoder->tsc_ctc_ratio_n) 353 decoder->tsc_ctc_ratio_d = 0; 354 355 if (decoder->tsc_ctc_ratio_d) { 356 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 357 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 358 decoder->tsc_ctc_ratio_d; 359 } 360 361 /* 362 * A TSC packet can slip past MTC packets so that the timestamp appears 363 * to go backwards. One estimate is that can be up to about 40 CPU 364 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 365 * slippage an order of magnitude more to be on the safe side. 366 */ 367 decoder->tsc_slip = 0x10000; 368 369 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 370 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 371 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 372 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 373 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 374 375 if (decoder->hop) 376 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 377 378 return decoder; 379 } 380 381 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, 382 uint64_t first_timestamp) 383 { 384 decoder->first_timestamp = first_timestamp; 385 } 386 387 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 388 { 389 struct intel_pt_blk *blk = stack->blk; 390 391 stack->blk = blk->prev; 392 if (!stack->spare) 393 stack->spare = blk; 394 else 395 free(blk); 396 } 397 398 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 399 { 400 if (!stack->pos) { 401 if (!stack->blk) 402 return 0; 403 intel_pt_pop_blk(stack); 404 if (!stack->blk) 405 return 0; 406 stack->pos = INTEL_PT_BLK_SIZE; 407 } 408 return stack->blk->ip[--stack->pos]; 409 } 410 411 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 412 { 413 struct intel_pt_blk *blk; 414 415 if (stack->spare) { 416 blk = stack->spare; 417 stack->spare = NULL; 418 } else { 419 blk = malloc(sizeof(struct intel_pt_blk)); 420 if (!blk) 421 return -ENOMEM; 422 } 423 424 blk->prev = stack->blk; 425 stack->blk = blk; 426 stack->pos = 0; 427 return 0; 428 } 429 430 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 431 { 432 int err; 433 434 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 435 err = intel_pt_alloc_blk(stack); 436 if (err) 437 return err; 438 } 439 440 stack->blk->ip[stack->pos++] = ip; 441 return 0; 442 } 443 444 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 445 { 446 while (stack->blk) 447 intel_pt_pop_blk(stack); 448 stack->pos = 0; 449 } 450 451 static void intel_pt_free_stack(struct intel_pt_stack *stack) 452 { 453 intel_pt_clear_stack(stack); 454 zfree(&stack->blk); 455 zfree(&stack->spare); 456 } 457 458 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 459 { 460 intel_pt_free_stack(&decoder->stack); 461 free(decoder); 462 } 463 464 static int intel_pt_ext_err(int code) 465 { 466 switch (code) { 467 case -ENOMEM: 468 return INTEL_PT_ERR_NOMEM; 469 case -ENOSYS: 470 return INTEL_PT_ERR_INTERN; 471 case -EBADMSG: 472 return INTEL_PT_ERR_BADPKT; 473 case -ENODATA: 474 return INTEL_PT_ERR_NODATA; 475 case -EILSEQ: 476 return INTEL_PT_ERR_NOINSN; 477 case -ENOENT: 478 return INTEL_PT_ERR_MISMAT; 479 case -EOVERFLOW: 480 return INTEL_PT_ERR_OVR; 481 case -ENOSPC: 482 return INTEL_PT_ERR_LOST; 483 case -ELOOP: 484 return INTEL_PT_ERR_NELOOP; 485 case -ECONNRESET: 486 return INTEL_PT_ERR_EPTW; 487 default: 488 return INTEL_PT_ERR_UNK; 489 } 490 } 491 492 static const char *intel_pt_err_msgs[] = { 493 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 494 [INTEL_PT_ERR_INTERN] = "Internal error", 495 [INTEL_PT_ERR_BADPKT] = "Bad packet", 496 [INTEL_PT_ERR_NODATA] = "No more data", 497 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 498 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 499 [INTEL_PT_ERR_OVR] = "Overflow packet", 500 [INTEL_PT_ERR_LOST] = "Lost trace data", 501 [INTEL_PT_ERR_UNK] = "Unknown error!", 502 [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)", 503 [INTEL_PT_ERR_EPTW] = "Broken emulated ptwrite", 504 }; 505 506 int intel_pt__strerror(int code, char *buf, size_t buflen) 507 { 508 if (code < 1 || code >= INTEL_PT_ERR_MAX) 509 code = INTEL_PT_ERR_UNK; 510 strlcpy(buf, intel_pt_err_msgs[code], buflen); 511 return 0; 512 } 513 514 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 515 uint64_t last_ip) 516 { 517 uint64_t ip; 518 519 switch (packet->count) { 520 case 1: 521 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 522 packet->payload; 523 break; 524 case 2: 525 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 526 packet->payload; 527 break; 528 case 3: 529 ip = packet->payload; 530 /* Sign-extend 6-byte ip */ 531 if (ip & (uint64_t)0x800000000000ULL) 532 ip |= (uint64_t)0xffff000000000000ULL; 533 break; 534 case 4: 535 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 536 packet->payload; 537 break; 538 case 6: 539 ip = packet->payload; 540 break; 541 default: 542 return 0; 543 } 544 545 return ip; 546 } 547 548 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 549 { 550 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 551 decoder->have_last_ip = true; 552 } 553 554 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 555 { 556 intel_pt_set_last_ip(decoder); 557 decoder->ip = decoder->last_ip; 558 } 559 560 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 561 { 562 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 563 decoder->buf); 564 } 565 566 static int intel_pt_bug(struct intel_pt_decoder *decoder) 567 { 568 intel_pt_log("ERROR: Internal error\n"); 569 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 570 return -ENOSYS; 571 } 572 573 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 574 { 575 decoder->tx_flags = 0; 576 } 577 578 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 579 { 580 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 581 } 582 583 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder) 584 { 585 decoder->pip_payload = decoder->packet.payload; 586 } 587 588 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder) 589 { 590 decoder->next_nr = decoder->pip_payload & 1; 591 } 592 593 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder) 594 { 595 decoder->nr = decoder->pip_payload & 1; 596 decoder->next_nr = decoder->nr; 597 } 598 599 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder) 600 { 601 intel_pt_update_pip(decoder); 602 intel_pt_set_nr(decoder); 603 } 604 605 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 606 { 607 intel_pt_clear_tx_flags(decoder); 608 decoder->have_tma = false; 609 decoder->pkt_len = 1; 610 decoder->pkt_step = 1; 611 intel_pt_decoder_log_packet(decoder); 612 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 613 intel_pt_log("ERROR: Bad packet\n"); 614 decoder->pkt_state = INTEL_PT_STATE_ERR1; 615 } 616 return -EBADMSG; 617 } 618 619 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 620 { 621 decoder->sample_timestamp = decoder->timestamp; 622 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 623 decoder->state.cycles = decoder->tot_cyc_cnt; 624 } 625 626 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 627 { 628 decoder->ip = 0; 629 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 630 decoder->timestamp = 0; 631 decoder->have_tma = false; 632 } 633 634 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 635 { 636 struct intel_pt_buffer buffer = { .buf = 0, }; 637 int ret; 638 639 decoder->pkt_step = 0; 640 641 intel_pt_log("Getting more data\n"); 642 ret = decoder->get_trace(&buffer, decoder->data); 643 if (ret) 644 return ret; 645 decoder->buf = buffer.buf; 646 decoder->len = buffer.len; 647 if (!decoder->len) { 648 intel_pt_log("No more data\n"); 649 return -ENODATA; 650 } 651 decoder->buf_timestamp = buffer.ref_timestamp; 652 if (!buffer.consecutive || reposition) { 653 intel_pt_reposition(decoder); 654 decoder->ref_timestamp = buffer.ref_timestamp; 655 decoder->state.trace_nr = buffer.trace_nr; 656 decoder->vm_tm_corr_same_buf = false; 657 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 658 decoder->ref_timestamp); 659 return -ENOLINK; 660 } 661 662 return 0; 663 } 664 665 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 666 bool reposition) 667 { 668 if (!decoder->next_buf) 669 return intel_pt_get_data(decoder, reposition); 670 671 decoder->buf = decoder->next_buf; 672 decoder->len = decoder->next_len; 673 decoder->next_buf = 0; 674 decoder->next_len = 0; 675 return 0; 676 } 677 678 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 679 { 680 unsigned char *buf = decoder->temp_buf; 681 size_t old_len, len, n; 682 int ret; 683 684 old_len = decoder->len; 685 len = decoder->len; 686 memcpy(buf, decoder->buf, len); 687 688 ret = intel_pt_get_data(decoder, false); 689 if (ret) { 690 decoder->pos += old_len; 691 return ret < 0 ? ret : -EINVAL; 692 } 693 694 n = INTEL_PT_PKT_MAX_SZ - len; 695 if (n > decoder->len) 696 n = decoder->len; 697 memcpy(buf + len, decoder->buf, n); 698 len += n; 699 700 decoder->prev_pkt_ctx = decoder->pkt_ctx; 701 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 702 if (ret < (int)old_len) { 703 decoder->next_buf = decoder->buf; 704 decoder->next_len = decoder->len; 705 decoder->buf = buf; 706 decoder->len = old_len; 707 return intel_pt_bad_packet(decoder); 708 } 709 710 decoder->next_buf = decoder->buf + (ret - old_len); 711 decoder->next_len = decoder->len - (ret - old_len); 712 713 decoder->buf = buf; 714 decoder->len = ret; 715 716 return ret; 717 } 718 719 struct intel_pt_pkt_info { 720 struct intel_pt_decoder *decoder; 721 struct intel_pt_pkt packet; 722 uint64_t pos; 723 int pkt_len; 724 int last_packet_type; 725 void *data; 726 }; 727 728 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 729 730 /* Lookahead packets in current buffer */ 731 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 732 intel_pt_pkt_cb_t cb, void *data) 733 { 734 struct intel_pt_pkt_info pkt_info; 735 const unsigned char *buf = decoder->buf; 736 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 737 size_t len = decoder->len; 738 int ret; 739 740 pkt_info.decoder = decoder; 741 pkt_info.pos = decoder->pos; 742 pkt_info.pkt_len = decoder->pkt_step; 743 pkt_info.last_packet_type = decoder->last_packet_type; 744 pkt_info.data = data; 745 746 while (1) { 747 do { 748 pkt_info.pos += pkt_info.pkt_len; 749 buf += pkt_info.pkt_len; 750 len -= pkt_info.pkt_len; 751 752 if (!len) 753 return INTEL_PT_NEED_MORE_BYTES; 754 755 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 756 &pkt_ctx); 757 if (!ret) 758 return INTEL_PT_NEED_MORE_BYTES; 759 if (ret < 0) 760 return ret; 761 762 pkt_info.pkt_len = ret; 763 } while (pkt_info.packet.type == INTEL_PT_PAD); 764 765 ret = cb(&pkt_info); 766 if (ret) 767 return 0; 768 769 pkt_info.last_packet_type = pkt_info.packet.type; 770 } 771 } 772 773 struct intel_pt_calc_cyc_to_tsc_info { 774 uint64_t cycle_cnt; 775 unsigned int cbr; 776 uint32_t last_mtc; 777 uint64_t ctc_timestamp; 778 uint64_t ctc_delta; 779 uint64_t tsc_timestamp; 780 uint64_t timestamp; 781 bool have_tma; 782 bool fixup_last_mtc; 783 bool from_mtc; 784 double cbr_cyc_to_tsc; 785 }; 786 787 /* 788 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 789 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 790 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 791 * packet by copying the missing bits from the current MTC assuming the least 792 * difference between the two, and that the current MTC comes after last_mtc. 793 */ 794 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 795 uint32_t *last_mtc) 796 { 797 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 798 uint32_t mask = ~(first_missing_bit - 1); 799 800 *last_mtc |= mtc & mask; 801 if (*last_mtc >= mtc) { 802 *last_mtc -= first_missing_bit; 803 *last_mtc &= 0xff; 804 } 805 } 806 807 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 808 { 809 struct intel_pt_decoder *decoder = pkt_info->decoder; 810 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 811 uint64_t timestamp; 812 double cyc_to_tsc; 813 unsigned int cbr; 814 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 815 816 switch (pkt_info->packet.type) { 817 case INTEL_PT_TNT: 818 case INTEL_PT_TIP_PGE: 819 case INTEL_PT_TIP: 820 case INTEL_PT_FUP: 821 case INTEL_PT_PSB: 822 case INTEL_PT_PIP: 823 case INTEL_PT_MODE_EXEC: 824 case INTEL_PT_MODE_TSX: 825 case INTEL_PT_PSBEND: 826 case INTEL_PT_PAD: 827 case INTEL_PT_VMCS: 828 case INTEL_PT_MNT: 829 case INTEL_PT_PTWRITE: 830 case INTEL_PT_PTWRITE_IP: 831 case INTEL_PT_BBP: 832 case INTEL_PT_BIP: 833 case INTEL_PT_BEP: 834 case INTEL_PT_BEP_IP: 835 case INTEL_PT_CFE: 836 case INTEL_PT_CFE_IP: 837 case INTEL_PT_EVD: 838 return 0; 839 840 case INTEL_PT_MTC: 841 if (!data->have_tma) 842 return 0; 843 844 mtc = pkt_info->packet.payload; 845 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 846 data->fixup_last_mtc = false; 847 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 848 &data->last_mtc); 849 } 850 if (mtc > data->last_mtc) 851 mtc_delta = mtc - data->last_mtc; 852 else 853 mtc_delta = mtc + 256 - data->last_mtc; 854 data->ctc_delta += mtc_delta << decoder->mtc_shift; 855 data->last_mtc = mtc; 856 857 if (decoder->tsc_ctc_mult) { 858 timestamp = data->ctc_timestamp + 859 data->ctc_delta * decoder->tsc_ctc_mult; 860 } else { 861 timestamp = data->ctc_timestamp + 862 multdiv(data->ctc_delta, 863 decoder->tsc_ctc_ratio_n, 864 decoder->tsc_ctc_ratio_d); 865 } 866 867 if (timestamp < data->timestamp) 868 return 1; 869 870 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 871 data->timestamp = timestamp; 872 return 0; 873 } 874 875 break; 876 877 case INTEL_PT_TSC: 878 /* 879 * For now, do not support using TSC packets - refer 880 * intel_pt_calc_cyc_to_tsc(). 881 */ 882 if (data->from_mtc) 883 return 1; 884 timestamp = pkt_info->packet.payload | 885 (data->timestamp & (0xffULL << 56)); 886 if (data->from_mtc && timestamp < data->timestamp && 887 data->timestamp - timestamp < decoder->tsc_slip) 888 return 1; 889 if (timestamp < data->timestamp) 890 timestamp += (1ULL << 56); 891 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 892 if (data->from_mtc) 893 return 1; 894 data->tsc_timestamp = timestamp; 895 data->timestamp = timestamp; 896 return 0; 897 } 898 break; 899 900 case INTEL_PT_TMA: 901 if (data->from_mtc) 902 return 1; 903 904 if (!decoder->tsc_ctc_ratio_d) 905 return 0; 906 907 ctc = pkt_info->packet.payload; 908 fc = pkt_info->packet.count; 909 ctc_rem = ctc & decoder->ctc_rem_mask; 910 911 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 912 913 data->ctc_timestamp = data->tsc_timestamp - fc; 914 if (decoder->tsc_ctc_mult) { 915 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 916 } else { 917 data->ctc_timestamp -= 918 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 919 decoder->tsc_ctc_ratio_d); 920 } 921 922 data->ctc_delta = 0; 923 data->have_tma = true; 924 data->fixup_last_mtc = true; 925 926 return 0; 927 928 case INTEL_PT_CYC: 929 data->cycle_cnt += pkt_info->packet.payload; 930 return 0; 931 932 case INTEL_PT_CBR: 933 cbr = pkt_info->packet.payload; 934 if (data->cbr && data->cbr != cbr) 935 return 1; 936 data->cbr = cbr; 937 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 938 return 0; 939 940 case INTEL_PT_TIP_PGD: 941 case INTEL_PT_TRACESTOP: 942 case INTEL_PT_EXSTOP: 943 case INTEL_PT_EXSTOP_IP: 944 case INTEL_PT_MWAIT: 945 case INTEL_PT_PWRE: 946 case INTEL_PT_PWRX: 947 case INTEL_PT_OVF: 948 case INTEL_PT_BAD: /* Does not happen */ 949 default: 950 return 1; 951 } 952 953 if (!data->cbr && decoder->cbr) { 954 data->cbr = decoder->cbr; 955 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 956 } 957 958 if (!data->cycle_cnt) 959 return 1; 960 961 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 962 963 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 964 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 965 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 966 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 967 return 1; 968 } 969 970 decoder->calc_cyc_to_tsc = cyc_to_tsc; 971 decoder->have_calc_cyc_to_tsc = true; 972 973 if (data->cbr) { 974 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 975 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 976 } else { 977 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 978 cyc_to_tsc, pkt_info->pos); 979 } 980 981 return 1; 982 } 983 984 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 985 bool from_mtc) 986 { 987 struct intel_pt_calc_cyc_to_tsc_info data = { 988 .cycle_cnt = 0, 989 .cbr = 0, 990 .last_mtc = decoder->last_mtc, 991 .ctc_timestamp = decoder->ctc_timestamp, 992 .ctc_delta = decoder->ctc_delta, 993 .tsc_timestamp = decoder->tsc_timestamp, 994 .timestamp = decoder->timestamp, 995 .have_tma = decoder->have_tma, 996 .fixup_last_mtc = decoder->fixup_last_mtc, 997 .from_mtc = from_mtc, 998 .cbr_cyc_to_tsc = 0, 999 }; 1000 1001 /* 1002 * For now, do not support using TSC packets for at least the reasons: 1003 * 1) timing might have stopped 1004 * 2) TSC packets within PSB+ can slip against CYC packets 1005 */ 1006 if (!from_mtc) 1007 return; 1008 1009 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 1010 } 1011 1012 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 1013 { 1014 int ret; 1015 1016 decoder->last_packet_type = decoder->packet.type; 1017 1018 do { 1019 decoder->pos += decoder->pkt_step; 1020 decoder->buf += decoder->pkt_step; 1021 decoder->len -= decoder->pkt_step; 1022 1023 if (!decoder->len) { 1024 ret = intel_pt_get_next_data(decoder, false); 1025 if (ret) 1026 return ret; 1027 } 1028 1029 decoder->prev_pkt_ctx = decoder->pkt_ctx; 1030 ret = intel_pt_get_packet(decoder->buf, decoder->len, 1031 &decoder->packet, &decoder->pkt_ctx); 1032 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 1033 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 1034 ret = intel_pt_get_split_packet(decoder); 1035 if (ret < 0) 1036 return ret; 1037 } 1038 if (ret <= 0) 1039 return intel_pt_bad_packet(decoder); 1040 1041 decoder->pkt_len = ret; 1042 decoder->pkt_step = ret; 1043 intel_pt_decoder_log_packet(decoder); 1044 } while (decoder->packet.type == INTEL_PT_PAD); 1045 1046 return 0; 1047 } 1048 1049 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 1050 { 1051 uint64_t timestamp, masked_timestamp; 1052 1053 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1054 masked_timestamp = timestamp & decoder->period_mask; 1055 if (decoder->continuous_period) { 1056 if (masked_timestamp > decoder->last_masked_timestamp) 1057 return 1; 1058 } else { 1059 timestamp += 1; 1060 masked_timestamp = timestamp & decoder->period_mask; 1061 if (masked_timestamp > decoder->last_masked_timestamp) { 1062 decoder->last_masked_timestamp = masked_timestamp; 1063 decoder->continuous_period = true; 1064 } 1065 } 1066 1067 if (masked_timestamp < decoder->last_masked_timestamp) 1068 return decoder->period_ticks; 1069 1070 return decoder->period_ticks - (timestamp - masked_timestamp); 1071 } 1072 1073 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 1074 { 1075 switch (decoder->period_type) { 1076 case INTEL_PT_PERIOD_INSTRUCTIONS: 1077 return decoder->period - decoder->period_insn_cnt; 1078 case INTEL_PT_PERIOD_TICKS: 1079 return intel_pt_next_period(decoder); 1080 case INTEL_PT_PERIOD_NONE: 1081 case INTEL_PT_PERIOD_MTC: 1082 default: 1083 return 0; 1084 } 1085 } 1086 1087 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 1088 { 1089 uint64_t timestamp, masked_timestamp; 1090 1091 switch (decoder->period_type) { 1092 case INTEL_PT_PERIOD_INSTRUCTIONS: 1093 decoder->period_insn_cnt = 0; 1094 break; 1095 case INTEL_PT_PERIOD_TICKS: 1096 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1097 masked_timestamp = timestamp & decoder->period_mask; 1098 if (masked_timestamp > decoder->last_masked_timestamp) 1099 decoder->last_masked_timestamp = masked_timestamp; 1100 else 1101 decoder->last_masked_timestamp += decoder->period_ticks; 1102 break; 1103 case INTEL_PT_PERIOD_NONE: 1104 case INTEL_PT_PERIOD_MTC: 1105 default: 1106 break; 1107 } 1108 1109 decoder->state.type |= INTEL_PT_INSTRUCTION; 1110 } 1111 1112 /* 1113 * Sample FUP instruction at the same time as reporting the FUP event, so the 1114 * instruction sample gets the same flags as the FUP event. 1115 */ 1116 static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder) 1117 { 1118 struct intel_pt_insn intel_pt_insn; 1119 uint64_t max_insn_cnt, insn_cnt = 0; 1120 int err; 1121 1122 decoder->state.insn_op = INTEL_PT_OP_OTHER; 1123 decoder->state.insn_len = 0; 1124 1125 if (!decoder->branch_enable || !decoder->pge || decoder->hop || 1126 decoder->ip != decoder->last_ip) 1127 return; 1128 1129 if (!decoder->mtc_insn) 1130 decoder->mtc_insn = true; 1131 1132 max_insn_cnt = intel_pt_next_sample(decoder); 1133 if (max_insn_cnt != 1) 1134 return; 1135 1136 err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip, 1137 0, max_insn_cnt, decoder->data); 1138 /* Ignore error, it will be reported next walk anyway */ 1139 if (err) 1140 return; 1141 1142 if (intel_pt_insn.branch != INTEL_PT_BR_NO_BRANCH) { 1143 intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip); 1144 return; 1145 } 1146 1147 decoder->tot_insn_cnt += insn_cnt; 1148 decoder->timestamp_insn_cnt += insn_cnt; 1149 decoder->sample_insn_cnt += insn_cnt; 1150 decoder->period_insn_cnt += insn_cnt; 1151 1152 intel_pt_sample_insn(decoder); 1153 1154 decoder->state.type |= INTEL_PT_INSTRUCTION; 1155 decoder->ip += intel_pt_insn.length; 1156 } 1157 1158 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1159 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1160 { 1161 uint64_t max_insn_cnt, insn_cnt = 0; 1162 int err; 1163 1164 if (!decoder->mtc_insn) 1165 decoder->mtc_insn = true; 1166 1167 max_insn_cnt = intel_pt_next_sample(decoder); 1168 1169 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1170 max_insn_cnt, decoder->data); 1171 1172 decoder->tot_insn_cnt += insn_cnt; 1173 decoder->timestamp_insn_cnt += insn_cnt; 1174 decoder->sample_insn_cnt += insn_cnt; 1175 decoder->period_insn_cnt += insn_cnt; 1176 1177 if (err) { 1178 decoder->no_progress = 0; 1179 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1180 intel_pt_log_at("ERROR: Failed to get instruction", 1181 decoder->ip); 1182 if (err == -ENOENT) 1183 return -ENOLINK; 1184 return -EILSEQ; 1185 } 1186 1187 if (ip && decoder->ip == ip) { 1188 err = -EAGAIN; 1189 goto out; 1190 } 1191 1192 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1193 intel_pt_sample_insn(decoder); 1194 1195 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1196 decoder->state.type = INTEL_PT_INSTRUCTION; 1197 decoder->state.from_ip = decoder->ip; 1198 decoder->state.to_ip = 0; 1199 decoder->ip += intel_pt_insn->length; 1200 err = INTEL_PT_RETURN; 1201 goto out; 1202 } 1203 1204 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1205 /* Zero-length calls are excluded */ 1206 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1207 intel_pt_insn->rel) { 1208 err = intel_pt_push(&decoder->stack, decoder->ip + 1209 intel_pt_insn->length); 1210 if (err) 1211 goto out; 1212 } 1213 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1214 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1215 } 1216 1217 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1218 int cnt = decoder->no_progress++; 1219 1220 decoder->state.from_ip = decoder->ip; 1221 decoder->ip += intel_pt_insn->length + 1222 intel_pt_insn->rel; 1223 decoder->state.to_ip = decoder->ip; 1224 err = INTEL_PT_RETURN; 1225 1226 /* 1227 * Check for being stuck in a loop. This can happen if a 1228 * decoder error results in the decoder erroneously setting the 1229 * ip to an address that is itself in an infinite loop that 1230 * consumes no packets. When that happens, there must be an 1231 * unconditional branch. 1232 */ 1233 if (cnt) { 1234 if (cnt == 1) { 1235 decoder->stuck_ip = decoder->state.to_ip; 1236 decoder->stuck_ip_prd = 1; 1237 decoder->stuck_ip_cnt = 1; 1238 } else if (cnt > decoder->max_loops || 1239 decoder->state.to_ip == decoder->stuck_ip) { 1240 intel_pt_log_at("ERROR: Never-ending loop", 1241 decoder->state.to_ip); 1242 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1243 err = -ELOOP; 1244 goto out; 1245 } else if (!--decoder->stuck_ip_cnt) { 1246 decoder->stuck_ip_prd += 1; 1247 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1248 decoder->stuck_ip = decoder->state.to_ip; 1249 } 1250 } 1251 goto out_no_progress; 1252 } 1253 out: 1254 decoder->no_progress = 0; 1255 out_no_progress: 1256 decoder->state.insn_op = intel_pt_insn->op; 1257 decoder->state.insn_len = intel_pt_insn->length; 1258 memcpy(decoder->state.insn, intel_pt_insn->buf, 1259 INTEL_PT_INSN_BUF_SZ); 1260 1261 if (decoder->tx_flags & INTEL_PT_IN_TX) 1262 decoder->state.flags |= INTEL_PT_IN_TX; 1263 1264 return err; 1265 } 1266 1267 static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder) 1268 { 1269 bool iflag = decoder->packet.count & INTEL_PT_IFLAG; 1270 1271 decoder->exec_mode = decoder->packet.payload; 1272 decoder->iflag = iflag; 1273 decoder->next_iflag = iflag; 1274 decoder->state.from_iflag = iflag; 1275 decoder->state.to_iflag = iflag; 1276 } 1277 1278 static void intel_pt_mode_exec(struct intel_pt_decoder *decoder) 1279 { 1280 bool iflag = decoder->packet.count & INTEL_PT_IFLAG; 1281 1282 decoder->exec_mode = decoder->packet.payload; 1283 decoder->next_iflag = iflag; 1284 } 1285 1286 static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder) 1287 { 1288 decoder->state.type |= INTEL_PT_IFLAG_CHG; 1289 decoder->state.from_iflag = decoder->iflag; 1290 decoder->state.to_iflag = decoder->next_iflag; 1291 decoder->iflag = decoder->next_iflag; 1292 } 1293 1294 static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder) 1295 { 1296 if (decoder->iflag != decoder->next_iflag) 1297 intel_pt_sample_iflag(decoder); 1298 } 1299 1300 static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder) 1301 { 1302 decoder->set_fup_tx_flags = false; 1303 decoder->set_fup_ptw = false; 1304 decoder->set_fup_mwait = false; 1305 decoder->set_fup_pwre = false; 1306 decoder->set_fup_exstop = false; 1307 decoder->set_fup_bep = false; 1308 decoder->set_fup_cfe_ip = false; 1309 decoder->set_fup_cfe = false; 1310 decoder->evd_cnt = 0; 1311 decoder->set_fup_mode_exec = false; 1312 decoder->iflag = decoder->next_iflag; 1313 } 1314 1315 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip) 1316 { 1317 enum intel_pt_sample_type type = decoder->state.type; 1318 bool sample_fup_insn = false; 1319 bool ret = false; 1320 1321 decoder->state.type &= ~INTEL_PT_BRANCH; 1322 decoder->state.insn_op = INTEL_PT_OP_OTHER; 1323 decoder->state.insn_len = 0; 1324 1325 if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) { 1326 bool ip = decoder->set_fup_cfe_ip; 1327 1328 decoder->set_fup_cfe_ip = false; 1329 decoder->set_fup_cfe = false; 1330 decoder->state.type |= INTEL_PT_EVT; 1331 if (!ip && decoder->pge) 1332 decoder->state.type |= INTEL_PT_BRANCH; 1333 decoder->state.cfe_type = decoder->fup_cfe_pkt.count; 1334 decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload; 1335 decoder->state.evd_cnt = decoder->evd_cnt; 1336 decoder->state.evd = decoder->evd; 1337 decoder->evd_cnt = 0; 1338 if (ip || decoder->pge) 1339 decoder->state.flags |= INTEL_PT_FUP_IP; 1340 ret = true; 1341 } 1342 if (decoder->set_fup_mode_exec) { 1343 decoder->set_fup_mode_exec = false; 1344 intel_pt_sample_iflag(decoder); 1345 sample_fup_insn = no_tip; 1346 ret = true; 1347 } 1348 if (decoder->set_fup_tx_flags) { 1349 decoder->set_fup_tx_flags = false; 1350 decoder->tx_flags = decoder->fup_tx_flags; 1351 decoder->state.type |= INTEL_PT_TRANSACTION; 1352 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) 1353 decoder->state.type |= INTEL_PT_BRANCH; 1354 decoder->state.flags = decoder->fup_tx_flags; 1355 ret = true; 1356 } 1357 if (decoder->set_fup_ptw) { 1358 decoder->set_fup_ptw = false; 1359 decoder->state.type |= INTEL_PT_PTW; 1360 decoder->state.flags |= INTEL_PT_FUP_IP; 1361 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1362 ret = true; 1363 } 1364 if (decoder->set_fup_mwait) { 1365 decoder->set_fup_mwait = false; 1366 decoder->state.type |= INTEL_PT_MWAIT_OP; 1367 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1368 ret = true; 1369 } 1370 if (decoder->set_fup_pwre) { 1371 decoder->set_fup_pwre = false; 1372 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1373 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1374 ret = true; 1375 } 1376 if (decoder->set_fup_exstop) { 1377 decoder->set_fup_exstop = false; 1378 decoder->state.type |= INTEL_PT_EX_STOP; 1379 decoder->state.flags |= INTEL_PT_FUP_IP; 1380 ret = true; 1381 } 1382 if (decoder->set_fup_bep) { 1383 decoder->set_fup_bep = false; 1384 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1385 ret = true; 1386 } 1387 if (decoder->overflow) { 1388 decoder->overflow = false; 1389 if (!ret && !decoder->pge) { 1390 if (decoder->hop) { 1391 decoder->state.type = 0; 1392 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 1393 } 1394 decoder->pge = true; 1395 decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN; 1396 decoder->state.from_ip = 0; 1397 decoder->state.to_ip = decoder->ip; 1398 return true; 1399 } 1400 } 1401 if (ret) { 1402 decoder->state.from_ip = decoder->ip; 1403 decoder->state.to_ip = 0; 1404 if (sample_fup_insn) 1405 intel_pt_sample_fup_insn(decoder); 1406 } else { 1407 decoder->state.type = type; 1408 } 1409 return ret; 1410 } 1411 1412 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1413 struct intel_pt_insn *intel_pt_insn, 1414 uint64_t ip, int err) 1415 { 1416 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1417 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1418 ip == decoder->ip + intel_pt_insn->length; 1419 } 1420 1421 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1422 { 1423 struct intel_pt_insn intel_pt_insn; 1424 uint64_t ip; 1425 int err; 1426 1427 ip = decoder->last_ip; 1428 1429 while (1) { 1430 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1431 if (err == INTEL_PT_RETURN) 1432 return 0; 1433 if (err == -EAGAIN || 1434 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1435 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; 1436 1437 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1438 if (intel_pt_fup_event(decoder, no_tip) && no_tip) 1439 return 0; 1440 return -EAGAIN; 1441 } 1442 decoder->set_fup_tx_flags = false; 1443 if (err) 1444 return err; 1445 1446 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1447 intel_pt_log_at("ERROR: Unexpected indirect branch", 1448 decoder->ip); 1449 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1450 return -ENOENT; 1451 } 1452 1453 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1454 intel_pt_log_at("ERROR: Unexpected conditional branch", 1455 decoder->ip); 1456 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1457 return -ENOENT; 1458 } 1459 1460 intel_pt_bug(decoder); 1461 } 1462 } 1463 1464 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1465 { 1466 struct intel_pt_insn intel_pt_insn; 1467 int err; 1468 1469 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1470 if (err == INTEL_PT_RETURN && 1471 decoder->pgd_ip && 1472 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1473 (decoder->state.type & INTEL_PT_BRANCH) && 1474 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1475 /* Unconditional branch leaving filter region */ 1476 decoder->no_progress = 0; 1477 decoder->pge = false; 1478 decoder->continuous_period = false; 1479 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1480 decoder->state.type |= INTEL_PT_TRACE_END; 1481 intel_pt_update_nr(decoder); 1482 return 0; 1483 } 1484 if (err == INTEL_PT_RETURN) 1485 return 0; 1486 if (err) 1487 return err; 1488 1489 intel_pt_update_nr(decoder); 1490 intel_pt_sample_iflag_chg(decoder); 1491 1492 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1493 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1494 decoder->pge = false; 1495 decoder->continuous_period = false; 1496 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1497 decoder->state.from_ip = decoder->ip; 1498 if (decoder->packet.count == 0) { 1499 decoder->state.to_ip = 0; 1500 } else { 1501 decoder->state.to_ip = decoder->last_ip; 1502 decoder->ip = decoder->last_ip; 1503 } 1504 decoder->state.type |= INTEL_PT_TRACE_END; 1505 } else { 1506 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1507 decoder->state.from_ip = decoder->ip; 1508 if (decoder->packet.count == 0) { 1509 decoder->state.to_ip = 0; 1510 } else { 1511 decoder->state.to_ip = decoder->last_ip; 1512 decoder->ip = decoder->last_ip; 1513 } 1514 } 1515 return 0; 1516 } 1517 1518 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1519 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1520 intel_pt_insn.rel; 1521 1522 if (decoder->pgd_ip && 1523 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1524 decoder->pgd_ip(to_ip, decoder->data)) { 1525 /* Conditional branch leaving filter region */ 1526 decoder->pge = false; 1527 decoder->continuous_period = false; 1528 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1529 decoder->ip = to_ip; 1530 decoder->state.from_ip = decoder->ip; 1531 decoder->state.to_ip = to_ip; 1532 decoder->state.type |= INTEL_PT_TRACE_END; 1533 return 0; 1534 } 1535 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1536 decoder->ip); 1537 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1538 return -ENOENT; 1539 } 1540 1541 return intel_pt_bug(decoder); 1542 } 1543 1544 struct eptw_data { 1545 int bit_countdown; 1546 uint64_t payload; 1547 }; 1548 1549 static int intel_pt_eptw_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 1550 { 1551 struct eptw_data *data = pkt_info->data; 1552 int nr_bits; 1553 1554 switch (pkt_info->packet.type) { 1555 case INTEL_PT_PAD: 1556 case INTEL_PT_MNT: 1557 case INTEL_PT_MODE_EXEC: 1558 case INTEL_PT_MODE_TSX: 1559 case INTEL_PT_MTC: 1560 case INTEL_PT_FUP: 1561 case INTEL_PT_CYC: 1562 case INTEL_PT_CBR: 1563 case INTEL_PT_TSC: 1564 case INTEL_PT_TMA: 1565 case INTEL_PT_PIP: 1566 case INTEL_PT_VMCS: 1567 case INTEL_PT_PSB: 1568 case INTEL_PT_PSBEND: 1569 case INTEL_PT_PTWRITE: 1570 case INTEL_PT_PTWRITE_IP: 1571 case INTEL_PT_EXSTOP: 1572 case INTEL_PT_EXSTOP_IP: 1573 case INTEL_PT_MWAIT: 1574 case INTEL_PT_PWRE: 1575 case INTEL_PT_PWRX: 1576 case INTEL_PT_BBP: 1577 case INTEL_PT_BIP: 1578 case INTEL_PT_BEP: 1579 case INTEL_PT_BEP_IP: 1580 case INTEL_PT_CFE: 1581 case INTEL_PT_CFE_IP: 1582 case INTEL_PT_EVD: 1583 break; 1584 1585 case INTEL_PT_TNT: 1586 nr_bits = data->bit_countdown; 1587 if (nr_bits > pkt_info->packet.count) 1588 nr_bits = pkt_info->packet.count; 1589 data->payload <<= nr_bits; 1590 data->payload |= pkt_info->packet.payload >> (64 - nr_bits); 1591 data->bit_countdown -= nr_bits; 1592 return !data->bit_countdown; 1593 1594 case INTEL_PT_TIP_PGE: 1595 case INTEL_PT_TIP_PGD: 1596 case INTEL_PT_TIP: 1597 case INTEL_PT_BAD: 1598 case INTEL_PT_OVF: 1599 case INTEL_PT_TRACESTOP: 1600 default: 1601 return 1; 1602 } 1603 1604 return 0; 1605 } 1606 1607 static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder) 1608 { 1609 int n = 64 - decoder->tnt.count; 1610 struct eptw_data data = { 1611 .bit_countdown = n, 1612 .payload = decoder->tnt.payload >> n, 1613 }; 1614 1615 decoder->emulated_ptwrite = false; 1616 intel_pt_log("Emulated ptwrite detected\n"); 1617 1618 intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data); 1619 if (data.bit_countdown) 1620 return -ECONNRESET; 1621 1622 decoder->state.type = INTEL_PT_PTW; 1623 decoder->state.from_ip = decoder->ip; 1624 decoder->state.to_ip = 0; 1625 decoder->state.ptw_payload = data.payload; 1626 return 0; 1627 } 1628 1629 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1630 { 1631 struct intel_pt_insn intel_pt_insn; 1632 int err; 1633 1634 while (1) { 1635 if (decoder->emulated_ptwrite) 1636 return intel_pt_emulated_ptwrite(decoder); 1637 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1638 if (err == INTEL_PT_RETURN) { 1639 decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite; 1640 return 0; 1641 } 1642 if (err) { 1643 decoder->emulated_ptwrite = false; 1644 return err; 1645 } 1646 1647 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1648 if (!decoder->return_compression) { 1649 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1650 decoder->ip); 1651 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1652 return -ENOENT; 1653 } 1654 if (!decoder->ret_addr) { 1655 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1656 decoder->ip); 1657 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1658 return -ENOENT; 1659 } 1660 if (!(decoder->tnt.payload & BIT63)) { 1661 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1662 decoder->ip); 1663 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1664 return -ENOENT; 1665 } 1666 decoder->tnt.count -= 1; 1667 if (decoder->tnt.count) 1668 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1669 else 1670 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1671 decoder->tnt.payload <<= 1; 1672 decoder->state.from_ip = decoder->ip; 1673 decoder->ip = decoder->ret_addr; 1674 decoder->state.to_ip = decoder->ip; 1675 return 0; 1676 } 1677 1678 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1679 /* Handle deferred TIPs */ 1680 err = intel_pt_get_next_packet(decoder); 1681 if (err) 1682 return err; 1683 if (decoder->packet.type != INTEL_PT_TIP || 1684 decoder->packet.count == 0) { 1685 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1686 decoder->ip); 1687 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1688 decoder->pkt_step = 0; 1689 return -ENOENT; 1690 } 1691 intel_pt_set_last_ip(decoder); 1692 decoder->state.from_ip = decoder->ip; 1693 decoder->state.to_ip = decoder->last_ip; 1694 decoder->ip = decoder->last_ip; 1695 intel_pt_update_nr(decoder); 1696 intel_pt_sample_iflag_chg(decoder); 1697 return 0; 1698 } 1699 1700 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1701 decoder->tnt.count -= 1; 1702 if (decoder->tnt.count) 1703 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1704 else 1705 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1706 if (decoder->tnt.payload & BIT63) { 1707 decoder->tnt.payload <<= 1; 1708 decoder->state.from_ip = decoder->ip; 1709 decoder->ip += intel_pt_insn.length + 1710 intel_pt_insn.rel; 1711 decoder->state.to_ip = decoder->ip; 1712 return 0; 1713 } 1714 /* Instruction sample for a non-taken branch */ 1715 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1716 decoder->tnt.payload <<= 1; 1717 decoder->state.type = INTEL_PT_INSTRUCTION; 1718 decoder->state.from_ip = decoder->ip; 1719 decoder->state.to_ip = 0; 1720 decoder->ip += intel_pt_insn.length; 1721 return 0; 1722 } 1723 decoder->sample_cyc = false; 1724 decoder->ip += intel_pt_insn.length; 1725 if (!decoder->tnt.count) { 1726 intel_pt_update_sample_time(decoder); 1727 return -EAGAIN; 1728 } 1729 decoder->tnt.payload <<= 1; 1730 continue; 1731 } 1732 1733 return intel_pt_bug(decoder); 1734 } 1735 } 1736 1737 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1738 { 1739 unsigned int fup_tx_flags; 1740 int err; 1741 1742 fup_tx_flags = decoder->packet.payload & 1743 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1744 err = intel_pt_get_next_packet(decoder); 1745 if (err) 1746 return err; 1747 if (decoder->packet.type == INTEL_PT_FUP) { 1748 decoder->fup_tx_flags = fup_tx_flags; 1749 decoder->set_fup_tx_flags = true; 1750 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1751 *no_tip = true; 1752 } else { 1753 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1754 decoder->pos); 1755 intel_pt_update_in_tx(decoder); 1756 } 1757 return 0; 1758 } 1759 1760 static int intel_pt_evd(struct intel_pt_decoder *decoder) 1761 { 1762 if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) { 1763 intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos); 1764 return -ENOSYS; 1765 } 1766 decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){ 1767 .type = decoder->packet.count, 1768 .payload = decoder->packet.payload, 1769 }; 1770 return 0; 1771 } 1772 1773 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1774 { 1775 timestamp |= (ref_timestamp & (0xffULL << 56)); 1776 1777 if (timestamp < ref_timestamp) { 1778 if (ref_timestamp - timestamp > (1ULL << 55)) 1779 timestamp += (1ULL << 56); 1780 } else { 1781 if (timestamp - ref_timestamp > (1ULL << 55)) 1782 timestamp -= (1ULL << 56); 1783 } 1784 1785 return timestamp; 1786 } 1787 1788 /* For use only when decoder->vm_time_correlation is true */ 1789 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder, 1790 uint64_t timestamp) 1791 { 1792 uint64_t max_timestamp = decoder->buf_timestamp; 1793 1794 if (!max_timestamp) { 1795 max_timestamp = decoder->last_reliable_timestamp + 1796 0x400000000ULL; 1797 } 1798 return timestamp >= decoder->last_reliable_timestamp && 1799 timestamp < decoder->buf_timestamp; 1800 } 1801 1802 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1803 { 1804 uint64_t timestamp; 1805 bool bad = false; 1806 1807 decoder->have_tma = false; 1808 1809 if (decoder->ref_timestamp) { 1810 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1811 decoder->ref_timestamp); 1812 decoder->tsc_timestamp = timestamp; 1813 decoder->timestamp = timestamp; 1814 decoder->ref_timestamp = 0; 1815 decoder->timestamp_insn_cnt = 0; 1816 } else if (decoder->timestamp) { 1817 timestamp = decoder->packet.payload | 1818 (decoder->timestamp & (0xffULL << 56)); 1819 decoder->tsc_timestamp = timestamp; 1820 if (timestamp < decoder->timestamp && 1821 decoder->timestamp - timestamp < decoder->tsc_slip) { 1822 intel_pt_log_to("Suppressing backwards timestamp", 1823 timestamp); 1824 timestamp = decoder->timestamp; 1825 } 1826 if (timestamp < decoder->timestamp) { 1827 if (!decoder->buf_timestamp || 1828 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { 1829 intel_pt_log_to("Wraparound timestamp", timestamp); 1830 timestamp += (1ULL << 56); 1831 decoder->tsc_timestamp = timestamp; 1832 } else { 1833 intel_pt_log_to("Suppressing bad timestamp", timestamp); 1834 timestamp = decoder->timestamp; 1835 bad = true; 1836 } 1837 } 1838 if (decoder->vm_time_correlation && 1839 (bad || !intel_pt_time_in_range(decoder, timestamp)) && 1840 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 1841 p_log("Timestamp out of range"); 1842 decoder->timestamp = timestamp; 1843 decoder->timestamp_insn_cnt = 0; 1844 } 1845 1846 if (decoder->last_packet_type == INTEL_PT_CYC) { 1847 decoder->cyc_ref_timestamp = decoder->timestamp; 1848 decoder->cycle_cnt = 0; 1849 decoder->have_calc_cyc_to_tsc = false; 1850 intel_pt_calc_cyc_to_tsc(decoder, false); 1851 } 1852 1853 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1854 } 1855 1856 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1857 { 1858 intel_pt_log("ERROR: Buffer overflow\n"); 1859 intel_pt_clear_tx_flags(decoder); 1860 intel_pt_set_nr(decoder); 1861 decoder->timestamp_insn_cnt = 0; 1862 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1863 decoder->state.from_ip = decoder->ip; 1864 decoder->ip = 0; 1865 decoder->pge = false; 1866 intel_pt_clear_fup_event(decoder); 1867 decoder->overflow = true; 1868 return -EOVERFLOW; 1869 } 1870 1871 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1872 { 1873 if (decoder->have_cyc) 1874 return; 1875 1876 decoder->cyc_cnt_timestamp = decoder->timestamp; 1877 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1878 } 1879 1880 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1881 { 1882 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1883 1884 if (decoder->pge) 1885 intel_pt_mtc_cyc_cnt_pge(decoder); 1886 } 1887 1888 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1889 { 1890 uint64_t tot_cyc_cnt, tsc_delta; 1891 1892 if (decoder->have_cyc) 1893 return; 1894 1895 decoder->sample_cyc = true; 1896 1897 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1898 return; 1899 1900 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1901 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1902 1903 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1904 decoder->tot_cyc_cnt = tot_cyc_cnt; 1905 } 1906 1907 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1908 { 1909 uint32_t ctc = decoder->packet.payload; 1910 uint32_t fc = decoder->packet.count; 1911 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1912 1913 if (!decoder->tsc_ctc_ratio_d) 1914 return; 1915 1916 if (decoder->pge && !decoder->in_psb) 1917 intel_pt_mtc_cyc_cnt_pge(decoder); 1918 else 1919 intel_pt_mtc_cyc_cnt_upd(decoder); 1920 1921 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1922 decoder->last_ctc = ctc - ctc_rem; 1923 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1924 if (decoder->tsc_ctc_mult) { 1925 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1926 } else { 1927 decoder->ctc_timestamp -= multdiv(ctc_rem, 1928 decoder->tsc_ctc_ratio_n, 1929 decoder->tsc_ctc_ratio_d); 1930 } 1931 decoder->ctc_delta = 0; 1932 decoder->have_tma = true; 1933 decoder->fixup_last_mtc = true; 1934 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1935 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1936 } 1937 1938 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1939 { 1940 uint64_t timestamp; 1941 uint32_t mtc, mtc_delta; 1942 1943 if (!decoder->have_tma) 1944 return; 1945 1946 mtc = decoder->packet.payload; 1947 1948 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1949 decoder->fixup_last_mtc = false; 1950 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1951 &decoder->last_mtc); 1952 } 1953 1954 if (mtc > decoder->last_mtc) 1955 mtc_delta = mtc - decoder->last_mtc; 1956 else 1957 mtc_delta = mtc + 256 - decoder->last_mtc; 1958 1959 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1960 1961 if (decoder->tsc_ctc_mult) { 1962 timestamp = decoder->ctc_timestamp + 1963 decoder->ctc_delta * decoder->tsc_ctc_mult; 1964 } else { 1965 timestamp = decoder->ctc_timestamp + 1966 multdiv(decoder->ctc_delta, 1967 decoder->tsc_ctc_ratio_n, 1968 decoder->tsc_ctc_ratio_d); 1969 } 1970 1971 if (timestamp < decoder->timestamp) 1972 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1973 timestamp, decoder->timestamp); 1974 else 1975 decoder->timestamp = timestamp; 1976 1977 intel_pt_mtc_cyc_cnt_upd(decoder); 1978 1979 decoder->timestamp_insn_cnt = 0; 1980 decoder->last_mtc = mtc; 1981 1982 if (decoder->last_packet_type == INTEL_PT_CYC) { 1983 decoder->cyc_ref_timestamp = decoder->timestamp; 1984 decoder->cycle_cnt = 0; 1985 decoder->have_calc_cyc_to_tsc = false; 1986 intel_pt_calc_cyc_to_tsc(decoder, true); 1987 } 1988 1989 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1990 } 1991 1992 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1993 { 1994 unsigned int cbr = decoder->packet.payload & 0xff; 1995 1996 decoder->cbr_payload = decoder->packet.payload; 1997 1998 if (decoder->cbr == cbr) 1999 return; 2000 2001 decoder->cbr = cbr; 2002 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 2003 decoder->cyc_ref_timestamp = decoder->timestamp; 2004 decoder->cycle_cnt = 0; 2005 2006 intel_pt_mtc_cyc_cnt_cbr(decoder); 2007 } 2008 2009 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 2010 { 2011 uint64_t timestamp = decoder->cyc_ref_timestamp; 2012 2013 decoder->have_cyc = true; 2014 2015 decoder->cycle_cnt += decoder->packet.payload; 2016 if (decoder->pge) 2017 decoder->tot_cyc_cnt += decoder->packet.payload; 2018 decoder->sample_cyc = true; 2019 2020 if (!decoder->cyc_ref_timestamp) 2021 return; 2022 2023 if (decoder->have_calc_cyc_to_tsc) 2024 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 2025 else if (decoder->cbr) 2026 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 2027 else 2028 return; 2029 2030 if (timestamp < decoder->timestamp) 2031 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 2032 timestamp, decoder->timestamp); 2033 else 2034 decoder->timestamp = timestamp; 2035 2036 decoder->timestamp_insn_cnt = 0; 2037 2038 intel_pt_log_to("Setting timestamp", decoder->timestamp); 2039 } 2040 2041 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 2042 { 2043 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 2044 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 2045 decoder->state.items.is_32_bit = false; 2046 } 2047 decoder->blk_type = decoder->packet.payload; 2048 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 2049 if (decoder->blk_type == INTEL_PT_GP_REGS) 2050 decoder->state.items.is_32_bit = decoder->packet.count; 2051 if (decoder->blk_type_pos < 0) { 2052 intel_pt_log("WARNING: Unknown block type %u\n", 2053 decoder->blk_type); 2054 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 2055 intel_pt_log("WARNING: Duplicate block type %u\n", 2056 decoder->blk_type); 2057 } 2058 } 2059 2060 static void intel_pt_bip(struct intel_pt_decoder *decoder) 2061 { 2062 uint32_t id = decoder->packet.count; 2063 uint32_t bit = 1 << id; 2064 int pos = decoder->blk_type_pos; 2065 2066 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 2067 intel_pt_log("WARNING: Unknown block item %u type %d\n", 2068 id, decoder->blk_type); 2069 return; 2070 } 2071 2072 if (decoder->state.items.mask[pos] & bit) { 2073 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 2074 id, decoder->blk_type); 2075 } 2076 2077 decoder->state.items.mask[pos] |= bit; 2078 decoder->state.items.val[pos][id] = decoder->packet.payload; 2079 } 2080 2081 /* Walk PSB+ packets when already in sync. */ 2082 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 2083 { 2084 int err; 2085 2086 decoder->in_psb = true; 2087 2088 while (1) { 2089 err = intel_pt_get_next_packet(decoder); 2090 if (err) 2091 goto out; 2092 2093 switch (decoder->packet.type) { 2094 case INTEL_PT_PSBEND: 2095 err = 0; 2096 goto out; 2097 2098 case INTEL_PT_TIP_PGD: 2099 case INTEL_PT_TIP_PGE: 2100 case INTEL_PT_TIP: 2101 case INTEL_PT_TNT: 2102 case INTEL_PT_TRACESTOP: 2103 case INTEL_PT_BAD: 2104 case INTEL_PT_PSB: 2105 case INTEL_PT_PTWRITE: 2106 case INTEL_PT_PTWRITE_IP: 2107 case INTEL_PT_EXSTOP: 2108 case INTEL_PT_EXSTOP_IP: 2109 case INTEL_PT_MWAIT: 2110 case INTEL_PT_PWRE: 2111 case INTEL_PT_PWRX: 2112 case INTEL_PT_BBP: 2113 case INTEL_PT_BIP: 2114 case INTEL_PT_BEP: 2115 case INTEL_PT_BEP_IP: 2116 case INTEL_PT_CFE: 2117 case INTEL_PT_CFE_IP: 2118 case INTEL_PT_EVD: 2119 decoder->have_tma = false; 2120 intel_pt_log("ERROR: Unexpected packet\n"); 2121 err = -EAGAIN; 2122 goto out; 2123 2124 case INTEL_PT_OVF: 2125 err = intel_pt_overflow(decoder); 2126 goto out; 2127 2128 case INTEL_PT_TSC: 2129 intel_pt_calc_tsc_timestamp(decoder); 2130 break; 2131 2132 case INTEL_PT_TMA: 2133 intel_pt_calc_tma(decoder); 2134 break; 2135 2136 case INTEL_PT_CBR: 2137 intel_pt_calc_cbr(decoder); 2138 break; 2139 2140 case INTEL_PT_MODE_EXEC: 2141 intel_pt_mode_exec_status(decoder); 2142 break; 2143 2144 case INTEL_PT_PIP: 2145 intel_pt_set_pip(decoder); 2146 break; 2147 2148 case INTEL_PT_FUP: 2149 decoder->pge = true; 2150 if (decoder->packet.count) { 2151 intel_pt_set_last_ip(decoder); 2152 decoder->psb_ip = decoder->last_ip; 2153 } 2154 break; 2155 2156 case INTEL_PT_MODE_TSX: 2157 intel_pt_update_in_tx(decoder); 2158 break; 2159 2160 case INTEL_PT_MTC: 2161 intel_pt_calc_mtc_timestamp(decoder); 2162 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 2163 decoder->state.type |= INTEL_PT_INSTRUCTION; 2164 break; 2165 2166 case INTEL_PT_CYC: 2167 intel_pt_calc_cyc_timestamp(decoder); 2168 break; 2169 2170 case INTEL_PT_VMCS: 2171 case INTEL_PT_MNT: 2172 case INTEL_PT_PAD: 2173 default: 2174 break; 2175 } 2176 } 2177 out: 2178 decoder->in_psb = false; 2179 2180 return err; 2181 } 2182 2183 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 2184 { 2185 int err; 2186 2187 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 2188 decoder->tx_flags = 0; 2189 decoder->state.flags &= ~INTEL_PT_IN_TX; 2190 decoder->state.flags |= INTEL_PT_ABORT_TX; 2191 } else { 2192 decoder->state.flags |= INTEL_PT_ASYNC; 2193 } 2194 2195 while (1) { 2196 err = intel_pt_get_next_packet(decoder); 2197 if (err) 2198 return err; 2199 2200 switch (decoder->packet.type) { 2201 case INTEL_PT_TNT: 2202 case INTEL_PT_FUP: 2203 case INTEL_PT_TRACESTOP: 2204 case INTEL_PT_PSB: 2205 case INTEL_PT_TSC: 2206 case INTEL_PT_TMA: 2207 case INTEL_PT_MODE_TSX: 2208 case INTEL_PT_BAD: 2209 case INTEL_PT_PSBEND: 2210 case INTEL_PT_PTWRITE: 2211 case INTEL_PT_PTWRITE_IP: 2212 case INTEL_PT_EXSTOP: 2213 case INTEL_PT_EXSTOP_IP: 2214 case INTEL_PT_MWAIT: 2215 case INTEL_PT_PWRE: 2216 case INTEL_PT_PWRX: 2217 case INTEL_PT_BBP: 2218 case INTEL_PT_BIP: 2219 case INTEL_PT_BEP: 2220 case INTEL_PT_BEP_IP: 2221 case INTEL_PT_CFE: 2222 case INTEL_PT_CFE_IP: 2223 case INTEL_PT_EVD: 2224 intel_pt_log("ERROR: Missing TIP after FUP\n"); 2225 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2226 decoder->pkt_step = 0; 2227 return -ENOENT; 2228 2229 case INTEL_PT_CBR: 2230 intel_pt_calc_cbr(decoder); 2231 break; 2232 2233 case INTEL_PT_OVF: 2234 return intel_pt_overflow(decoder); 2235 2236 case INTEL_PT_TIP_PGD: 2237 decoder->state.from_ip = decoder->ip; 2238 if (decoder->packet.count == 0) { 2239 decoder->state.to_ip = 0; 2240 } else { 2241 intel_pt_set_ip(decoder); 2242 decoder->state.to_ip = decoder->ip; 2243 } 2244 decoder->pge = false; 2245 decoder->continuous_period = false; 2246 decoder->state.type |= INTEL_PT_TRACE_END; 2247 intel_pt_update_nr(decoder); 2248 return 0; 2249 2250 case INTEL_PT_TIP_PGE: 2251 decoder->pge = true; 2252 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 2253 decoder->ip); 2254 decoder->state.from_ip = 0; 2255 if (decoder->packet.count == 0) { 2256 decoder->state.to_ip = 0; 2257 } else { 2258 intel_pt_set_ip(decoder); 2259 decoder->state.to_ip = decoder->ip; 2260 } 2261 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2262 intel_pt_mtc_cyc_cnt_pge(decoder); 2263 intel_pt_set_nr(decoder); 2264 return 0; 2265 2266 case INTEL_PT_TIP: 2267 decoder->state.from_ip = decoder->ip; 2268 if (decoder->packet.count == 0) { 2269 decoder->state.to_ip = 0; 2270 } else { 2271 intel_pt_set_ip(decoder); 2272 decoder->state.to_ip = decoder->ip; 2273 } 2274 intel_pt_update_nr(decoder); 2275 intel_pt_sample_iflag_chg(decoder); 2276 return 0; 2277 2278 case INTEL_PT_PIP: 2279 intel_pt_update_pip(decoder); 2280 break; 2281 2282 case INTEL_PT_MTC: 2283 intel_pt_calc_mtc_timestamp(decoder); 2284 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 2285 decoder->state.type |= INTEL_PT_INSTRUCTION; 2286 break; 2287 2288 case INTEL_PT_CYC: 2289 intel_pt_calc_cyc_timestamp(decoder); 2290 break; 2291 2292 case INTEL_PT_MODE_EXEC: 2293 intel_pt_mode_exec(decoder); 2294 break; 2295 2296 case INTEL_PT_VMCS: 2297 case INTEL_PT_MNT: 2298 case INTEL_PT_PAD: 2299 break; 2300 2301 default: 2302 return intel_pt_bug(decoder); 2303 } 2304 } 2305 } 2306 2307 static int intel_pt_resample(struct intel_pt_decoder *decoder) 2308 { 2309 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2310 decoder->state.type = INTEL_PT_INSTRUCTION; 2311 decoder->state.from_ip = decoder->ip; 2312 decoder->state.to_ip = 0; 2313 return 0; 2314 } 2315 2316 struct intel_pt_vm_tsc_info { 2317 struct intel_pt_pkt pip_packet; 2318 struct intel_pt_pkt vmcs_packet; 2319 struct intel_pt_pkt tma_packet; 2320 bool tsc, pip, vmcs, tma, psbend; 2321 uint64_t ctc_delta; 2322 uint64_t last_ctc; 2323 int max_lookahead; 2324 }; 2325 2326 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */ 2327 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2328 { 2329 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2330 2331 switch (pkt_info->packet.type) { 2332 case INTEL_PT_PAD: 2333 case INTEL_PT_MNT: 2334 case INTEL_PT_MODE_EXEC: 2335 case INTEL_PT_MODE_TSX: 2336 case INTEL_PT_MTC: 2337 case INTEL_PT_FUP: 2338 case INTEL_PT_CYC: 2339 case INTEL_PT_CBR: 2340 break; 2341 2342 case INTEL_PT_TSC: 2343 data->tsc = true; 2344 break; 2345 2346 case INTEL_PT_TMA: 2347 data->tma_packet = pkt_info->packet; 2348 data->tma = true; 2349 break; 2350 2351 case INTEL_PT_PIP: 2352 data->pip_packet = pkt_info->packet; 2353 data->pip = true; 2354 break; 2355 2356 case INTEL_PT_VMCS: 2357 data->vmcs_packet = pkt_info->packet; 2358 data->vmcs = true; 2359 break; 2360 2361 case INTEL_PT_PSBEND: 2362 data->psbend = true; 2363 return 1; 2364 2365 case INTEL_PT_TIP_PGE: 2366 case INTEL_PT_PTWRITE: 2367 case INTEL_PT_PTWRITE_IP: 2368 case INTEL_PT_EXSTOP: 2369 case INTEL_PT_EXSTOP_IP: 2370 case INTEL_PT_MWAIT: 2371 case INTEL_PT_PWRE: 2372 case INTEL_PT_PWRX: 2373 case INTEL_PT_BBP: 2374 case INTEL_PT_BIP: 2375 case INTEL_PT_BEP: 2376 case INTEL_PT_BEP_IP: 2377 case INTEL_PT_OVF: 2378 case INTEL_PT_BAD: 2379 case INTEL_PT_TNT: 2380 case INTEL_PT_TIP_PGD: 2381 case INTEL_PT_TIP: 2382 case INTEL_PT_PSB: 2383 case INTEL_PT_TRACESTOP: 2384 case INTEL_PT_CFE: 2385 case INTEL_PT_CFE_IP: 2386 case INTEL_PT_EVD: 2387 default: 2388 return 1; 2389 } 2390 2391 return 0; 2392 } 2393 2394 struct intel_pt_ovf_fup_info { 2395 int max_lookahead; 2396 bool found; 2397 }; 2398 2399 /* Lookahead to detect a FUP packet after OVF */ 2400 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2401 { 2402 struct intel_pt_ovf_fup_info *data = pkt_info->data; 2403 2404 if (pkt_info->packet.type == INTEL_PT_CYC || 2405 pkt_info->packet.type == INTEL_PT_MTC || 2406 pkt_info->packet.type == INTEL_PT_TSC) 2407 return !--(data->max_lookahead); 2408 data->found = pkt_info->packet.type == INTEL_PT_FUP; 2409 return 1; 2410 } 2411 2412 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder) 2413 { 2414 struct intel_pt_ovf_fup_info data = { 2415 .max_lookahead = 16, 2416 .found = false, 2417 }; 2418 2419 intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data); 2420 return data.found; 2421 } 2422 2423 /* Lookahead and get the TMA packet after TSC */ 2424 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2425 { 2426 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2427 2428 if (pkt_info->packet.type == INTEL_PT_CYC || 2429 pkt_info->packet.type == INTEL_PT_MTC) 2430 return !--(data->max_lookahead); 2431 2432 if (pkt_info->packet.type == INTEL_PT_TMA) { 2433 data->tma_packet = pkt_info->packet; 2434 data->tma = true; 2435 } 2436 return 1; 2437 } 2438 2439 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc) 2440 { 2441 if (decoder->tsc_ctc_mult) 2442 return ctc * decoder->tsc_ctc_mult; 2443 else 2444 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); 2445 } 2446 2447 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder, 2448 uint32_t ctc, 2449 uint32_t fc, 2450 uint64_t last_ctc_timestamp, 2451 uint64_t ctc_delta, 2452 uint32_t last_ctc) 2453 { 2454 /* Number of CTC ticks from last_ctc_timestamp to last_mtc */ 2455 uint64_t last_mtc_ctc = last_ctc + ctc_delta; 2456 /* 2457 * Number of CTC ticks from there until current TMA packet. We would 2458 * expect last_mtc_ctc to be before ctc, but the TSC packet can slip 2459 * past an MTC, so a sign-extended value is used. 2460 */ 2461 uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); 2462 /* Total CTC ticks from last_ctc_timestamp to current TMA packet */ 2463 uint64_t new_ctc_delta = ctc_delta + delta; 2464 uint64_t expected_tsc; 2465 2466 /* 2467 * Convert CTC ticks to TSC ticks, add the starting point 2468 * (last_ctc_timestamp) and the fast counter from the TMA packet. 2469 */ 2470 expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc; 2471 2472 if (intel_pt_enable_logging) { 2473 intel_pt_log_x64(last_mtc_ctc); 2474 intel_pt_log_x32(last_ctc); 2475 intel_pt_log_x64(ctc_delta); 2476 intel_pt_log_x64(delta); 2477 intel_pt_log_x32(ctc); 2478 intel_pt_log_x64(new_ctc_delta); 2479 intel_pt_log_x64(last_ctc_timestamp); 2480 intel_pt_log_x32(fc); 2481 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta)); 2482 intel_pt_log_x64(expected_tsc); 2483 } 2484 2485 return expected_tsc; 2486 } 2487 2488 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder, 2489 struct intel_pt_vm_tsc_info *data) 2490 { 2491 uint32_t ctc = data->tma_packet.payload; 2492 uint32_t fc = data->tma_packet.count; 2493 2494 return intel_pt_calc_expected_tsc(decoder, ctc, fc, 2495 decoder->ctc_timestamp, 2496 data->ctc_delta, data->last_ctc); 2497 } 2498 2499 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder, 2500 struct intel_pt_vmcs_info *vmcs_info) 2501 { 2502 uint64_t payload = decoder->packet.payload; 2503 2504 /* VMX adds the TSC Offset, so subtract to get host TSC */ 2505 decoder->packet.payload -= vmcs_info->tsc_offset; 2506 /* TSC packet has only 7 bytes */ 2507 decoder->packet.payload &= SEVEN_BYTES; 2508 2509 /* 2510 * The buffer is mmapped from the data file, so this also updates the 2511 * data file. 2512 */ 2513 if (!decoder->vm_tm_corr_dry_run) 2514 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7); 2515 2516 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2517 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2518 payload, decoder->packet.payload, vmcs_info->vmcs, 2519 vmcs_info->tsc_offset); 2520 } 2521 2522 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder, 2523 uint64_t tsc_offset) 2524 { 2525 struct intel_pt_vmcs_info vmcs_info = { 2526 .vmcs = NO_VMCS, 2527 .tsc_offset = tsc_offset 2528 }; 2529 2530 intel_pt_translate_vm_tsc(decoder, &vmcs_info); 2531 } 2532 2533 static inline bool in_vm(uint64_t pip_payload) 2534 { 2535 return pip_payload & 1; 2536 } 2537 2538 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet) 2539 { 2540 return pip_packet->payload & 1; 2541 } 2542 2543 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info) 2544 { 2545 p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64, 2546 vmcs_info->vmcs, vmcs_info->tsc_offset); 2547 } 2548 2549 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder, 2550 struct intel_pt_vm_tsc_info *data) 2551 { 2552 memset(data, 0, sizeof(*data)); 2553 data->ctc_delta = decoder->ctc_delta; 2554 data->last_ctc = decoder->last_ctc; 2555 intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data); 2556 if (data->tsc && !data->psbend) 2557 p_log("ERROR: PSB without PSBEND"); 2558 decoder->in_psb = data->psbend; 2559 } 2560 2561 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder, 2562 struct intel_pt_vm_tsc_info *data, 2563 struct intel_pt_vmcs_info *vmcs_info, 2564 uint64_t host_tsc) 2565 { 2566 if (!decoder->in_psb) { 2567 /* Can't happen */ 2568 p_log("ERROR: First TSC is not in PSB+"); 2569 } 2570 2571 if (data->pip) { 2572 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2573 if (vmcs_info && vmcs_info->tsc_offset) { 2574 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2575 decoder->vm_tm_corr_reliable = true; 2576 } else { 2577 p_log("ERROR: First TSC, unknown TSC Offset"); 2578 } 2579 } else { /* Host */ 2580 decoder->vm_tm_corr_reliable = true; 2581 } 2582 } else { /* Host or Guest */ 2583 decoder->vm_tm_corr_reliable = false; 2584 if (intel_pt_time_in_range(decoder, host_tsc)) { 2585 /* Assume Host */ 2586 } else { 2587 /* Assume Guest */ 2588 if (vmcs_info && vmcs_info->tsc_offset) 2589 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2590 else 2591 p_log("ERROR: First TSC, no PIP, unknown TSC Offset"); 2592 } 2593 } 2594 } 2595 2596 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder, 2597 struct intel_pt_vm_tsc_info *data) 2598 { 2599 struct intel_pt_vmcs_info *vmcs_info; 2600 uint64_t tsc_offset = 0; 2601 uint64_t vmcs; 2602 bool reliable = true; 2603 uint64_t expected_tsc; 2604 uint64_t host_tsc; 2605 uint64_t ref_timestamp; 2606 2607 bool assign = false; 2608 bool assign_reliable = false; 2609 2610 /* Already have 'data' for the in_psb case */ 2611 if (!decoder->in_psb) { 2612 memset(data, 0, sizeof(*data)); 2613 data->ctc_delta = decoder->ctc_delta; 2614 data->last_ctc = decoder->last_ctc; 2615 data->max_lookahead = 16; 2616 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data); 2617 if (decoder->pge) { 2618 data->pip = true; 2619 data->pip_packet.payload = decoder->pip_payload; 2620 } 2621 } 2622 2623 /* Calculations depend on having TMA packets */ 2624 if (!data->tma) { 2625 p_log("ERROR: TSC without TMA"); 2626 return; 2627 } 2628 2629 vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; 2630 if (vmcs == NO_VMCS) 2631 vmcs = 0; 2632 2633 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2634 2635 ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; 2636 host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp); 2637 2638 if (!decoder->ctc_timestamp) { 2639 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc); 2640 return; 2641 } 2642 2643 expected_tsc = intel_pt_expected_tsc(decoder, data); 2644 2645 tsc_offset = host_tsc - expected_tsc; 2646 2647 /* Determine if TSC is from Host or Guest */ 2648 if (data->pip) { 2649 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2650 if (!vmcs_info) { 2651 /* PIP NR=1 without VMCS cannot happen */ 2652 p_log("ERROR: Missing VMCS"); 2653 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2654 decoder->vm_tm_corr_reliable = false; 2655 return; 2656 } 2657 } else { /* Host */ 2658 decoder->last_reliable_timestamp = host_tsc; 2659 decoder->vm_tm_corr_reliable = true; 2660 return; 2661 } 2662 } else { /* Host or Guest */ 2663 reliable = false; /* Host/Guest is a guess, so not reliable */ 2664 if (decoder->in_psb) { 2665 if (!tsc_offset) 2666 return; /* Zero TSC Offset, assume Host */ 2667 /* 2668 * TSC packet has only 7 bytes of TSC. We have no 2669 * information about the Guest's 8th byte, but it 2670 * doesn't matter because we only need 7 bytes. 2671 * Here, since the 8th byte is unreliable and 2672 * irrelevant, compare only 7 byes. 2673 */ 2674 if (vmcs_info && 2675 (tsc_offset & SEVEN_BYTES) == 2676 (vmcs_info->tsc_offset & SEVEN_BYTES)) { 2677 /* Same TSC Offset as last VMCS, assume Guest */ 2678 goto guest; 2679 } 2680 } 2681 /* 2682 * Check if the host_tsc is within the expected range. 2683 * Note, we could narrow the range more by looking ahead for 2684 * the next host TSC in the same buffer, but we don't bother to 2685 * do that because this is probably good enough. 2686 */ 2687 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) { 2688 /* Within expected range for Host TSC, assume Host */ 2689 decoder->vm_tm_corr_reliable = false; 2690 return; 2691 } 2692 } 2693 2694 guest: /* Assuming Guest */ 2695 2696 /* Determine whether to assign TSC Offset */ 2697 if (vmcs_info && vmcs_info->vmcs) { 2698 if (vmcs_info->tsc_offset && vmcs_info->reliable) { 2699 assign = false; 2700 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && 2701 decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { 2702 /* Continuous tracing, TSC in a PSB is not a time loss */ 2703 assign = true; 2704 assign_reliable = true; 2705 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { 2706 /* 2707 * Unlikely to be a time loss TSC in a PSB which is not 2708 * at the start of a buffer. 2709 */ 2710 assign = true; 2711 assign_reliable = false; 2712 } 2713 } 2714 2715 /* Record VMCS TSC Offset */ 2716 if (assign && (vmcs_info->tsc_offset != tsc_offset || 2717 vmcs_info->reliable != assign_reliable)) { 2718 bool print = vmcs_info->tsc_offset != tsc_offset; 2719 2720 vmcs_info->tsc_offset = tsc_offset; 2721 vmcs_info->reliable = assign_reliable; 2722 if (print) 2723 intel_pt_print_vmcs_info(vmcs_info); 2724 } 2725 2726 /* Determine what TSC Offset to use */ 2727 if (vmcs_info && vmcs_info->tsc_offset) { 2728 if (!vmcs_info->reliable) 2729 reliable = false; 2730 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2731 } else { 2732 reliable = false; 2733 if (vmcs_info) { 2734 if (!vmcs_info->error_printed) { 2735 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64, 2736 vmcs_info->vmcs); 2737 vmcs_info->error_printed = true; 2738 } 2739 } else { 2740 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2741 p_log("ERROR: Unknown VMCS"); 2742 } 2743 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2744 } 2745 2746 decoder->vm_tm_corr_reliable = reliable; 2747 } 2748 2749 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder) 2750 { 2751 uint64_t host_tsc = decoder->packet.payload; 2752 uint64_t guest_tsc = decoder->packet.payload; 2753 struct intel_pt_vmcs_info *vmcs_info; 2754 uint64_t vmcs; 2755 2756 vmcs = decoder->vmcs; 2757 if (vmcs == NO_VMCS) 2758 vmcs = 0; 2759 2760 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2761 2762 if (decoder->pge) { 2763 if (in_vm(decoder->pip_payload)) { /* Guest */ 2764 if (!vmcs_info) { 2765 /* PIP NR=1 without VMCS cannot happen */ 2766 p_log("ERROR: Missing VMCS"); 2767 } 2768 } else { /* Host */ 2769 return; 2770 } 2771 } else { /* Host or Guest */ 2772 if (intel_pt_time_in_range(decoder, host_tsc)) { 2773 /* Within expected range for Host TSC, assume Host */ 2774 return; 2775 } 2776 } 2777 2778 if (vmcs_info) { 2779 /* Translate Guest TSC to Host TSC */ 2780 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; 2781 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); 2782 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2783 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2784 guest_tsc, host_tsc, vmcs_info->vmcs, 2785 vmcs_info->tsc_offset); 2786 if (!intel_pt_time_in_range(decoder, host_tsc) && 2787 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 2788 p_log("Timestamp out of range"); 2789 } else { 2790 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2791 p_log("ERROR: Unknown VMCS"); 2792 host_tsc = decoder->timestamp; 2793 } 2794 2795 decoder->packet.payload = host_tsc; 2796 2797 if (!decoder->vm_tm_corr_dry_run) 2798 memcpy((void *)decoder->buf + 1, &host_tsc, 8); 2799 } 2800 2801 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder) 2802 { 2803 struct intel_pt_vm_tsc_info data = { .psbend = false }; 2804 bool pge; 2805 int err; 2806 2807 if (decoder->in_psb) 2808 intel_pt_vm_tm_corr_psb(decoder, &data); 2809 2810 while (1) { 2811 err = intel_pt_get_next_packet(decoder); 2812 if (err == -ENOLINK) 2813 continue; 2814 if (err) 2815 break; 2816 2817 switch (decoder->packet.type) { 2818 case INTEL_PT_TIP_PGD: 2819 decoder->pge = false; 2820 decoder->vm_tm_corr_continuous = false; 2821 break; 2822 2823 case INTEL_PT_TNT: 2824 case INTEL_PT_TIP: 2825 case INTEL_PT_TIP_PGE: 2826 decoder->pge = true; 2827 break; 2828 2829 case INTEL_PT_OVF: 2830 decoder->in_psb = false; 2831 pge = decoder->pge; 2832 decoder->pge = intel_pt_ovf_fup_lookahead(decoder); 2833 if (pge != decoder->pge) 2834 intel_pt_log("Surprising PGE change in OVF!"); 2835 if (!decoder->pge) 2836 decoder->vm_tm_corr_continuous = false; 2837 break; 2838 2839 case INTEL_PT_FUP: 2840 if (decoder->in_psb) 2841 decoder->pge = true; 2842 break; 2843 2844 case INTEL_PT_TRACESTOP: 2845 decoder->pge = false; 2846 decoder->vm_tm_corr_continuous = false; 2847 decoder->have_tma = false; 2848 break; 2849 2850 case INTEL_PT_PSB: 2851 intel_pt_vm_tm_corr_psb(decoder, &data); 2852 break; 2853 2854 case INTEL_PT_PIP: 2855 decoder->pip_payload = decoder->packet.payload; 2856 break; 2857 2858 case INTEL_PT_MTC: 2859 intel_pt_calc_mtc_timestamp(decoder); 2860 break; 2861 2862 case INTEL_PT_TSC: 2863 intel_pt_vm_tm_corr_tsc(decoder, &data); 2864 intel_pt_calc_tsc_timestamp(decoder); 2865 decoder->vm_tm_corr_same_buf = true; 2866 decoder->vm_tm_corr_continuous = decoder->pge; 2867 break; 2868 2869 case INTEL_PT_TMA: 2870 intel_pt_calc_tma(decoder); 2871 break; 2872 2873 case INTEL_PT_CYC: 2874 intel_pt_calc_cyc_timestamp(decoder); 2875 break; 2876 2877 case INTEL_PT_CBR: 2878 intel_pt_calc_cbr(decoder); 2879 break; 2880 2881 case INTEL_PT_PSBEND: 2882 decoder->in_psb = false; 2883 data.psbend = false; 2884 break; 2885 2886 case INTEL_PT_VMCS: 2887 if (decoder->packet.payload != NO_VMCS) 2888 decoder->vmcs = decoder->packet.payload; 2889 break; 2890 2891 case INTEL_PT_BBP: 2892 decoder->blk_type = decoder->packet.payload; 2893 break; 2894 2895 case INTEL_PT_BIP: 2896 if (decoder->blk_type == INTEL_PT_PEBS_BASIC && 2897 decoder->packet.count == 2) 2898 intel_pt_vm_tm_corr_pebs_tsc(decoder); 2899 break; 2900 2901 case INTEL_PT_BEP: 2902 case INTEL_PT_BEP_IP: 2903 decoder->blk_type = 0; 2904 break; 2905 2906 case INTEL_PT_CFE: 2907 case INTEL_PT_CFE_IP: 2908 case INTEL_PT_EVD: 2909 case INTEL_PT_MODE_EXEC: 2910 case INTEL_PT_MODE_TSX: 2911 case INTEL_PT_MNT: 2912 case INTEL_PT_PAD: 2913 case INTEL_PT_PTWRITE_IP: 2914 case INTEL_PT_PTWRITE: 2915 case INTEL_PT_MWAIT: 2916 case INTEL_PT_PWRE: 2917 case INTEL_PT_EXSTOP_IP: 2918 case INTEL_PT_EXSTOP: 2919 case INTEL_PT_PWRX: 2920 case INTEL_PT_BAD: /* Does not happen */ 2921 default: 2922 break; 2923 } 2924 } 2925 2926 return err; 2927 } 2928 2929 #define HOP_PROCESS 0 2930 #define HOP_IGNORE 1 2931 #define HOP_RETURN 2 2932 #define HOP_AGAIN 3 2933 2934 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 2935 2936 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 2937 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 2938 { 2939 *err = 0; 2940 2941 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 2942 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 2943 *err = intel_pt_scan_for_psb(decoder); 2944 if (*err) 2945 return HOP_RETURN; 2946 } 2947 2948 switch (decoder->packet.type) { 2949 case INTEL_PT_TNT: 2950 return HOP_IGNORE; 2951 2952 case INTEL_PT_TIP_PGD: 2953 decoder->pge = false; 2954 if (!decoder->packet.count) { 2955 intel_pt_set_nr(decoder); 2956 return HOP_IGNORE; 2957 } 2958 intel_pt_set_ip(decoder); 2959 decoder->state.type |= INTEL_PT_TRACE_END; 2960 decoder->state.from_ip = 0; 2961 decoder->state.to_ip = decoder->ip; 2962 intel_pt_update_nr(decoder); 2963 return HOP_RETURN; 2964 2965 case INTEL_PT_TIP: 2966 if (!decoder->packet.count) { 2967 intel_pt_set_nr(decoder); 2968 return HOP_IGNORE; 2969 } 2970 intel_pt_set_ip(decoder); 2971 decoder->state.type = INTEL_PT_INSTRUCTION; 2972 decoder->state.from_ip = decoder->ip; 2973 decoder->state.to_ip = 0; 2974 intel_pt_update_nr(decoder); 2975 intel_pt_sample_iflag_chg(decoder); 2976 return HOP_RETURN; 2977 2978 case INTEL_PT_FUP: 2979 if (!decoder->packet.count) 2980 return HOP_IGNORE; 2981 intel_pt_set_ip(decoder); 2982 if (decoder->set_fup_mwait || decoder->set_fup_pwre) 2983 *no_tip = true; 2984 if (!decoder->branch_enable || !decoder->pge) 2985 *no_tip = true; 2986 if (*no_tip) { 2987 decoder->state.type = INTEL_PT_INSTRUCTION; 2988 decoder->state.from_ip = decoder->ip; 2989 decoder->state.to_ip = 0; 2990 intel_pt_fup_event(decoder, *no_tip); 2991 return HOP_RETURN; 2992 } 2993 intel_pt_fup_event(decoder, *no_tip); 2994 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH; 2995 *err = intel_pt_walk_fup_tip(decoder); 2996 if (!*err && decoder->state.to_ip) 2997 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2998 return HOP_RETURN; 2999 3000 case INTEL_PT_PSB: 3001 decoder->state.psb_offset = decoder->pos; 3002 decoder->psb_ip = 0; 3003 decoder->last_ip = 0; 3004 decoder->have_last_ip = true; 3005 *err = intel_pt_walk_psbend(decoder); 3006 if (*err == -EAGAIN) 3007 return HOP_AGAIN; 3008 if (*err) 3009 return HOP_RETURN; 3010 decoder->state.type = INTEL_PT_PSB_EVT; 3011 if (decoder->psb_ip) { 3012 decoder->state.type |= INTEL_PT_INSTRUCTION; 3013 decoder->ip = decoder->psb_ip; 3014 } 3015 decoder->state.from_ip = decoder->psb_ip; 3016 decoder->state.to_ip = 0; 3017 return HOP_RETURN; 3018 3019 case INTEL_PT_BAD: 3020 case INTEL_PT_PAD: 3021 case INTEL_PT_TIP_PGE: 3022 case INTEL_PT_TSC: 3023 case INTEL_PT_TMA: 3024 case INTEL_PT_MODE_EXEC: 3025 case INTEL_PT_MODE_TSX: 3026 case INTEL_PT_MTC: 3027 case INTEL_PT_CYC: 3028 case INTEL_PT_VMCS: 3029 case INTEL_PT_PSBEND: 3030 case INTEL_PT_CBR: 3031 case INTEL_PT_TRACESTOP: 3032 case INTEL_PT_PIP: 3033 case INTEL_PT_OVF: 3034 case INTEL_PT_MNT: 3035 case INTEL_PT_PTWRITE: 3036 case INTEL_PT_PTWRITE_IP: 3037 case INTEL_PT_EXSTOP: 3038 case INTEL_PT_EXSTOP_IP: 3039 case INTEL_PT_MWAIT: 3040 case INTEL_PT_PWRE: 3041 case INTEL_PT_PWRX: 3042 case INTEL_PT_BBP: 3043 case INTEL_PT_BIP: 3044 case INTEL_PT_BEP: 3045 case INTEL_PT_BEP_IP: 3046 case INTEL_PT_CFE: 3047 case INTEL_PT_CFE_IP: 3048 case INTEL_PT_EVD: 3049 default: 3050 return HOP_PROCESS; 3051 } 3052 } 3053 3054 struct intel_pt_psb_info { 3055 struct intel_pt_pkt fup_packet; 3056 bool fup; 3057 int after_psbend; 3058 }; 3059 3060 /* Lookahead and get the FUP packet from PSB+ */ 3061 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 3062 { 3063 struct intel_pt_psb_info *data = pkt_info->data; 3064 3065 switch (pkt_info->packet.type) { 3066 case INTEL_PT_PAD: 3067 case INTEL_PT_MNT: 3068 case INTEL_PT_TSC: 3069 case INTEL_PT_TMA: 3070 case INTEL_PT_MODE_EXEC: 3071 case INTEL_PT_MODE_TSX: 3072 case INTEL_PT_MTC: 3073 case INTEL_PT_CYC: 3074 case INTEL_PT_VMCS: 3075 case INTEL_PT_CBR: 3076 case INTEL_PT_PIP: 3077 if (data->after_psbend) { 3078 data->after_psbend -= 1; 3079 if (!data->after_psbend) 3080 return 1; 3081 } 3082 break; 3083 3084 case INTEL_PT_FUP: 3085 if (data->after_psbend) 3086 return 1; 3087 if (data->fup || pkt_info->packet.count == 0) 3088 return 1; 3089 data->fup_packet = pkt_info->packet; 3090 data->fup = true; 3091 break; 3092 3093 case INTEL_PT_PSBEND: 3094 if (!data->fup) 3095 return 1; 3096 /* Keep going to check for a TIP.PGE */ 3097 data->after_psbend = 6; 3098 break; 3099 3100 case INTEL_PT_TIP_PGE: 3101 /* Ignore FUP in PSB+ if followed by TIP.PGE */ 3102 if (data->after_psbend) 3103 data->fup = false; 3104 return 1; 3105 3106 case INTEL_PT_PTWRITE: 3107 case INTEL_PT_PTWRITE_IP: 3108 case INTEL_PT_EXSTOP: 3109 case INTEL_PT_EXSTOP_IP: 3110 case INTEL_PT_MWAIT: 3111 case INTEL_PT_PWRE: 3112 case INTEL_PT_PWRX: 3113 case INTEL_PT_BBP: 3114 case INTEL_PT_BIP: 3115 case INTEL_PT_BEP: 3116 case INTEL_PT_BEP_IP: 3117 case INTEL_PT_CFE: 3118 case INTEL_PT_CFE_IP: 3119 case INTEL_PT_EVD: 3120 if (data->after_psbend) { 3121 data->after_psbend -= 1; 3122 if (!data->after_psbend) 3123 return 1; 3124 break; 3125 } 3126 return 1; 3127 3128 case INTEL_PT_OVF: 3129 case INTEL_PT_BAD: 3130 case INTEL_PT_TNT: 3131 case INTEL_PT_TIP_PGD: 3132 case INTEL_PT_TIP: 3133 case INTEL_PT_PSB: 3134 case INTEL_PT_TRACESTOP: 3135 default: 3136 return 1; 3137 } 3138 3139 return 0; 3140 } 3141 3142 static int intel_pt_psb(struct intel_pt_decoder *decoder) 3143 { 3144 int err; 3145 3146 decoder->last_ip = 0; 3147 decoder->psb_ip = 0; 3148 decoder->have_last_ip = true; 3149 intel_pt_clear_stack(&decoder->stack); 3150 err = intel_pt_walk_psbend(decoder); 3151 if (err) 3152 return err; 3153 decoder->state.type = INTEL_PT_PSB_EVT; 3154 decoder->state.from_ip = decoder->psb_ip; 3155 decoder->state.to_ip = 0; 3156 return 0; 3157 } 3158 3159 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder) 3160 { 3161 int err; 3162 3163 if (decoder->ip != decoder->last_ip) { 3164 err = intel_pt_walk_fup(decoder); 3165 if (!err || err != -EAGAIN) 3166 return err; 3167 } 3168 3169 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3170 err = intel_pt_psb(decoder); 3171 if (err) { 3172 decoder->pkt_state = INTEL_PT_STATE_ERR3; 3173 return -ENOENT; 3174 } 3175 3176 return 0; 3177 } 3178 3179 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err) 3180 { 3181 struct intel_pt_psb_info data = { .fup = false }; 3182 3183 if (!decoder->branch_enable) 3184 return false; 3185 3186 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); 3187 if (!data.fup) 3188 return false; 3189 3190 decoder->packet = data.fup_packet; 3191 intel_pt_set_last_ip(decoder); 3192 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; 3193 3194 *err = intel_pt_fup_in_psb(decoder); 3195 3196 return true; 3197 } 3198 3199 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 3200 { 3201 int last_packet_type = INTEL_PT_PAD; 3202 bool no_tip = false; 3203 int err; 3204 3205 while (1) { 3206 err = intel_pt_get_next_packet(decoder); 3207 if (err) 3208 return err; 3209 next: 3210 err = 0; 3211 if (decoder->cyc_threshold) { 3212 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 3213 decoder->sample_cyc = false; 3214 last_packet_type = decoder->packet.type; 3215 } 3216 3217 if (decoder->hop) { 3218 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 3219 case HOP_IGNORE: 3220 continue; 3221 case HOP_RETURN: 3222 return err; 3223 case HOP_AGAIN: 3224 goto next; 3225 default: 3226 break; 3227 } 3228 } 3229 3230 switch (decoder->packet.type) { 3231 case INTEL_PT_TNT: 3232 if (!decoder->packet.count) 3233 break; 3234 decoder->tnt = decoder->packet; 3235 decoder->pkt_state = INTEL_PT_STATE_TNT; 3236 err = intel_pt_walk_tnt(decoder); 3237 if (err == -EAGAIN) 3238 break; 3239 return err; 3240 3241 case INTEL_PT_TIP_PGD: 3242 if (decoder->packet.count != 0) 3243 intel_pt_set_last_ip(decoder); 3244 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 3245 return intel_pt_walk_tip(decoder); 3246 3247 case INTEL_PT_TIP_PGE: { 3248 decoder->pge = true; 3249 decoder->overflow = false; 3250 intel_pt_mtc_cyc_cnt_pge(decoder); 3251 intel_pt_set_nr(decoder); 3252 if (decoder->packet.count == 0) { 3253 intel_pt_log_at("Skipping zero TIP.PGE", 3254 decoder->pos); 3255 break; 3256 } 3257 intel_pt_sample_iflag_chg(decoder); 3258 intel_pt_set_ip(decoder); 3259 decoder->state.from_ip = 0; 3260 decoder->state.to_ip = decoder->ip; 3261 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3262 /* 3263 * In hop mode, resample to get the to_ip as an 3264 * "instruction" sample. 3265 */ 3266 if (decoder->hop) 3267 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3268 return 0; 3269 } 3270 3271 case INTEL_PT_OVF: 3272 return intel_pt_overflow(decoder); 3273 3274 case INTEL_PT_TIP: 3275 if (decoder->packet.count != 0) 3276 intel_pt_set_last_ip(decoder); 3277 decoder->pkt_state = INTEL_PT_STATE_TIP; 3278 return intel_pt_walk_tip(decoder); 3279 3280 case INTEL_PT_FUP: 3281 if (decoder->packet.count == 0) { 3282 intel_pt_log_at("Skipping zero FUP", 3283 decoder->pos); 3284 no_tip = false; 3285 break; 3286 } 3287 intel_pt_set_last_ip(decoder); 3288 if (!decoder->branch_enable || !decoder->pge) { 3289 decoder->ip = decoder->last_ip; 3290 if (intel_pt_fup_event(decoder, no_tip)) 3291 return 0; 3292 no_tip = false; 3293 break; 3294 } 3295 if (decoder->set_fup_mwait) 3296 no_tip = true; 3297 if (no_tip) 3298 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 3299 else 3300 decoder->pkt_state = INTEL_PT_STATE_FUP; 3301 err = intel_pt_walk_fup(decoder); 3302 if (err != -EAGAIN) 3303 return err; 3304 if (no_tip) { 3305 no_tip = false; 3306 break; 3307 } 3308 return intel_pt_walk_fup_tip(decoder); 3309 3310 case INTEL_PT_TRACESTOP: 3311 decoder->pge = false; 3312 decoder->continuous_period = false; 3313 intel_pt_clear_tx_flags(decoder); 3314 decoder->have_tma = false; 3315 break; 3316 3317 case INTEL_PT_PSB: 3318 decoder->state.psb_offset = decoder->pos; 3319 decoder->psb_ip = 0; 3320 if (intel_pt_psb_with_fup(decoder, &err)) 3321 return err; 3322 err = intel_pt_psb(decoder); 3323 if (err == -EAGAIN) 3324 goto next; 3325 return err; 3326 3327 case INTEL_PT_PIP: 3328 intel_pt_update_pip(decoder); 3329 break; 3330 3331 case INTEL_PT_MTC: 3332 intel_pt_calc_mtc_timestamp(decoder); 3333 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 3334 break; 3335 /* 3336 * Ensure that there has been an instruction since the 3337 * last MTC. 3338 */ 3339 if (!decoder->mtc_insn) 3340 break; 3341 decoder->mtc_insn = false; 3342 /* Ensure that there is a timestamp */ 3343 if (!decoder->timestamp) 3344 break; 3345 decoder->state.type = INTEL_PT_INSTRUCTION; 3346 decoder->state.from_ip = decoder->ip; 3347 decoder->state.to_ip = 0; 3348 decoder->mtc_insn = false; 3349 return 0; 3350 3351 case INTEL_PT_TSC: 3352 intel_pt_calc_tsc_timestamp(decoder); 3353 break; 3354 3355 case INTEL_PT_TMA: 3356 intel_pt_calc_tma(decoder); 3357 break; 3358 3359 case INTEL_PT_CYC: 3360 intel_pt_calc_cyc_timestamp(decoder); 3361 break; 3362 3363 case INTEL_PT_CBR: 3364 intel_pt_calc_cbr(decoder); 3365 if (decoder->cbr != decoder->cbr_seen) { 3366 decoder->state.type = 0; 3367 return 0; 3368 } 3369 break; 3370 3371 case INTEL_PT_MODE_EXEC: 3372 intel_pt_mode_exec(decoder); 3373 err = intel_pt_get_next_packet(decoder); 3374 if (err) 3375 return err; 3376 if (decoder->packet.type == INTEL_PT_FUP) { 3377 decoder->set_fup_mode_exec = true; 3378 no_tip = true; 3379 } 3380 goto next; 3381 3382 case INTEL_PT_MODE_TSX: 3383 /* MODE_TSX need not be followed by FUP */ 3384 if (!decoder->pge || decoder->in_psb) { 3385 intel_pt_update_in_tx(decoder); 3386 break; 3387 } 3388 err = intel_pt_mode_tsx(decoder, &no_tip); 3389 if (err) 3390 return err; 3391 goto next; 3392 3393 case INTEL_PT_BAD: /* Does not happen */ 3394 return intel_pt_bug(decoder); 3395 3396 case INTEL_PT_PSBEND: 3397 case INTEL_PT_VMCS: 3398 case INTEL_PT_MNT: 3399 case INTEL_PT_PAD: 3400 break; 3401 3402 case INTEL_PT_PTWRITE_IP: 3403 decoder->fup_ptw_payload = decoder->packet.payload; 3404 err = intel_pt_get_next_packet(decoder); 3405 if (err) 3406 return err; 3407 if (decoder->packet.type == INTEL_PT_FUP) { 3408 decoder->set_fup_ptw = true; 3409 no_tip = true; 3410 } else { 3411 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 3412 decoder->pos); 3413 } 3414 goto next; 3415 3416 case INTEL_PT_PTWRITE: 3417 decoder->state.type = INTEL_PT_PTW; 3418 decoder->state.from_ip = decoder->ip; 3419 decoder->state.to_ip = 0; 3420 decoder->state.ptw_payload = decoder->packet.payload; 3421 return 0; 3422 3423 case INTEL_PT_MWAIT: 3424 decoder->fup_mwait_payload = decoder->packet.payload; 3425 decoder->set_fup_mwait = true; 3426 break; 3427 3428 case INTEL_PT_PWRE: 3429 if (decoder->set_fup_mwait) { 3430 decoder->fup_pwre_payload = 3431 decoder->packet.payload; 3432 decoder->set_fup_pwre = true; 3433 break; 3434 } 3435 decoder->state.type = INTEL_PT_PWR_ENTRY; 3436 decoder->state.from_ip = decoder->ip; 3437 decoder->state.to_ip = 0; 3438 decoder->state.pwrx_payload = decoder->packet.payload; 3439 return 0; 3440 3441 case INTEL_PT_EXSTOP_IP: 3442 err = intel_pt_get_next_packet(decoder); 3443 if (err) 3444 return err; 3445 if (decoder->packet.type == INTEL_PT_FUP) { 3446 decoder->set_fup_exstop = true; 3447 no_tip = true; 3448 } else { 3449 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 3450 decoder->pos); 3451 } 3452 goto next; 3453 3454 case INTEL_PT_EXSTOP: 3455 decoder->state.type = INTEL_PT_EX_STOP; 3456 decoder->state.from_ip = decoder->ip; 3457 decoder->state.to_ip = 0; 3458 return 0; 3459 3460 case INTEL_PT_PWRX: 3461 decoder->state.type = INTEL_PT_PWR_EXIT; 3462 decoder->state.from_ip = decoder->ip; 3463 decoder->state.to_ip = 0; 3464 decoder->state.pwrx_payload = decoder->packet.payload; 3465 return 0; 3466 3467 case INTEL_PT_BBP: 3468 intel_pt_bbp(decoder); 3469 break; 3470 3471 case INTEL_PT_BIP: 3472 intel_pt_bip(decoder); 3473 break; 3474 3475 case INTEL_PT_BEP: 3476 decoder->state.type = INTEL_PT_BLK_ITEMS; 3477 decoder->state.from_ip = decoder->ip; 3478 decoder->state.to_ip = 0; 3479 return 0; 3480 3481 case INTEL_PT_BEP_IP: 3482 err = intel_pt_get_next_packet(decoder); 3483 if (err) 3484 return err; 3485 if (decoder->packet.type == INTEL_PT_FUP) { 3486 decoder->set_fup_bep = true; 3487 no_tip = true; 3488 } else { 3489 intel_pt_log_at("ERROR: Missing FUP after BEP", 3490 decoder->pos); 3491 } 3492 goto next; 3493 3494 case INTEL_PT_CFE: 3495 decoder->fup_cfe_pkt = decoder->packet; 3496 decoder->set_fup_cfe = true; 3497 if (!decoder->pge) { 3498 intel_pt_fup_event(decoder, true); 3499 return 0; 3500 } 3501 break; 3502 3503 case INTEL_PT_CFE_IP: 3504 decoder->fup_cfe_pkt = decoder->packet; 3505 err = intel_pt_get_next_packet(decoder); 3506 if (err) 3507 return err; 3508 if (decoder->packet.type == INTEL_PT_FUP) { 3509 decoder->set_fup_cfe_ip = true; 3510 no_tip = true; 3511 } else { 3512 intel_pt_log_at("ERROR: Missing FUP after CFE", 3513 decoder->pos); 3514 } 3515 goto next; 3516 3517 case INTEL_PT_EVD: 3518 err = intel_pt_evd(decoder); 3519 if (err) 3520 return err; 3521 break; 3522 3523 default: 3524 return intel_pt_bug(decoder); 3525 } 3526 } 3527 } 3528 3529 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 3530 { 3531 return decoder->packet.count && 3532 (decoder->have_last_ip || decoder->packet.count == 3 || 3533 decoder->packet.count == 6); 3534 } 3535 3536 /* Walk PSB+ packets to get in sync. */ 3537 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 3538 { 3539 int err; 3540 3541 decoder->in_psb = true; 3542 3543 while (1) { 3544 err = intel_pt_get_next_packet(decoder); 3545 if (err) 3546 goto out; 3547 3548 switch (decoder->packet.type) { 3549 case INTEL_PT_TIP_PGD: 3550 decoder->continuous_period = false; 3551 fallthrough; 3552 case INTEL_PT_TIP_PGE: 3553 case INTEL_PT_TIP: 3554 case INTEL_PT_PTWRITE: 3555 case INTEL_PT_PTWRITE_IP: 3556 case INTEL_PT_EXSTOP: 3557 case INTEL_PT_EXSTOP_IP: 3558 case INTEL_PT_MWAIT: 3559 case INTEL_PT_PWRE: 3560 case INTEL_PT_PWRX: 3561 case INTEL_PT_BBP: 3562 case INTEL_PT_BIP: 3563 case INTEL_PT_BEP: 3564 case INTEL_PT_BEP_IP: 3565 case INTEL_PT_CFE: 3566 case INTEL_PT_CFE_IP: 3567 case INTEL_PT_EVD: 3568 intel_pt_log("ERROR: Unexpected packet\n"); 3569 err = -ENOENT; 3570 goto out; 3571 3572 case INTEL_PT_FUP: 3573 decoder->pge = true; 3574 if (intel_pt_have_ip(decoder)) { 3575 uint64_t current_ip = decoder->ip; 3576 3577 intel_pt_set_ip(decoder); 3578 decoder->psb_ip = decoder->ip; 3579 if (current_ip) 3580 intel_pt_log_to("Setting IP", 3581 decoder->ip); 3582 } 3583 break; 3584 3585 case INTEL_PT_MTC: 3586 intel_pt_calc_mtc_timestamp(decoder); 3587 break; 3588 3589 case INTEL_PT_TSC: 3590 intel_pt_calc_tsc_timestamp(decoder); 3591 break; 3592 3593 case INTEL_PT_TMA: 3594 intel_pt_calc_tma(decoder); 3595 break; 3596 3597 case INTEL_PT_CYC: 3598 intel_pt_calc_cyc_timestamp(decoder); 3599 break; 3600 3601 case INTEL_PT_CBR: 3602 intel_pt_calc_cbr(decoder); 3603 break; 3604 3605 case INTEL_PT_PIP: 3606 intel_pt_set_pip(decoder); 3607 break; 3608 3609 case INTEL_PT_MODE_EXEC: 3610 intel_pt_mode_exec_status(decoder); 3611 break; 3612 3613 case INTEL_PT_MODE_TSX: 3614 intel_pt_update_in_tx(decoder); 3615 break; 3616 3617 case INTEL_PT_TRACESTOP: 3618 decoder->pge = false; 3619 decoder->continuous_period = false; 3620 intel_pt_clear_tx_flags(decoder); 3621 fallthrough; 3622 3623 case INTEL_PT_TNT: 3624 decoder->have_tma = false; 3625 intel_pt_log("ERROR: Unexpected packet\n"); 3626 if (decoder->ip) 3627 decoder->pkt_state = INTEL_PT_STATE_ERR4; 3628 else 3629 decoder->pkt_state = INTEL_PT_STATE_ERR3; 3630 err = -ENOENT; 3631 goto out; 3632 3633 case INTEL_PT_BAD: /* Does not happen */ 3634 err = intel_pt_bug(decoder); 3635 goto out; 3636 3637 case INTEL_PT_OVF: 3638 err = intel_pt_overflow(decoder); 3639 goto out; 3640 3641 case INTEL_PT_PSBEND: 3642 err = 0; 3643 goto out; 3644 3645 case INTEL_PT_PSB: 3646 case INTEL_PT_VMCS: 3647 case INTEL_PT_MNT: 3648 case INTEL_PT_PAD: 3649 default: 3650 break; 3651 } 3652 } 3653 out: 3654 decoder->in_psb = false; 3655 3656 return err; 3657 } 3658 3659 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 3660 { 3661 int err; 3662 3663 while (1) { 3664 err = intel_pt_get_next_packet(decoder); 3665 if (err) 3666 return err; 3667 3668 switch (decoder->packet.type) { 3669 case INTEL_PT_TIP_PGD: 3670 decoder->continuous_period = false; 3671 decoder->pge = false; 3672 if (intel_pt_have_ip(decoder)) 3673 intel_pt_set_ip(decoder); 3674 if (!decoder->ip) 3675 break; 3676 decoder->state.type |= INTEL_PT_TRACE_END; 3677 return 0; 3678 3679 case INTEL_PT_TIP_PGE: 3680 decoder->pge = true; 3681 intel_pt_mtc_cyc_cnt_pge(decoder); 3682 if (intel_pt_have_ip(decoder)) 3683 intel_pt_set_ip(decoder); 3684 if (!decoder->ip) 3685 break; 3686 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3687 return 0; 3688 3689 case INTEL_PT_TIP: 3690 decoder->pge = true; 3691 if (intel_pt_have_ip(decoder)) 3692 intel_pt_set_ip(decoder); 3693 if (!decoder->ip) 3694 break; 3695 return 0; 3696 3697 case INTEL_PT_FUP: 3698 if (intel_pt_have_ip(decoder)) 3699 intel_pt_set_ip(decoder); 3700 if (decoder->ip) 3701 return 0; 3702 break; 3703 3704 case INTEL_PT_MTC: 3705 intel_pt_calc_mtc_timestamp(decoder); 3706 break; 3707 3708 case INTEL_PT_TSC: 3709 intel_pt_calc_tsc_timestamp(decoder); 3710 break; 3711 3712 case INTEL_PT_TMA: 3713 intel_pt_calc_tma(decoder); 3714 break; 3715 3716 case INTEL_PT_CYC: 3717 intel_pt_calc_cyc_timestamp(decoder); 3718 break; 3719 3720 case INTEL_PT_CBR: 3721 intel_pt_calc_cbr(decoder); 3722 break; 3723 3724 case INTEL_PT_PIP: 3725 intel_pt_set_pip(decoder); 3726 break; 3727 3728 case INTEL_PT_MODE_EXEC: 3729 intel_pt_mode_exec_status(decoder); 3730 break; 3731 3732 case INTEL_PT_MODE_TSX: 3733 intel_pt_update_in_tx(decoder); 3734 break; 3735 3736 case INTEL_PT_OVF: 3737 return intel_pt_overflow(decoder); 3738 3739 case INTEL_PT_BAD: /* Does not happen */ 3740 return intel_pt_bug(decoder); 3741 3742 case INTEL_PT_TRACESTOP: 3743 decoder->pge = false; 3744 decoder->continuous_period = false; 3745 intel_pt_clear_tx_flags(decoder); 3746 decoder->have_tma = false; 3747 break; 3748 3749 case INTEL_PT_PSB: 3750 decoder->state.psb_offset = decoder->pos; 3751 decoder->psb_ip = 0; 3752 decoder->last_ip = 0; 3753 decoder->have_last_ip = true; 3754 intel_pt_clear_stack(&decoder->stack); 3755 err = intel_pt_walk_psb(decoder); 3756 if (err) 3757 return err; 3758 decoder->state.type = INTEL_PT_PSB_EVT; 3759 decoder->state.from_ip = decoder->psb_ip; 3760 decoder->state.to_ip = 0; 3761 return 0; 3762 3763 case INTEL_PT_TNT: 3764 case INTEL_PT_PSBEND: 3765 case INTEL_PT_VMCS: 3766 case INTEL_PT_MNT: 3767 case INTEL_PT_PAD: 3768 case INTEL_PT_PTWRITE: 3769 case INTEL_PT_PTWRITE_IP: 3770 case INTEL_PT_EXSTOP: 3771 case INTEL_PT_EXSTOP_IP: 3772 case INTEL_PT_MWAIT: 3773 case INTEL_PT_PWRE: 3774 case INTEL_PT_PWRX: 3775 case INTEL_PT_BBP: 3776 case INTEL_PT_BIP: 3777 case INTEL_PT_BEP: 3778 case INTEL_PT_BEP_IP: 3779 case INTEL_PT_CFE: 3780 case INTEL_PT_CFE_IP: 3781 case INTEL_PT_EVD: 3782 default: 3783 break; 3784 } 3785 } 3786 } 3787 3788 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 3789 { 3790 int err; 3791 3792 intel_pt_clear_fup_event(decoder); 3793 decoder->overflow = false; 3794 3795 if (!decoder->branch_enable) { 3796 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3797 decoder->state.type = 0; /* Do not have a sample */ 3798 return 0; 3799 } 3800 3801 intel_pt_log("Scanning for full IP\n"); 3802 err = intel_pt_walk_to_ip(decoder); 3803 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) 3804 return err; 3805 3806 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 3807 if (decoder->hop) 3808 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3809 else 3810 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3811 3812 decoder->state.from_ip = 0; 3813 decoder->state.to_ip = decoder->ip; 3814 intel_pt_log_to("Setting IP", decoder->ip); 3815 3816 return 0; 3817 } 3818 3819 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 3820 { 3821 const unsigned char *end = decoder->buf + decoder->len; 3822 size_t i; 3823 3824 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 3825 if (i > decoder->len) 3826 continue; 3827 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 3828 return i; 3829 } 3830 return 0; 3831 } 3832 3833 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 3834 { 3835 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 3836 const char *psb = INTEL_PT_PSB_STR; 3837 3838 if (rest_psb > decoder->len || 3839 memcmp(decoder->buf, psb + part_psb, rest_psb)) 3840 return 0; 3841 3842 return rest_psb; 3843 } 3844 3845 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 3846 int part_psb) 3847 { 3848 int rest_psb, ret; 3849 3850 decoder->pos += decoder->len; 3851 decoder->len = 0; 3852 3853 ret = intel_pt_get_next_data(decoder, false); 3854 if (ret) 3855 return ret; 3856 3857 rest_psb = intel_pt_rest_psb(decoder, part_psb); 3858 if (!rest_psb) 3859 return 0; 3860 3861 decoder->pos -= part_psb; 3862 decoder->next_buf = decoder->buf + rest_psb; 3863 decoder->next_len = decoder->len - rest_psb; 3864 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3865 decoder->buf = decoder->temp_buf; 3866 decoder->len = INTEL_PT_PSB_LEN; 3867 3868 return 0; 3869 } 3870 3871 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 3872 { 3873 unsigned char *next; 3874 int ret; 3875 3876 intel_pt_log("Scanning for PSB\n"); 3877 while (1) { 3878 if (!decoder->len) { 3879 ret = intel_pt_get_next_data(decoder, false); 3880 if (ret) 3881 return ret; 3882 } 3883 3884 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 3885 INTEL_PT_PSB_LEN); 3886 if (!next) { 3887 int part_psb; 3888 3889 part_psb = intel_pt_part_psb(decoder); 3890 if (part_psb) { 3891 ret = intel_pt_get_split_psb(decoder, part_psb); 3892 if (ret) 3893 return ret; 3894 } else { 3895 decoder->pos += decoder->len; 3896 decoder->len = 0; 3897 } 3898 continue; 3899 } 3900 3901 decoder->pkt_step = next - decoder->buf; 3902 return intel_pt_get_next_packet(decoder); 3903 } 3904 } 3905 3906 static int intel_pt_sync(struct intel_pt_decoder *decoder) 3907 { 3908 int err; 3909 3910 decoder->pge = false; 3911 decoder->continuous_period = false; 3912 decoder->have_last_ip = false; 3913 decoder->last_ip = 0; 3914 decoder->psb_ip = 0; 3915 decoder->ip = 0; 3916 intel_pt_clear_stack(&decoder->stack); 3917 3918 err = intel_pt_scan_for_psb(decoder); 3919 if (err) 3920 return err; 3921 3922 if (decoder->vm_time_correlation) { 3923 decoder->in_psb = true; 3924 if (!decoder->timestamp) 3925 decoder->timestamp = 1; 3926 decoder->state.type = 0; 3927 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; 3928 return 0; 3929 } 3930 3931 decoder->have_last_ip = true; 3932 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3933 3934 err = intel_pt_walk_psb(decoder); 3935 if (err) 3936 return err; 3937 3938 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ 3939 decoder->state.from_ip = decoder->psb_ip; 3940 decoder->state.to_ip = 0; 3941 3942 if (decoder->ip) { 3943 /* 3944 * In hop mode, resample to get the PSB FUP ip as an 3945 * "instruction" sample. 3946 */ 3947 if (decoder->hop) 3948 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3949 else 3950 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3951 } 3952 3953 return 0; 3954 } 3955 3956 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 3957 { 3958 uint64_t est = decoder->sample_insn_cnt << 1; 3959 3960 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 3961 goto out; 3962 3963 est *= decoder->max_non_turbo_ratio; 3964 est /= decoder->cbr; 3965 out: 3966 return decoder->sample_timestamp + est; 3967 } 3968 3969 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 3970 { 3971 int err; 3972 3973 do { 3974 decoder->state.type = INTEL_PT_BRANCH; 3975 decoder->state.flags = 0; 3976 3977 switch (decoder->pkt_state) { 3978 case INTEL_PT_STATE_NO_PSB: 3979 err = intel_pt_sync(decoder); 3980 break; 3981 case INTEL_PT_STATE_NO_IP: 3982 decoder->have_last_ip = false; 3983 decoder->last_ip = 0; 3984 decoder->ip = 0; 3985 fallthrough; 3986 case INTEL_PT_STATE_ERR_RESYNC: 3987 err = intel_pt_sync_ip(decoder); 3988 break; 3989 case INTEL_PT_STATE_IN_SYNC: 3990 err = intel_pt_walk_trace(decoder); 3991 break; 3992 case INTEL_PT_STATE_TNT: 3993 case INTEL_PT_STATE_TNT_CONT: 3994 err = intel_pt_walk_tnt(decoder); 3995 if (err == -EAGAIN) 3996 err = intel_pt_walk_trace(decoder); 3997 break; 3998 case INTEL_PT_STATE_TIP: 3999 case INTEL_PT_STATE_TIP_PGD: 4000 err = intel_pt_walk_tip(decoder); 4001 break; 4002 case INTEL_PT_STATE_FUP: 4003 err = intel_pt_walk_fup(decoder); 4004 if (err == -EAGAIN) 4005 err = intel_pt_walk_fup_tip(decoder); 4006 break; 4007 case INTEL_PT_STATE_FUP_NO_TIP: 4008 err = intel_pt_walk_fup(decoder); 4009 if (err == -EAGAIN) 4010 err = intel_pt_walk_trace(decoder); 4011 break; 4012 case INTEL_PT_STATE_FUP_IN_PSB: 4013 err = intel_pt_fup_in_psb(decoder); 4014 break; 4015 case INTEL_PT_STATE_RESAMPLE: 4016 err = intel_pt_resample(decoder); 4017 break; 4018 case INTEL_PT_STATE_VM_TIME_CORRELATION: 4019 err = intel_pt_vm_time_correlation(decoder); 4020 break; 4021 default: 4022 err = intel_pt_bug(decoder); 4023 break; 4024 } 4025 } while (err == -ENOLINK); 4026 4027 if (err) { 4028 decoder->state.err = intel_pt_ext_err(err); 4029 if (err != -EOVERFLOW) 4030 decoder->state.from_ip = decoder->ip; 4031 intel_pt_update_sample_time(decoder); 4032 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 4033 intel_pt_set_nr(decoder); 4034 } else { 4035 decoder->state.err = 0; 4036 if (decoder->cbr != decoder->cbr_seen) { 4037 decoder->cbr_seen = decoder->cbr; 4038 if (!decoder->state.type) { 4039 decoder->state.from_ip = decoder->ip; 4040 decoder->state.to_ip = 0; 4041 } 4042 decoder->state.type |= INTEL_PT_CBR_CHG; 4043 decoder->state.cbr_payload = decoder->cbr_payload; 4044 decoder->state.cbr = decoder->cbr; 4045 } 4046 if (intel_pt_sample_time(decoder->pkt_state)) { 4047 intel_pt_update_sample_time(decoder); 4048 if (decoder->sample_cyc) { 4049 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 4050 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 4051 decoder->sample_cyc = false; 4052 } 4053 } 4054 /* 4055 * When using only TSC/MTC to compute cycles, IPC can be 4056 * sampled as soon as the cycle count changes. 4057 */ 4058 if (!decoder->have_cyc) 4059 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 4060 } 4061 4062 /* Let PSB event always have TSC timestamp */ 4063 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) 4064 decoder->sample_timestamp = decoder->tsc_timestamp; 4065 4066 decoder->state.from_nr = decoder->nr; 4067 decoder->state.to_nr = decoder->next_nr; 4068 decoder->nr = decoder->next_nr; 4069 4070 decoder->state.timestamp = decoder->sample_timestamp; 4071 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 4072 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 4073 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 4074 4075 return &decoder->state; 4076 } 4077 4078 /** 4079 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 4080 * @buf: pointer to buffer pointer 4081 * @len: size of buffer 4082 * 4083 * Updates the buffer pointer to point to the start of the next PSB packet if 4084 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 4085 * @len is adjusted accordingly. 4086 * 4087 * Return: %true if a PSB packet is found, %false otherwise. 4088 */ 4089 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 4090 { 4091 unsigned char *next; 4092 4093 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 4094 if (next) { 4095 *len -= next - *buf; 4096 *buf = next; 4097 return true; 4098 } 4099 return false; 4100 } 4101 4102 /** 4103 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 4104 * packet. 4105 * @buf: pointer to buffer pointer 4106 * @len: size of buffer 4107 * 4108 * Updates the buffer pointer to point to the start of the following PSB packet 4109 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 4110 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 4111 * 4112 * Return: %true if a PSB packet is found, %false otherwise. 4113 */ 4114 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 4115 { 4116 unsigned char *next; 4117 4118 if (!*len) 4119 return false; 4120 4121 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 4122 if (next) { 4123 *len -= next - *buf; 4124 *buf = next; 4125 return true; 4126 } 4127 return false; 4128 } 4129 4130 /** 4131 * intel_pt_last_psb - find the last PSB packet in a buffer. 4132 * @buf: buffer 4133 * @len: size of buffer 4134 * 4135 * This function finds the last PSB in a buffer. 4136 * 4137 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 4138 */ 4139 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 4140 { 4141 const char *n = INTEL_PT_PSB_STR; 4142 unsigned char *p; 4143 size_t k; 4144 4145 if (len < INTEL_PT_PSB_LEN) 4146 return NULL; 4147 4148 k = len - INTEL_PT_PSB_LEN + 1; 4149 while (1) { 4150 p = memrchr(buf, n[0], k); 4151 if (!p) 4152 return NULL; 4153 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 4154 return p; 4155 k = p - buf; 4156 if (!k) 4157 return NULL; 4158 } 4159 } 4160 4161 /** 4162 * intel_pt_next_tsc - find and return next TSC. 4163 * @buf: buffer 4164 * @len: size of buffer 4165 * @tsc: TSC value returned 4166 * @rem: returns remaining size when TSC is found 4167 * 4168 * Find a TSC packet in @buf and return the TSC value. This function assumes 4169 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 4170 * PSBEND packet is found. 4171 * 4172 * Return: %true if TSC is found, false otherwise. 4173 */ 4174 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 4175 size_t *rem) 4176 { 4177 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 4178 struct intel_pt_pkt packet; 4179 int ret; 4180 4181 while (len) { 4182 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 4183 if (ret <= 0) 4184 return false; 4185 if (packet.type == INTEL_PT_TSC) { 4186 *tsc = packet.payload; 4187 *rem = len; 4188 return true; 4189 } 4190 if (packet.type == INTEL_PT_PSBEND) 4191 return false; 4192 buf += ret; 4193 len -= ret; 4194 } 4195 return false; 4196 } 4197 4198 /** 4199 * intel_pt_tsc_cmp - compare 7-byte TSCs. 4200 * @tsc1: first TSC to compare 4201 * @tsc2: second TSC to compare 4202 * 4203 * This function compares 7-byte TSC values allowing for the possibility that 4204 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 4205 * around so for that purpose this function assumes the absolute difference is 4206 * less than half the maximum difference. 4207 * 4208 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 4209 * after @tsc2. 4210 */ 4211 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 4212 { 4213 const uint64_t halfway = (1ULL << 55); 4214 4215 if (tsc1 == tsc2) 4216 return 0; 4217 4218 if (tsc1 < tsc2) { 4219 if (tsc2 - tsc1 < halfway) 4220 return -1; 4221 else 4222 return 1; 4223 } else { 4224 if (tsc1 - tsc2 < halfway) 4225 return 1; 4226 else 4227 return -1; 4228 } 4229 } 4230 4231 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 4232 4233 /** 4234 * adj_for_padding - adjust overlap to account for padding. 4235 * @buf_b: second buffer 4236 * @buf_a: first buffer 4237 * @len_a: size of first buffer 4238 * 4239 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 4240 * accordingly. 4241 * 4242 * Return: A pointer into @buf_b from where non-overlapped data starts 4243 */ 4244 static unsigned char *adj_for_padding(unsigned char *buf_b, 4245 unsigned char *buf_a, size_t len_a) 4246 { 4247 unsigned char *p = buf_b - MAX_PADDING; 4248 unsigned char *q = buf_a + len_a - MAX_PADDING; 4249 int i; 4250 4251 for (i = MAX_PADDING; i; i--, p++, q++) { 4252 if (*p != *q) 4253 break; 4254 } 4255 4256 return p; 4257 } 4258 4259 /** 4260 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 4261 * using TSC. 4262 * @buf_a: first buffer 4263 * @len_a: size of first buffer 4264 * @buf_b: second buffer 4265 * @len_b: size of second buffer 4266 * @consecutive: returns true if there is data in buf_b that is consecutive 4267 * to buf_a 4268 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 4269 * 4270 * If the trace contains TSC we can look at the last TSC of @buf_a and the 4271 * first TSC of @buf_b in order to determine if the buffers overlap, and then 4272 * walk forward in @buf_b until a later TSC is found. A precondition is that 4273 * @buf_a and @buf_b are positioned at a PSB. 4274 * 4275 * Return: A pointer into @buf_b from where non-overlapped data starts, or 4276 * @buf_b + @len_b if there is no non-overlapped data. 4277 */ 4278 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 4279 size_t len_a, 4280 unsigned char *buf_b, 4281 size_t len_b, bool *consecutive, 4282 bool ooo_tsc) 4283 { 4284 uint64_t tsc_a, tsc_b; 4285 unsigned char *p; 4286 size_t len, rem_a, rem_b; 4287 4288 p = intel_pt_last_psb(buf_a, len_a); 4289 if (!p) 4290 return buf_b; /* No PSB in buf_a => no overlap */ 4291 4292 len = len_a - (p - buf_a); 4293 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 4294 /* The last PSB+ in buf_a is incomplete, so go back one more */ 4295 len_a -= len; 4296 p = intel_pt_last_psb(buf_a, len_a); 4297 if (!p) 4298 return buf_b; /* No full PSB+ => assume no overlap */ 4299 len = len_a - (p - buf_a); 4300 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 4301 return buf_b; /* No TSC in buf_a => assume no overlap */ 4302 } 4303 4304 while (1) { 4305 /* Ignore PSB+ with no TSC */ 4306 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 4307 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 4308 4309 /* Same TSC, so buffers are consecutive */ 4310 if (!cmp && rem_b >= rem_a) { 4311 unsigned char *start; 4312 4313 *consecutive = true; 4314 start = buf_b + len_b - (rem_b - rem_a); 4315 return adj_for_padding(start, buf_a, len_a); 4316 } 4317 if (cmp < 0 && !ooo_tsc) 4318 return buf_b; /* tsc_a < tsc_b => no overlap */ 4319 } 4320 4321 if (!intel_pt_step_psb(&buf_b, &len_b)) 4322 return buf_b + len_b; /* No PSB in buf_b => no data */ 4323 } 4324 } 4325 4326 /** 4327 * intel_pt_find_overlap - determine start of non-overlapped trace data. 4328 * @buf_a: first buffer 4329 * @len_a: size of first buffer 4330 * @buf_b: second buffer 4331 * @len_b: size of second buffer 4332 * @have_tsc: can use TSC packets to detect overlap 4333 * @consecutive: returns true if there is data in buf_b that is consecutive 4334 * to buf_a 4335 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 4336 * 4337 * When trace samples or snapshots are recorded there is the possibility that 4338 * the data overlaps. Note that, for the purposes of decoding, data is only 4339 * useful if it begins with a PSB packet. 4340 * 4341 * Return: A pointer into @buf_b from where non-overlapped data starts, or 4342 * @buf_b + @len_b if there is no non-overlapped data. 4343 */ 4344 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 4345 unsigned char *buf_b, size_t len_b, 4346 bool have_tsc, bool *consecutive, 4347 bool ooo_tsc) 4348 { 4349 unsigned char *found; 4350 4351 /* Buffer 'b' must start at PSB so throw away everything before that */ 4352 if (!intel_pt_next_psb(&buf_b, &len_b)) 4353 return buf_b + len_b; /* No PSB */ 4354 4355 if (!intel_pt_next_psb(&buf_a, &len_a)) 4356 return buf_b; /* No overlap */ 4357 4358 if (have_tsc) { 4359 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 4360 consecutive, ooo_tsc); 4361 if (found) 4362 return found; 4363 } 4364 4365 /* 4366 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 4367 * we can ignore the first part of buffer 'a'. 4368 */ 4369 while (len_b < len_a) { 4370 if (!intel_pt_step_psb(&buf_a, &len_a)) 4371 return buf_b; /* No overlap */ 4372 } 4373 4374 /* Now len_b >= len_a */ 4375 while (1) { 4376 /* Potential overlap so check the bytes */ 4377 found = memmem(buf_a, len_a, buf_b, len_a); 4378 if (found) { 4379 *consecutive = true; 4380 return adj_for_padding(buf_b + len_a, buf_a, len_a); 4381 } 4382 4383 /* Try again at next PSB in buffer 'a' */ 4384 if (!intel_pt_step_psb(&buf_a, &len_a)) 4385 return buf_b; /* No overlap */ 4386 } 4387 } 4388 4389 /** 4390 * struct fast_forward_data - data used by intel_pt_ff_cb(). 4391 * @timestamp: timestamp to fast forward towards 4392 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 4393 * the fast forward timestamp. 4394 */ 4395 struct fast_forward_data { 4396 uint64_t timestamp; 4397 uint64_t buf_timestamp; 4398 }; 4399 4400 /** 4401 * intel_pt_ff_cb - fast forward lookahead callback. 4402 * @buffer: Intel PT trace buffer 4403 * @data: opaque pointer to fast forward data (struct fast_forward_data) 4404 * 4405 * Determine if @buffer trace is past the fast forward timestamp. 4406 * 4407 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 4408 * timestamp, and 0 otherwise. 4409 */ 4410 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 4411 { 4412 struct fast_forward_data *d = data; 4413 unsigned char *buf; 4414 uint64_t tsc; 4415 size_t rem; 4416 size_t len; 4417 4418 buf = (unsigned char *)buffer->buf; 4419 len = buffer->len; 4420 4421 if (!intel_pt_next_psb(&buf, &len) || 4422 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 4423 return 0; 4424 4425 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 4426 4427 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 4428 tsc, buffer->ref_timestamp); 4429 4430 /* 4431 * If the buffer contains a timestamp earlier that the fast forward 4432 * timestamp, then record it, else stop. 4433 */ 4434 if (tsc < d->timestamp) 4435 d->buf_timestamp = buffer->ref_timestamp; 4436 else 4437 return 1; 4438 4439 return 0; 4440 } 4441 4442 /** 4443 * intel_pt_fast_forward - reposition decoder forwards. 4444 * @decoder: Intel PT decoder 4445 * @timestamp: timestamp to fast forward towards 4446 * 4447 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 4448 * 4449 * Return: 0 on success or negative error code on failure. 4450 */ 4451 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 4452 { 4453 struct fast_forward_data d = { .timestamp = timestamp }; 4454 unsigned char *buf; 4455 size_t len; 4456 int err; 4457 4458 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 4459 4460 /* Find buffer timestamp of buffer to fast forward to */ 4461 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 4462 if (err < 0) 4463 return err; 4464 4465 /* Walk to buffer with same buffer timestamp */ 4466 if (d.buf_timestamp) { 4467 do { 4468 decoder->pos += decoder->len; 4469 decoder->len = 0; 4470 err = intel_pt_get_next_data(decoder, true); 4471 /* -ENOLINK means non-consecutive trace */ 4472 if (err && err != -ENOLINK) 4473 return err; 4474 } while (decoder->buf_timestamp != d.buf_timestamp); 4475 } 4476 4477 if (!decoder->buf) 4478 return 0; 4479 4480 buf = (unsigned char *)decoder->buf; 4481 len = decoder->len; 4482 4483 if (!intel_pt_next_psb(&buf, &len)) 4484 return 0; 4485 4486 /* 4487 * Walk PSBs while the PSB timestamp is less than the fast forward 4488 * timestamp. 4489 */ 4490 do { 4491 uint64_t tsc; 4492 size_t rem; 4493 4494 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 4495 break; 4496 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 4497 /* 4498 * A TSC packet can slip past MTC packets but, after fast 4499 * forward, decoding starts at the TSC timestamp. That means 4500 * the timestamps may not be exactly the same as the timestamps 4501 * that would have been decoded without fast forward. 4502 */ 4503 if (tsc < timestamp) { 4504 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 4505 decoder->pos += decoder->len - len; 4506 decoder->buf = buf; 4507 decoder->len = len; 4508 intel_pt_reposition(decoder); 4509 } else { 4510 break; 4511 } 4512 } while (intel_pt_step_psb(&buf, &len)); 4513 4514 return 0; 4515 } 4516