1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define BITULL(x) (1ULL << (x)) 28 29 /* IA32_RTIT_CTL MSR bits */ 30 #define INTEL_PT_CYC_ENABLE BITULL(1) 31 #define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32 #define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define SEVEN_BYTES 0xffffffffffffffULL 39 40 #define NO_VMCS 0xffffffffffULL 41 42 #define INTEL_PT_RETURN 1 43 44 /* 45 * Default maximum number of loops with no packets consumed i.e. stuck in a 46 * loop. 47 */ 48 #define INTEL_PT_MAX_LOOPS 100000 49 50 struct intel_pt_blk { 51 struct intel_pt_blk *prev; 52 uint64_t ip[INTEL_PT_BLK_SIZE]; 53 }; 54 55 struct intel_pt_stack { 56 struct intel_pt_blk *blk; 57 struct intel_pt_blk *spare; 58 int pos; 59 }; 60 61 enum intel_pt_p_once { 62 INTEL_PT_PRT_ONCE_UNK_VMCS, 63 INTEL_PT_PRT_ONCE_ERANGE, 64 }; 65 66 enum intel_pt_pkt_state { 67 INTEL_PT_STATE_NO_PSB, 68 INTEL_PT_STATE_NO_IP, 69 INTEL_PT_STATE_ERR_RESYNC, 70 INTEL_PT_STATE_IN_SYNC, 71 INTEL_PT_STATE_TNT_CONT, 72 INTEL_PT_STATE_TNT, 73 INTEL_PT_STATE_TIP, 74 INTEL_PT_STATE_TIP_PGD, 75 INTEL_PT_STATE_FUP, 76 INTEL_PT_STATE_FUP_NO_TIP, 77 INTEL_PT_STATE_FUP_IN_PSB, 78 INTEL_PT_STATE_RESAMPLE, 79 INTEL_PT_STATE_VM_TIME_CORRELATION, 80 }; 81 82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 83 { 84 switch (pkt_state) { 85 case INTEL_PT_STATE_NO_PSB: 86 case INTEL_PT_STATE_NO_IP: 87 case INTEL_PT_STATE_ERR_RESYNC: 88 case INTEL_PT_STATE_IN_SYNC: 89 case INTEL_PT_STATE_TNT_CONT: 90 case INTEL_PT_STATE_RESAMPLE: 91 case INTEL_PT_STATE_VM_TIME_CORRELATION: 92 return true; 93 case INTEL_PT_STATE_TNT: 94 case INTEL_PT_STATE_TIP: 95 case INTEL_PT_STATE_TIP_PGD: 96 case INTEL_PT_STATE_FUP: 97 case INTEL_PT_STATE_FUP_NO_TIP: 98 case INTEL_PT_STATE_FUP_IN_PSB: 99 return false; 100 default: 101 return true; 102 }; 103 } 104 105 #ifdef INTEL_PT_STRICT 106 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 107 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 108 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 109 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 110 #else 111 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 112 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 113 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 114 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 115 #endif 116 117 struct intel_pt_decoder { 118 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 119 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 120 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 121 uint64_t max_insn_cnt, void *data); 122 bool (*pgd_ip)(uint64_t ip, void *data); 123 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 124 struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); 125 void *data; 126 struct intel_pt_state state; 127 const unsigned char *buf; 128 size_t len; 129 bool return_compression; 130 bool branch_enable; 131 bool mtc_insn; 132 bool pge; 133 bool have_tma; 134 bool have_cyc; 135 bool fixup_last_mtc; 136 bool have_last_ip; 137 bool in_psb; 138 bool hop; 139 bool leap; 140 bool emulated_ptwrite; 141 bool vm_time_correlation; 142 bool vm_tm_corr_dry_run; 143 bool vm_tm_corr_reliable; 144 bool vm_tm_corr_same_buf; 145 bool vm_tm_corr_continuous; 146 bool nr; 147 bool next_nr; 148 bool iflag; 149 bool next_iflag; 150 enum intel_pt_param_flags flags; 151 uint64_t pos; 152 uint64_t last_ip; 153 uint64_t ip; 154 uint64_t pip_payload; 155 uint64_t timestamp; 156 uint64_t tsc_timestamp; 157 uint64_t ref_timestamp; 158 uint64_t buf_timestamp; 159 uint64_t sample_timestamp; 160 uint64_t ret_addr; 161 uint64_t ctc_timestamp; 162 uint64_t ctc_delta; 163 uint64_t cycle_cnt; 164 uint64_t cyc_ref_timestamp; 165 uint64_t first_timestamp; 166 uint64_t last_reliable_timestamp; 167 uint64_t vmcs; 168 uint64_t print_once; 169 uint64_t last_ctc; 170 uint32_t last_mtc; 171 uint32_t tsc_ctc_ratio_n; 172 uint32_t tsc_ctc_ratio_d; 173 uint32_t tsc_ctc_mult; 174 uint32_t tsc_slip; 175 uint32_t ctc_rem_mask; 176 int mtc_shift; 177 struct intel_pt_stack stack; 178 enum intel_pt_pkt_state pkt_state; 179 enum intel_pt_pkt_ctx pkt_ctx; 180 enum intel_pt_pkt_ctx prev_pkt_ctx; 181 enum intel_pt_blk_type blk_type; 182 int blk_type_pos; 183 struct intel_pt_pkt packet; 184 struct intel_pt_pkt tnt; 185 int pkt_step; 186 int pkt_len; 187 int last_packet_type; 188 unsigned int cbr; 189 unsigned int cbr_seen; 190 unsigned int max_non_turbo_ratio; 191 double max_non_turbo_ratio_fp; 192 double cbr_cyc_to_tsc; 193 double calc_cyc_to_tsc; 194 bool have_calc_cyc_to_tsc; 195 int exec_mode; 196 unsigned int insn_bytes; 197 uint64_t period; 198 enum intel_pt_period_type period_type; 199 uint64_t tot_insn_cnt; 200 uint64_t period_insn_cnt; 201 uint64_t period_mask; 202 uint64_t period_ticks; 203 uint64_t last_masked_timestamp; 204 uint64_t tot_cyc_cnt; 205 uint64_t sample_tot_cyc_cnt; 206 uint64_t base_cyc_cnt; 207 uint64_t cyc_cnt_timestamp; 208 uint64_t ctl; 209 uint64_t cyc_threshold; 210 double tsc_to_cyc; 211 bool continuous_period; 212 bool overflow; 213 bool set_fup_tx_flags; 214 bool set_fup_ptw; 215 bool set_fup_mwait; 216 bool set_fup_pwre; 217 bool set_fup_exstop; 218 bool set_fup_bep; 219 bool set_fup_cfe_ip; 220 bool set_fup_cfe; 221 bool set_fup_mode_exec; 222 bool sample_cyc; 223 unsigned int fup_tx_flags; 224 unsigned int tx_flags; 225 uint64_t fup_ptw_payload; 226 uint64_t fup_mwait_payload; 227 uint64_t fup_pwre_payload; 228 uint64_t cbr_payload; 229 uint64_t timestamp_insn_cnt; 230 uint64_t sample_insn_cnt; 231 uint64_t stuck_ip; 232 struct intel_pt_pkt fup_cfe_pkt; 233 int max_loops; 234 int no_progress; 235 int stuck_ip_prd; 236 int stuck_ip_cnt; 237 uint64_t psb_ip; 238 const unsigned char *next_buf; 239 size_t next_len; 240 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 241 int evd_cnt; 242 struct intel_pt_evd evd[INTEL_PT_MAX_EVDS]; 243 }; 244 245 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 246 { 247 int i; 248 249 for (i = 0; x != 1; i++) 250 x >>= 1; 251 252 return x << i; 253 } 254 255 __printf(1, 2) 256 static void p_log(const char *fmt, ...) 257 { 258 char buf[512]; 259 va_list args; 260 261 va_start(args, fmt); 262 vsnprintf(buf, sizeof(buf), fmt, args); 263 va_end(args); 264 265 fprintf(stderr, "%s\n", buf); 266 intel_pt_log("%s\n", buf); 267 } 268 269 static bool intel_pt_print_once(struct intel_pt_decoder *decoder, 270 enum intel_pt_p_once id) 271 { 272 uint64_t bit = 1ULL << id; 273 274 if (decoder->print_once & bit) 275 return false; 276 decoder->print_once |= bit; 277 return true; 278 } 279 280 static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 281 { 282 if (!(ctl & INTEL_PT_CYC_ENABLE)) 283 return 0; 284 285 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 286 } 287 288 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 289 { 290 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 291 uint64_t period; 292 293 period = intel_pt_lower_power_of_2(decoder->period); 294 decoder->period_mask = ~(period - 1); 295 decoder->period_ticks = period; 296 } 297 } 298 299 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 300 { 301 if (!d) 302 return 0; 303 return (t / d) * n + ((t % d) * n) / d; 304 } 305 306 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 307 { 308 struct intel_pt_decoder *decoder; 309 310 if (!params->get_trace || !params->walk_insn) 311 return NULL; 312 313 decoder = zalloc(sizeof(struct intel_pt_decoder)); 314 if (!decoder) 315 return NULL; 316 317 decoder->get_trace = params->get_trace; 318 decoder->walk_insn = params->walk_insn; 319 decoder->pgd_ip = params->pgd_ip; 320 decoder->lookahead = params->lookahead; 321 decoder->findnew_vmcs_info = params->findnew_vmcs_info; 322 decoder->data = params->data; 323 decoder->return_compression = params->return_compression; 324 decoder->branch_enable = params->branch_enable; 325 decoder->hop = params->quick >= 1; 326 decoder->leap = params->quick >= 2; 327 decoder->vm_time_correlation = params->vm_time_correlation; 328 decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; 329 decoder->first_timestamp = params->first_timestamp; 330 decoder->last_reliable_timestamp = params->first_timestamp; 331 decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS; 332 333 decoder->flags = params->flags; 334 335 decoder->ctl = params->ctl; 336 decoder->period = params->period; 337 decoder->period_type = params->period_type; 338 339 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 340 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 341 342 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 343 344 intel_pt_setup_period(decoder); 345 346 decoder->mtc_shift = params->mtc_period; 347 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 348 349 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 350 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 351 352 if (!decoder->tsc_ctc_ratio_n) 353 decoder->tsc_ctc_ratio_d = 0; 354 355 if (decoder->tsc_ctc_ratio_d) { 356 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 357 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 358 decoder->tsc_ctc_ratio_d; 359 } 360 361 /* 362 * A TSC packet can slip past MTC packets so that the timestamp appears 363 * to go backwards. One estimate is that can be up to about 40 CPU 364 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 365 * slippage an order of magnitude more to be on the safe side. 366 */ 367 decoder->tsc_slip = 0x10000; 368 369 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 370 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 371 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 372 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 373 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 374 375 if (decoder->hop) 376 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 377 378 return decoder; 379 } 380 381 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, 382 uint64_t first_timestamp) 383 { 384 decoder->first_timestamp = first_timestamp; 385 } 386 387 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 388 { 389 struct intel_pt_blk *blk = stack->blk; 390 391 stack->blk = blk->prev; 392 if (!stack->spare) 393 stack->spare = blk; 394 else 395 free(blk); 396 } 397 398 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 399 { 400 if (!stack->pos) { 401 if (!stack->blk) 402 return 0; 403 intel_pt_pop_blk(stack); 404 if (!stack->blk) 405 return 0; 406 stack->pos = INTEL_PT_BLK_SIZE; 407 } 408 return stack->blk->ip[--stack->pos]; 409 } 410 411 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 412 { 413 struct intel_pt_blk *blk; 414 415 if (stack->spare) { 416 blk = stack->spare; 417 stack->spare = NULL; 418 } else { 419 blk = malloc(sizeof(struct intel_pt_blk)); 420 if (!blk) 421 return -ENOMEM; 422 } 423 424 blk->prev = stack->blk; 425 stack->blk = blk; 426 stack->pos = 0; 427 return 0; 428 } 429 430 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 431 { 432 int err; 433 434 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 435 err = intel_pt_alloc_blk(stack); 436 if (err) 437 return err; 438 } 439 440 stack->blk->ip[stack->pos++] = ip; 441 return 0; 442 } 443 444 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 445 { 446 while (stack->blk) 447 intel_pt_pop_blk(stack); 448 stack->pos = 0; 449 } 450 451 static void intel_pt_free_stack(struct intel_pt_stack *stack) 452 { 453 intel_pt_clear_stack(stack); 454 zfree(&stack->blk); 455 zfree(&stack->spare); 456 } 457 458 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 459 { 460 intel_pt_free_stack(&decoder->stack); 461 free(decoder); 462 } 463 464 static int intel_pt_ext_err(int code) 465 { 466 switch (code) { 467 case -ENOMEM: 468 return INTEL_PT_ERR_NOMEM; 469 case -ENOSYS: 470 return INTEL_PT_ERR_INTERN; 471 case -EBADMSG: 472 return INTEL_PT_ERR_BADPKT; 473 case -ENODATA: 474 return INTEL_PT_ERR_NODATA; 475 case -EILSEQ: 476 return INTEL_PT_ERR_NOINSN; 477 case -ENOENT: 478 return INTEL_PT_ERR_MISMAT; 479 case -EOVERFLOW: 480 return INTEL_PT_ERR_OVR; 481 case -ENOSPC: 482 return INTEL_PT_ERR_LOST; 483 case -ELOOP: 484 return INTEL_PT_ERR_NELOOP; 485 case -ECONNRESET: 486 return INTEL_PT_ERR_EPTW; 487 default: 488 return INTEL_PT_ERR_UNK; 489 } 490 } 491 492 static const char *intel_pt_err_msgs[] = { 493 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 494 [INTEL_PT_ERR_INTERN] = "Internal error", 495 [INTEL_PT_ERR_BADPKT] = "Bad packet", 496 [INTEL_PT_ERR_NODATA] = "No more data", 497 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 498 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 499 [INTEL_PT_ERR_OVR] = "Overflow packet", 500 [INTEL_PT_ERR_LOST] = "Lost trace data", 501 [INTEL_PT_ERR_UNK] = "Unknown error!", 502 [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)", 503 [INTEL_PT_ERR_EPTW] = "Broken emulated ptwrite", 504 }; 505 506 int intel_pt__strerror(int code, char *buf, size_t buflen) 507 { 508 if (code < 1 || code >= INTEL_PT_ERR_MAX) 509 code = INTEL_PT_ERR_UNK; 510 strlcpy(buf, intel_pt_err_msgs[code], buflen); 511 return 0; 512 } 513 514 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 515 uint64_t last_ip) 516 { 517 uint64_t ip; 518 519 switch (packet->count) { 520 case 1: 521 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 522 packet->payload; 523 break; 524 case 2: 525 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 526 packet->payload; 527 break; 528 case 3: 529 ip = packet->payload; 530 /* Sign-extend 6-byte ip */ 531 if (ip & (uint64_t)0x800000000000ULL) 532 ip |= (uint64_t)0xffff000000000000ULL; 533 break; 534 case 4: 535 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 536 packet->payload; 537 break; 538 case 6: 539 ip = packet->payload; 540 break; 541 default: 542 return 0; 543 } 544 545 return ip; 546 } 547 548 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 549 { 550 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 551 decoder->have_last_ip = true; 552 } 553 554 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 555 { 556 intel_pt_set_last_ip(decoder); 557 decoder->ip = decoder->last_ip; 558 } 559 560 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 561 { 562 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 563 decoder->buf); 564 } 565 566 static int intel_pt_bug(struct intel_pt_decoder *decoder) 567 { 568 intel_pt_log("ERROR: Internal error\n"); 569 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 570 return -ENOSYS; 571 } 572 573 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 574 { 575 decoder->tx_flags = 0; 576 } 577 578 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 579 { 580 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 581 } 582 583 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder) 584 { 585 decoder->pip_payload = decoder->packet.payload; 586 } 587 588 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder) 589 { 590 decoder->next_nr = decoder->pip_payload & 1; 591 } 592 593 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder) 594 { 595 decoder->nr = decoder->pip_payload & 1; 596 decoder->next_nr = decoder->nr; 597 } 598 599 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder) 600 { 601 intel_pt_update_pip(decoder); 602 intel_pt_set_nr(decoder); 603 } 604 605 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 606 { 607 intel_pt_clear_tx_flags(decoder); 608 decoder->have_tma = false; 609 decoder->pkt_len = 1; 610 decoder->pkt_step = 1; 611 intel_pt_decoder_log_packet(decoder); 612 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 613 intel_pt_log("ERROR: Bad packet\n"); 614 decoder->pkt_state = INTEL_PT_STATE_ERR1; 615 } 616 return -EBADMSG; 617 } 618 619 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 620 { 621 decoder->sample_timestamp = decoder->timestamp; 622 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 623 decoder->state.cycles = decoder->tot_cyc_cnt; 624 } 625 626 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 627 { 628 decoder->ip = 0; 629 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 630 decoder->timestamp = 0; 631 decoder->have_tma = false; 632 } 633 634 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 635 { 636 struct intel_pt_buffer buffer = { .buf = 0, }; 637 int ret; 638 639 decoder->pkt_step = 0; 640 641 intel_pt_log("Getting more data\n"); 642 ret = decoder->get_trace(&buffer, decoder->data); 643 if (ret) 644 return ret; 645 decoder->buf = buffer.buf; 646 decoder->len = buffer.len; 647 if (!decoder->len) { 648 intel_pt_log("No more data\n"); 649 return -ENODATA; 650 } 651 decoder->buf_timestamp = buffer.ref_timestamp; 652 if (!buffer.consecutive || reposition) { 653 intel_pt_reposition(decoder); 654 decoder->ref_timestamp = buffer.ref_timestamp; 655 decoder->state.trace_nr = buffer.trace_nr; 656 decoder->vm_tm_corr_same_buf = false; 657 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 658 decoder->ref_timestamp); 659 return -ENOLINK; 660 } 661 662 return 0; 663 } 664 665 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 666 bool reposition) 667 { 668 if (!decoder->next_buf) 669 return intel_pt_get_data(decoder, reposition); 670 671 decoder->buf = decoder->next_buf; 672 decoder->len = decoder->next_len; 673 decoder->next_buf = 0; 674 decoder->next_len = 0; 675 return 0; 676 } 677 678 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 679 { 680 unsigned char *buf = decoder->temp_buf; 681 size_t old_len, len, n; 682 int ret; 683 684 old_len = decoder->len; 685 len = decoder->len; 686 memcpy(buf, decoder->buf, len); 687 688 ret = intel_pt_get_data(decoder, false); 689 if (ret) { 690 decoder->pos += old_len; 691 return ret < 0 ? ret : -EINVAL; 692 } 693 694 n = INTEL_PT_PKT_MAX_SZ - len; 695 if (n > decoder->len) 696 n = decoder->len; 697 memcpy(buf + len, decoder->buf, n); 698 len += n; 699 700 decoder->prev_pkt_ctx = decoder->pkt_ctx; 701 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 702 if (ret < (int)old_len) { 703 decoder->next_buf = decoder->buf; 704 decoder->next_len = decoder->len; 705 decoder->buf = buf; 706 decoder->len = old_len; 707 return intel_pt_bad_packet(decoder); 708 } 709 710 decoder->next_buf = decoder->buf + (ret - old_len); 711 decoder->next_len = decoder->len - (ret - old_len); 712 713 decoder->buf = buf; 714 decoder->len = ret; 715 716 return ret; 717 } 718 719 struct intel_pt_pkt_info { 720 struct intel_pt_decoder *decoder; 721 struct intel_pt_pkt packet; 722 uint64_t pos; 723 int pkt_len; 724 int last_packet_type; 725 void *data; 726 }; 727 728 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 729 730 /* Lookahead packets in current buffer */ 731 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 732 intel_pt_pkt_cb_t cb, void *data) 733 { 734 struct intel_pt_pkt_info pkt_info; 735 const unsigned char *buf = decoder->buf; 736 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 737 size_t len = decoder->len; 738 int ret; 739 740 pkt_info.decoder = decoder; 741 pkt_info.pos = decoder->pos; 742 pkt_info.pkt_len = decoder->pkt_step; 743 pkt_info.last_packet_type = decoder->last_packet_type; 744 pkt_info.data = data; 745 746 while (1) { 747 do { 748 pkt_info.pos += pkt_info.pkt_len; 749 buf += pkt_info.pkt_len; 750 len -= pkt_info.pkt_len; 751 752 if (!len) 753 return INTEL_PT_NEED_MORE_BYTES; 754 755 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 756 &pkt_ctx); 757 if (!ret) 758 return INTEL_PT_NEED_MORE_BYTES; 759 if (ret < 0) 760 return ret; 761 762 pkt_info.pkt_len = ret; 763 } while (pkt_info.packet.type == INTEL_PT_PAD); 764 765 ret = cb(&pkt_info); 766 if (ret) 767 return 0; 768 769 pkt_info.last_packet_type = pkt_info.packet.type; 770 } 771 } 772 773 struct intel_pt_calc_cyc_to_tsc_info { 774 uint64_t cycle_cnt; 775 unsigned int cbr; 776 uint32_t last_mtc; 777 uint64_t ctc_timestamp; 778 uint64_t ctc_delta; 779 uint64_t tsc_timestamp; 780 uint64_t timestamp; 781 bool have_tma; 782 bool fixup_last_mtc; 783 bool from_mtc; 784 double cbr_cyc_to_tsc; 785 }; 786 787 /* 788 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 789 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 790 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 791 * packet by copying the missing bits from the current MTC assuming the least 792 * difference between the two, and that the current MTC comes after last_mtc. 793 */ 794 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 795 uint32_t *last_mtc) 796 { 797 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 798 uint32_t mask = ~(first_missing_bit - 1); 799 800 *last_mtc |= mtc & mask; 801 if (*last_mtc >= mtc) { 802 *last_mtc -= first_missing_bit; 803 *last_mtc &= 0xff; 804 } 805 } 806 807 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 808 { 809 struct intel_pt_decoder *decoder = pkt_info->decoder; 810 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 811 uint64_t timestamp; 812 double cyc_to_tsc; 813 unsigned int cbr; 814 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 815 816 switch (pkt_info->packet.type) { 817 case INTEL_PT_TNT: 818 case INTEL_PT_TIP_PGE: 819 case INTEL_PT_TIP: 820 case INTEL_PT_FUP: 821 case INTEL_PT_PSB: 822 case INTEL_PT_PIP: 823 case INTEL_PT_MODE_EXEC: 824 case INTEL_PT_MODE_TSX: 825 case INTEL_PT_PSBEND: 826 case INTEL_PT_PAD: 827 case INTEL_PT_VMCS: 828 case INTEL_PT_MNT: 829 case INTEL_PT_PTWRITE: 830 case INTEL_PT_PTWRITE_IP: 831 case INTEL_PT_BBP: 832 case INTEL_PT_BIP: 833 case INTEL_PT_BEP: 834 case INTEL_PT_BEP_IP: 835 case INTEL_PT_CFE: 836 case INTEL_PT_CFE_IP: 837 case INTEL_PT_EVD: 838 return 0; 839 840 case INTEL_PT_MTC: 841 if (!data->have_tma) 842 return 0; 843 844 mtc = pkt_info->packet.payload; 845 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 846 data->fixup_last_mtc = false; 847 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 848 &data->last_mtc); 849 } 850 if (mtc > data->last_mtc) 851 mtc_delta = mtc - data->last_mtc; 852 else 853 mtc_delta = mtc + 256 - data->last_mtc; 854 data->ctc_delta += mtc_delta << decoder->mtc_shift; 855 data->last_mtc = mtc; 856 857 if (decoder->tsc_ctc_mult) { 858 timestamp = data->ctc_timestamp + 859 data->ctc_delta * decoder->tsc_ctc_mult; 860 } else { 861 timestamp = data->ctc_timestamp + 862 multdiv(data->ctc_delta, 863 decoder->tsc_ctc_ratio_n, 864 decoder->tsc_ctc_ratio_d); 865 } 866 867 if (timestamp < data->timestamp) 868 return 1; 869 870 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 871 data->timestamp = timestamp; 872 return 0; 873 } 874 875 break; 876 877 case INTEL_PT_TSC: 878 /* 879 * For now, do not support using TSC packets - refer 880 * intel_pt_calc_cyc_to_tsc(). 881 */ 882 if (data->from_mtc) 883 return 1; 884 timestamp = pkt_info->packet.payload | 885 (data->timestamp & (0xffULL << 56)); 886 if (data->from_mtc && timestamp < data->timestamp && 887 data->timestamp - timestamp < decoder->tsc_slip) 888 return 1; 889 if (timestamp < data->timestamp) 890 timestamp += (1ULL << 56); 891 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 892 if (data->from_mtc) 893 return 1; 894 data->tsc_timestamp = timestamp; 895 data->timestamp = timestamp; 896 return 0; 897 } 898 break; 899 900 case INTEL_PT_TMA: 901 if (data->from_mtc) 902 return 1; 903 904 if (!decoder->tsc_ctc_ratio_d) 905 return 0; 906 907 ctc = pkt_info->packet.payload; 908 fc = pkt_info->packet.count; 909 ctc_rem = ctc & decoder->ctc_rem_mask; 910 911 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 912 913 data->ctc_timestamp = data->tsc_timestamp - fc; 914 if (decoder->tsc_ctc_mult) { 915 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 916 } else { 917 data->ctc_timestamp -= 918 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 919 decoder->tsc_ctc_ratio_d); 920 } 921 922 data->ctc_delta = 0; 923 data->have_tma = true; 924 data->fixup_last_mtc = true; 925 926 return 0; 927 928 case INTEL_PT_CYC: 929 data->cycle_cnt += pkt_info->packet.payload; 930 return 0; 931 932 case INTEL_PT_CBR: 933 cbr = pkt_info->packet.payload; 934 if (data->cbr && data->cbr != cbr) 935 return 1; 936 data->cbr = cbr; 937 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 938 return 0; 939 940 case INTEL_PT_TIP_PGD: 941 case INTEL_PT_TRACESTOP: 942 case INTEL_PT_EXSTOP: 943 case INTEL_PT_EXSTOP_IP: 944 case INTEL_PT_MWAIT: 945 case INTEL_PT_PWRE: 946 case INTEL_PT_PWRX: 947 case INTEL_PT_OVF: 948 case INTEL_PT_BAD: /* Does not happen */ 949 default: 950 return 1; 951 } 952 953 if (!data->cbr && decoder->cbr) { 954 data->cbr = decoder->cbr; 955 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 956 } 957 958 if (!data->cycle_cnt) 959 return 1; 960 961 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 962 963 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 964 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 965 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 966 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 967 return 1; 968 } 969 970 decoder->calc_cyc_to_tsc = cyc_to_tsc; 971 decoder->have_calc_cyc_to_tsc = true; 972 973 if (data->cbr) { 974 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 975 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 976 } else { 977 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 978 cyc_to_tsc, pkt_info->pos); 979 } 980 981 return 1; 982 } 983 984 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 985 bool from_mtc) 986 { 987 struct intel_pt_calc_cyc_to_tsc_info data = { 988 .cycle_cnt = 0, 989 .cbr = 0, 990 .last_mtc = decoder->last_mtc, 991 .ctc_timestamp = decoder->ctc_timestamp, 992 .ctc_delta = decoder->ctc_delta, 993 .tsc_timestamp = decoder->tsc_timestamp, 994 .timestamp = decoder->timestamp, 995 .have_tma = decoder->have_tma, 996 .fixup_last_mtc = decoder->fixup_last_mtc, 997 .from_mtc = from_mtc, 998 .cbr_cyc_to_tsc = 0, 999 }; 1000 1001 /* 1002 * For now, do not support using TSC packets for at least the reasons: 1003 * 1) timing might have stopped 1004 * 2) TSC packets within PSB+ can slip against CYC packets 1005 */ 1006 if (!from_mtc) 1007 return; 1008 1009 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 1010 } 1011 1012 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 1013 { 1014 int ret; 1015 1016 decoder->last_packet_type = decoder->packet.type; 1017 1018 do { 1019 decoder->pos += decoder->pkt_step; 1020 decoder->buf += decoder->pkt_step; 1021 decoder->len -= decoder->pkt_step; 1022 1023 if (!decoder->len) { 1024 ret = intel_pt_get_next_data(decoder, false); 1025 if (ret) 1026 return ret; 1027 } 1028 1029 decoder->prev_pkt_ctx = decoder->pkt_ctx; 1030 ret = intel_pt_get_packet(decoder->buf, decoder->len, 1031 &decoder->packet, &decoder->pkt_ctx); 1032 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 1033 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 1034 ret = intel_pt_get_split_packet(decoder); 1035 if (ret < 0) 1036 return ret; 1037 } 1038 if (ret <= 0) 1039 return intel_pt_bad_packet(decoder); 1040 1041 decoder->pkt_len = ret; 1042 decoder->pkt_step = ret; 1043 intel_pt_decoder_log_packet(decoder); 1044 } while (decoder->packet.type == INTEL_PT_PAD); 1045 1046 return 0; 1047 } 1048 1049 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 1050 { 1051 uint64_t timestamp, masked_timestamp; 1052 1053 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1054 masked_timestamp = timestamp & decoder->period_mask; 1055 if (decoder->continuous_period) { 1056 if (masked_timestamp > decoder->last_masked_timestamp) 1057 return 1; 1058 } else { 1059 timestamp += 1; 1060 masked_timestamp = timestamp & decoder->period_mask; 1061 if (masked_timestamp > decoder->last_masked_timestamp) { 1062 decoder->last_masked_timestamp = masked_timestamp; 1063 decoder->continuous_period = true; 1064 } 1065 } 1066 1067 if (masked_timestamp < decoder->last_masked_timestamp) 1068 return decoder->period_ticks; 1069 1070 return decoder->period_ticks - (timestamp - masked_timestamp); 1071 } 1072 1073 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 1074 { 1075 switch (decoder->period_type) { 1076 case INTEL_PT_PERIOD_INSTRUCTIONS: 1077 return decoder->period - decoder->period_insn_cnt; 1078 case INTEL_PT_PERIOD_TICKS: 1079 return intel_pt_next_period(decoder); 1080 case INTEL_PT_PERIOD_NONE: 1081 case INTEL_PT_PERIOD_MTC: 1082 default: 1083 return 0; 1084 } 1085 } 1086 1087 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 1088 { 1089 uint64_t timestamp, masked_timestamp; 1090 1091 switch (decoder->period_type) { 1092 case INTEL_PT_PERIOD_INSTRUCTIONS: 1093 decoder->period_insn_cnt = 0; 1094 break; 1095 case INTEL_PT_PERIOD_TICKS: 1096 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1097 masked_timestamp = timestamp & decoder->period_mask; 1098 if (masked_timestamp > decoder->last_masked_timestamp) 1099 decoder->last_masked_timestamp = masked_timestamp; 1100 else 1101 decoder->last_masked_timestamp += decoder->period_ticks; 1102 break; 1103 case INTEL_PT_PERIOD_NONE: 1104 case INTEL_PT_PERIOD_MTC: 1105 default: 1106 break; 1107 } 1108 1109 decoder->state.type |= INTEL_PT_INSTRUCTION; 1110 } 1111 1112 /* 1113 * Sample FUP instruction at the same time as reporting the FUP event, so the 1114 * instruction sample gets the same flags as the FUP event. 1115 */ 1116 static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder) 1117 { 1118 struct intel_pt_insn intel_pt_insn; 1119 uint64_t max_insn_cnt, insn_cnt = 0; 1120 int err; 1121 1122 decoder->state.insn_op = INTEL_PT_OP_OTHER; 1123 decoder->state.insn_len = 0; 1124 1125 if (!decoder->branch_enable || !decoder->pge || decoder->hop || 1126 decoder->ip != decoder->last_ip) 1127 return; 1128 1129 if (!decoder->mtc_insn) 1130 decoder->mtc_insn = true; 1131 1132 max_insn_cnt = intel_pt_next_sample(decoder); 1133 if (max_insn_cnt != 1) 1134 return; 1135 1136 err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip, 1137 0, max_insn_cnt, decoder->data); 1138 /* Ignore error, it will be reported next walk anyway */ 1139 if (err) 1140 return; 1141 1142 if (intel_pt_insn.branch != INTEL_PT_BR_NO_BRANCH) { 1143 intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip); 1144 return; 1145 } 1146 1147 decoder->tot_insn_cnt += insn_cnt; 1148 decoder->timestamp_insn_cnt += insn_cnt; 1149 decoder->sample_insn_cnt += insn_cnt; 1150 decoder->period_insn_cnt += insn_cnt; 1151 1152 intel_pt_sample_insn(decoder); 1153 1154 decoder->state.type |= INTEL_PT_INSTRUCTION; 1155 decoder->ip += intel_pt_insn.length; 1156 } 1157 1158 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1159 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1160 { 1161 uint64_t max_insn_cnt, insn_cnt = 0; 1162 int err; 1163 1164 if (!decoder->mtc_insn) 1165 decoder->mtc_insn = true; 1166 1167 max_insn_cnt = intel_pt_next_sample(decoder); 1168 1169 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1170 max_insn_cnt, decoder->data); 1171 1172 decoder->tot_insn_cnt += insn_cnt; 1173 decoder->timestamp_insn_cnt += insn_cnt; 1174 decoder->sample_insn_cnt += insn_cnt; 1175 decoder->period_insn_cnt += insn_cnt; 1176 1177 if (err) { 1178 decoder->no_progress = 0; 1179 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1180 intel_pt_log_at("ERROR: Failed to get instruction", 1181 decoder->ip); 1182 if (err == -ENOENT) 1183 return -ENOLINK; 1184 return -EILSEQ; 1185 } 1186 1187 if (ip && decoder->ip == ip) { 1188 err = -EAGAIN; 1189 goto out; 1190 } 1191 1192 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1193 intel_pt_sample_insn(decoder); 1194 1195 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1196 decoder->state.type = INTEL_PT_INSTRUCTION; 1197 decoder->state.from_ip = decoder->ip; 1198 decoder->state.to_ip = 0; 1199 decoder->ip += intel_pt_insn->length; 1200 err = INTEL_PT_RETURN; 1201 goto out; 1202 } 1203 1204 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1205 /* Zero-length calls are excluded */ 1206 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1207 intel_pt_insn->rel) { 1208 err = intel_pt_push(&decoder->stack, decoder->ip + 1209 intel_pt_insn->length); 1210 if (err) 1211 goto out; 1212 } 1213 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1214 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1215 } 1216 1217 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1218 int cnt = decoder->no_progress++; 1219 1220 decoder->state.from_ip = decoder->ip; 1221 decoder->ip += intel_pt_insn->length + 1222 intel_pt_insn->rel; 1223 decoder->state.to_ip = decoder->ip; 1224 err = INTEL_PT_RETURN; 1225 1226 /* 1227 * Check for being stuck in a loop. This can happen if a 1228 * decoder error results in the decoder erroneously setting the 1229 * ip to an address that is itself in an infinite loop that 1230 * consumes no packets. When that happens, there must be an 1231 * unconditional branch. 1232 */ 1233 if (cnt) { 1234 if (cnt == 1) { 1235 decoder->stuck_ip = decoder->state.to_ip; 1236 decoder->stuck_ip_prd = 1; 1237 decoder->stuck_ip_cnt = 1; 1238 } else if (cnt > decoder->max_loops || 1239 decoder->state.to_ip == decoder->stuck_ip) { 1240 intel_pt_log_at("ERROR: Never-ending loop", 1241 decoder->state.to_ip); 1242 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1243 err = -ELOOP; 1244 goto out; 1245 } else if (!--decoder->stuck_ip_cnt) { 1246 decoder->stuck_ip_prd += 1; 1247 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1248 decoder->stuck_ip = decoder->state.to_ip; 1249 } 1250 } 1251 goto out_no_progress; 1252 } 1253 out: 1254 decoder->no_progress = 0; 1255 out_no_progress: 1256 decoder->state.insn_op = intel_pt_insn->op; 1257 decoder->state.insn_len = intel_pt_insn->length; 1258 memcpy(decoder->state.insn, intel_pt_insn->buf, 1259 INTEL_PT_INSN_BUF_SZ); 1260 1261 if (decoder->tx_flags & INTEL_PT_IN_TX) 1262 decoder->state.flags |= INTEL_PT_IN_TX; 1263 1264 return err; 1265 } 1266 1267 static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder) 1268 { 1269 bool iflag = decoder->packet.count & INTEL_PT_IFLAG; 1270 1271 decoder->exec_mode = decoder->packet.payload; 1272 decoder->iflag = iflag; 1273 decoder->next_iflag = iflag; 1274 decoder->state.from_iflag = iflag; 1275 decoder->state.to_iflag = iflag; 1276 } 1277 1278 static void intel_pt_mode_exec(struct intel_pt_decoder *decoder) 1279 { 1280 bool iflag = decoder->packet.count & INTEL_PT_IFLAG; 1281 1282 decoder->exec_mode = decoder->packet.payload; 1283 decoder->next_iflag = iflag; 1284 } 1285 1286 static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder) 1287 { 1288 decoder->state.type |= INTEL_PT_IFLAG_CHG; 1289 decoder->state.from_iflag = decoder->iflag; 1290 decoder->state.to_iflag = decoder->next_iflag; 1291 decoder->iflag = decoder->next_iflag; 1292 } 1293 1294 static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder) 1295 { 1296 if (decoder->iflag != decoder->next_iflag) 1297 intel_pt_sample_iflag(decoder); 1298 } 1299 1300 static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder) 1301 { 1302 decoder->set_fup_tx_flags = false; 1303 decoder->set_fup_ptw = false; 1304 decoder->set_fup_mwait = false; 1305 decoder->set_fup_pwre = false; 1306 decoder->set_fup_exstop = false; 1307 decoder->set_fup_bep = false; 1308 decoder->set_fup_cfe_ip = false; 1309 decoder->set_fup_cfe = false; 1310 decoder->evd_cnt = 0; 1311 decoder->set_fup_mode_exec = false; 1312 decoder->iflag = decoder->next_iflag; 1313 } 1314 1315 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip) 1316 { 1317 enum intel_pt_sample_type type = decoder->state.type; 1318 bool sample_fup_insn = false; 1319 bool ret = false; 1320 1321 decoder->state.type &= ~INTEL_PT_BRANCH; 1322 1323 if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) { 1324 bool ip = decoder->set_fup_cfe_ip; 1325 1326 decoder->set_fup_cfe_ip = false; 1327 decoder->set_fup_cfe = false; 1328 decoder->state.type |= INTEL_PT_EVT; 1329 if (!ip && decoder->pge) 1330 decoder->state.type |= INTEL_PT_BRANCH; 1331 decoder->state.cfe_type = decoder->fup_cfe_pkt.count; 1332 decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload; 1333 decoder->state.evd_cnt = decoder->evd_cnt; 1334 decoder->state.evd = decoder->evd; 1335 decoder->evd_cnt = 0; 1336 if (ip || decoder->pge) 1337 decoder->state.flags |= INTEL_PT_FUP_IP; 1338 ret = true; 1339 } 1340 if (decoder->set_fup_mode_exec) { 1341 decoder->set_fup_mode_exec = false; 1342 intel_pt_sample_iflag(decoder); 1343 sample_fup_insn = no_tip; 1344 ret = true; 1345 } 1346 if (decoder->set_fup_tx_flags) { 1347 decoder->set_fup_tx_flags = false; 1348 decoder->tx_flags = decoder->fup_tx_flags; 1349 decoder->state.type |= INTEL_PT_TRANSACTION; 1350 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) 1351 decoder->state.type |= INTEL_PT_BRANCH; 1352 decoder->state.flags = decoder->fup_tx_flags; 1353 ret = true; 1354 } 1355 if (decoder->set_fup_ptw) { 1356 decoder->set_fup_ptw = false; 1357 decoder->state.type |= INTEL_PT_PTW; 1358 decoder->state.flags |= INTEL_PT_FUP_IP; 1359 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1360 ret = true; 1361 } 1362 if (decoder->set_fup_mwait) { 1363 decoder->set_fup_mwait = false; 1364 decoder->state.type |= INTEL_PT_MWAIT_OP; 1365 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1366 ret = true; 1367 } 1368 if (decoder->set_fup_pwre) { 1369 decoder->set_fup_pwre = false; 1370 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1371 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1372 ret = true; 1373 } 1374 if (decoder->set_fup_exstop) { 1375 decoder->set_fup_exstop = false; 1376 decoder->state.type |= INTEL_PT_EX_STOP; 1377 decoder->state.flags |= INTEL_PT_FUP_IP; 1378 ret = true; 1379 } 1380 if (decoder->set_fup_bep) { 1381 decoder->set_fup_bep = false; 1382 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1383 ret = true; 1384 } 1385 if (decoder->overflow) { 1386 decoder->overflow = false; 1387 if (!ret && !decoder->pge) { 1388 if (decoder->hop) { 1389 decoder->state.type = 0; 1390 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 1391 } 1392 decoder->pge = true; 1393 decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN; 1394 decoder->state.from_ip = 0; 1395 decoder->state.to_ip = decoder->ip; 1396 return true; 1397 } 1398 } 1399 if (ret) { 1400 decoder->state.from_ip = decoder->ip; 1401 decoder->state.to_ip = 0; 1402 if (sample_fup_insn) 1403 intel_pt_sample_fup_insn(decoder); 1404 } else { 1405 decoder->state.type = type; 1406 } 1407 return ret; 1408 } 1409 1410 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1411 struct intel_pt_insn *intel_pt_insn, 1412 uint64_t ip, int err) 1413 { 1414 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1415 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1416 ip == decoder->ip + intel_pt_insn->length; 1417 } 1418 1419 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1420 { 1421 struct intel_pt_insn intel_pt_insn; 1422 uint64_t ip; 1423 int err; 1424 1425 ip = decoder->last_ip; 1426 1427 while (1) { 1428 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1429 if (err == INTEL_PT_RETURN) 1430 return 0; 1431 if (err == -EAGAIN || 1432 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1433 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; 1434 1435 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1436 if (intel_pt_fup_event(decoder, no_tip) && no_tip) 1437 return 0; 1438 return -EAGAIN; 1439 } 1440 decoder->set_fup_tx_flags = false; 1441 if (err) 1442 return err; 1443 1444 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1445 intel_pt_log_at("ERROR: Unexpected indirect branch", 1446 decoder->ip); 1447 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1448 return -ENOENT; 1449 } 1450 1451 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1452 intel_pt_log_at("ERROR: Unexpected conditional branch", 1453 decoder->ip); 1454 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1455 return -ENOENT; 1456 } 1457 1458 intel_pt_bug(decoder); 1459 } 1460 } 1461 1462 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1463 { 1464 struct intel_pt_insn intel_pt_insn; 1465 int err; 1466 1467 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1468 if (err == INTEL_PT_RETURN && 1469 decoder->pgd_ip && 1470 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1471 (decoder->state.type & INTEL_PT_BRANCH) && 1472 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1473 /* Unconditional branch leaving filter region */ 1474 decoder->no_progress = 0; 1475 decoder->pge = false; 1476 decoder->continuous_period = false; 1477 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1478 decoder->state.type |= INTEL_PT_TRACE_END; 1479 intel_pt_update_nr(decoder); 1480 return 0; 1481 } 1482 if (err == INTEL_PT_RETURN) 1483 return 0; 1484 if (err) 1485 return err; 1486 1487 intel_pt_update_nr(decoder); 1488 intel_pt_sample_iflag_chg(decoder); 1489 1490 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1491 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1492 decoder->pge = false; 1493 decoder->continuous_period = false; 1494 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1495 decoder->state.from_ip = decoder->ip; 1496 if (decoder->packet.count == 0) { 1497 decoder->state.to_ip = 0; 1498 } else { 1499 decoder->state.to_ip = decoder->last_ip; 1500 decoder->ip = decoder->last_ip; 1501 } 1502 decoder->state.type |= INTEL_PT_TRACE_END; 1503 } else { 1504 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1505 decoder->state.from_ip = decoder->ip; 1506 if (decoder->packet.count == 0) { 1507 decoder->state.to_ip = 0; 1508 } else { 1509 decoder->state.to_ip = decoder->last_ip; 1510 decoder->ip = decoder->last_ip; 1511 } 1512 } 1513 return 0; 1514 } 1515 1516 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1517 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1518 intel_pt_insn.rel; 1519 1520 if (decoder->pgd_ip && 1521 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1522 decoder->pgd_ip(to_ip, decoder->data)) { 1523 /* Conditional branch leaving filter region */ 1524 decoder->pge = false; 1525 decoder->continuous_period = false; 1526 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1527 decoder->ip = to_ip; 1528 decoder->state.from_ip = decoder->ip; 1529 decoder->state.to_ip = to_ip; 1530 decoder->state.type |= INTEL_PT_TRACE_END; 1531 return 0; 1532 } 1533 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1534 decoder->ip); 1535 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1536 return -ENOENT; 1537 } 1538 1539 return intel_pt_bug(decoder); 1540 } 1541 1542 struct eptw_data { 1543 int bit_countdown; 1544 uint64_t payload; 1545 }; 1546 1547 static int intel_pt_eptw_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 1548 { 1549 struct eptw_data *data = pkt_info->data; 1550 int nr_bits; 1551 1552 switch (pkt_info->packet.type) { 1553 case INTEL_PT_PAD: 1554 case INTEL_PT_MNT: 1555 case INTEL_PT_MODE_EXEC: 1556 case INTEL_PT_MODE_TSX: 1557 case INTEL_PT_MTC: 1558 case INTEL_PT_FUP: 1559 case INTEL_PT_CYC: 1560 case INTEL_PT_CBR: 1561 case INTEL_PT_TSC: 1562 case INTEL_PT_TMA: 1563 case INTEL_PT_PIP: 1564 case INTEL_PT_VMCS: 1565 case INTEL_PT_PSB: 1566 case INTEL_PT_PSBEND: 1567 case INTEL_PT_PTWRITE: 1568 case INTEL_PT_PTWRITE_IP: 1569 case INTEL_PT_EXSTOP: 1570 case INTEL_PT_EXSTOP_IP: 1571 case INTEL_PT_MWAIT: 1572 case INTEL_PT_PWRE: 1573 case INTEL_PT_PWRX: 1574 case INTEL_PT_BBP: 1575 case INTEL_PT_BIP: 1576 case INTEL_PT_BEP: 1577 case INTEL_PT_BEP_IP: 1578 case INTEL_PT_CFE: 1579 case INTEL_PT_CFE_IP: 1580 case INTEL_PT_EVD: 1581 break; 1582 1583 case INTEL_PT_TNT: 1584 nr_bits = data->bit_countdown; 1585 if (nr_bits > pkt_info->packet.count) 1586 nr_bits = pkt_info->packet.count; 1587 data->payload <<= nr_bits; 1588 data->payload |= pkt_info->packet.payload >> (64 - nr_bits); 1589 data->bit_countdown -= nr_bits; 1590 return !data->bit_countdown; 1591 1592 case INTEL_PT_TIP_PGE: 1593 case INTEL_PT_TIP_PGD: 1594 case INTEL_PT_TIP: 1595 case INTEL_PT_BAD: 1596 case INTEL_PT_OVF: 1597 case INTEL_PT_TRACESTOP: 1598 default: 1599 return 1; 1600 } 1601 1602 return 0; 1603 } 1604 1605 static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder) 1606 { 1607 int n = 64 - decoder->tnt.count; 1608 struct eptw_data data = { 1609 .bit_countdown = n, 1610 .payload = decoder->tnt.payload >> n, 1611 }; 1612 1613 decoder->emulated_ptwrite = false; 1614 intel_pt_log("Emulated ptwrite detected\n"); 1615 1616 intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data); 1617 if (data.bit_countdown) 1618 return -ECONNRESET; 1619 1620 decoder->state.type = INTEL_PT_PTW; 1621 decoder->state.from_ip = decoder->ip; 1622 decoder->state.to_ip = 0; 1623 decoder->state.ptw_payload = data.payload; 1624 return 0; 1625 } 1626 1627 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1628 { 1629 struct intel_pt_insn intel_pt_insn; 1630 int err; 1631 1632 while (1) { 1633 if (decoder->emulated_ptwrite) 1634 return intel_pt_emulated_ptwrite(decoder); 1635 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1636 if (err == INTEL_PT_RETURN) { 1637 decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite; 1638 return 0; 1639 } 1640 if (err) { 1641 decoder->emulated_ptwrite = false; 1642 return err; 1643 } 1644 1645 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1646 if (!decoder->return_compression) { 1647 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1648 decoder->ip); 1649 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1650 return -ENOENT; 1651 } 1652 if (!decoder->ret_addr) { 1653 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1654 decoder->ip); 1655 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1656 return -ENOENT; 1657 } 1658 if (!(decoder->tnt.payload & BIT63)) { 1659 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1660 decoder->ip); 1661 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1662 return -ENOENT; 1663 } 1664 decoder->tnt.count -= 1; 1665 if (decoder->tnt.count) 1666 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1667 else 1668 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1669 decoder->tnt.payload <<= 1; 1670 decoder->state.from_ip = decoder->ip; 1671 decoder->ip = decoder->ret_addr; 1672 decoder->state.to_ip = decoder->ip; 1673 return 0; 1674 } 1675 1676 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1677 /* Handle deferred TIPs */ 1678 err = intel_pt_get_next_packet(decoder); 1679 if (err) 1680 return err; 1681 if (decoder->packet.type != INTEL_PT_TIP || 1682 decoder->packet.count == 0) { 1683 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1684 decoder->ip); 1685 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1686 decoder->pkt_step = 0; 1687 return -ENOENT; 1688 } 1689 intel_pt_set_last_ip(decoder); 1690 decoder->state.from_ip = decoder->ip; 1691 decoder->state.to_ip = decoder->last_ip; 1692 decoder->ip = decoder->last_ip; 1693 intel_pt_update_nr(decoder); 1694 intel_pt_sample_iflag_chg(decoder); 1695 return 0; 1696 } 1697 1698 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1699 decoder->tnt.count -= 1; 1700 if (decoder->tnt.count) 1701 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1702 else 1703 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1704 if (decoder->tnt.payload & BIT63) { 1705 decoder->tnt.payload <<= 1; 1706 decoder->state.from_ip = decoder->ip; 1707 decoder->ip += intel_pt_insn.length + 1708 intel_pt_insn.rel; 1709 decoder->state.to_ip = decoder->ip; 1710 return 0; 1711 } 1712 /* Instruction sample for a non-taken branch */ 1713 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1714 decoder->tnt.payload <<= 1; 1715 decoder->state.type = INTEL_PT_INSTRUCTION; 1716 decoder->state.from_ip = decoder->ip; 1717 decoder->state.to_ip = 0; 1718 decoder->ip += intel_pt_insn.length; 1719 return 0; 1720 } 1721 decoder->sample_cyc = false; 1722 decoder->ip += intel_pt_insn.length; 1723 if (!decoder->tnt.count) { 1724 intel_pt_update_sample_time(decoder); 1725 return -EAGAIN; 1726 } 1727 decoder->tnt.payload <<= 1; 1728 continue; 1729 } 1730 1731 return intel_pt_bug(decoder); 1732 } 1733 } 1734 1735 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1736 { 1737 unsigned int fup_tx_flags; 1738 int err; 1739 1740 fup_tx_flags = decoder->packet.payload & 1741 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1742 err = intel_pt_get_next_packet(decoder); 1743 if (err) 1744 return err; 1745 if (decoder->packet.type == INTEL_PT_FUP) { 1746 decoder->fup_tx_flags = fup_tx_flags; 1747 decoder->set_fup_tx_flags = true; 1748 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1749 *no_tip = true; 1750 } else { 1751 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1752 decoder->pos); 1753 intel_pt_update_in_tx(decoder); 1754 } 1755 return 0; 1756 } 1757 1758 static int intel_pt_evd(struct intel_pt_decoder *decoder) 1759 { 1760 if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) { 1761 intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos); 1762 return -ENOSYS; 1763 } 1764 decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){ 1765 .type = decoder->packet.count, 1766 .payload = decoder->packet.payload, 1767 }; 1768 return 0; 1769 } 1770 1771 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1772 { 1773 timestamp |= (ref_timestamp & (0xffULL << 56)); 1774 1775 if (timestamp < ref_timestamp) { 1776 if (ref_timestamp - timestamp > (1ULL << 55)) 1777 timestamp += (1ULL << 56); 1778 } else { 1779 if (timestamp - ref_timestamp > (1ULL << 55)) 1780 timestamp -= (1ULL << 56); 1781 } 1782 1783 return timestamp; 1784 } 1785 1786 /* For use only when decoder->vm_time_correlation is true */ 1787 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder, 1788 uint64_t timestamp) 1789 { 1790 uint64_t max_timestamp = decoder->buf_timestamp; 1791 1792 if (!max_timestamp) { 1793 max_timestamp = decoder->last_reliable_timestamp + 1794 0x400000000ULL; 1795 } 1796 return timestamp >= decoder->last_reliable_timestamp && 1797 timestamp < decoder->buf_timestamp; 1798 } 1799 1800 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1801 { 1802 uint64_t timestamp; 1803 bool bad = false; 1804 1805 decoder->have_tma = false; 1806 1807 if (decoder->ref_timestamp) { 1808 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1809 decoder->ref_timestamp); 1810 decoder->tsc_timestamp = timestamp; 1811 decoder->timestamp = timestamp; 1812 decoder->ref_timestamp = 0; 1813 decoder->timestamp_insn_cnt = 0; 1814 } else if (decoder->timestamp) { 1815 timestamp = decoder->packet.payload | 1816 (decoder->timestamp & (0xffULL << 56)); 1817 decoder->tsc_timestamp = timestamp; 1818 if (timestamp < decoder->timestamp && 1819 decoder->timestamp - timestamp < decoder->tsc_slip) { 1820 intel_pt_log_to("Suppressing backwards timestamp", 1821 timestamp); 1822 timestamp = decoder->timestamp; 1823 } 1824 if (timestamp < decoder->timestamp) { 1825 if (!decoder->buf_timestamp || 1826 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { 1827 intel_pt_log_to("Wraparound timestamp", timestamp); 1828 timestamp += (1ULL << 56); 1829 decoder->tsc_timestamp = timestamp; 1830 } else { 1831 intel_pt_log_to("Suppressing bad timestamp", timestamp); 1832 timestamp = decoder->timestamp; 1833 bad = true; 1834 } 1835 } 1836 if (decoder->vm_time_correlation && 1837 (bad || !intel_pt_time_in_range(decoder, timestamp)) && 1838 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 1839 p_log("Timestamp out of range"); 1840 decoder->timestamp = timestamp; 1841 decoder->timestamp_insn_cnt = 0; 1842 } 1843 1844 if (decoder->last_packet_type == INTEL_PT_CYC) { 1845 decoder->cyc_ref_timestamp = decoder->timestamp; 1846 decoder->cycle_cnt = 0; 1847 decoder->have_calc_cyc_to_tsc = false; 1848 intel_pt_calc_cyc_to_tsc(decoder, false); 1849 } 1850 1851 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1852 } 1853 1854 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1855 { 1856 intel_pt_log("ERROR: Buffer overflow\n"); 1857 intel_pt_clear_tx_flags(decoder); 1858 intel_pt_set_nr(decoder); 1859 decoder->timestamp_insn_cnt = 0; 1860 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1861 decoder->state.from_ip = decoder->ip; 1862 decoder->ip = 0; 1863 decoder->pge = false; 1864 intel_pt_clear_fup_event(decoder); 1865 decoder->overflow = true; 1866 return -EOVERFLOW; 1867 } 1868 1869 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1870 { 1871 if (decoder->have_cyc) 1872 return; 1873 1874 decoder->cyc_cnt_timestamp = decoder->timestamp; 1875 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1876 } 1877 1878 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1879 { 1880 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1881 1882 if (decoder->pge) 1883 intel_pt_mtc_cyc_cnt_pge(decoder); 1884 } 1885 1886 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1887 { 1888 uint64_t tot_cyc_cnt, tsc_delta; 1889 1890 if (decoder->have_cyc) 1891 return; 1892 1893 decoder->sample_cyc = true; 1894 1895 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1896 return; 1897 1898 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1899 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1900 1901 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1902 decoder->tot_cyc_cnt = tot_cyc_cnt; 1903 } 1904 1905 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1906 { 1907 uint32_t ctc = decoder->packet.payload; 1908 uint32_t fc = decoder->packet.count; 1909 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1910 1911 if (!decoder->tsc_ctc_ratio_d) 1912 return; 1913 1914 if (decoder->pge && !decoder->in_psb) 1915 intel_pt_mtc_cyc_cnt_pge(decoder); 1916 else 1917 intel_pt_mtc_cyc_cnt_upd(decoder); 1918 1919 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1920 decoder->last_ctc = ctc - ctc_rem; 1921 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1922 if (decoder->tsc_ctc_mult) { 1923 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1924 } else { 1925 decoder->ctc_timestamp -= multdiv(ctc_rem, 1926 decoder->tsc_ctc_ratio_n, 1927 decoder->tsc_ctc_ratio_d); 1928 } 1929 decoder->ctc_delta = 0; 1930 decoder->have_tma = true; 1931 decoder->fixup_last_mtc = true; 1932 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1933 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1934 } 1935 1936 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1937 { 1938 uint64_t timestamp; 1939 uint32_t mtc, mtc_delta; 1940 1941 if (!decoder->have_tma) 1942 return; 1943 1944 mtc = decoder->packet.payload; 1945 1946 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1947 decoder->fixup_last_mtc = false; 1948 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1949 &decoder->last_mtc); 1950 } 1951 1952 if (mtc > decoder->last_mtc) 1953 mtc_delta = mtc - decoder->last_mtc; 1954 else 1955 mtc_delta = mtc + 256 - decoder->last_mtc; 1956 1957 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1958 1959 if (decoder->tsc_ctc_mult) { 1960 timestamp = decoder->ctc_timestamp + 1961 decoder->ctc_delta * decoder->tsc_ctc_mult; 1962 } else { 1963 timestamp = decoder->ctc_timestamp + 1964 multdiv(decoder->ctc_delta, 1965 decoder->tsc_ctc_ratio_n, 1966 decoder->tsc_ctc_ratio_d); 1967 } 1968 1969 if (timestamp < decoder->timestamp) 1970 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1971 timestamp, decoder->timestamp); 1972 else 1973 decoder->timestamp = timestamp; 1974 1975 intel_pt_mtc_cyc_cnt_upd(decoder); 1976 1977 decoder->timestamp_insn_cnt = 0; 1978 decoder->last_mtc = mtc; 1979 1980 if (decoder->last_packet_type == INTEL_PT_CYC) { 1981 decoder->cyc_ref_timestamp = decoder->timestamp; 1982 decoder->cycle_cnt = 0; 1983 decoder->have_calc_cyc_to_tsc = false; 1984 intel_pt_calc_cyc_to_tsc(decoder, true); 1985 } 1986 1987 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1988 } 1989 1990 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1991 { 1992 unsigned int cbr = decoder->packet.payload & 0xff; 1993 1994 decoder->cbr_payload = decoder->packet.payload; 1995 1996 if (decoder->cbr == cbr) 1997 return; 1998 1999 decoder->cbr = cbr; 2000 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 2001 2002 intel_pt_mtc_cyc_cnt_cbr(decoder); 2003 } 2004 2005 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 2006 { 2007 uint64_t timestamp = decoder->cyc_ref_timestamp; 2008 2009 decoder->have_cyc = true; 2010 2011 decoder->cycle_cnt += decoder->packet.payload; 2012 if (decoder->pge) 2013 decoder->tot_cyc_cnt += decoder->packet.payload; 2014 decoder->sample_cyc = true; 2015 2016 if (!decoder->cyc_ref_timestamp) 2017 return; 2018 2019 if (decoder->have_calc_cyc_to_tsc) 2020 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 2021 else if (decoder->cbr) 2022 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 2023 else 2024 return; 2025 2026 if (timestamp < decoder->timestamp) 2027 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 2028 timestamp, decoder->timestamp); 2029 else 2030 decoder->timestamp = timestamp; 2031 2032 decoder->timestamp_insn_cnt = 0; 2033 2034 intel_pt_log_to("Setting timestamp", decoder->timestamp); 2035 } 2036 2037 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 2038 { 2039 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 2040 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 2041 decoder->state.items.is_32_bit = false; 2042 } 2043 decoder->blk_type = decoder->packet.payload; 2044 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 2045 if (decoder->blk_type == INTEL_PT_GP_REGS) 2046 decoder->state.items.is_32_bit = decoder->packet.count; 2047 if (decoder->blk_type_pos < 0) { 2048 intel_pt_log("WARNING: Unknown block type %u\n", 2049 decoder->blk_type); 2050 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 2051 intel_pt_log("WARNING: Duplicate block type %u\n", 2052 decoder->blk_type); 2053 } 2054 } 2055 2056 static void intel_pt_bip(struct intel_pt_decoder *decoder) 2057 { 2058 uint32_t id = decoder->packet.count; 2059 uint32_t bit = 1 << id; 2060 int pos = decoder->blk_type_pos; 2061 2062 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 2063 intel_pt_log("WARNING: Unknown block item %u type %d\n", 2064 id, decoder->blk_type); 2065 return; 2066 } 2067 2068 if (decoder->state.items.mask[pos] & bit) { 2069 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 2070 id, decoder->blk_type); 2071 } 2072 2073 decoder->state.items.mask[pos] |= bit; 2074 decoder->state.items.val[pos][id] = decoder->packet.payload; 2075 } 2076 2077 /* Walk PSB+ packets when already in sync. */ 2078 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 2079 { 2080 int err; 2081 2082 decoder->in_psb = true; 2083 2084 while (1) { 2085 err = intel_pt_get_next_packet(decoder); 2086 if (err) 2087 goto out; 2088 2089 switch (decoder->packet.type) { 2090 case INTEL_PT_PSBEND: 2091 err = 0; 2092 goto out; 2093 2094 case INTEL_PT_TIP_PGD: 2095 case INTEL_PT_TIP_PGE: 2096 case INTEL_PT_TIP: 2097 case INTEL_PT_TNT: 2098 case INTEL_PT_TRACESTOP: 2099 case INTEL_PT_BAD: 2100 case INTEL_PT_PSB: 2101 case INTEL_PT_PTWRITE: 2102 case INTEL_PT_PTWRITE_IP: 2103 case INTEL_PT_EXSTOP: 2104 case INTEL_PT_EXSTOP_IP: 2105 case INTEL_PT_MWAIT: 2106 case INTEL_PT_PWRE: 2107 case INTEL_PT_PWRX: 2108 case INTEL_PT_BBP: 2109 case INTEL_PT_BIP: 2110 case INTEL_PT_BEP: 2111 case INTEL_PT_BEP_IP: 2112 case INTEL_PT_CFE: 2113 case INTEL_PT_CFE_IP: 2114 case INTEL_PT_EVD: 2115 decoder->have_tma = false; 2116 intel_pt_log("ERROR: Unexpected packet\n"); 2117 err = -EAGAIN; 2118 goto out; 2119 2120 case INTEL_PT_OVF: 2121 err = intel_pt_overflow(decoder); 2122 goto out; 2123 2124 case INTEL_PT_TSC: 2125 intel_pt_calc_tsc_timestamp(decoder); 2126 break; 2127 2128 case INTEL_PT_TMA: 2129 intel_pt_calc_tma(decoder); 2130 break; 2131 2132 case INTEL_PT_CBR: 2133 intel_pt_calc_cbr(decoder); 2134 break; 2135 2136 case INTEL_PT_MODE_EXEC: 2137 intel_pt_mode_exec_status(decoder); 2138 break; 2139 2140 case INTEL_PT_PIP: 2141 intel_pt_set_pip(decoder); 2142 break; 2143 2144 case INTEL_PT_FUP: 2145 decoder->pge = true; 2146 if (decoder->packet.count) { 2147 intel_pt_set_last_ip(decoder); 2148 decoder->psb_ip = decoder->last_ip; 2149 } 2150 break; 2151 2152 case INTEL_PT_MODE_TSX: 2153 intel_pt_update_in_tx(decoder); 2154 break; 2155 2156 case INTEL_PT_MTC: 2157 intel_pt_calc_mtc_timestamp(decoder); 2158 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 2159 decoder->state.type |= INTEL_PT_INSTRUCTION; 2160 break; 2161 2162 case INTEL_PT_CYC: 2163 intel_pt_calc_cyc_timestamp(decoder); 2164 break; 2165 2166 case INTEL_PT_VMCS: 2167 case INTEL_PT_MNT: 2168 case INTEL_PT_PAD: 2169 default: 2170 break; 2171 } 2172 } 2173 out: 2174 decoder->in_psb = false; 2175 2176 return err; 2177 } 2178 2179 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 2180 { 2181 int err; 2182 2183 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 2184 decoder->tx_flags = 0; 2185 decoder->state.flags &= ~INTEL_PT_IN_TX; 2186 decoder->state.flags |= INTEL_PT_ABORT_TX; 2187 } else { 2188 decoder->state.flags |= INTEL_PT_ASYNC; 2189 } 2190 2191 while (1) { 2192 err = intel_pt_get_next_packet(decoder); 2193 if (err) 2194 return err; 2195 2196 switch (decoder->packet.type) { 2197 case INTEL_PT_TNT: 2198 case INTEL_PT_FUP: 2199 case INTEL_PT_TRACESTOP: 2200 case INTEL_PT_PSB: 2201 case INTEL_PT_TSC: 2202 case INTEL_PT_TMA: 2203 case INTEL_PT_MODE_TSX: 2204 case INTEL_PT_BAD: 2205 case INTEL_PT_PSBEND: 2206 case INTEL_PT_PTWRITE: 2207 case INTEL_PT_PTWRITE_IP: 2208 case INTEL_PT_EXSTOP: 2209 case INTEL_PT_EXSTOP_IP: 2210 case INTEL_PT_MWAIT: 2211 case INTEL_PT_PWRE: 2212 case INTEL_PT_PWRX: 2213 case INTEL_PT_BBP: 2214 case INTEL_PT_BIP: 2215 case INTEL_PT_BEP: 2216 case INTEL_PT_BEP_IP: 2217 case INTEL_PT_CFE: 2218 case INTEL_PT_CFE_IP: 2219 case INTEL_PT_EVD: 2220 intel_pt_log("ERROR: Missing TIP after FUP\n"); 2221 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2222 decoder->pkt_step = 0; 2223 return -ENOENT; 2224 2225 case INTEL_PT_CBR: 2226 intel_pt_calc_cbr(decoder); 2227 break; 2228 2229 case INTEL_PT_OVF: 2230 return intel_pt_overflow(decoder); 2231 2232 case INTEL_PT_TIP_PGD: 2233 decoder->state.from_ip = decoder->ip; 2234 if (decoder->packet.count == 0) { 2235 decoder->state.to_ip = 0; 2236 } else { 2237 intel_pt_set_ip(decoder); 2238 decoder->state.to_ip = decoder->ip; 2239 } 2240 decoder->pge = false; 2241 decoder->continuous_period = false; 2242 decoder->state.type |= INTEL_PT_TRACE_END; 2243 intel_pt_update_nr(decoder); 2244 return 0; 2245 2246 case INTEL_PT_TIP_PGE: 2247 decoder->pge = true; 2248 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 2249 decoder->ip); 2250 decoder->state.from_ip = 0; 2251 if (decoder->packet.count == 0) { 2252 decoder->state.to_ip = 0; 2253 } else { 2254 intel_pt_set_ip(decoder); 2255 decoder->state.to_ip = decoder->ip; 2256 } 2257 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2258 intel_pt_mtc_cyc_cnt_pge(decoder); 2259 intel_pt_set_nr(decoder); 2260 return 0; 2261 2262 case INTEL_PT_TIP: 2263 decoder->state.from_ip = decoder->ip; 2264 if (decoder->packet.count == 0) { 2265 decoder->state.to_ip = 0; 2266 } else { 2267 intel_pt_set_ip(decoder); 2268 decoder->state.to_ip = decoder->ip; 2269 } 2270 intel_pt_update_nr(decoder); 2271 intel_pt_sample_iflag_chg(decoder); 2272 return 0; 2273 2274 case INTEL_PT_PIP: 2275 intel_pt_update_pip(decoder); 2276 break; 2277 2278 case INTEL_PT_MTC: 2279 intel_pt_calc_mtc_timestamp(decoder); 2280 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 2281 decoder->state.type |= INTEL_PT_INSTRUCTION; 2282 break; 2283 2284 case INTEL_PT_CYC: 2285 intel_pt_calc_cyc_timestamp(decoder); 2286 break; 2287 2288 case INTEL_PT_MODE_EXEC: 2289 intel_pt_mode_exec(decoder); 2290 break; 2291 2292 case INTEL_PT_VMCS: 2293 case INTEL_PT_MNT: 2294 case INTEL_PT_PAD: 2295 break; 2296 2297 default: 2298 return intel_pt_bug(decoder); 2299 } 2300 } 2301 } 2302 2303 static int intel_pt_resample(struct intel_pt_decoder *decoder) 2304 { 2305 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2306 decoder->state.type = INTEL_PT_INSTRUCTION; 2307 decoder->state.from_ip = decoder->ip; 2308 decoder->state.to_ip = 0; 2309 return 0; 2310 } 2311 2312 struct intel_pt_vm_tsc_info { 2313 struct intel_pt_pkt pip_packet; 2314 struct intel_pt_pkt vmcs_packet; 2315 struct intel_pt_pkt tma_packet; 2316 bool tsc, pip, vmcs, tma, psbend; 2317 uint64_t ctc_delta; 2318 uint64_t last_ctc; 2319 int max_lookahead; 2320 }; 2321 2322 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */ 2323 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2324 { 2325 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2326 2327 switch (pkt_info->packet.type) { 2328 case INTEL_PT_PAD: 2329 case INTEL_PT_MNT: 2330 case INTEL_PT_MODE_EXEC: 2331 case INTEL_PT_MODE_TSX: 2332 case INTEL_PT_MTC: 2333 case INTEL_PT_FUP: 2334 case INTEL_PT_CYC: 2335 case INTEL_PT_CBR: 2336 break; 2337 2338 case INTEL_PT_TSC: 2339 data->tsc = true; 2340 break; 2341 2342 case INTEL_PT_TMA: 2343 data->tma_packet = pkt_info->packet; 2344 data->tma = true; 2345 break; 2346 2347 case INTEL_PT_PIP: 2348 data->pip_packet = pkt_info->packet; 2349 data->pip = true; 2350 break; 2351 2352 case INTEL_PT_VMCS: 2353 data->vmcs_packet = pkt_info->packet; 2354 data->vmcs = true; 2355 break; 2356 2357 case INTEL_PT_PSBEND: 2358 data->psbend = true; 2359 return 1; 2360 2361 case INTEL_PT_TIP_PGE: 2362 case INTEL_PT_PTWRITE: 2363 case INTEL_PT_PTWRITE_IP: 2364 case INTEL_PT_EXSTOP: 2365 case INTEL_PT_EXSTOP_IP: 2366 case INTEL_PT_MWAIT: 2367 case INTEL_PT_PWRE: 2368 case INTEL_PT_PWRX: 2369 case INTEL_PT_BBP: 2370 case INTEL_PT_BIP: 2371 case INTEL_PT_BEP: 2372 case INTEL_PT_BEP_IP: 2373 case INTEL_PT_OVF: 2374 case INTEL_PT_BAD: 2375 case INTEL_PT_TNT: 2376 case INTEL_PT_TIP_PGD: 2377 case INTEL_PT_TIP: 2378 case INTEL_PT_PSB: 2379 case INTEL_PT_TRACESTOP: 2380 case INTEL_PT_CFE: 2381 case INTEL_PT_CFE_IP: 2382 case INTEL_PT_EVD: 2383 default: 2384 return 1; 2385 } 2386 2387 return 0; 2388 } 2389 2390 struct intel_pt_ovf_fup_info { 2391 int max_lookahead; 2392 bool found; 2393 }; 2394 2395 /* Lookahead to detect a FUP packet after OVF */ 2396 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2397 { 2398 struct intel_pt_ovf_fup_info *data = pkt_info->data; 2399 2400 if (pkt_info->packet.type == INTEL_PT_CYC || 2401 pkt_info->packet.type == INTEL_PT_MTC || 2402 pkt_info->packet.type == INTEL_PT_TSC) 2403 return !--(data->max_lookahead); 2404 data->found = pkt_info->packet.type == INTEL_PT_FUP; 2405 return 1; 2406 } 2407 2408 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder) 2409 { 2410 struct intel_pt_ovf_fup_info data = { 2411 .max_lookahead = 16, 2412 .found = false, 2413 }; 2414 2415 intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data); 2416 return data.found; 2417 } 2418 2419 /* Lookahead and get the TMA packet after TSC */ 2420 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2421 { 2422 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2423 2424 if (pkt_info->packet.type == INTEL_PT_CYC || 2425 pkt_info->packet.type == INTEL_PT_MTC) 2426 return !--(data->max_lookahead); 2427 2428 if (pkt_info->packet.type == INTEL_PT_TMA) { 2429 data->tma_packet = pkt_info->packet; 2430 data->tma = true; 2431 } 2432 return 1; 2433 } 2434 2435 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc) 2436 { 2437 if (decoder->tsc_ctc_mult) 2438 return ctc * decoder->tsc_ctc_mult; 2439 else 2440 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); 2441 } 2442 2443 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder, 2444 uint32_t ctc, 2445 uint32_t fc, 2446 uint64_t last_ctc_timestamp, 2447 uint64_t ctc_delta, 2448 uint32_t last_ctc) 2449 { 2450 /* Number of CTC ticks from last_ctc_timestamp to last_mtc */ 2451 uint64_t last_mtc_ctc = last_ctc + ctc_delta; 2452 /* 2453 * Number of CTC ticks from there until current TMA packet. We would 2454 * expect last_mtc_ctc to be before ctc, but the TSC packet can slip 2455 * past an MTC, so a sign-extended value is used. 2456 */ 2457 uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); 2458 /* Total CTC ticks from last_ctc_timestamp to current TMA packet */ 2459 uint64_t new_ctc_delta = ctc_delta + delta; 2460 uint64_t expected_tsc; 2461 2462 /* 2463 * Convert CTC ticks to TSC ticks, add the starting point 2464 * (last_ctc_timestamp) and the fast counter from the TMA packet. 2465 */ 2466 expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc; 2467 2468 if (intel_pt_enable_logging) { 2469 intel_pt_log_x64(last_mtc_ctc); 2470 intel_pt_log_x32(last_ctc); 2471 intel_pt_log_x64(ctc_delta); 2472 intel_pt_log_x64(delta); 2473 intel_pt_log_x32(ctc); 2474 intel_pt_log_x64(new_ctc_delta); 2475 intel_pt_log_x64(last_ctc_timestamp); 2476 intel_pt_log_x32(fc); 2477 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta)); 2478 intel_pt_log_x64(expected_tsc); 2479 } 2480 2481 return expected_tsc; 2482 } 2483 2484 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder, 2485 struct intel_pt_vm_tsc_info *data) 2486 { 2487 uint32_t ctc = data->tma_packet.payload; 2488 uint32_t fc = data->tma_packet.count; 2489 2490 return intel_pt_calc_expected_tsc(decoder, ctc, fc, 2491 decoder->ctc_timestamp, 2492 data->ctc_delta, data->last_ctc); 2493 } 2494 2495 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder, 2496 struct intel_pt_vmcs_info *vmcs_info) 2497 { 2498 uint64_t payload = decoder->packet.payload; 2499 2500 /* VMX adds the TSC Offset, so subtract to get host TSC */ 2501 decoder->packet.payload -= vmcs_info->tsc_offset; 2502 /* TSC packet has only 7 bytes */ 2503 decoder->packet.payload &= SEVEN_BYTES; 2504 2505 /* 2506 * The buffer is mmapped from the data file, so this also updates the 2507 * data file. 2508 */ 2509 if (!decoder->vm_tm_corr_dry_run) 2510 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7); 2511 2512 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2513 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2514 payload, decoder->packet.payload, vmcs_info->vmcs, 2515 vmcs_info->tsc_offset); 2516 } 2517 2518 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder, 2519 uint64_t tsc_offset) 2520 { 2521 struct intel_pt_vmcs_info vmcs_info = { 2522 .vmcs = NO_VMCS, 2523 .tsc_offset = tsc_offset 2524 }; 2525 2526 intel_pt_translate_vm_tsc(decoder, &vmcs_info); 2527 } 2528 2529 static inline bool in_vm(uint64_t pip_payload) 2530 { 2531 return pip_payload & 1; 2532 } 2533 2534 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet) 2535 { 2536 return pip_packet->payload & 1; 2537 } 2538 2539 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info) 2540 { 2541 p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64, 2542 vmcs_info->vmcs, vmcs_info->tsc_offset); 2543 } 2544 2545 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder, 2546 struct intel_pt_vm_tsc_info *data) 2547 { 2548 memset(data, 0, sizeof(*data)); 2549 data->ctc_delta = decoder->ctc_delta; 2550 data->last_ctc = decoder->last_ctc; 2551 intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data); 2552 if (data->tsc && !data->psbend) 2553 p_log("ERROR: PSB without PSBEND"); 2554 decoder->in_psb = data->psbend; 2555 } 2556 2557 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder, 2558 struct intel_pt_vm_tsc_info *data, 2559 struct intel_pt_vmcs_info *vmcs_info, 2560 uint64_t host_tsc) 2561 { 2562 if (!decoder->in_psb) { 2563 /* Can't happen */ 2564 p_log("ERROR: First TSC is not in PSB+"); 2565 } 2566 2567 if (data->pip) { 2568 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2569 if (vmcs_info && vmcs_info->tsc_offset) { 2570 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2571 decoder->vm_tm_corr_reliable = true; 2572 } else { 2573 p_log("ERROR: First TSC, unknown TSC Offset"); 2574 } 2575 } else { /* Host */ 2576 decoder->vm_tm_corr_reliable = true; 2577 } 2578 } else { /* Host or Guest */ 2579 decoder->vm_tm_corr_reliable = false; 2580 if (intel_pt_time_in_range(decoder, host_tsc)) { 2581 /* Assume Host */ 2582 } else { 2583 /* Assume Guest */ 2584 if (vmcs_info && vmcs_info->tsc_offset) 2585 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2586 else 2587 p_log("ERROR: First TSC, no PIP, unknown TSC Offset"); 2588 } 2589 } 2590 } 2591 2592 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder, 2593 struct intel_pt_vm_tsc_info *data) 2594 { 2595 struct intel_pt_vmcs_info *vmcs_info; 2596 uint64_t tsc_offset = 0; 2597 uint64_t vmcs; 2598 bool reliable = true; 2599 uint64_t expected_tsc; 2600 uint64_t host_tsc; 2601 uint64_t ref_timestamp; 2602 2603 bool assign = false; 2604 bool assign_reliable = false; 2605 2606 /* Already have 'data' for the in_psb case */ 2607 if (!decoder->in_psb) { 2608 memset(data, 0, sizeof(*data)); 2609 data->ctc_delta = decoder->ctc_delta; 2610 data->last_ctc = decoder->last_ctc; 2611 data->max_lookahead = 16; 2612 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data); 2613 if (decoder->pge) { 2614 data->pip = true; 2615 data->pip_packet.payload = decoder->pip_payload; 2616 } 2617 } 2618 2619 /* Calculations depend on having TMA packets */ 2620 if (!data->tma) { 2621 p_log("ERROR: TSC without TMA"); 2622 return; 2623 } 2624 2625 vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; 2626 if (vmcs == NO_VMCS) 2627 vmcs = 0; 2628 2629 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2630 2631 ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; 2632 host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp); 2633 2634 if (!decoder->ctc_timestamp) { 2635 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc); 2636 return; 2637 } 2638 2639 expected_tsc = intel_pt_expected_tsc(decoder, data); 2640 2641 tsc_offset = host_tsc - expected_tsc; 2642 2643 /* Determine if TSC is from Host or Guest */ 2644 if (data->pip) { 2645 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2646 if (!vmcs_info) { 2647 /* PIP NR=1 without VMCS cannot happen */ 2648 p_log("ERROR: Missing VMCS"); 2649 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2650 decoder->vm_tm_corr_reliable = false; 2651 return; 2652 } 2653 } else { /* Host */ 2654 decoder->last_reliable_timestamp = host_tsc; 2655 decoder->vm_tm_corr_reliable = true; 2656 return; 2657 } 2658 } else { /* Host or Guest */ 2659 reliable = false; /* Host/Guest is a guess, so not reliable */ 2660 if (decoder->in_psb) { 2661 if (!tsc_offset) 2662 return; /* Zero TSC Offset, assume Host */ 2663 /* 2664 * TSC packet has only 7 bytes of TSC. We have no 2665 * information about the Guest's 8th byte, but it 2666 * doesn't matter because we only need 7 bytes. 2667 * Here, since the 8th byte is unreliable and 2668 * irrelevant, compare only 7 byes. 2669 */ 2670 if (vmcs_info && 2671 (tsc_offset & SEVEN_BYTES) == 2672 (vmcs_info->tsc_offset & SEVEN_BYTES)) { 2673 /* Same TSC Offset as last VMCS, assume Guest */ 2674 goto guest; 2675 } 2676 } 2677 /* 2678 * Check if the host_tsc is within the expected range. 2679 * Note, we could narrow the range more by looking ahead for 2680 * the next host TSC in the same buffer, but we don't bother to 2681 * do that because this is probably good enough. 2682 */ 2683 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) { 2684 /* Within expected range for Host TSC, assume Host */ 2685 decoder->vm_tm_corr_reliable = false; 2686 return; 2687 } 2688 } 2689 2690 guest: /* Assuming Guest */ 2691 2692 /* Determine whether to assign TSC Offset */ 2693 if (vmcs_info && vmcs_info->vmcs) { 2694 if (vmcs_info->tsc_offset && vmcs_info->reliable) { 2695 assign = false; 2696 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && 2697 decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { 2698 /* Continuous tracing, TSC in a PSB is not a time loss */ 2699 assign = true; 2700 assign_reliable = true; 2701 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { 2702 /* 2703 * Unlikely to be a time loss TSC in a PSB which is not 2704 * at the start of a buffer. 2705 */ 2706 assign = true; 2707 assign_reliable = false; 2708 } 2709 } 2710 2711 /* Record VMCS TSC Offset */ 2712 if (assign && (vmcs_info->tsc_offset != tsc_offset || 2713 vmcs_info->reliable != assign_reliable)) { 2714 bool print = vmcs_info->tsc_offset != tsc_offset; 2715 2716 vmcs_info->tsc_offset = tsc_offset; 2717 vmcs_info->reliable = assign_reliable; 2718 if (print) 2719 intel_pt_print_vmcs_info(vmcs_info); 2720 } 2721 2722 /* Determine what TSC Offset to use */ 2723 if (vmcs_info && vmcs_info->tsc_offset) { 2724 if (!vmcs_info->reliable) 2725 reliable = false; 2726 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2727 } else { 2728 reliable = false; 2729 if (vmcs_info) { 2730 if (!vmcs_info->error_printed) { 2731 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64, 2732 vmcs_info->vmcs); 2733 vmcs_info->error_printed = true; 2734 } 2735 } else { 2736 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2737 p_log("ERROR: Unknown VMCS"); 2738 } 2739 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2740 } 2741 2742 decoder->vm_tm_corr_reliable = reliable; 2743 } 2744 2745 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder) 2746 { 2747 uint64_t host_tsc = decoder->packet.payload; 2748 uint64_t guest_tsc = decoder->packet.payload; 2749 struct intel_pt_vmcs_info *vmcs_info; 2750 uint64_t vmcs; 2751 2752 vmcs = decoder->vmcs; 2753 if (vmcs == NO_VMCS) 2754 vmcs = 0; 2755 2756 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2757 2758 if (decoder->pge) { 2759 if (in_vm(decoder->pip_payload)) { /* Guest */ 2760 if (!vmcs_info) { 2761 /* PIP NR=1 without VMCS cannot happen */ 2762 p_log("ERROR: Missing VMCS"); 2763 } 2764 } else { /* Host */ 2765 return; 2766 } 2767 } else { /* Host or Guest */ 2768 if (intel_pt_time_in_range(decoder, host_tsc)) { 2769 /* Within expected range for Host TSC, assume Host */ 2770 return; 2771 } 2772 } 2773 2774 if (vmcs_info) { 2775 /* Translate Guest TSC to Host TSC */ 2776 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; 2777 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); 2778 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2779 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2780 guest_tsc, host_tsc, vmcs_info->vmcs, 2781 vmcs_info->tsc_offset); 2782 if (!intel_pt_time_in_range(decoder, host_tsc) && 2783 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 2784 p_log("Timestamp out of range"); 2785 } else { 2786 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2787 p_log("ERROR: Unknown VMCS"); 2788 host_tsc = decoder->timestamp; 2789 } 2790 2791 decoder->packet.payload = host_tsc; 2792 2793 if (!decoder->vm_tm_corr_dry_run) 2794 memcpy((void *)decoder->buf + 1, &host_tsc, 8); 2795 } 2796 2797 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder) 2798 { 2799 struct intel_pt_vm_tsc_info data = { .psbend = false }; 2800 bool pge; 2801 int err; 2802 2803 if (decoder->in_psb) 2804 intel_pt_vm_tm_corr_psb(decoder, &data); 2805 2806 while (1) { 2807 err = intel_pt_get_next_packet(decoder); 2808 if (err == -ENOLINK) 2809 continue; 2810 if (err) 2811 break; 2812 2813 switch (decoder->packet.type) { 2814 case INTEL_PT_TIP_PGD: 2815 decoder->pge = false; 2816 decoder->vm_tm_corr_continuous = false; 2817 break; 2818 2819 case INTEL_PT_TNT: 2820 case INTEL_PT_TIP: 2821 case INTEL_PT_TIP_PGE: 2822 decoder->pge = true; 2823 break; 2824 2825 case INTEL_PT_OVF: 2826 decoder->in_psb = false; 2827 pge = decoder->pge; 2828 decoder->pge = intel_pt_ovf_fup_lookahead(decoder); 2829 if (pge != decoder->pge) 2830 intel_pt_log("Surprising PGE change in OVF!"); 2831 if (!decoder->pge) 2832 decoder->vm_tm_corr_continuous = false; 2833 break; 2834 2835 case INTEL_PT_FUP: 2836 if (decoder->in_psb) 2837 decoder->pge = true; 2838 break; 2839 2840 case INTEL_PT_TRACESTOP: 2841 decoder->pge = false; 2842 decoder->vm_tm_corr_continuous = false; 2843 decoder->have_tma = false; 2844 break; 2845 2846 case INTEL_PT_PSB: 2847 intel_pt_vm_tm_corr_psb(decoder, &data); 2848 break; 2849 2850 case INTEL_PT_PIP: 2851 decoder->pip_payload = decoder->packet.payload; 2852 break; 2853 2854 case INTEL_PT_MTC: 2855 intel_pt_calc_mtc_timestamp(decoder); 2856 break; 2857 2858 case INTEL_PT_TSC: 2859 intel_pt_vm_tm_corr_tsc(decoder, &data); 2860 intel_pt_calc_tsc_timestamp(decoder); 2861 decoder->vm_tm_corr_same_buf = true; 2862 decoder->vm_tm_corr_continuous = decoder->pge; 2863 break; 2864 2865 case INTEL_PT_TMA: 2866 intel_pt_calc_tma(decoder); 2867 break; 2868 2869 case INTEL_PT_CYC: 2870 intel_pt_calc_cyc_timestamp(decoder); 2871 break; 2872 2873 case INTEL_PT_CBR: 2874 intel_pt_calc_cbr(decoder); 2875 break; 2876 2877 case INTEL_PT_PSBEND: 2878 decoder->in_psb = false; 2879 data.psbend = false; 2880 break; 2881 2882 case INTEL_PT_VMCS: 2883 if (decoder->packet.payload != NO_VMCS) 2884 decoder->vmcs = decoder->packet.payload; 2885 break; 2886 2887 case INTEL_PT_BBP: 2888 decoder->blk_type = decoder->packet.payload; 2889 break; 2890 2891 case INTEL_PT_BIP: 2892 if (decoder->blk_type == INTEL_PT_PEBS_BASIC && 2893 decoder->packet.count == 2) 2894 intel_pt_vm_tm_corr_pebs_tsc(decoder); 2895 break; 2896 2897 case INTEL_PT_BEP: 2898 case INTEL_PT_BEP_IP: 2899 decoder->blk_type = 0; 2900 break; 2901 2902 case INTEL_PT_CFE: 2903 case INTEL_PT_CFE_IP: 2904 case INTEL_PT_EVD: 2905 case INTEL_PT_MODE_EXEC: 2906 case INTEL_PT_MODE_TSX: 2907 case INTEL_PT_MNT: 2908 case INTEL_PT_PAD: 2909 case INTEL_PT_PTWRITE_IP: 2910 case INTEL_PT_PTWRITE: 2911 case INTEL_PT_MWAIT: 2912 case INTEL_PT_PWRE: 2913 case INTEL_PT_EXSTOP_IP: 2914 case INTEL_PT_EXSTOP: 2915 case INTEL_PT_PWRX: 2916 case INTEL_PT_BAD: /* Does not happen */ 2917 default: 2918 break; 2919 } 2920 } 2921 2922 return err; 2923 } 2924 2925 #define HOP_PROCESS 0 2926 #define HOP_IGNORE 1 2927 #define HOP_RETURN 2 2928 #define HOP_AGAIN 3 2929 2930 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 2931 2932 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 2933 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 2934 { 2935 *err = 0; 2936 2937 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 2938 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 2939 *err = intel_pt_scan_for_psb(decoder); 2940 if (*err) 2941 return HOP_RETURN; 2942 } 2943 2944 switch (decoder->packet.type) { 2945 case INTEL_PT_TNT: 2946 return HOP_IGNORE; 2947 2948 case INTEL_PT_TIP_PGD: 2949 decoder->pge = false; 2950 if (!decoder->packet.count) { 2951 intel_pt_set_nr(decoder); 2952 return HOP_IGNORE; 2953 } 2954 intel_pt_set_ip(decoder); 2955 decoder->state.type |= INTEL_PT_TRACE_END; 2956 decoder->state.from_ip = 0; 2957 decoder->state.to_ip = decoder->ip; 2958 intel_pt_update_nr(decoder); 2959 return HOP_RETURN; 2960 2961 case INTEL_PT_TIP: 2962 if (!decoder->packet.count) { 2963 intel_pt_set_nr(decoder); 2964 return HOP_IGNORE; 2965 } 2966 intel_pt_set_ip(decoder); 2967 decoder->state.type = INTEL_PT_INSTRUCTION; 2968 decoder->state.from_ip = decoder->ip; 2969 decoder->state.to_ip = 0; 2970 intel_pt_update_nr(decoder); 2971 intel_pt_sample_iflag_chg(decoder); 2972 return HOP_RETURN; 2973 2974 case INTEL_PT_FUP: 2975 if (!decoder->packet.count) 2976 return HOP_IGNORE; 2977 intel_pt_set_ip(decoder); 2978 if (decoder->set_fup_mwait || decoder->set_fup_pwre) 2979 *no_tip = true; 2980 if (!decoder->branch_enable || !decoder->pge) 2981 *no_tip = true; 2982 if (*no_tip) { 2983 decoder->state.type = INTEL_PT_INSTRUCTION; 2984 decoder->state.from_ip = decoder->ip; 2985 decoder->state.to_ip = 0; 2986 intel_pt_fup_event(decoder, *no_tip); 2987 return HOP_RETURN; 2988 } 2989 intel_pt_fup_event(decoder, *no_tip); 2990 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH; 2991 *err = intel_pt_walk_fup_tip(decoder); 2992 if (!*err && decoder->state.to_ip) 2993 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2994 return HOP_RETURN; 2995 2996 case INTEL_PT_PSB: 2997 decoder->state.psb_offset = decoder->pos; 2998 decoder->psb_ip = 0; 2999 decoder->last_ip = 0; 3000 decoder->have_last_ip = true; 3001 *err = intel_pt_walk_psbend(decoder); 3002 if (*err == -EAGAIN) 3003 return HOP_AGAIN; 3004 if (*err) 3005 return HOP_RETURN; 3006 decoder->state.type = INTEL_PT_PSB_EVT; 3007 if (decoder->psb_ip) { 3008 decoder->state.type |= INTEL_PT_INSTRUCTION; 3009 decoder->ip = decoder->psb_ip; 3010 } 3011 decoder->state.from_ip = decoder->psb_ip; 3012 decoder->state.to_ip = 0; 3013 return HOP_RETURN; 3014 3015 case INTEL_PT_BAD: 3016 case INTEL_PT_PAD: 3017 case INTEL_PT_TIP_PGE: 3018 case INTEL_PT_TSC: 3019 case INTEL_PT_TMA: 3020 case INTEL_PT_MODE_EXEC: 3021 case INTEL_PT_MODE_TSX: 3022 case INTEL_PT_MTC: 3023 case INTEL_PT_CYC: 3024 case INTEL_PT_VMCS: 3025 case INTEL_PT_PSBEND: 3026 case INTEL_PT_CBR: 3027 case INTEL_PT_TRACESTOP: 3028 case INTEL_PT_PIP: 3029 case INTEL_PT_OVF: 3030 case INTEL_PT_MNT: 3031 case INTEL_PT_PTWRITE: 3032 case INTEL_PT_PTWRITE_IP: 3033 case INTEL_PT_EXSTOP: 3034 case INTEL_PT_EXSTOP_IP: 3035 case INTEL_PT_MWAIT: 3036 case INTEL_PT_PWRE: 3037 case INTEL_PT_PWRX: 3038 case INTEL_PT_BBP: 3039 case INTEL_PT_BIP: 3040 case INTEL_PT_BEP: 3041 case INTEL_PT_BEP_IP: 3042 case INTEL_PT_CFE: 3043 case INTEL_PT_CFE_IP: 3044 case INTEL_PT_EVD: 3045 default: 3046 return HOP_PROCESS; 3047 } 3048 } 3049 3050 struct intel_pt_psb_info { 3051 struct intel_pt_pkt fup_packet; 3052 bool fup; 3053 int after_psbend; 3054 }; 3055 3056 /* Lookahead and get the FUP packet from PSB+ */ 3057 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 3058 { 3059 struct intel_pt_psb_info *data = pkt_info->data; 3060 3061 switch (pkt_info->packet.type) { 3062 case INTEL_PT_PAD: 3063 case INTEL_PT_MNT: 3064 case INTEL_PT_TSC: 3065 case INTEL_PT_TMA: 3066 case INTEL_PT_MODE_EXEC: 3067 case INTEL_PT_MODE_TSX: 3068 case INTEL_PT_MTC: 3069 case INTEL_PT_CYC: 3070 case INTEL_PT_VMCS: 3071 case INTEL_PT_CBR: 3072 case INTEL_PT_PIP: 3073 if (data->after_psbend) { 3074 data->after_psbend -= 1; 3075 if (!data->after_psbend) 3076 return 1; 3077 } 3078 break; 3079 3080 case INTEL_PT_FUP: 3081 if (data->after_psbend) 3082 return 1; 3083 if (data->fup || pkt_info->packet.count == 0) 3084 return 1; 3085 data->fup_packet = pkt_info->packet; 3086 data->fup = true; 3087 break; 3088 3089 case INTEL_PT_PSBEND: 3090 if (!data->fup) 3091 return 1; 3092 /* Keep going to check for a TIP.PGE */ 3093 data->after_psbend = 6; 3094 break; 3095 3096 case INTEL_PT_TIP_PGE: 3097 /* Ignore FUP in PSB+ if followed by TIP.PGE */ 3098 if (data->after_psbend) 3099 data->fup = false; 3100 return 1; 3101 3102 case INTEL_PT_PTWRITE: 3103 case INTEL_PT_PTWRITE_IP: 3104 case INTEL_PT_EXSTOP: 3105 case INTEL_PT_EXSTOP_IP: 3106 case INTEL_PT_MWAIT: 3107 case INTEL_PT_PWRE: 3108 case INTEL_PT_PWRX: 3109 case INTEL_PT_BBP: 3110 case INTEL_PT_BIP: 3111 case INTEL_PT_BEP: 3112 case INTEL_PT_BEP_IP: 3113 case INTEL_PT_CFE: 3114 case INTEL_PT_CFE_IP: 3115 case INTEL_PT_EVD: 3116 if (data->after_psbend) { 3117 data->after_psbend -= 1; 3118 if (!data->after_psbend) 3119 return 1; 3120 break; 3121 } 3122 return 1; 3123 3124 case INTEL_PT_OVF: 3125 case INTEL_PT_BAD: 3126 case INTEL_PT_TNT: 3127 case INTEL_PT_TIP_PGD: 3128 case INTEL_PT_TIP: 3129 case INTEL_PT_PSB: 3130 case INTEL_PT_TRACESTOP: 3131 default: 3132 return 1; 3133 } 3134 3135 return 0; 3136 } 3137 3138 static int intel_pt_psb(struct intel_pt_decoder *decoder) 3139 { 3140 int err; 3141 3142 decoder->last_ip = 0; 3143 decoder->psb_ip = 0; 3144 decoder->have_last_ip = true; 3145 intel_pt_clear_stack(&decoder->stack); 3146 err = intel_pt_walk_psbend(decoder); 3147 if (err) 3148 return err; 3149 decoder->state.type = INTEL_PT_PSB_EVT; 3150 decoder->state.from_ip = decoder->psb_ip; 3151 decoder->state.to_ip = 0; 3152 return 0; 3153 } 3154 3155 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder) 3156 { 3157 int err; 3158 3159 if (decoder->ip != decoder->last_ip) { 3160 err = intel_pt_walk_fup(decoder); 3161 if (!err || err != -EAGAIN) 3162 return err; 3163 } 3164 3165 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3166 err = intel_pt_psb(decoder); 3167 if (err) { 3168 decoder->pkt_state = INTEL_PT_STATE_ERR3; 3169 return -ENOENT; 3170 } 3171 3172 return 0; 3173 } 3174 3175 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err) 3176 { 3177 struct intel_pt_psb_info data = { .fup = false }; 3178 3179 if (!decoder->branch_enable) 3180 return false; 3181 3182 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); 3183 if (!data.fup) 3184 return false; 3185 3186 decoder->packet = data.fup_packet; 3187 intel_pt_set_last_ip(decoder); 3188 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; 3189 3190 *err = intel_pt_fup_in_psb(decoder); 3191 3192 return true; 3193 } 3194 3195 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 3196 { 3197 int last_packet_type = INTEL_PT_PAD; 3198 bool no_tip = false; 3199 int err; 3200 3201 while (1) { 3202 err = intel_pt_get_next_packet(decoder); 3203 if (err) 3204 return err; 3205 next: 3206 err = 0; 3207 if (decoder->cyc_threshold) { 3208 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 3209 decoder->sample_cyc = false; 3210 last_packet_type = decoder->packet.type; 3211 } 3212 3213 if (decoder->hop) { 3214 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 3215 case HOP_IGNORE: 3216 continue; 3217 case HOP_RETURN: 3218 return err; 3219 case HOP_AGAIN: 3220 goto next; 3221 default: 3222 break; 3223 } 3224 } 3225 3226 switch (decoder->packet.type) { 3227 case INTEL_PT_TNT: 3228 if (!decoder->packet.count) 3229 break; 3230 decoder->tnt = decoder->packet; 3231 decoder->pkt_state = INTEL_PT_STATE_TNT; 3232 err = intel_pt_walk_tnt(decoder); 3233 if (err == -EAGAIN) 3234 break; 3235 return err; 3236 3237 case INTEL_PT_TIP_PGD: 3238 if (decoder->packet.count != 0) 3239 intel_pt_set_last_ip(decoder); 3240 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 3241 return intel_pt_walk_tip(decoder); 3242 3243 case INTEL_PT_TIP_PGE: { 3244 decoder->pge = true; 3245 decoder->overflow = false; 3246 intel_pt_mtc_cyc_cnt_pge(decoder); 3247 intel_pt_set_nr(decoder); 3248 if (decoder->packet.count == 0) { 3249 intel_pt_log_at("Skipping zero TIP.PGE", 3250 decoder->pos); 3251 break; 3252 } 3253 intel_pt_sample_iflag_chg(decoder); 3254 intel_pt_set_ip(decoder); 3255 decoder->state.from_ip = 0; 3256 decoder->state.to_ip = decoder->ip; 3257 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3258 /* 3259 * In hop mode, resample to get the to_ip as an 3260 * "instruction" sample. 3261 */ 3262 if (decoder->hop) 3263 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3264 return 0; 3265 } 3266 3267 case INTEL_PT_OVF: 3268 return intel_pt_overflow(decoder); 3269 3270 case INTEL_PT_TIP: 3271 if (decoder->packet.count != 0) 3272 intel_pt_set_last_ip(decoder); 3273 decoder->pkt_state = INTEL_PT_STATE_TIP; 3274 return intel_pt_walk_tip(decoder); 3275 3276 case INTEL_PT_FUP: 3277 if (decoder->packet.count == 0) { 3278 intel_pt_log_at("Skipping zero FUP", 3279 decoder->pos); 3280 no_tip = false; 3281 break; 3282 } 3283 intel_pt_set_last_ip(decoder); 3284 if (!decoder->branch_enable || !decoder->pge) { 3285 decoder->ip = decoder->last_ip; 3286 if (intel_pt_fup_event(decoder, no_tip)) 3287 return 0; 3288 no_tip = false; 3289 break; 3290 } 3291 if (decoder->set_fup_mwait) 3292 no_tip = true; 3293 if (no_tip) 3294 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 3295 else 3296 decoder->pkt_state = INTEL_PT_STATE_FUP; 3297 err = intel_pt_walk_fup(decoder); 3298 if (err != -EAGAIN) 3299 return err; 3300 if (no_tip) { 3301 no_tip = false; 3302 break; 3303 } 3304 return intel_pt_walk_fup_tip(decoder); 3305 3306 case INTEL_PT_TRACESTOP: 3307 decoder->pge = false; 3308 decoder->continuous_period = false; 3309 intel_pt_clear_tx_flags(decoder); 3310 decoder->have_tma = false; 3311 break; 3312 3313 case INTEL_PT_PSB: 3314 decoder->state.psb_offset = decoder->pos; 3315 decoder->psb_ip = 0; 3316 if (intel_pt_psb_with_fup(decoder, &err)) 3317 return err; 3318 err = intel_pt_psb(decoder); 3319 if (err == -EAGAIN) 3320 goto next; 3321 return err; 3322 3323 case INTEL_PT_PIP: 3324 intel_pt_update_pip(decoder); 3325 break; 3326 3327 case INTEL_PT_MTC: 3328 intel_pt_calc_mtc_timestamp(decoder); 3329 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 3330 break; 3331 /* 3332 * Ensure that there has been an instruction since the 3333 * last MTC. 3334 */ 3335 if (!decoder->mtc_insn) 3336 break; 3337 decoder->mtc_insn = false; 3338 /* Ensure that there is a timestamp */ 3339 if (!decoder->timestamp) 3340 break; 3341 decoder->state.type = INTEL_PT_INSTRUCTION; 3342 decoder->state.from_ip = decoder->ip; 3343 decoder->state.to_ip = 0; 3344 decoder->mtc_insn = false; 3345 return 0; 3346 3347 case INTEL_PT_TSC: 3348 intel_pt_calc_tsc_timestamp(decoder); 3349 break; 3350 3351 case INTEL_PT_TMA: 3352 intel_pt_calc_tma(decoder); 3353 break; 3354 3355 case INTEL_PT_CYC: 3356 intel_pt_calc_cyc_timestamp(decoder); 3357 break; 3358 3359 case INTEL_PT_CBR: 3360 intel_pt_calc_cbr(decoder); 3361 if (decoder->cbr != decoder->cbr_seen) { 3362 decoder->state.type = 0; 3363 return 0; 3364 } 3365 break; 3366 3367 case INTEL_PT_MODE_EXEC: 3368 intel_pt_mode_exec(decoder); 3369 err = intel_pt_get_next_packet(decoder); 3370 if (err) 3371 return err; 3372 if (decoder->packet.type == INTEL_PT_FUP) { 3373 decoder->set_fup_mode_exec = true; 3374 no_tip = true; 3375 } 3376 goto next; 3377 3378 case INTEL_PT_MODE_TSX: 3379 /* MODE_TSX need not be followed by FUP */ 3380 if (!decoder->pge || decoder->in_psb) { 3381 intel_pt_update_in_tx(decoder); 3382 break; 3383 } 3384 err = intel_pt_mode_tsx(decoder, &no_tip); 3385 if (err) 3386 return err; 3387 goto next; 3388 3389 case INTEL_PT_BAD: /* Does not happen */ 3390 return intel_pt_bug(decoder); 3391 3392 case INTEL_PT_PSBEND: 3393 case INTEL_PT_VMCS: 3394 case INTEL_PT_MNT: 3395 case INTEL_PT_PAD: 3396 break; 3397 3398 case INTEL_PT_PTWRITE_IP: 3399 decoder->fup_ptw_payload = decoder->packet.payload; 3400 err = intel_pt_get_next_packet(decoder); 3401 if (err) 3402 return err; 3403 if (decoder->packet.type == INTEL_PT_FUP) { 3404 decoder->set_fup_ptw = true; 3405 no_tip = true; 3406 } else { 3407 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 3408 decoder->pos); 3409 } 3410 goto next; 3411 3412 case INTEL_PT_PTWRITE: 3413 decoder->state.type = INTEL_PT_PTW; 3414 decoder->state.from_ip = decoder->ip; 3415 decoder->state.to_ip = 0; 3416 decoder->state.ptw_payload = decoder->packet.payload; 3417 return 0; 3418 3419 case INTEL_PT_MWAIT: 3420 decoder->fup_mwait_payload = decoder->packet.payload; 3421 decoder->set_fup_mwait = true; 3422 break; 3423 3424 case INTEL_PT_PWRE: 3425 if (decoder->set_fup_mwait) { 3426 decoder->fup_pwre_payload = 3427 decoder->packet.payload; 3428 decoder->set_fup_pwre = true; 3429 break; 3430 } 3431 decoder->state.type = INTEL_PT_PWR_ENTRY; 3432 decoder->state.from_ip = decoder->ip; 3433 decoder->state.to_ip = 0; 3434 decoder->state.pwrx_payload = decoder->packet.payload; 3435 return 0; 3436 3437 case INTEL_PT_EXSTOP_IP: 3438 err = intel_pt_get_next_packet(decoder); 3439 if (err) 3440 return err; 3441 if (decoder->packet.type == INTEL_PT_FUP) { 3442 decoder->set_fup_exstop = true; 3443 no_tip = true; 3444 } else { 3445 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 3446 decoder->pos); 3447 } 3448 goto next; 3449 3450 case INTEL_PT_EXSTOP: 3451 decoder->state.type = INTEL_PT_EX_STOP; 3452 decoder->state.from_ip = decoder->ip; 3453 decoder->state.to_ip = 0; 3454 return 0; 3455 3456 case INTEL_PT_PWRX: 3457 decoder->state.type = INTEL_PT_PWR_EXIT; 3458 decoder->state.from_ip = decoder->ip; 3459 decoder->state.to_ip = 0; 3460 decoder->state.pwrx_payload = decoder->packet.payload; 3461 return 0; 3462 3463 case INTEL_PT_BBP: 3464 intel_pt_bbp(decoder); 3465 break; 3466 3467 case INTEL_PT_BIP: 3468 intel_pt_bip(decoder); 3469 break; 3470 3471 case INTEL_PT_BEP: 3472 decoder->state.type = INTEL_PT_BLK_ITEMS; 3473 decoder->state.from_ip = decoder->ip; 3474 decoder->state.to_ip = 0; 3475 return 0; 3476 3477 case INTEL_PT_BEP_IP: 3478 err = intel_pt_get_next_packet(decoder); 3479 if (err) 3480 return err; 3481 if (decoder->packet.type == INTEL_PT_FUP) { 3482 decoder->set_fup_bep = true; 3483 no_tip = true; 3484 } else { 3485 intel_pt_log_at("ERROR: Missing FUP after BEP", 3486 decoder->pos); 3487 } 3488 goto next; 3489 3490 case INTEL_PT_CFE: 3491 decoder->fup_cfe_pkt = decoder->packet; 3492 decoder->set_fup_cfe = true; 3493 if (!decoder->pge) { 3494 intel_pt_fup_event(decoder, true); 3495 return 0; 3496 } 3497 break; 3498 3499 case INTEL_PT_CFE_IP: 3500 decoder->fup_cfe_pkt = decoder->packet; 3501 err = intel_pt_get_next_packet(decoder); 3502 if (err) 3503 return err; 3504 if (decoder->packet.type == INTEL_PT_FUP) { 3505 decoder->set_fup_cfe_ip = true; 3506 no_tip = true; 3507 } else { 3508 intel_pt_log_at("ERROR: Missing FUP after CFE", 3509 decoder->pos); 3510 } 3511 goto next; 3512 3513 case INTEL_PT_EVD: 3514 err = intel_pt_evd(decoder); 3515 if (err) 3516 return err; 3517 break; 3518 3519 default: 3520 return intel_pt_bug(decoder); 3521 } 3522 } 3523 } 3524 3525 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 3526 { 3527 return decoder->packet.count && 3528 (decoder->have_last_ip || decoder->packet.count == 3 || 3529 decoder->packet.count == 6); 3530 } 3531 3532 /* Walk PSB+ packets to get in sync. */ 3533 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 3534 { 3535 int err; 3536 3537 decoder->in_psb = true; 3538 3539 while (1) { 3540 err = intel_pt_get_next_packet(decoder); 3541 if (err) 3542 goto out; 3543 3544 switch (decoder->packet.type) { 3545 case INTEL_PT_TIP_PGD: 3546 decoder->continuous_period = false; 3547 __fallthrough; 3548 case INTEL_PT_TIP_PGE: 3549 case INTEL_PT_TIP: 3550 case INTEL_PT_PTWRITE: 3551 case INTEL_PT_PTWRITE_IP: 3552 case INTEL_PT_EXSTOP: 3553 case INTEL_PT_EXSTOP_IP: 3554 case INTEL_PT_MWAIT: 3555 case INTEL_PT_PWRE: 3556 case INTEL_PT_PWRX: 3557 case INTEL_PT_BBP: 3558 case INTEL_PT_BIP: 3559 case INTEL_PT_BEP: 3560 case INTEL_PT_BEP_IP: 3561 case INTEL_PT_CFE: 3562 case INTEL_PT_CFE_IP: 3563 case INTEL_PT_EVD: 3564 intel_pt_log("ERROR: Unexpected packet\n"); 3565 err = -ENOENT; 3566 goto out; 3567 3568 case INTEL_PT_FUP: 3569 decoder->pge = true; 3570 if (intel_pt_have_ip(decoder)) { 3571 uint64_t current_ip = decoder->ip; 3572 3573 intel_pt_set_ip(decoder); 3574 decoder->psb_ip = decoder->ip; 3575 if (current_ip) 3576 intel_pt_log_to("Setting IP", 3577 decoder->ip); 3578 } 3579 break; 3580 3581 case INTEL_PT_MTC: 3582 intel_pt_calc_mtc_timestamp(decoder); 3583 break; 3584 3585 case INTEL_PT_TSC: 3586 intel_pt_calc_tsc_timestamp(decoder); 3587 break; 3588 3589 case INTEL_PT_TMA: 3590 intel_pt_calc_tma(decoder); 3591 break; 3592 3593 case INTEL_PT_CYC: 3594 intel_pt_calc_cyc_timestamp(decoder); 3595 break; 3596 3597 case INTEL_PT_CBR: 3598 intel_pt_calc_cbr(decoder); 3599 break; 3600 3601 case INTEL_PT_PIP: 3602 intel_pt_set_pip(decoder); 3603 break; 3604 3605 case INTEL_PT_MODE_EXEC: 3606 intel_pt_mode_exec_status(decoder); 3607 break; 3608 3609 case INTEL_PT_MODE_TSX: 3610 intel_pt_update_in_tx(decoder); 3611 break; 3612 3613 case INTEL_PT_TRACESTOP: 3614 decoder->pge = false; 3615 decoder->continuous_period = false; 3616 intel_pt_clear_tx_flags(decoder); 3617 __fallthrough; 3618 3619 case INTEL_PT_TNT: 3620 decoder->have_tma = false; 3621 intel_pt_log("ERROR: Unexpected packet\n"); 3622 if (decoder->ip) 3623 decoder->pkt_state = INTEL_PT_STATE_ERR4; 3624 else 3625 decoder->pkt_state = INTEL_PT_STATE_ERR3; 3626 err = -ENOENT; 3627 goto out; 3628 3629 case INTEL_PT_BAD: /* Does not happen */ 3630 err = intel_pt_bug(decoder); 3631 goto out; 3632 3633 case INTEL_PT_OVF: 3634 err = intel_pt_overflow(decoder); 3635 goto out; 3636 3637 case INTEL_PT_PSBEND: 3638 err = 0; 3639 goto out; 3640 3641 case INTEL_PT_PSB: 3642 case INTEL_PT_VMCS: 3643 case INTEL_PT_MNT: 3644 case INTEL_PT_PAD: 3645 default: 3646 break; 3647 } 3648 } 3649 out: 3650 decoder->in_psb = false; 3651 3652 return err; 3653 } 3654 3655 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 3656 { 3657 int err; 3658 3659 while (1) { 3660 err = intel_pt_get_next_packet(decoder); 3661 if (err) 3662 return err; 3663 3664 switch (decoder->packet.type) { 3665 case INTEL_PT_TIP_PGD: 3666 decoder->continuous_period = false; 3667 decoder->pge = false; 3668 if (intel_pt_have_ip(decoder)) 3669 intel_pt_set_ip(decoder); 3670 if (!decoder->ip) 3671 break; 3672 decoder->state.type |= INTEL_PT_TRACE_END; 3673 return 0; 3674 3675 case INTEL_PT_TIP_PGE: 3676 decoder->pge = true; 3677 intel_pt_mtc_cyc_cnt_pge(decoder); 3678 if (intel_pt_have_ip(decoder)) 3679 intel_pt_set_ip(decoder); 3680 if (!decoder->ip) 3681 break; 3682 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3683 return 0; 3684 3685 case INTEL_PT_TIP: 3686 decoder->pge = true; 3687 if (intel_pt_have_ip(decoder)) 3688 intel_pt_set_ip(decoder); 3689 if (!decoder->ip) 3690 break; 3691 return 0; 3692 3693 case INTEL_PT_FUP: 3694 if (intel_pt_have_ip(decoder)) 3695 intel_pt_set_ip(decoder); 3696 if (decoder->ip) 3697 return 0; 3698 break; 3699 3700 case INTEL_PT_MTC: 3701 intel_pt_calc_mtc_timestamp(decoder); 3702 break; 3703 3704 case INTEL_PT_TSC: 3705 intel_pt_calc_tsc_timestamp(decoder); 3706 break; 3707 3708 case INTEL_PT_TMA: 3709 intel_pt_calc_tma(decoder); 3710 break; 3711 3712 case INTEL_PT_CYC: 3713 intel_pt_calc_cyc_timestamp(decoder); 3714 break; 3715 3716 case INTEL_PT_CBR: 3717 intel_pt_calc_cbr(decoder); 3718 break; 3719 3720 case INTEL_PT_PIP: 3721 intel_pt_set_pip(decoder); 3722 break; 3723 3724 case INTEL_PT_MODE_EXEC: 3725 intel_pt_mode_exec_status(decoder); 3726 break; 3727 3728 case INTEL_PT_MODE_TSX: 3729 intel_pt_update_in_tx(decoder); 3730 break; 3731 3732 case INTEL_PT_OVF: 3733 return intel_pt_overflow(decoder); 3734 3735 case INTEL_PT_BAD: /* Does not happen */ 3736 return intel_pt_bug(decoder); 3737 3738 case INTEL_PT_TRACESTOP: 3739 decoder->pge = false; 3740 decoder->continuous_period = false; 3741 intel_pt_clear_tx_flags(decoder); 3742 decoder->have_tma = false; 3743 break; 3744 3745 case INTEL_PT_PSB: 3746 decoder->state.psb_offset = decoder->pos; 3747 decoder->psb_ip = 0; 3748 decoder->last_ip = 0; 3749 decoder->have_last_ip = true; 3750 intel_pt_clear_stack(&decoder->stack); 3751 err = intel_pt_walk_psb(decoder); 3752 if (err) 3753 return err; 3754 decoder->state.type = INTEL_PT_PSB_EVT; 3755 decoder->state.from_ip = decoder->psb_ip; 3756 decoder->state.to_ip = 0; 3757 return 0; 3758 3759 case INTEL_PT_TNT: 3760 case INTEL_PT_PSBEND: 3761 case INTEL_PT_VMCS: 3762 case INTEL_PT_MNT: 3763 case INTEL_PT_PAD: 3764 case INTEL_PT_PTWRITE: 3765 case INTEL_PT_PTWRITE_IP: 3766 case INTEL_PT_EXSTOP: 3767 case INTEL_PT_EXSTOP_IP: 3768 case INTEL_PT_MWAIT: 3769 case INTEL_PT_PWRE: 3770 case INTEL_PT_PWRX: 3771 case INTEL_PT_BBP: 3772 case INTEL_PT_BIP: 3773 case INTEL_PT_BEP: 3774 case INTEL_PT_BEP_IP: 3775 case INTEL_PT_CFE: 3776 case INTEL_PT_CFE_IP: 3777 case INTEL_PT_EVD: 3778 default: 3779 break; 3780 } 3781 } 3782 } 3783 3784 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 3785 { 3786 int err; 3787 3788 intel_pt_clear_fup_event(decoder); 3789 decoder->overflow = false; 3790 3791 if (!decoder->branch_enable) { 3792 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3793 decoder->state.type = 0; /* Do not have a sample */ 3794 return 0; 3795 } 3796 3797 intel_pt_log("Scanning for full IP\n"); 3798 err = intel_pt_walk_to_ip(decoder); 3799 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) 3800 return err; 3801 3802 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 3803 if (decoder->hop) 3804 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3805 else 3806 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3807 3808 decoder->state.from_ip = 0; 3809 decoder->state.to_ip = decoder->ip; 3810 intel_pt_log_to("Setting IP", decoder->ip); 3811 3812 return 0; 3813 } 3814 3815 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 3816 { 3817 const unsigned char *end = decoder->buf + decoder->len; 3818 size_t i; 3819 3820 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 3821 if (i > decoder->len) 3822 continue; 3823 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 3824 return i; 3825 } 3826 return 0; 3827 } 3828 3829 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 3830 { 3831 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 3832 const char *psb = INTEL_PT_PSB_STR; 3833 3834 if (rest_psb > decoder->len || 3835 memcmp(decoder->buf, psb + part_psb, rest_psb)) 3836 return 0; 3837 3838 return rest_psb; 3839 } 3840 3841 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 3842 int part_psb) 3843 { 3844 int rest_psb, ret; 3845 3846 decoder->pos += decoder->len; 3847 decoder->len = 0; 3848 3849 ret = intel_pt_get_next_data(decoder, false); 3850 if (ret) 3851 return ret; 3852 3853 rest_psb = intel_pt_rest_psb(decoder, part_psb); 3854 if (!rest_psb) 3855 return 0; 3856 3857 decoder->pos -= part_psb; 3858 decoder->next_buf = decoder->buf + rest_psb; 3859 decoder->next_len = decoder->len - rest_psb; 3860 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3861 decoder->buf = decoder->temp_buf; 3862 decoder->len = INTEL_PT_PSB_LEN; 3863 3864 return 0; 3865 } 3866 3867 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 3868 { 3869 unsigned char *next; 3870 int ret; 3871 3872 intel_pt_log("Scanning for PSB\n"); 3873 while (1) { 3874 if (!decoder->len) { 3875 ret = intel_pt_get_next_data(decoder, false); 3876 if (ret) 3877 return ret; 3878 } 3879 3880 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 3881 INTEL_PT_PSB_LEN); 3882 if (!next) { 3883 int part_psb; 3884 3885 part_psb = intel_pt_part_psb(decoder); 3886 if (part_psb) { 3887 ret = intel_pt_get_split_psb(decoder, part_psb); 3888 if (ret) 3889 return ret; 3890 } else { 3891 decoder->pos += decoder->len; 3892 decoder->len = 0; 3893 } 3894 continue; 3895 } 3896 3897 decoder->pkt_step = next - decoder->buf; 3898 return intel_pt_get_next_packet(decoder); 3899 } 3900 } 3901 3902 static int intel_pt_sync(struct intel_pt_decoder *decoder) 3903 { 3904 int err; 3905 3906 decoder->pge = false; 3907 decoder->continuous_period = false; 3908 decoder->have_last_ip = false; 3909 decoder->last_ip = 0; 3910 decoder->psb_ip = 0; 3911 decoder->ip = 0; 3912 intel_pt_clear_stack(&decoder->stack); 3913 3914 err = intel_pt_scan_for_psb(decoder); 3915 if (err) 3916 return err; 3917 3918 if (decoder->vm_time_correlation) { 3919 decoder->in_psb = true; 3920 if (!decoder->timestamp) 3921 decoder->timestamp = 1; 3922 decoder->state.type = 0; 3923 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; 3924 return 0; 3925 } 3926 3927 decoder->have_last_ip = true; 3928 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3929 3930 err = intel_pt_walk_psb(decoder); 3931 if (err) 3932 return err; 3933 3934 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ 3935 decoder->state.from_ip = decoder->psb_ip; 3936 decoder->state.to_ip = 0; 3937 3938 if (decoder->ip) { 3939 /* 3940 * In hop mode, resample to get the PSB FUP ip as an 3941 * "instruction" sample. 3942 */ 3943 if (decoder->hop) 3944 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3945 else 3946 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3947 } 3948 3949 return 0; 3950 } 3951 3952 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 3953 { 3954 uint64_t est = decoder->sample_insn_cnt << 1; 3955 3956 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 3957 goto out; 3958 3959 est *= decoder->max_non_turbo_ratio; 3960 est /= decoder->cbr; 3961 out: 3962 return decoder->sample_timestamp + est; 3963 } 3964 3965 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 3966 { 3967 int err; 3968 3969 do { 3970 decoder->state.type = INTEL_PT_BRANCH; 3971 decoder->state.flags = 0; 3972 3973 switch (decoder->pkt_state) { 3974 case INTEL_PT_STATE_NO_PSB: 3975 err = intel_pt_sync(decoder); 3976 break; 3977 case INTEL_PT_STATE_NO_IP: 3978 decoder->have_last_ip = false; 3979 decoder->last_ip = 0; 3980 decoder->ip = 0; 3981 __fallthrough; 3982 case INTEL_PT_STATE_ERR_RESYNC: 3983 err = intel_pt_sync_ip(decoder); 3984 break; 3985 case INTEL_PT_STATE_IN_SYNC: 3986 err = intel_pt_walk_trace(decoder); 3987 break; 3988 case INTEL_PT_STATE_TNT: 3989 case INTEL_PT_STATE_TNT_CONT: 3990 err = intel_pt_walk_tnt(decoder); 3991 if (err == -EAGAIN) 3992 err = intel_pt_walk_trace(decoder); 3993 break; 3994 case INTEL_PT_STATE_TIP: 3995 case INTEL_PT_STATE_TIP_PGD: 3996 err = intel_pt_walk_tip(decoder); 3997 break; 3998 case INTEL_PT_STATE_FUP: 3999 err = intel_pt_walk_fup(decoder); 4000 if (err == -EAGAIN) 4001 err = intel_pt_walk_fup_tip(decoder); 4002 break; 4003 case INTEL_PT_STATE_FUP_NO_TIP: 4004 err = intel_pt_walk_fup(decoder); 4005 if (err == -EAGAIN) 4006 err = intel_pt_walk_trace(decoder); 4007 break; 4008 case INTEL_PT_STATE_FUP_IN_PSB: 4009 err = intel_pt_fup_in_psb(decoder); 4010 break; 4011 case INTEL_PT_STATE_RESAMPLE: 4012 err = intel_pt_resample(decoder); 4013 break; 4014 case INTEL_PT_STATE_VM_TIME_CORRELATION: 4015 err = intel_pt_vm_time_correlation(decoder); 4016 break; 4017 default: 4018 err = intel_pt_bug(decoder); 4019 break; 4020 } 4021 } while (err == -ENOLINK); 4022 4023 if (err) { 4024 decoder->state.err = intel_pt_ext_err(err); 4025 if (err != -EOVERFLOW) 4026 decoder->state.from_ip = decoder->ip; 4027 intel_pt_update_sample_time(decoder); 4028 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 4029 intel_pt_set_nr(decoder); 4030 } else { 4031 decoder->state.err = 0; 4032 if (decoder->cbr != decoder->cbr_seen) { 4033 decoder->cbr_seen = decoder->cbr; 4034 if (!decoder->state.type) { 4035 decoder->state.from_ip = decoder->ip; 4036 decoder->state.to_ip = 0; 4037 } 4038 decoder->state.type |= INTEL_PT_CBR_CHG; 4039 decoder->state.cbr_payload = decoder->cbr_payload; 4040 decoder->state.cbr = decoder->cbr; 4041 } 4042 if (intel_pt_sample_time(decoder->pkt_state)) { 4043 intel_pt_update_sample_time(decoder); 4044 if (decoder->sample_cyc) { 4045 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 4046 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 4047 decoder->sample_cyc = false; 4048 } 4049 } 4050 /* 4051 * When using only TSC/MTC to compute cycles, IPC can be 4052 * sampled as soon as the cycle count changes. 4053 */ 4054 if (!decoder->have_cyc) 4055 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 4056 } 4057 4058 /* Let PSB event always have TSC timestamp */ 4059 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) 4060 decoder->sample_timestamp = decoder->tsc_timestamp; 4061 4062 decoder->state.from_nr = decoder->nr; 4063 decoder->state.to_nr = decoder->next_nr; 4064 decoder->nr = decoder->next_nr; 4065 4066 decoder->state.timestamp = decoder->sample_timestamp; 4067 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 4068 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 4069 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 4070 4071 return &decoder->state; 4072 } 4073 4074 /** 4075 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 4076 * @buf: pointer to buffer pointer 4077 * @len: size of buffer 4078 * 4079 * Updates the buffer pointer to point to the start of the next PSB packet if 4080 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 4081 * @len is adjusted accordingly. 4082 * 4083 * Return: %true if a PSB packet is found, %false otherwise. 4084 */ 4085 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 4086 { 4087 unsigned char *next; 4088 4089 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 4090 if (next) { 4091 *len -= next - *buf; 4092 *buf = next; 4093 return true; 4094 } 4095 return false; 4096 } 4097 4098 /** 4099 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 4100 * packet. 4101 * @buf: pointer to buffer pointer 4102 * @len: size of buffer 4103 * 4104 * Updates the buffer pointer to point to the start of the following PSB packet 4105 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 4106 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 4107 * 4108 * Return: %true if a PSB packet is found, %false otherwise. 4109 */ 4110 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 4111 { 4112 unsigned char *next; 4113 4114 if (!*len) 4115 return false; 4116 4117 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 4118 if (next) { 4119 *len -= next - *buf; 4120 *buf = next; 4121 return true; 4122 } 4123 return false; 4124 } 4125 4126 /** 4127 * intel_pt_last_psb - find the last PSB packet in a buffer. 4128 * @buf: buffer 4129 * @len: size of buffer 4130 * 4131 * This function finds the last PSB in a buffer. 4132 * 4133 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 4134 */ 4135 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 4136 { 4137 const char *n = INTEL_PT_PSB_STR; 4138 unsigned char *p; 4139 size_t k; 4140 4141 if (len < INTEL_PT_PSB_LEN) 4142 return NULL; 4143 4144 k = len - INTEL_PT_PSB_LEN + 1; 4145 while (1) { 4146 p = memrchr(buf, n[0], k); 4147 if (!p) 4148 return NULL; 4149 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 4150 return p; 4151 k = p - buf; 4152 if (!k) 4153 return NULL; 4154 } 4155 } 4156 4157 /** 4158 * intel_pt_next_tsc - find and return next TSC. 4159 * @buf: buffer 4160 * @len: size of buffer 4161 * @tsc: TSC value returned 4162 * @rem: returns remaining size when TSC is found 4163 * 4164 * Find a TSC packet in @buf and return the TSC value. This function assumes 4165 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 4166 * PSBEND packet is found. 4167 * 4168 * Return: %true if TSC is found, false otherwise. 4169 */ 4170 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 4171 size_t *rem) 4172 { 4173 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 4174 struct intel_pt_pkt packet; 4175 int ret; 4176 4177 while (len) { 4178 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 4179 if (ret <= 0) 4180 return false; 4181 if (packet.type == INTEL_PT_TSC) { 4182 *tsc = packet.payload; 4183 *rem = len; 4184 return true; 4185 } 4186 if (packet.type == INTEL_PT_PSBEND) 4187 return false; 4188 buf += ret; 4189 len -= ret; 4190 } 4191 return false; 4192 } 4193 4194 /** 4195 * intel_pt_tsc_cmp - compare 7-byte TSCs. 4196 * @tsc1: first TSC to compare 4197 * @tsc2: second TSC to compare 4198 * 4199 * This function compares 7-byte TSC values allowing for the possibility that 4200 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 4201 * around so for that purpose this function assumes the absolute difference is 4202 * less than half the maximum difference. 4203 * 4204 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 4205 * after @tsc2. 4206 */ 4207 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 4208 { 4209 const uint64_t halfway = (1ULL << 55); 4210 4211 if (tsc1 == tsc2) 4212 return 0; 4213 4214 if (tsc1 < tsc2) { 4215 if (tsc2 - tsc1 < halfway) 4216 return -1; 4217 else 4218 return 1; 4219 } else { 4220 if (tsc1 - tsc2 < halfway) 4221 return 1; 4222 else 4223 return -1; 4224 } 4225 } 4226 4227 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 4228 4229 /** 4230 * adj_for_padding - adjust overlap to account for padding. 4231 * @buf_b: second buffer 4232 * @buf_a: first buffer 4233 * @len_a: size of first buffer 4234 * 4235 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 4236 * accordingly. 4237 * 4238 * Return: A pointer into @buf_b from where non-overlapped data starts 4239 */ 4240 static unsigned char *adj_for_padding(unsigned char *buf_b, 4241 unsigned char *buf_a, size_t len_a) 4242 { 4243 unsigned char *p = buf_b - MAX_PADDING; 4244 unsigned char *q = buf_a + len_a - MAX_PADDING; 4245 int i; 4246 4247 for (i = MAX_PADDING; i; i--, p++, q++) { 4248 if (*p != *q) 4249 break; 4250 } 4251 4252 return p; 4253 } 4254 4255 /** 4256 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 4257 * using TSC. 4258 * @buf_a: first buffer 4259 * @len_a: size of first buffer 4260 * @buf_b: second buffer 4261 * @len_b: size of second buffer 4262 * @consecutive: returns true if there is data in buf_b that is consecutive 4263 * to buf_a 4264 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 4265 * 4266 * If the trace contains TSC we can look at the last TSC of @buf_a and the 4267 * first TSC of @buf_b in order to determine if the buffers overlap, and then 4268 * walk forward in @buf_b until a later TSC is found. A precondition is that 4269 * @buf_a and @buf_b are positioned at a PSB. 4270 * 4271 * Return: A pointer into @buf_b from where non-overlapped data starts, or 4272 * @buf_b + @len_b if there is no non-overlapped data. 4273 */ 4274 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 4275 size_t len_a, 4276 unsigned char *buf_b, 4277 size_t len_b, bool *consecutive, 4278 bool ooo_tsc) 4279 { 4280 uint64_t tsc_a, tsc_b; 4281 unsigned char *p; 4282 size_t len, rem_a, rem_b; 4283 4284 p = intel_pt_last_psb(buf_a, len_a); 4285 if (!p) 4286 return buf_b; /* No PSB in buf_a => no overlap */ 4287 4288 len = len_a - (p - buf_a); 4289 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 4290 /* The last PSB+ in buf_a is incomplete, so go back one more */ 4291 len_a -= len; 4292 p = intel_pt_last_psb(buf_a, len_a); 4293 if (!p) 4294 return buf_b; /* No full PSB+ => assume no overlap */ 4295 len = len_a - (p - buf_a); 4296 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 4297 return buf_b; /* No TSC in buf_a => assume no overlap */ 4298 } 4299 4300 while (1) { 4301 /* Ignore PSB+ with no TSC */ 4302 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 4303 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 4304 4305 /* Same TSC, so buffers are consecutive */ 4306 if (!cmp && rem_b >= rem_a) { 4307 unsigned char *start; 4308 4309 *consecutive = true; 4310 start = buf_b + len_b - (rem_b - rem_a); 4311 return adj_for_padding(start, buf_a, len_a); 4312 } 4313 if (cmp < 0 && !ooo_tsc) 4314 return buf_b; /* tsc_a < tsc_b => no overlap */ 4315 } 4316 4317 if (!intel_pt_step_psb(&buf_b, &len_b)) 4318 return buf_b + len_b; /* No PSB in buf_b => no data */ 4319 } 4320 } 4321 4322 /** 4323 * intel_pt_find_overlap - determine start of non-overlapped trace data. 4324 * @buf_a: first buffer 4325 * @len_a: size of first buffer 4326 * @buf_b: second buffer 4327 * @len_b: size of second buffer 4328 * @have_tsc: can use TSC packets to detect overlap 4329 * @consecutive: returns true if there is data in buf_b that is consecutive 4330 * to buf_a 4331 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 4332 * 4333 * When trace samples or snapshots are recorded there is the possibility that 4334 * the data overlaps. Note that, for the purposes of decoding, data is only 4335 * useful if it begins with a PSB packet. 4336 * 4337 * Return: A pointer into @buf_b from where non-overlapped data starts, or 4338 * @buf_b + @len_b if there is no non-overlapped data. 4339 */ 4340 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 4341 unsigned char *buf_b, size_t len_b, 4342 bool have_tsc, bool *consecutive, 4343 bool ooo_tsc) 4344 { 4345 unsigned char *found; 4346 4347 /* Buffer 'b' must start at PSB so throw away everything before that */ 4348 if (!intel_pt_next_psb(&buf_b, &len_b)) 4349 return buf_b + len_b; /* No PSB */ 4350 4351 if (!intel_pt_next_psb(&buf_a, &len_a)) 4352 return buf_b; /* No overlap */ 4353 4354 if (have_tsc) { 4355 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 4356 consecutive, ooo_tsc); 4357 if (found) 4358 return found; 4359 } 4360 4361 /* 4362 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 4363 * we can ignore the first part of buffer 'a'. 4364 */ 4365 while (len_b < len_a) { 4366 if (!intel_pt_step_psb(&buf_a, &len_a)) 4367 return buf_b; /* No overlap */ 4368 } 4369 4370 /* Now len_b >= len_a */ 4371 while (1) { 4372 /* Potential overlap so check the bytes */ 4373 found = memmem(buf_a, len_a, buf_b, len_a); 4374 if (found) { 4375 *consecutive = true; 4376 return adj_for_padding(buf_b + len_a, buf_a, len_a); 4377 } 4378 4379 /* Try again at next PSB in buffer 'a' */ 4380 if (!intel_pt_step_psb(&buf_a, &len_a)) 4381 return buf_b; /* No overlap */ 4382 } 4383 } 4384 4385 /** 4386 * struct fast_forward_data - data used by intel_pt_ff_cb(). 4387 * @timestamp: timestamp to fast forward towards 4388 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 4389 * the fast forward timestamp. 4390 */ 4391 struct fast_forward_data { 4392 uint64_t timestamp; 4393 uint64_t buf_timestamp; 4394 }; 4395 4396 /** 4397 * intel_pt_ff_cb - fast forward lookahead callback. 4398 * @buffer: Intel PT trace buffer 4399 * @data: opaque pointer to fast forward data (struct fast_forward_data) 4400 * 4401 * Determine if @buffer trace is past the fast forward timestamp. 4402 * 4403 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 4404 * timestamp, and 0 otherwise. 4405 */ 4406 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 4407 { 4408 struct fast_forward_data *d = data; 4409 unsigned char *buf; 4410 uint64_t tsc; 4411 size_t rem; 4412 size_t len; 4413 4414 buf = (unsigned char *)buffer->buf; 4415 len = buffer->len; 4416 4417 if (!intel_pt_next_psb(&buf, &len) || 4418 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 4419 return 0; 4420 4421 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 4422 4423 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 4424 tsc, buffer->ref_timestamp); 4425 4426 /* 4427 * If the buffer contains a timestamp earlier that the fast forward 4428 * timestamp, then record it, else stop. 4429 */ 4430 if (tsc < d->timestamp) 4431 d->buf_timestamp = buffer->ref_timestamp; 4432 else 4433 return 1; 4434 4435 return 0; 4436 } 4437 4438 /** 4439 * intel_pt_fast_forward - reposition decoder forwards. 4440 * @decoder: Intel PT decoder 4441 * @timestamp: timestamp to fast forward towards 4442 * 4443 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 4444 * 4445 * Return: 0 on success or negative error code on failure. 4446 */ 4447 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 4448 { 4449 struct fast_forward_data d = { .timestamp = timestamp }; 4450 unsigned char *buf; 4451 size_t len; 4452 int err; 4453 4454 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 4455 4456 /* Find buffer timestamp of buffer to fast forward to */ 4457 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 4458 if (err < 0) 4459 return err; 4460 4461 /* Walk to buffer with same buffer timestamp */ 4462 if (d.buf_timestamp) { 4463 do { 4464 decoder->pos += decoder->len; 4465 decoder->len = 0; 4466 err = intel_pt_get_next_data(decoder, true); 4467 /* -ENOLINK means non-consecutive trace */ 4468 if (err && err != -ENOLINK) 4469 return err; 4470 } while (decoder->buf_timestamp != d.buf_timestamp); 4471 } 4472 4473 if (!decoder->buf) 4474 return 0; 4475 4476 buf = (unsigned char *)decoder->buf; 4477 len = decoder->len; 4478 4479 if (!intel_pt_next_psb(&buf, &len)) 4480 return 0; 4481 4482 /* 4483 * Walk PSBs while the PSB timestamp is less than the fast forward 4484 * timestamp. 4485 */ 4486 do { 4487 uint64_t tsc; 4488 size_t rem; 4489 4490 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 4491 break; 4492 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 4493 /* 4494 * A TSC packet can slip past MTC packets but, after fast 4495 * forward, decoding starts at the TSC timestamp. That means 4496 * the timestamps may not be exactly the same as the timestamps 4497 * that would have been decoded without fast forward. 4498 */ 4499 if (tsc < timestamp) { 4500 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 4501 decoder->pos += decoder->len - len; 4502 decoder->buf = buf; 4503 decoder->len = len; 4504 intel_pt_reposition(decoder); 4505 } else { 4506 break; 4507 } 4508 } while (intel_pt_step_psb(&buf, &len)); 4509 4510 return 0; 4511 } 4512