1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_pt_decoder.c: Intel Processor Trace support 4 * Copyright (c) 2013-2014, Intel Corporation. 5 */ 6 7 #ifndef _GNU_SOURCE 8 #define _GNU_SOURCE 9 #endif 10 #include <stdlib.h> 11 #include <stdbool.h> 12 #include <string.h> 13 #include <errno.h> 14 #include <stdint.h> 15 #include <inttypes.h> 16 #include <linux/compiler.h> 17 #include <linux/string.h> 18 #include <linux/zalloc.h> 19 20 #include "../auxtrace.h" 21 22 #include "intel-pt-insn-decoder.h" 23 #include "intel-pt-pkt-decoder.h" 24 #include "intel-pt-decoder.h" 25 #include "intel-pt-log.h" 26 27 #define BITULL(x) (1ULL << (x)) 28 29 /* IA32_RTIT_CTL MSR bits */ 30 #define INTEL_PT_CYC_ENABLE BITULL(1) 31 #define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19)) 32 #define INTEL_PT_CYC_THRESHOLD_SHIFT 19 33 34 #define INTEL_PT_BLK_SIZE 1024 35 36 #define BIT63 (((uint64_t)1 << 63)) 37 38 #define SEVEN_BYTES 0xffffffffffffffULL 39 40 #define NO_VMCS 0xffffffffffULL 41 42 #define INTEL_PT_RETURN 1 43 44 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */ 45 #define INTEL_PT_MAX_LOOPS 10000 46 47 struct intel_pt_blk { 48 struct intel_pt_blk *prev; 49 uint64_t ip[INTEL_PT_BLK_SIZE]; 50 }; 51 52 struct intel_pt_stack { 53 struct intel_pt_blk *blk; 54 struct intel_pt_blk *spare; 55 int pos; 56 }; 57 58 enum intel_pt_p_once { 59 INTEL_PT_PRT_ONCE_UNK_VMCS, 60 INTEL_PT_PRT_ONCE_ERANGE, 61 }; 62 63 enum intel_pt_pkt_state { 64 INTEL_PT_STATE_NO_PSB, 65 INTEL_PT_STATE_NO_IP, 66 INTEL_PT_STATE_ERR_RESYNC, 67 INTEL_PT_STATE_IN_SYNC, 68 INTEL_PT_STATE_TNT_CONT, 69 INTEL_PT_STATE_TNT, 70 INTEL_PT_STATE_TIP, 71 INTEL_PT_STATE_TIP_PGD, 72 INTEL_PT_STATE_FUP, 73 INTEL_PT_STATE_FUP_NO_TIP, 74 INTEL_PT_STATE_FUP_IN_PSB, 75 INTEL_PT_STATE_RESAMPLE, 76 INTEL_PT_STATE_VM_TIME_CORRELATION, 77 }; 78 79 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) 80 { 81 switch (pkt_state) { 82 case INTEL_PT_STATE_NO_PSB: 83 case INTEL_PT_STATE_NO_IP: 84 case INTEL_PT_STATE_ERR_RESYNC: 85 case INTEL_PT_STATE_IN_SYNC: 86 case INTEL_PT_STATE_TNT_CONT: 87 case INTEL_PT_STATE_RESAMPLE: 88 case INTEL_PT_STATE_VM_TIME_CORRELATION: 89 return true; 90 case INTEL_PT_STATE_TNT: 91 case INTEL_PT_STATE_TIP: 92 case INTEL_PT_STATE_TIP_PGD: 93 case INTEL_PT_STATE_FUP: 94 case INTEL_PT_STATE_FUP_NO_TIP: 95 case INTEL_PT_STATE_FUP_IN_PSB: 96 return false; 97 default: 98 return true; 99 }; 100 } 101 102 #ifdef INTEL_PT_STRICT 103 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB 104 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB 105 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB 106 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB 107 #else 108 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state) 109 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP 110 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC 111 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC 112 #endif 113 114 struct intel_pt_decoder { 115 int (*get_trace)(struct intel_pt_buffer *buffer, void *data); 116 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn, 117 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip, 118 uint64_t max_insn_cnt, void *data); 119 bool (*pgd_ip)(uint64_t ip, void *data); 120 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); 121 struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); 122 void *data; 123 struct intel_pt_state state; 124 const unsigned char *buf; 125 size_t len; 126 bool return_compression; 127 bool branch_enable; 128 bool mtc_insn; 129 bool pge; 130 bool have_tma; 131 bool have_cyc; 132 bool fixup_last_mtc; 133 bool have_last_ip; 134 bool in_psb; 135 bool hop; 136 bool leap; 137 bool vm_time_correlation; 138 bool vm_tm_corr_dry_run; 139 bool vm_tm_corr_reliable; 140 bool vm_tm_corr_same_buf; 141 bool vm_tm_corr_continuous; 142 bool nr; 143 bool next_nr; 144 enum intel_pt_param_flags flags; 145 uint64_t pos; 146 uint64_t last_ip; 147 uint64_t ip; 148 uint64_t pip_payload; 149 uint64_t timestamp; 150 uint64_t tsc_timestamp; 151 uint64_t ref_timestamp; 152 uint64_t buf_timestamp; 153 uint64_t sample_timestamp; 154 uint64_t ret_addr; 155 uint64_t ctc_timestamp; 156 uint64_t ctc_delta; 157 uint64_t cycle_cnt; 158 uint64_t cyc_ref_timestamp; 159 uint64_t first_timestamp; 160 uint64_t last_reliable_timestamp; 161 uint64_t vmcs; 162 uint64_t print_once; 163 uint64_t last_ctc; 164 uint32_t last_mtc; 165 uint32_t tsc_ctc_ratio_n; 166 uint32_t tsc_ctc_ratio_d; 167 uint32_t tsc_ctc_mult; 168 uint32_t tsc_slip; 169 uint32_t ctc_rem_mask; 170 int mtc_shift; 171 struct intel_pt_stack stack; 172 enum intel_pt_pkt_state pkt_state; 173 enum intel_pt_pkt_ctx pkt_ctx; 174 enum intel_pt_pkt_ctx prev_pkt_ctx; 175 enum intel_pt_blk_type blk_type; 176 int blk_type_pos; 177 struct intel_pt_pkt packet; 178 struct intel_pt_pkt tnt; 179 int pkt_step; 180 int pkt_len; 181 int last_packet_type; 182 unsigned int cbr; 183 unsigned int cbr_seen; 184 unsigned int max_non_turbo_ratio; 185 double max_non_turbo_ratio_fp; 186 double cbr_cyc_to_tsc; 187 double calc_cyc_to_tsc; 188 bool have_calc_cyc_to_tsc; 189 int exec_mode; 190 unsigned int insn_bytes; 191 uint64_t period; 192 enum intel_pt_period_type period_type; 193 uint64_t tot_insn_cnt; 194 uint64_t period_insn_cnt; 195 uint64_t period_mask; 196 uint64_t period_ticks; 197 uint64_t last_masked_timestamp; 198 uint64_t tot_cyc_cnt; 199 uint64_t sample_tot_cyc_cnt; 200 uint64_t base_cyc_cnt; 201 uint64_t cyc_cnt_timestamp; 202 uint64_t ctl; 203 uint64_t cyc_threshold; 204 double tsc_to_cyc; 205 bool continuous_period; 206 bool overflow; 207 bool set_fup_tx_flags; 208 bool set_fup_ptw; 209 bool set_fup_mwait; 210 bool set_fup_pwre; 211 bool set_fup_exstop; 212 bool set_fup_bep; 213 bool sample_cyc; 214 unsigned int fup_tx_flags; 215 unsigned int tx_flags; 216 uint64_t fup_ptw_payload; 217 uint64_t fup_mwait_payload; 218 uint64_t fup_pwre_payload; 219 uint64_t cbr_payload; 220 uint64_t timestamp_insn_cnt; 221 uint64_t sample_insn_cnt; 222 uint64_t stuck_ip; 223 int no_progress; 224 int stuck_ip_prd; 225 int stuck_ip_cnt; 226 uint64_t psb_ip; 227 const unsigned char *next_buf; 228 size_t next_len; 229 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ]; 230 }; 231 232 static uint64_t intel_pt_lower_power_of_2(uint64_t x) 233 { 234 int i; 235 236 for (i = 0; x != 1; i++) 237 x >>= 1; 238 239 return x << i; 240 } 241 242 __printf(1, 2) 243 static void p_log(const char *fmt, ...) 244 { 245 char buf[512]; 246 va_list args; 247 248 va_start(args, fmt); 249 vsnprintf(buf, sizeof(buf), fmt, args); 250 va_end(args); 251 252 fprintf(stderr, "%s\n", buf); 253 intel_pt_log("%s\n", buf); 254 } 255 256 static bool intel_pt_print_once(struct intel_pt_decoder *decoder, 257 enum intel_pt_p_once id) 258 { 259 uint64_t bit = 1ULL << id; 260 261 if (decoder->print_once & bit) 262 return false; 263 decoder->print_once |= bit; 264 return true; 265 } 266 267 static uint64_t intel_pt_cyc_threshold(uint64_t ctl) 268 { 269 if (!(ctl & INTEL_PT_CYC_ENABLE)) 270 return 0; 271 272 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT; 273 } 274 275 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) 276 { 277 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { 278 uint64_t period; 279 280 period = intel_pt_lower_power_of_2(decoder->period); 281 decoder->period_mask = ~(period - 1); 282 decoder->period_ticks = period; 283 } 284 } 285 286 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d) 287 { 288 if (!d) 289 return 0; 290 return (t / d) * n + ((t % d) * n) / d; 291 } 292 293 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) 294 { 295 struct intel_pt_decoder *decoder; 296 297 if (!params->get_trace || !params->walk_insn) 298 return NULL; 299 300 decoder = zalloc(sizeof(struct intel_pt_decoder)); 301 if (!decoder) 302 return NULL; 303 304 decoder->get_trace = params->get_trace; 305 decoder->walk_insn = params->walk_insn; 306 decoder->pgd_ip = params->pgd_ip; 307 decoder->lookahead = params->lookahead; 308 decoder->findnew_vmcs_info = params->findnew_vmcs_info; 309 decoder->data = params->data; 310 decoder->return_compression = params->return_compression; 311 decoder->branch_enable = params->branch_enable; 312 decoder->hop = params->quick >= 1; 313 decoder->leap = params->quick >= 2; 314 decoder->vm_time_correlation = params->vm_time_correlation; 315 decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; 316 decoder->first_timestamp = params->first_timestamp; 317 decoder->last_reliable_timestamp = params->first_timestamp; 318 319 decoder->flags = params->flags; 320 321 decoder->ctl = params->ctl; 322 decoder->period = params->period; 323 decoder->period_type = params->period_type; 324 325 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; 326 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; 327 328 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); 329 330 intel_pt_setup_period(decoder); 331 332 decoder->mtc_shift = params->mtc_period; 333 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; 334 335 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; 336 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; 337 338 if (!decoder->tsc_ctc_ratio_n) 339 decoder->tsc_ctc_ratio_d = 0; 340 341 if (decoder->tsc_ctc_ratio_d) { 342 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) 343 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / 344 decoder->tsc_ctc_ratio_d; 345 } 346 347 /* 348 * A TSC packet can slip past MTC packets so that the timestamp appears 349 * to go backwards. One estimate is that can be up to about 40 CPU 350 * cycles, which is certainly less than 0x1000 TSC ticks, but accept 351 * slippage an order of magnitude more to be on the safe side. 352 */ 353 decoder->tsc_slip = 0x10000; 354 355 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); 356 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); 357 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); 358 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); 359 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); 360 361 if (decoder->hop) 362 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n"); 363 364 return decoder; 365 } 366 367 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, 368 uint64_t first_timestamp) 369 { 370 decoder->first_timestamp = first_timestamp; 371 } 372 373 static void intel_pt_pop_blk(struct intel_pt_stack *stack) 374 { 375 struct intel_pt_blk *blk = stack->blk; 376 377 stack->blk = blk->prev; 378 if (!stack->spare) 379 stack->spare = blk; 380 else 381 free(blk); 382 } 383 384 static uint64_t intel_pt_pop(struct intel_pt_stack *stack) 385 { 386 if (!stack->pos) { 387 if (!stack->blk) 388 return 0; 389 intel_pt_pop_blk(stack); 390 if (!stack->blk) 391 return 0; 392 stack->pos = INTEL_PT_BLK_SIZE; 393 } 394 return stack->blk->ip[--stack->pos]; 395 } 396 397 static int intel_pt_alloc_blk(struct intel_pt_stack *stack) 398 { 399 struct intel_pt_blk *blk; 400 401 if (stack->spare) { 402 blk = stack->spare; 403 stack->spare = NULL; 404 } else { 405 blk = malloc(sizeof(struct intel_pt_blk)); 406 if (!blk) 407 return -ENOMEM; 408 } 409 410 blk->prev = stack->blk; 411 stack->blk = blk; 412 stack->pos = 0; 413 return 0; 414 } 415 416 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip) 417 { 418 int err; 419 420 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) { 421 err = intel_pt_alloc_blk(stack); 422 if (err) 423 return err; 424 } 425 426 stack->blk->ip[stack->pos++] = ip; 427 return 0; 428 } 429 430 static void intel_pt_clear_stack(struct intel_pt_stack *stack) 431 { 432 while (stack->blk) 433 intel_pt_pop_blk(stack); 434 stack->pos = 0; 435 } 436 437 static void intel_pt_free_stack(struct intel_pt_stack *stack) 438 { 439 intel_pt_clear_stack(stack); 440 zfree(&stack->blk); 441 zfree(&stack->spare); 442 } 443 444 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) 445 { 446 intel_pt_free_stack(&decoder->stack); 447 free(decoder); 448 } 449 450 static int intel_pt_ext_err(int code) 451 { 452 switch (code) { 453 case -ENOMEM: 454 return INTEL_PT_ERR_NOMEM; 455 case -ENOSYS: 456 return INTEL_PT_ERR_INTERN; 457 case -EBADMSG: 458 return INTEL_PT_ERR_BADPKT; 459 case -ENODATA: 460 return INTEL_PT_ERR_NODATA; 461 case -EILSEQ: 462 return INTEL_PT_ERR_NOINSN; 463 case -ENOENT: 464 return INTEL_PT_ERR_MISMAT; 465 case -EOVERFLOW: 466 return INTEL_PT_ERR_OVR; 467 case -ENOSPC: 468 return INTEL_PT_ERR_LOST; 469 case -ELOOP: 470 return INTEL_PT_ERR_NELOOP; 471 default: 472 return INTEL_PT_ERR_UNK; 473 } 474 } 475 476 static const char *intel_pt_err_msgs[] = { 477 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed", 478 [INTEL_PT_ERR_INTERN] = "Internal error", 479 [INTEL_PT_ERR_BADPKT] = "Bad packet", 480 [INTEL_PT_ERR_NODATA] = "No more data", 481 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction", 482 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction", 483 [INTEL_PT_ERR_OVR] = "Overflow packet", 484 [INTEL_PT_ERR_LOST] = "Lost trace data", 485 [INTEL_PT_ERR_UNK] = "Unknown error!", 486 [INTEL_PT_ERR_NELOOP] = "Never-ending loop", 487 }; 488 489 int intel_pt__strerror(int code, char *buf, size_t buflen) 490 { 491 if (code < 1 || code >= INTEL_PT_ERR_MAX) 492 code = INTEL_PT_ERR_UNK; 493 strlcpy(buf, intel_pt_err_msgs[code], buflen); 494 return 0; 495 } 496 497 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet, 498 uint64_t last_ip) 499 { 500 uint64_t ip; 501 502 switch (packet->count) { 503 case 1: 504 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) | 505 packet->payload; 506 break; 507 case 2: 508 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) | 509 packet->payload; 510 break; 511 case 3: 512 ip = packet->payload; 513 /* Sign-extend 6-byte ip */ 514 if (ip & (uint64_t)0x800000000000ULL) 515 ip |= (uint64_t)0xffff000000000000ULL; 516 break; 517 case 4: 518 ip = (last_ip & (uint64_t)0xffff000000000000ULL) | 519 packet->payload; 520 break; 521 case 6: 522 ip = packet->payload; 523 break; 524 default: 525 return 0; 526 } 527 528 return ip; 529 } 530 531 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) 532 { 533 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); 534 decoder->have_last_ip = true; 535 } 536 537 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) 538 { 539 intel_pt_set_last_ip(decoder); 540 decoder->ip = decoder->last_ip; 541 } 542 543 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) 544 { 545 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, 546 decoder->buf); 547 } 548 549 static int intel_pt_bug(struct intel_pt_decoder *decoder) 550 { 551 intel_pt_log("ERROR: Internal error\n"); 552 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 553 return -ENOSYS; 554 } 555 556 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) 557 { 558 decoder->tx_flags = 0; 559 } 560 561 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) 562 { 563 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; 564 } 565 566 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder) 567 { 568 decoder->pip_payload = decoder->packet.payload; 569 } 570 571 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder) 572 { 573 decoder->next_nr = decoder->pip_payload & 1; 574 } 575 576 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder) 577 { 578 decoder->nr = decoder->pip_payload & 1; 579 decoder->next_nr = decoder->nr; 580 } 581 582 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder) 583 { 584 intel_pt_update_pip(decoder); 585 intel_pt_set_nr(decoder); 586 } 587 588 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) 589 { 590 intel_pt_clear_tx_flags(decoder); 591 decoder->have_tma = false; 592 decoder->pkt_len = 1; 593 decoder->pkt_step = 1; 594 intel_pt_decoder_log_packet(decoder); 595 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { 596 intel_pt_log("ERROR: Bad packet\n"); 597 decoder->pkt_state = INTEL_PT_STATE_ERR1; 598 } 599 return -EBADMSG; 600 } 601 602 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) 603 { 604 decoder->sample_timestamp = decoder->timestamp; 605 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; 606 } 607 608 static void intel_pt_reposition(struct intel_pt_decoder *decoder) 609 { 610 decoder->ip = 0; 611 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; 612 decoder->timestamp = 0; 613 decoder->have_tma = false; 614 } 615 616 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) 617 { 618 struct intel_pt_buffer buffer = { .buf = 0, }; 619 int ret; 620 621 decoder->pkt_step = 0; 622 623 intel_pt_log("Getting more data\n"); 624 ret = decoder->get_trace(&buffer, decoder->data); 625 if (ret) 626 return ret; 627 decoder->buf = buffer.buf; 628 decoder->len = buffer.len; 629 if (!decoder->len) { 630 intel_pt_log("No more data\n"); 631 return -ENODATA; 632 } 633 decoder->buf_timestamp = buffer.ref_timestamp; 634 if (!buffer.consecutive || reposition) { 635 intel_pt_reposition(decoder); 636 decoder->ref_timestamp = buffer.ref_timestamp; 637 decoder->state.trace_nr = buffer.trace_nr; 638 decoder->vm_tm_corr_same_buf = false; 639 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", 640 decoder->ref_timestamp); 641 return -ENOLINK; 642 } 643 644 return 0; 645 } 646 647 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, 648 bool reposition) 649 { 650 if (!decoder->next_buf) 651 return intel_pt_get_data(decoder, reposition); 652 653 decoder->buf = decoder->next_buf; 654 decoder->len = decoder->next_len; 655 decoder->next_buf = 0; 656 decoder->next_len = 0; 657 return 0; 658 } 659 660 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) 661 { 662 unsigned char *buf = decoder->temp_buf; 663 size_t old_len, len, n; 664 int ret; 665 666 old_len = decoder->len; 667 len = decoder->len; 668 memcpy(buf, decoder->buf, len); 669 670 ret = intel_pt_get_data(decoder, false); 671 if (ret) { 672 decoder->pos += old_len; 673 return ret < 0 ? ret : -EINVAL; 674 } 675 676 n = INTEL_PT_PKT_MAX_SZ - len; 677 if (n > decoder->len) 678 n = decoder->len; 679 memcpy(buf + len, decoder->buf, n); 680 len += n; 681 682 decoder->prev_pkt_ctx = decoder->pkt_ctx; 683 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); 684 if (ret < (int)old_len) { 685 decoder->next_buf = decoder->buf; 686 decoder->next_len = decoder->len; 687 decoder->buf = buf; 688 decoder->len = old_len; 689 return intel_pt_bad_packet(decoder); 690 } 691 692 decoder->next_buf = decoder->buf + (ret - old_len); 693 decoder->next_len = decoder->len - (ret - old_len); 694 695 decoder->buf = buf; 696 decoder->len = ret; 697 698 return ret; 699 } 700 701 struct intel_pt_pkt_info { 702 struct intel_pt_decoder *decoder; 703 struct intel_pt_pkt packet; 704 uint64_t pos; 705 int pkt_len; 706 int last_packet_type; 707 void *data; 708 }; 709 710 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info); 711 712 /* Lookahead packets in current buffer */ 713 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, 714 intel_pt_pkt_cb_t cb, void *data) 715 { 716 struct intel_pt_pkt_info pkt_info; 717 const unsigned char *buf = decoder->buf; 718 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; 719 size_t len = decoder->len; 720 int ret; 721 722 pkt_info.decoder = decoder; 723 pkt_info.pos = decoder->pos; 724 pkt_info.pkt_len = decoder->pkt_step; 725 pkt_info.last_packet_type = decoder->last_packet_type; 726 pkt_info.data = data; 727 728 while (1) { 729 do { 730 pkt_info.pos += pkt_info.pkt_len; 731 buf += pkt_info.pkt_len; 732 len -= pkt_info.pkt_len; 733 734 if (!len) 735 return INTEL_PT_NEED_MORE_BYTES; 736 737 ret = intel_pt_get_packet(buf, len, &pkt_info.packet, 738 &pkt_ctx); 739 if (!ret) 740 return INTEL_PT_NEED_MORE_BYTES; 741 if (ret < 0) 742 return ret; 743 744 pkt_info.pkt_len = ret; 745 } while (pkt_info.packet.type == INTEL_PT_PAD); 746 747 ret = cb(&pkt_info); 748 if (ret) 749 return 0; 750 751 pkt_info.last_packet_type = pkt_info.packet.type; 752 } 753 } 754 755 struct intel_pt_calc_cyc_to_tsc_info { 756 uint64_t cycle_cnt; 757 unsigned int cbr; 758 uint32_t last_mtc; 759 uint64_t ctc_timestamp; 760 uint64_t ctc_delta; 761 uint64_t tsc_timestamp; 762 uint64_t timestamp; 763 bool have_tma; 764 bool fixup_last_mtc; 765 bool from_mtc; 766 double cbr_cyc_to_tsc; 767 }; 768 769 /* 770 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower 771 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC 772 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA 773 * packet by copying the missing bits from the current MTC assuming the least 774 * difference between the two, and that the current MTC comes after last_mtc. 775 */ 776 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift, 777 uint32_t *last_mtc) 778 { 779 uint32_t first_missing_bit = 1U << (16 - mtc_shift); 780 uint32_t mask = ~(first_missing_bit - 1); 781 782 *last_mtc |= mtc & mask; 783 if (*last_mtc >= mtc) { 784 *last_mtc -= first_missing_bit; 785 *last_mtc &= 0xff; 786 } 787 } 788 789 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info) 790 { 791 struct intel_pt_decoder *decoder = pkt_info->decoder; 792 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data; 793 uint64_t timestamp; 794 double cyc_to_tsc; 795 unsigned int cbr; 796 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem; 797 798 switch (pkt_info->packet.type) { 799 case INTEL_PT_TNT: 800 case INTEL_PT_TIP_PGE: 801 case INTEL_PT_TIP: 802 case INTEL_PT_FUP: 803 case INTEL_PT_PSB: 804 case INTEL_PT_PIP: 805 case INTEL_PT_MODE_EXEC: 806 case INTEL_PT_MODE_TSX: 807 case INTEL_PT_PSBEND: 808 case INTEL_PT_PAD: 809 case INTEL_PT_VMCS: 810 case INTEL_PT_MNT: 811 case INTEL_PT_PTWRITE: 812 case INTEL_PT_PTWRITE_IP: 813 case INTEL_PT_BBP: 814 case INTEL_PT_BIP: 815 case INTEL_PT_BEP: 816 case INTEL_PT_BEP_IP: 817 return 0; 818 819 case INTEL_PT_MTC: 820 if (!data->have_tma) 821 return 0; 822 823 mtc = pkt_info->packet.payload; 824 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { 825 data->fixup_last_mtc = false; 826 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 827 &data->last_mtc); 828 } 829 if (mtc > data->last_mtc) 830 mtc_delta = mtc - data->last_mtc; 831 else 832 mtc_delta = mtc + 256 - data->last_mtc; 833 data->ctc_delta += mtc_delta << decoder->mtc_shift; 834 data->last_mtc = mtc; 835 836 if (decoder->tsc_ctc_mult) { 837 timestamp = data->ctc_timestamp + 838 data->ctc_delta * decoder->tsc_ctc_mult; 839 } else { 840 timestamp = data->ctc_timestamp + 841 multdiv(data->ctc_delta, 842 decoder->tsc_ctc_ratio_n, 843 decoder->tsc_ctc_ratio_d); 844 } 845 846 if (timestamp < data->timestamp) 847 return 1; 848 849 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 850 data->timestamp = timestamp; 851 return 0; 852 } 853 854 break; 855 856 case INTEL_PT_TSC: 857 /* 858 * For now, do not support using TSC packets - refer 859 * intel_pt_calc_cyc_to_tsc(). 860 */ 861 if (data->from_mtc) 862 return 1; 863 timestamp = pkt_info->packet.payload | 864 (data->timestamp & (0xffULL << 56)); 865 if (data->from_mtc && timestamp < data->timestamp && 866 data->timestamp - timestamp < decoder->tsc_slip) 867 return 1; 868 if (timestamp < data->timestamp) 869 timestamp += (1ULL << 56); 870 if (pkt_info->last_packet_type != INTEL_PT_CYC) { 871 if (data->from_mtc) 872 return 1; 873 data->tsc_timestamp = timestamp; 874 data->timestamp = timestamp; 875 return 0; 876 } 877 break; 878 879 case INTEL_PT_TMA: 880 if (data->from_mtc) 881 return 1; 882 883 if (!decoder->tsc_ctc_ratio_d) 884 return 0; 885 886 ctc = pkt_info->packet.payload; 887 fc = pkt_info->packet.count; 888 ctc_rem = ctc & decoder->ctc_rem_mask; 889 890 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 891 892 data->ctc_timestamp = data->tsc_timestamp - fc; 893 if (decoder->tsc_ctc_mult) { 894 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 895 } else { 896 data->ctc_timestamp -= 897 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, 898 decoder->tsc_ctc_ratio_d); 899 } 900 901 data->ctc_delta = 0; 902 data->have_tma = true; 903 data->fixup_last_mtc = true; 904 905 return 0; 906 907 case INTEL_PT_CYC: 908 data->cycle_cnt += pkt_info->packet.payload; 909 return 0; 910 911 case INTEL_PT_CBR: 912 cbr = pkt_info->packet.payload; 913 if (data->cbr && data->cbr != cbr) 914 return 1; 915 data->cbr = cbr; 916 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 917 return 0; 918 919 case INTEL_PT_TIP_PGD: 920 case INTEL_PT_TRACESTOP: 921 case INTEL_PT_EXSTOP: 922 case INTEL_PT_EXSTOP_IP: 923 case INTEL_PT_MWAIT: 924 case INTEL_PT_PWRE: 925 case INTEL_PT_PWRX: 926 case INTEL_PT_OVF: 927 case INTEL_PT_BAD: /* Does not happen */ 928 default: 929 return 1; 930 } 931 932 if (!data->cbr && decoder->cbr) { 933 data->cbr = decoder->cbr; 934 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; 935 } 936 937 if (!data->cycle_cnt) 938 return 1; 939 940 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; 941 942 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc && 943 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) { 944 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n", 945 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 946 return 1; 947 } 948 949 decoder->calc_cyc_to_tsc = cyc_to_tsc; 950 decoder->have_calc_cyc_to_tsc = true; 951 952 if (data->cbr) { 953 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n", 954 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos); 955 } else { 956 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n", 957 cyc_to_tsc, pkt_info->pos); 958 } 959 960 return 1; 961 } 962 963 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, 964 bool from_mtc) 965 { 966 struct intel_pt_calc_cyc_to_tsc_info data = { 967 .cycle_cnt = 0, 968 .cbr = 0, 969 .last_mtc = decoder->last_mtc, 970 .ctc_timestamp = decoder->ctc_timestamp, 971 .ctc_delta = decoder->ctc_delta, 972 .tsc_timestamp = decoder->tsc_timestamp, 973 .timestamp = decoder->timestamp, 974 .have_tma = decoder->have_tma, 975 .fixup_last_mtc = decoder->fixup_last_mtc, 976 .from_mtc = from_mtc, 977 .cbr_cyc_to_tsc = 0, 978 }; 979 980 /* 981 * For now, do not support using TSC packets for at least the reasons: 982 * 1) timing might have stopped 983 * 2) TSC packets within PSB+ can slip against CYC packets 984 */ 985 if (!from_mtc) 986 return; 987 988 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); 989 } 990 991 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) 992 { 993 int ret; 994 995 decoder->last_packet_type = decoder->packet.type; 996 997 do { 998 decoder->pos += decoder->pkt_step; 999 decoder->buf += decoder->pkt_step; 1000 decoder->len -= decoder->pkt_step; 1001 1002 if (!decoder->len) { 1003 ret = intel_pt_get_next_data(decoder, false); 1004 if (ret) 1005 return ret; 1006 } 1007 1008 decoder->prev_pkt_ctx = decoder->pkt_ctx; 1009 ret = intel_pt_get_packet(decoder->buf, decoder->len, 1010 &decoder->packet, &decoder->pkt_ctx); 1011 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 && 1012 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { 1013 ret = intel_pt_get_split_packet(decoder); 1014 if (ret < 0) 1015 return ret; 1016 } 1017 if (ret <= 0) 1018 return intel_pt_bad_packet(decoder); 1019 1020 decoder->pkt_len = ret; 1021 decoder->pkt_step = ret; 1022 intel_pt_decoder_log_packet(decoder); 1023 } while (decoder->packet.type == INTEL_PT_PAD); 1024 1025 return 0; 1026 } 1027 1028 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) 1029 { 1030 uint64_t timestamp, masked_timestamp; 1031 1032 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1033 masked_timestamp = timestamp & decoder->period_mask; 1034 if (decoder->continuous_period) { 1035 if (masked_timestamp > decoder->last_masked_timestamp) 1036 return 1; 1037 } else { 1038 timestamp += 1; 1039 masked_timestamp = timestamp & decoder->period_mask; 1040 if (masked_timestamp > decoder->last_masked_timestamp) { 1041 decoder->last_masked_timestamp = masked_timestamp; 1042 decoder->continuous_period = true; 1043 } 1044 } 1045 1046 if (masked_timestamp < decoder->last_masked_timestamp) 1047 return decoder->period_ticks; 1048 1049 return decoder->period_ticks - (timestamp - masked_timestamp); 1050 } 1051 1052 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) 1053 { 1054 switch (decoder->period_type) { 1055 case INTEL_PT_PERIOD_INSTRUCTIONS: 1056 return decoder->period - decoder->period_insn_cnt; 1057 case INTEL_PT_PERIOD_TICKS: 1058 return intel_pt_next_period(decoder); 1059 case INTEL_PT_PERIOD_NONE: 1060 case INTEL_PT_PERIOD_MTC: 1061 default: 1062 return 0; 1063 } 1064 } 1065 1066 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) 1067 { 1068 uint64_t timestamp, masked_timestamp; 1069 1070 switch (decoder->period_type) { 1071 case INTEL_PT_PERIOD_INSTRUCTIONS: 1072 decoder->period_insn_cnt = 0; 1073 break; 1074 case INTEL_PT_PERIOD_TICKS: 1075 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; 1076 masked_timestamp = timestamp & decoder->period_mask; 1077 if (masked_timestamp > decoder->last_masked_timestamp) 1078 decoder->last_masked_timestamp = masked_timestamp; 1079 else 1080 decoder->last_masked_timestamp += decoder->period_ticks; 1081 break; 1082 case INTEL_PT_PERIOD_NONE: 1083 case INTEL_PT_PERIOD_MTC: 1084 default: 1085 break; 1086 } 1087 1088 decoder->state.type |= INTEL_PT_INSTRUCTION; 1089 } 1090 1091 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, 1092 struct intel_pt_insn *intel_pt_insn, uint64_t ip) 1093 { 1094 uint64_t max_insn_cnt, insn_cnt = 0; 1095 int err; 1096 1097 if (!decoder->mtc_insn) 1098 decoder->mtc_insn = true; 1099 1100 max_insn_cnt = intel_pt_next_sample(decoder); 1101 1102 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, 1103 max_insn_cnt, decoder->data); 1104 1105 decoder->tot_insn_cnt += insn_cnt; 1106 decoder->timestamp_insn_cnt += insn_cnt; 1107 decoder->sample_insn_cnt += insn_cnt; 1108 decoder->period_insn_cnt += insn_cnt; 1109 1110 if (err) { 1111 decoder->no_progress = 0; 1112 decoder->pkt_state = INTEL_PT_STATE_ERR2; 1113 intel_pt_log_at("ERROR: Failed to get instruction", 1114 decoder->ip); 1115 if (err == -ENOENT) 1116 return -ENOLINK; 1117 return -EILSEQ; 1118 } 1119 1120 if (ip && decoder->ip == ip) { 1121 err = -EAGAIN; 1122 goto out; 1123 } 1124 1125 if (max_insn_cnt && insn_cnt >= max_insn_cnt) 1126 intel_pt_sample_insn(decoder); 1127 1128 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) { 1129 decoder->state.type = INTEL_PT_INSTRUCTION; 1130 decoder->state.from_ip = decoder->ip; 1131 decoder->state.to_ip = 0; 1132 decoder->ip += intel_pt_insn->length; 1133 err = INTEL_PT_RETURN; 1134 goto out; 1135 } 1136 1137 if (intel_pt_insn->op == INTEL_PT_OP_CALL) { 1138 /* Zero-length calls are excluded */ 1139 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL || 1140 intel_pt_insn->rel) { 1141 err = intel_pt_push(&decoder->stack, decoder->ip + 1142 intel_pt_insn->length); 1143 if (err) 1144 goto out; 1145 } 1146 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) { 1147 decoder->ret_addr = intel_pt_pop(&decoder->stack); 1148 } 1149 1150 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) { 1151 int cnt = decoder->no_progress++; 1152 1153 decoder->state.from_ip = decoder->ip; 1154 decoder->ip += intel_pt_insn->length + 1155 intel_pt_insn->rel; 1156 decoder->state.to_ip = decoder->ip; 1157 err = INTEL_PT_RETURN; 1158 1159 /* 1160 * Check for being stuck in a loop. This can happen if a 1161 * decoder error results in the decoder erroneously setting the 1162 * ip to an address that is itself in an infinite loop that 1163 * consumes no packets. When that happens, there must be an 1164 * unconditional branch. 1165 */ 1166 if (cnt) { 1167 if (cnt == 1) { 1168 decoder->stuck_ip = decoder->state.to_ip; 1169 decoder->stuck_ip_prd = 1; 1170 decoder->stuck_ip_cnt = 1; 1171 } else if (cnt > INTEL_PT_MAX_LOOPS || 1172 decoder->state.to_ip == decoder->stuck_ip) { 1173 intel_pt_log_at("ERROR: Never-ending loop", 1174 decoder->state.to_ip); 1175 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1176 err = -ELOOP; 1177 goto out; 1178 } else if (!--decoder->stuck_ip_cnt) { 1179 decoder->stuck_ip_prd += 1; 1180 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; 1181 decoder->stuck_ip = decoder->state.to_ip; 1182 } 1183 } 1184 goto out_no_progress; 1185 } 1186 out: 1187 decoder->no_progress = 0; 1188 out_no_progress: 1189 decoder->state.insn_op = intel_pt_insn->op; 1190 decoder->state.insn_len = intel_pt_insn->length; 1191 memcpy(decoder->state.insn, intel_pt_insn->buf, 1192 INTEL_PT_INSN_BUF_SZ); 1193 1194 if (decoder->tx_flags & INTEL_PT_IN_TX) 1195 decoder->state.flags |= INTEL_PT_IN_TX; 1196 1197 return err; 1198 } 1199 1200 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) 1201 { 1202 bool ret = false; 1203 1204 if (decoder->set_fup_tx_flags) { 1205 decoder->set_fup_tx_flags = false; 1206 decoder->tx_flags = decoder->fup_tx_flags; 1207 decoder->state.type = INTEL_PT_TRANSACTION; 1208 decoder->state.from_ip = decoder->ip; 1209 decoder->state.to_ip = 0; 1210 decoder->state.flags = decoder->fup_tx_flags; 1211 return true; 1212 } 1213 if (decoder->set_fup_ptw) { 1214 decoder->set_fup_ptw = false; 1215 decoder->state.type = INTEL_PT_PTW; 1216 decoder->state.flags |= INTEL_PT_FUP_IP; 1217 decoder->state.from_ip = decoder->ip; 1218 decoder->state.to_ip = 0; 1219 decoder->state.ptw_payload = decoder->fup_ptw_payload; 1220 return true; 1221 } 1222 if (decoder->set_fup_mwait) { 1223 decoder->set_fup_mwait = false; 1224 decoder->state.type = INTEL_PT_MWAIT_OP; 1225 decoder->state.from_ip = decoder->ip; 1226 decoder->state.to_ip = 0; 1227 decoder->state.mwait_payload = decoder->fup_mwait_payload; 1228 ret = true; 1229 } 1230 if (decoder->set_fup_pwre) { 1231 decoder->set_fup_pwre = false; 1232 decoder->state.type |= INTEL_PT_PWR_ENTRY; 1233 decoder->state.type &= ~INTEL_PT_BRANCH; 1234 decoder->state.from_ip = decoder->ip; 1235 decoder->state.to_ip = 0; 1236 decoder->state.pwre_payload = decoder->fup_pwre_payload; 1237 ret = true; 1238 } 1239 if (decoder->set_fup_exstop) { 1240 decoder->set_fup_exstop = false; 1241 decoder->state.type |= INTEL_PT_EX_STOP; 1242 decoder->state.type &= ~INTEL_PT_BRANCH; 1243 decoder->state.flags |= INTEL_PT_FUP_IP; 1244 decoder->state.from_ip = decoder->ip; 1245 decoder->state.to_ip = 0; 1246 ret = true; 1247 } 1248 if (decoder->set_fup_bep) { 1249 decoder->set_fup_bep = false; 1250 decoder->state.type |= INTEL_PT_BLK_ITEMS; 1251 decoder->state.type &= ~INTEL_PT_BRANCH; 1252 decoder->state.from_ip = decoder->ip; 1253 decoder->state.to_ip = 0; 1254 ret = true; 1255 } 1256 return ret; 1257 } 1258 1259 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, 1260 struct intel_pt_insn *intel_pt_insn, 1261 uint64_t ip, int err) 1262 { 1263 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && 1264 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT && 1265 ip == decoder->ip + intel_pt_insn->length; 1266 } 1267 1268 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) 1269 { 1270 struct intel_pt_insn intel_pt_insn; 1271 uint64_t ip; 1272 int err; 1273 1274 ip = decoder->last_ip; 1275 1276 while (1) { 1277 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); 1278 if (err == INTEL_PT_RETURN) 1279 return 0; 1280 if (err == -EAGAIN || 1281 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { 1282 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1283 if (intel_pt_fup_event(decoder)) 1284 return 0; 1285 return -EAGAIN; 1286 } 1287 decoder->set_fup_tx_flags = false; 1288 if (err) 1289 return err; 1290 1291 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1292 intel_pt_log_at("ERROR: Unexpected indirect branch", 1293 decoder->ip); 1294 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1295 return -ENOENT; 1296 } 1297 1298 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1299 intel_pt_log_at("ERROR: Unexpected conditional branch", 1300 decoder->ip); 1301 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1302 return -ENOENT; 1303 } 1304 1305 intel_pt_bug(decoder); 1306 } 1307 } 1308 1309 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) 1310 { 1311 struct intel_pt_insn intel_pt_insn; 1312 int err; 1313 1314 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1315 if (err == INTEL_PT_RETURN && 1316 decoder->pgd_ip && 1317 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1318 (decoder->state.type & INTEL_PT_BRANCH) && 1319 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { 1320 /* Unconditional branch leaving filter region */ 1321 decoder->no_progress = 0; 1322 decoder->pge = false; 1323 decoder->continuous_period = false; 1324 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1325 decoder->state.type |= INTEL_PT_TRACE_END; 1326 intel_pt_update_nr(decoder); 1327 return 0; 1328 } 1329 if (err == INTEL_PT_RETURN) 1330 return 0; 1331 if (err) 1332 return err; 1333 1334 intel_pt_update_nr(decoder); 1335 1336 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1337 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { 1338 decoder->pge = false; 1339 decoder->continuous_period = false; 1340 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1341 decoder->state.from_ip = decoder->ip; 1342 if (decoder->packet.count == 0) { 1343 decoder->state.to_ip = 0; 1344 } else { 1345 decoder->state.to_ip = decoder->last_ip; 1346 decoder->ip = decoder->last_ip; 1347 } 1348 decoder->state.type |= INTEL_PT_TRACE_END; 1349 } else { 1350 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1351 decoder->state.from_ip = decoder->ip; 1352 if (decoder->packet.count == 0) { 1353 decoder->state.to_ip = 0; 1354 } else { 1355 decoder->state.to_ip = decoder->last_ip; 1356 decoder->ip = decoder->last_ip; 1357 } 1358 } 1359 return 0; 1360 } 1361 1362 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1363 uint64_t to_ip = decoder->ip + intel_pt_insn.length + 1364 intel_pt_insn.rel; 1365 1366 if (decoder->pgd_ip && 1367 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && 1368 decoder->pgd_ip(to_ip, decoder->data)) { 1369 /* Conditional branch leaving filter region */ 1370 decoder->pge = false; 1371 decoder->continuous_period = false; 1372 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1373 decoder->ip = to_ip; 1374 decoder->state.from_ip = decoder->ip; 1375 decoder->state.to_ip = to_ip; 1376 decoder->state.type |= INTEL_PT_TRACE_END; 1377 return 0; 1378 } 1379 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch", 1380 decoder->ip); 1381 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1382 return -ENOENT; 1383 } 1384 1385 return intel_pt_bug(decoder); 1386 } 1387 1388 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) 1389 { 1390 struct intel_pt_insn intel_pt_insn; 1391 int err; 1392 1393 while (1) { 1394 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); 1395 if (err == INTEL_PT_RETURN) 1396 return 0; 1397 if (err) 1398 return err; 1399 1400 if (intel_pt_insn.op == INTEL_PT_OP_RET) { 1401 if (!decoder->return_compression) { 1402 intel_pt_log_at("ERROR: RET when expecting conditional branch", 1403 decoder->ip); 1404 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1405 return -ENOENT; 1406 } 1407 if (!decoder->ret_addr) { 1408 intel_pt_log_at("ERROR: Bad RET compression (stack empty)", 1409 decoder->ip); 1410 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1411 return -ENOENT; 1412 } 1413 if (!(decoder->tnt.payload & BIT63)) { 1414 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)", 1415 decoder->ip); 1416 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1417 return -ENOENT; 1418 } 1419 decoder->tnt.count -= 1; 1420 if (decoder->tnt.count) 1421 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1422 else 1423 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1424 decoder->tnt.payload <<= 1; 1425 decoder->state.from_ip = decoder->ip; 1426 decoder->ip = decoder->ret_addr; 1427 decoder->state.to_ip = decoder->ip; 1428 return 0; 1429 } 1430 1431 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) { 1432 /* Handle deferred TIPs */ 1433 err = intel_pt_get_next_packet(decoder); 1434 if (err) 1435 return err; 1436 if (decoder->packet.type != INTEL_PT_TIP || 1437 decoder->packet.count == 0) { 1438 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch", 1439 decoder->ip); 1440 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1441 decoder->pkt_step = 0; 1442 return -ENOENT; 1443 } 1444 intel_pt_set_last_ip(decoder); 1445 decoder->state.from_ip = decoder->ip; 1446 decoder->state.to_ip = decoder->last_ip; 1447 decoder->ip = decoder->last_ip; 1448 intel_pt_update_nr(decoder); 1449 return 0; 1450 } 1451 1452 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) { 1453 decoder->tnt.count -= 1; 1454 if (decoder->tnt.count) 1455 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; 1456 else 1457 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 1458 if (decoder->tnt.payload & BIT63) { 1459 decoder->tnt.payload <<= 1; 1460 decoder->state.from_ip = decoder->ip; 1461 decoder->ip += intel_pt_insn.length + 1462 intel_pt_insn.rel; 1463 decoder->state.to_ip = decoder->ip; 1464 return 0; 1465 } 1466 /* Instruction sample for a non-taken branch */ 1467 if (decoder->state.type & INTEL_PT_INSTRUCTION) { 1468 decoder->tnt.payload <<= 1; 1469 decoder->state.type = INTEL_PT_INSTRUCTION; 1470 decoder->state.from_ip = decoder->ip; 1471 decoder->state.to_ip = 0; 1472 decoder->ip += intel_pt_insn.length; 1473 return 0; 1474 } 1475 decoder->sample_cyc = false; 1476 decoder->ip += intel_pt_insn.length; 1477 if (!decoder->tnt.count) { 1478 intel_pt_update_sample_time(decoder); 1479 return -EAGAIN; 1480 } 1481 decoder->tnt.payload <<= 1; 1482 continue; 1483 } 1484 1485 return intel_pt_bug(decoder); 1486 } 1487 } 1488 1489 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) 1490 { 1491 unsigned int fup_tx_flags; 1492 int err; 1493 1494 fup_tx_flags = decoder->packet.payload & 1495 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX); 1496 err = intel_pt_get_next_packet(decoder); 1497 if (err) 1498 return err; 1499 if (decoder->packet.type == INTEL_PT_FUP) { 1500 decoder->fup_tx_flags = fup_tx_flags; 1501 decoder->set_fup_tx_flags = true; 1502 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) 1503 *no_tip = true; 1504 } else { 1505 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX", 1506 decoder->pos); 1507 intel_pt_update_in_tx(decoder); 1508 } 1509 return 0; 1510 } 1511 1512 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) 1513 { 1514 timestamp |= (ref_timestamp & (0xffULL << 56)); 1515 1516 if (timestamp < ref_timestamp) { 1517 if (ref_timestamp - timestamp > (1ULL << 55)) 1518 timestamp += (1ULL << 56); 1519 } else { 1520 if (timestamp - ref_timestamp > (1ULL << 55)) 1521 timestamp -= (1ULL << 56); 1522 } 1523 1524 return timestamp; 1525 } 1526 1527 /* For use only when decoder->vm_time_correlation is true */ 1528 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder, 1529 uint64_t timestamp) 1530 { 1531 uint64_t max_timestamp = decoder->buf_timestamp; 1532 1533 if (!max_timestamp) { 1534 max_timestamp = decoder->last_reliable_timestamp + 1535 0x400000000ULL; 1536 } 1537 return timestamp >= decoder->last_reliable_timestamp && 1538 timestamp < decoder->buf_timestamp; 1539 } 1540 1541 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) 1542 { 1543 uint64_t timestamp; 1544 bool bad = false; 1545 1546 decoder->have_tma = false; 1547 1548 if (decoder->ref_timestamp) { 1549 timestamp = intel_pt_8b_tsc(decoder->packet.payload, 1550 decoder->ref_timestamp); 1551 decoder->tsc_timestamp = timestamp; 1552 decoder->timestamp = timestamp; 1553 decoder->ref_timestamp = 0; 1554 decoder->timestamp_insn_cnt = 0; 1555 } else if (decoder->timestamp) { 1556 timestamp = decoder->packet.payload | 1557 (decoder->timestamp & (0xffULL << 56)); 1558 decoder->tsc_timestamp = timestamp; 1559 if (timestamp < decoder->timestamp && 1560 decoder->timestamp - timestamp < decoder->tsc_slip) { 1561 intel_pt_log_to("Suppressing backwards timestamp", 1562 timestamp); 1563 timestamp = decoder->timestamp; 1564 } 1565 if (timestamp < decoder->timestamp) { 1566 if (!decoder->buf_timestamp || 1567 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { 1568 intel_pt_log_to("Wraparound timestamp", timestamp); 1569 timestamp += (1ULL << 56); 1570 decoder->tsc_timestamp = timestamp; 1571 } else { 1572 intel_pt_log_to("Suppressing bad timestamp", timestamp); 1573 timestamp = decoder->timestamp; 1574 bad = true; 1575 } 1576 } 1577 if (decoder->vm_time_correlation && 1578 (bad || !intel_pt_time_in_range(decoder, timestamp)) && 1579 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 1580 p_log("Timestamp out of range"); 1581 decoder->timestamp = timestamp; 1582 decoder->timestamp_insn_cnt = 0; 1583 } 1584 1585 if (decoder->last_packet_type == INTEL_PT_CYC) { 1586 decoder->cyc_ref_timestamp = decoder->timestamp; 1587 decoder->cycle_cnt = 0; 1588 decoder->have_calc_cyc_to_tsc = false; 1589 intel_pt_calc_cyc_to_tsc(decoder, false); 1590 } 1591 1592 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1593 } 1594 1595 static int intel_pt_overflow(struct intel_pt_decoder *decoder) 1596 { 1597 intel_pt_log("ERROR: Buffer overflow\n"); 1598 intel_pt_clear_tx_flags(decoder); 1599 intel_pt_set_nr(decoder); 1600 decoder->timestamp_insn_cnt = 0; 1601 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; 1602 decoder->overflow = true; 1603 return -EOVERFLOW; 1604 } 1605 1606 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) 1607 { 1608 if (decoder->have_cyc) 1609 return; 1610 1611 decoder->cyc_cnt_timestamp = decoder->timestamp; 1612 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; 1613 } 1614 1615 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) 1616 { 1617 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; 1618 1619 if (decoder->pge) 1620 intel_pt_mtc_cyc_cnt_pge(decoder); 1621 } 1622 1623 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) 1624 { 1625 uint64_t tot_cyc_cnt, tsc_delta; 1626 1627 if (decoder->have_cyc) 1628 return; 1629 1630 decoder->sample_cyc = true; 1631 1632 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) 1633 return; 1634 1635 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; 1636 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; 1637 1638 if (tot_cyc_cnt > decoder->tot_cyc_cnt) 1639 decoder->tot_cyc_cnt = tot_cyc_cnt; 1640 } 1641 1642 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) 1643 { 1644 uint32_t ctc = decoder->packet.payload; 1645 uint32_t fc = decoder->packet.count; 1646 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; 1647 1648 if (!decoder->tsc_ctc_ratio_d) 1649 return; 1650 1651 if (decoder->pge && !decoder->in_psb) 1652 intel_pt_mtc_cyc_cnt_pge(decoder); 1653 else 1654 intel_pt_mtc_cyc_cnt_upd(decoder); 1655 1656 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; 1657 decoder->last_ctc = ctc - ctc_rem; 1658 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; 1659 if (decoder->tsc_ctc_mult) { 1660 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; 1661 } else { 1662 decoder->ctc_timestamp -= multdiv(ctc_rem, 1663 decoder->tsc_ctc_ratio_n, 1664 decoder->tsc_ctc_ratio_d); 1665 } 1666 decoder->ctc_delta = 0; 1667 decoder->have_tma = true; 1668 decoder->fixup_last_mtc = true; 1669 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n", 1670 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); 1671 } 1672 1673 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) 1674 { 1675 uint64_t timestamp; 1676 uint32_t mtc, mtc_delta; 1677 1678 if (!decoder->have_tma) 1679 return; 1680 1681 mtc = decoder->packet.payload; 1682 1683 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { 1684 decoder->fixup_last_mtc = false; 1685 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, 1686 &decoder->last_mtc); 1687 } 1688 1689 if (mtc > decoder->last_mtc) 1690 mtc_delta = mtc - decoder->last_mtc; 1691 else 1692 mtc_delta = mtc + 256 - decoder->last_mtc; 1693 1694 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; 1695 1696 if (decoder->tsc_ctc_mult) { 1697 timestamp = decoder->ctc_timestamp + 1698 decoder->ctc_delta * decoder->tsc_ctc_mult; 1699 } else { 1700 timestamp = decoder->ctc_timestamp + 1701 multdiv(decoder->ctc_delta, 1702 decoder->tsc_ctc_ratio_n, 1703 decoder->tsc_ctc_ratio_d); 1704 } 1705 1706 if (timestamp < decoder->timestamp) 1707 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1708 timestamp, decoder->timestamp); 1709 else 1710 decoder->timestamp = timestamp; 1711 1712 intel_pt_mtc_cyc_cnt_upd(decoder); 1713 1714 decoder->timestamp_insn_cnt = 0; 1715 decoder->last_mtc = mtc; 1716 1717 if (decoder->last_packet_type == INTEL_PT_CYC) { 1718 decoder->cyc_ref_timestamp = decoder->timestamp; 1719 decoder->cycle_cnt = 0; 1720 decoder->have_calc_cyc_to_tsc = false; 1721 intel_pt_calc_cyc_to_tsc(decoder, true); 1722 } 1723 1724 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1725 } 1726 1727 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) 1728 { 1729 unsigned int cbr = decoder->packet.payload & 0xff; 1730 1731 decoder->cbr_payload = decoder->packet.payload; 1732 1733 if (decoder->cbr == cbr) 1734 return; 1735 1736 decoder->cbr = cbr; 1737 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; 1738 1739 intel_pt_mtc_cyc_cnt_cbr(decoder); 1740 } 1741 1742 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) 1743 { 1744 uint64_t timestamp = decoder->cyc_ref_timestamp; 1745 1746 decoder->have_cyc = true; 1747 1748 decoder->cycle_cnt += decoder->packet.payload; 1749 if (decoder->pge) 1750 decoder->tot_cyc_cnt += decoder->packet.payload; 1751 decoder->sample_cyc = true; 1752 1753 if (!decoder->cyc_ref_timestamp) 1754 return; 1755 1756 if (decoder->have_calc_cyc_to_tsc) 1757 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; 1758 else if (decoder->cbr) 1759 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; 1760 else 1761 return; 1762 1763 if (timestamp < decoder->timestamp) 1764 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n", 1765 timestamp, decoder->timestamp); 1766 else 1767 decoder->timestamp = timestamp; 1768 1769 decoder->timestamp_insn_cnt = 0; 1770 1771 intel_pt_log_to("Setting timestamp", decoder->timestamp); 1772 } 1773 1774 static void intel_pt_bbp(struct intel_pt_decoder *decoder) 1775 { 1776 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { 1777 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); 1778 decoder->state.items.is_32_bit = false; 1779 } 1780 decoder->blk_type = decoder->packet.payload; 1781 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); 1782 if (decoder->blk_type == INTEL_PT_GP_REGS) 1783 decoder->state.items.is_32_bit = decoder->packet.count; 1784 if (decoder->blk_type_pos < 0) { 1785 intel_pt_log("WARNING: Unknown block type %u\n", 1786 decoder->blk_type); 1787 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { 1788 intel_pt_log("WARNING: Duplicate block type %u\n", 1789 decoder->blk_type); 1790 } 1791 } 1792 1793 static void intel_pt_bip(struct intel_pt_decoder *decoder) 1794 { 1795 uint32_t id = decoder->packet.count; 1796 uint32_t bit = 1 << id; 1797 int pos = decoder->blk_type_pos; 1798 1799 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) { 1800 intel_pt_log("WARNING: Unknown block item %u type %d\n", 1801 id, decoder->blk_type); 1802 return; 1803 } 1804 1805 if (decoder->state.items.mask[pos] & bit) { 1806 intel_pt_log("WARNING: Duplicate block item %u type %d\n", 1807 id, decoder->blk_type); 1808 } 1809 1810 decoder->state.items.mask[pos] |= bit; 1811 decoder->state.items.val[pos][id] = decoder->packet.payload; 1812 } 1813 1814 /* Walk PSB+ packets when already in sync. */ 1815 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) 1816 { 1817 int err; 1818 1819 decoder->in_psb = true; 1820 1821 while (1) { 1822 err = intel_pt_get_next_packet(decoder); 1823 if (err) 1824 goto out; 1825 1826 switch (decoder->packet.type) { 1827 case INTEL_PT_PSBEND: 1828 err = 0; 1829 goto out; 1830 1831 case INTEL_PT_TIP_PGD: 1832 case INTEL_PT_TIP_PGE: 1833 case INTEL_PT_TIP: 1834 case INTEL_PT_TNT: 1835 case INTEL_PT_TRACESTOP: 1836 case INTEL_PT_BAD: 1837 case INTEL_PT_PSB: 1838 case INTEL_PT_PTWRITE: 1839 case INTEL_PT_PTWRITE_IP: 1840 case INTEL_PT_EXSTOP: 1841 case INTEL_PT_EXSTOP_IP: 1842 case INTEL_PT_MWAIT: 1843 case INTEL_PT_PWRE: 1844 case INTEL_PT_PWRX: 1845 case INTEL_PT_BBP: 1846 case INTEL_PT_BIP: 1847 case INTEL_PT_BEP: 1848 case INTEL_PT_BEP_IP: 1849 decoder->have_tma = false; 1850 intel_pt_log("ERROR: Unexpected packet\n"); 1851 err = -EAGAIN; 1852 goto out; 1853 1854 case INTEL_PT_OVF: 1855 err = intel_pt_overflow(decoder); 1856 goto out; 1857 1858 case INTEL_PT_TSC: 1859 intel_pt_calc_tsc_timestamp(decoder); 1860 break; 1861 1862 case INTEL_PT_TMA: 1863 intel_pt_calc_tma(decoder); 1864 break; 1865 1866 case INTEL_PT_CBR: 1867 intel_pt_calc_cbr(decoder); 1868 break; 1869 1870 case INTEL_PT_MODE_EXEC: 1871 decoder->exec_mode = decoder->packet.payload; 1872 break; 1873 1874 case INTEL_PT_PIP: 1875 intel_pt_set_pip(decoder); 1876 break; 1877 1878 case INTEL_PT_FUP: 1879 decoder->pge = true; 1880 if (decoder->packet.count) { 1881 intel_pt_set_last_ip(decoder); 1882 decoder->psb_ip = decoder->last_ip; 1883 } 1884 break; 1885 1886 case INTEL_PT_MODE_TSX: 1887 intel_pt_update_in_tx(decoder); 1888 break; 1889 1890 case INTEL_PT_MTC: 1891 intel_pt_calc_mtc_timestamp(decoder); 1892 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 1893 decoder->state.type |= INTEL_PT_INSTRUCTION; 1894 break; 1895 1896 case INTEL_PT_CYC: 1897 intel_pt_calc_cyc_timestamp(decoder); 1898 break; 1899 1900 case INTEL_PT_VMCS: 1901 case INTEL_PT_MNT: 1902 case INTEL_PT_PAD: 1903 default: 1904 break; 1905 } 1906 } 1907 out: 1908 decoder->in_psb = false; 1909 1910 return err; 1911 } 1912 1913 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) 1914 { 1915 int err; 1916 1917 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { 1918 decoder->tx_flags = 0; 1919 decoder->state.flags &= ~INTEL_PT_IN_TX; 1920 decoder->state.flags |= INTEL_PT_ABORT_TX; 1921 } else { 1922 decoder->state.flags |= INTEL_PT_ASYNC; 1923 } 1924 1925 while (1) { 1926 err = intel_pt_get_next_packet(decoder); 1927 if (err) 1928 return err; 1929 1930 switch (decoder->packet.type) { 1931 case INTEL_PT_TNT: 1932 case INTEL_PT_FUP: 1933 case INTEL_PT_TRACESTOP: 1934 case INTEL_PT_PSB: 1935 case INTEL_PT_TSC: 1936 case INTEL_PT_TMA: 1937 case INTEL_PT_MODE_TSX: 1938 case INTEL_PT_BAD: 1939 case INTEL_PT_PSBEND: 1940 case INTEL_PT_PTWRITE: 1941 case INTEL_PT_PTWRITE_IP: 1942 case INTEL_PT_EXSTOP: 1943 case INTEL_PT_EXSTOP_IP: 1944 case INTEL_PT_MWAIT: 1945 case INTEL_PT_PWRE: 1946 case INTEL_PT_PWRX: 1947 case INTEL_PT_BBP: 1948 case INTEL_PT_BIP: 1949 case INTEL_PT_BEP: 1950 case INTEL_PT_BEP_IP: 1951 intel_pt_log("ERROR: Missing TIP after FUP\n"); 1952 decoder->pkt_state = INTEL_PT_STATE_ERR3; 1953 decoder->pkt_step = 0; 1954 return -ENOENT; 1955 1956 case INTEL_PT_CBR: 1957 intel_pt_calc_cbr(decoder); 1958 break; 1959 1960 case INTEL_PT_OVF: 1961 return intel_pt_overflow(decoder); 1962 1963 case INTEL_PT_TIP_PGD: 1964 decoder->state.from_ip = decoder->ip; 1965 if (decoder->packet.count == 0) { 1966 decoder->state.to_ip = 0; 1967 } else { 1968 intel_pt_set_ip(decoder); 1969 decoder->state.to_ip = decoder->ip; 1970 } 1971 decoder->pge = false; 1972 decoder->continuous_period = false; 1973 decoder->state.type |= INTEL_PT_TRACE_END; 1974 intel_pt_update_nr(decoder); 1975 return 0; 1976 1977 case INTEL_PT_TIP_PGE: 1978 decoder->pge = true; 1979 intel_pt_log("Omitting PGE ip " x64_fmt "\n", 1980 decoder->ip); 1981 decoder->state.from_ip = 0; 1982 if (decoder->packet.count == 0) { 1983 decoder->state.to_ip = 0; 1984 } else { 1985 intel_pt_set_ip(decoder); 1986 decoder->state.to_ip = decoder->ip; 1987 } 1988 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 1989 intel_pt_mtc_cyc_cnt_pge(decoder); 1990 intel_pt_set_nr(decoder); 1991 return 0; 1992 1993 case INTEL_PT_TIP: 1994 decoder->state.from_ip = decoder->ip; 1995 if (decoder->packet.count == 0) { 1996 decoder->state.to_ip = 0; 1997 } else { 1998 intel_pt_set_ip(decoder); 1999 decoder->state.to_ip = decoder->ip; 2000 } 2001 intel_pt_update_nr(decoder); 2002 return 0; 2003 2004 case INTEL_PT_PIP: 2005 intel_pt_update_pip(decoder); 2006 break; 2007 2008 case INTEL_PT_MTC: 2009 intel_pt_calc_mtc_timestamp(decoder); 2010 if (decoder->period_type == INTEL_PT_PERIOD_MTC) 2011 decoder->state.type |= INTEL_PT_INSTRUCTION; 2012 break; 2013 2014 case INTEL_PT_CYC: 2015 intel_pt_calc_cyc_timestamp(decoder); 2016 break; 2017 2018 case INTEL_PT_MODE_EXEC: 2019 decoder->exec_mode = decoder->packet.payload; 2020 break; 2021 2022 case INTEL_PT_VMCS: 2023 case INTEL_PT_MNT: 2024 case INTEL_PT_PAD: 2025 break; 2026 2027 default: 2028 return intel_pt_bug(decoder); 2029 } 2030 } 2031 } 2032 2033 static int intel_pt_resample(struct intel_pt_decoder *decoder) 2034 { 2035 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2036 decoder->state.type = INTEL_PT_INSTRUCTION; 2037 decoder->state.from_ip = decoder->ip; 2038 decoder->state.to_ip = 0; 2039 return 0; 2040 } 2041 2042 struct intel_pt_vm_tsc_info { 2043 struct intel_pt_pkt pip_packet; 2044 struct intel_pt_pkt vmcs_packet; 2045 struct intel_pt_pkt tma_packet; 2046 bool tsc, pip, vmcs, tma, psbend; 2047 uint64_t ctc_delta; 2048 uint64_t last_ctc; 2049 int max_lookahead; 2050 }; 2051 2052 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */ 2053 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2054 { 2055 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2056 2057 switch (pkt_info->packet.type) { 2058 case INTEL_PT_PAD: 2059 case INTEL_PT_MNT: 2060 case INTEL_PT_MODE_EXEC: 2061 case INTEL_PT_MODE_TSX: 2062 case INTEL_PT_MTC: 2063 case INTEL_PT_FUP: 2064 case INTEL_PT_CYC: 2065 case INTEL_PT_CBR: 2066 break; 2067 2068 case INTEL_PT_TSC: 2069 data->tsc = true; 2070 break; 2071 2072 case INTEL_PT_TMA: 2073 data->tma_packet = pkt_info->packet; 2074 data->tma = true; 2075 break; 2076 2077 case INTEL_PT_PIP: 2078 data->pip_packet = pkt_info->packet; 2079 data->pip = true; 2080 break; 2081 2082 case INTEL_PT_VMCS: 2083 data->vmcs_packet = pkt_info->packet; 2084 data->vmcs = true; 2085 break; 2086 2087 case INTEL_PT_PSBEND: 2088 data->psbend = true; 2089 return 1; 2090 2091 case INTEL_PT_TIP_PGE: 2092 case INTEL_PT_PTWRITE: 2093 case INTEL_PT_PTWRITE_IP: 2094 case INTEL_PT_EXSTOP: 2095 case INTEL_PT_EXSTOP_IP: 2096 case INTEL_PT_MWAIT: 2097 case INTEL_PT_PWRE: 2098 case INTEL_PT_PWRX: 2099 case INTEL_PT_BBP: 2100 case INTEL_PT_BIP: 2101 case INTEL_PT_BEP: 2102 case INTEL_PT_BEP_IP: 2103 case INTEL_PT_OVF: 2104 case INTEL_PT_BAD: 2105 case INTEL_PT_TNT: 2106 case INTEL_PT_TIP_PGD: 2107 case INTEL_PT_TIP: 2108 case INTEL_PT_PSB: 2109 case INTEL_PT_TRACESTOP: 2110 default: 2111 return 1; 2112 } 2113 2114 return 0; 2115 } 2116 2117 struct intel_pt_ovf_fup_info { 2118 int max_lookahead; 2119 bool found; 2120 }; 2121 2122 /* Lookahead to detect a FUP packet after OVF */ 2123 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2124 { 2125 struct intel_pt_ovf_fup_info *data = pkt_info->data; 2126 2127 if (pkt_info->packet.type == INTEL_PT_CYC || 2128 pkt_info->packet.type == INTEL_PT_MTC || 2129 pkt_info->packet.type == INTEL_PT_TSC) 2130 return !--(data->max_lookahead); 2131 data->found = pkt_info->packet.type == INTEL_PT_FUP; 2132 return 1; 2133 } 2134 2135 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder) 2136 { 2137 struct intel_pt_ovf_fup_info data = { 2138 .max_lookahead = 16, 2139 .found = false, 2140 }; 2141 2142 intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data); 2143 return data.found; 2144 } 2145 2146 /* Lookahead and get the TMA packet after TSC */ 2147 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2148 { 2149 struct intel_pt_vm_tsc_info *data = pkt_info->data; 2150 2151 if (pkt_info->packet.type == INTEL_PT_CYC || 2152 pkt_info->packet.type == INTEL_PT_MTC) 2153 return !--(data->max_lookahead); 2154 2155 if (pkt_info->packet.type == INTEL_PT_TMA) { 2156 data->tma_packet = pkt_info->packet; 2157 data->tma = true; 2158 } 2159 return 1; 2160 } 2161 2162 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc) 2163 { 2164 if (decoder->tsc_ctc_mult) 2165 return ctc * decoder->tsc_ctc_mult; 2166 else 2167 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); 2168 } 2169 2170 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder, 2171 uint32_t ctc, 2172 uint32_t fc, 2173 uint64_t last_ctc_timestamp, 2174 uint64_t ctc_delta, 2175 uint32_t last_ctc) 2176 { 2177 /* Number of CTC ticks from last_ctc_timestamp to last_mtc */ 2178 uint64_t last_mtc_ctc = last_ctc + ctc_delta; 2179 /* 2180 * Number of CTC ticks from there until current TMA packet. We would 2181 * expect last_mtc_ctc to be before ctc, but the TSC packet can slip 2182 * past an MTC, so a sign-extended value is used. 2183 */ 2184 uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); 2185 /* Total CTC ticks from last_ctc_timestamp to current TMA packet */ 2186 uint64_t new_ctc_delta = ctc_delta + delta; 2187 uint64_t expected_tsc; 2188 2189 /* 2190 * Convert CTC ticks to TSC ticks, add the starting point 2191 * (last_ctc_timestamp) and the fast counter from the TMA packet. 2192 */ 2193 expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc; 2194 2195 if (intel_pt_enable_logging) { 2196 intel_pt_log_x64(last_mtc_ctc); 2197 intel_pt_log_x32(last_ctc); 2198 intel_pt_log_x64(ctc_delta); 2199 intel_pt_log_x64(delta); 2200 intel_pt_log_x32(ctc); 2201 intel_pt_log_x64(new_ctc_delta); 2202 intel_pt_log_x64(last_ctc_timestamp); 2203 intel_pt_log_x32(fc); 2204 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta)); 2205 intel_pt_log_x64(expected_tsc); 2206 } 2207 2208 return expected_tsc; 2209 } 2210 2211 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder, 2212 struct intel_pt_vm_tsc_info *data) 2213 { 2214 uint32_t ctc = data->tma_packet.payload; 2215 uint32_t fc = data->tma_packet.count; 2216 2217 return intel_pt_calc_expected_tsc(decoder, ctc, fc, 2218 decoder->ctc_timestamp, 2219 data->ctc_delta, data->last_ctc); 2220 } 2221 2222 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder, 2223 struct intel_pt_vmcs_info *vmcs_info) 2224 { 2225 uint64_t payload = decoder->packet.payload; 2226 2227 /* VMX adds the TSC Offset, so subtract to get host TSC */ 2228 decoder->packet.payload -= vmcs_info->tsc_offset; 2229 /* TSC packet has only 7 bytes */ 2230 decoder->packet.payload &= SEVEN_BYTES; 2231 2232 /* 2233 * The buffer is mmapped from the data file, so this also updates the 2234 * data file. 2235 */ 2236 if (!decoder->vm_tm_corr_dry_run) 2237 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7); 2238 2239 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2240 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2241 payload, decoder->packet.payload, vmcs_info->vmcs, 2242 vmcs_info->tsc_offset); 2243 } 2244 2245 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder, 2246 uint64_t tsc_offset) 2247 { 2248 struct intel_pt_vmcs_info vmcs_info = { 2249 .vmcs = NO_VMCS, 2250 .tsc_offset = tsc_offset 2251 }; 2252 2253 intel_pt_translate_vm_tsc(decoder, &vmcs_info); 2254 } 2255 2256 static inline bool in_vm(uint64_t pip_payload) 2257 { 2258 return pip_payload & 1; 2259 } 2260 2261 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet) 2262 { 2263 return pip_packet->payload & 1; 2264 } 2265 2266 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info) 2267 { 2268 p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64, 2269 vmcs_info->vmcs, vmcs_info->tsc_offset); 2270 } 2271 2272 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder, 2273 struct intel_pt_vm_tsc_info *data) 2274 { 2275 memset(data, 0, sizeof(*data)); 2276 data->ctc_delta = decoder->ctc_delta; 2277 data->last_ctc = decoder->last_ctc; 2278 intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data); 2279 if (data->tsc && !data->psbend) 2280 p_log("ERROR: PSB without PSBEND"); 2281 decoder->in_psb = data->psbend; 2282 } 2283 2284 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder, 2285 struct intel_pt_vm_tsc_info *data, 2286 struct intel_pt_vmcs_info *vmcs_info, 2287 uint64_t host_tsc) 2288 { 2289 if (!decoder->in_psb) { 2290 /* Can't happen */ 2291 p_log("ERROR: First TSC is not in PSB+"); 2292 } 2293 2294 if (data->pip) { 2295 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2296 if (vmcs_info && vmcs_info->tsc_offset) { 2297 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2298 decoder->vm_tm_corr_reliable = true; 2299 } else { 2300 p_log("ERROR: First TSC, unknown TSC Offset"); 2301 } 2302 } else { /* Host */ 2303 decoder->vm_tm_corr_reliable = true; 2304 } 2305 } else { /* Host or Guest */ 2306 decoder->vm_tm_corr_reliable = false; 2307 if (intel_pt_time_in_range(decoder, host_tsc)) { 2308 /* Assume Host */ 2309 } else { 2310 /* Assume Guest */ 2311 if (vmcs_info && vmcs_info->tsc_offset) 2312 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2313 else 2314 p_log("ERROR: First TSC, no PIP, unknown TSC Offset"); 2315 } 2316 } 2317 } 2318 2319 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder, 2320 struct intel_pt_vm_tsc_info *data) 2321 { 2322 struct intel_pt_vmcs_info *vmcs_info; 2323 uint64_t tsc_offset = 0; 2324 uint64_t vmcs; 2325 bool reliable = true; 2326 uint64_t expected_tsc; 2327 uint64_t host_tsc; 2328 uint64_t ref_timestamp; 2329 2330 bool assign = false; 2331 bool assign_reliable = false; 2332 2333 /* Already have 'data' for the in_psb case */ 2334 if (!decoder->in_psb) { 2335 memset(data, 0, sizeof(*data)); 2336 data->ctc_delta = decoder->ctc_delta; 2337 data->last_ctc = decoder->last_ctc; 2338 data->max_lookahead = 16; 2339 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data); 2340 if (decoder->pge) { 2341 data->pip = true; 2342 data->pip_packet.payload = decoder->pip_payload; 2343 } 2344 } 2345 2346 /* Calculations depend on having TMA packets */ 2347 if (!data->tma) { 2348 p_log("ERROR: TSC without TMA"); 2349 return; 2350 } 2351 2352 vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; 2353 if (vmcs == NO_VMCS) 2354 vmcs = 0; 2355 2356 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2357 2358 ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; 2359 host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp); 2360 2361 if (!decoder->ctc_timestamp) { 2362 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc); 2363 return; 2364 } 2365 2366 expected_tsc = intel_pt_expected_tsc(decoder, data); 2367 2368 tsc_offset = host_tsc - expected_tsc; 2369 2370 /* Determine if TSC is from Host or Guest */ 2371 if (data->pip) { 2372 if (pip_in_vm(&data->pip_packet)) { /* Guest */ 2373 if (!vmcs_info) { 2374 /* PIP NR=1 without VMCS cannot happen */ 2375 p_log("ERROR: Missing VMCS"); 2376 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2377 decoder->vm_tm_corr_reliable = false; 2378 return; 2379 } 2380 } else { /* Host */ 2381 decoder->last_reliable_timestamp = host_tsc; 2382 decoder->vm_tm_corr_reliable = true; 2383 return; 2384 } 2385 } else { /* Host or Guest */ 2386 reliable = false; /* Host/Guest is a guess, so not reliable */ 2387 if (decoder->in_psb) { 2388 if (!tsc_offset) 2389 return; /* Zero TSC Offset, assume Host */ 2390 /* 2391 * TSC packet has only 7 bytes of TSC. We have no 2392 * information about the Guest's 8th byte, but it 2393 * doesn't matter because we only need 7 bytes. 2394 * Here, since the 8th byte is unreliable and 2395 * irrelevant, compare only 7 byes. 2396 */ 2397 if (vmcs_info && 2398 (tsc_offset & SEVEN_BYTES) == 2399 (vmcs_info->tsc_offset & SEVEN_BYTES)) { 2400 /* Same TSC Offset as last VMCS, assume Guest */ 2401 goto guest; 2402 } 2403 } 2404 /* 2405 * Check if the host_tsc is within the expected range. 2406 * Note, we could narrow the range more by looking ahead for 2407 * the next host TSC in the same buffer, but we don't bother to 2408 * do that because this is probably good enough. 2409 */ 2410 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) { 2411 /* Within expected range for Host TSC, assume Host */ 2412 decoder->vm_tm_corr_reliable = false; 2413 return; 2414 } 2415 } 2416 2417 guest: /* Assuming Guest */ 2418 2419 /* Determine whether to assign TSC Offset */ 2420 if (vmcs_info && vmcs_info->vmcs) { 2421 if (vmcs_info->tsc_offset && vmcs_info->reliable) { 2422 assign = false; 2423 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && 2424 decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { 2425 /* Continuous tracing, TSC in a PSB is not a time loss */ 2426 assign = true; 2427 assign_reliable = true; 2428 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { 2429 /* 2430 * Unlikely to be a time loss TSC in a PSB which is not 2431 * at the start of a buffer. 2432 */ 2433 assign = true; 2434 assign_reliable = false; 2435 } 2436 } 2437 2438 /* Record VMCS TSC Offset */ 2439 if (assign && (vmcs_info->tsc_offset != tsc_offset || 2440 vmcs_info->reliable != assign_reliable)) { 2441 bool print = vmcs_info->tsc_offset != tsc_offset; 2442 2443 vmcs_info->tsc_offset = tsc_offset; 2444 vmcs_info->reliable = assign_reliable; 2445 if (print) 2446 intel_pt_print_vmcs_info(vmcs_info); 2447 } 2448 2449 /* Determine what TSC Offset to use */ 2450 if (vmcs_info && vmcs_info->tsc_offset) { 2451 if (!vmcs_info->reliable) 2452 reliable = false; 2453 intel_pt_translate_vm_tsc(decoder, vmcs_info); 2454 } else { 2455 reliable = false; 2456 if (vmcs_info) { 2457 if (!vmcs_info->error_printed) { 2458 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64, 2459 vmcs_info->vmcs); 2460 vmcs_info->error_printed = true; 2461 } 2462 } else { 2463 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2464 p_log("ERROR: Unknown VMCS"); 2465 } 2466 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); 2467 } 2468 2469 decoder->vm_tm_corr_reliable = reliable; 2470 } 2471 2472 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder) 2473 { 2474 uint64_t host_tsc = decoder->packet.payload; 2475 uint64_t guest_tsc = decoder->packet.payload; 2476 struct intel_pt_vmcs_info *vmcs_info; 2477 uint64_t vmcs; 2478 2479 vmcs = decoder->vmcs; 2480 if (vmcs == NO_VMCS) 2481 vmcs = 0; 2482 2483 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); 2484 2485 if (decoder->pge) { 2486 if (in_vm(decoder->pip_payload)) { /* Guest */ 2487 if (!vmcs_info) { 2488 /* PIP NR=1 without VMCS cannot happen */ 2489 p_log("ERROR: Missing VMCS"); 2490 } 2491 } else { /* Host */ 2492 return; 2493 } 2494 } else { /* Host or Guest */ 2495 if (intel_pt_time_in_range(decoder, host_tsc)) { 2496 /* Within expected range for Host TSC, assume Host */ 2497 return; 2498 } 2499 } 2500 2501 if (vmcs_info) { 2502 /* Translate Guest TSC to Host TSC */ 2503 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; 2504 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); 2505 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 2506 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", 2507 guest_tsc, host_tsc, vmcs_info->vmcs, 2508 vmcs_info->tsc_offset); 2509 if (!intel_pt_time_in_range(decoder, host_tsc) && 2510 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) 2511 p_log("Timestamp out of range"); 2512 } else { 2513 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) 2514 p_log("ERROR: Unknown VMCS"); 2515 host_tsc = decoder->timestamp; 2516 } 2517 2518 decoder->packet.payload = host_tsc; 2519 2520 if (!decoder->vm_tm_corr_dry_run) 2521 memcpy((void *)decoder->buf + 1, &host_tsc, 8); 2522 } 2523 2524 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder) 2525 { 2526 struct intel_pt_vm_tsc_info data = { .psbend = false }; 2527 bool pge; 2528 int err; 2529 2530 if (decoder->in_psb) 2531 intel_pt_vm_tm_corr_psb(decoder, &data); 2532 2533 while (1) { 2534 err = intel_pt_get_next_packet(decoder); 2535 if (err == -ENOLINK) 2536 continue; 2537 if (err) 2538 break; 2539 2540 switch (decoder->packet.type) { 2541 case INTEL_PT_TIP_PGD: 2542 decoder->pge = false; 2543 decoder->vm_tm_corr_continuous = false; 2544 break; 2545 2546 case INTEL_PT_TNT: 2547 case INTEL_PT_TIP: 2548 case INTEL_PT_TIP_PGE: 2549 decoder->pge = true; 2550 break; 2551 2552 case INTEL_PT_OVF: 2553 decoder->in_psb = false; 2554 pge = decoder->pge; 2555 decoder->pge = intel_pt_ovf_fup_lookahead(decoder); 2556 if (pge != decoder->pge) 2557 intel_pt_log("Surprising PGE change in OVF!"); 2558 if (!decoder->pge) 2559 decoder->vm_tm_corr_continuous = false; 2560 break; 2561 2562 case INTEL_PT_FUP: 2563 if (decoder->in_psb) 2564 decoder->pge = true; 2565 break; 2566 2567 case INTEL_PT_TRACESTOP: 2568 decoder->pge = false; 2569 decoder->vm_tm_corr_continuous = false; 2570 decoder->have_tma = false; 2571 break; 2572 2573 case INTEL_PT_PSB: 2574 intel_pt_vm_tm_corr_psb(decoder, &data); 2575 break; 2576 2577 case INTEL_PT_PIP: 2578 decoder->pip_payload = decoder->packet.payload; 2579 break; 2580 2581 case INTEL_PT_MTC: 2582 intel_pt_calc_mtc_timestamp(decoder); 2583 break; 2584 2585 case INTEL_PT_TSC: 2586 intel_pt_vm_tm_corr_tsc(decoder, &data); 2587 intel_pt_calc_tsc_timestamp(decoder); 2588 decoder->vm_tm_corr_same_buf = true; 2589 decoder->vm_tm_corr_continuous = decoder->pge; 2590 break; 2591 2592 case INTEL_PT_TMA: 2593 intel_pt_calc_tma(decoder); 2594 break; 2595 2596 case INTEL_PT_CYC: 2597 intel_pt_calc_cyc_timestamp(decoder); 2598 break; 2599 2600 case INTEL_PT_CBR: 2601 intel_pt_calc_cbr(decoder); 2602 break; 2603 2604 case INTEL_PT_PSBEND: 2605 decoder->in_psb = false; 2606 data.psbend = false; 2607 break; 2608 2609 case INTEL_PT_VMCS: 2610 if (decoder->packet.payload != NO_VMCS) 2611 decoder->vmcs = decoder->packet.payload; 2612 break; 2613 2614 case INTEL_PT_BBP: 2615 decoder->blk_type = decoder->packet.payload; 2616 break; 2617 2618 case INTEL_PT_BIP: 2619 if (decoder->blk_type == INTEL_PT_PEBS_BASIC && 2620 decoder->packet.count == 2) 2621 intel_pt_vm_tm_corr_pebs_tsc(decoder); 2622 break; 2623 2624 case INTEL_PT_BEP: 2625 case INTEL_PT_BEP_IP: 2626 decoder->blk_type = 0; 2627 break; 2628 2629 case INTEL_PT_MODE_EXEC: 2630 case INTEL_PT_MODE_TSX: 2631 case INTEL_PT_MNT: 2632 case INTEL_PT_PAD: 2633 case INTEL_PT_PTWRITE_IP: 2634 case INTEL_PT_PTWRITE: 2635 case INTEL_PT_MWAIT: 2636 case INTEL_PT_PWRE: 2637 case INTEL_PT_EXSTOP_IP: 2638 case INTEL_PT_EXSTOP: 2639 case INTEL_PT_PWRX: 2640 case INTEL_PT_BAD: /* Does not happen */ 2641 default: 2642 break; 2643 } 2644 } 2645 2646 return err; 2647 } 2648 2649 #define HOP_PROCESS 0 2650 #define HOP_IGNORE 1 2651 #define HOP_RETURN 2 2652 #define HOP_AGAIN 3 2653 2654 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder); 2655 2656 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */ 2657 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) 2658 { 2659 /* Leap from PSB to PSB, getting ip from FUP within PSB+ */ 2660 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { 2661 *err = intel_pt_scan_for_psb(decoder); 2662 if (*err) 2663 return HOP_RETURN; 2664 } 2665 2666 switch (decoder->packet.type) { 2667 case INTEL_PT_TNT: 2668 return HOP_IGNORE; 2669 2670 case INTEL_PT_TIP_PGD: 2671 if (!decoder->packet.count) { 2672 intel_pt_set_nr(decoder); 2673 return HOP_IGNORE; 2674 } 2675 intel_pt_set_ip(decoder); 2676 decoder->state.type |= INTEL_PT_TRACE_END; 2677 decoder->state.from_ip = 0; 2678 decoder->state.to_ip = decoder->ip; 2679 intel_pt_update_nr(decoder); 2680 return HOP_RETURN; 2681 2682 case INTEL_PT_TIP: 2683 if (!decoder->packet.count) { 2684 intel_pt_set_nr(decoder); 2685 return HOP_IGNORE; 2686 } 2687 intel_pt_set_ip(decoder); 2688 decoder->state.type = INTEL_PT_INSTRUCTION; 2689 decoder->state.from_ip = decoder->ip; 2690 decoder->state.to_ip = 0; 2691 intel_pt_update_nr(decoder); 2692 return HOP_RETURN; 2693 2694 case INTEL_PT_FUP: 2695 if (!decoder->packet.count) 2696 return HOP_IGNORE; 2697 intel_pt_set_ip(decoder); 2698 if (intel_pt_fup_event(decoder)) 2699 return HOP_RETURN; 2700 if (!decoder->branch_enable) 2701 *no_tip = true; 2702 if (*no_tip) { 2703 decoder->state.type = INTEL_PT_INSTRUCTION; 2704 decoder->state.from_ip = decoder->ip; 2705 decoder->state.to_ip = 0; 2706 return HOP_RETURN; 2707 } 2708 *err = intel_pt_walk_fup_tip(decoder); 2709 if (!*err) 2710 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2711 return HOP_RETURN; 2712 2713 case INTEL_PT_PSB: 2714 decoder->state.psb_offset = decoder->pos; 2715 decoder->psb_ip = 0; 2716 decoder->last_ip = 0; 2717 decoder->have_last_ip = true; 2718 *err = intel_pt_walk_psbend(decoder); 2719 if (*err == -EAGAIN) 2720 return HOP_AGAIN; 2721 if (*err) 2722 return HOP_RETURN; 2723 decoder->state.type = INTEL_PT_PSB_EVT; 2724 if (decoder->psb_ip) { 2725 decoder->state.type |= INTEL_PT_INSTRUCTION; 2726 decoder->ip = decoder->psb_ip; 2727 } 2728 decoder->state.from_ip = decoder->psb_ip; 2729 decoder->state.to_ip = 0; 2730 return HOP_RETURN; 2731 2732 case INTEL_PT_BAD: 2733 case INTEL_PT_PAD: 2734 case INTEL_PT_TIP_PGE: 2735 case INTEL_PT_TSC: 2736 case INTEL_PT_TMA: 2737 case INTEL_PT_MODE_EXEC: 2738 case INTEL_PT_MODE_TSX: 2739 case INTEL_PT_MTC: 2740 case INTEL_PT_CYC: 2741 case INTEL_PT_VMCS: 2742 case INTEL_PT_PSBEND: 2743 case INTEL_PT_CBR: 2744 case INTEL_PT_TRACESTOP: 2745 case INTEL_PT_PIP: 2746 case INTEL_PT_OVF: 2747 case INTEL_PT_MNT: 2748 case INTEL_PT_PTWRITE: 2749 case INTEL_PT_PTWRITE_IP: 2750 case INTEL_PT_EXSTOP: 2751 case INTEL_PT_EXSTOP_IP: 2752 case INTEL_PT_MWAIT: 2753 case INTEL_PT_PWRE: 2754 case INTEL_PT_PWRX: 2755 case INTEL_PT_BBP: 2756 case INTEL_PT_BIP: 2757 case INTEL_PT_BEP: 2758 case INTEL_PT_BEP_IP: 2759 default: 2760 return HOP_PROCESS; 2761 } 2762 } 2763 2764 struct intel_pt_psb_info { 2765 struct intel_pt_pkt fup_packet; 2766 bool fup; 2767 int after_psbend; 2768 }; 2769 2770 /* Lookahead and get the FUP packet from PSB+ */ 2771 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) 2772 { 2773 struct intel_pt_psb_info *data = pkt_info->data; 2774 2775 switch (pkt_info->packet.type) { 2776 case INTEL_PT_PAD: 2777 case INTEL_PT_MNT: 2778 case INTEL_PT_TSC: 2779 case INTEL_PT_TMA: 2780 case INTEL_PT_MODE_EXEC: 2781 case INTEL_PT_MODE_TSX: 2782 case INTEL_PT_MTC: 2783 case INTEL_PT_CYC: 2784 case INTEL_PT_VMCS: 2785 case INTEL_PT_CBR: 2786 case INTEL_PT_PIP: 2787 if (data->after_psbend) { 2788 data->after_psbend -= 1; 2789 if (!data->after_psbend) 2790 return 1; 2791 } 2792 break; 2793 2794 case INTEL_PT_FUP: 2795 if (data->after_psbend) 2796 return 1; 2797 if (data->fup || pkt_info->packet.count == 0) 2798 return 1; 2799 data->fup_packet = pkt_info->packet; 2800 data->fup = true; 2801 break; 2802 2803 case INTEL_PT_PSBEND: 2804 if (!data->fup) 2805 return 1; 2806 /* Keep going to check for a TIP.PGE */ 2807 data->after_psbend = 6; 2808 break; 2809 2810 case INTEL_PT_TIP_PGE: 2811 /* Ignore FUP in PSB+ if followed by TIP.PGE */ 2812 if (data->after_psbend) 2813 data->fup = false; 2814 return 1; 2815 2816 case INTEL_PT_PTWRITE: 2817 case INTEL_PT_PTWRITE_IP: 2818 case INTEL_PT_EXSTOP: 2819 case INTEL_PT_EXSTOP_IP: 2820 case INTEL_PT_MWAIT: 2821 case INTEL_PT_PWRE: 2822 case INTEL_PT_PWRX: 2823 case INTEL_PT_BBP: 2824 case INTEL_PT_BIP: 2825 case INTEL_PT_BEP: 2826 case INTEL_PT_BEP_IP: 2827 if (data->after_psbend) { 2828 data->after_psbend -= 1; 2829 if (!data->after_psbend) 2830 return 1; 2831 break; 2832 } 2833 return 1; 2834 2835 case INTEL_PT_OVF: 2836 case INTEL_PT_BAD: 2837 case INTEL_PT_TNT: 2838 case INTEL_PT_TIP_PGD: 2839 case INTEL_PT_TIP: 2840 case INTEL_PT_PSB: 2841 case INTEL_PT_TRACESTOP: 2842 default: 2843 return 1; 2844 } 2845 2846 return 0; 2847 } 2848 2849 static int intel_pt_psb(struct intel_pt_decoder *decoder) 2850 { 2851 int err; 2852 2853 decoder->last_ip = 0; 2854 decoder->psb_ip = 0; 2855 decoder->have_last_ip = true; 2856 intel_pt_clear_stack(&decoder->stack); 2857 err = intel_pt_walk_psbend(decoder); 2858 if (err) 2859 return err; 2860 decoder->state.type = INTEL_PT_PSB_EVT; 2861 decoder->state.from_ip = decoder->psb_ip; 2862 decoder->state.to_ip = 0; 2863 return 0; 2864 } 2865 2866 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder) 2867 { 2868 int err; 2869 2870 if (decoder->ip != decoder->last_ip) { 2871 err = intel_pt_walk_fup(decoder); 2872 if (!err || err != -EAGAIN) 2873 return err; 2874 } 2875 2876 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 2877 err = intel_pt_psb(decoder); 2878 if (err) { 2879 decoder->pkt_state = INTEL_PT_STATE_ERR3; 2880 return -ENOENT; 2881 } 2882 2883 return 0; 2884 } 2885 2886 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err) 2887 { 2888 struct intel_pt_psb_info data = { .fup = false }; 2889 2890 if (!decoder->branch_enable || !decoder->pge) 2891 return false; 2892 2893 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data); 2894 if (!data.fup) 2895 return false; 2896 2897 decoder->packet = data.fup_packet; 2898 intel_pt_set_last_ip(decoder); 2899 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB; 2900 2901 *err = intel_pt_fup_in_psb(decoder); 2902 2903 return true; 2904 } 2905 2906 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) 2907 { 2908 int last_packet_type = INTEL_PT_PAD; 2909 bool no_tip = false; 2910 int err; 2911 2912 while (1) { 2913 err = intel_pt_get_next_packet(decoder); 2914 if (err) 2915 return err; 2916 next: 2917 if (decoder->cyc_threshold) { 2918 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) 2919 decoder->sample_cyc = false; 2920 last_packet_type = decoder->packet.type; 2921 } 2922 2923 if (decoder->hop) { 2924 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { 2925 case HOP_IGNORE: 2926 continue; 2927 case HOP_RETURN: 2928 return err; 2929 case HOP_AGAIN: 2930 goto next; 2931 default: 2932 break; 2933 } 2934 } 2935 2936 switch (decoder->packet.type) { 2937 case INTEL_PT_TNT: 2938 if (!decoder->packet.count) 2939 break; 2940 decoder->tnt = decoder->packet; 2941 decoder->pkt_state = INTEL_PT_STATE_TNT; 2942 err = intel_pt_walk_tnt(decoder); 2943 if (err == -EAGAIN) 2944 break; 2945 return err; 2946 2947 case INTEL_PT_TIP_PGD: 2948 if (decoder->packet.count != 0) 2949 intel_pt_set_last_ip(decoder); 2950 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; 2951 return intel_pt_walk_tip(decoder); 2952 2953 case INTEL_PT_TIP_PGE: { 2954 decoder->pge = true; 2955 intel_pt_mtc_cyc_cnt_pge(decoder); 2956 intel_pt_set_nr(decoder); 2957 if (decoder->packet.count == 0) { 2958 intel_pt_log_at("Skipping zero TIP.PGE", 2959 decoder->pos); 2960 break; 2961 } 2962 intel_pt_set_ip(decoder); 2963 decoder->state.from_ip = 0; 2964 decoder->state.to_ip = decoder->ip; 2965 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 2966 /* 2967 * In hop mode, resample to get the to_ip as an 2968 * "instruction" sample. 2969 */ 2970 if (decoder->hop) 2971 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 2972 return 0; 2973 } 2974 2975 case INTEL_PT_OVF: 2976 return intel_pt_overflow(decoder); 2977 2978 case INTEL_PT_TIP: 2979 if (decoder->packet.count != 0) 2980 intel_pt_set_last_ip(decoder); 2981 decoder->pkt_state = INTEL_PT_STATE_TIP; 2982 return intel_pt_walk_tip(decoder); 2983 2984 case INTEL_PT_FUP: 2985 if (decoder->packet.count == 0) { 2986 intel_pt_log_at("Skipping zero FUP", 2987 decoder->pos); 2988 no_tip = false; 2989 break; 2990 } 2991 intel_pt_set_last_ip(decoder); 2992 if (!decoder->branch_enable) { 2993 decoder->ip = decoder->last_ip; 2994 if (intel_pt_fup_event(decoder)) 2995 return 0; 2996 no_tip = false; 2997 break; 2998 } 2999 if (decoder->set_fup_mwait) 3000 no_tip = true; 3001 if (no_tip) 3002 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; 3003 else 3004 decoder->pkt_state = INTEL_PT_STATE_FUP; 3005 err = intel_pt_walk_fup(decoder); 3006 if (err != -EAGAIN) 3007 return err; 3008 if (no_tip) { 3009 no_tip = false; 3010 break; 3011 } 3012 return intel_pt_walk_fup_tip(decoder); 3013 3014 case INTEL_PT_TRACESTOP: 3015 decoder->pge = false; 3016 decoder->continuous_period = false; 3017 intel_pt_clear_tx_flags(decoder); 3018 decoder->have_tma = false; 3019 break; 3020 3021 case INTEL_PT_PSB: 3022 decoder->state.psb_offset = decoder->pos; 3023 decoder->psb_ip = 0; 3024 if (intel_pt_psb_with_fup(decoder, &err)) 3025 return err; 3026 err = intel_pt_psb(decoder); 3027 if (err == -EAGAIN) 3028 goto next; 3029 return err; 3030 3031 case INTEL_PT_PIP: 3032 intel_pt_update_pip(decoder); 3033 break; 3034 3035 case INTEL_PT_MTC: 3036 intel_pt_calc_mtc_timestamp(decoder); 3037 if (decoder->period_type != INTEL_PT_PERIOD_MTC) 3038 break; 3039 /* 3040 * Ensure that there has been an instruction since the 3041 * last MTC. 3042 */ 3043 if (!decoder->mtc_insn) 3044 break; 3045 decoder->mtc_insn = false; 3046 /* Ensure that there is a timestamp */ 3047 if (!decoder->timestamp) 3048 break; 3049 decoder->state.type = INTEL_PT_INSTRUCTION; 3050 decoder->state.from_ip = decoder->ip; 3051 decoder->state.to_ip = 0; 3052 decoder->mtc_insn = false; 3053 return 0; 3054 3055 case INTEL_PT_TSC: 3056 intel_pt_calc_tsc_timestamp(decoder); 3057 break; 3058 3059 case INTEL_PT_TMA: 3060 intel_pt_calc_tma(decoder); 3061 break; 3062 3063 case INTEL_PT_CYC: 3064 intel_pt_calc_cyc_timestamp(decoder); 3065 break; 3066 3067 case INTEL_PT_CBR: 3068 intel_pt_calc_cbr(decoder); 3069 if (decoder->cbr != decoder->cbr_seen) { 3070 decoder->state.type = 0; 3071 return 0; 3072 } 3073 break; 3074 3075 case INTEL_PT_MODE_EXEC: 3076 decoder->exec_mode = decoder->packet.payload; 3077 break; 3078 3079 case INTEL_PT_MODE_TSX: 3080 /* MODE_TSX need not be followed by FUP */ 3081 if (!decoder->pge || decoder->in_psb) { 3082 intel_pt_update_in_tx(decoder); 3083 break; 3084 } 3085 err = intel_pt_mode_tsx(decoder, &no_tip); 3086 if (err) 3087 return err; 3088 goto next; 3089 3090 case INTEL_PT_BAD: /* Does not happen */ 3091 return intel_pt_bug(decoder); 3092 3093 case INTEL_PT_PSBEND: 3094 case INTEL_PT_VMCS: 3095 case INTEL_PT_MNT: 3096 case INTEL_PT_PAD: 3097 break; 3098 3099 case INTEL_PT_PTWRITE_IP: 3100 decoder->fup_ptw_payload = decoder->packet.payload; 3101 err = intel_pt_get_next_packet(decoder); 3102 if (err) 3103 return err; 3104 if (decoder->packet.type == INTEL_PT_FUP) { 3105 decoder->set_fup_ptw = true; 3106 no_tip = true; 3107 } else { 3108 intel_pt_log_at("ERROR: Missing FUP after PTWRITE", 3109 decoder->pos); 3110 } 3111 goto next; 3112 3113 case INTEL_PT_PTWRITE: 3114 decoder->state.type = INTEL_PT_PTW; 3115 decoder->state.from_ip = decoder->ip; 3116 decoder->state.to_ip = 0; 3117 decoder->state.ptw_payload = decoder->packet.payload; 3118 return 0; 3119 3120 case INTEL_PT_MWAIT: 3121 decoder->fup_mwait_payload = decoder->packet.payload; 3122 decoder->set_fup_mwait = true; 3123 break; 3124 3125 case INTEL_PT_PWRE: 3126 if (decoder->set_fup_mwait) { 3127 decoder->fup_pwre_payload = 3128 decoder->packet.payload; 3129 decoder->set_fup_pwre = true; 3130 break; 3131 } 3132 decoder->state.type = INTEL_PT_PWR_ENTRY; 3133 decoder->state.from_ip = decoder->ip; 3134 decoder->state.to_ip = 0; 3135 decoder->state.pwrx_payload = decoder->packet.payload; 3136 return 0; 3137 3138 case INTEL_PT_EXSTOP_IP: 3139 err = intel_pt_get_next_packet(decoder); 3140 if (err) 3141 return err; 3142 if (decoder->packet.type == INTEL_PT_FUP) { 3143 decoder->set_fup_exstop = true; 3144 no_tip = true; 3145 } else { 3146 intel_pt_log_at("ERROR: Missing FUP after EXSTOP", 3147 decoder->pos); 3148 } 3149 goto next; 3150 3151 case INTEL_PT_EXSTOP: 3152 decoder->state.type = INTEL_PT_EX_STOP; 3153 decoder->state.from_ip = decoder->ip; 3154 decoder->state.to_ip = 0; 3155 return 0; 3156 3157 case INTEL_PT_PWRX: 3158 decoder->state.type = INTEL_PT_PWR_EXIT; 3159 decoder->state.from_ip = decoder->ip; 3160 decoder->state.to_ip = 0; 3161 decoder->state.pwrx_payload = decoder->packet.payload; 3162 return 0; 3163 3164 case INTEL_PT_BBP: 3165 intel_pt_bbp(decoder); 3166 break; 3167 3168 case INTEL_PT_BIP: 3169 intel_pt_bip(decoder); 3170 break; 3171 3172 case INTEL_PT_BEP: 3173 decoder->state.type = INTEL_PT_BLK_ITEMS; 3174 decoder->state.from_ip = decoder->ip; 3175 decoder->state.to_ip = 0; 3176 return 0; 3177 3178 case INTEL_PT_BEP_IP: 3179 err = intel_pt_get_next_packet(decoder); 3180 if (err) 3181 return err; 3182 if (decoder->packet.type == INTEL_PT_FUP) { 3183 decoder->set_fup_bep = true; 3184 no_tip = true; 3185 } else { 3186 intel_pt_log_at("ERROR: Missing FUP after BEP", 3187 decoder->pos); 3188 } 3189 goto next; 3190 3191 default: 3192 return intel_pt_bug(decoder); 3193 } 3194 } 3195 } 3196 3197 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) 3198 { 3199 return decoder->packet.count && 3200 (decoder->have_last_ip || decoder->packet.count == 3 || 3201 decoder->packet.count == 6); 3202 } 3203 3204 /* Walk PSB+ packets to get in sync. */ 3205 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) 3206 { 3207 int err; 3208 3209 decoder->in_psb = true; 3210 3211 while (1) { 3212 err = intel_pt_get_next_packet(decoder); 3213 if (err) 3214 goto out; 3215 3216 switch (decoder->packet.type) { 3217 case INTEL_PT_TIP_PGD: 3218 decoder->continuous_period = false; 3219 __fallthrough; 3220 case INTEL_PT_TIP_PGE: 3221 case INTEL_PT_TIP: 3222 case INTEL_PT_PTWRITE: 3223 case INTEL_PT_PTWRITE_IP: 3224 case INTEL_PT_EXSTOP: 3225 case INTEL_PT_EXSTOP_IP: 3226 case INTEL_PT_MWAIT: 3227 case INTEL_PT_PWRE: 3228 case INTEL_PT_PWRX: 3229 case INTEL_PT_BBP: 3230 case INTEL_PT_BIP: 3231 case INTEL_PT_BEP: 3232 case INTEL_PT_BEP_IP: 3233 intel_pt_log("ERROR: Unexpected packet\n"); 3234 err = -ENOENT; 3235 goto out; 3236 3237 case INTEL_PT_FUP: 3238 decoder->pge = true; 3239 if (intel_pt_have_ip(decoder)) { 3240 uint64_t current_ip = decoder->ip; 3241 3242 intel_pt_set_ip(decoder); 3243 decoder->psb_ip = decoder->ip; 3244 if (current_ip) 3245 intel_pt_log_to("Setting IP", 3246 decoder->ip); 3247 } 3248 break; 3249 3250 case INTEL_PT_MTC: 3251 intel_pt_calc_mtc_timestamp(decoder); 3252 break; 3253 3254 case INTEL_PT_TSC: 3255 intel_pt_calc_tsc_timestamp(decoder); 3256 break; 3257 3258 case INTEL_PT_TMA: 3259 intel_pt_calc_tma(decoder); 3260 break; 3261 3262 case INTEL_PT_CYC: 3263 intel_pt_calc_cyc_timestamp(decoder); 3264 break; 3265 3266 case INTEL_PT_CBR: 3267 intel_pt_calc_cbr(decoder); 3268 break; 3269 3270 case INTEL_PT_PIP: 3271 intel_pt_set_pip(decoder); 3272 break; 3273 3274 case INTEL_PT_MODE_EXEC: 3275 decoder->exec_mode = decoder->packet.payload; 3276 break; 3277 3278 case INTEL_PT_MODE_TSX: 3279 intel_pt_update_in_tx(decoder); 3280 break; 3281 3282 case INTEL_PT_TRACESTOP: 3283 decoder->pge = false; 3284 decoder->continuous_period = false; 3285 intel_pt_clear_tx_flags(decoder); 3286 __fallthrough; 3287 3288 case INTEL_PT_TNT: 3289 decoder->have_tma = false; 3290 intel_pt_log("ERROR: Unexpected packet\n"); 3291 if (decoder->ip) 3292 decoder->pkt_state = INTEL_PT_STATE_ERR4; 3293 else 3294 decoder->pkt_state = INTEL_PT_STATE_ERR3; 3295 err = -ENOENT; 3296 goto out; 3297 3298 case INTEL_PT_BAD: /* Does not happen */ 3299 err = intel_pt_bug(decoder); 3300 goto out; 3301 3302 case INTEL_PT_OVF: 3303 err = intel_pt_overflow(decoder); 3304 goto out; 3305 3306 case INTEL_PT_PSBEND: 3307 err = 0; 3308 goto out; 3309 3310 case INTEL_PT_PSB: 3311 case INTEL_PT_VMCS: 3312 case INTEL_PT_MNT: 3313 case INTEL_PT_PAD: 3314 default: 3315 break; 3316 } 3317 } 3318 out: 3319 decoder->in_psb = false; 3320 3321 return err; 3322 } 3323 3324 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) 3325 { 3326 int err; 3327 3328 while (1) { 3329 err = intel_pt_get_next_packet(decoder); 3330 if (err) 3331 return err; 3332 3333 switch (decoder->packet.type) { 3334 case INTEL_PT_TIP_PGD: 3335 decoder->continuous_period = false; 3336 decoder->pge = false; 3337 if (intel_pt_have_ip(decoder)) 3338 intel_pt_set_ip(decoder); 3339 if (!decoder->ip) 3340 break; 3341 decoder->state.type |= INTEL_PT_TRACE_END; 3342 return 0; 3343 3344 case INTEL_PT_TIP_PGE: 3345 decoder->pge = true; 3346 intel_pt_mtc_cyc_cnt_pge(decoder); 3347 if (intel_pt_have_ip(decoder)) 3348 intel_pt_set_ip(decoder); 3349 if (!decoder->ip) 3350 break; 3351 decoder->state.type |= INTEL_PT_TRACE_BEGIN; 3352 return 0; 3353 3354 case INTEL_PT_TIP: 3355 decoder->pge = true; 3356 if (intel_pt_have_ip(decoder)) 3357 intel_pt_set_ip(decoder); 3358 if (!decoder->ip) 3359 break; 3360 return 0; 3361 3362 case INTEL_PT_FUP: 3363 if (intel_pt_have_ip(decoder)) 3364 intel_pt_set_ip(decoder); 3365 if (decoder->ip) 3366 return 0; 3367 break; 3368 3369 case INTEL_PT_MTC: 3370 intel_pt_calc_mtc_timestamp(decoder); 3371 break; 3372 3373 case INTEL_PT_TSC: 3374 intel_pt_calc_tsc_timestamp(decoder); 3375 break; 3376 3377 case INTEL_PT_TMA: 3378 intel_pt_calc_tma(decoder); 3379 break; 3380 3381 case INTEL_PT_CYC: 3382 intel_pt_calc_cyc_timestamp(decoder); 3383 break; 3384 3385 case INTEL_PT_CBR: 3386 intel_pt_calc_cbr(decoder); 3387 break; 3388 3389 case INTEL_PT_PIP: 3390 intel_pt_set_pip(decoder); 3391 break; 3392 3393 case INTEL_PT_MODE_EXEC: 3394 decoder->exec_mode = decoder->packet.payload; 3395 break; 3396 3397 case INTEL_PT_MODE_TSX: 3398 intel_pt_update_in_tx(decoder); 3399 break; 3400 3401 case INTEL_PT_OVF: 3402 return intel_pt_overflow(decoder); 3403 3404 case INTEL_PT_BAD: /* Does not happen */ 3405 return intel_pt_bug(decoder); 3406 3407 case INTEL_PT_TRACESTOP: 3408 decoder->pge = false; 3409 decoder->continuous_period = false; 3410 intel_pt_clear_tx_flags(decoder); 3411 decoder->have_tma = false; 3412 break; 3413 3414 case INTEL_PT_PSB: 3415 decoder->state.psb_offset = decoder->pos; 3416 decoder->psb_ip = 0; 3417 decoder->last_ip = 0; 3418 decoder->have_last_ip = true; 3419 intel_pt_clear_stack(&decoder->stack); 3420 err = intel_pt_walk_psb(decoder); 3421 if (err) 3422 return err; 3423 decoder->state.type = INTEL_PT_PSB_EVT; 3424 decoder->state.from_ip = decoder->psb_ip; 3425 decoder->state.to_ip = 0; 3426 return 0; 3427 3428 case INTEL_PT_TNT: 3429 case INTEL_PT_PSBEND: 3430 case INTEL_PT_VMCS: 3431 case INTEL_PT_MNT: 3432 case INTEL_PT_PAD: 3433 case INTEL_PT_PTWRITE: 3434 case INTEL_PT_PTWRITE_IP: 3435 case INTEL_PT_EXSTOP: 3436 case INTEL_PT_EXSTOP_IP: 3437 case INTEL_PT_MWAIT: 3438 case INTEL_PT_PWRE: 3439 case INTEL_PT_PWRX: 3440 case INTEL_PT_BBP: 3441 case INTEL_PT_BIP: 3442 case INTEL_PT_BEP: 3443 case INTEL_PT_BEP_IP: 3444 default: 3445 break; 3446 } 3447 } 3448 } 3449 3450 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) 3451 { 3452 int err; 3453 3454 decoder->set_fup_tx_flags = false; 3455 decoder->set_fup_ptw = false; 3456 decoder->set_fup_mwait = false; 3457 decoder->set_fup_pwre = false; 3458 decoder->set_fup_exstop = false; 3459 decoder->set_fup_bep = false; 3460 3461 if (!decoder->branch_enable) { 3462 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3463 decoder->overflow = false; 3464 decoder->state.type = 0; /* Do not have a sample */ 3465 return 0; 3466 } 3467 3468 intel_pt_log("Scanning for full IP\n"); 3469 err = intel_pt_walk_to_ip(decoder); 3470 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip)) 3471 return err; 3472 3473 /* In hop mode, resample to get the to_ip as an "instruction" sample */ 3474 if (decoder->hop) 3475 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3476 else 3477 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3478 decoder->overflow = false; 3479 3480 decoder->state.from_ip = 0; 3481 decoder->state.to_ip = decoder->ip; 3482 intel_pt_log_to("Setting IP", decoder->ip); 3483 3484 return 0; 3485 } 3486 3487 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) 3488 { 3489 const unsigned char *end = decoder->buf + decoder->len; 3490 size_t i; 3491 3492 for (i = INTEL_PT_PSB_LEN - 1; i; i--) { 3493 if (i > decoder->len) 3494 continue; 3495 if (!memcmp(end - i, INTEL_PT_PSB_STR, i)) 3496 return i; 3497 } 3498 return 0; 3499 } 3500 3501 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) 3502 { 3503 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb; 3504 const char *psb = INTEL_PT_PSB_STR; 3505 3506 if (rest_psb > decoder->len || 3507 memcmp(decoder->buf, psb + part_psb, rest_psb)) 3508 return 0; 3509 3510 return rest_psb; 3511 } 3512 3513 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, 3514 int part_psb) 3515 { 3516 int rest_psb, ret; 3517 3518 decoder->pos += decoder->len; 3519 decoder->len = 0; 3520 3521 ret = intel_pt_get_next_data(decoder, false); 3522 if (ret) 3523 return ret; 3524 3525 rest_psb = intel_pt_rest_psb(decoder, part_psb); 3526 if (!rest_psb) 3527 return 0; 3528 3529 decoder->pos -= part_psb; 3530 decoder->next_buf = decoder->buf + rest_psb; 3531 decoder->next_len = decoder->len - rest_psb; 3532 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3533 decoder->buf = decoder->temp_buf; 3534 decoder->len = INTEL_PT_PSB_LEN; 3535 3536 return 0; 3537 } 3538 3539 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) 3540 { 3541 unsigned char *next; 3542 int ret; 3543 3544 intel_pt_log("Scanning for PSB\n"); 3545 while (1) { 3546 if (!decoder->len) { 3547 ret = intel_pt_get_next_data(decoder, false); 3548 if (ret) 3549 return ret; 3550 } 3551 3552 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, 3553 INTEL_PT_PSB_LEN); 3554 if (!next) { 3555 int part_psb; 3556 3557 part_psb = intel_pt_part_psb(decoder); 3558 if (part_psb) { 3559 ret = intel_pt_get_split_psb(decoder, part_psb); 3560 if (ret) 3561 return ret; 3562 } else { 3563 decoder->pos += decoder->len; 3564 decoder->len = 0; 3565 } 3566 continue; 3567 } 3568 3569 decoder->pkt_step = next - decoder->buf; 3570 return intel_pt_get_next_packet(decoder); 3571 } 3572 } 3573 3574 static int intel_pt_sync(struct intel_pt_decoder *decoder) 3575 { 3576 int err; 3577 3578 decoder->pge = false; 3579 decoder->continuous_period = false; 3580 decoder->have_last_ip = false; 3581 decoder->last_ip = 0; 3582 decoder->psb_ip = 0; 3583 decoder->ip = 0; 3584 intel_pt_clear_stack(&decoder->stack); 3585 3586 err = intel_pt_scan_for_psb(decoder); 3587 if (err) 3588 return err; 3589 3590 if (decoder->vm_time_correlation) { 3591 decoder->in_psb = true; 3592 if (!decoder->timestamp) 3593 decoder->timestamp = 1; 3594 decoder->state.type = 0; 3595 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; 3596 return 0; 3597 } 3598 3599 decoder->have_last_ip = true; 3600 decoder->pkt_state = INTEL_PT_STATE_NO_IP; 3601 3602 err = intel_pt_walk_psb(decoder); 3603 if (err) 3604 return err; 3605 3606 decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */ 3607 decoder->state.from_ip = decoder->psb_ip; 3608 decoder->state.to_ip = 0; 3609 3610 if (decoder->ip) { 3611 /* 3612 * In hop mode, resample to get the PSB FUP ip as an 3613 * "instruction" sample. 3614 */ 3615 if (decoder->hop) 3616 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; 3617 else 3618 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; 3619 } 3620 3621 return 0; 3622 } 3623 3624 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) 3625 { 3626 uint64_t est = decoder->sample_insn_cnt << 1; 3627 3628 if (!decoder->cbr || !decoder->max_non_turbo_ratio) 3629 goto out; 3630 3631 est *= decoder->max_non_turbo_ratio; 3632 est /= decoder->cbr; 3633 out: 3634 return decoder->sample_timestamp + est; 3635 } 3636 3637 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) 3638 { 3639 int err; 3640 3641 do { 3642 decoder->state.type = INTEL_PT_BRANCH; 3643 decoder->state.flags = 0; 3644 3645 switch (decoder->pkt_state) { 3646 case INTEL_PT_STATE_NO_PSB: 3647 err = intel_pt_sync(decoder); 3648 break; 3649 case INTEL_PT_STATE_NO_IP: 3650 decoder->have_last_ip = false; 3651 decoder->last_ip = 0; 3652 decoder->ip = 0; 3653 __fallthrough; 3654 case INTEL_PT_STATE_ERR_RESYNC: 3655 err = intel_pt_sync_ip(decoder); 3656 break; 3657 case INTEL_PT_STATE_IN_SYNC: 3658 err = intel_pt_walk_trace(decoder); 3659 break; 3660 case INTEL_PT_STATE_TNT: 3661 case INTEL_PT_STATE_TNT_CONT: 3662 err = intel_pt_walk_tnt(decoder); 3663 if (err == -EAGAIN) 3664 err = intel_pt_walk_trace(decoder); 3665 break; 3666 case INTEL_PT_STATE_TIP: 3667 case INTEL_PT_STATE_TIP_PGD: 3668 err = intel_pt_walk_tip(decoder); 3669 break; 3670 case INTEL_PT_STATE_FUP: 3671 err = intel_pt_walk_fup(decoder); 3672 if (err == -EAGAIN) 3673 err = intel_pt_walk_fup_tip(decoder); 3674 break; 3675 case INTEL_PT_STATE_FUP_NO_TIP: 3676 err = intel_pt_walk_fup(decoder); 3677 if (err == -EAGAIN) 3678 err = intel_pt_walk_trace(decoder); 3679 break; 3680 case INTEL_PT_STATE_FUP_IN_PSB: 3681 err = intel_pt_fup_in_psb(decoder); 3682 break; 3683 case INTEL_PT_STATE_RESAMPLE: 3684 err = intel_pt_resample(decoder); 3685 break; 3686 case INTEL_PT_STATE_VM_TIME_CORRELATION: 3687 err = intel_pt_vm_time_correlation(decoder); 3688 break; 3689 default: 3690 err = intel_pt_bug(decoder); 3691 break; 3692 } 3693 } while (err == -ENOLINK); 3694 3695 if (err) { 3696 decoder->state.err = intel_pt_ext_err(err); 3697 decoder->state.from_ip = decoder->ip; 3698 intel_pt_update_sample_time(decoder); 3699 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3700 intel_pt_set_nr(decoder); 3701 } else { 3702 decoder->state.err = 0; 3703 if (decoder->cbr != decoder->cbr_seen) { 3704 decoder->cbr_seen = decoder->cbr; 3705 if (!decoder->state.type) { 3706 decoder->state.from_ip = decoder->ip; 3707 decoder->state.to_ip = 0; 3708 } 3709 decoder->state.type |= INTEL_PT_CBR_CHG; 3710 decoder->state.cbr_payload = decoder->cbr_payload; 3711 decoder->state.cbr = decoder->cbr; 3712 } 3713 if (intel_pt_sample_time(decoder->pkt_state)) { 3714 intel_pt_update_sample_time(decoder); 3715 if (decoder->sample_cyc) { 3716 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; 3717 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3718 decoder->sample_cyc = false; 3719 } 3720 } 3721 /* 3722 * When using only TSC/MTC to compute cycles, IPC can be 3723 * sampled as soon as the cycle count changes. 3724 */ 3725 if (!decoder->have_cyc) 3726 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; 3727 } 3728 3729 /* Let PSB event always have TSC timestamp */ 3730 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp) 3731 decoder->sample_timestamp = decoder->tsc_timestamp; 3732 3733 decoder->state.from_nr = decoder->nr; 3734 decoder->state.to_nr = decoder->next_nr; 3735 decoder->nr = decoder->next_nr; 3736 3737 decoder->state.timestamp = decoder->sample_timestamp; 3738 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); 3739 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; 3740 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; 3741 3742 return &decoder->state; 3743 } 3744 3745 /** 3746 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet. 3747 * @buf: pointer to buffer pointer 3748 * @len: size of buffer 3749 * 3750 * Updates the buffer pointer to point to the start of the next PSB packet if 3751 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated, 3752 * @len is adjusted accordingly. 3753 * 3754 * Return: %true if a PSB packet is found, %false otherwise. 3755 */ 3756 static bool intel_pt_next_psb(unsigned char **buf, size_t *len) 3757 { 3758 unsigned char *next; 3759 3760 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3761 if (next) { 3762 *len -= next - *buf; 3763 *buf = next; 3764 return true; 3765 } 3766 return false; 3767 } 3768 3769 /** 3770 * intel_pt_step_psb - move buffer pointer to the start of the following PSB 3771 * packet. 3772 * @buf: pointer to buffer pointer 3773 * @len: size of buffer 3774 * 3775 * Updates the buffer pointer to point to the start of the following PSB packet 3776 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer 3777 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly. 3778 * 3779 * Return: %true if a PSB packet is found, %false otherwise. 3780 */ 3781 static bool intel_pt_step_psb(unsigned char **buf, size_t *len) 3782 { 3783 unsigned char *next; 3784 3785 if (!*len) 3786 return false; 3787 3788 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); 3789 if (next) { 3790 *len -= next - *buf; 3791 *buf = next; 3792 return true; 3793 } 3794 return false; 3795 } 3796 3797 /** 3798 * intel_pt_last_psb - find the last PSB packet in a buffer. 3799 * @buf: buffer 3800 * @len: size of buffer 3801 * 3802 * This function finds the last PSB in a buffer. 3803 * 3804 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise. 3805 */ 3806 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len) 3807 { 3808 const char *n = INTEL_PT_PSB_STR; 3809 unsigned char *p; 3810 size_t k; 3811 3812 if (len < INTEL_PT_PSB_LEN) 3813 return NULL; 3814 3815 k = len - INTEL_PT_PSB_LEN + 1; 3816 while (1) { 3817 p = memrchr(buf, n[0], k); 3818 if (!p) 3819 return NULL; 3820 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1)) 3821 return p; 3822 k = p - buf; 3823 if (!k) 3824 return NULL; 3825 } 3826 } 3827 3828 /** 3829 * intel_pt_next_tsc - find and return next TSC. 3830 * @buf: buffer 3831 * @len: size of buffer 3832 * @tsc: TSC value returned 3833 * @rem: returns remaining size when TSC is found 3834 * 3835 * Find a TSC packet in @buf and return the TSC value. This function assumes 3836 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a 3837 * PSBEND packet is found. 3838 * 3839 * Return: %true if TSC is found, false otherwise. 3840 */ 3841 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc, 3842 size_t *rem) 3843 { 3844 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX; 3845 struct intel_pt_pkt packet; 3846 int ret; 3847 3848 while (len) { 3849 ret = intel_pt_get_packet(buf, len, &packet, &ctx); 3850 if (ret <= 0) 3851 return false; 3852 if (packet.type == INTEL_PT_TSC) { 3853 *tsc = packet.payload; 3854 *rem = len; 3855 return true; 3856 } 3857 if (packet.type == INTEL_PT_PSBEND) 3858 return false; 3859 buf += ret; 3860 len -= ret; 3861 } 3862 return false; 3863 } 3864 3865 /** 3866 * intel_pt_tsc_cmp - compare 7-byte TSCs. 3867 * @tsc1: first TSC to compare 3868 * @tsc2: second TSC to compare 3869 * 3870 * This function compares 7-byte TSC values allowing for the possibility that 3871 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped 3872 * around so for that purpose this function assumes the absolute difference is 3873 * less than half the maximum difference. 3874 * 3875 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is 3876 * after @tsc2. 3877 */ 3878 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2) 3879 { 3880 const uint64_t halfway = (1ULL << 55); 3881 3882 if (tsc1 == tsc2) 3883 return 0; 3884 3885 if (tsc1 < tsc2) { 3886 if (tsc2 - tsc1 < halfway) 3887 return -1; 3888 else 3889 return 1; 3890 } else { 3891 if (tsc1 - tsc2 < halfway) 3892 return 1; 3893 else 3894 return -1; 3895 } 3896 } 3897 3898 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1) 3899 3900 /** 3901 * adj_for_padding - adjust overlap to account for padding. 3902 * @buf_b: second buffer 3903 * @buf_a: first buffer 3904 * @len_a: size of first buffer 3905 * 3906 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap 3907 * accordingly. 3908 * 3909 * Return: A pointer into @buf_b from where non-overlapped data starts 3910 */ 3911 static unsigned char *adj_for_padding(unsigned char *buf_b, 3912 unsigned char *buf_a, size_t len_a) 3913 { 3914 unsigned char *p = buf_b - MAX_PADDING; 3915 unsigned char *q = buf_a + len_a - MAX_PADDING; 3916 int i; 3917 3918 for (i = MAX_PADDING; i; i--, p++, q++) { 3919 if (*p != *q) 3920 break; 3921 } 3922 3923 return p; 3924 } 3925 3926 /** 3927 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data 3928 * using TSC. 3929 * @buf_a: first buffer 3930 * @len_a: size of first buffer 3931 * @buf_b: second buffer 3932 * @len_b: size of second buffer 3933 * @consecutive: returns true if there is data in buf_b that is consecutive 3934 * to buf_a 3935 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 3936 * 3937 * If the trace contains TSC we can look at the last TSC of @buf_a and the 3938 * first TSC of @buf_b in order to determine if the buffers overlap, and then 3939 * walk forward in @buf_b until a later TSC is found. A precondition is that 3940 * @buf_a and @buf_b are positioned at a PSB. 3941 * 3942 * Return: A pointer into @buf_b from where non-overlapped data starts, or 3943 * @buf_b + @len_b if there is no non-overlapped data. 3944 */ 3945 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, 3946 size_t len_a, 3947 unsigned char *buf_b, 3948 size_t len_b, bool *consecutive, 3949 bool ooo_tsc) 3950 { 3951 uint64_t tsc_a, tsc_b; 3952 unsigned char *p; 3953 size_t len, rem_a, rem_b; 3954 3955 p = intel_pt_last_psb(buf_a, len_a); 3956 if (!p) 3957 return buf_b; /* No PSB in buf_a => no overlap */ 3958 3959 len = len_a - (p - buf_a); 3960 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) { 3961 /* The last PSB+ in buf_a is incomplete, so go back one more */ 3962 len_a -= len; 3963 p = intel_pt_last_psb(buf_a, len_a); 3964 if (!p) 3965 return buf_b; /* No full PSB+ => assume no overlap */ 3966 len = len_a - (p - buf_a); 3967 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) 3968 return buf_b; /* No TSC in buf_a => assume no overlap */ 3969 } 3970 3971 while (1) { 3972 /* Ignore PSB+ with no TSC */ 3973 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) { 3974 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b); 3975 3976 /* Same TSC, so buffers are consecutive */ 3977 if (!cmp && rem_b >= rem_a) { 3978 unsigned char *start; 3979 3980 *consecutive = true; 3981 start = buf_b + len_b - (rem_b - rem_a); 3982 return adj_for_padding(start, buf_a, len_a); 3983 } 3984 if (cmp < 0 && !ooo_tsc) 3985 return buf_b; /* tsc_a < tsc_b => no overlap */ 3986 } 3987 3988 if (!intel_pt_step_psb(&buf_b, &len_b)) 3989 return buf_b + len_b; /* No PSB in buf_b => no data */ 3990 } 3991 } 3992 3993 /** 3994 * intel_pt_find_overlap - determine start of non-overlapped trace data. 3995 * @buf_a: first buffer 3996 * @len_a: size of first buffer 3997 * @buf_b: second buffer 3998 * @len_b: size of second buffer 3999 * @have_tsc: can use TSC packets to detect overlap 4000 * @consecutive: returns true if there is data in buf_b that is consecutive 4001 * to buf_a 4002 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling 4003 * 4004 * When trace samples or snapshots are recorded there is the possibility that 4005 * the data overlaps. Note that, for the purposes of decoding, data is only 4006 * useful if it begins with a PSB packet. 4007 * 4008 * Return: A pointer into @buf_b from where non-overlapped data starts, or 4009 * @buf_b + @len_b if there is no non-overlapped data. 4010 */ 4011 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, 4012 unsigned char *buf_b, size_t len_b, 4013 bool have_tsc, bool *consecutive, 4014 bool ooo_tsc) 4015 { 4016 unsigned char *found; 4017 4018 /* Buffer 'b' must start at PSB so throw away everything before that */ 4019 if (!intel_pt_next_psb(&buf_b, &len_b)) 4020 return buf_b + len_b; /* No PSB */ 4021 4022 if (!intel_pt_next_psb(&buf_a, &len_a)) 4023 return buf_b; /* No overlap */ 4024 4025 if (have_tsc) { 4026 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, 4027 consecutive, ooo_tsc); 4028 if (found) 4029 return found; 4030 } 4031 4032 /* 4033 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes, 4034 * we can ignore the first part of buffer 'a'. 4035 */ 4036 while (len_b < len_a) { 4037 if (!intel_pt_step_psb(&buf_a, &len_a)) 4038 return buf_b; /* No overlap */ 4039 } 4040 4041 /* Now len_b >= len_a */ 4042 while (1) { 4043 /* Potential overlap so check the bytes */ 4044 found = memmem(buf_a, len_a, buf_b, len_a); 4045 if (found) { 4046 *consecutive = true; 4047 return adj_for_padding(buf_b + len_a, buf_a, len_a); 4048 } 4049 4050 /* Try again at next PSB in buffer 'a' */ 4051 if (!intel_pt_step_psb(&buf_a, &len_a)) 4052 return buf_b; /* No overlap */ 4053 } 4054 } 4055 4056 /** 4057 * struct fast_forward_data - data used by intel_pt_ff_cb(). 4058 * @timestamp: timestamp to fast forward towards 4059 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than 4060 * the fast forward timestamp. 4061 */ 4062 struct fast_forward_data { 4063 uint64_t timestamp; 4064 uint64_t buf_timestamp; 4065 }; 4066 4067 /** 4068 * intel_pt_ff_cb - fast forward lookahead callback. 4069 * @buffer: Intel PT trace buffer 4070 * @data: opaque pointer to fast forward data (struct fast_forward_data) 4071 * 4072 * Determine if @buffer trace is past the fast forward timestamp. 4073 * 4074 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward 4075 * timestamp, and 0 otherwise. 4076 */ 4077 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data) 4078 { 4079 struct fast_forward_data *d = data; 4080 unsigned char *buf; 4081 uint64_t tsc; 4082 size_t rem; 4083 size_t len; 4084 4085 buf = (unsigned char *)buffer->buf; 4086 len = buffer->len; 4087 4088 if (!intel_pt_next_psb(&buf, &len) || 4089 !intel_pt_next_tsc(buf, len, &tsc, &rem)) 4090 return 0; 4091 4092 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp); 4093 4094 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n", 4095 tsc, buffer->ref_timestamp); 4096 4097 /* 4098 * If the buffer contains a timestamp earlier that the fast forward 4099 * timestamp, then record it, else stop. 4100 */ 4101 if (tsc < d->timestamp) 4102 d->buf_timestamp = buffer->ref_timestamp; 4103 else 4104 return 1; 4105 4106 return 0; 4107 } 4108 4109 /** 4110 * intel_pt_fast_forward - reposition decoder forwards. 4111 * @decoder: Intel PT decoder 4112 * @timestamp: timestamp to fast forward towards 4113 * 4114 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp. 4115 * 4116 * Return: 0 on success or negative error code on failure. 4117 */ 4118 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) 4119 { 4120 struct fast_forward_data d = { .timestamp = timestamp }; 4121 unsigned char *buf; 4122 size_t len; 4123 int err; 4124 4125 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp); 4126 4127 /* Find buffer timestamp of buffer to fast forward to */ 4128 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); 4129 if (err < 0) 4130 return err; 4131 4132 /* Walk to buffer with same buffer timestamp */ 4133 if (d.buf_timestamp) { 4134 do { 4135 decoder->pos += decoder->len; 4136 decoder->len = 0; 4137 err = intel_pt_get_next_data(decoder, true); 4138 /* -ENOLINK means non-consecutive trace */ 4139 if (err && err != -ENOLINK) 4140 return err; 4141 } while (decoder->buf_timestamp != d.buf_timestamp); 4142 } 4143 4144 if (!decoder->buf) 4145 return 0; 4146 4147 buf = (unsigned char *)decoder->buf; 4148 len = decoder->len; 4149 4150 if (!intel_pt_next_psb(&buf, &len)) 4151 return 0; 4152 4153 /* 4154 * Walk PSBs while the PSB timestamp is less than the fast forward 4155 * timestamp. 4156 */ 4157 do { 4158 uint64_t tsc; 4159 size_t rem; 4160 4161 if (!intel_pt_next_tsc(buf, len, &tsc, &rem)) 4162 break; 4163 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); 4164 /* 4165 * A TSC packet can slip past MTC packets but, after fast 4166 * forward, decoding starts at the TSC timestamp. That means 4167 * the timestamps may not be exactly the same as the timestamps 4168 * that would have been decoded without fast forward. 4169 */ 4170 if (tsc < timestamp) { 4171 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc); 4172 decoder->pos += decoder->len - len; 4173 decoder->buf = buf; 4174 decoder->len = len; 4175 intel_pt_reposition(decoder); 4176 } else { 4177 break; 4178 } 4179 } while (intel_pt_step_psb(&buf, &len)); 4180 4181 return 0; 4182 } 4183