1 /* 2 * intel_pt.c: Intel Processor Trace support 3 * Copyright (c) 2013-2015, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #include <stdbool.h> 17 #include <linux/kernel.h> 18 #include <linux/types.h> 19 #include <linux/bitops.h> 20 #include <linux/log2.h> 21 #include <cpuid.h> 22 23 #include "../../perf.h" 24 #include "../../util/session.h" 25 #include "../../util/event.h" 26 #include "../../util/evlist.h" 27 #include "../../util/evsel.h" 28 #include "../../util/cpumap.h" 29 #include "../../util/parse-options.h" 30 #include "../../util/parse-events.h" 31 #include "../../util/pmu.h" 32 #include "../../util/debug.h" 33 #include "../../util/auxtrace.h" 34 #include "../../util/tsc.h" 35 #include "../../util/intel-pt.h" 36 37 #define KiB(x) ((x) * 1024) 38 #define MiB(x) ((x) * 1024 * 1024) 39 #define KiB_MASK(x) (KiB(x) - 1) 40 #define MiB_MASK(x) (MiB(x) - 1) 41 42 #define INTEL_PT_DEFAULT_SAMPLE_SIZE KiB(4) 43 44 #define INTEL_PT_MAX_SAMPLE_SIZE KiB(60) 45 46 #define INTEL_PT_PSB_PERIOD_NEAR 256 47 48 struct intel_pt_snapshot_ref { 49 void *ref_buf; 50 size_t ref_offset; 51 bool wrapped; 52 }; 53 54 struct intel_pt_recording { 55 struct auxtrace_record itr; 56 struct perf_pmu *intel_pt_pmu; 57 int have_sched_switch; 58 struct perf_evlist *evlist; 59 bool snapshot_mode; 60 bool snapshot_init_done; 61 size_t snapshot_size; 62 size_t snapshot_ref_buf_size; 63 int snapshot_ref_cnt; 64 struct intel_pt_snapshot_ref *snapshot_refs; 65 }; 66 67 static int intel_pt_parse_terms_with_default(struct list_head *formats, 68 const char *str, 69 u64 *config) 70 { 71 struct list_head *terms; 72 struct perf_event_attr attr = { .size = 0, }; 73 int err; 74 75 terms = malloc(sizeof(struct list_head)); 76 if (!terms) 77 return -ENOMEM; 78 79 INIT_LIST_HEAD(terms); 80 81 err = parse_events_terms(terms, str); 82 if (err) 83 goto out_free; 84 85 attr.config = *config; 86 err = perf_pmu__config_terms(formats, &attr, terms, true, NULL); 87 if (err) 88 goto out_free; 89 90 *config = attr.config; 91 out_free: 92 parse_events__free_terms(terms); 93 return err; 94 } 95 96 static int intel_pt_parse_terms(struct list_head *formats, const char *str, 97 u64 *config) 98 { 99 *config = 0; 100 return intel_pt_parse_terms_with_default(formats, str, config); 101 } 102 103 static u64 intel_pt_masked_bits(u64 mask, u64 bits) 104 { 105 const u64 top_bit = 1ULL << 63; 106 u64 res = 0; 107 int i; 108 109 for (i = 0; i < 64; i++) { 110 if (mask & top_bit) { 111 res <<= 1; 112 if (bits & top_bit) 113 res |= 1; 114 } 115 mask <<= 1; 116 bits <<= 1; 117 } 118 119 return res; 120 } 121 122 static int intel_pt_read_config(struct perf_pmu *intel_pt_pmu, const char *str, 123 struct perf_evlist *evlist, u64 *res) 124 { 125 struct perf_evsel *evsel; 126 u64 mask; 127 128 *res = 0; 129 130 mask = perf_pmu__format_bits(&intel_pt_pmu->format, str); 131 if (!mask) 132 return -EINVAL; 133 134 evlist__for_each(evlist, evsel) { 135 if (evsel->attr.type == intel_pt_pmu->type) { 136 *res = intel_pt_masked_bits(mask, evsel->attr.config); 137 return 0; 138 } 139 } 140 141 return -EINVAL; 142 } 143 144 static size_t intel_pt_psb_period(struct perf_pmu *intel_pt_pmu, 145 struct perf_evlist *evlist) 146 { 147 u64 val; 148 int err, topa_multiple_entries; 149 size_t psb_period; 150 151 if (perf_pmu__scan_file(intel_pt_pmu, "caps/topa_multiple_entries", 152 "%d", &topa_multiple_entries) != 1) 153 topa_multiple_entries = 0; 154 155 /* 156 * Use caps/topa_multiple_entries to indicate early hardware that had 157 * extra frequent PSBs. 158 */ 159 if (!topa_multiple_entries) { 160 psb_period = 256; 161 goto out; 162 } 163 164 err = intel_pt_read_config(intel_pt_pmu, "psb_period", evlist, &val); 165 if (err) 166 val = 0; 167 168 psb_period = 1 << (val + 11); 169 out: 170 pr_debug2("%s psb_period %zu\n", intel_pt_pmu->name, psb_period); 171 return psb_period; 172 } 173 174 static int intel_pt_pick_bit(int bits, int target) 175 { 176 int pos, pick = -1; 177 178 for (pos = 0; bits; bits >>= 1, pos++) { 179 if (bits & 1) { 180 if (pos <= target || pick < 0) 181 pick = pos; 182 if (pos >= target) 183 break; 184 } 185 } 186 187 return pick; 188 } 189 190 static u64 intel_pt_default_config(struct perf_pmu *intel_pt_pmu) 191 { 192 char buf[256]; 193 int mtc, mtc_periods = 0, mtc_period; 194 int psb_cyc, psb_periods, psb_period; 195 int pos = 0; 196 u64 config; 197 198 pos += scnprintf(buf + pos, sizeof(buf) - pos, "tsc"); 199 200 if (perf_pmu__scan_file(intel_pt_pmu, "caps/mtc", "%d", 201 &mtc) != 1) 202 mtc = 1; 203 204 if (mtc) { 205 if (perf_pmu__scan_file(intel_pt_pmu, "caps/mtc_periods", "%x", 206 &mtc_periods) != 1) 207 mtc_periods = 0; 208 if (mtc_periods) { 209 mtc_period = intel_pt_pick_bit(mtc_periods, 3); 210 pos += scnprintf(buf + pos, sizeof(buf) - pos, 211 ",mtc,mtc_period=%d", mtc_period); 212 } 213 } 214 215 if (perf_pmu__scan_file(intel_pt_pmu, "caps/psb_cyc", "%d", 216 &psb_cyc) != 1) 217 psb_cyc = 1; 218 219 if (psb_cyc && mtc_periods) { 220 if (perf_pmu__scan_file(intel_pt_pmu, "caps/psb_periods", "%x", 221 &psb_periods) != 1) 222 psb_periods = 0; 223 if (psb_periods) { 224 psb_period = intel_pt_pick_bit(psb_periods, 3); 225 pos += scnprintf(buf + pos, sizeof(buf) - pos, 226 ",psb_period=%d", psb_period); 227 } 228 } 229 230 pr_debug2("%s default config: %s\n", intel_pt_pmu->name, buf); 231 232 intel_pt_parse_terms(&intel_pt_pmu->format, buf, &config); 233 234 return config; 235 } 236 237 static int intel_pt_parse_snapshot_options(struct auxtrace_record *itr, 238 struct record_opts *opts, 239 const char *str) 240 { 241 struct intel_pt_recording *ptr = 242 container_of(itr, struct intel_pt_recording, itr); 243 unsigned long long snapshot_size = 0; 244 char *endptr; 245 246 if (str) { 247 snapshot_size = strtoull(str, &endptr, 0); 248 if (*endptr || snapshot_size > SIZE_MAX) 249 return -1; 250 } 251 252 opts->auxtrace_snapshot_mode = true; 253 opts->auxtrace_snapshot_size = snapshot_size; 254 255 ptr->snapshot_size = snapshot_size; 256 257 return 0; 258 } 259 260 struct perf_event_attr * 261 intel_pt_pmu_default_config(struct perf_pmu *intel_pt_pmu) 262 { 263 struct perf_event_attr *attr; 264 265 attr = zalloc(sizeof(struct perf_event_attr)); 266 if (!attr) 267 return NULL; 268 269 attr->config = intel_pt_default_config(intel_pt_pmu); 270 271 intel_pt_pmu->selectable = true; 272 273 return attr; 274 } 275 276 static size_t intel_pt_info_priv_size(struct auxtrace_record *itr __maybe_unused) 277 { 278 return INTEL_PT_AUXTRACE_PRIV_SIZE; 279 } 280 281 static void intel_pt_tsc_ctc_ratio(u32 *n, u32 *d) 282 { 283 unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; 284 285 __get_cpuid(0x15, &eax, &ebx, &ecx, &edx); 286 *n = ebx; 287 *d = eax; 288 } 289 290 static int intel_pt_info_fill(struct auxtrace_record *itr, 291 struct perf_session *session, 292 struct auxtrace_info_event *auxtrace_info, 293 size_t priv_size) 294 { 295 struct intel_pt_recording *ptr = 296 container_of(itr, struct intel_pt_recording, itr); 297 struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; 298 struct perf_event_mmap_page *pc; 299 struct perf_tsc_conversion tc = { .time_mult = 0, }; 300 bool cap_user_time_zero = false, per_cpu_mmaps; 301 u64 tsc_bit, mtc_bit, mtc_freq_bits, cyc_bit, noretcomp_bit; 302 u32 tsc_ctc_ratio_n, tsc_ctc_ratio_d; 303 int err; 304 305 if (priv_size != INTEL_PT_AUXTRACE_PRIV_SIZE) 306 return -EINVAL; 307 308 intel_pt_parse_terms(&intel_pt_pmu->format, "tsc", &tsc_bit); 309 intel_pt_parse_terms(&intel_pt_pmu->format, "noretcomp", 310 &noretcomp_bit); 311 intel_pt_parse_terms(&intel_pt_pmu->format, "mtc", &mtc_bit); 312 mtc_freq_bits = perf_pmu__format_bits(&intel_pt_pmu->format, 313 "mtc_period"); 314 intel_pt_parse_terms(&intel_pt_pmu->format, "cyc", &cyc_bit); 315 316 intel_pt_tsc_ctc_ratio(&tsc_ctc_ratio_n, &tsc_ctc_ratio_d); 317 318 if (!session->evlist->nr_mmaps) 319 return -EINVAL; 320 321 pc = session->evlist->mmap[0].base; 322 if (pc) { 323 err = perf_read_tsc_conversion(pc, &tc); 324 if (err) { 325 if (err != -EOPNOTSUPP) 326 return err; 327 } else { 328 cap_user_time_zero = tc.time_mult != 0; 329 } 330 if (!cap_user_time_zero) 331 ui__warning("Intel Processor Trace: TSC not available\n"); 332 } 333 334 per_cpu_mmaps = !cpu_map__empty(session->evlist->cpus); 335 336 auxtrace_info->type = PERF_AUXTRACE_INTEL_PT; 337 auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type; 338 auxtrace_info->priv[INTEL_PT_TIME_SHIFT] = tc.time_shift; 339 auxtrace_info->priv[INTEL_PT_TIME_MULT] = tc.time_mult; 340 auxtrace_info->priv[INTEL_PT_TIME_ZERO] = tc.time_zero; 341 auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO] = cap_user_time_zero; 342 auxtrace_info->priv[INTEL_PT_TSC_BIT] = tsc_bit; 343 auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT] = noretcomp_bit; 344 auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH] = ptr->have_sched_switch; 345 auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE] = ptr->snapshot_mode; 346 auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS] = per_cpu_mmaps; 347 auxtrace_info->priv[INTEL_PT_MTC_BIT] = mtc_bit; 348 auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS] = mtc_freq_bits; 349 auxtrace_info->priv[INTEL_PT_TSC_CTC_N] = tsc_ctc_ratio_n; 350 auxtrace_info->priv[INTEL_PT_TSC_CTC_D] = tsc_ctc_ratio_d; 351 auxtrace_info->priv[INTEL_PT_CYC_BIT] = cyc_bit; 352 353 return 0; 354 } 355 356 static int intel_pt_track_switches(struct perf_evlist *evlist) 357 { 358 const char *sched_switch = "sched:sched_switch"; 359 struct perf_evsel *evsel; 360 int err; 361 362 if (!perf_evlist__can_select_event(evlist, sched_switch)) 363 return -EPERM; 364 365 err = parse_events(evlist, sched_switch, NULL); 366 if (err) { 367 pr_debug2("%s: failed to parse %s, error %d\n", 368 __func__, sched_switch, err); 369 return err; 370 } 371 372 evsel = perf_evlist__last(evlist); 373 374 perf_evsel__set_sample_bit(evsel, CPU); 375 perf_evsel__set_sample_bit(evsel, TIME); 376 377 evsel->system_wide = true; 378 evsel->no_aux_samples = true; 379 evsel->immediate = true; 380 381 return 0; 382 } 383 384 static void intel_pt_valid_str(char *str, size_t len, u64 valid) 385 { 386 unsigned int val, last = 0, state = 1; 387 int p = 0; 388 389 str[0] = '\0'; 390 391 for (val = 0; val <= 64; val++, valid >>= 1) { 392 if (valid & 1) { 393 last = val; 394 switch (state) { 395 case 0: 396 p += scnprintf(str + p, len - p, ","); 397 /* Fall through */ 398 case 1: 399 p += scnprintf(str + p, len - p, "%u", val); 400 state = 2; 401 break; 402 case 2: 403 state = 3; 404 break; 405 case 3: 406 state = 4; 407 break; 408 default: 409 break; 410 } 411 } else { 412 switch (state) { 413 case 3: 414 p += scnprintf(str + p, len - p, ",%u", last); 415 state = 0; 416 break; 417 case 4: 418 p += scnprintf(str + p, len - p, "-%u", last); 419 state = 0; 420 break; 421 default: 422 break; 423 } 424 if (state != 1) 425 state = 0; 426 } 427 } 428 } 429 430 static int intel_pt_val_config_term(struct perf_pmu *intel_pt_pmu, 431 const char *caps, const char *name, 432 const char *supported, u64 config) 433 { 434 char valid_str[256]; 435 unsigned int shift; 436 unsigned long long valid; 437 u64 bits; 438 int ok; 439 440 if (perf_pmu__scan_file(intel_pt_pmu, caps, "%llx", &valid) != 1) 441 valid = 0; 442 443 if (supported && 444 perf_pmu__scan_file(intel_pt_pmu, supported, "%d", &ok) == 1 && !ok) 445 valid = 0; 446 447 valid |= 1; 448 449 bits = perf_pmu__format_bits(&intel_pt_pmu->format, name); 450 451 config &= bits; 452 453 for (shift = 0; bits && !(bits & 1); shift++) 454 bits >>= 1; 455 456 config >>= shift; 457 458 if (config > 63) 459 goto out_err; 460 461 if (valid & (1 << config)) 462 return 0; 463 out_err: 464 intel_pt_valid_str(valid_str, sizeof(valid_str), valid); 465 pr_err("Invalid %s for %s. Valid values are: %s\n", 466 name, INTEL_PT_PMU_NAME, valid_str); 467 return -EINVAL; 468 } 469 470 static int intel_pt_validate_config(struct perf_pmu *intel_pt_pmu, 471 struct perf_evsel *evsel) 472 { 473 int err; 474 475 if (!evsel) 476 return 0; 477 478 err = intel_pt_val_config_term(intel_pt_pmu, "caps/cycle_thresholds", 479 "cyc_thresh", "caps/psb_cyc", 480 evsel->attr.config); 481 if (err) 482 return err; 483 484 err = intel_pt_val_config_term(intel_pt_pmu, "caps/mtc_periods", 485 "mtc_period", "caps/mtc", 486 evsel->attr.config); 487 if (err) 488 return err; 489 490 return intel_pt_val_config_term(intel_pt_pmu, "caps/psb_periods", 491 "psb_period", "caps/psb_cyc", 492 evsel->attr.config); 493 } 494 495 static int intel_pt_recording_options(struct auxtrace_record *itr, 496 struct perf_evlist *evlist, 497 struct record_opts *opts) 498 { 499 struct intel_pt_recording *ptr = 500 container_of(itr, struct intel_pt_recording, itr); 501 struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; 502 bool have_timing_info; 503 struct perf_evsel *evsel, *intel_pt_evsel = NULL; 504 const struct cpu_map *cpus = evlist->cpus; 505 bool privileged = geteuid() == 0 || perf_event_paranoid() < 0; 506 u64 tsc_bit; 507 int err; 508 509 ptr->evlist = evlist; 510 ptr->snapshot_mode = opts->auxtrace_snapshot_mode; 511 512 evlist__for_each(evlist, evsel) { 513 if (evsel->attr.type == intel_pt_pmu->type) { 514 if (intel_pt_evsel) { 515 pr_err("There may be only one " INTEL_PT_PMU_NAME " event\n"); 516 return -EINVAL; 517 } 518 evsel->attr.freq = 0; 519 evsel->attr.sample_period = 1; 520 intel_pt_evsel = evsel; 521 opts->full_auxtrace = true; 522 } 523 } 524 525 if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) { 526 pr_err("Snapshot mode (-S option) requires " INTEL_PT_PMU_NAME " PMU event (-e " INTEL_PT_PMU_NAME ")\n"); 527 return -EINVAL; 528 } 529 530 if (opts->use_clockid) { 531 pr_err("Cannot use clockid (-k option) with " INTEL_PT_PMU_NAME "\n"); 532 return -EINVAL; 533 } 534 535 if (!opts->full_auxtrace) 536 return 0; 537 538 err = intel_pt_validate_config(intel_pt_pmu, intel_pt_evsel); 539 if (err) 540 return err; 541 542 /* Set default sizes for snapshot mode */ 543 if (opts->auxtrace_snapshot_mode) { 544 size_t psb_period = intel_pt_psb_period(intel_pt_pmu, evlist); 545 546 if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) { 547 if (privileged) { 548 opts->auxtrace_mmap_pages = MiB(4) / page_size; 549 } else { 550 opts->auxtrace_mmap_pages = KiB(128) / page_size; 551 if (opts->mmap_pages == UINT_MAX) 552 opts->mmap_pages = KiB(256) / page_size; 553 } 554 } else if (!opts->auxtrace_mmap_pages && !privileged && 555 opts->mmap_pages == UINT_MAX) { 556 opts->mmap_pages = KiB(256) / page_size; 557 } 558 if (!opts->auxtrace_snapshot_size) 559 opts->auxtrace_snapshot_size = 560 opts->auxtrace_mmap_pages * (size_t)page_size; 561 if (!opts->auxtrace_mmap_pages) { 562 size_t sz = opts->auxtrace_snapshot_size; 563 564 sz = round_up(sz, page_size) / page_size; 565 opts->auxtrace_mmap_pages = roundup_pow_of_two(sz); 566 } 567 if (opts->auxtrace_snapshot_size > 568 opts->auxtrace_mmap_pages * (size_t)page_size) { 569 pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n", 570 opts->auxtrace_snapshot_size, 571 opts->auxtrace_mmap_pages * (size_t)page_size); 572 return -EINVAL; 573 } 574 if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) { 575 pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n"); 576 return -EINVAL; 577 } 578 pr_debug2("Intel PT snapshot size: %zu\n", 579 opts->auxtrace_snapshot_size); 580 if (psb_period && 581 opts->auxtrace_snapshot_size <= psb_period + 582 INTEL_PT_PSB_PERIOD_NEAR) 583 ui__warning("Intel PT snapshot size (%zu) may be too small for PSB period (%zu)\n", 584 opts->auxtrace_snapshot_size, psb_period); 585 } 586 587 /* Set default sizes for full trace mode */ 588 if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) { 589 if (privileged) { 590 opts->auxtrace_mmap_pages = MiB(4) / page_size; 591 } else { 592 opts->auxtrace_mmap_pages = KiB(128) / page_size; 593 if (opts->mmap_pages == UINT_MAX) 594 opts->mmap_pages = KiB(256) / page_size; 595 } 596 } 597 598 /* Validate auxtrace_mmap_pages */ 599 if (opts->auxtrace_mmap_pages) { 600 size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size; 601 size_t min_sz; 602 603 if (opts->auxtrace_snapshot_mode) 604 min_sz = KiB(4); 605 else 606 min_sz = KiB(8); 607 608 if (sz < min_sz || !is_power_of_2(sz)) { 609 pr_err("Invalid mmap size for Intel Processor Trace: must be at least %zuKiB and a power of 2\n", 610 min_sz / 1024); 611 return -EINVAL; 612 } 613 } 614 615 intel_pt_parse_terms(&intel_pt_pmu->format, "tsc", &tsc_bit); 616 617 if (opts->full_auxtrace && (intel_pt_evsel->attr.config & tsc_bit)) 618 have_timing_info = true; 619 else 620 have_timing_info = false; 621 622 /* 623 * Per-cpu recording needs sched_switch events to distinguish different 624 * threads. 625 */ 626 if (have_timing_info && !cpu_map__empty(cpus)) { 627 err = intel_pt_track_switches(evlist); 628 if (err == -EPERM) 629 pr_debug2("Unable to select sched:sched_switch\n"); 630 else if (err) 631 return err; 632 else 633 ptr->have_sched_switch = 1; 634 } 635 636 if (intel_pt_evsel) { 637 /* 638 * To obtain the auxtrace buffer file descriptor, the auxtrace 639 * event must come first. 640 */ 641 perf_evlist__to_front(evlist, intel_pt_evsel); 642 /* 643 * In the case of per-cpu mmaps, we need the CPU on the 644 * AUX event. 645 */ 646 if (!cpu_map__empty(cpus)) 647 perf_evsel__set_sample_bit(intel_pt_evsel, CPU); 648 } 649 650 /* Add dummy event to keep tracking */ 651 if (opts->full_auxtrace) { 652 struct perf_evsel *tracking_evsel; 653 654 err = parse_events(evlist, "dummy:u", NULL); 655 if (err) 656 return err; 657 658 tracking_evsel = perf_evlist__last(evlist); 659 660 perf_evlist__set_tracking_event(evlist, tracking_evsel); 661 662 tracking_evsel->attr.freq = 0; 663 tracking_evsel->attr.sample_period = 1; 664 665 /* In per-cpu case, always need the time of mmap events etc */ 666 if (!cpu_map__empty(cpus)) 667 perf_evsel__set_sample_bit(tracking_evsel, TIME); 668 } 669 670 /* 671 * Warn the user when we do not have enough information to decode i.e. 672 * per-cpu with no sched_switch (except workload-only). 673 */ 674 if (!ptr->have_sched_switch && !cpu_map__empty(cpus) && 675 !target__none(&opts->target)) 676 ui__warning("Intel Processor Trace decoding will not be possible except for kernel tracing!\n"); 677 678 return 0; 679 } 680 681 static int intel_pt_snapshot_start(struct auxtrace_record *itr) 682 { 683 struct intel_pt_recording *ptr = 684 container_of(itr, struct intel_pt_recording, itr); 685 struct perf_evsel *evsel; 686 687 evlist__for_each(ptr->evlist, evsel) { 688 if (evsel->attr.type == ptr->intel_pt_pmu->type) 689 return perf_evlist__disable_event(ptr->evlist, evsel); 690 } 691 return -EINVAL; 692 } 693 694 static int intel_pt_snapshot_finish(struct auxtrace_record *itr) 695 { 696 struct intel_pt_recording *ptr = 697 container_of(itr, struct intel_pt_recording, itr); 698 struct perf_evsel *evsel; 699 700 evlist__for_each(ptr->evlist, evsel) { 701 if (evsel->attr.type == ptr->intel_pt_pmu->type) 702 return perf_evlist__enable_event(ptr->evlist, evsel); 703 } 704 return -EINVAL; 705 } 706 707 static int intel_pt_alloc_snapshot_refs(struct intel_pt_recording *ptr, int idx) 708 { 709 const size_t sz = sizeof(struct intel_pt_snapshot_ref); 710 int cnt = ptr->snapshot_ref_cnt, new_cnt = cnt * 2; 711 struct intel_pt_snapshot_ref *refs; 712 713 if (!new_cnt) 714 new_cnt = 16; 715 716 while (new_cnt <= idx) 717 new_cnt *= 2; 718 719 refs = calloc(new_cnt, sz); 720 if (!refs) 721 return -ENOMEM; 722 723 memcpy(refs, ptr->snapshot_refs, cnt * sz); 724 725 ptr->snapshot_refs = refs; 726 ptr->snapshot_ref_cnt = new_cnt; 727 728 return 0; 729 } 730 731 static void intel_pt_free_snapshot_refs(struct intel_pt_recording *ptr) 732 { 733 int i; 734 735 for (i = 0; i < ptr->snapshot_ref_cnt; i++) 736 zfree(&ptr->snapshot_refs[i].ref_buf); 737 zfree(&ptr->snapshot_refs); 738 } 739 740 static void intel_pt_recording_free(struct auxtrace_record *itr) 741 { 742 struct intel_pt_recording *ptr = 743 container_of(itr, struct intel_pt_recording, itr); 744 745 intel_pt_free_snapshot_refs(ptr); 746 free(ptr); 747 } 748 749 static int intel_pt_alloc_snapshot_ref(struct intel_pt_recording *ptr, int idx, 750 size_t snapshot_buf_size) 751 { 752 size_t ref_buf_size = ptr->snapshot_ref_buf_size; 753 void *ref_buf; 754 755 ref_buf = zalloc(ref_buf_size); 756 if (!ref_buf) 757 return -ENOMEM; 758 759 ptr->snapshot_refs[idx].ref_buf = ref_buf; 760 ptr->snapshot_refs[idx].ref_offset = snapshot_buf_size - ref_buf_size; 761 762 return 0; 763 } 764 765 static size_t intel_pt_snapshot_ref_buf_size(struct intel_pt_recording *ptr, 766 size_t snapshot_buf_size) 767 { 768 const size_t max_size = 256 * 1024; 769 size_t buf_size = 0, psb_period; 770 771 if (ptr->snapshot_size <= 64 * 1024) 772 return 0; 773 774 psb_period = intel_pt_psb_period(ptr->intel_pt_pmu, ptr->evlist); 775 if (psb_period) 776 buf_size = psb_period * 2; 777 778 if (!buf_size || buf_size > max_size) 779 buf_size = max_size; 780 781 if (buf_size >= snapshot_buf_size) 782 return 0; 783 784 if (buf_size >= ptr->snapshot_size / 2) 785 return 0; 786 787 return buf_size; 788 } 789 790 static int intel_pt_snapshot_init(struct intel_pt_recording *ptr, 791 size_t snapshot_buf_size) 792 { 793 if (ptr->snapshot_init_done) 794 return 0; 795 796 ptr->snapshot_init_done = true; 797 798 ptr->snapshot_ref_buf_size = intel_pt_snapshot_ref_buf_size(ptr, 799 snapshot_buf_size); 800 801 return 0; 802 } 803 804 /** 805 * intel_pt_compare_buffers - compare bytes in a buffer to a circular buffer. 806 * @buf1: first buffer 807 * @compare_size: number of bytes to compare 808 * @buf2: second buffer (a circular buffer) 809 * @offs2: offset in second buffer 810 * @buf2_size: size of second buffer 811 * 812 * The comparison allows for the possibility that the bytes to compare in the 813 * circular buffer are not contiguous. It is assumed that @compare_size <= 814 * @buf2_size. This function returns %false if the bytes are identical, %true 815 * otherwise. 816 */ 817 static bool intel_pt_compare_buffers(void *buf1, size_t compare_size, 818 void *buf2, size_t offs2, size_t buf2_size) 819 { 820 size_t end2 = offs2 + compare_size, part_size; 821 822 if (end2 <= buf2_size) 823 return memcmp(buf1, buf2 + offs2, compare_size); 824 825 part_size = end2 - buf2_size; 826 if (memcmp(buf1, buf2 + offs2, part_size)) 827 return true; 828 829 compare_size -= part_size; 830 831 return memcmp(buf1 + part_size, buf2, compare_size); 832 } 833 834 static bool intel_pt_compare_ref(void *ref_buf, size_t ref_offset, 835 size_t ref_size, size_t buf_size, 836 void *data, size_t head) 837 { 838 size_t ref_end = ref_offset + ref_size; 839 840 if (ref_end > buf_size) { 841 if (head > ref_offset || head < ref_end - buf_size) 842 return true; 843 } else if (head > ref_offset && head < ref_end) { 844 return true; 845 } 846 847 return intel_pt_compare_buffers(ref_buf, ref_size, data, ref_offset, 848 buf_size); 849 } 850 851 static void intel_pt_copy_ref(void *ref_buf, size_t ref_size, size_t buf_size, 852 void *data, size_t head) 853 { 854 if (head >= ref_size) { 855 memcpy(ref_buf, data + head - ref_size, ref_size); 856 } else { 857 memcpy(ref_buf, data, head); 858 ref_size -= head; 859 memcpy(ref_buf + head, data + buf_size - ref_size, ref_size); 860 } 861 } 862 863 static bool intel_pt_wrapped(struct intel_pt_recording *ptr, int idx, 864 struct auxtrace_mmap *mm, unsigned char *data, 865 u64 head) 866 { 867 struct intel_pt_snapshot_ref *ref = &ptr->snapshot_refs[idx]; 868 bool wrapped; 869 870 wrapped = intel_pt_compare_ref(ref->ref_buf, ref->ref_offset, 871 ptr->snapshot_ref_buf_size, mm->len, 872 data, head); 873 874 intel_pt_copy_ref(ref->ref_buf, ptr->snapshot_ref_buf_size, mm->len, 875 data, head); 876 877 return wrapped; 878 } 879 880 static bool intel_pt_first_wrap(u64 *data, size_t buf_size) 881 { 882 int i, a, b; 883 884 b = buf_size >> 3; 885 a = b - 512; 886 if (a < 0) 887 a = 0; 888 889 for (i = a; i < b; i++) { 890 if (data[i]) 891 return true; 892 } 893 894 return false; 895 } 896 897 static int intel_pt_find_snapshot(struct auxtrace_record *itr, int idx, 898 struct auxtrace_mmap *mm, unsigned char *data, 899 u64 *head, u64 *old) 900 { 901 struct intel_pt_recording *ptr = 902 container_of(itr, struct intel_pt_recording, itr); 903 bool wrapped; 904 int err; 905 906 pr_debug3("%s: mmap index %d old head %zu new head %zu\n", 907 __func__, idx, (size_t)*old, (size_t)*head); 908 909 err = intel_pt_snapshot_init(ptr, mm->len); 910 if (err) 911 goto out_err; 912 913 if (idx >= ptr->snapshot_ref_cnt) { 914 err = intel_pt_alloc_snapshot_refs(ptr, idx); 915 if (err) 916 goto out_err; 917 } 918 919 if (ptr->snapshot_ref_buf_size) { 920 if (!ptr->snapshot_refs[idx].ref_buf) { 921 err = intel_pt_alloc_snapshot_ref(ptr, idx, mm->len); 922 if (err) 923 goto out_err; 924 } 925 wrapped = intel_pt_wrapped(ptr, idx, mm, data, *head); 926 } else { 927 wrapped = ptr->snapshot_refs[idx].wrapped; 928 if (!wrapped && intel_pt_first_wrap((u64 *)data, mm->len)) { 929 ptr->snapshot_refs[idx].wrapped = true; 930 wrapped = true; 931 } 932 } 933 934 /* 935 * In full trace mode 'head' continually increases. However in snapshot 936 * mode 'head' is an offset within the buffer. Here 'old' and 'head' 937 * are adjusted to match the full trace case which expects that 'old' is 938 * always less than 'head'. 939 */ 940 if (wrapped) { 941 *old = *head; 942 *head += mm->len; 943 } else { 944 if (mm->mask) 945 *old &= mm->mask; 946 else 947 *old %= mm->len; 948 if (*old > *head) 949 *head += mm->len; 950 } 951 952 pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n", 953 __func__, wrapped ? "" : "not ", (size_t)*old, (size_t)*head); 954 955 return 0; 956 957 out_err: 958 pr_err("%s: failed, error %d\n", __func__, err); 959 return err; 960 } 961 962 static u64 intel_pt_reference(struct auxtrace_record *itr __maybe_unused) 963 { 964 return rdtsc(); 965 } 966 967 static int intel_pt_read_finish(struct auxtrace_record *itr, int idx) 968 { 969 struct intel_pt_recording *ptr = 970 container_of(itr, struct intel_pt_recording, itr); 971 struct perf_evsel *evsel; 972 973 evlist__for_each(ptr->evlist, evsel) { 974 if (evsel->attr.type == ptr->intel_pt_pmu->type) 975 return perf_evlist__enable_event_idx(ptr->evlist, evsel, 976 idx); 977 } 978 return -EINVAL; 979 } 980 981 struct auxtrace_record *intel_pt_recording_init(int *err) 982 { 983 struct perf_pmu *intel_pt_pmu = perf_pmu__find(INTEL_PT_PMU_NAME); 984 struct intel_pt_recording *ptr; 985 986 if (!intel_pt_pmu) 987 return NULL; 988 989 ptr = zalloc(sizeof(struct intel_pt_recording)); 990 if (!ptr) { 991 *err = -ENOMEM; 992 return NULL; 993 } 994 995 ptr->intel_pt_pmu = intel_pt_pmu; 996 ptr->itr.recording_options = intel_pt_recording_options; 997 ptr->itr.info_priv_size = intel_pt_info_priv_size; 998 ptr->itr.info_fill = intel_pt_info_fill; 999 ptr->itr.free = intel_pt_recording_free; 1000 ptr->itr.snapshot_start = intel_pt_snapshot_start; 1001 ptr->itr.snapshot_finish = intel_pt_snapshot_finish; 1002 ptr->itr.find_snapshot = intel_pt_find_snapshot; 1003 ptr->itr.parse_snapshot_options = intel_pt_parse_snapshot_options; 1004 ptr->itr.reference = intel_pt_reference; 1005 ptr->itr.read_finish = intel_pt_read_finish; 1006 return &ptr->itr; 1007 } 1008