1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched/mm.h> 24 #include <linux/sched/clock.h> 25 #include <linux/uaccess.h> 26 #include <linux/slab.h> 27 #include <linux/cpu.h> 28 #include <linux/bitops.h> 29 #include <linux/device.h> 30 31 #include <asm/apic.h> 32 #include <asm/stacktrace.h> 33 #include <asm/nmi.h> 34 #include <asm/smp.h> 35 #include <asm/alternative.h> 36 #include <asm/mmu_context.h> 37 #include <asm/tlbflush.h> 38 #include <asm/timer.h> 39 #include <asm/desc.h> 40 #include <asm/ldt.h> 41 #include <asm/unwind.h> 42 43 #include "perf_event.h" 44 45 struct x86_pmu x86_pmu __read_mostly; 46 47 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 48 .enabled = 1, 49 }; 50 51 struct static_key rdpmc_always_available = STATIC_KEY_INIT_FALSE; 52 53 u64 __read_mostly hw_cache_event_ids 54 [PERF_COUNT_HW_CACHE_MAX] 55 [PERF_COUNT_HW_CACHE_OP_MAX] 56 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 57 u64 __read_mostly hw_cache_extra_regs 58 [PERF_COUNT_HW_CACHE_MAX] 59 [PERF_COUNT_HW_CACHE_OP_MAX] 60 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 61 62 /* 63 * Propagate event elapsed time into the generic event. 64 * Can only be executed on the CPU where the event is active. 65 * Returns the delta events processed. 66 */ 67 u64 x86_perf_event_update(struct perf_event *event) 68 { 69 struct hw_perf_event *hwc = &event->hw; 70 int shift = 64 - x86_pmu.cntval_bits; 71 u64 prev_raw_count, new_raw_count; 72 int idx = hwc->idx; 73 u64 delta; 74 75 if (idx == INTEL_PMC_IDX_FIXED_BTS) 76 return 0; 77 78 /* 79 * Careful: an NMI might modify the previous event value. 80 * 81 * Our tactic to handle this is to first atomically read and 82 * exchange a new raw count - then add that new-prev delta 83 * count to the generic event atomically: 84 */ 85 again: 86 prev_raw_count = local64_read(&hwc->prev_count); 87 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 88 89 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 90 new_raw_count) != prev_raw_count) 91 goto again; 92 93 /* 94 * Now we have the new raw value and have updated the prev 95 * timestamp already. We can now calculate the elapsed delta 96 * (event-)time and add that to the generic event. 97 * 98 * Careful, not all hw sign-extends above the physical width 99 * of the count. 100 */ 101 delta = (new_raw_count << shift) - (prev_raw_count << shift); 102 delta >>= shift; 103 104 local64_add(delta, &event->count); 105 local64_sub(delta, &hwc->period_left); 106 107 return new_raw_count; 108 } 109 110 /* 111 * Find and validate any extra registers to set up. 112 */ 113 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 114 { 115 struct hw_perf_event_extra *reg; 116 struct extra_reg *er; 117 118 reg = &event->hw.extra_reg; 119 120 if (!x86_pmu.extra_regs) 121 return 0; 122 123 for (er = x86_pmu.extra_regs; er->msr; er++) { 124 if (er->event != (config & er->config_mask)) 125 continue; 126 if (event->attr.config1 & ~er->valid_mask) 127 return -EINVAL; 128 /* Check if the extra msrs can be safely accessed*/ 129 if (!er->extra_msr_access) 130 return -ENXIO; 131 132 reg->idx = er->idx; 133 reg->config = event->attr.config1; 134 reg->reg = er->msr; 135 break; 136 } 137 return 0; 138 } 139 140 static atomic_t active_events; 141 static atomic_t pmc_refcount; 142 static DEFINE_MUTEX(pmc_reserve_mutex); 143 144 #ifdef CONFIG_X86_LOCAL_APIC 145 146 static bool reserve_pmc_hardware(void) 147 { 148 int i; 149 150 for (i = 0; i < x86_pmu.num_counters; i++) { 151 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 152 goto perfctr_fail; 153 } 154 155 for (i = 0; i < x86_pmu.num_counters; i++) { 156 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 157 goto eventsel_fail; 158 } 159 160 return true; 161 162 eventsel_fail: 163 for (i--; i >= 0; i--) 164 release_evntsel_nmi(x86_pmu_config_addr(i)); 165 166 i = x86_pmu.num_counters; 167 168 perfctr_fail: 169 for (i--; i >= 0; i--) 170 release_perfctr_nmi(x86_pmu_event_addr(i)); 171 172 return false; 173 } 174 175 static void release_pmc_hardware(void) 176 { 177 int i; 178 179 for (i = 0; i < x86_pmu.num_counters; i++) { 180 release_perfctr_nmi(x86_pmu_event_addr(i)); 181 release_evntsel_nmi(x86_pmu_config_addr(i)); 182 } 183 } 184 185 #else 186 187 static bool reserve_pmc_hardware(void) { return true; } 188 static void release_pmc_hardware(void) {} 189 190 #endif 191 192 static bool check_hw_exists(void) 193 { 194 u64 val, val_fail = -1, val_new= ~0; 195 int i, reg, reg_fail = -1, ret = 0; 196 int bios_fail = 0; 197 int reg_safe = -1; 198 199 /* 200 * Check to see if the BIOS enabled any of the counters, if so 201 * complain and bail. 202 */ 203 for (i = 0; i < x86_pmu.num_counters; i++) { 204 reg = x86_pmu_config_addr(i); 205 ret = rdmsrl_safe(reg, &val); 206 if (ret) 207 goto msr_fail; 208 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 209 bios_fail = 1; 210 val_fail = val; 211 reg_fail = reg; 212 } else { 213 reg_safe = i; 214 } 215 } 216 217 if (x86_pmu.num_counters_fixed) { 218 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 219 ret = rdmsrl_safe(reg, &val); 220 if (ret) 221 goto msr_fail; 222 for (i = 0; i < x86_pmu.num_counters_fixed; i++) { 223 if (val & (0x03 << i*4)) { 224 bios_fail = 1; 225 val_fail = val; 226 reg_fail = reg; 227 } 228 } 229 } 230 231 /* 232 * If all the counters are enabled, the below test will always 233 * fail. The tools will also become useless in this scenario. 234 * Just fail and disable the hardware counters. 235 */ 236 237 if (reg_safe == -1) { 238 reg = reg_safe; 239 goto msr_fail; 240 } 241 242 /* 243 * Read the current value, change it and read it back to see if it 244 * matches, this is needed to detect certain hardware emulators 245 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 246 */ 247 reg = x86_pmu_event_addr(reg_safe); 248 if (rdmsrl_safe(reg, &val)) 249 goto msr_fail; 250 val ^= 0xffffUL; 251 ret = wrmsrl_safe(reg, val); 252 ret |= rdmsrl_safe(reg, &val_new); 253 if (ret || val != val_new) 254 goto msr_fail; 255 256 /* 257 * We still allow the PMU driver to operate: 258 */ 259 if (bios_fail) { 260 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 261 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 262 reg_fail, val_fail); 263 } 264 265 return true; 266 267 msr_fail: 268 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 269 pr_cont("PMU not available due to virtualization, using software events only.\n"); 270 } else { 271 pr_cont("Broken PMU hardware detected, using software events only.\n"); 272 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 273 reg, val_new); 274 } 275 276 return false; 277 } 278 279 static void hw_perf_event_destroy(struct perf_event *event) 280 { 281 x86_release_hardware(); 282 atomic_dec(&active_events); 283 } 284 285 void hw_perf_lbr_event_destroy(struct perf_event *event) 286 { 287 hw_perf_event_destroy(event); 288 289 /* undo the lbr/bts event accounting */ 290 x86_del_exclusive(x86_lbr_exclusive_lbr); 291 } 292 293 static inline int x86_pmu_initialized(void) 294 { 295 return x86_pmu.handle_irq != NULL; 296 } 297 298 static inline int 299 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 300 { 301 struct perf_event_attr *attr = &event->attr; 302 unsigned int cache_type, cache_op, cache_result; 303 u64 config, val; 304 305 config = attr->config; 306 307 cache_type = (config >> 0) & 0xff; 308 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 309 return -EINVAL; 310 311 cache_op = (config >> 8) & 0xff; 312 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 313 return -EINVAL; 314 315 cache_result = (config >> 16) & 0xff; 316 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 317 return -EINVAL; 318 319 val = hw_cache_event_ids[cache_type][cache_op][cache_result]; 320 321 if (val == 0) 322 return -ENOENT; 323 324 if (val == -1) 325 return -EINVAL; 326 327 hwc->config |= val; 328 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result]; 329 return x86_pmu_extra_regs(val, event); 330 } 331 332 int x86_reserve_hardware(void) 333 { 334 int err = 0; 335 336 if (!atomic_inc_not_zero(&pmc_refcount)) { 337 mutex_lock(&pmc_reserve_mutex); 338 if (atomic_read(&pmc_refcount) == 0) { 339 if (!reserve_pmc_hardware()) 340 err = -EBUSY; 341 else 342 reserve_ds_buffers(); 343 } 344 if (!err) 345 atomic_inc(&pmc_refcount); 346 mutex_unlock(&pmc_reserve_mutex); 347 } 348 349 return err; 350 } 351 352 void x86_release_hardware(void) 353 { 354 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 355 release_pmc_hardware(); 356 release_ds_buffers(); 357 mutex_unlock(&pmc_reserve_mutex); 358 } 359 } 360 361 /* 362 * Check if we can create event of a certain type (that no conflicting events 363 * are present). 364 */ 365 int x86_add_exclusive(unsigned int what) 366 { 367 int i; 368 369 /* 370 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 371 * LBR and BTS are still mutually exclusive. 372 */ 373 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 374 return 0; 375 376 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 377 mutex_lock(&pmc_reserve_mutex); 378 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 379 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 380 goto fail_unlock; 381 } 382 atomic_inc(&x86_pmu.lbr_exclusive[what]); 383 mutex_unlock(&pmc_reserve_mutex); 384 } 385 386 atomic_inc(&active_events); 387 return 0; 388 389 fail_unlock: 390 mutex_unlock(&pmc_reserve_mutex); 391 return -EBUSY; 392 } 393 394 void x86_del_exclusive(unsigned int what) 395 { 396 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 397 return; 398 399 atomic_dec(&x86_pmu.lbr_exclusive[what]); 400 atomic_dec(&active_events); 401 } 402 403 int x86_setup_perfctr(struct perf_event *event) 404 { 405 struct perf_event_attr *attr = &event->attr; 406 struct hw_perf_event *hwc = &event->hw; 407 u64 config; 408 409 if (!is_sampling_event(event)) { 410 hwc->sample_period = x86_pmu.max_period; 411 hwc->last_period = hwc->sample_period; 412 local64_set(&hwc->period_left, hwc->sample_period); 413 } 414 415 if (attr->type == PERF_TYPE_RAW) 416 return x86_pmu_extra_regs(event->attr.config, event); 417 418 if (attr->type == PERF_TYPE_HW_CACHE) 419 return set_ext_hw_attr(hwc, event); 420 421 if (attr->config >= x86_pmu.max_events) 422 return -EINVAL; 423 424 /* 425 * The generic map: 426 */ 427 config = x86_pmu.event_map(attr->config); 428 429 if (config == 0) 430 return -ENOENT; 431 432 if (config == -1LL) 433 return -EINVAL; 434 435 /* 436 * Branch tracing: 437 */ 438 if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS && 439 !attr->freq && hwc->sample_period == 1) { 440 /* BTS is not supported by this architecture. */ 441 if (!x86_pmu.bts_active) 442 return -EOPNOTSUPP; 443 444 /* BTS is currently only allowed for user-mode. */ 445 if (!attr->exclude_kernel) 446 return -EOPNOTSUPP; 447 448 /* disallow bts if conflicting events are present */ 449 if (x86_add_exclusive(x86_lbr_exclusive_lbr)) 450 return -EBUSY; 451 452 event->destroy = hw_perf_lbr_event_destroy; 453 } 454 455 hwc->config |= config; 456 457 return 0; 458 } 459 460 /* 461 * check that branch_sample_type is compatible with 462 * settings needed for precise_ip > 1 which implies 463 * using the LBR to capture ALL taken branches at the 464 * priv levels of the measurement 465 */ 466 static inline int precise_br_compat(struct perf_event *event) 467 { 468 u64 m = event->attr.branch_sample_type; 469 u64 b = 0; 470 471 /* must capture all branches */ 472 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 473 return 0; 474 475 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 476 477 if (!event->attr.exclude_user) 478 b |= PERF_SAMPLE_BRANCH_USER; 479 480 if (!event->attr.exclude_kernel) 481 b |= PERF_SAMPLE_BRANCH_KERNEL; 482 483 /* 484 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 485 */ 486 487 return m == b; 488 } 489 490 int x86_pmu_max_precise(void) 491 { 492 int precise = 0; 493 494 /* Support for constant skid */ 495 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 496 precise++; 497 498 /* Support for IP fixup */ 499 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 500 precise++; 501 502 if (x86_pmu.pebs_prec_dist) 503 precise++; 504 } 505 return precise; 506 } 507 508 int x86_pmu_hw_config(struct perf_event *event) 509 { 510 if (event->attr.precise_ip) { 511 int precise = x86_pmu_max_precise(); 512 513 if (event->attr.precise_ip > precise) 514 return -EOPNOTSUPP; 515 516 /* There's no sense in having PEBS for non sampling events: */ 517 if (!is_sampling_event(event)) 518 return -EINVAL; 519 } 520 /* 521 * check that PEBS LBR correction does not conflict with 522 * whatever the user is asking with attr->branch_sample_type 523 */ 524 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 525 u64 *br_type = &event->attr.branch_sample_type; 526 527 if (has_branch_stack(event)) { 528 if (!precise_br_compat(event)) 529 return -EOPNOTSUPP; 530 531 /* branch_sample_type is compatible */ 532 533 } else { 534 /* 535 * user did not specify branch_sample_type 536 * 537 * For PEBS fixups, we capture all 538 * the branches at the priv level of the 539 * event. 540 */ 541 *br_type = PERF_SAMPLE_BRANCH_ANY; 542 543 if (!event->attr.exclude_user) 544 *br_type |= PERF_SAMPLE_BRANCH_USER; 545 546 if (!event->attr.exclude_kernel) 547 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 548 } 549 } 550 551 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK) 552 event->attach_state |= PERF_ATTACH_TASK_DATA; 553 554 /* 555 * Generate PMC IRQs: 556 * (keep 'enabled' bit clear for now) 557 */ 558 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 559 560 /* 561 * Count user and OS events unless requested not to 562 */ 563 if (!event->attr.exclude_user) 564 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 565 if (!event->attr.exclude_kernel) 566 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 567 568 if (event->attr.type == PERF_TYPE_RAW) 569 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; 570 571 if (event->attr.sample_period && x86_pmu.limit_period) { 572 if (x86_pmu.limit_period(event, event->attr.sample_period) > 573 event->attr.sample_period) 574 return -EINVAL; 575 } 576 577 return x86_setup_perfctr(event); 578 } 579 580 /* 581 * Setup the hardware configuration for a given attr_type 582 */ 583 static int __x86_pmu_event_init(struct perf_event *event) 584 { 585 int err; 586 587 if (!x86_pmu_initialized()) 588 return -ENODEV; 589 590 err = x86_reserve_hardware(); 591 if (err) 592 return err; 593 594 atomic_inc(&active_events); 595 event->destroy = hw_perf_event_destroy; 596 597 event->hw.idx = -1; 598 event->hw.last_cpu = -1; 599 event->hw.last_tag = ~0ULL; 600 601 /* mark unused */ 602 event->hw.extra_reg.idx = EXTRA_REG_NONE; 603 event->hw.branch_reg.idx = EXTRA_REG_NONE; 604 605 return x86_pmu.hw_config(event); 606 } 607 608 void x86_pmu_disable_all(void) 609 { 610 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 611 int idx; 612 613 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 614 u64 val; 615 616 if (!test_bit(idx, cpuc->active_mask)) 617 continue; 618 rdmsrl(x86_pmu_config_addr(idx), val); 619 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 620 continue; 621 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 622 wrmsrl(x86_pmu_config_addr(idx), val); 623 } 624 } 625 626 /* 627 * There may be PMI landing after enabled=0. The PMI hitting could be before or 628 * after disable_all. 629 * 630 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 631 * It will not be re-enabled in the NMI handler again, because enabled=0. After 632 * handling the NMI, disable_all will be called, which will not change the 633 * state either. If PMI hits after disable_all, the PMU is already disabled 634 * before entering NMI handler. The NMI handler will not change the state 635 * either. 636 * 637 * So either situation is harmless. 638 */ 639 static void x86_pmu_disable(struct pmu *pmu) 640 { 641 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 642 643 if (!x86_pmu_initialized()) 644 return; 645 646 if (!cpuc->enabled) 647 return; 648 649 cpuc->n_added = 0; 650 cpuc->enabled = 0; 651 barrier(); 652 653 x86_pmu.disable_all(); 654 } 655 656 void x86_pmu_enable_all(int added) 657 { 658 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 659 int idx; 660 661 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 662 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 663 664 if (!test_bit(idx, cpuc->active_mask)) 665 continue; 666 667 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 668 } 669 } 670 671 static struct pmu pmu; 672 673 static inline int is_x86_event(struct perf_event *event) 674 { 675 return event->pmu == &pmu; 676 } 677 678 /* 679 * Event scheduler state: 680 * 681 * Assign events iterating over all events and counters, beginning 682 * with events with least weights first. Keep the current iterator 683 * state in struct sched_state. 684 */ 685 struct sched_state { 686 int weight; 687 int event; /* event index */ 688 int counter; /* counter index */ 689 int unassigned; /* number of events to be assigned left */ 690 int nr_gp; /* number of GP counters used */ 691 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 692 }; 693 694 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 695 #define SCHED_STATES_MAX 2 696 697 struct perf_sched { 698 int max_weight; 699 int max_events; 700 int max_gp; 701 int saved_states; 702 struct event_constraint **constraints; 703 struct sched_state state; 704 struct sched_state saved[SCHED_STATES_MAX]; 705 }; 706 707 /* 708 * Initialize interator that runs through all events and counters. 709 */ 710 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 711 int num, int wmin, int wmax, int gpmax) 712 { 713 int idx; 714 715 memset(sched, 0, sizeof(*sched)); 716 sched->max_events = num; 717 sched->max_weight = wmax; 718 sched->max_gp = gpmax; 719 sched->constraints = constraints; 720 721 for (idx = 0; idx < num; idx++) { 722 if (constraints[idx]->weight == wmin) 723 break; 724 } 725 726 sched->state.event = idx; /* start with min weight */ 727 sched->state.weight = wmin; 728 sched->state.unassigned = num; 729 } 730 731 static void perf_sched_save_state(struct perf_sched *sched) 732 { 733 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 734 return; 735 736 sched->saved[sched->saved_states] = sched->state; 737 sched->saved_states++; 738 } 739 740 static bool perf_sched_restore_state(struct perf_sched *sched) 741 { 742 if (!sched->saved_states) 743 return false; 744 745 sched->saved_states--; 746 sched->state = sched->saved[sched->saved_states]; 747 748 /* continue with next counter: */ 749 clear_bit(sched->state.counter++, sched->state.used); 750 751 return true; 752 } 753 754 /* 755 * Select a counter for the current event to schedule. Return true on 756 * success. 757 */ 758 static bool __perf_sched_find_counter(struct perf_sched *sched) 759 { 760 struct event_constraint *c; 761 int idx; 762 763 if (!sched->state.unassigned) 764 return false; 765 766 if (sched->state.event >= sched->max_events) 767 return false; 768 769 c = sched->constraints[sched->state.event]; 770 /* Prefer fixed purpose counters */ 771 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 772 idx = INTEL_PMC_IDX_FIXED; 773 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 774 if (!__test_and_set_bit(idx, sched->state.used)) 775 goto done; 776 } 777 } 778 779 /* Grab the first unused counter starting with idx */ 780 idx = sched->state.counter; 781 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 782 if (!__test_and_set_bit(idx, sched->state.used)) { 783 if (sched->state.nr_gp++ >= sched->max_gp) 784 return false; 785 786 goto done; 787 } 788 } 789 790 return false; 791 792 done: 793 sched->state.counter = idx; 794 795 if (c->overlap) 796 perf_sched_save_state(sched); 797 798 return true; 799 } 800 801 static bool perf_sched_find_counter(struct perf_sched *sched) 802 { 803 while (!__perf_sched_find_counter(sched)) { 804 if (!perf_sched_restore_state(sched)) 805 return false; 806 } 807 808 return true; 809 } 810 811 /* 812 * Go through all unassigned events and find the next one to schedule. 813 * Take events with the least weight first. Return true on success. 814 */ 815 static bool perf_sched_next_event(struct perf_sched *sched) 816 { 817 struct event_constraint *c; 818 819 if (!sched->state.unassigned || !--sched->state.unassigned) 820 return false; 821 822 do { 823 /* next event */ 824 sched->state.event++; 825 if (sched->state.event >= sched->max_events) { 826 /* next weight */ 827 sched->state.event = 0; 828 sched->state.weight++; 829 if (sched->state.weight > sched->max_weight) 830 return false; 831 } 832 c = sched->constraints[sched->state.event]; 833 } while (c->weight != sched->state.weight); 834 835 sched->state.counter = 0; /* start with first counter */ 836 837 return true; 838 } 839 840 /* 841 * Assign a counter for each event. 842 */ 843 int perf_assign_events(struct event_constraint **constraints, int n, 844 int wmin, int wmax, int gpmax, int *assign) 845 { 846 struct perf_sched sched; 847 848 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 849 850 do { 851 if (!perf_sched_find_counter(&sched)) 852 break; /* failed */ 853 if (assign) 854 assign[sched.state.event] = sched.state.counter; 855 } while (perf_sched_next_event(&sched)); 856 857 return sched.state.unassigned; 858 } 859 EXPORT_SYMBOL_GPL(perf_assign_events); 860 861 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 862 { 863 struct event_constraint *c; 864 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 865 struct perf_event *e; 866 int i, wmin, wmax, unsched = 0; 867 struct hw_perf_event *hwc; 868 869 bitmap_zero(used_mask, X86_PMC_IDX_MAX); 870 871 if (x86_pmu.start_scheduling) 872 x86_pmu.start_scheduling(cpuc); 873 874 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 875 cpuc->event_constraint[i] = NULL; 876 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]); 877 cpuc->event_constraint[i] = c; 878 879 wmin = min(wmin, c->weight); 880 wmax = max(wmax, c->weight); 881 } 882 883 /* 884 * fastpath, try to reuse previous register 885 */ 886 for (i = 0; i < n; i++) { 887 hwc = &cpuc->event_list[i]->hw; 888 c = cpuc->event_constraint[i]; 889 890 /* never assigned */ 891 if (hwc->idx == -1) 892 break; 893 894 /* constraint still honored */ 895 if (!test_bit(hwc->idx, c->idxmsk)) 896 break; 897 898 /* not already used */ 899 if (test_bit(hwc->idx, used_mask)) 900 break; 901 902 __set_bit(hwc->idx, used_mask); 903 if (assign) 904 assign[i] = hwc->idx; 905 } 906 907 /* slow path */ 908 if (i != n) { 909 int gpmax = x86_pmu.num_counters; 910 911 /* 912 * Do not allow scheduling of more than half the available 913 * generic counters. 914 * 915 * This helps avoid counter starvation of sibling thread by 916 * ensuring at most half the counters cannot be in exclusive 917 * mode. There is no designated counters for the limits. Any 918 * N/2 counters can be used. This helps with events with 919 * specific counter constraints. 920 */ 921 if (is_ht_workaround_enabled() && !cpuc->is_fake && 922 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 923 gpmax /= 2; 924 925 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 926 wmax, gpmax, assign); 927 } 928 929 /* 930 * In case of success (unsched = 0), mark events as committed, 931 * so we do not put_constraint() in case new events are added 932 * and fail to be scheduled 933 * 934 * We invoke the lower level commit callback to lock the resource 935 * 936 * We do not need to do all of this in case we are called to 937 * validate an event group (assign == NULL) 938 */ 939 if (!unsched && assign) { 940 for (i = 0; i < n; i++) { 941 e = cpuc->event_list[i]; 942 e->hw.flags |= PERF_X86_EVENT_COMMITTED; 943 if (x86_pmu.commit_scheduling) 944 x86_pmu.commit_scheduling(cpuc, i, assign[i]); 945 } 946 } else { 947 for (i = 0; i < n; i++) { 948 e = cpuc->event_list[i]; 949 /* 950 * do not put_constraint() on comitted events, 951 * because they are good to go 952 */ 953 if ((e->hw.flags & PERF_X86_EVENT_COMMITTED)) 954 continue; 955 956 /* 957 * release events that failed scheduling 958 */ 959 if (x86_pmu.put_event_constraints) 960 x86_pmu.put_event_constraints(cpuc, e); 961 } 962 } 963 964 if (x86_pmu.stop_scheduling) 965 x86_pmu.stop_scheduling(cpuc); 966 967 return unsched ? -EINVAL : 0; 968 } 969 970 /* 971 * dogrp: true if must collect siblings events (group) 972 * returns total number of events and error code 973 */ 974 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 975 { 976 struct perf_event *event; 977 int n, max_count; 978 979 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed; 980 981 /* current number of events already accepted */ 982 n = cpuc->n_events; 983 984 if (is_x86_event(leader)) { 985 if (n >= max_count) 986 return -EINVAL; 987 cpuc->event_list[n] = leader; 988 n++; 989 } 990 if (!dogrp) 991 return n; 992 993 list_for_each_entry(event, &leader->sibling_list, group_entry) { 994 if (!is_x86_event(event) || 995 event->state <= PERF_EVENT_STATE_OFF) 996 continue; 997 998 if (n >= max_count) 999 return -EINVAL; 1000 1001 cpuc->event_list[n] = event; 1002 n++; 1003 } 1004 return n; 1005 } 1006 1007 static inline void x86_assign_hw_event(struct perf_event *event, 1008 struct cpu_hw_events *cpuc, int i) 1009 { 1010 struct hw_perf_event *hwc = &event->hw; 1011 1012 hwc->idx = cpuc->assign[i]; 1013 hwc->last_cpu = smp_processor_id(); 1014 hwc->last_tag = ++cpuc->tags[i]; 1015 1016 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) { 1017 hwc->config_base = 0; 1018 hwc->event_base = 0; 1019 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) { 1020 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1021 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED); 1022 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30; 1023 } else { 1024 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1025 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1026 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1027 } 1028 } 1029 1030 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1031 struct cpu_hw_events *cpuc, 1032 int i) 1033 { 1034 return hwc->idx == cpuc->assign[i] && 1035 hwc->last_cpu == smp_processor_id() && 1036 hwc->last_tag == cpuc->tags[i]; 1037 } 1038 1039 static void x86_pmu_start(struct perf_event *event, int flags); 1040 1041 static void x86_pmu_enable(struct pmu *pmu) 1042 { 1043 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1044 struct perf_event *event; 1045 struct hw_perf_event *hwc; 1046 int i, added = cpuc->n_added; 1047 1048 if (!x86_pmu_initialized()) 1049 return; 1050 1051 if (cpuc->enabled) 1052 return; 1053 1054 if (cpuc->n_added) { 1055 int n_running = cpuc->n_events - cpuc->n_added; 1056 /* 1057 * apply assignment obtained either from 1058 * hw_perf_group_sched_in() or x86_pmu_enable() 1059 * 1060 * step1: save events moving to new counters 1061 */ 1062 for (i = 0; i < n_running; i++) { 1063 event = cpuc->event_list[i]; 1064 hwc = &event->hw; 1065 1066 /* 1067 * we can avoid reprogramming counter if: 1068 * - assigned same counter as last time 1069 * - running on same CPU as last time 1070 * - no other event has used the counter since 1071 */ 1072 if (hwc->idx == -1 || 1073 match_prev_assignment(hwc, cpuc, i)) 1074 continue; 1075 1076 /* 1077 * Ensure we don't accidentally enable a stopped 1078 * counter simply because we rescheduled. 1079 */ 1080 if (hwc->state & PERF_HES_STOPPED) 1081 hwc->state |= PERF_HES_ARCH; 1082 1083 x86_pmu_stop(event, PERF_EF_UPDATE); 1084 } 1085 1086 /* 1087 * step2: reprogram moved events into new counters 1088 */ 1089 for (i = 0; i < cpuc->n_events; i++) { 1090 event = cpuc->event_list[i]; 1091 hwc = &event->hw; 1092 1093 if (!match_prev_assignment(hwc, cpuc, i)) 1094 x86_assign_hw_event(event, cpuc, i); 1095 else if (i < n_running) 1096 continue; 1097 1098 if (hwc->state & PERF_HES_ARCH) 1099 continue; 1100 1101 x86_pmu_start(event, PERF_EF_RELOAD); 1102 } 1103 cpuc->n_added = 0; 1104 perf_events_lapic_init(); 1105 } 1106 1107 cpuc->enabled = 1; 1108 barrier(); 1109 1110 x86_pmu.enable_all(added); 1111 } 1112 1113 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1114 1115 /* 1116 * Set the next IRQ period, based on the hwc->period_left value. 1117 * To be called with the event disabled in hw: 1118 */ 1119 int x86_perf_event_set_period(struct perf_event *event) 1120 { 1121 struct hw_perf_event *hwc = &event->hw; 1122 s64 left = local64_read(&hwc->period_left); 1123 s64 period = hwc->sample_period; 1124 int ret = 0, idx = hwc->idx; 1125 1126 if (idx == INTEL_PMC_IDX_FIXED_BTS) 1127 return 0; 1128 1129 /* 1130 * If we are way outside a reasonable range then just skip forward: 1131 */ 1132 if (unlikely(left <= -period)) { 1133 left = period; 1134 local64_set(&hwc->period_left, left); 1135 hwc->last_period = period; 1136 ret = 1; 1137 } 1138 1139 if (unlikely(left <= 0)) { 1140 left += period; 1141 local64_set(&hwc->period_left, left); 1142 hwc->last_period = period; 1143 ret = 1; 1144 } 1145 /* 1146 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1147 */ 1148 if (unlikely(left < 2)) 1149 left = 2; 1150 1151 if (left > x86_pmu.max_period) 1152 left = x86_pmu.max_period; 1153 1154 if (x86_pmu.limit_period) 1155 left = x86_pmu.limit_period(event, left); 1156 1157 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; 1158 1159 if (!(hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) || 1160 local64_read(&hwc->prev_count) != (u64)-left) { 1161 /* 1162 * The hw event starts counting from this event offset, 1163 * mark it to be able to extra future deltas: 1164 */ 1165 local64_set(&hwc->prev_count, (u64)-left); 1166 1167 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1168 } 1169 1170 /* 1171 * Due to erratum on certan cpu we need 1172 * a second write to be sure the register 1173 * is updated properly 1174 */ 1175 if (x86_pmu.perfctr_second_write) { 1176 wrmsrl(hwc->event_base, 1177 (u64)(-left) & x86_pmu.cntval_mask); 1178 } 1179 1180 perf_event_update_userpage(event); 1181 1182 return ret; 1183 } 1184 1185 void x86_pmu_enable_event(struct perf_event *event) 1186 { 1187 if (__this_cpu_read(cpu_hw_events.enabled)) 1188 __x86_pmu_enable_event(&event->hw, 1189 ARCH_PERFMON_EVENTSEL_ENABLE); 1190 } 1191 1192 /* 1193 * Add a single event to the PMU. 1194 * 1195 * The event is added to the group of enabled events 1196 * but only if it can be scehduled with existing events. 1197 */ 1198 static int x86_pmu_add(struct perf_event *event, int flags) 1199 { 1200 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1201 struct hw_perf_event *hwc; 1202 int assign[X86_PMC_IDX_MAX]; 1203 int n, n0, ret; 1204 1205 hwc = &event->hw; 1206 1207 n0 = cpuc->n_events; 1208 ret = n = collect_events(cpuc, event, false); 1209 if (ret < 0) 1210 goto out; 1211 1212 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1213 if (!(flags & PERF_EF_START)) 1214 hwc->state |= PERF_HES_ARCH; 1215 1216 /* 1217 * If group events scheduling transaction was started, 1218 * skip the schedulability test here, it will be performed 1219 * at commit time (->commit_txn) as a whole. 1220 * 1221 * If commit fails, we'll call ->del() on all events 1222 * for which ->add() was called. 1223 */ 1224 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1225 goto done_collect; 1226 1227 ret = x86_pmu.schedule_events(cpuc, n, assign); 1228 if (ret) 1229 goto out; 1230 /* 1231 * copy new assignment, now we know it is possible 1232 * will be used by hw_perf_enable() 1233 */ 1234 memcpy(cpuc->assign, assign, n*sizeof(int)); 1235 1236 done_collect: 1237 /* 1238 * Commit the collect_events() state. See x86_pmu_del() and 1239 * x86_pmu_*_txn(). 1240 */ 1241 cpuc->n_events = n; 1242 cpuc->n_added += n - n0; 1243 cpuc->n_txn += n - n0; 1244 1245 if (x86_pmu.add) { 1246 /* 1247 * This is before x86_pmu_enable() will call x86_pmu_start(), 1248 * so we enable LBRs before an event needs them etc.. 1249 */ 1250 x86_pmu.add(event); 1251 } 1252 1253 ret = 0; 1254 out: 1255 return ret; 1256 } 1257 1258 static void x86_pmu_start(struct perf_event *event, int flags) 1259 { 1260 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1261 int idx = event->hw.idx; 1262 1263 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1264 return; 1265 1266 if (WARN_ON_ONCE(idx == -1)) 1267 return; 1268 1269 if (flags & PERF_EF_RELOAD) { 1270 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1271 x86_perf_event_set_period(event); 1272 } 1273 1274 event->hw.state = 0; 1275 1276 cpuc->events[idx] = event; 1277 __set_bit(idx, cpuc->active_mask); 1278 __set_bit(idx, cpuc->running); 1279 x86_pmu.enable(event); 1280 perf_event_update_userpage(event); 1281 } 1282 1283 void perf_event_print_debug(void) 1284 { 1285 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1286 u64 pebs, debugctl; 1287 struct cpu_hw_events *cpuc; 1288 unsigned long flags; 1289 int cpu, idx; 1290 1291 if (!x86_pmu.num_counters) 1292 return; 1293 1294 local_irq_save(flags); 1295 1296 cpu = smp_processor_id(); 1297 cpuc = &per_cpu(cpu_hw_events, cpu); 1298 1299 if (x86_pmu.version >= 2) { 1300 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1301 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1302 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1303 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1304 1305 pr_info("\n"); 1306 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1307 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1308 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1309 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1310 if (x86_pmu.pebs_constraints) { 1311 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1312 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1313 } 1314 if (x86_pmu.lbr_nr) { 1315 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1316 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1317 } 1318 } 1319 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1320 1321 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1322 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1323 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1324 1325 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1326 1327 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1328 cpu, idx, pmc_ctrl); 1329 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1330 cpu, idx, pmc_count); 1331 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1332 cpu, idx, prev_left); 1333 } 1334 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) { 1335 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); 1336 1337 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1338 cpu, idx, pmc_count); 1339 } 1340 local_irq_restore(flags); 1341 } 1342 1343 void x86_pmu_stop(struct perf_event *event, int flags) 1344 { 1345 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1346 struct hw_perf_event *hwc = &event->hw; 1347 1348 if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) { 1349 x86_pmu.disable(event); 1350 cpuc->events[hwc->idx] = NULL; 1351 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1352 hwc->state |= PERF_HES_STOPPED; 1353 } 1354 1355 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1356 /* 1357 * Drain the remaining delta count out of a event 1358 * that we are disabling: 1359 */ 1360 x86_perf_event_update(event); 1361 hwc->state |= PERF_HES_UPTODATE; 1362 } 1363 } 1364 1365 static void x86_pmu_del(struct perf_event *event, int flags) 1366 { 1367 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1368 int i; 1369 1370 /* 1371 * event is descheduled 1372 */ 1373 event->hw.flags &= ~PERF_X86_EVENT_COMMITTED; 1374 1375 /* 1376 * If we're called during a txn, we only need to undo x86_pmu.add. 1377 * The events never got scheduled and ->cancel_txn will truncate 1378 * the event_list. 1379 * 1380 * XXX assumes any ->del() called during a TXN will only be on 1381 * an event added during that same TXN. 1382 */ 1383 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1384 goto do_del; 1385 1386 /* 1387 * Not a TXN, therefore cleanup properly. 1388 */ 1389 x86_pmu_stop(event, PERF_EF_UPDATE); 1390 1391 for (i = 0; i < cpuc->n_events; i++) { 1392 if (event == cpuc->event_list[i]) 1393 break; 1394 } 1395 1396 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1397 return; 1398 1399 /* If we have a newly added event; make sure to decrease n_added. */ 1400 if (i >= cpuc->n_events - cpuc->n_added) 1401 --cpuc->n_added; 1402 1403 if (x86_pmu.put_event_constraints) 1404 x86_pmu.put_event_constraints(cpuc, event); 1405 1406 /* Delete the array entry. */ 1407 while (++i < cpuc->n_events) { 1408 cpuc->event_list[i-1] = cpuc->event_list[i]; 1409 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1410 } 1411 --cpuc->n_events; 1412 1413 perf_event_update_userpage(event); 1414 1415 do_del: 1416 if (x86_pmu.del) { 1417 /* 1418 * This is after x86_pmu_stop(); so we disable LBRs after any 1419 * event can need them etc.. 1420 */ 1421 x86_pmu.del(event); 1422 } 1423 } 1424 1425 int x86_pmu_handle_irq(struct pt_regs *regs) 1426 { 1427 struct perf_sample_data data; 1428 struct cpu_hw_events *cpuc; 1429 struct perf_event *event; 1430 int idx, handled = 0; 1431 u64 val; 1432 1433 cpuc = this_cpu_ptr(&cpu_hw_events); 1434 1435 /* 1436 * Some chipsets need to unmask the LVTPC in a particular spot 1437 * inside the nmi handler. As a result, the unmasking was pushed 1438 * into all the nmi handlers. 1439 * 1440 * This generic handler doesn't seem to have any issues where the 1441 * unmasking occurs so it was left at the top. 1442 */ 1443 apic_write(APIC_LVTPC, APIC_DM_NMI); 1444 1445 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1446 if (!test_bit(idx, cpuc->active_mask)) { 1447 /* 1448 * Though we deactivated the counter some cpus 1449 * might still deliver spurious interrupts still 1450 * in flight. Catch them: 1451 */ 1452 if (__test_and_clear_bit(idx, cpuc->running)) 1453 handled++; 1454 continue; 1455 } 1456 1457 event = cpuc->events[idx]; 1458 1459 val = x86_perf_event_update(event); 1460 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1461 continue; 1462 1463 /* 1464 * event overflow 1465 */ 1466 handled++; 1467 perf_sample_data_init(&data, 0, event->hw.last_period); 1468 1469 if (!x86_perf_event_set_period(event)) 1470 continue; 1471 1472 if (perf_event_overflow(event, &data, regs)) 1473 x86_pmu_stop(event, 0); 1474 } 1475 1476 if (handled) 1477 inc_irq_stat(apic_perf_irqs); 1478 1479 return handled; 1480 } 1481 1482 void perf_events_lapic_init(void) 1483 { 1484 if (!x86_pmu.apic || !x86_pmu_initialized()) 1485 return; 1486 1487 /* 1488 * Always use NMI for PMU 1489 */ 1490 apic_write(APIC_LVTPC, APIC_DM_NMI); 1491 } 1492 1493 static int 1494 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1495 { 1496 u64 start_clock; 1497 u64 finish_clock; 1498 int ret; 1499 1500 /* 1501 * All PMUs/events that share this PMI handler should make sure to 1502 * increment active_events for their events. 1503 */ 1504 if (!atomic_read(&active_events)) 1505 return NMI_DONE; 1506 1507 start_clock = sched_clock(); 1508 ret = x86_pmu.handle_irq(regs); 1509 finish_clock = sched_clock(); 1510 1511 perf_sample_event_took(finish_clock - start_clock); 1512 1513 return ret; 1514 } 1515 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1516 1517 struct event_constraint emptyconstraint; 1518 struct event_constraint unconstrained; 1519 1520 static int x86_pmu_prepare_cpu(unsigned int cpu) 1521 { 1522 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1523 int i; 1524 1525 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1526 cpuc->kfree_on_online[i] = NULL; 1527 if (x86_pmu.cpu_prepare) 1528 return x86_pmu.cpu_prepare(cpu); 1529 return 0; 1530 } 1531 1532 static int x86_pmu_dead_cpu(unsigned int cpu) 1533 { 1534 if (x86_pmu.cpu_dead) 1535 x86_pmu.cpu_dead(cpu); 1536 return 0; 1537 } 1538 1539 static int x86_pmu_online_cpu(unsigned int cpu) 1540 { 1541 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1542 int i; 1543 1544 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1545 kfree(cpuc->kfree_on_online[i]); 1546 cpuc->kfree_on_online[i] = NULL; 1547 } 1548 return 0; 1549 } 1550 1551 static int x86_pmu_starting_cpu(unsigned int cpu) 1552 { 1553 if (x86_pmu.cpu_starting) 1554 x86_pmu.cpu_starting(cpu); 1555 return 0; 1556 } 1557 1558 static int x86_pmu_dying_cpu(unsigned int cpu) 1559 { 1560 if (x86_pmu.cpu_dying) 1561 x86_pmu.cpu_dying(cpu); 1562 return 0; 1563 } 1564 1565 static void __init pmu_check_apic(void) 1566 { 1567 if (boot_cpu_has(X86_FEATURE_APIC)) 1568 return; 1569 1570 x86_pmu.apic = 0; 1571 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1572 pr_info("no hardware sampling interrupt available.\n"); 1573 1574 /* 1575 * If we have a PMU initialized but no APIC 1576 * interrupts, we cannot sample hardware 1577 * events (user-space has to fall back and 1578 * sample via a hrtimer based software event): 1579 */ 1580 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1581 1582 } 1583 1584 static struct attribute_group x86_pmu_format_group = { 1585 .name = "format", 1586 .attrs = NULL, 1587 }; 1588 1589 /* 1590 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1591 * out of events_attr attributes. 1592 */ 1593 static void __init filter_events(struct attribute **attrs) 1594 { 1595 struct device_attribute *d; 1596 struct perf_pmu_events_attr *pmu_attr; 1597 int offset = 0; 1598 int i, j; 1599 1600 for (i = 0; attrs[i]; i++) { 1601 d = (struct device_attribute *)attrs[i]; 1602 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr); 1603 /* str trumps id */ 1604 if (pmu_attr->event_str) 1605 continue; 1606 if (x86_pmu.event_map(i + offset)) 1607 continue; 1608 1609 for (j = i; attrs[j]; j++) 1610 attrs[j] = attrs[j + 1]; 1611 1612 /* Check the shifted attr. */ 1613 i--; 1614 1615 /* 1616 * event_map() is index based, the attrs array is organized 1617 * by increasing event index. If we shift the events, then 1618 * we need to compensate for the event_map(), otherwise 1619 * we are looking up the wrong event in the map 1620 */ 1621 offset++; 1622 } 1623 } 1624 1625 /* Merge two pointer arrays */ 1626 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b) 1627 { 1628 struct attribute **new; 1629 int j, i; 1630 1631 for (j = 0; a[j]; j++) 1632 ; 1633 for (i = 0; b[i]; i++) 1634 j++; 1635 j++; 1636 1637 new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL); 1638 if (!new) 1639 return NULL; 1640 1641 j = 0; 1642 for (i = 0; a[i]; i++) 1643 new[j++] = a[i]; 1644 for (i = 0; b[i]; i++) 1645 new[j++] = b[i]; 1646 new[j] = NULL; 1647 1648 return new; 1649 } 1650 1651 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1652 { 1653 struct perf_pmu_events_attr *pmu_attr = \ 1654 container_of(attr, struct perf_pmu_events_attr, attr); 1655 u64 config = x86_pmu.event_map(pmu_attr->id); 1656 1657 /* string trumps id */ 1658 if (pmu_attr->event_str) 1659 return sprintf(page, "%s", pmu_attr->event_str); 1660 1661 return x86_pmu.events_sysfs_show(page, config); 1662 } 1663 EXPORT_SYMBOL_GPL(events_sysfs_show); 1664 1665 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1666 char *page) 1667 { 1668 struct perf_pmu_events_ht_attr *pmu_attr = 1669 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1670 1671 /* 1672 * Report conditional events depending on Hyper-Threading. 1673 * 1674 * This is overly conservative as usually the HT special 1675 * handling is not needed if the other CPU thread is idle. 1676 * 1677 * Note this does not (and cannot) handle the case when thread 1678 * siblings are invisible, for example with virtualization 1679 * if they are owned by some other guest. The user tool 1680 * has to re-read when a thread sibling gets onlined later. 1681 */ 1682 return sprintf(page, "%s", 1683 topology_max_smt_threads() > 1 ? 1684 pmu_attr->event_str_ht : 1685 pmu_attr->event_str_noht); 1686 } 1687 1688 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1689 EVENT_ATTR(instructions, INSTRUCTIONS ); 1690 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1691 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1692 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1693 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1694 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1695 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1696 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1697 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1698 1699 static struct attribute *empty_attrs; 1700 1701 static struct attribute *events_attr[] = { 1702 EVENT_PTR(CPU_CYCLES), 1703 EVENT_PTR(INSTRUCTIONS), 1704 EVENT_PTR(CACHE_REFERENCES), 1705 EVENT_PTR(CACHE_MISSES), 1706 EVENT_PTR(BRANCH_INSTRUCTIONS), 1707 EVENT_PTR(BRANCH_MISSES), 1708 EVENT_PTR(BUS_CYCLES), 1709 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1710 EVENT_PTR(STALLED_CYCLES_BACKEND), 1711 EVENT_PTR(REF_CPU_CYCLES), 1712 NULL, 1713 }; 1714 1715 static struct attribute_group x86_pmu_events_group = { 1716 .name = "events", 1717 .attrs = events_attr, 1718 }; 1719 1720 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1721 { 1722 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1723 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1724 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1725 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1726 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1727 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1728 ssize_t ret; 1729 1730 /* 1731 * We have whole page size to spend and just little data 1732 * to write, so we can safely use sprintf. 1733 */ 1734 ret = sprintf(page, "event=0x%02llx", event); 1735 1736 if (umask) 1737 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1738 1739 if (edge) 1740 ret += sprintf(page + ret, ",edge"); 1741 1742 if (pc) 1743 ret += sprintf(page + ret, ",pc"); 1744 1745 if (any) 1746 ret += sprintf(page + ret, ",any"); 1747 1748 if (inv) 1749 ret += sprintf(page + ret, ",inv"); 1750 1751 if (cmask) 1752 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1753 1754 ret += sprintf(page + ret, "\n"); 1755 1756 return ret; 1757 } 1758 1759 static struct attribute_group x86_pmu_attr_group; 1760 static struct attribute_group x86_pmu_caps_group; 1761 1762 static int __init init_hw_perf_events(void) 1763 { 1764 struct x86_pmu_quirk *quirk; 1765 int err; 1766 1767 pr_info("Performance Events: "); 1768 1769 switch (boot_cpu_data.x86_vendor) { 1770 case X86_VENDOR_INTEL: 1771 err = intel_pmu_init(); 1772 break; 1773 case X86_VENDOR_AMD: 1774 err = amd_pmu_init(); 1775 break; 1776 default: 1777 err = -ENOTSUPP; 1778 } 1779 if (err != 0) { 1780 pr_cont("no PMU driver, software events only.\n"); 1781 return 0; 1782 } 1783 1784 pmu_check_apic(); 1785 1786 /* sanity check that the hardware exists or is emulated */ 1787 if (!check_hw_exists()) 1788 return 0; 1789 1790 pr_cont("%s PMU driver.\n", x86_pmu.name); 1791 1792 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 1793 1794 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 1795 quirk->func(); 1796 1797 if (!x86_pmu.intel_ctrl) 1798 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; 1799 1800 perf_events_lapic_init(); 1801 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 1802 1803 unconstrained = (struct event_constraint) 1804 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 1805 0, x86_pmu.num_counters, 0, 0); 1806 1807 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 1808 1809 if (x86_pmu.caps_attrs) { 1810 struct attribute **tmp; 1811 1812 tmp = merge_attr(x86_pmu_caps_group.attrs, x86_pmu.caps_attrs); 1813 if (!WARN_ON(!tmp)) 1814 x86_pmu_caps_group.attrs = tmp; 1815 } 1816 1817 if (x86_pmu.event_attrs) 1818 x86_pmu_events_group.attrs = x86_pmu.event_attrs; 1819 1820 if (!x86_pmu.events_sysfs_show) 1821 x86_pmu_events_group.attrs = &empty_attrs; 1822 else 1823 filter_events(x86_pmu_events_group.attrs); 1824 1825 if (x86_pmu.cpu_events) { 1826 struct attribute **tmp; 1827 1828 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events); 1829 if (!WARN_ON(!tmp)) 1830 x86_pmu_events_group.attrs = tmp; 1831 } 1832 1833 if (x86_pmu.attrs) { 1834 struct attribute **tmp; 1835 1836 tmp = merge_attr(x86_pmu_attr_group.attrs, x86_pmu.attrs); 1837 if (!WARN_ON(!tmp)) 1838 x86_pmu_attr_group.attrs = tmp; 1839 } 1840 1841 pr_info("... version: %d\n", x86_pmu.version); 1842 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 1843 pr_info("... generic registers: %d\n", x86_pmu.num_counters); 1844 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 1845 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 1846 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed); 1847 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl); 1848 1849 /* 1850 * Install callbacks. Core will call them for each online 1851 * cpu. 1852 */ 1853 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 1854 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 1855 if (err) 1856 return err; 1857 1858 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 1859 "perf/x86:starting", x86_pmu_starting_cpu, 1860 x86_pmu_dying_cpu); 1861 if (err) 1862 goto out; 1863 1864 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 1865 x86_pmu_online_cpu, NULL); 1866 if (err) 1867 goto out1; 1868 1869 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 1870 if (err) 1871 goto out2; 1872 1873 return 0; 1874 1875 out2: 1876 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 1877 out1: 1878 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 1879 out: 1880 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 1881 return err; 1882 } 1883 early_initcall(init_hw_perf_events); 1884 1885 static inline void x86_pmu_read(struct perf_event *event) 1886 { 1887 x86_perf_event_update(event); 1888 } 1889 1890 /* 1891 * Start group events scheduling transaction 1892 * Set the flag to make pmu::enable() not perform the 1893 * schedulability test, it will be performed at commit time 1894 * 1895 * We only support PERF_PMU_TXN_ADD transactions. Save the 1896 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 1897 * transactions. 1898 */ 1899 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 1900 { 1901 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1902 1903 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 1904 1905 cpuc->txn_flags = txn_flags; 1906 if (txn_flags & ~PERF_PMU_TXN_ADD) 1907 return; 1908 1909 perf_pmu_disable(pmu); 1910 __this_cpu_write(cpu_hw_events.n_txn, 0); 1911 } 1912 1913 /* 1914 * Stop group events scheduling transaction 1915 * Clear the flag and pmu::enable() will perform the 1916 * schedulability test. 1917 */ 1918 static void x86_pmu_cancel_txn(struct pmu *pmu) 1919 { 1920 unsigned int txn_flags; 1921 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1922 1923 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1924 1925 txn_flags = cpuc->txn_flags; 1926 cpuc->txn_flags = 0; 1927 if (txn_flags & ~PERF_PMU_TXN_ADD) 1928 return; 1929 1930 /* 1931 * Truncate collected array by the number of events added in this 1932 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 1933 */ 1934 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 1935 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 1936 perf_pmu_enable(pmu); 1937 } 1938 1939 /* 1940 * Commit group events scheduling transaction 1941 * Perform the group schedulability test as a whole 1942 * Return 0 if success 1943 * 1944 * Does not cancel the transaction on failure; expects the caller to do this. 1945 */ 1946 static int x86_pmu_commit_txn(struct pmu *pmu) 1947 { 1948 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1949 int assign[X86_PMC_IDX_MAX]; 1950 int n, ret; 1951 1952 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1953 1954 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 1955 cpuc->txn_flags = 0; 1956 return 0; 1957 } 1958 1959 n = cpuc->n_events; 1960 1961 if (!x86_pmu_initialized()) 1962 return -EAGAIN; 1963 1964 ret = x86_pmu.schedule_events(cpuc, n, assign); 1965 if (ret) 1966 return ret; 1967 1968 /* 1969 * copy new assignment, now we know it is possible 1970 * will be used by hw_perf_enable() 1971 */ 1972 memcpy(cpuc->assign, assign, n*sizeof(int)); 1973 1974 cpuc->txn_flags = 0; 1975 perf_pmu_enable(pmu); 1976 return 0; 1977 } 1978 /* 1979 * a fake_cpuc is used to validate event groups. Due to 1980 * the extra reg logic, we need to also allocate a fake 1981 * per_core and per_cpu structure. Otherwise, group events 1982 * using extra reg may conflict without the kernel being 1983 * able to catch this when the last event gets added to 1984 * the group. 1985 */ 1986 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 1987 { 1988 kfree(cpuc->shared_regs); 1989 kfree(cpuc); 1990 } 1991 1992 static struct cpu_hw_events *allocate_fake_cpuc(void) 1993 { 1994 struct cpu_hw_events *cpuc; 1995 int cpu = raw_smp_processor_id(); 1996 1997 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 1998 if (!cpuc) 1999 return ERR_PTR(-ENOMEM); 2000 2001 /* only needed, if we have extra_regs */ 2002 if (x86_pmu.extra_regs) { 2003 cpuc->shared_regs = allocate_shared_regs(cpu); 2004 if (!cpuc->shared_regs) 2005 goto error; 2006 } 2007 cpuc->is_fake = 1; 2008 return cpuc; 2009 error: 2010 free_fake_cpuc(cpuc); 2011 return ERR_PTR(-ENOMEM); 2012 } 2013 2014 /* 2015 * validate that we can schedule this event 2016 */ 2017 static int validate_event(struct perf_event *event) 2018 { 2019 struct cpu_hw_events *fake_cpuc; 2020 struct event_constraint *c; 2021 int ret = 0; 2022 2023 fake_cpuc = allocate_fake_cpuc(); 2024 if (IS_ERR(fake_cpuc)) 2025 return PTR_ERR(fake_cpuc); 2026 2027 c = x86_pmu.get_event_constraints(fake_cpuc, -1, event); 2028 2029 if (!c || !c->weight) 2030 ret = -EINVAL; 2031 2032 if (x86_pmu.put_event_constraints) 2033 x86_pmu.put_event_constraints(fake_cpuc, event); 2034 2035 free_fake_cpuc(fake_cpuc); 2036 2037 return ret; 2038 } 2039 2040 /* 2041 * validate a single event group 2042 * 2043 * validation include: 2044 * - check events are compatible which each other 2045 * - events do not compete for the same counter 2046 * - number of events <= number of counters 2047 * 2048 * validation ensures the group can be loaded onto the 2049 * PMU if it was the only group available. 2050 */ 2051 static int validate_group(struct perf_event *event) 2052 { 2053 struct perf_event *leader = event->group_leader; 2054 struct cpu_hw_events *fake_cpuc; 2055 int ret = -EINVAL, n; 2056 2057 fake_cpuc = allocate_fake_cpuc(); 2058 if (IS_ERR(fake_cpuc)) 2059 return PTR_ERR(fake_cpuc); 2060 /* 2061 * the event is not yet connected with its 2062 * siblings therefore we must first collect 2063 * existing siblings, then add the new event 2064 * before we can simulate the scheduling 2065 */ 2066 n = collect_events(fake_cpuc, leader, true); 2067 if (n < 0) 2068 goto out; 2069 2070 fake_cpuc->n_events = n; 2071 n = collect_events(fake_cpuc, event, false); 2072 if (n < 0) 2073 goto out; 2074 2075 fake_cpuc->n_events = n; 2076 2077 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2078 2079 out: 2080 free_fake_cpuc(fake_cpuc); 2081 return ret; 2082 } 2083 2084 static int x86_pmu_event_init(struct perf_event *event) 2085 { 2086 struct pmu *tmp; 2087 int err; 2088 2089 switch (event->attr.type) { 2090 case PERF_TYPE_RAW: 2091 case PERF_TYPE_HARDWARE: 2092 case PERF_TYPE_HW_CACHE: 2093 break; 2094 2095 default: 2096 return -ENOENT; 2097 } 2098 2099 err = __x86_pmu_event_init(event); 2100 if (!err) { 2101 /* 2102 * we temporarily connect event to its pmu 2103 * such that validate_group() can classify 2104 * it as an x86 event using is_x86_event() 2105 */ 2106 tmp = event->pmu; 2107 event->pmu = &pmu; 2108 2109 if (event->group_leader != event) 2110 err = validate_group(event); 2111 else 2112 err = validate_event(event); 2113 2114 event->pmu = tmp; 2115 } 2116 if (err) { 2117 if (event->destroy) 2118 event->destroy(event); 2119 } 2120 2121 if (READ_ONCE(x86_pmu.attr_rdpmc)) 2122 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; 2123 2124 return err; 2125 } 2126 2127 static void refresh_pce(void *ignored) 2128 { 2129 load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm)); 2130 } 2131 2132 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) 2133 { 2134 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2135 return; 2136 2137 /* 2138 * This function relies on not being called concurrently in two 2139 * tasks in the same mm. Otherwise one task could observe 2140 * perf_rdpmc_allowed > 1 and return all the way back to 2141 * userspace with CR4.PCE clear while another task is still 2142 * doing on_each_cpu_mask() to propagate CR4.PCE. 2143 * 2144 * For now, this can't happen because all callers hold mmap_sem 2145 * for write. If this changes, we'll need a different solution. 2146 */ 2147 lockdep_assert_held_exclusive(&mm->mmap_sem); 2148 2149 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) 2150 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2151 } 2152 2153 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) 2154 { 2155 2156 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2157 return; 2158 2159 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) 2160 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2161 } 2162 2163 static int x86_pmu_event_idx(struct perf_event *event) 2164 { 2165 int idx = event->hw.idx; 2166 2167 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2168 return 0; 2169 2170 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) { 2171 idx -= INTEL_PMC_IDX_FIXED; 2172 idx |= 1 << 30; 2173 } 2174 2175 return idx + 1; 2176 } 2177 2178 static ssize_t get_attr_rdpmc(struct device *cdev, 2179 struct device_attribute *attr, 2180 char *buf) 2181 { 2182 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2183 } 2184 2185 static ssize_t set_attr_rdpmc(struct device *cdev, 2186 struct device_attribute *attr, 2187 const char *buf, size_t count) 2188 { 2189 unsigned long val; 2190 ssize_t ret; 2191 2192 ret = kstrtoul(buf, 0, &val); 2193 if (ret) 2194 return ret; 2195 2196 if (val > 2) 2197 return -EINVAL; 2198 2199 if (x86_pmu.attr_rdpmc_broken) 2200 return -ENOTSUPP; 2201 2202 if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) { 2203 /* 2204 * Changing into or out of always available, aka 2205 * perf-event-bypassing mode. This path is extremely slow, 2206 * but only root can trigger it, so it's okay. 2207 */ 2208 if (val == 2) 2209 static_key_slow_inc(&rdpmc_always_available); 2210 else 2211 static_key_slow_dec(&rdpmc_always_available); 2212 on_each_cpu(refresh_pce, NULL, 1); 2213 } 2214 2215 x86_pmu.attr_rdpmc = val; 2216 2217 return count; 2218 } 2219 2220 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2221 2222 static struct attribute *x86_pmu_attrs[] = { 2223 &dev_attr_rdpmc.attr, 2224 NULL, 2225 }; 2226 2227 static struct attribute_group x86_pmu_attr_group = { 2228 .attrs = x86_pmu_attrs, 2229 }; 2230 2231 static ssize_t max_precise_show(struct device *cdev, 2232 struct device_attribute *attr, 2233 char *buf) 2234 { 2235 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise()); 2236 } 2237 2238 static DEVICE_ATTR_RO(max_precise); 2239 2240 static struct attribute *x86_pmu_caps_attrs[] = { 2241 &dev_attr_max_precise.attr, 2242 NULL 2243 }; 2244 2245 static struct attribute_group x86_pmu_caps_group = { 2246 .name = "caps", 2247 .attrs = x86_pmu_caps_attrs, 2248 }; 2249 2250 static const struct attribute_group *x86_pmu_attr_groups[] = { 2251 &x86_pmu_attr_group, 2252 &x86_pmu_format_group, 2253 &x86_pmu_events_group, 2254 &x86_pmu_caps_group, 2255 NULL, 2256 }; 2257 2258 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) 2259 { 2260 if (x86_pmu.sched_task) 2261 x86_pmu.sched_task(ctx, sched_in); 2262 } 2263 2264 void perf_check_microcode(void) 2265 { 2266 if (x86_pmu.check_microcode) 2267 x86_pmu.check_microcode(); 2268 } 2269 2270 static struct pmu pmu = { 2271 .pmu_enable = x86_pmu_enable, 2272 .pmu_disable = x86_pmu_disable, 2273 2274 .attr_groups = x86_pmu_attr_groups, 2275 2276 .event_init = x86_pmu_event_init, 2277 2278 .event_mapped = x86_pmu_event_mapped, 2279 .event_unmapped = x86_pmu_event_unmapped, 2280 2281 .add = x86_pmu_add, 2282 .del = x86_pmu_del, 2283 .start = x86_pmu_start, 2284 .stop = x86_pmu_stop, 2285 .read = x86_pmu_read, 2286 2287 .start_txn = x86_pmu_start_txn, 2288 .cancel_txn = x86_pmu_cancel_txn, 2289 .commit_txn = x86_pmu_commit_txn, 2290 2291 .event_idx = x86_pmu_event_idx, 2292 .sched_task = x86_pmu_sched_task, 2293 .task_ctx_size = sizeof(struct x86_perf_task_context), 2294 }; 2295 2296 void arch_perf_update_userpage(struct perf_event *event, 2297 struct perf_event_mmap_page *userpg, u64 now) 2298 { 2299 struct cyc2ns_data data; 2300 u64 offset; 2301 2302 userpg->cap_user_time = 0; 2303 userpg->cap_user_time_zero = 0; 2304 userpg->cap_user_rdpmc = 2305 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); 2306 userpg->pmc_width = x86_pmu.cntval_bits; 2307 2308 if (!using_native_sched_clock() || !sched_clock_stable()) 2309 return; 2310 2311 cyc2ns_read_begin(&data); 2312 2313 offset = data.cyc2ns_offset + __sched_clock_offset; 2314 2315 /* 2316 * Internal timekeeping for enabled/running/stopped times 2317 * is always in the local_clock domain. 2318 */ 2319 userpg->cap_user_time = 1; 2320 userpg->time_mult = data.cyc2ns_mul; 2321 userpg->time_shift = data.cyc2ns_shift; 2322 userpg->time_offset = offset - now; 2323 2324 /* 2325 * cap_user_time_zero doesn't make sense when we're using a different 2326 * time base for the records. 2327 */ 2328 if (!event->attr.use_clockid) { 2329 userpg->cap_user_time_zero = 1; 2330 userpg->time_zero = offset; 2331 } 2332 2333 cyc2ns_read_end(); 2334 } 2335 2336 void 2337 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2338 { 2339 struct unwind_state state; 2340 unsigned long addr; 2341 2342 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2343 /* TODO: We don't support guest os callchain now */ 2344 return; 2345 } 2346 2347 if (perf_callchain_store(entry, regs->ip)) 2348 return; 2349 2350 for (unwind_start(&state, current, regs, NULL); !unwind_done(&state); 2351 unwind_next_frame(&state)) { 2352 addr = unwind_get_return_address(&state); 2353 if (!addr || perf_callchain_store(entry, addr)) 2354 return; 2355 } 2356 } 2357 2358 static inline int 2359 valid_user_frame(const void __user *fp, unsigned long size) 2360 { 2361 return (__range_not_ok(fp, size, TASK_SIZE) == 0); 2362 } 2363 2364 static unsigned long get_segment_base(unsigned int segment) 2365 { 2366 struct desc_struct *desc; 2367 unsigned int idx = segment >> 3; 2368 2369 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2370 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2371 struct ldt_struct *ldt; 2372 2373 /* IRQs are off, so this synchronizes with smp_store_release */ 2374 ldt = READ_ONCE(current->active_mm->context.ldt); 2375 if (!ldt || idx >= ldt->nr_entries) 2376 return 0; 2377 2378 desc = &ldt->entries[idx]; 2379 #else 2380 return 0; 2381 #endif 2382 } else { 2383 if (idx >= GDT_ENTRIES) 2384 return 0; 2385 2386 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2387 } 2388 2389 return get_desc_base(desc); 2390 } 2391 2392 #ifdef CONFIG_IA32_EMULATION 2393 2394 #include <asm/compat.h> 2395 2396 static inline int 2397 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2398 { 2399 /* 32-bit process in 64-bit kernel. */ 2400 unsigned long ss_base, cs_base; 2401 struct stack_frame_ia32 frame; 2402 const void __user *fp; 2403 2404 if (!test_thread_flag(TIF_IA32)) 2405 return 0; 2406 2407 cs_base = get_segment_base(regs->cs); 2408 ss_base = get_segment_base(regs->ss); 2409 2410 fp = compat_ptr(ss_base + regs->bp); 2411 pagefault_disable(); 2412 while (entry->nr < entry->max_stack) { 2413 unsigned long bytes; 2414 frame.next_frame = 0; 2415 frame.return_address = 0; 2416 2417 if (!valid_user_frame(fp, sizeof(frame))) 2418 break; 2419 2420 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4); 2421 if (bytes != 0) 2422 break; 2423 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4); 2424 if (bytes != 0) 2425 break; 2426 2427 perf_callchain_store(entry, cs_base + frame.return_address); 2428 fp = compat_ptr(ss_base + frame.next_frame); 2429 } 2430 pagefault_enable(); 2431 return 1; 2432 } 2433 #else 2434 static inline int 2435 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2436 { 2437 return 0; 2438 } 2439 #endif 2440 2441 void 2442 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2443 { 2444 struct stack_frame frame; 2445 const unsigned long __user *fp; 2446 2447 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2448 /* TODO: We don't support guest os callchain now */ 2449 return; 2450 } 2451 2452 /* 2453 * We don't know what to do with VM86 stacks.. ignore them for now. 2454 */ 2455 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2456 return; 2457 2458 fp = (unsigned long __user *)regs->bp; 2459 2460 perf_callchain_store(entry, regs->ip); 2461 2462 if (!current->mm) 2463 return; 2464 2465 if (perf_callchain_user32(regs, entry)) 2466 return; 2467 2468 pagefault_disable(); 2469 while (entry->nr < entry->max_stack) { 2470 unsigned long bytes; 2471 2472 frame.next_frame = NULL; 2473 frame.return_address = 0; 2474 2475 if (!valid_user_frame(fp, sizeof(frame))) 2476 break; 2477 2478 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp)); 2479 if (bytes != 0) 2480 break; 2481 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp)); 2482 if (bytes != 0) 2483 break; 2484 2485 perf_callchain_store(entry, frame.return_address); 2486 fp = (void __user *)frame.next_frame; 2487 } 2488 pagefault_enable(); 2489 } 2490 2491 /* 2492 * Deal with code segment offsets for the various execution modes: 2493 * 2494 * VM86 - the good olde 16 bit days, where the linear address is 2495 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2496 * 2497 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2498 * to figure out what the 32bit base address is. 2499 * 2500 * X32 - has TIF_X32 set, but is running in x86_64 2501 * 2502 * X86_64 - CS,DS,SS,ES are all zero based. 2503 */ 2504 static unsigned long code_segment_base(struct pt_regs *regs) 2505 { 2506 /* 2507 * For IA32 we look at the GDT/LDT segment base to convert the 2508 * effective IP to a linear address. 2509 */ 2510 2511 #ifdef CONFIG_X86_32 2512 /* 2513 * If we are in VM86 mode, add the segment offset to convert to a 2514 * linear address. 2515 */ 2516 if (regs->flags & X86_VM_MASK) 2517 return 0x10 * regs->cs; 2518 2519 if (user_mode(regs) && regs->cs != __USER_CS) 2520 return get_segment_base(regs->cs); 2521 #else 2522 if (user_mode(regs) && !user_64bit_mode(regs) && 2523 regs->cs != __USER32_CS) 2524 return get_segment_base(regs->cs); 2525 #endif 2526 return 0; 2527 } 2528 2529 unsigned long perf_instruction_pointer(struct pt_regs *regs) 2530 { 2531 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 2532 return perf_guest_cbs->get_guest_ip(); 2533 2534 return regs->ip + code_segment_base(regs); 2535 } 2536 2537 unsigned long perf_misc_flags(struct pt_regs *regs) 2538 { 2539 int misc = 0; 2540 2541 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2542 if (perf_guest_cbs->is_user_mode()) 2543 misc |= PERF_RECORD_MISC_GUEST_USER; 2544 else 2545 misc |= PERF_RECORD_MISC_GUEST_KERNEL; 2546 } else { 2547 if (user_mode(regs)) 2548 misc |= PERF_RECORD_MISC_USER; 2549 else 2550 misc |= PERF_RECORD_MISC_KERNEL; 2551 } 2552 2553 if (regs->flags & PERF_EFLAGS_EXACT) 2554 misc |= PERF_RECORD_MISC_EXACT_IP; 2555 2556 return misc; 2557 } 2558 2559 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 2560 { 2561 cap->version = x86_pmu.version; 2562 cap->num_counters_gp = x86_pmu.num_counters; 2563 cap->num_counters_fixed = x86_pmu.num_counters_fixed; 2564 cap->bit_width_gp = x86_pmu.cntval_bits; 2565 cap->bit_width_fixed = x86_pmu.cntval_bits; 2566 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 2567 cap->events_mask_len = x86_pmu.events_mask_len; 2568 } 2569 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 2570