1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched/mm.h> 24 #include <linux/sched/clock.h> 25 #include <linux/uaccess.h> 26 #include <linux/slab.h> 27 #include <linux/cpu.h> 28 #include <linux/bitops.h> 29 #include <linux/device.h> 30 #include <linux/nospec.h> 31 #include <linux/static_call.h> 32 33 #include <asm/apic.h> 34 #include <asm/stacktrace.h> 35 #include <asm/nmi.h> 36 #include <asm/smp.h> 37 #include <asm/alternative.h> 38 #include <asm/mmu_context.h> 39 #include <asm/tlbflush.h> 40 #include <asm/timer.h> 41 #include <asm/desc.h> 42 #include <asm/ldt.h> 43 #include <asm/unwind.h> 44 #include <asm/uprobes.h> 45 #include <asm/ibt.h> 46 47 #include "perf_event.h" 48 49 struct x86_pmu x86_pmu __read_mostly; 50 static struct pmu pmu; 51 52 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 53 .enabled = 1, 54 .pmu = &pmu, 55 }; 56 57 DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key); 58 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key); 59 DEFINE_STATIC_KEY_FALSE(perf_is_hybrid); 60 61 /* 62 * This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined 63 * from just a typename, as opposed to an actual function. 64 */ 65 DEFINE_STATIC_CALL_NULL(x86_pmu_handle_irq, *x86_pmu.handle_irq); 66 DEFINE_STATIC_CALL_NULL(x86_pmu_disable_all, *x86_pmu.disable_all); 67 DEFINE_STATIC_CALL_NULL(x86_pmu_enable_all, *x86_pmu.enable_all); 68 DEFINE_STATIC_CALL_NULL(x86_pmu_enable, *x86_pmu.enable); 69 DEFINE_STATIC_CALL_NULL(x86_pmu_disable, *x86_pmu.disable); 70 71 DEFINE_STATIC_CALL_NULL(x86_pmu_assign, *x86_pmu.assign); 72 73 DEFINE_STATIC_CALL_NULL(x86_pmu_add, *x86_pmu.add); 74 DEFINE_STATIC_CALL_NULL(x86_pmu_del, *x86_pmu.del); 75 DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read); 76 77 DEFINE_STATIC_CALL_NULL(x86_pmu_set_period, *x86_pmu.set_period); 78 DEFINE_STATIC_CALL_NULL(x86_pmu_update, *x86_pmu.update); 79 DEFINE_STATIC_CALL_NULL(x86_pmu_limit_period, *x86_pmu.limit_period); 80 81 DEFINE_STATIC_CALL_NULL(x86_pmu_schedule_events, *x86_pmu.schedule_events); 82 DEFINE_STATIC_CALL_NULL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints); 83 DEFINE_STATIC_CALL_NULL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints); 84 85 DEFINE_STATIC_CALL_NULL(x86_pmu_start_scheduling, *x86_pmu.start_scheduling); 86 DEFINE_STATIC_CALL_NULL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling); 87 DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling); 88 89 DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task, *x86_pmu.sched_task); 90 DEFINE_STATIC_CALL_NULL(x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx); 91 92 DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs); 93 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases); 94 95 DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter); 96 97 /* 98 * This one is magic, it will get called even when PMU init fails (because 99 * there is no PMU), in which case it should simply return NULL. 100 */ 101 DEFINE_STATIC_CALL_RET0(x86_pmu_guest_get_msrs, *x86_pmu.guest_get_msrs); 102 103 u64 __read_mostly hw_cache_event_ids 104 [PERF_COUNT_HW_CACHE_MAX] 105 [PERF_COUNT_HW_CACHE_OP_MAX] 106 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 107 u64 __read_mostly hw_cache_extra_regs 108 [PERF_COUNT_HW_CACHE_MAX] 109 [PERF_COUNT_HW_CACHE_OP_MAX] 110 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 111 112 /* 113 * Propagate event elapsed time into the generic event. 114 * Can only be executed on the CPU where the event is active. 115 * Returns the delta events processed. 116 */ 117 u64 x86_perf_event_update(struct perf_event *event) 118 { 119 struct hw_perf_event *hwc = &event->hw; 120 int shift = 64 - x86_pmu.cntval_bits; 121 u64 prev_raw_count, new_raw_count; 122 u64 delta; 123 124 if (unlikely(!hwc->event_base)) 125 return 0; 126 127 /* 128 * Careful: an NMI might modify the previous event value. 129 * 130 * Our tactic to handle this is to first atomically read and 131 * exchange a new raw count - then add that new-prev delta 132 * count to the generic event atomically: 133 */ 134 prev_raw_count = local64_read(&hwc->prev_count); 135 do { 136 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 137 } while (!local64_try_cmpxchg(&hwc->prev_count, 138 &prev_raw_count, new_raw_count)); 139 140 /* 141 * Now we have the new raw value and have updated the prev 142 * timestamp already. We can now calculate the elapsed delta 143 * (event-)time and add that to the generic event. 144 * 145 * Careful, not all hw sign-extends above the physical width 146 * of the count. 147 */ 148 delta = (new_raw_count << shift) - (prev_raw_count << shift); 149 delta >>= shift; 150 151 local64_add(delta, &event->count); 152 local64_sub(delta, &hwc->period_left); 153 154 return new_raw_count; 155 } 156 157 /* 158 * Find and validate any extra registers to set up. 159 */ 160 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 161 { 162 struct extra_reg *extra_regs = hybrid(event->pmu, extra_regs); 163 struct hw_perf_event_extra *reg; 164 struct extra_reg *er; 165 166 reg = &event->hw.extra_reg; 167 168 if (!extra_regs) 169 return 0; 170 171 for (er = extra_regs; er->msr; er++) { 172 if (er->event != (config & er->config_mask)) 173 continue; 174 if (event->attr.config1 & ~er->valid_mask) 175 return -EINVAL; 176 /* Check if the extra msrs can be safely accessed*/ 177 if (!er->extra_msr_access) 178 return -ENXIO; 179 180 reg->idx = er->idx; 181 reg->config = event->attr.config1; 182 reg->reg = er->msr; 183 break; 184 } 185 return 0; 186 } 187 188 static atomic_t active_events; 189 static atomic_t pmc_refcount; 190 static DEFINE_MUTEX(pmc_reserve_mutex); 191 192 #ifdef CONFIG_X86_LOCAL_APIC 193 194 static inline u64 get_possible_counter_mask(void) 195 { 196 u64 cntr_mask = x86_pmu.cntr_mask64; 197 int i; 198 199 if (!is_hybrid()) 200 return cntr_mask; 201 202 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) 203 cntr_mask |= x86_pmu.hybrid_pmu[i].cntr_mask64; 204 205 return cntr_mask; 206 } 207 208 static bool reserve_pmc_hardware(void) 209 { 210 u64 cntr_mask = get_possible_counter_mask(); 211 int i, end; 212 213 for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) { 214 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 215 goto perfctr_fail; 216 } 217 218 for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) { 219 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 220 goto eventsel_fail; 221 } 222 223 return true; 224 225 eventsel_fail: 226 end = i; 227 for_each_set_bit(i, (unsigned long *)&cntr_mask, end) 228 release_evntsel_nmi(x86_pmu_config_addr(i)); 229 i = X86_PMC_IDX_MAX; 230 231 perfctr_fail: 232 end = i; 233 for_each_set_bit(i, (unsigned long *)&cntr_mask, end) 234 release_perfctr_nmi(x86_pmu_event_addr(i)); 235 236 return false; 237 } 238 239 static void release_pmc_hardware(void) 240 { 241 u64 cntr_mask = get_possible_counter_mask(); 242 int i; 243 244 for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) { 245 release_perfctr_nmi(x86_pmu_event_addr(i)); 246 release_evntsel_nmi(x86_pmu_config_addr(i)); 247 } 248 } 249 250 #else 251 252 static bool reserve_pmc_hardware(void) { return true; } 253 static void release_pmc_hardware(void) {} 254 255 #endif 256 257 bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask, 258 unsigned long *fixed_cntr_mask) 259 { 260 u64 val, val_fail = -1, val_new= ~0; 261 int i, reg, reg_fail = -1, ret = 0; 262 int bios_fail = 0; 263 int reg_safe = -1; 264 265 /* 266 * Check to see if the BIOS enabled any of the counters, if so 267 * complain and bail. 268 */ 269 for_each_set_bit(i, cntr_mask, X86_PMC_IDX_MAX) { 270 reg = x86_pmu_config_addr(i); 271 ret = rdmsrl_safe(reg, &val); 272 if (ret) 273 goto msr_fail; 274 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 275 bios_fail = 1; 276 val_fail = val; 277 reg_fail = reg; 278 } else { 279 reg_safe = i; 280 } 281 } 282 283 if (*(u64 *)fixed_cntr_mask) { 284 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 285 ret = rdmsrl_safe(reg, &val); 286 if (ret) 287 goto msr_fail; 288 for_each_set_bit(i, fixed_cntr_mask, X86_PMC_IDX_MAX) { 289 if (fixed_counter_disabled(i, pmu)) 290 continue; 291 if (val & (0x03ULL << i*4)) { 292 bios_fail = 1; 293 val_fail = val; 294 reg_fail = reg; 295 } 296 } 297 } 298 299 /* 300 * If all the counters are enabled, the below test will always 301 * fail. The tools will also become useless in this scenario. 302 * Just fail and disable the hardware counters. 303 */ 304 305 if (reg_safe == -1) { 306 reg = reg_safe; 307 goto msr_fail; 308 } 309 310 /* 311 * Read the current value, change it and read it back to see if it 312 * matches, this is needed to detect certain hardware emulators 313 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 314 */ 315 reg = x86_pmu_event_addr(reg_safe); 316 if (rdmsrl_safe(reg, &val)) 317 goto msr_fail; 318 val ^= 0xffffUL; 319 ret = wrmsrl_safe(reg, val); 320 ret |= rdmsrl_safe(reg, &val_new); 321 if (ret || val != val_new) 322 goto msr_fail; 323 324 /* 325 * We still allow the PMU driver to operate: 326 */ 327 if (bios_fail) { 328 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 329 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 330 reg_fail, val_fail); 331 } 332 333 return true; 334 335 msr_fail: 336 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 337 pr_cont("PMU not available due to virtualization, using software events only.\n"); 338 } else { 339 pr_cont("Broken PMU hardware detected, using software events only.\n"); 340 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 341 reg, val_new); 342 } 343 344 return false; 345 } 346 347 static void hw_perf_event_destroy(struct perf_event *event) 348 { 349 x86_release_hardware(); 350 atomic_dec(&active_events); 351 } 352 353 void hw_perf_lbr_event_destroy(struct perf_event *event) 354 { 355 hw_perf_event_destroy(event); 356 357 /* undo the lbr/bts event accounting */ 358 x86_del_exclusive(x86_lbr_exclusive_lbr); 359 } 360 361 static inline int x86_pmu_initialized(void) 362 { 363 return x86_pmu.handle_irq != NULL; 364 } 365 366 static inline int 367 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 368 { 369 struct perf_event_attr *attr = &event->attr; 370 unsigned int cache_type, cache_op, cache_result; 371 u64 config, val; 372 373 config = attr->config; 374 375 cache_type = (config >> 0) & 0xff; 376 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 377 return -EINVAL; 378 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX); 379 380 cache_op = (config >> 8) & 0xff; 381 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 382 return -EINVAL; 383 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX); 384 385 cache_result = (config >> 16) & 0xff; 386 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 387 return -EINVAL; 388 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX); 389 390 val = hybrid_var(event->pmu, hw_cache_event_ids)[cache_type][cache_op][cache_result]; 391 if (val == 0) 392 return -ENOENT; 393 394 if (val == -1) 395 return -EINVAL; 396 397 hwc->config |= val; 398 attr->config1 = hybrid_var(event->pmu, hw_cache_extra_regs)[cache_type][cache_op][cache_result]; 399 return x86_pmu_extra_regs(val, event); 400 } 401 402 int x86_reserve_hardware(void) 403 { 404 int err = 0; 405 406 if (!atomic_inc_not_zero(&pmc_refcount)) { 407 mutex_lock(&pmc_reserve_mutex); 408 if (atomic_read(&pmc_refcount) == 0) { 409 if (!reserve_pmc_hardware()) { 410 err = -EBUSY; 411 } else { 412 reserve_ds_buffers(); 413 reserve_lbr_buffers(); 414 } 415 } 416 if (!err) 417 atomic_inc(&pmc_refcount); 418 mutex_unlock(&pmc_reserve_mutex); 419 } 420 421 return err; 422 } 423 424 void x86_release_hardware(void) 425 { 426 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 427 release_pmc_hardware(); 428 release_ds_buffers(); 429 release_lbr_buffers(); 430 mutex_unlock(&pmc_reserve_mutex); 431 } 432 } 433 434 /* 435 * Check if we can create event of a certain type (that no conflicting events 436 * are present). 437 */ 438 int x86_add_exclusive(unsigned int what) 439 { 440 int i; 441 442 /* 443 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 444 * LBR and BTS are still mutually exclusive. 445 */ 446 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 447 goto out; 448 449 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 450 mutex_lock(&pmc_reserve_mutex); 451 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 452 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 453 goto fail_unlock; 454 } 455 atomic_inc(&x86_pmu.lbr_exclusive[what]); 456 mutex_unlock(&pmc_reserve_mutex); 457 } 458 459 out: 460 atomic_inc(&active_events); 461 return 0; 462 463 fail_unlock: 464 mutex_unlock(&pmc_reserve_mutex); 465 return -EBUSY; 466 } 467 468 void x86_del_exclusive(unsigned int what) 469 { 470 atomic_dec(&active_events); 471 472 /* 473 * See the comment in x86_add_exclusive(). 474 */ 475 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 476 return; 477 478 atomic_dec(&x86_pmu.lbr_exclusive[what]); 479 } 480 481 int x86_setup_perfctr(struct perf_event *event) 482 { 483 struct perf_event_attr *attr = &event->attr; 484 struct hw_perf_event *hwc = &event->hw; 485 u64 config; 486 487 if (!is_sampling_event(event)) { 488 hwc->sample_period = x86_pmu.max_period; 489 hwc->last_period = hwc->sample_period; 490 local64_set(&hwc->period_left, hwc->sample_period); 491 } 492 493 if (attr->type == event->pmu->type) 494 return x86_pmu_extra_regs(event->attr.config, event); 495 496 if (attr->type == PERF_TYPE_HW_CACHE) 497 return set_ext_hw_attr(hwc, event); 498 499 if (attr->config >= x86_pmu.max_events) 500 return -EINVAL; 501 502 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events); 503 504 /* 505 * The generic map: 506 */ 507 config = x86_pmu.event_map(attr->config); 508 509 if (config == 0) 510 return -ENOENT; 511 512 if (config == -1LL) 513 return -EINVAL; 514 515 hwc->config |= config; 516 517 return 0; 518 } 519 520 /* 521 * check that branch_sample_type is compatible with 522 * settings needed for precise_ip > 1 which implies 523 * using the LBR to capture ALL taken branches at the 524 * priv levels of the measurement 525 */ 526 static inline int precise_br_compat(struct perf_event *event) 527 { 528 u64 m = event->attr.branch_sample_type; 529 u64 b = 0; 530 531 /* must capture all branches */ 532 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 533 return 0; 534 535 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 536 537 if (!event->attr.exclude_user) 538 b |= PERF_SAMPLE_BRANCH_USER; 539 540 if (!event->attr.exclude_kernel) 541 b |= PERF_SAMPLE_BRANCH_KERNEL; 542 543 /* 544 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 545 */ 546 547 return m == b; 548 } 549 550 int x86_pmu_max_precise(void) 551 { 552 int precise = 0; 553 554 /* Support for constant skid */ 555 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 556 precise++; 557 558 /* Support for IP fixup */ 559 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 560 precise++; 561 562 if (x86_pmu.pebs_prec_dist) 563 precise++; 564 } 565 return precise; 566 } 567 568 int x86_pmu_hw_config(struct perf_event *event) 569 { 570 if (event->attr.precise_ip) { 571 int precise = x86_pmu_max_precise(); 572 573 if (event->attr.precise_ip > precise) 574 return -EOPNOTSUPP; 575 576 /* There's no sense in having PEBS for non sampling events: */ 577 if (!is_sampling_event(event)) 578 return -EINVAL; 579 } 580 /* 581 * check that PEBS LBR correction does not conflict with 582 * whatever the user is asking with attr->branch_sample_type 583 */ 584 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 585 u64 *br_type = &event->attr.branch_sample_type; 586 587 if (has_branch_stack(event)) { 588 if (!precise_br_compat(event)) 589 return -EOPNOTSUPP; 590 591 /* branch_sample_type is compatible */ 592 593 } else { 594 /* 595 * user did not specify branch_sample_type 596 * 597 * For PEBS fixups, we capture all 598 * the branches at the priv level of the 599 * event. 600 */ 601 *br_type = PERF_SAMPLE_BRANCH_ANY; 602 603 if (!event->attr.exclude_user) 604 *br_type |= PERF_SAMPLE_BRANCH_USER; 605 606 if (!event->attr.exclude_kernel) 607 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 608 } 609 } 610 611 if (branch_sample_call_stack(event)) 612 event->attach_state |= PERF_ATTACH_TASK_DATA; 613 614 /* 615 * Generate PMC IRQs: 616 * (keep 'enabled' bit clear for now) 617 */ 618 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 619 620 /* 621 * Count user and OS events unless requested not to 622 */ 623 if (!event->attr.exclude_user) 624 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 625 if (!event->attr.exclude_kernel) 626 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 627 628 if (event->attr.type == event->pmu->type) 629 event->hw.config |= x86_pmu_get_event_config(event); 630 631 if (event->attr.sample_period && x86_pmu.limit_period) { 632 s64 left = event->attr.sample_period; 633 x86_pmu.limit_period(event, &left); 634 if (left > event->attr.sample_period) 635 return -EINVAL; 636 } 637 638 /* sample_regs_user never support XMM registers */ 639 if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) 640 return -EINVAL; 641 /* 642 * Besides the general purpose registers, XMM registers may 643 * be collected in PEBS on some platforms, e.g. Icelake 644 */ 645 if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) { 646 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) 647 return -EINVAL; 648 649 if (!event->attr.precise_ip) 650 return -EINVAL; 651 } 652 653 return x86_setup_perfctr(event); 654 } 655 656 /* 657 * Setup the hardware configuration for a given attr_type 658 */ 659 static int __x86_pmu_event_init(struct perf_event *event) 660 { 661 int err; 662 663 if (!x86_pmu_initialized()) 664 return -ENODEV; 665 666 err = x86_reserve_hardware(); 667 if (err) 668 return err; 669 670 atomic_inc(&active_events); 671 event->destroy = hw_perf_event_destroy; 672 673 event->hw.idx = -1; 674 event->hw.last_cpu = -1; 675 event->hw.last_tag = ~0ULL; 676 677 /* mark unused */ 678 event->hw.extra_reg.idx = EXTRA_REG_NONE; 679 event->hw.branch_reg.idx = EXTRA_REG_NONE; 680 681 return x86_pmu.hw_config(event); 682 } 683 684 void x86_pmu_disable_all(void) 685 { 686 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 687 int idx; 688 689 for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { 690 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 691 u64 val; 692 693 if (!test_bit(idx, cpuc->active_mask)) 694 continue; 695 rdmsrl(x86_pmu_config_addr(idx), val); 696 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 697 continue; 698 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 699 wrmsrl(x86_pmu_config_addr(idx), val); 700 if (is_counter_pair(hwc)) 701 wrmsrl(x86_pmu_config_addr(idx + 1), 0); 702 } 703 } 704 705 struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data) 706 { 707 return static_call(x86_pmu_guest_get_msrs)(nr, data); 708 } 709 EXPORT_SYMBOL_GPL(perf_guest_get_msrs); 710 711 /* 712 * There may be PMI landing after enabled=0. The PMI hitting could be before or 713 * after disable_all. 714 * 715 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 716 * It will not be re-enabled in the NMI handler again, because enabled=0. After 717 * handling the NMI, disable_all will be called, which will not change the 718 * state either. If PMI hits after disable_all, the PMU is already disabled 719 * before entering NMI handler. The NMI handler will not change the state 720 * either. 721 * 722 * So either situation is harmless. 723 */ 724 static void x86_pmu_disable(struct pmu *pmu) 725 { 726 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 727 728 if (!x86_pmu_initialized()) 729 return; 730 731 if (!cpuc->enabled) 732 return; 733 734 cpuc->n_added = 0; 735 cpuc->enabled = 0; 736 barrier(); 737 738 static_call(x86_pmu_disable_all)(); 739 } 740 741 void x86_pmu_enable_all(int added) 742 { 743 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 744 int idx; 745 746 for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { 747 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 748 749 if (!test_bit(idx, cpuc->active_mask)) 750 continue; 751 752 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 753 } 754 } 755 756 static inline int is_x86_event(struct perf_event *event) 757 { 758 int i; 759 760 if (!is_hybrid()) 761 return event->pmu == &pmu; 762 763 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { 764 if (event->pmu == &x86_pmu.hybrid_pmu[i].pmu) 765 return true; 766 } 767 768 return false; 769 } 770 771 struct pmu *x86_get_pmu(unsigned int cpu) 772 { 773 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 774 775 /* 776 * All CPUs of the hybrid type have been offline. 777 * The x86_get_pmu() should not be invoked. 778 */ 779 if (WARN_ON_ONCE(!cpuc->pmu)) 780 return &pmu; 781 782 return cpuc->pmu; 783 } 784 /* 785 * Event scheduler state: 786 * 787 * Assign events iterating over all events and counters, beginning 788 * with events with least weights first. Keep the current iterator 789 * state in struct sched_state. 790 */ 791 struct sched_state { 792 int weight; 793 int event; /* event index */ 794 int counter; /* counter index */ 795 int unassigned; /* number of events to be assigned left */ 796 int nr_gp; /* number of GP counters used */ 797 u64 used; 798 }; 799 800 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 801 #define SCHED_STATES_MAX 2 802 803 struct perf_sched { 804 int max_weight; 805 int max_events; 806 int max_gp; 807 int saved_states; 808 struct event_constraint **constraints; 809 struct sched_state state; 810 struct sched_state saved[SCHED_STATES_MAX]; 811 }; 812 813 /* 814 * Initialize iterator that runs through all events and counters. 815 */ 816 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 817 int num, int wmin, int wmax, int gpmax) 818 { 819 int idx; 820 821 memset(sched, 0, sizeof(*sched)); 822 sched->max_events = num; 823 sched->max_weight = wmax; 824 sched->max_gp = gpmax; 825 sched->constraints = constraints; 826 827 for (idx = 0; idx < num; idx++) { 828 if (constraints[idx]->weight == wmin) 829 break; 830 } 831 832 sched->state.event = idx; /* start with min weight */ 833 sched->state.weight = wmin; 834 sched->state.unassigned = num; 835 } 836 837 static void perf_sched_save_state(struct perf_sched *sched) 838 { 839 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 840 return; 841 842 sched->saved[sched->saved_states] = sched->state; 843 sched->saved_states++; 844 } 845 846 static bool perf_sched_restore_state(struct perf_sched *sched) 847 { 848 if (!sched->saved_states) 849 return false; 850 851 sched->saved_states--; 852 sched->state = sched->saved[sched->saved_states]; 853 854 /* this assignment didn't work out */ 855 /* XXX broken vs EVENT_PAIR */ 856 sched->state.used &= ~BIT_ULL(sched->state.counter); 857 858 /* try the next one */ 859 sched->state.counter++; 860 861 return true; 862 } 863 864 /* 865 * Select a counter for the current event to schedule. Return true on 866 * success. 867 */ 868 static bool __perf_sched_find_counter(struct perf_sched *sched) 869 { 870 struct event_constraint *c; 871 int idx; 872 873 if (!sched->state.unassigned) 874 return false; 875 876 if (sched->state.event >= sched->max_events) 877 return false; 878 879 c = sched->constraints[sched->state.event]; 880 /* Prefer fixed purpose counters */ 881 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 882 idx = INTEL_PMC_IDX_FIXED; 883 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 884 u64 mask = BIT_ULL(idx); 885 886 if (sched->state.used & mask) 887 continue; 888 889 sched->state.used |= mask; 890 goto done; 891 } 892 } 893 894 /* Grab the first unused counter starting with idx */ 895 idx = sched->state.counter; 896 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 897 u64 mask = BIT_ULL(idx); 898 899 if (c->flags & PERF_X86_EVENT_PAIR) 900 mask |= mask << 1; 901 902 if (sched->state.used & mask) 903 continue; 904 905 if (sched->state.nr_gp++ >= sched->max_gp) 906 return false; 907 908 sched->state.used |= mask; 909 goto done; 910 } 911 912 return false; 913 914 done: 915 sched->state.counter = idx; 916 917 if (c->overlap) 918 perf_sched_save_state(sched); 919 920 return true; 921 } 922 923 static bool perf_sched_find_counter(struct perf_sched *sched) 924 { 925 while (!__perf_sched_find_counter(sched)) { 926 if (!perf_sched_restore_state(sched)) 927 return false; 928 } 929 930 return true; 931 } 932 933 /* 934 * Go through all unassigned events and find the next one to schedule. 935 * Take events with the least weight first. Return true on success. 936 */ 937 static bool perf_sched_next_event(struct perf_sched *sched) 938 { 939 struct event_constraint *c; 940 941 if (!sched->state.unassigned || !--sched->state.unassigned) 942 return false; 943 944 do { 945 /* next event */ 946 sched->state.event++; 947 if (sched->state.event >= sched->max_events) { 948 /* next weight */ 949 sched->state.event = 0; 950 sched->state.weight++; 951 if (sched->state.weight > sched->max_weight) 952 return false; 953 } 954 c = sched->constraints[sched->state.event]; 955 } while (c->weight != sched->state.weight); 956 957 sched->state.counter = 0; /* start with first counter */ 958 959 return true; 960 } 961 962 /* 963 * Assign a counter for each event. 964 */ 965 int perf_assign_events(struct event_constraint **constraints, int n, 966 int wmin, int wmax, int gpmax, int *assign) 967 { 968 struct perf_sched sched; 969 970 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 971 972 do { 973 if (!perf_sched_find_counter(&sched)) 974 break; /* failed */ 975 if (assign) 976 assign[sched.state.event] = sched.state.counter; 977 } while (perf_sched_next_event(&sched)); 978 979 return sched.state.unassigned; 980 } 981 EXPORT_SYMBOL_GPL(perf_assign_events); 982 983 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 984 { 985 struct event_constraint *c; 986 struct perf_event *e; 987 int n0, i, wmin, wmax, unsched = 0; 988 struct hw_perf_event *hwc; 989 u64 used_mask = 0; 990 991 /* 992 * Compute the number of events already present; see x86_pmu_add(), 993 * validate_group() and x86_pmu_commit_txn(). For the former two 994 * cpuc->n_events hasn't been updated yet, while for the latter 995 * cpuc->n_txn contains the number of events added in the current 996 * transaction. 997 */ 998 n0 = cpuc->n_events; 999 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1000 n0 -= cpuc->n_txn; 1001 1002 static_call_cond(x86_pmu_start_scheduling)(cpuc); 1003 1004 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 1005 c = cpuc->event_constraint[i]; 1006 1007 /* 1008 * Previously scheduled events should have a cached constraint, 1009 * while new events should not have one. 1010 */ 1011 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0)); 1012 1013 /* 1014 * Request constraints for new events; or for those events that 1015 * have a dynamic constraint -- for those the constraint can 1016 * change due to external factors (sibling state, allow_tfa). 1017 */ 1018 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) { 1019 c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]); 1020 cpuc->event_constraint[i] = c; 1021 } 1022 1023 wmin = min(wmin, c->weight); 1024 wmax = max(wmax, c->weight); 1025 } 1026 1027 /* 1028 * fastpath, try to reuse previous register 1029 */ 1030 for (i = 0; i < n; i++) { 1031 u64 mask; 1032 1033 hwc = &cpuc->event_list[i]->hw; 1034 c = cpuc->event_constraint[i]; 1035 1036 /* never assigned */ 1037 if (hwc->idx == -1) 1038 break; 1039 1040 /* constraint still honored */ 1041 if (!test_bit(hwc->idx, c->idxmsk)) 1042 break; 1043 1044 mask = BIT_ULL(hwc->idx); 1045 if (is_counter_pair(hwc)) 1046 mask |= mask << 1; 1047 1048 /* not already used */ 1049 if (used_mask & mask) 1050 break; 1051 1052 used_mask |= mask; 1053 1054 if (assign) 1055 assign[i] = hwc->idx; 1056 } 1057 1058 /* slow path */ 1059 if (i != n) { 1060 int gpmax = x86_pmu_max_num_counters(cpuc->pmu); 1061 1062 /* 1063 * Do not allow scheduling of more than half the available 1064 * generic counters. 1065 * 1066 * This helps avoid counter starvation of sibling thread by 1067 * ensuring at most half the counters cannot be in exclusive 1068 * mode. There is no designated counters for the limits. Any 1069 * N/2 counters can be used. This helps with events with 1070 * specific counter constraints. 1071 */ 1072 if (is_ht_workaround_enabled() && !cpuc->is_fake && 1073 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 1074 gpmax /= 2; 1075 1076 /* 1077 * Reduce the amount of available counters to allow fitting 1078 * the extra Merge events needed by large increment events. 1079 */ 1080 if (x86_pmu.flags & PMU_FL_PAIR) { 1081 gpmax -= cpuc->n_pair; 1082 WARN_ON(gpmax <= 0); 1083 } 1084 1085 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 1086 wmax, gpmax, assign); 1087 } 1088 1089 /* 1090 * In case of success (unsched = 0), mark events as committed, 1091 * so we do not put_constraint() in case new events are added 1092 * and fail to be scheduled 1093 * 1094 * We invoke the lower level commit callback to lock the resource 1095 * 1096 * We do not need to do all of this in case we are called to 1097 * validate an event group (assign == NULL) 1098 */ 1099 if (!unsched && assign) { 1100 for (i = 0; i < n; i++) 1101 static_call_cond(x86_pmu_commit_scheduling)(cpuc, i, assign[i]); 1102 } else { 1103 for (i = n0; i < n; i++) { 1104 e = cpuc->event_list[i]; 1105 1106 /* 1107 * release events that failed scheduling 1108 */ 1109 static_call_cond(x86_pmu_put_event_constraints)(cpuc, e); 1110 1111 cpuc->event_constraint[i] = NULL; 1112 } 1113 } 1114 1115 static_call_cond(x86_pmu_stop_scheduling)(cpuc); 1116 1117 return unsched ? -EINVAL : 0; 1118 } 1119 1120 static int add_nr_metric_event(struct cpu_hw_events *cpuc, 1121 struct perf_event *event) 1122 { 1123 if (is_metric_event(event)) { 1124 if (cpuc->n_metric == INTEL_TD_METRIC_NUM) 1125 return -EINVAL; 1126 cpuc->n_metric++; 1127 cpuc->n_txn_metric++; 1128 } 1129 1130 return 0; 1131 } 1132 1133 static void del_nr_metric_event(struct cpu_hw_events *cpuc, 1134 struct perf_event *event) 1135 { 1136 if (is_metric_event(event)) 1137 cpuc->n_metric--; 1138 } 1139 1140 static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event, 1141 int max_count, int n) 1142 { 1143 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap); 1144 1145 if (intel_cap.perf_metrics && add_nr_metric_event(cpuc, event)) 1146 return -EINVAL; 1147 1148 if (n >= max_count + cpuc->n_metric) 1149 return -EINVAL; 1150 1151 cpuc->event_list[n] = event; 1152 if (is_counter_pair(&event->hw)) { 1153 cpuc->n_pair++; 1154 cpuc->n_txn_pair++; 1155 } 1156 1157 return 0; 1158 } 1159 1160 /* 1161 * dogrp: true if must collect siblings events (group) 1162 * returns total number of events and error code 1163 */ 1164 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 1165 { 1166 struct perf_event *event; 1167 int n, max_count; 1168 1169 max_count = x86_pmu_num_counters(cpuc->pmu) + x86_pmu_num_counters_fixed(cpuc->pmu); 1170 1171 /* current number of events already accepted */ 1172 n = cpuc->n_events; 1173 if (!cpuc->n_events) 1174 cpuc->pebs_output = 0; 1175 1176 if (!cpuc->is_fake && leader->attr.precise_ip) { 1177 /* 1178 * For PEBS->PT, if !aux_event, the group leader (PT) went 1179 * away, the group was broken down and this singleton event 1180 * can't schedule any more. 1181 */ 1182 if (is_pebs_pt(leader) && !leader->aux_event) 1183 return -EINVAL; 1184 1185 /* 1186 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS 1187 */ 1188 if (cpuc->pebs_output && 1189 cpuc->pebs_output != is_pebs_pt(leader) + 1) 1190 return -EINVAL; 1191 1192 cpuc->pebs_output = is_pebs_pt(leader) + 1; 1193 } 1194 1195 if (is_x86_event(leader)) { 1196 if (collect_event(cpuc, leader, max_count, n)) 1197 return -EINVAL; 1198 n++; 1199 } 1200 1201 if (!dogrp) 1202 return n; 1203 1204 for_each_sibling_event(event, leader) { 1205 if (!is_x86_event(event) || event->state <= PERF_EVENT_STATE_OFF) 1206 continue; 1207 1208 if (collect_event(cpuc, event, max_count, n)) 1209 return -EINVAL; 1210 1211 n++; 1212 } 1213 return n; 1214 } 1215 1216 static inline void x86_assign_hw_event(struct perf_event *event, 1217 struct cpu_hw_events *cpuc, int i) 1218 { 1219 struct hw_perf_event *hwc = &event->hw; 1220 int idx; 1221 1222 idx = hwc->idx = cpuc->assign[i]; 1223 hwc->last_cpu = smp_processor_id(); 1224 hwc->last_tag = ++cpuc->tags[i]; 1225 1226 static_call_cond(x86_pmu_assign)(event, idx); 1227 1228 switch (hwc->idx) { 1229 case INTEL_PMC_IDX_FIXED_BTS: 1230 case INTEL_PMC_IDX_FIXED_VLBR: 1231 hwc->config_base = 0; 1232 hwc->event_base = 0; 1233 break; 1234 1235 case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END: 1236 /* All the metric events are mapped onto the fixed counter 3. */ 1237 idx = INTEL_PMC_IDX_FIXED_SLOTS; 1238 fallthrough; 1239 case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1: 1240 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1241 hwc->event_base = x86_pmu_fixed_ctr_addr(idx - INTEL_PMC_IDX_FIXED); 1242 hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) | 1243 INTEL_PMC_FIXED_RDPMC_BASE; 1244 break; 1245 1246 default: 1247 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1248 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1249 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1250 break; 1251 } 1252 } 1253 1254 /** 1255 * x86_perf_rdpmc_index - Return PMC counter used for event 1256 * @event: the perf_event to which the PMC counter was assigned 1257 * 1258 * The counter assigned to this performance event may change if interrupts 1259 * are enabled. This counter should thus never be used while interrupts are 1260 * enabled. Before this function is used to obtain the assigned counter the 1261 * event should be checked for validity using, for example, 1262 * perf_event_read_local(), within the same interrupt disabled section in 1263 * which this counter is planned to be used. 1264 * 1265 * Return: The index of the performance monitoring counter assigned to 1266 * @perf_event. 1267 */ 1268 int x86_perf_rdpmc_index(struct perf_event *event) 1269 { 1270 lockdep_assert_irqs_disabled(); 1271 1272 return event->hw.event_base_rdpmc; 1273 } 1274 1275 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1276 struct cpu_hw_events *cpuc, 1277 int i) 1278 { 1279 return hwc->idx == cpuc->assign[i] && 1280 hwc->last_cpu == smp_processor_id() && 1281 hwc->last_tag == cpuc->tags[i]; 1282 } 1283 1284 static void x86_pmu_start(struct perf_event *event, int flags); 1285 1286 static void x86_pmu_enable(struct pmu *pmu) 1287 { 1288 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1289 struct perf_event *event; 1290 struct hw_perf_event *hwc; 1291 int i, added = cpuc->n_added; 1292 1293 if (!x86_pmu_initialized()) 1294 return; 1295 1296 if (cpuc->enabled) 1297 return; 1298 1299 if (cpuc->n_added) { 1300 int n_running = cpuc->n_events - cpuc->n_added; 1301 /* 1302 * apply assignment obtained either from 1303 * hw_perf_group_sched_in() or x86_pmu_enable() 1304 * 1305 * step1: save events moving to new counters 1306 */ 1307 for (i = 0; i < n_running; i++) { 1308 event = cpuc->event_list[i]; 1309 hwc = &event->hw; 1310 1311 /* 1312 * we can avoid reprogramming counter if: 1313 * - assigned same counter as last time 1314 * - running on same CPU as last time 1315 * - no other event has used the counter since 1316 */ 1317 if (hwc->idx == -1 || 1318 match_prev_assignment(hwc, cpuc, i)) 1319 continue; 1320 1321 /* 1322 * Ensure we don't accidentally enable a stopped 1323 * counter simply because we rescheduled. 1324 */ 1325 if (hwc->state & PERF_HES_STOPPED) 1326 hwc->state |= PERF_HES_ARCH; 1327 1328 x86_pmu_stop(event, PERF_EF_UPDATE); 1329 } 1330 1331 /* 1332 * step2: reprogram moved events into new counters 1333 */ 1334 for (i = 0; i < cpuc->n_events; i++) { 1335 event = cpuc->event_list[i]; 1336 hwc = &event->hw; 1337 1338 if (!match_prev_assignment(hwc, cpuc, i)) 1339 x86_assign_hw_event(event, cpuc, i); 1340 else if (i < n_running) 1341 continue; 1342 1343 if (hwc->state & PERF_HES_ARCH) 1344 continue; 1345 1346 /* 1347 * if cpuc->enabled = 0, then no wrmsr as 1348 * per x86_pmu_enable_event() 1349 */ 1350 x86_pmu_start(event, PERF_EF_RELOAD); 1351 } 1352 cpuc->n_added = 0; 1353 perf_events_lapic_init(); 1354 } 1355 1356 cpuc->enabled = 1; 1357 barrier(); 1358 1359 static_call(x86_pmu_enable_all)(added); 1360 } 1361 1362 DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1363 1364 /* 1365 * Set the next IRQ period, based on the hwc->period_left value. 1366 * To be called with the event disabled in hw: 1367 */ 1368 int x86_perf_event_set_period(struct perf_event *event) 1369 { 1370 struct hw_perf_event *hwc = &event->hw; 1371 s64 left = local64_read(&hwc->period_left); 1372 s64 period = hwc->sample_period; 1373 int ret = 0, idx = hwc->idx; 1374 1375 if (unlikely(!hwc->event_base)) 1376 return 0; 1377 1378 /* 1379 * If we are way outside a reasonable range then just skip forward: 1380 */ 1381 if (unlikely(left <= -period)) { 1382 left = period; 1383 local64_set(&hwc->period_left, left); 1384 hwc->last_period = period; 1385 ret = 1; 1386 } 1387 1388 if (unlikely(left <= 0)) { 1389 left += period; 1390 local64_set(&hwc->period_left, left); 1391 hwc->last_period = period; 1392 ret = 1; 1393 } 1394 /* 1395 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1396 */ 1397 if (unlikely(left < 2)) 1398 left = 2; 1399 1400 if (left > x86_pmu.max_period) 1401 left = x86_pmu.max_period; 1402 1403 static_call_cond(x86_pmu_limit_period)(event, &left); 1404 1405 this_cpu_write(pmc_prev_left[idx], left); 1406 1407 /* 1408 * The hw event starts counting from this event offset, 1409 * mark it to be able to extra future deltas: 1410 */ 1411 local64_set(&hwc->prev_count, (u64)-left); 1412 1413 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1414 1415 /* 1416 * Sign extend the Merge event counter's upper 16 bits since 1417 * we currently declare a 48-bit counter width 1418 */ 1419 if (is_counter_pair(hwc)) 1420 wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff); 1421 1422 perf_event_update_userpage(event); 1423 1424 return ret; 1425 } 1426 1427 void x86_pmu_enable_event(struct perf_event *event) 1428 { 1429 if (__this_cpu_read(cpu_hw_events.enabled)) 1430 __x86_pmu_enable_event(&event->hw, 1431 ARCH_PERFMON_EVENTSEL_ENABLE); 1432 } 1433 1434 /* 1435 * Add a single event to the PMU. 1436 * 1437 * The event is added to the group of enabled events 1438 * but only if it can be scheduled with existing events. 1439 */ 1440 static int x86_pmu_add(struct perf_event *event, int flags) 1441 { 1442 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1443 struct hw_perf_event *hwc; 1444 int assign[X86_PMC_IDX_MAX]; 1445 int n, n0, ret; 1446 1447 hwc = &event->hw; 1448 1449 n0 = cpuc->n_events; 1450 ret = n = collect_events(cpuc, event, false); 1451 if (ret < 0) 1452 goto out; 1453 1454 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1455 if (!(flags & PERF_EF_START)) 1456 hwc->state |= PERF_HES_ARCH; 1457 1458 /* 1459 * If group events scheduling transaction was started, 1460 * skip the schedulability test here, it will be performed 1461 * at commit time (->commit_txn) as a whole. 1462 * 1463 * If commit fails, we'll call ->del() on all events 1464 * for which ->add() was called. 1465 */ 1466 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1467 goto done_collect; 1468 1469 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign); 1470 if (ret) 1471 goto out; 1472 /* 1473 * copy new assignment, now we know it is possible 1474 * will be used by hw_perf_enable() 1475 */ 1476 memcpy(cpuc->assign, assign, n*sizeof(int)); 1477 1478 done_collect: 1479 /* 1480 * Commit the collect_events() state. See x86_pmu_del() and 1481 * x86_pmu_*_txn(). 1482 */ 1483 cpuc->n_events = n; 1484 cpuc->n_added += n - n0; 1485 cpuc->n_txn += n - n0; 1486 1487 /* 1488 * This is before x86_pmu_enable() will call x86_pmu_start(), 1489 * so we enable LBRs before an event needs them etc.. 1490 */ 1491 static_call_cond(x86_pmu_add)(event); 1492 1493 ret = 0; 1494 out: 1495 return ret; 1496 } 1497 1498 static void x86_pmu_start(struct perf_event *event, int flags) 1499 { 1500 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1501 int idx = event->hw.idx; 1502 1503 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1504 return; 1505 1506 if (WARN_ON_ONCE(idx == -1)) 1507 return; 1508 1509 if (flags & PERF_EF_RELOAD) { 1510 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1511 static_call(x86_pmu_set_period)(event); 1512 } 1513 1514 event->hw.state = 0; 1515 1516 cpuc->events[idx] = event; 1517 __set_bit(idx, cpuc->active_mask); 1518 static_call(x86_pmu_enable)(event); 1519 perf_event_update_userpage(event); 1520 } 1521 1522 void perf_event_print_debug(void) 1523 { 1524 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1525 unsigned long *cntr_mask, *fixed_cntr_mask; 1526 struct event_constraint *pebs_constraints; 1527 struct cpu_hw_events *cpuc; 1528 u64 pebs, debugctl; 1529 int cpu, idx; 1530 1531 guard(irqsave)(); 1532 1533 cpu = smp_processor_id(); 1534 cpuc = &per_cpu(cpu_hw_events, cpu); 1535 cntr_mask = hybrid(cpuc->pmu, cntr_mask); 1536 fixed_cntr_mask = hybrid(cpuc->pmu, fixed_cntr_mask); 1537 pebs_constraints = hybrid(cpuc->pmu, pebs_constraints); 1538 1539 if (!*(u64 *)cntr_mask) 1540 return; 1541 1542 if (x86_pmu.version >= 2) { 1543 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1544 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1545 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1546 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1547 1548 pr_info("\n"); 1549 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1550 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1551 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1552 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1553 if (pebs_constraints) { 1554 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1555 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1556 } 1557 if (x86_pmu.lbr_nr) { 1558 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1559 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1560 } 1561 } 1562 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1563 1564 for_each_set_bit(idx, cntr_mask, X86_PMC_IDX_MAX) { 1565 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1566 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1567 1568 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1569 1570 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1571 cpu, idx, pmc_ctrl); 1572 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1573 cpu, idx, pmc_count); 1574 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1575 cpu, idx, prev_left); 1576 } 1577 for_each_set_bit(idx, fixed_cntr_mask, X86_PMC_IDX_MAX) { 1578 if (fixed_counter_disabled(idx, cpuc->pmu)) 1579 continue; 1580 rdmsrl(x86_pmu_fixed_ctr_addr(idx), pmc_count); 1581 1582 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1583 cpu, idx, pmc_count); 1584 } 1585 } 1586 1587 void x86_pmu_stop(struct perf_event *event, int flags) 1588 { 1589 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1590 struct hw_perf_event *hwc = &event->hw; 1591 1592 if (test_bit(hwc->idx, cpuc->active_mask)) { 1593 static_call(x86_pmu_disable)(event); 1594 __clear_bit(hwc->idx, cpuc->active_mask); 1595 cpuc->events[hwc->idx] = NULL; 1596 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1597 hwc->state |= PERF_HES_STOPPED; 1598 } 1599 1600 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1601 /* 1602 * Drain the remaining delta count out of a event 1603 * that we are disabling: 1604 */ 1605 static_call(x86_pmu_update)(event); 1606 hwc->state |= PERF_HES_UPTODATE; 1607 } 1608 } 1609 1610 static void x86_pmu_del(struct perf_event *event, int flags) 1611 { 1612 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1613 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap); 1614 int i; 1615 1616 /* 1617 * If we're called during a txn, we only need to undo x86_pmu.add. 1618 * The events never got scheduled and ->cancel_txn will truncate 1619 * the event_list. 1620 * 1621 * XXX assumes any ->del() called during a TXN will only be on 1622 * an event added during that same TXN. 1623 */ 1624 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1625 goto do_del; 1626 1627 __set_bit(event->hw.idx, cpuc->dirty); 1628 1629 /* 1630 * Not a TXN, therefore cleanup properly. 1631 */ 1632 x86_pmu_stop(event, PERF_EF_UPDATE); 1633 1634 for (i = 0; i < cpuc->n_events; i++) { 1635 if (event == cpuc->event_list[i]) 1636 break; 1637 } 1638 1639 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1640 return; 1641 1642 /* If we have a newly added event; make sure to decrease n_added. */ 1643 if (i >= cpuc->n_events - cpuc->n_added) 1644 --cpuc->n_added; 1645 1646 static_call_cond(x86_pmu_put_event_constraints)(cpuc, event); 1647 1648 /* Delete the array entry. */ 1649 while (++i < cpuc->n_events) { 1650 cpuc->event_list[i-1] = cpuc->event_list[i]; 1651 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1652 cpuc->assign[i-1] = cpuc->assign[i]; 1653 } 1654 cpuc->event_constraint[i-1] = NULL; 1655 --cpuc->n_events; 1656 if (intel_cap.perf_metrics) 1657 del_nr_metric_event(cpuc, event); 1658 1659 perf_event_update_userpage(event); 1660 1661 do_del: 1662 1663 /* 1664 * This is after x86_pmu_stop(); so we disable LBRs after any 1665 * event can need them etc.. 1666 */ 1667 static_call_cond(x86_pmu_del)(event); 1668 } 1669 1670 int x86_pmu_handle_irq(struct pt_regs *regs) 1671 { 1672 struct perf_sample_data data; 1673 struct cpu_hw_events *cpuc; 1674 struct perf_event *event; 1675 int idx, handled = 0; 1676 u64 val; 1677 1678 cpuc = this_cpu_ptr(&cpu_hw_events); 1679 1680 /* 1681 * Some chipsets need to unmask the LVTPC in a particular spot 1682 * inside the nmi handler. As a result, the unmasking was pushed 1683 * into all the nmi handlers. 1684 * 1685 * This generic handler doesn't seem to have any issues where the 1686 * unmasking occurs so it was left at the top. 1687 */ 1688 apic_write(APIC_LVTPC, APIC_DM_NMI); 1689 1690 for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { 1691 if (!test_bit(idx, cpuc->active_mask)) 1692 continue; 1693 1694 event = cpuc->events[idx]; 1695 1696 val = static_call(x86_pmu_update)(event); 1697 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1698 continue; 1699 1700 /* 1701 * event overflow 1702 */ 1703 handled++; 1704 1705 if (!static_call(x86_pmu_set_period)(event)) 1706 continue; 1707 1708 perf_sample_data_init(&data, 0, event->hw.last_period); 1709 1710 if (has_branch_stack(event)) 1711 perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL); 1712 1713 if (perf_event_overflow(event, &data, regs)) 1714 x86_pmu_stop(event, 0); 1715 } 1716 1717 if (handled) 1718 inc_irq_stat(apic_perf_irqs); 1719 1720 return handled; 1721 } 1722 1723 void perf_events_lapic_init(void) 1724 { 1725 if (!x86_pmu.apic || !x86_pmu_initialized()) 1726 return; 1727 1728 /* 1729 * Always use NMI for PMU 1730 */ 1731 apic_write(APIC_LVTPC, APIC_DM_NMI); 1732 } 1733 1734 static int 1735 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1736 { 1737 u64 start_clock; 1738 u64 finish_clock; 1739 int ret; 1740 1741 /* 1742 * All PMUs/events that share this PMI handler should make sure to 1743 * increment active_events for their events. 1744 */ 1745 if (!atomic_read(&active_events)) 1746 return NMI_DONE; 1747 1748 start_clock = sched_clock(); 1749 ret = static_call(x86_pmu_handle_irq)(regs); 1750 finish_clock = sched_clock(); 1751 1752 perf_sample_event_took(finish_clock - start_clock); 1753 1754 return ret; 1755 } 1756 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1757 1758 struct event_constraint emptyconstraint; 1759 struct event_constraint unconstrained; 1760 1761 static int x86_pmu_prepare_cpu(unsigned int cpu) 1762 { 1763 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1764 int i; 1765 1766 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1767 cpuc->kfree_on_online[i] = NULL; 1768 if (x86_pmu.cpu_prepare) 1769 return x86_pmu.cpu_prepare(cpu); 1770 return 0; 1771 } 1772 1773 static int x86_pmu_dead_cpu(unsigned int cpu) 1774 { 1775 if (x86_pmu.cpu_dead) 1776 x86_pmu.cpu_dead(cpu); 1777 return 0; 1778 } 1779 1780 static int x86_pmu_online_cpu(unsigned int cpu) 1781 { 1782 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1783 int i; 1784 1785 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1786 kfree(cpuc->kfree_on_online[i]); 1787 cpuc->kfree_on_online[i] = NULL; 1788 } 1789 return 0; 1790 } 1791 1792 static int x86_pmu_starting_cpu(unsigned int cpu) 1793 { 1794 if (x86_pmu.cpu_starting) 1795 x86_pmu.cpu_starting(cpu); 1796 return 0; 1797 } 1798 1799 static int x86_pmu_dying_cpu(unsigned int cpu) 1800 { 1801 if (x86_pmu.cpu_dying) 1802 x86_pmu.cpu_dying(cpu); 1803 return 0; 1804 } 1805 1806 static void __init pmu_check_apic(void) 1807 { 1808 if (boot_cpu_has(X86_FEATURE_APIC)) 1809 return; 1810 1811 x86_pmu.apic = 0; 1812 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1813 pr_info("no hardware sampling interrupt available.\n"); 1814 1815 /* 1816 * If we have a PMU initialized but no APIC 1817 * interrupts, we cannot sample hardware 1818 * events (user-space has to fall back and 1819 * sample via a hrtimer based software event): 1820 */ 1821 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1822 1823 } 1824 1825 static struct attribute_group x86_pmu_format_group __ro_after_init = { 1826 .name = "format", 1827 .attrs = NULL, 1828 }; 1829 1830 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1831 { 1832 struct perf_pmu_events_attr *pmu_attr = 1833 container_of(attr, struct perf_pmu_events_attr, attr); 1834 u64 config = 0; 1835 1836 if (pmu_attr->id < x86_pmu.max_events) 1837 config = x86_pmu.event_map(pmu_attr->id); 1838 1839 /* string trumps id */ 1840 if (pmu_attr->event_str) 1841 return sprintf(page, "%s\n", pmu_attr->event_str); 1842 1843 return x86_pmu.events_sysfs_show(page, config); 1844 } 1845 EXPORT_SYMBOL_GPL(events_sysfs_show); 1846 1847 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1848 char *page) 1849 { 1850 struct perf_pmu_events_ht_attr *pmu_attr = 1851 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1852 1853 /* 1854 * Report conditional events depending on Hyper-Threading. 1855 * 1856 * This is overly conservative as usually the HT special 1857 * handling is not needed if the other CPU thread is idle. 1858 * 1859 * Note this does not (and cannot) handle the case when thread 1860 * siblings are invisible, for example with virtualization 1861 * if they are owned by some other guest. The user tool 1862 * has to re-read when a thread sibling gets onlined later. 1863 */ 1864 return sprintf(page, "%s", 1865 topology_max_smt_threads() > 1 ? 1866 pmu_attr->event_str_ht : 1867 pmu_attr->event_str_noht); 1868 } 1869 1870 ssize_t events_hybrid_sysfs_show(struct device *dev, 1871 struct device_attribute *attr, 1872 char *page) 1873 { 1874 struct perf_pmu_events_hybrid_attr *pmu_attr = 1875 container_of(attr, struct perf_pmu_events_hybrid_attr, attr); 1876 struct x86_hybrid_pmu *pmu; 1877 const char *str, *next_str; 1878 int i; 1879 1880 if (hweight64(pmu_attr->pmu_type) == 1) 1881 return sprintf(page, "%s", pmu_attr->event_str); 1882 1883 /* 1884 * Hybrid PMUs may support the same event name, but with different 1885 * event encoding, e.g., the mem-loads event on an Atom PMU has 1886 * different event encoding from a Core PMU. 1887 * 1888 * The event_str includes all event encodings. Each event encoding 1889 * is divided by ";". The order of the event encodings must follow 1890 * the order of the hybrid PMU index. 1891 */ 1892 pmu = container_of(dev_get_drvdata(dev), struct x86_hybrid_pmu, pmu); 1893 1894 str = pmu_attr->event_str; 1895 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { 1896 if (!(x86_pmu.hybrid_pmu[i].pmu_type & pmu_attr->pmu_type)) 1897 continue; 1898 if (x86_pmu.hybrid_pmu[i].pmu_type & pmu->pmu_type) { 1899 next_str = strchr(str, ';'); 1900 if (next_str) 1901 return snprintf(page, next_str - str + 1, "%s", str); 1902 else 1903 return sprintf(page, "%s", str); 1904 } 1905 str = strchr(str, ';'); 1906 str++; 1907 } 1908 1909 return 0; 1910 } 1911 EXPORT_SYMBOL_GPL(events_hybrid_sysfs_show); 1912 1913 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1914 EVENT_ATTR(instructions, INSTRUCTIONS ); 1915 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1916 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1917 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1918 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1919 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1920 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1921 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1922 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1923 1924 static struct attribute *empty_attrs; 1925 1926 static struct attribute *events_attr[] = { 1927 EVENT_PTR(CPU_CYCLES), 1928 EVENT_PTR(INSTRUCTIONS), 1929 EVENT_PTR(CACHE_REFERENCES), 1930 EVENT_PTR(CACHE_MISSES), 1931 EVENT_PTR(BRANCH_INSTRUCTIONS), 1932 EVENT_PTR(BRANCH_MISSES), 1933 EVENT_PTR(BUS_CYCLES), 1934 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1935 EVENT_PTR(STALLED_CYCLES_BACKEND), 1936 EVENT_PTR(REF_CPU_CYCLES), 1937 NULL, 1938 }; 1939 1940 /* 1941 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1942 * out of events_attr attributes. 1943 */ 1944 static umode_t 1945 is_visible(struct kobject *kobj, struct attribute *attr, int idx) 1946 { 1947 struct perf_pmu_events_attr *pmu_attr; 1948 1949 if (idx >= x86_pmu.max_events) 1950 return 0; 1951 1952 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr); 1953 /* str trumps id */ 1954 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0; 1955 } 1956 1957 static struct attribute_group x86_pmu_events_group __ro_after_init = { 1958 .name = "events", 1959 .attrs = events_attr, 1960 .is_visible = is_visible, 1961 }; 1962 1963 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1964 { 1965 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1966 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1967 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1968 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1969 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1970 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1971 ssize_t ret; 1972 1973 /* 1974 * We have whole page size to spend and just little data 1975 * to write, so we can safely use sprintf. 1976 */ 1977 ret = sprintf(page, "event=0x%02llx", event); 1978 1979 if (umask) 1980 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1981 1982 if (edge) 1983 ret += sprintf(page + ret, ",edge"); 1984 1985 if (pc) 1986 ret += sprintf(page + ret, ",pc"); 1987 1988 if (any) 1989 ret += sprintf(page + ret, ",any"); 1990 1991 if (inv) 1992 ret += sprintf(page + ret, ",inv"); 1993 1994 if (cmask) 1995 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1996 1997 ret += sprintf(page + ret, "\n"); 1998 1999 return ret; 2000 } 2001 2002 static struct attribute_group x86_pmu_attr_group; 2003 static struct attribute_group x86_pmu_caps_group; 2004 2005 static void x86_pmu_static_call_update(void) 2006 { 2007 static_call_update(x86_pmu_handle_irq, x86_pmu.handle_irq); 2008 static_call_update(x86_pmu_disable_all, x86_pmu.disable_all); 2009 static_call_update(x86_pmu_enable_all, x86_pmu.enable_all); 2010 static_call_update(x86_pmu_enable, x86_pmu.enable); 2011 static_call_update(x86_pmu_disable, x86_pmu.disable); 2012 2013 static_call_update(x86_pmu_assign, x86_pmu.assign); 2014 2015 static_call_update(x86_pmu_add, x86_pmu.add); 2016 static_call_update(x86_pmu_del, x86_pmu.del); 2017 static_call_update(x86_pmu_read, x86_pmu.read); 2018 2019 static_call_update(x86_pmu_set_period, x86_pmu.set_period); 2020 static_call_update(x86_pmu_update, x86_pmu.update); 2021 static_call_update(x86_pmu_limit_period, x86_pmu.limit_period); 2022 2023 static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events); 2024 static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints); 2025 static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints); 2026 2027 static_call_update(x86_pmu_start_scheduling, x86_pmu.start_scheduling); 2028 static_call_update(x86_pmu_commit_scheduling, x86_pmu.commit_scheduling); 2029 static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling); 2030 2031 static_call_update(x86_pmu_sched_task, x86_pmu.sched_task); 2032 static_call_update(x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx); 2033 2034 static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs); 2035 static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases); 2036 2037 static_call_update(x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs); 2038 static_call_update(x86_pmu_filter, x86_pmu.filter); 2039 } 2040 2041 static void _x86_pmu_read(struct perf_event *event) 2042 { 2043 static_call(x86_pmu_update)(event); 2044 } 2045 2046 void x86_pmu_show_pmu_cap(struct pmu *pmu) 2047 { 2048 pr_info("... version: %d\n", x86_pmu.version); 2049 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 2050 pr_info("... generic registers: %d\n", x86_pmu_num_counters(pmu)); 2051 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 2052 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 2053 pr_info("... fixed-purpose events: %d\n", x86_pmu_num_counters_fixed(pmu)); 2054 pr_info("... event mask: %016Lx\n", hybrid(pmu, intel_ctrl)); 2055 } 2056 2057 static int __init init_hw_perf_events(void) 2058 { 2059 struct x86_pmu_quirk *quirk; 2060 int err; 2061 2062 pr_info("Performance Events: "); 2063 2064 switch (boot_cpu_data.x86_vendor) { 2065 case X86_VENDOR_INTEL: 2066 err = intel_pmu_init(); 2067 break; 2068 case X86_VENDOR_AMD: 2069 err = amd_pmu_init(); 2070 break; 2071 case X86_VENDOR_HYGON: 2072 err = amd_pmu_init(); 2073 x86_pmu.name = "HYGON"; 2074 break; 2075 case X86_VENDOR_ZHAOXIN: 2076 case X86_VENDOR_CENTAUR: 2077 err = zhaoxin_pmu_init(); 2078 break; 2079 default: 2080 err = -ENOTSUPP; 2081 } 2082 if (err != 0) { 2083 pr_cont("no PMU driver, software events only.\n"); 2084 err = 0; 2085 goto out_bad_pmu; 2086 } 2087 2088 pmu_check_apic(); 2089 2090 /* sanity check that the hardware exists or is emulated */ 2091 if (!check_hw_exists(&pmu, x86_pmu.cntr_mask, x86_pmu.fixed_cntr_mask)) 2092 goto out_bad_pmu; 2093 2094 pr_cont("%s PMU driver.\n", x86_pmu.name); 2095 2096 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 2097 2098 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 2099 quirk->func(); 2100 2101 if (!x86_pmu.intel_ctrl) 2102 x86_pmu.intel_ctrl = x86_pmu.cntr_mask64; 2103 2104 if (!x86_pmu.config_mask) 2105 x86_pmu.config_mask = X86_RAW_EVENT_MASK; 2106 2107 perf_events_lapic_init(); 2108 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 2109 2110 unconstrained = (struct event_constraint) 2111 __EVENT_CONSTRAINT(0, x86_pmu.cntr_mask64, 2112 0, x86_pmu_num_counters(NULL), 0, 0); 2113 2114 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 2115 2116 if (!x86_pmu.events_sysfs_show) 2117 x86_pmu_events_group.attrs = &empty_attrs; 2118 2119 pmu.attr_update = x86_pmu.attr_update; 2120 2121 if (!is_hybrid()) 2122 x86_pmu_show_pmu_cap(NULL); 2123 2124 if (!x86_pmu.read) 2125 x86_pmu.read = _x86_pmu_read; 2126 2127 if (!x86_pmu.guest_get_msrs) 2128 x86_pmu.guest_get_msrs = (void *)&__static_call_return0; 2129 2130 if (!x86_pmu.set_period) 2131 x86_pmu.set_period = x86_perf_event_set_period; 2132 2133 if (!x86_pmu.update) 2134 x86_pmu.update = x86_perf_event_update; 2135 2136 x86_pmu_static_call_update(); 2137 2138 /* 2139 * Install callbacks. Core will call them for each online 2140 * cpu. 2141 */ 2142 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 2143 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 2144 if (err) 2145 return err; 2146 2147 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 2148 "perf/x86:starting", x86_pmu_starting_cpu, 2149 x86_pmu_dying_cpu); 2150 if (err) 2151 goto out; 2152 2153 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 2154 x86_pmu_online_cpu, NULL); 2155 if (err) 2156 goto out1; 2157 2158 if (!is_hybrid()) { 2159 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 2160 if (err) 2161 goto out2; 2162 } else { 2163 struct x86_hybrid_pmu *hybrid_pmu; 2164 int i, j; 2165 2166 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { 2167 hybrid_pmu = &x86_pmu.hybrid_pmu[i]; 2168 2169 hybrid_pmu->pmu = pmu; 2170 hybrid_pmu->pmu.type = -1; 2171 hybrid_pmu->pmu.attr_update = x86_pmu.attr_update; 2172 hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE; 2173 2174 err = perf_pmu_register(&hybrid_pmu->pmu, hybrid_pmu->name, 2175 (hybrid_pmu->pmu_type == hybrid_big) ? PERF_TYPE_RAW : -1); 2176 if (err) 2177 break; 2178 } 2179 2180 if (i < x86_pmu.num_hybrid_pmus) { 2181 for (j = 0; j < i; j++) 2182 perf_pmu_unregister(&x86_pmu.hybrid_pmu[j].pmu); 2183 pr_warn("Failed to register hybrid PMUs\n"); 2184 kfree(x86_pmu.hybrid_pmu); 2185 x86_pmu.hybrid_pmu = NULL; 2186 x86_pmu.num_hybrid_pmus = 0; 2187 goto out2; 2188 } 2189 } 2190 2191 return 0; 2192 2193 out2: 2194 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 2195 out1: 2196 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 2197 out: 2198 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 2199 out_bad_pmu: 2200 memset(&x86_pmu, 0, sizeof(x86_pmu)); 2201 return err; 2202 } 2203 early_initcall(init_hw_perf_events); 2204 2205 static void x86_pmu_read(struct perf_event *event) 2206 { 2207 static_call(x86_pmu_read)(event); 2208 } 2209 2210 /* 2211 * Start group events scheduling transaction 2212 * Set the flag to make pmu::enable() not perform the 2213 * schedulability test, it will be performed at commit time 2214 * 2215 * We only support PERF_PMU_TXN_ADD transactions. Save the 2216 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 2217 * transactions. 2218 */ 2219 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 2220 { 2221 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2222 2223 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 2224 2225 cpuc->txn_flags = txn_flags; 2226 if (txn_flags & ~PERF_PMU_TXN_ADD) 2227 return; 2228 2229 perf_pmu_disable(pmu); 2230 __this_cpu_write(cpu_hw_events.n_txn, 0); 2231 __this_cpu_write(cpu_hw_events.n_txn_pair, 0); 2232 __this_cpu_write(cpu_hw_events.n_txn_metric, 0); 2233 } 2234 2235 /* 2236 * Stop group events scheduling transaction 2237 * Clear the flag and pmu::enable() will perform the 2238 * schedulability test. 2239 */ 2240 static void x86_pmu_cancel_txn(struct pmu *pmu) 2241 { 2242 unsigned int txn_flags; 2243 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2244 2245 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 2246 2247 txn_flags = cpuc->txn_flags; 2248 cpuc->txn_flags = 0; 2249 if (txn_flags & ~PERF_PMU_TXN_ADD) 2250 return; 2251 2252 /* 2253 * Truncate collected array by the number of events added in this 2254 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 2255 */ 2256 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 2257 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 2258 __this_cpu_sub(cpu_hw_events.n_pair, __this_cpu_read(cpu_hw_events.n_txn_pair)); 2259 __this_cpu_sub(cpu_hw_events.n_metric, __this_cpu_read(cpu_hw_events.n_txn_metric)); 2260 perf_pmu_enable(pmu); 2261 } 2262 2263 /* 2264 * Commit group events scheduling transaction 2265 * Perform the group schedulability test as a whole 2266 * Return 0 if success 2267 * 2268 * Does not cancel the transaction on failure; expects the caller to do this. 2269 */ 2270 static int x86_pmu_commit_txn(struct pmu *pmu) 2271 { 2272 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2273 int assign[X86_PMC_IDX_MAX]; 2274 int n, ret; 2275 2276 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 2277 2278 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 2279 cpuc->txn_flags = 0; 2280 return 0; 2281 } 2282 2283 n = cpuc->n_events; 2284 2285 if (!x86_pmu_initialized()) 2286 return -EAGAIN; 2287 2288 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign); 2289 if (ret) 2290 return ret; 2291 2292 /* 2293 * copy new assignment, now we know it is possible 2294 * will be used by hw_perf_enable() 2295 */ 2296 memcpy(cpuc->assign, assign, n*sizeof(int)); 2297 2298 cpuc->txn_flags = 0; 2299 perf_pmu_enable(pmu); 2300 return 0; 2301 } 2302 /* 2303 * a fake_cpuc is used to validate event groups. Due to 2304 * the extra reg logic, we need to also allocate a fake 2305 * per_core and per_cpu structure. Otherwise, group events 2306 * using extra reg may conflict without the kernel being 2307 * able to catch this when the last event gets added to 2308 * the group. 2309 */ 2310 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 2311 { 2312 intel_cpuc_finish(cpuc); 2313 kfree(cpuc); 2314 } 2315 2316 static struct cpu_hw_events *allocate_fake_cpuc(struct pmu *event_pmu) 2317 { 2318 struct cpu_hw_events *cpuc; 2319 int cpu; 2320 2321 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 2322 if (!cpuc) 2323 return ERR_PTR(-ENOMEM); 2324 cpuc->is_fake = 1; 2325 2326 if (is_hybrid()) { 2327 struct x86_hybrid_pmu *h_pmu; 2328 2329 h_pmu = hybrid_pmu(event_pmu); 2330 if (cpumask_empty(&h_pmu->supported_cpus)) 2331 goto error; 2332 cpu = cpumask_first(&h_pmu->supported_cpus); 2333 } else 2334 cpu = raw_smp_processor_id(); 2335 cpuc->pmu = event_pmu; 2336 2337 if (intel_cpuc_prepare(cpuc, cpu)) 2338 goto error; 2339 2340 return cpuc; 2341 error: 2342 free_fake_cpuc(cpuc); 2343 return ERR_PTR(-ENOMEM); 2344 } 2345 2346 /* 2347 * validate that we can schedule this event 2348 */ 2349 static int validate_event(struct perf_event *event) 2350 { 2351 struct cpu_hw_events *fake_cpuc; 2352 struct event_constraint *c; 2353 int ret = 0; 2354 2355 fake_cpuc = allocate_fake_cpuc(event->pmu); 2356 if (IS_ERR(fake_cpuc)) 2357 return PTR_ERR(fake_cpuc); 2358 2359 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event); 2360 2361 if (!c || !c->weight) 2362 ret = -EINVAL; 2363 2364 if (x86_pmu.put_event_constraints) 2365 x86_pmu.put_event_constraints(fake_cpuc, event); 2366 2367 free_fake_cpuc(fake_cpuc); 2368 2369 return ret; 2370 } 2371 2372 /* 2373 * validate a single event group 2374 * 2375 * validation include: 2376 * - check events are compatible which each other 2377 * - events do not compete for the same counter 2378 * - number of events <= number of counters 2379 * 2380 * validation ensures the group can be loaded onto the 2381 * PMU if it was the only group available. 2382 */ 2383 static int validate_group(struct perf_event *event) 2384 { 2385 struct perf_event *leader = event->group_leader; 2386 struct cpu_hw_events *fake_cpuc; 2387 int ret = -EINVAL, n; 2388 2389 /* 2390 * Reject events from different hybrid PMUs. 2391 */ 2392 if (is_hybrid()) { 2393 struct perf_event *sibling; 2394 struct pmu *pmu = NULL; 2395 2396 if (is_x86_event(leader)) 2397 pmu = leader->pmu; 2398 2399 for_each_sibling_event(sibling, leader) { 2400 if (!is_x86_event(sibling)) 2401 continue; 2402 if (!pmu) 2403 pmu = sibling->pmu; 2404 else if (pmu != sibling->pmu) 2405 return ret; 2406 } 2407 } 2408 2409 fake_cpuc = allocate_fake_cpuc(event->pmu); 2410 if (IS_ERR(fake_cpuc)) 2411 return PTR_ERR(fake_cpuc); 2412 /* 2413 * the event is not yet connected with its 2414 * siblings therefore we must first collect 2415 * existing siblings, then add the new event 2416 * before we can simulate the scheduling 2417 */ 2418 n = collect_events(fake_cpuc, leader, true); 2419 if (n < 0) 2420 goto out; 2421 2422 fake_cpuc->n_events = n; 2423 n = collect_events(fake_cpuc, event, false); 2424 if (n < 0) 2425 goto out; 2426 2427 fake_cpuc->n_events = 0; 2428 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2429 2430 out: 2431 free_fake_cpuc(fake_cpuc); 2432 return ret; 2433 } 2434 2435 static int x86_pmu_event_init(struct perf_event *event) 2436 { 2437 struct x86_hybrid_pmu *pmu = NULL; 2438 int err; 2439 2440 if ((event->attr.type != event->pmu->type) && 2441 (event->attr.type != PERF_TYPE_HARDWARE) && 2442 (event->attr.type != PERF_TYPE_HW_CACHE)) 2443 return -ENOENT; 2444 2445 if (is_hybrid() && (event->cpu != -1)) { 2446 pmu = hybrid_pmu(event->pmu); 2447 if (!cpumask_test_cpu(event->cpu, &pmu->supported_cpus)) 2448 return -ENOENT; 2449 } 2450 2451 err = __x86_pmu_event_init(event); 2452 if (!err) { 2453 if (event->group_leader != event) 2454 err = validate_group(event); 2455 else 2456 err = validate_event(event); 2457 } 2458 if (err) { 2459 if (event->destroy) 2460 event->destroy(event); 2461 event->destroy = NULL; 2462 } 2463 2464 if (READ_ONCE(x86_pmu.attr_rdpmc) && 2465 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) 2466 event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; 2467 2468 return err; 2469 } 2470 2471 void perf_clear_dirty_counters(void) 2472 { 2473 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2474 int i; 2475 2476 /* Don't need to clear the assigned counter. */ 2477 for (i = 0; i < cpuc->n_events; i++) 2478 __clear_bit(cpuc->assign[i], cpuc->dirty); 2479 2480 if (bitmap_empty(cpuc->dirty, X86_PMC_IDX_MAX)) 2481 return; 2482 2483 for_each_set_bit(i, cpuc->dirty, X86_PMC_IDX_MAX) { 2484 if (i >= INTEL_PMC_IDX_FIXED) { 2485 /* Metrics and fake events don't have corresponding HW counters. */ 2486 if (!test_bit(i - INTEL_PMC_IDX_FIXED, hybrid(cpuc->pmu, fixed_cntr_mask))) 2487 continue; 2488 2489 wrmsrl(x86_pmu_fixed_ctr_addr(i - INTEL_PMC_IDX_FIXED), 0); 2490 } else { 2491 wrmsrl(x86_pmu_event_addr(i), 0); 2492 } 2493 } 2494 2495 bitmap_zero(cpuc->dirty, X86_PMC_IDX_MAX); 2496 } 2497 2498 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) 2499 { 2500 if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) 2501 return; 2502 2503 /* 2504 * This function relies on not being called concurrently in two 2505 * tasks in the same mm. Otherwise one task could observe 2506 * perf_rdpmc_allowed > 1 and return all the way back to 2507 * userspace with CR4.PCE clear while another task is still 2508 * doing on_each_cpu_mask() to propagate CR4.PCE. 2509 * 2510 * For now, this can't happen because all callers hold mmap_lock 2511 * for write. If this changes, we'll need a different solution. 2512 */ 2513 mmap_assert_write_locked(mm); 2514 2515 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) 2516 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1); 2517 } 2518 2519 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) 2520 { 2521 if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) 2522 return; 2523 2524 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) 2525 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1); 2526 } 2527 2528 static int x86_pmu_event_idx(struct perf_event *event) 2529 { 2530 struct hw_perf_event *hwc = &event->hw; 2531 2532 if (!(hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT)) 2533 return 0; 2534 2535 if (is_metric_idx(hwc->idx)) 2536 return INTEL_PMC_FIXED_RDPMC_METRICS + 1; 2537 else 2538 return hwc->event_base_rdpmc + 1; 2539 } 2540 2541 static ssize_t get_attr_rdpmc(struct device *cdev, 2542 struct device_attribute *attr, 2543 char *buf) 2544 { 2545 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2546 } 2547 2548 static ssize_t set_attr_rdpmc(struct device *cdev, 2549 struct device_attribute *attr, 2550 const char *buf, size_t count) 2551 { 2552 static DEFINE_MUTEX(rdpmc_mutex); 2553 unsigned long val; 2554 ssize_t ret; 2555 2556 ret = kstrtoul(buf, 0, &val); 2557 if (ret) 2558 return ret; 2559 2560 if (val > 2) 2561 return -EINVAL; 2562 2563 if (x86_pmu.attr_rdpmc_broken) 2564 return -ENOTSUPP; 2565 2566 guard(mutex)(&rdpmc_mutex); 2567 2568 if (val != x86_pmu.attr_rdpmc) { 2569 /* 2570 * Changing into or out of never available or always available, 2571 * aka perf-event-bypassing mode. This path is extremely slow, 2572 * but only root can trigger it, so it's okay. 2573 */ 2574 if (val == 0) 2575 static_branch_inc(&rdpmc_never_available_key); 2576 else if (x86_pmu.attr_rdpmc == 0) 2577 static_branch_dec(&rdpmc_never_available_key); 2578 2579 if (val == 2) 2580 static_branch_inc(&rdpmc_always_available_key); 2581 else if (x86_pmu.attr_rdpmc == 2) 2582 static_branch_dec(&rdpmc_always_available_key); 2583 2584 on_each_cpu(cr4_update_pce, NULL, 1); 2585 x86_pmu.attr_rdpmc = val; 2586 } 2587 2588 return count; 2589 } 2590 2591 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2592 2593 static struct attribute *x86_pmu_attrs[] = { 2594 &dev_attr_rdpmc.attr, 2595 NULL, 2596 }; 2597 2598 static struct attribute_group x86_pmu_attr_group __ro_after_init = { 2599 .attrs = x86_pmu_attrs, 2600 }; 2601 2602 static ssize_t max_precise_show(struct device *cdev, 2603 struct device_attribute *attr, 2604 char *buf) 2605 { 2606 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise()); 2607 } 2608 2609 static DEVICE_ATTR_RO(max_precise); 2610 2611 static struct attribute *x86_pmu_caps_attrs[] = { 2612 &dev_attr_max_precise.attr, 2613 NULL 2614 }; 2615 2616 static struct attribute_group x86_pmu_caps_group __ro_after_init = { 2617 .name = "caps", 2618 .attrs = x86_pmu_caps_attrs, 2619 }; 2620 2621 static const struct attribute_group *x86_pmu_attr_groups[] = { 2622 &x86_pmu_attr_group, 2623 &x86_pmu_format_group, 2624 &x86_pmu_events_group, 2625 &x86_pmu_caps_group, 2626 NULL, 2627 }; 2628 2629 static void x86_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) 2630 { 2631 static_call_cond(x86_pmu_sched_task)(pmu_ctx, sched_in); 2632 } 2633 2634 static void x86_pmu_swap_task_ctx(struct perf_event_pmu_context *prev_epc, 2635 struct perf_event_pmu_context *next_epc) 2636 { 2637 static_call_cond(x86_pmu_swap_task_ctx)(prev_epc, next_epc); 2638 } 2639 2640 void perf_check_microcode(void) 2641 { 2642 if (x86_pmu.check_microcode) 2643 x86_pmu.check_microcode(); 2644 } 2645 2646 static int x86_pmu_check_period(struct perf_event *event, u64 value) 2647 { 2648 if (x86_pmu.check_period && x86_pmu.check_period(event, value)) 2649 return -EINVAL; 2650 2651 if (value && x86_pmu.limit_period) { 2652 s64 left = value; 2653 x86_pmu.limit_period(event, &left); 2654 if (left > value) 2655 return -EINVAL; 2656 } 2657 2658 return 0; 2659 } 2660 2661 static int x86_pmu_aux_output_match(struct perf_event *event) 2662 { 2663 if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT)) 2664 return 0; 2665 2666 if (x86_pmu.aux_output_match) 2667 return x86_pmu.aux_output_match(event); 2668 2669 return 0; 2670 } 2671 2672 static bool x86_pmu_filter(struct pmu *pmu, int cpu) 2673 { 2674 bool ret = false; 2675 2676 static_call_cond(x86_pmu_filter)(pmu, cpu, &ret); 2677 2678 return ret; 2679 } 2680 2681 static struct pmu pmu = { 2682 .pmu_enable = x86_pmu_enable, 2683 .pmu_disable = x86_pmu_disable, 2684 2685 .attr_groups = x86_pmu_attr_groups, 2686 2687 .event_init = x86_pmu_event_init, 2688 2689 .event_mapped = x86_pmu_event_mapped, 2690 .event_unmapped = x86_pmu_event_unmapped, 2691 2692 .add = x86_pmu_add, 2693 .del = x86_pmu_del, 2694 .start = x86_pmu_start, 2695 .stop = x86_pmu_stop, 2696 .read = x86_pmu_read, 2697 2698 .start_txn = x86_pmu_start_txn, 2699 .cancel_txn = x86_pmu_cancel_txn, 2700 .commit_txn = x86_pmu_commit_txn, 2701 2702 .event_idx = x86_pmu_event_idx, 2703 .sched_task = x86_pmu_sched_task, 2704 .swap_task_ctx = x86_pmu_swap_task_ctx, 2705 .check_period = x86_pmu_check_period, 2706 2707 .aux_output_match = x86_pmu_aux_output_match, 2708 2709 .filter = x86_pmu_filter, 2710 }; 2711 2712 void arch_perf_update_userpage(struct perf_event *event, 2713 struct perf_event_mmap_page *userpg, u64 now) 2714 { 2715 struct cyc2ns_data data; 2716 u64 offset; 2717 2718 userpg->cap_user_time = 0; 2719 userpg->cap_user_time_zero = 0; 2720 userpg->cap_user_rdpmc = 2721 !!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT); 2722 userpg->pmc_width = x86_pmu.cntval_bits; 2723 2724 if (!using_native_sched_clock() || !sched_clock_stable()) 2725 return; 2726 2727 cyc2ns_read_begin(&data); 2728 2729 offset = data.cyc2ns_offset + __sched_clock_offset; 2730 2731 /* 2732 * Internal timekeeping for enabled/running/stopped times 2733 * is always in the local_clock domain. 2734 */ 2735 userpg->cap_user_time = 1; 2736 userpg->time_mult = data.cyc2ns_mul; 2737 userpg->time_shift = data.cyc2ns_shift; 2738 userpg->time_offset = offset - now; 2739 2740 /* 2741 * cap_user_time_zero doesn't make sense when we're using a different 2742 * time base for the records. 2743 */ 2744 if (!event->attr.use_clockid) { 2745 userpg->cap_user_time_zero = 1; 2746 userpg->time_zero = offset; 2747 } 2748 2749 cyc2ns_read_end(); 2750 } 2751 2752 /* 2753 * Determine whether the regs were taken from an irq/exception handler rather 2754 * than from perf_arch_fetch_caller_regs(). 2755 */ 2756 static bool perf_hw_regs(struct pt_regs *regs) 2757 { 2758 return regs->flags & X86_EFLAGS_FIXED; 2759 } 2760 2761 void 2762 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2763 { 2764 struct unwind_state state; 2765 unsigned long addr; 2766 2767 if (perf_guest_state()) { 2768 /* TODO: We don't support guest os callchain now */ 2769 return; 2770 } 2771 2772 if (perf_callchain_store(entry, regs->ip)) 2773 return; 2774 2775 if (perf_hw_regs(regs)) 2776 unwind_start(&state, current, regs, NULL); 2777 else 2778 unwind_start(&state, current, NULL, (void *)regs->sp); 2779 2780 for (; !unwind_done(&state); unwind_next_frame(&state)) { 2781 addr = unwind_get_return_address(&state); 2782 if (!addr || perf_callchain_store(entry, addr)) 2783 return; 2784 } 2785 } 2786 2787 static inline int 2788 valid_user_frame(const void __user *fp, unsigned long size) 2789 { 2790 return __access_ok(fp, size); 2791 } 2792 2793 static unsigned long get_segment_base(unsigned int segment) 2794 { 2795 struct desc_struct *desc; 2796 unsigned int idx = segment >> 3; 2797 2798 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2799 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2800 struct ldt_struct *ldt; 2801 2802 /* IRQs are off, so this synchronizes with smp_store_release */ 2803 ldt = READ_ONCE(current->active_mm->context.ldt); 2804 if (!ldt || idx >= ldt->nr_entries) 2805 return 0; 2806 2807 desc = &ldt->entries[idx]; 2808 #else 2809 return 0; 2810 #endif 2811 } else { 2812 if (idx >= GDT_ENTRIES) 2813 return 0; 2814 2815 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2816 } 2817 2818 return get_desc_base(desc); 2819 } 2820 2821 #ifdef CONFIG_UPROBES 2822 /* 2823 * Heuristic-based check if uprobe is installed at the function entry. 2824 * 2825 * Under assumption of user code being compiled with frame pointers, 2826 * `push %rbp/%ebp` is a good indicator that we indeed are. 2827 * 2828 * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern. 2829 * If we get this wrong, captured stack trace might have one extra bogus 2830 * entry, but the rest of stack trace will still be meaningful. 2831 */ 2832 static bool is_uprobe_at_func_entry(struct pt_regs *regs) 2833 { 2834 struct arch_uprobe *auprobe; 2835 2836 if (!current->utask) 2837 return false; 2838 2839 auprobe = current->utask->auprobe; 2840 if (!auprobe) 2841 return false; 2842 2843 /* push %rbp/%ebp */ 2844 if (auprobe->insn[0] == 0x55) 2845 return true; 2846 2847 /* endbr64 (64-bit only) */ 2848 if (user_64bit_mode(regs) && is_endbr(*(u32 *)auprobe->insn)) 2849 return true; 2850 2851 return false; 2852 } 2853 2854 #else 2855 static bool is_uprobe_at_func_entry(struct pt_regs *regs) 2856 { 2857 return false; 2858 } 2859 #endif /* CONFIG_UPROBES */ 2860 2861 #ifdef CONFIG_IA32_EMULATION 2862 2863 #include <linux/compat.h> 2864 2865 static inline int 2866 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2867 { 2868 /* 32-bit process in 64-bit kernel. */ 2869 unsigned long ss_base, cs_base; 2870 struct stack_frame_ia32 frame; 2871 const struct stack_frame_ia32 __user *fp; 2872 u32 ret_addr; 2873 2874 if (user_64bit_mode(regs)) 2875 return 0; 2876 2877 cs_base = get_segment_base(regs->cs); 2878 ss_base = get_segment_base(regs->ss); 2879 2880 fp = compat_ptr(ss_base + regs->bp); 2881 pagefault_disable(); 2882 2883 /* see perf_callchain_user() below for why we do this */ 2884 if (is_uprobe_at_func_entry(regs) && 2885 !get_user(ret_addr, (const u32 __user *)regs->sp)) 2886 perf_callchain_store(entry, ret_addr); 2887 2888 while (entry->nr < entry->max_stack) { 2889 if (!valid_user_frame(fp, sizeof(frame))) 2890 break; 2891 2892 if (__get_user(frame.next_frame, &fp->next_frame)) 2893 break; 2894 if (__get_user(frame.return_address, &fp->return_address)) 2895 break; 2896 2897 perf_callchain_store(entry, cs_base + frame.return_address); 2898 fp = compat_ptr(ss_base + frame.next_frame); 2899 } 2900 pagefault_enable(); 2901 return 1; 2902 } 2903 #else 2904 static inline int 2905 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2906 { 2907 return 0; 2908 } 2909 #endif 2910 2911 void 2912 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2913 { 2914 struct stack_frame frame; 2915 const struct stack_frame __user *fp; 2916 unsigned long ret_addr; 2917 2918 if (perf_guest_state()) { 2919 /* TODO: We don't support guest os callchain now */ 2920 return; 2921 } 2922 2923 /* 2924 * We don't know what to do with VM86 stacks.. ignore them for now. 2925 */ 2926 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2927 return; 2928 2929 fp = (void __user *)regs->bp; 2930 2931 perf_callchain_store(entry, regs->ip); 2932 2933 if (!nmi_uaccess_okay()) 2934 return; 2935 2936 if (perf_callchain_user32(regs, entry)) 2937 return; 2938 2939 pagefault_disable(); 2940 2941 /* 2942 * If we are called from uprobe handler, and we are indeed at the very 2943 * entry to user function (which is normally a `push %rbp` instruction, 2944 * under assumption of application being compiled with frame pointers), 2945 * we should read return address from *regs->sp before proceeding 2946 * to follow frame pointers, otherwise we'll skip immediate caller 2947 * as %rbp is not yet setup. 2948 */ 2949 if (is_uprobe_at_func_entry(regs) && 2950 !get_user(ret_addr, (const unsigned long __user *)regs->sp)) 2951 perf_callchain_store(entry, ret_addr); 2952 2953 while (entry->nr < entry->max_stack) { 2954 if (!valid_user_frame(fp, sizeof(frame))) 2955 break; 2956 2957 if (__get_user(frame.next_frame, &fp->next_frame)) 2958 break; 2959 if (__get_user(frame.return_address, &fp->return_address)) 2960 break; 2961 2962 perf_callchain_store(entry, frame.return_address); 2963 fp = (void __user *)frame.next_frame; 2964 } 2965 pagefault_enable(); 2966 } 2967 2968 /* 2969 * Deal with code segment offsets for the various execution modes: 2970 * 2971 * VM86 - the good olde 16 bit days, where the linear address is 2972 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2973 * 2974 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2975 * to figure out what the 32bit base address is. 2976 * 2977 * X32 - has TIF_X32 set, but is running in x86_64 2978 * 2979 * X86_64 - CS,DS,SS,ES are all zero based. 2980 */ 2981 static unsigned long code_segment_base(struct pt_regs *regs) 2982 { 2983 /* 2984 * For IA32 we look at the GDT/LDT segment base to convert the 2985 * effective IP to a linear address. 2986 */ 2987 2988 #ifdef CONFIG_X86_32 2989 /* 2990 * If we are in VM86 mode, add the segment offset to convert to a 2991 * linear address. 2992 */ 2993 if (regs->flags & X86_VM_MASK) 2994 return 0x10 * regs->cs; 2995 2996 if (user_mode(regs) && regs->cs != __USER_CS) 2997 return get_segment_base(regs->cs); 2998 #else 2999 if (user_mode(regs) && !user_64bit_mode(regs) && 3000 regs->cs != __USER32_CS) 3001 return get_segment_base(regs->cs); 3002 #endif 3003 return 0; 3004 } 3005 3006 unsigned long perf_arch_instruction_pointer(struct pt_regs *regs) 3007 { 3008 return regs->ip + code_segment_base(regs); 3009 } 3010 3011 static unsigned long common_misc_flags(struct pt_regs *regs) 3012 { 3013 if (regs->flags & PERF_EFLAGS_EXACT) 3014 return PERF_RECORD_MISC_EXACT_IP; 3015 3016 return 0; 3017 } 3018 3019 static unsigned long guest_misc_flags(struct pt_regs *regs) 3020 { 3021 unsigned long guest_state = perf_guest_state(); 3022 3023 if (!(guest_state & PERF_GUEST_ACTIVE)) 3024 return 0; 3025 3026 if (guest_state & PERF_GUEST_USER) 3027 return PERF_RECORD_MISC_GUEST_USER; 3028 else 3029 return PERF_RECORD_MISC_GUEST_KERNEL; 3030 3031 } 3032 3033 static unsigned long host_misc_flags(struct pt_regs *regs) 3034 { 3035 if (user_mode(regs)) 3036 return PERF_RECORD_MISC_USER; 3037 else 3038 return PERF_RECORD_MISC_KERNEL; 3039 } 3040 3041 unsigned long perf_arch_guest_misc_flags(struct pt_regs *regs) 3042 { 3043 unsigned long flags = common_misc_flags(regs); 3044 3045 flags |= guest_misc_flags(regs); 3046 3047 return flags; 3048 } 3049 3050 unsigned long perf_arch_misc_flags(struct pt_regs *regs) 3051 { 3052 unsigned long flags = common_misc_flags(regs); 3053 3054 flags |= host_misc_flags(regs); 3055 3056 return flags; 3057 } 3058 3059 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 3060 { 3061 /* This API doesn't currently support enumerating hybrid PMUs. */ 3062 if (WARN_ON_ONCE(cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) || 3063 !x86_pmu_initialized()) { 3064 memset(cap, 0, sizeof(*cap)); 3065 return; 3066 } 3067 3068 /* 3069 * Note, hybrid CPU models get tracked as having hybrid PMUs even when 3070 * all E-cores are disabled via BIOS. When E-cores are disabled, the 3071 * base PMU holds the correct number of counters for P-cores. 3072 */ 3073 cap->version = x86_pmu.version; 3074 cap->num_counters_gp = x86_pmu_num_counters(NULL); 3075 cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL); 3076 cap->bit_width_gp = x86_pmu.cntval_bits; 3077 cap->bit_width_fixed = x86_pmu.cntval_bits; 3078 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 3079 cap->events_mask_len = x86_pmu.events_mask_len; 3080 cap->pebs_ept = x86_pmu.pebs_ept; 3081 } 3082 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 3083 3084 u64 perf_get_hw_event_config(int hw_event) 3085 { 3086 int max = x86_pmu.max_events; 3087 3088 if (hw_event < max) 3089 return x86_pmu.event_map(array_index_nospec(hw_event, max)); 3090 3091 return 0; 3092 } 3093 EXPORT_SYMBOL_GPL(perf_get_hw_event_config); 3094