1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <tglx@kernel.org> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/kvm_types.h> 24 #include <linux/sched/mm.h> 25 #include <linux/sched/clock.h> 26 #include <linux/uaccess.h> 27 #include <linux/slab.h> 28 #include <linux/cpu.h> 29 #include <linux/bitops.h> 30 #include <linux/device.h> 31 #include <linux/nospec.h> 32 #include <linux/static_call.h> 33 #include <linux/kvm_types.h> 34 35 #include <asm/apic.h> 36 #include <asm/stacktrace.h> 37 #include <asm/msr.h> 38 #include <asm/nmi.h> 39 #include <asm/smp.h> 40 #include <asm/alternative.h> 41 #include <asm/mmu_context.h> 42 #include <asm/tlbflush.h> 43 #include <asm/timer.h> 44 #include <asm/desc.h> 45 #include <asm/ldt.h> 46 #include <asm/unwind.h> 47 #include <asm/uprobes.h> 48 #include <asm/ibt.h> 49 50 #include "perf_event.h" 51 52 struct x86_pmu x86_pmu __read_mostly; 53 static struct pmu pmu; 54 55 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 56 .enabled = 1, 57 .pmu = &pmu, 58 }; 59 60 static DEFINE_PER_CPU(bool, guest_lvtpc_loaded); 61 62 DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key); 63 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key); 64 DEFINE_STATIC_KEY_FALSE(perf_is_hybrid); 65 66 /* 67 * This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined 68 * from just a typename, as opposed to an actual function. 69 */ 70 DEFINE_STATIC_CALL_NULL(x86_pmu_handle_irq, *x86_pmu.handle_irq); 71 DEFINE_STATIC_CALL_NULL(x86_pmu_disable_all, *x86_pmu.disable_all); 72 DEFINE_STATIC_CALL_NULL(x86_pmu_enable_all, *x86_pmu.enable_all); 73 DEFINE_STATIC_CALL_NULL(x86_pmu_enable, *x86_pmu.enable); 74 DEFINE_STATIC_CALL_NULL(x86_pmu_disable, *x86_pmu.disable); 75 76 DEFINE_STATIC_CALL_NULL(x86_pmu_assign, *x86_pmu.assign); 77 78 DEFINE_STATIC_CALL_NULL(x86_pmu_add, *x86_pmu.add); 79 DEFINE_STATIC_CALL_NULL(x86_pmu_del, *x86_pmu.del); 80 DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read); 81 82 DEFINE_STATIC_CALL_NULL(x86_pmu_set_period, *x86_pmu.set_period); 83 DEFINE_STATIC_CALL_NULL(x86_pmu_update, *x86_pmu.update); 84 DEFINE_STATIC_CALL_NULL(x86_pmu_limit_period, *x86_pmu.limit_period); 85 86 DEFINE_STATIC_CALL_NULL(x86_pmu_schedule_events, *x86_pmu.schedule_events); 87 DEFINE_STATIC_CALL_NULL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints); 88 DEFINE_STATIC_CALL_NULL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints); 89 90 DEFINE_STATIC_CALL_NULL(x86_pmu_start_scheduling, *x86_pmu.start_scheduling); 91 DEFINE_STATIC_CALL_NULL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling); 92 DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling); 93 94 DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task, *x86_pmu.sched_task); 95 96 DEFINE_STATIC_CALL_RET0(x86_pmu_drain_pebs, *x86_pmu.drain_pebs); 97 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases); 98 99 DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter); 100 101 DEFINE_STATIC_CALL_NULL(x86_pmu_late_setup, *x86_pmu.late_setup); 102 103 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_enable, *x86_pmu.pebs_enable); 104 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_disable, *x86_pmu.pebs_disable); 105 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_enable_all, *x86_pmu.pebs_enable_all); 106 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_disable_all, *x86_pmu.pebs_disable_all); 107 108 /* 109 * This one is magic, it will get called even when PMU init fails (because 110 * there is no PMU), in which case it should simply return NULL. 111 */ 112 DEFINE_STATIC_CALL_RET0(x86_pmu_guest_get_msrs, *x86_pmu.guest_get_msrs); 113 114 u64 __read_mostly hw_cache_event_ids 115 [PERF_COUNT_HW_CACHE_MAX] 116 [PERF_COUNT_HW_CACHE_OP_MAX] 117 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 118 u64 __read_mostly hw_cache_extra_regs 119 [PERF_COUNT_HW_CACHE_MAX] 120 [PERF_COUNT_HW_CACHE_OP_MAX] 121 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 122 123 /* 124 * Propagate event elapsed time into the generic event. 125 * Can only be executed on the CPU where the event is active. 126 * Returns the delta events processed. 127 */ 128 u64 x86_perf_event_update(struct perf_event *event) 129 { 130 struct hw_perf_event *hwc = &event->hw; 131 int shift = 64 - x86_pmu.cntval_bits; 132 u64 prev_raw_count, new_raw_count; 133 u64 delta; 134 135 if (unlikely(!hwc->event_base)) 136 return 0; 137 138 /* 139 * Careful: an NMI might modify the previous event value. 140 * 141 * Our tactic to handle this is to first atomically read and 142 * exchange a new raw count - then add that new-prev delta 143 * count to the generic event atomically: 144 */ 145 prev_raw_count = local64_read(&hwc->prev_count); 146 do { 147 new_raw_count = rdpmc(hwc->event_base_rdpmc); 148 } while (!local64_try_cmpxchg(&hwc->prev_count, 149 &prev_raw_count, new_raw_count)); 150 151 /* 152 * Now we have the new raw value and have updated the prev 153 * timestamp already. We can now calculate the elapsed delta 154 * (event-)time and add that to the generic event. 155 * 156 * Careful, not all hw sign-extends above the physical width 157 * of the count. 158 */ 159 delta = (new_raw_count << shift) - (prev_raw_count << shift); 160 delta >>= shift; 161 162 local64_add(delta, &event->count); 163 local64_sub(delta, &hwc->period_left); 164 165 return new_raw_count; 166 } 167 168 /* 169 * Find and validate any extra registers to set up. 170 */ 171 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 172 { 173 struct extra_reg *extra_regs = hybrid(event->pmu, extra_regs); 174 struct hw_perf_event_extra *reg; 175 struct extra_reg *er; 176 177 reg = &event->hw.extra_reg; 178 179 if (!extra_regs) 180 return 0; 181 182 for (er = extra_regs; er->msr; er++) { 183 if (er->event != (config & er->config_mask)) 184 continue; 185 if (event->attr.config1 & ~er->valid_mask) 186 return -EINVAL; 187 /* Check if the extra msrs can be safely accessed*/ 188 if (!er->extra_msr_access) 189 return -ENXIO; 190 191 reg->idx = er->idx; 192 reg->config = event->attr.config1; 193 reg->reg = er->msr; 194 break; 195 } 196 return 0; 197 } 198 199 static atomic_t active_events; 200 static atomic_t pmc_refcount; 201 static DEFINE_MUTEX(pmc_reserve_mutex); 202 203 #ifdef CONFIG_X86_LOCAL_APIC 204 205 static inline u64 get_possible_counter_mask(void) 206 { 207 u64 cntr_mask = x86_pmu.cntr_mask64; 208 int i; 209 210 if (!is_hybrid()) 211 return cntr_mask; 212 213 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) 214 cntr_mask |= x86_pmu.hybrid_pmu[i].cntr_mask64; 215 216 return cntr_mask; 217 } 218 219 static bool reserve_pmc_hardware(void) 220 { 221 u64 cntr_mask = get_possible_counter_mask(); 222 int i, end; 223 224 for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) { 225 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 226 goto perfctr_fail; 227 } 228 229 for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) { 230 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 231 goto eventsel_fail; 232 } 233 234 return true; 235 236 eventsel_fail: 237 end = i; 238 for_each_set_bit(i, (unsigned long *)&cntr_mask, end) 239 release_evntsel_nmi(x86_pmu_config_addr(i)); 240 i = X86_PMC_IDX_MAX; 241 242 perfctr_fail: 243 end = i; 244 for_each_set_bit(i, (unsigned long *)&cntr_mask, end) 245 release_perfctr_nmi(x86_pmu_event_addr(i)); 246 247 return false; 248 } 249 250 static void release_pmc_hardware(void) 251 { 252 u64 cntr_mask = get_possible_counter_mask(); 253 int i; 254 255 for_each_set_bit(i, (unsigned long *)&cntr_mask, X86_PMC_IDX_MAX) { 256 release_perfctr_nmi(x86_pmu_event_addr(i)); 257 release_evntsel_nmi(x86_pmu_config_addr(i)); 258 } 259 } 260 261 #else 262 263 static bool reserve_pmc_hardware(void) { return true; } 264 static void release_pmc_hardware(void) {} 265 266 #endif 267 268 bool check_hw_exists(unsigned long *cntr_mask, 269 unsigned long *fixed_cntr_mask) 270 { 271 u64 val, val_fail = -1, val_new= ~0; 272 int i, reg, reg_fail = -1, ret = 0; 273 int bios_fail = 0; 274 int reg_safe = -1; 275 276 /* 277 * Check to see if the BIOS enabled any of the counters, if so 278 * complain and bail. 279 */ 280 for_each_set_bit(i, cntr_mask, X86_PMC_IDX_MAX) { 281 reg = x86_pmu_config_addr(i); 282 ret = rdmsrq_safe(reg, &val); 283 if (ret) 284 goto msr_fail; 285 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 286 bios_fail = 1; 287 val_fail = val; 288 reg_fail = reg; 289 } else { 290 reg_safe = i; 291 } 292 } 293 294 if (*(u64 *)fixed_cntr_mask) { 295 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 296 ret = rdmsrq_safe(reg, &val); 297 if (ret) 298 goto msr_fail; 299 for_each_set_bit(i, fixed_cntr_mask, X86_PMC_IDX_MAX) { 300 if (val & (0x03ULL << i*4)) { 301 bios_fail = 1; 302 val_fail = val; 303 reg_fail = reg; 304 } 305 } 306 } 307 308 /* 309 * If all the counters are enabled, the below test will always 310 * fail. The tools will also become useless in this scenario. 311 * Just fail and disable the hardware counters. 312 */ 313 314 if (reg_safe == -1) { 315 reg = reg_safe; 316 goto msr_fail; 317 } 318 319 /* 320 * Read the current value, change it and read it back to see if it 321 * matches, this is needed to detect certain hardware emulators 322 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 323 */ 324 reg = x86_pmu_event_addr(reg_safe); 325 if (rdmsrq_safe(reg, &val)) 326 goto msr_fail; 327 val ^= 0xffffUL; 328 ret = wrmsrq_safe(reg, val); 329 ret |= rdmsrq_safe(reg, &val_new); 330 if (ret || val != val_new) 331 goto msr_fail; 332 333 /* 334 * We still allow the PMU driver to operate: 335 */ 336 if (bios_fail) { 337 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 338 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 339 reg_fail, val_fail); 340 } 341 342 return true; 343 344 msr_fail: 345 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 346 pr_cont("PMU not available due to virtualization, using software events only.\n"); 347 } else { 348 pr_cont("Broken PMU hardware detected, using software events only.\n"); 349 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 350 reg, val_new); 351 } 352 353 return false; 354 } 355 356 static void hw_perf_event_destroy(struct perf_event *event) 357 { 358 x86_release_hardware(); 359 atomic_dec(&active_events); 360 } 361 362 void hw_perf_lbr_event_destroy(struct perf_event *event) 363 { 364 hw_perf_event_destroy(event); 365 366 /* undo the lbr/bts event accounting */ 367 x86_del_exclusive(x86_lbr_exclusive_lbr); 368 } 369 370 static inline int x86_pmu_initialized(void) 371 { 372 return x86_pmu.handle_irq != NULL; 373 } 374 375 static inline int 376 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 377 { 378 struct perf_event_attr *attr = &event->attr; 379 unsigned int cache_type, cache_op, cache_result; 380 u64 config, val; 381 382 config = attr->config; 383 384 cache_type = (config >> 0) & 0xff; 385 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 386 return -EINVAL; 387 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX); 388 389 cache_op = (config >> 8) & 0xff; 390 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 391 return -EINVAL; 392 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX); 393 394 cache_result = (config >> 16) & 0xff; 395 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 396 return -EINVAL; 397 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX); 398 399 val = hybrid_var(event->pmu, hw_cache_event_ids)[cache_type][cache_op][cache_result]; 400 if (val == 0) 401 return -ENOENT; 402 403 if (val == -1) 404 return -EINVAL; 405 406 hwc->config |= val; 407 attr->config1 = hybrid_var(event->pmu, hw_cache_extra_regs)[cache_type][cache_op][cache_result]; 408 return x86_pmu_extra_regs(val, event); 409 } 410 411 static DEFINE_PER_CPU(struct xregs_state *, ext_regs_buf); 412 413 static void release_ext_regs_buffers(void) 414 { 415 int cpu; 416 417 if (!x86_pmu.ext_regs_mask) 418 return; 419 420 for_each_possible_cpu(cpu) { 421 kfree(per_cpu(ext_regs_buf, cpu)); 422 per_cpu(ext_regs_buf, cpu) = NULL; 423 } 424 } 425 426 static void reserve_ext_regs_buffers(void) 427 { 428 bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); 429 unsigned int size; 430 int cpu; 431 432 if (!x86_pmu.ext_regs_mask) 433 return; 434 435 /* Add 64 bytes to satisfy the XSAVE area's 64-byte alignment. */ 436 size = xstate_calculate_size(x86_pmu.ext_regs_mask, compacted) + 64; 437 438 for_each_possible_cpu(cpu) { 439 per_cpu(ext_regs_buf, cpu) = kzalloc_node(size, GFP_KERNEL, 440 cpu_to_node(cpu)); 441 if (!per_cpu(ext_regs_buf, cpu)) 442 goto err; 443 } 444 445 return; 446 447 err: 448 release_ext_regs_buffers(); 449 } 450 451 static inline struct xregs_state *get_ext_regs_buf(int cpu) 452 { 453 void *buf = per_cpu(ext_regs_buf, cpu); 454 455 return buf ? PTR_ALIGN(buf, 64) : NULL; 456 } 457 458 int x86_reserve_hardware(void) 459 { 460 int err = 0; 461 462 if (!atomic_inc_not_zero(&pmc_refcount)) { 463 mutex_lock(&pmc_reserve_mutex); 464 if (atomic_read(&pmc_refcount) == 0) { 465 if (!reserve_pmc_hardware()) { 466 err = -EBUSY; 467 } else { 468 reserve_ds_buffers(); 469 reserve_lbr_buffers(); 470 reserve_ext_regs_buffers(); 471 } 472 } 473 if (!err) 474 atomic_inc(&pmc_refcount); 475 mutex_unlock(&pmc_reserve_mutex); 476 } 477 478 return err; 479 } 480 481 void x86_release_hardware(void) 482 { 483 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 484 release_pmc_hardware(); 485 release_ds_buffers(); 486 release_lbr_buffers(); 487 release_ext_regs_buffers(); 488 mutex_unlock(&pmc_reserve_mutex); 489 } 490 } 491 492 /* 493 * Check if we can create event of a certain type (that no conflicting events 494 * are present). 495 */ 496 int x86_add_exclusive(unsigned int what) 497 { 498 int i; 499 500 /* 501 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 502 * LBR and BTS are still mutually exclusive. 503 */ 504 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 505 goto out; 506 507 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 508 mutex_lock(&pmc_reserve_mutex); 509 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 510 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 511 goto fail_unlock; 512 } 513 atomic_inc(&x86_pmu.lbr_exclusive[what]); 514 mutex_unlock(&pmc_reserve_mutex); 515 } 516 517 out: 518 atomic_inc(&active_events); 519 return 0; 520 521 fail_unlock: 522 mutex_unlock(&pmc_reserve_mutex); 523 return -EBUSY; 524 } 525 526 void x86_del_exclusive(unsigned int what) 527 { 528 atomic_dec(&active_events); 529 530 /* 531 * See the comment in x86_add_exclusive(). 532 */ 533 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 534 return; 535 536 atomic_dec(&x86_pmu.lbr_exclusive[what]); 537 } 538 539 int x86_setup_perfctr(struct perf_event *event) 540 { 541 struct perf_event_attr *attr = &event->attr; 542 struct hw_perf_event *hwc = &event->hw; 543 u64 config; 544 545 if (!is_sampling_event(event)) { 546 hwc->sample_period = x86_pmu.max_period; 547 hwc->last_period = hwc->sample_period; 548 local64_set(&hwc->period_left, hwc->sample_period); 549 } 550 551 if (attr->type == event->pmu->type) 552 return x86_pmu_extra_regs(event->attr.config, event); 553 554 if (attr->type == PERF_TYPE_HW_CACHE) 555 return set_ext_hw_attr(hwc, event); 556 557 if (attr->config >= x86_pmu.max_events) 558 return -EINVAL; 559 560 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events); 561 562 /* 563 * The generic map: 564 */ 565 config = x86_pmu.event_map(attr->config); 566 567 if (config == 0) 568 return -ENOENT; 569 570 if (config == -1LL) 571 return -EINVAL; 572 573 hwc->config |= config; 574 575 return 0; 576 } 577 578 /* 579 * check that branch_sample_type is compatible with 580 * settings needed for precise_ip > 1 which implies 581 * using the LBR to capture ALL taken branches at the 582 * priv levels of the measurement 583 */ 584 static inline int precise_br_compat(struct perf_event *event) 585 { 586 u64 m = event->attr.branch_sample_type; 587 u64 b = 0; 588 589 /* must capture all branches */ 590 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 591 return 0; 592 593 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 594 595 if (!event->attr.exclude_user) 596 b |= PERF_SAMPLE_BRANCH_USER; 597 598 if (!event->attr.exclude_kernel) 599 b |= PERF_SAMPLE_BRANCH_KERNEL; 600 601 /* 602 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 603 */ 604 605 return m == b; 606 } 607 608 int x86_pmu_max_precise(struct pmu *pmu) 609 { 610 int precise = 0; 611 612 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 613 /* arch PEBS */ 614 if (x86_pmu.arch_pebs) { 615 precise = 2; 616 if (hybrid(pmu, arch_pebs_cap).pdists) 617 precise++; 618 619 return precise; 620 } 621 622 /* legacy PEBS - support for constant skid */ 623 precise++; 624 /* Support for IP fixup */ 625 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 626 precise++; 627 628 if (x86_pmu.pebs_prec_dist) 629 precise++; 630 } 631 632 return precise; 633 } 634 635 static int pebs_simd_regs_validate(struct perf_event *event) 636 { 637 u64 caps = hybrid(event->pmu, arch_pebs_cap).caps; 638 639 if (event_needs_xmm(event) && 640 !x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline) 641 return -EINVAL; 642 if (event_needs_xmm(event) && 643 x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) 644 return -EINVAL; 645 646 if (event_needs_ssp(event) && 647 !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_GPR))) 648 return -EINVAL; 649 if (event_needs_ymm(event) && 650 !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_YMMH))) 651 return -EINVAL; 652 if (event_needs_egprs(event) && 653 !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_EGPRS))) 654 return -EINVAL; 655 if (event_needs_opmask(event) && 656 !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_OPMASK))) 657 return -EINVAL; 658 if (event_needs_low16_zmm(event) && 659 !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_ZMMH))) 660 return -EINVAL; 661 if (event_needs_high16_zmm(event) && 662 !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_H16ZMM))) 663 return -EINVAL; 664 665 return 0; 666 } 667 668 static int event_simd_regs_validate(struct perf_event *event) 669 { 670 u64 reserved = ~GENMASK_ULL(PERF_REG_MISC_MAX - 1, 0); 671 672 if (!get_ext_regs_buf(raw_smp_processor_id())) 673 return -ENOMEM; 674 /* 675 * The XMM space in the perf_event_x86_regs is reclaimed 676 * for eGPRs and other general registers. 677 */ 678 if (((event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && 679 (event->attr.sample_regs_intr & reserved)) || 680 ((event->attr.sample_type & PERF_SAMPLE_REGS_USER) && 681 (event->attr.sample_regs_user & reserved))) 682 return -EINVAL; 683 if (event_needs_egprs(event) && 684 !(x86_pmu.ext_regs_mask & XFEATURE_MASK_APX)) 685 return -EINVAL; 686 if (event_needs_ssp(event) && 687 !(x86_pmu.ext_regs_mask & XFEATURE_MASK_CET_USER)) 688 return -EINVAL; 689 if (event_needs_xmm(event) && 690 !(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE)) 691 return -EINVAL; 692 if (event_needs_ymm(event) && 693 !(x86_pmu.ext_regs_mask & XFEATURE_MASK_YMM)) 694 return -EINVAL; 695 if (event_needs_low16_zmm(event) && 696 !(x86_pmu.ext_regs_mask & XFEATURE_MASK_ZMM_Hi256)) 697 return -EINVAL; 698 if (event_needs_high16_zmm(event) && 699 !(x86_pmu.ext_regs_mask & XFEATURE_MASK_Hi16_ZMM)) 700 return -EINVAL; 701 if (event_needs_opmask(event) && 702 !(x86_pmu.ext_regs_mask & XFEATURE_MASK_OPMASK)) 703 return -EINVAL; 704 705 return 0; 706 } 707 708 int x86_pmu_hw_config(struct perf_event *event) 709 { 710 if (event->attr.precise_ip) { 711 int precise = x86_pmu_max_precise(event->pmu); 712 713 if (event->attr.precise_ip > precise) 714 return -EOPNOTSUPP; 715 716 /* There's no sense in having PEBS for non sampling events: */ 717 if (!is_sampling_event(event)) 718 return -EINVAL; 719 } 720 /* 721 * check that PEBS LBR correction does not conflict with 722 * whatever the user is asking with attr->branch_sample_type 723 */ 724 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 725 u64 *br_type = &event->attr.branch_sample_type; 726 727 if (has_branch_stack(event)) { 728 if (!precise_br_compat(event)) 729 return -EOPNOTSUPP; 730 731 /* branch_sample_type is compatible */ 732 733 } else { 734 /* 735 * user did not specify branch_sample_type 736 * 737 * For PEBS fixups, we capture all 738 * the branches at the priv level of the 739 * event. 740 */ 741 *br_type = PERF_SAMPLE_BRANCH_ANY; 742 743 if (!event->attr.exclude_user) 744 *br_type |= PERF_SAMPLE_BRANCH_USER; 745 746 if (!event->attr.exclude_kernel) 747 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 748 } 749 } 750 751 if (branch_sample_call_stack(event)) 752 event->attach_state |= PERF_ATTACH_TASK_DATA; 753 754 /* 755 * Generate PMC IRQs: 756 * (keep 'enabled' bit clear for now) 757 */ 758 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 759 760 /* 761 * Count user and OS events unless requested not to 762 */ 763 if (!event->attr.exclude_user) 764 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 765 if (!event->attr.exclude_kernel) 766 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 767 768 if (event->attr.type == event->pmu->type) 769 event->hw.config |= x86_pmu_get_event_config(event); 770 771 if (is_sampling_event(event) && !event->attr.freq && x86_pmu.limit_period) { 772 s64 left = event->attr.sample_period; 773 x86_pmu.limit_period(event, &left); 774 if (left > event->attr.sample_period) 775 return -EINVAL; 776 } 777 778 if (event->attr.sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) { 779 int ret; 780 781 if (event->attr.sample_simd_regs_enabled) { 782 if (!(event->pmu->capabilities & PERF_PMU_CAP_SIMD_REGS)) 783 return -EINVAL; 784 785 if (event->attr.precise_ip) { 786 ret = pebs_simd_regs_validate(event); 787 if (ret) 788 return ret; 789 } 790 ret = event_simd_regs_validate(event); 791 if (ret) 792 return ret; 793 } else if (event_has_extended_regs(event)) { 794 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) 795 return -EINVAL; 796 797 if (event->attr.precise_ip) { 798 u64 caps = hybrid(event->pmu, arch_pebs_cap).caps; 799 800 if (x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) 801 return -EINVAL; 802 if (!x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline) 803 return -EINVAL; 804 } 805 if (!get_ext_regs_buf(raw_smp_processor_id())) 806 return -ENOMEM; 807 if (!(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE)) 808 return -EINVAL; 809 } 810 } 811 812 return x86_setup_perfctr(event); 813 } 814 815 /* 816 * Setup the hardware configuration for a given attr_type 817 */ 818 static int __x86_pmu_event_init(struct perf_event *event) 819 { 820 int err; 821 822 if (!x86_pmu_initialized()) 823 return -ENODEV; 824 825 err = x86_reserve_hardware(); 826 if (err) 827 return err; 828 829 atomic_inc(&active_events); 830 event->destroy = hw_perf_event_destroy; 831 832 event->hw.idx = -1; 833 event->hw.last_cpu = -1; 834 event->hw.last_tag = ~0ULL; 835 event->hw.dyn_constraint = ~0ULL; 836 837 /* mark unused */ 838 event->hw.extra_reg.idx = EXTRA_REG_NONE; 839 event->hw.branch_reg.idx = EXTRA_REG_NONE; 840 841 return x86_pmu.hw_config(event); 842 } 843 844 void x86_pmu_disable_all(void) 845 { 846 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 847 int idx; 848 849 for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { 850 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 851 u64 val; 852 853 if (!test_bit(idx, cpuc->active_mask)) 854 continue; 855 rdmsrq(x86_pmu_config_addr(idx), val); 856 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 857 continue; 858 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 859 wrmsrq(x86_pmu_config_addr(idx), val); 860 if (is_counter_pair(hwc)) 861 wrmsrq(x86_pmu_config_addr(idx + 1), 0); 862 } 863 } 864 865 struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, 866 struct x86_guest_pebs *guest_pebs) 867 { 868 return static_call(x86_pmu_guest_get_msrs)(nr, guest_pebs); 869 } 870 EXPORT_SYMBOL_FOR_KVM(perf_guest_get_msrs); 871 872 /* 873 * There may be PMI landing after enabled=0. The PMI hitting could be before or 874 * after disable_all. 875 * 876 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 877 * It will not be re-enabled in the NMI handler again, because enabled=0. After 878 * handling the NMI, disable_all will be called, which will not change the 879 * state either. If PMI hits after disable_all, the PMU is already disabled 880 * before entering NMI handler. The NMI handler will not change the state 881 * either. 882 * 883 * So either situation is harmless. 884 */ 885 static void x86_pmu_disable(struct pmu *pmu) 886 { 887 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 888 889 if (!x86_pmu_initialized()) 890 return; 891 892 if (!cpuc->enabled) 893 return; 894 895 cpuc->n_added = 0; 896 cpuc->enabled = 0; 897 barrier(); 898 899 static_call(x86_pmu_disable_all)(); 900 } 901 902 void x86_pmu_enable_all(int added) 903 { 904 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 905 int idx; 906 907 for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { 908 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 909 910 if (!test_bit(idx, cpuc->active_mask)) 911 continue; 912 913 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 914 } 915 } 916 917 int is_x86_event(struct perf_event *event) 918 { 919 /* 920 * For a non-hybrid platforms, the type of X86 pmu is 921 * always PERF_TYPE_RAW. 922 * For a hybrid platform, the PERF_PMU_CAP_EXTENDED_HW_TYPE 923 * is a unique capability for the X86 PMU. 924 * Use them to detect a X86 event. 925 */ 926 if (event->pmu->type == PERF_TYPE_RAW || 927 event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE) 928 return true; 929 930 return false; 931 } 932 933 inline struct pmu *x86_get_static_pmu(void) 934 { 935 return &pmu; 936 } 937 938 struct pmu *x86_get_pmu(unsigned int cpu) 939 { 940 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 941 942 /* 943 * All CPUs of the hybrid type have been offline. 944 * The x86_get_pmu() should not be invoked. 945 */ 946 if (WARN_ON_ONCE(!cpuc->pmu)) 947 return &pmu; 948 949 return cpuc->pmu; 950 } 951 /* 952 * Event scheduler state: 953 * 954 * Assign events iterating over all events and counters, beginning 955 * with events with least weights first. Keep the current iterator 956 * state in struct sched_state. 957 */ 958 struct sched_state { 959 int weight; 960 int event; /* event index */ 961 int counter; /* counter index */ 962 int unassigned; /* number of events to be assigned left */ 963 int nr_gp; /* number of GP counters used */ 964 u64 used; 965 }; 966 967 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 968 #define SCHED_STATES_MAX 2 969 970 struct perf_sched { 971 int max_weight; 972 int max_events; 973 int max_gp; 974 int saved_states; 975 struct event_constraint **constraints; 976 struct sched_state state; 977 struct sched_state saved[SCHED_STATES_MAX]; 978 }; 979 980 /* 981 * Initialize iterator that runs through all events and counters. 982 */ 983 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 984 int num, int wmin, int wmax, int gpmax) 985 { 986 int idx; 987 988 memset(sched, 0, sizeof(*sched)); 989 sched->max_events = num; 990 sched->max_weight = wmax; 991 sched->max_gp = gpmax; 992 sched->constraints = constraints; 993 994 for (idx = 0; idx < num; idx++) { 995 if (constraints[idx]->weight == wmin) 996 break; 997 } 998 999 sched->state.event = idx; /* start with min weight */ 1000 sched->state.weight = wmin; 1001 sched->state.unassigned = num; 1002 } 1003 1004 static void perf_sched_save_state(struct perf_sched *sched) 1005 { 1006 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 1007 return; 1008 1009 sched->saved[sched->saved_states] = sched->state; 1010 sched->saved_states++; 1011 } 1012 1013 static bool perf_sched_restore_state(struct perf_sched *sched) 1014 { 1015 if (!sched->saved_states) 1016 return false; 1017 1018 sched->saved_states--; 1019 sched->state = sched->saved[sched->saved_states]; 1020 1021 /* this assignment didn't work out */ 1022 /* XXX broken vs EVENT_PAIR */ 1023 sched->state.used &= ~BIT_ULL(sched->state.counter); 1024 1025 /* try the next one */ 1026 sched->state.counter++; 1027 1028 return true; 1029 } 1030 1031 /* 1032 * Select a counter for the current event to schedule. Return true on 1033 * success. 1034 */ 1035 static bool __perf_sched_find_counter(struct perf_sched *sched) 1036 { 1037 struct event_constraint *c; 1038 int idx; 1039 1040 if (!sched->state.unassigned) 1041 return false; 1042 1043 if (sched->state.event >= sched->max_events) 1044 return false; 1045 1046 c = sched->constraints[sched->state.event]; 1047 /* Prefer fixed purpose counters */ 1048 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 1049 idx = INTEL_PMC_IDX_FIXED; 1050 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 1051 u64 mask = BIT_ULL(idx); 1052 1053 if (sched->state.used & mask) 1054 continue; 1055 1056 sched->state.used |= mask; 1057 goto done; 1058 } 1059 } 1060 1061 /* Grab the first unused counter starting with idx */ 1062 idx = sched->state.counter; 1063 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 1064 u64 mask = BIT_ULL(idx); 1065 1066 if (c->flags & PERF_X86_EVENT_PAIR) 1067 mask |= mask << 1; 1068 1069 if (sched->state.used & mask) 1070 continue; 1071 1072 if (sched->state.nr_gp++ >= sched->max_gp) 1073 return false; 1074 1075 sched->state.used |= mask; 1076 goto done; 1077 } 1078 1079 return false; 1080 1081 done: 1082 sched->state.counter = idx; 1083 1084 if (c->overlap) 1085 perf_sched_save_state(sched); 1086 1087 return true; 1088 } 1089 1090 static bool perf_sched_find_counter(struct perf_sched *sched) 1091 { 1092 while (!__perf_sched_find_counter(sched)) { 1093 if (!perf_sched_restore_state(sched)) 1094 return false; 1095 } 1096 1097 return true; 1098 } 1099 1100 /* 1101 * Go through all unassigned events and find the next one to schedule. 1102 * Take events with the least weight first. Return true on success. 1103 */ 1104 static bool perf_sched_next_event(struct perf_sched *sched) 1105 { 1106 struct event_constraint *c; 1107 1108 if (!sched->state.unassigned || !--sched->state.unassigned) 1109 return false; 1110 1111 do { 1112 /* next event */ 1113 sched->state.event++; 1114 if (sched->state.event >= sched->max_events) { 1115 /* next weight */ 1116 sched->state.event = 0; 1117 sched->state.weight++; 1118 if (sched->state.weight > sched->max_weight) 1119 return false; 1120 } 1121 c = sched->constraints[sched->state.event]; 1122 } while (c->weight != sched->state.weight); 1123 1124 sched->state.counter = 0; /* start with first counter */ 1125 1126 return true; 1127 } 1128 1129 /* 1130 * Assign a counter for each event. 1131 */ 1132 int perf_assign_events(struct event_constraint **constraints, int n, 1133 int wmin, int wmax, int gpmax, int *assign) 1134 { 1135 struct perf_sched sched; 1136 1137 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 1138 1139 do { 1140 if (!perf_sched_find_counter(&sched)) 1141 break; /* failed */ 1142 if (assign) 1143 assign[sched.state.event] = sched.state.counter; 1144 } while (perf_sched_next_event(&sched)); 1145 1146 return sched.state.unassigned; 1147 } 1148 EXPORT_SYMBOL_GPL(perf_assign_events); 1149 1150 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 1151 { 1152 struct event_constraint *c; 1153 struct perf_event *e; 1154 int n0, i, wmin, wmax, unsched = 0; 1155 struct hw_perf_event *hwc; 1156 u64 used_mask = 0; 1157 1158 /* 1159 * Compute the number of events already present; see x86_pmu_add(), 1160 * validate_group() and x86_pmu_commit_txn(). For the former two 1161 * cpuc->n_events hasn't been updated yet, while for the latter 1162 * cpuc->n_txn contains the number of events added in the current 1163 * transaction. 1164 */ 1165 n0 = cpuc->n_events; 1166 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1167 n0 -= cpuc->n_txn; 1168 1169 static_call_cond(x86_pmu_start_scheduling)(cpuc); 1170 1171 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 1172 c = cpuc->event_constraint[i]; 1173 1174 /* 1175 * Previously scheduled events should have a cached constraint, 1176 * while new events should not have one. 1177 */ 1178 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0)); 1179 1180 /* 1181 * Request constraints for new events; or for those events that 1182 * have a dynamic constraint -- for those the constraint can 1183 * change due to external factors (sibling state, allow_tfa). 1184 */ 1185 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) { 1186 c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]); 1187 cpuc->event_constraint[i] = c; 1188 } 1189 1190 wmin = min(wmin, c->weight); 1191 wmax = max(wmax, c->weight); 1192 } 1193 1194 /* 1195 * fastpath, try to reuse previous register 1196 */ 1197 for (i = 0; i < n; i++) { 1198 u64 mask; 1199 1200 hwc = &cpuc->event_list[i]->hw; 1201 c = cpuc->event_constraint[i]; 1202 1203 /* never assigned */ 1204 if (hwc->idx == -1) 1205 break; 1206 1207 /* constraint still honored */ 1208 if (!test_bit(hwc->idx, c->idxmsk)) 1209 break; 1210 1211 mask = BIT_ULL(hwc->idx); 1212 if (is_counter_pair(hwc)) 1213 mask |= mask << 1; 1214 1215 /* not already used */ 1216 if (used_mask & mask) 1217 break; 1218 1219 used_mask |= mask; 1220 1221 if (assign) 1222 assign[i] = hwc->idx; 1223 } 1224 1225 /* slow path */ 1226 if (i != n) { 1227 int gpmax = x86_pmu_max_num_counters(cpuc->pmu); 1228 1229 /* 1230 * Do not allow scheduling of more than half the available 1231 * generic counters. 1232 * 1233 * This helps avoid counter starvation of sibling thread by 1234 * ensuring at most half the counters cannot be in exclusive 1235 * mode. There is no designated counters for the limits. Any 1236 * N/2 counters can be used. This helps with events with 1237 * specific counter constraints. 1238 */ 1239 if (is_ht_workaround_enabled() && !cpuc->is_fake && 1240 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 1241 gpmax /= 2; 1242 1243 /* 1244 * Reduce the amount of available counters to allow fitting 1245 * the extra Merge events needed by large increment events. 1246 */ 1247 if (x86_pmu.flags & PMU_FL_PAIR) { 1248 gpmax -= cpuc->n_pair; 1249 WARN_ON(gpmax <= 0); 1250 } 1251 1252 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 1253 wmax, gpmax, assign); 1254 } 1255 1256 /* 1257 * In case of success (unsched = 0), mark events as committed, 1258 * so we do not put_constraint() in case new events are added 1259 * and fail to be scheduled 1260 * 1261 * We invoke the lower level commit callback to lock the resource 1262 * 1263 * We do not need to do all of this in case we are called to 1264 * validate an event group (assign == NULL) 1265 */ 1266 if (!unsched && assign) { 1267 for (i = 0; i < n; i++) 1268 static_call_cond(x86_pmu_commit_scheduling)(cpuc, i, assign[i]); 1269 } else { 1270 for (i = n0; i < n; i++) { 1271 e = cpuc->event_list[i]; 1272 1273 /* 1274 * release events that failed scheduling 1275 */ 1276 static_call_cond(x86_pmu_put_event_constraints)(cpuc, e); 1277 1278 cpuc->event_constraint[i] = NULL; 1279 } 1280 } 1281 1282 static_call_cond(x86_pmu_stop_scheduling)(cpuc); 1283 1284 return unsched ? -EINVAL : 0; 1285 } 1286 1287 static int add_nr_metric_event(struct cpu_hw_events *cpuc, 1288 struct perf_event *event) 1289 { 1290 if (is_metric_event(event)) { 1291 if (cpuc->n_metric == INTEL_TD_METRIC_NUM) 1292 return -EINVAL; 1293 cpuc->n_metric++; 1294 cpuc->n_txn_metric++; 1295 } 1296 1297 return 0; 1298 } 1299 1300 static void del_nr_metric_event(struct cpu_hw_events *cpuc, 1301 struct perf_event *event) 1302 { 1303 if (is_metric_event(event)) 1304 cpuc->n_metric--; 1305 } 1306 1307 static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event, 1308 int max_count, int n) 1309 { 1310 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap); 1311 1312 if (intel_cap.perf_metrics && add_nr_metric_event(cpuc, event)) 1313 return -EINVAL; 1314 1315 if (n >= max_count + cpuc->n_metric) 1316 return -EINVAL; 1317 1318 cpuc->event_list[n] = event; 1319 if (is_counter_pair(&event->hw)) { 1320 cpuc->n_pair++; 1321 cpuc->n_txn_pair++; 1322 } 1323 1324 return 0; 1325 } 1326 1327 /* 1328 * dogrp: true if must collect siblings events (group) 1329 * returns total number of events and error code 1330 */ 1331 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 1332 { 1333 struct perf_event *event; 1334 int n, max_count; 1335 1336 max_count = x86_pmu_num_counters(cpuc->pmu) + x86_pmu_num_counters_fixed(cpuc->pmu); 1337 1338 /* current number of events already accepted */ 1339 n = cpuc->n_events; 1340 if (!cpuc->n_events) 1341 cpuc->pebs_output = 0; 1342 1343 if (!cpuc->is_fake && leader->attr.precise_ip) { 1344 /* 1345 * For PEBS->PT, if !aux_event, the group leader (PT) went 1346 * away, the group was broken down and this singleton event 1347 * can't schedule any more. 1348 */ 1349 if (is_pebs_pt(leader) && !leader->aux_event) 1350 return -EINVAL; 1351 1352 /* 1353 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS 1354 */ 1355 if (cpuc->pebs_output && 1356 cpuc->pebs_output != is_pebs_pt(leader) + 1) 1357 return -EINVAL; 1358 1359 cpuc->pebs_output = is_pebs_pt(leader) + 1; 1360 } 1361 1362 if (is_x86_event(leader)) { 1363 if (collect_event(cpuc, leader, max_count, n)) 1364 return -EINVAL; 1365 n++; 1366 } 1367 1368 if (!dogrp) 1369 return n; 1370 1371 for_each_sibling_event(event, leader) { 1372 if (!is_x86_event(event) || event->state <= PERF_EVENT_STATE_OFF) 1373 continue; 1374 1375 if (collect_event(cpuc, event, max_count, n)) 1376 return -EINVAL; 1377 1378 n++; 1379 } 1380 return n; 1381 } 1382 1383 static inline void x86_assign_hw_event(struct perf_event *event, 1384 struct cpu_hw_events *cpuc, int i) 1385 { 1386 struct hw_perf_event *hwc = &event->hw; 1387 int idx; 1388 1389 idx = hwc->idx = cpuc->assign[i]; 1390 hwc->last_cpu = smp_processor_id(); 1391 hwc->last_tag = ++cpuc->tags[i]; 1392 1393 static_call_cond(x86_pmu_assign)(event, idx); 1394 1395 switch (hwc->idx) { 1396 case INTEL_PMC_IDX_FIXED_BTS: 1397 case INTEL_PMC_IDX_FIXED_VLBR: 1398 hwc->config_base = 0; 1399 hwc->event_base = 0; 1400 break; 1401 1402 case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END: 1403 /* All the metric events are mapped onto the fixed counter 3. */ 1404 idx = INTEL_PMC_IDX_FIXED_SLOTS; 1405 fallthrough; 1406 case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1: 1407 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1408 hwc->event_base = x86_pmu_fixed_ctr_addr(idx - INTEL_PMC_IDX_FIXED); 1409 hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) | 1410 INTEL_PMC_FIXED_RDPMC_BASE; 1411 break; 1412 1413 default: 1414 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1415 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1416 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1417 break; 1418 } 1419 } 1420 1421 /** 1422 * x86_perf_rdpmc_index - Return PMC counter used for event 1423 * @event: the perf_event to which the PMC counter was assigned 1424 * 1425 * The counter assigned to this performance event may change if interrupts 1426 * are enabled. This counter should thus never be used while interrupts are 1427 * enabled. Before this function is used to obtain the assigned counter the 1428 * event should be checked for validity using, for example, 1429 * perf_event_read_local(), within the same interrupt disabled section in 1430 * which this counter is planned to be used. 1431 * 1432 * Return: The index of the performance monitoring counter assigned to 1433 * @perf_event. 1434 */ 1435 int x86_perf_rdpmc_index(struct perf_event *event) 1436 { 1437 lockdep_assert_irqs_disabled(); 1438 1439 return event->hw.event_base_rdpmc; 1440 } 1441 1442 static inline bool acr_match_prev_indices(struct perf_event *event, 1443 struct cpu_hw_events *cpuc) 1444 { 1445 struct hw_perf_event *hwc = &event->hw; 1446 1447 if (!is_acr_event_group(event)) 1448 return true; 1449 /* ACR counter indices don't change. */ 1450 return hwc->config1 == cpuc->acr_cfg_b[hwc->idx]; 1451 } 1452 1453 static inline int match_prev_assignment(struct perf_event *event, 1454 struct cpu_hw_events *cpuc, 1455 int i) 1456 { 1457 struct hw_perf_event *hwc = &event->hw; 1458 1459 return hwc->idx == cpuc->assign[i] && 1460 hwc->last_cpu == smp_processor_id() && 1461 hwc->last_tag == cpuc->tags[i] && 1462 acr_match_prev_indices(event, cpuc); 1463 } 1464 1465 static void x86_pmu_start(struct perf_event *event, int flags); 1466 1467 static void x86_pmu_enable(struct pmu *pmu) 1468 { 1469 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1470 struct perf_event *event; 1471 struct hw_perf_event *hwc; 1472 int i, added = cpuc->n_added; 1473 1474 if (!x86_pmu_initialized()) 1475 return; 1476 1477 if (cpuc->enabled) 1478 return; 1479 1480 if (cpuc->n_added) { 1481 int n_running = cpuc->n_events - cpuc->n_added; 1482 1483 /* 1484 * The late setup (after counters are scheduled) 1485 * is required for some cases, e.g., PEBS counters 1486 * snapshotting. Because an accurate counter index 1487 * is needed. 1488 */ 1489 static_call_cond(x86_pmu_late_setup)(); 1490 1491 /* 1492 * apply assignment obtained either from 1493 * hw_perf_group_sched_in() or x86_pmu_enable() 1494 * 1495 * step1: save events moving to new counters 1496 */ 1497 for (i = 0; i < n_running; i++) { 1498 event = cpuc->event_list[i]; 1499 hwc = &event->hw; 1500 1501 /* 1502 * we can avoid reprogramming counter if: 1503 * - assigned same counter as last time 1504 * - running on same CPU as last time 1505 * - no other event has used the counter since 1506 */ 1507 if (hwc->idx == -1 || 1508 match_prev_assignment(event, cpuc, i)) 1509 continue; 1510 1511 /* 1512 * Ensure we don't accidentally enable a stopped 1513 * counter simply because we rescheduled. 1514 */ 1515 if (hwc->state & PERF_HES_STOPPED) 1516 hwc->state |= PERF_HES_ARCH; 1517 1518 x86_pmu_stop(event, PERF_EF_UPDATE); 1519 cpuc->events[hwc->idx] = NULL; 1520 } 1521 1522 /* 1523 * step2: reprogram moved events into new counters 1524 */ 1525 for (i = 0; i < cpuc->n_events; i++) { 1526 event = cpuc->event_list[i]; 1527 hwc = &event->hw; 1528 1529 if (!match_prev_assignment(event, cpuc, i)) 1530 x86_assign_hw_event(event, cpuc, i); 1531 else if (i < n_running) 1532 continue; 1533 1534 cpuc->events[hwc->idx] = event; 1535 1536 if (hwc->state & PERF_HES_ARCH) { 1537 static_call(x86_pmu_set_period)(event); 1538 continue; 1539 } 1540 1541 /* 1542 * if cpuc->enabled = 0, then no wrmsr as 1543 * per x86_pmu_enable_event() 1544 */ 1545 x86_pmu_start(event, PERF_EF_RELOAD); 1546 } 1547 cpuc->n_added = 0; 1548 perf_events_lapic_init(); 1549 } 1550 1551 cpuc->enabled = 1; 1552 barrier(); 1553 1554 static_call(x86_pmu_enable_all)(added); 1555 } 1556 1557 DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1558 1559 /* 1560 * Set the next IRQ period, based on the hwc->period_left value. 1561 * To be called with the event disabled in hw: 1562 */ 1563 int x86_perf_event_set_period(struct perf_event *event) 1564 { 1565 struct hw_perf_event *hwc = &event->hw; 1566 s64 left = local64_read(&hwc->period_left); 1567 s64 period = hwc->sample_period; 1568 int ret = 0, idx = hwc->idx; 1569 1570 if (unlikely(!hwc->event_base)) 1571 return 0; 1572 1573 /* 1574 * If we are way outside a reasonable range then just skip forward: 1575 */ 1576 if (unlikely(left <= -period)) { 1577 left = period; 1578 local64_set(&hwc->period_left, left); 1579 hwc->last_period = period; 1580 ret = 1; 1581 } 1582 1583 if (unlikely(left <= 0)) { 1584 left += period; 1585 local64_set(&hwc->period_left, left); 1586 hwc->last_period = period; 1587 ret = 1; 1588 } 1589 /* 1590 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1591 */ 1592 if (unlikely(left < 2)) 1593 left = 2; 1594 1595 if (left > x86_pmu.max_period) 1596 left = x86_pmu.max_period; 1597 1598 static_call_cond(x86_pmu_limit_period)(event, &left); 1599 1600 this_cpu_write(pmc_prev_left[idx], left); 1601 1602 /* 1603 * The hw event starts counting from this event offset, 1604 * mark it to be able to extra future deltas: 1605 */ 1606 local64_set(&hwc->prev_count, (u64)-left); 1607 1608 wrmsrq(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1609 1610 /* 1611 * Sign extend the Merge event counter's upper 16 bits since 1612 * we currently declare a 48-bit counter width 1613 */ 1614 if (is_counter_pair(hwc)) 1615 wrmsrq(x86_pmu_event_addr(idx + 1), 0xffff); 1616 1617 perf_event_update_userpage(event); 1618 1619 return ret; 1620 } 1621 1622 void x86_pmu_enable_event(struct perf_event *event) 1623 { 1624 if (__this_cpu_read(cpu_hw_events.enabled)) 1625 __x86_pmu_enable_event(&event->hw, 1626 ARCH_PERFMON_EVENTSEL_ENABLE); 1627 } 1628 1629 /* 1630 * Add a single event to the PMU. 1631 * 1632 * The event is added to the group of enabled events 1633 * but only if it can be scheduled with existing events. 1634 */ 1635 static int x86_pmu_add(struct perf_event *event, int flags) 1636 { 1637 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1638 struct hw_perf_event *hwc; 1639 int assign[X86_PMC_IDX_MAX]; 1640 int n, n0, ret; 1641 1642 hwc = &event->hw; 1643 1644 n0 = cpuc->n_events; 1645 ret = n = collect_events(cpuc, event, false); 1646 if (ret < 0) 1647 goto out; 1648 1649 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1650 if (!(flags & PERF_EF_START)) 1651 hwc->state |= PERF_HES_ARCH; 1652 1653 /* 1654 * If group events scheduling transaction was started, 1655 * skip the schedulability test here, it will be performed 1656 * at commit time (->commit_txn) as a whole. 1657 * 1658 * If commit fails, we'll call ->del() on all events 1659 * for which ->add() was called. 1660 */ 1661 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1662 goto done_collect; 1663 1664 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign); 1665 if (ret) 1666 goto out; 1667 /* 1668 * copy new assignment, now we know it is possible 1669 * will be used by hw_perf_enable() 1670 */ 1671 memcpy(cpuc->assign, assign, n*sizeof(int)); 1672 1673 done_collect: 1674 /* 1675 * Commit the collect_events() state. See x86_pmu_del() and 1676 * x86_pmu_*_txn(). 1677 */ 1678 cpuc->n_events = n; 1679 cpuc->n_added += n - n0; 1680 cpuc->n_txn += n - n0; 1681 1682 /* 1683 * This is before x86_pmu_enable() will call x86_pmu_start(), 1684 * so we enable LBRs before an event needs them etc.. 1685 */ 1686 static_call_cond(x86_pmu_add)(event); 1687 1688 ret = 0; 1689 out: 1690 return ret; 1691 } 1692 1693 static void x86_pmu_start(struct perf_event *event, int flags) 1694 { 1695 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1696 int idx = event->hw.idx; 1697 1698 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1699 return; 1700 1701 if (WARN_ON_ONCE(idx == -1)) 1702 return; 1703 1704 if (flags & PERF_EF_RELOAD) { 1705 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1706 static_call(x86_pmu_set_period)(event); 1707 } 1708 1709 event->hw.state = 0; 1710 1711 __set_bit(idx, cpuc->active_mask); 1712 static_call(x86_pmu_enable)(event); 1713 perf_event_update_userpage(event); 1714 } 1715 1716 void perf_event_print_debug(void) 1717 { 1718 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1719 unsigned long *cntr_mask, *fixed_cntr_mask; 1720 struct event_constraint *pebs_constraints; 1721 struct cpu_hw_events *cpuc; 1722 u64 pebs, debugctl; 1723 int cpu, idx; 1724 1725 guard(irqsave)(); 1726 1727 cpu = smp_processor_id(); 1728 cpuc = &per_cpu(cpu_hw_events, cpu); 1729 cntr_mask = hybrid(cpuc->pmu, cntr_mask); 1730 fixed_cntr_mask = hybrid(cpuc->pmu, fixed_cntr_mask); 1731 pebs_constraints = hybrid(cpuc->pmu, pebs_constraints); 1732 1733 if (!*(u64 *)cntr_mask) 1734 return; 1735 1736 if (x86_pmu.version >= 2) { 1737 rdmsrq(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1738 rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, status); 1739 rdmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1740 rdmsrq(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1741 1742 pr_info("\n"); 1743 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1744 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1745 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1746 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1747 if (pebs_constraints) { 1748 rdmsrq(MSR_IA32_PEBS_ENABLE, pebs); 1749 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1750 } 1751 if (x86_pmu.lbr_nr) { 1752 rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); 1753 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1754 } 1755 } 1756 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1757 1758 for_each_set_bit(idx, cntr_mask, X86_PMC_IDX_MAX) { 1759 rdmsrq(x86_pmu_config_addr(idx), pmc_ctrl); 1760 rdmsrq(x86_pmu_event_addr(idx), pmc_count); 1761 1762 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1763 1764 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1765 cpu, idx, pmc_ctrl); 1766 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1767 cpu, idx, pmc_count); 1768 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1769 cpu, idx, prev_left); 1770 } 1771 for_each_set_bit(idx, fixed_cntr_mask, X86_PMC_IDX_MAX) { 1772 rdmsrq(x86_pmu_fixed_ctr_addr(idx), pmc_count); 1773 1774 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1775 cpu, idx, pmc_count); 1776 } 1777 } 1778 1779 void x86_pmu_stop(struct perf_event *event, int flags) 1780 { 1781 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1782 struct hw_perf_event *hwc = &event->hw; 1783 1784 if (test_bit(hwc->idx, cpuc->active_mask)) { 1785 static_call(x86_pmu_disable)(event); 1786 __clear_bit(hwc->idx, cpuc->active_mask); 1787 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1788 hwc->state |= PERF_HES_STOPPED; 1789 } 1790 1791 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1792 /* 1793 * Drain the remaining delta count out of a event 1794 * that we are disabling: 1795 */ 1796 static_call(x86_pmu_update)(event); 1797 hwc->state |= PERF_HES_UPTODATE; 1798 } 1799 } 1800 1801 static void x86_pmu_del(struct perf_event *event, int flags) 1802 { 1803 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1804 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap); 1805 int i; 1806 1807 /* 1808 * If we're called during a txn, we only need to undo x86_pmu.add. 1809 * The events never got scheduled and ->cancel_txn will truncate 1810 * the event_list. 1811 * 1812 * XXX assumes any ->del() called during a TXN will only be on 1813 * an event added during that same TXN. 1814 */ 1815 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1816 goto do_del; 1817 1818 __set_bit(event->hw.idx, cpuc->dirty); 1819 1820 /* 1821 * Not a TXN, therefore cleanup properly. 1822 */ 1823 x86_pmu_stop(event, PERF_EF_UPDATE); 1824 cpuc->events[event->hw.idx] = NULL; 1825 1826 for (i = 0; i < cpuc->n_events; i++) { 1827 if (event == cpuc->event_list[i]) 1828 break; 1829 } 1830 1831 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1832 return; 1833 1834 /* If we have a newly added event; make sure to decrease n_added. */ 1835 if (i >= cpuc->n_events - cpuc->n_added) 1836 --cpuc->n_added; 1837 1838 static_call_cond(x86_pmu_put_event_constraints)(cpuc, event); 1839 1840 /* Delete the array entry. */ 1841 while (++i < cpuc->n_events) { 1842 cpuc->event_list[i-1] = cpuc->event_list[i]; 1843 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1844 cpuc->assign[i-1] = cpuc->assign[i]; 1845 } 1846 cpuc->event_constraint[i-1] = NULL; 1847 --cpuc->n_events; 1848 if (intel_cap.perf_metrics) 1849 del_nr_metric_event(cpuc, event); 1850 1851 perf_event_update_userpage(event); 1852 1853 do_del: 1854 1855 /* 1856 * This is after x86_pmu_stop(); so we disable LBRs after any 1857 * event can need them etc.. 1858 */ 1859 static_call_cond(x86_pmu_del)(event); 1860 } 1861 1862 void x86_pmu_clear_perf_regs(struct pt_regs *regs) 1863 { 1864 struct x86_perf_regs *perf_regs = container_of(regs, struct x86_perf_regs, regs); 1865 1866 perf_regs->abi = PERF_SAMPLE_REGS_ABI_NONE; 1867 perf_regs->xmm_regs = NULL; 1868 perf_regs->ymmh_regs = NULL; 1869 perf_regs->zmmh_regs = NULL; 1870 perf_regs->h16zmm_regs = NULL; 1871 perf_regs->opmask_regs = NULL; 1872 perf_regs->egpr_regs = NULL; 1873 perf_regs->ssp = NULL; 1874 } 1875 1876 static void update_perf_regs(struct x86_perf_regs *perf_regs, 1877 struct xregs_state *xsave, u64 bitmap) 1878 { 1879 struct cet_user_state *cet; 1880 u64 mask; 1881 1882 if (!xsave) 1883 return; 1884 1885 /* Restrict to features actually saved by XSAVES */ 1886 mask = bitmap & xsave->header.xfeatures; 1887 1888 if (mask & XFEATURE_MASK_SSE) 1889 perf_regs->xmm_space = xsave->i387.xmm_space; 1890 if (mask & XFEATURE_MASK_YMM) 1891 perf_regs->ymmh = get_xsave_addr(xsave, XFEATURE_YMM); 1892 if (mask & XFEATURE_MASK_ZMM_Hi256) 1893 perf_regs->zmmh = get_xsave_addr(xsave, XFEATURE_ZMM_Hi256); 1894 if (mask & XFEATURE_MASK_Hi16_ZMM) 1895 perf_regs->h16zmm = get_xsave_addr(xsave, XFEATURE_Hi16_ZMM); 1896 if (mask & XFEATURE_MASK_OPMASK) 1897 perf_regs->opmask = get_xsave_addr(xsave, XFEATURE_OPMASK); 1898 if (mask & XFEATURE_MASK_APX) 1899 perf_regs->egpr = get_xsave_addr(xsave, XFEATURE_APX); 1900 if (mask & XFEATURE_MASK_CET_USER) { 1901 cet = get_xsave_addr(xsave, XFEATURE_CET_USER); 1902 perf_regs->ssp = cet ? &cet->user_ssp : NULL; 1903 } 1904 } 1905 1906 /* 1907 * The x86 specific variant of perf_sample_regs_intr(). 1908 * Update data->regs_intr fields for extended registers (e.g., SIMD). 1909 */ 1910 static void x86_pmu_update_regs_intr(struct perf_event *event, 1911 struct perf_sample_data *data, 1912 struct pt_regs *regs, 1913 bool exclude_kernel) 1914 { 1915 struct x86_perf_regs *perf_regs; 1916 1917 if (exclude_kernel && !user_mode(regs)) { 1918 data->regs_intr.regs = NULL; 1919 data->regs_intr.abi = PERF_SAMPLE_REGS_ABI_NONE; 1920 } else { 1921 data->regs_intr.regs = regs; 1922 data->regs_intr.abi = perf_reg_abi(current); 1923 } 1924 1925 data->dyn_size += sizeof(u64); 1926 if (data->regs_intr.regs) { 1927 data->dyn_size += hweight64(event->attr.sample_regs_intr) * 1928 sizeof(u64); 1929 if (event_has_simd_regs(event)) { 1930 data->dyn_size += perf_update_xregs_size(event, true); 1931 data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD; 1932 } 1933 1934 perf_regs = container_of(data->regs_intr.regs, 1935 struct x86_perf_regs, regs); 1936 perf_regs->abi = data->regs_intr.abi; 1937 } 1938 1939 /* 1940 * Set PERF_SAMPLE_REGS_INTR to bypass perf_sample_regs_intr() call 1941 * in perf_prepare_sample() function. 1942 */ 1943 data->sample_flags |= PERF_SAMPLE_REGS_INTR; 1944 } 1945 1946 static DEFINE_PER_CPU(struct x86_perf_regs, x86_user_regs); 1947 1948 static void x86_pmu_get_regs_user(struct perf_sample_data *data, 1949 struct pt_regs *regs) 1950 { 1951 struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs); 1952 struct perf_regs regs_user; 1953 1954 x86_pmu_clear_perf_regs(&x86_regs_user->regs); 1955 1956 perf_get_regs_user(®s_user, regs); 1957 data->regs_user.abi = regs_user.abi; 1958 if (regs_user.regs) { 1959 x86_regs_user->regs = *regs_user.regs; 1960 data->regs_user.regs = &x86_regs_user->regs; 1961 } else { 1962 data->regs_user.regs = NULL; 1963 } 1964 } 1965 1966 /* 1967 * The x86 specific variant of perf_sample_regs_user(). 1968 * Update data->regs_user fields for extended registers (e.g., SIMD). 1969 */ 1970 static void x86_pmu_update_regs_user(struct perf_event *event, 1971 struct perf_sample_data *data, 1972 struct pt_regs *regs) 1973 { 1974 struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs); 1975 struct perf_event_attr *attr = &event->attr; 1976 struct x86_perf_regs *perf_regs; 1977 1978 /* 1979 * PERF_SAMPLE_REGS_INTR and PERF_SAMPLE_REGS_USER can both be 1980 * requested for one event. Keep user regs in a separate x86_perf_regs 1981 * instance, so intr-reg collection does not overwrite user-reg data. 1982 */ 1983 if (user_mode(regs)) { 1984 x86_pmu_clear_perf_regs(&x86_regs_user->regs); 1985 perf_regs = container_of(regs, struct x86_perf_regs, regs); 1986 /* Copy all sampled regs data to x86_regs_user. */ 1987 *x86_regs_user = *perf_regs; 1988 data->regs_user.regs = &x86_regs_user->regs; 1989 data->regs_user.abi = perf_reg_abi(current); 1990 } else if (is_user_task(current)) { 1991 x86_pmu_get_regs_user(data, regs); 1992 } else { 1993 data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE; 1994 data->regs_user.regs = NULL; 1995 } 1996 1997 data->dyn_size += sizeof(u64); 1998 if (data->regs_user.regs) { 1999 data->dyn_size += hweight64(attr->sample_regs_user) * sizeof(u64); 2000 if (event_has_simd_regs(event)) { 2001 data->dyn_size += perf_update_xregs_size(event, false); 2002 data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD; 2003 } 2004 2005 x86_regs_user->abi = data->regs_user.abi; 2006 } 2007 2008 /* 2009 * Set PERF_SAMPLE_REGS_USER to bypass perf_sample_regs_user() call 2010 * in perf_prepare_sample() function. 2011 */ 2012 data->sample_flags |= PERF_SAMPLE_REGS_USER; 2013 } 2014 2015 /* 2016 * This function retrieves cached user-space fpu registers (XMM/YMM/ZMM). 2017 * If TIF_NEED_FPU_LOAD is set or PMI hits into guest, it indicates that 2018 * the user-space FPU state is cached. Otherwise, the data should be read 2019 * directly from the hardware registers. 2020 */ 2021 static inline u64 x86_pmu_update_user_xregs(struct perf_sample_data *data, 2022 struct pt_regs *regs, 2023 u64 mask, bool from_pebs) 2024 { 2025 struct x86_perf_regs *perf_regs; 2026 struct xregs_state *xsave; 2027 struct fpu *fpu; 2028 struct fpstate *fps; 2029 u64 user_mask = mask; 2030 2031 if (!is_user_task(current)) 2032 return 0; 2033 2034 if (data->regs_user.abi == PERF_SAMPLE_REGS_ABI_NONE) 2035 return 0; 2036 2037 /* 2038 * If PEBS hits kernel space, need to re-sample extended 2039 * registers for user space. 2040 */ 2041 if (user_mode(regs)) 2042 user_mask = from_pebs ? 0 : mask; 2043 2044 fpu = x86_task_fpu(current); 2045 fps = READ_ONCE(fpu->__task_fpstate); 2046 /* 2047 * If fpu->__task_fpstate is set, it points to the cached user 2048 * FPU state (e.g. with KVM guest-state swapping). Otherwise, 2049 * when TIF_NEED_FPU_LOAD is set, fpu->fpstate holds the cached 2050 * user state. If neither is true, the user state is live in hardware. 2051 */ 2052 if (user_mask && (test_thread_flag(TIF_NEED_FPU_LOAD) || fps)) { 2053 perf_regs = container_of(data->regs_user.regs, 2054 struct x86_perf_regs, regs); 2055 if (!fps) 2056 fps = fpu->fpstate; 2057 xsave = &fps->regs.xsave; 2058 2059 update_perf_regs(perf_regs, xsave, user_mask); 2060 return 0; 2061 } 2062 2063 return user_mask; 2064 } 2065 2066 static u64 get_simd_sample_mask(struct perf_event *event, u64 sample_type) 2067 { 2068 u64 mask = 0; 2069 2070 if (__event_needs_xmm(event, sample_type)) 2071 mask |= XFEATURE_MASK_SSE; 2072 if (__event_needs_ymm(event, sample_type)) 2073 mask |= XFEATURE_MASK_YMM; 2074 if (__event_needs_low16_zmm(event, sample_type)) 2075 mask |= XFEATURE_MASK_ZMM_Hi256; 2076 if (__event_needs_high16_zmm(event, sample_type)) 2077 mask |= XFEATURE_MASK_Hi16_ZMM; 2078 if (__event_needs_opmask(event, sample_type)) 2079 mask |= XFEATURE_MASK_OPMASK; 2080 if (__event_needs_egprs(event, sample_type)) 2081 mask |= XFEATURE_MASK_APX; 2082 if (__event_needs_ssp(event, sample_type)) 2083 mask |= XFEATURE_MASK_CET_USER; 2084 2085 return mask; 2086 } 2087 2088 static void x86_pmu_sample_xregs(struct perf_event *event, 2089 struct perf_sample_data *data, 2090 struct pt_regs *regs, 2091 bool from_pebs) 2092 { 2093 struct xregs_state *xsave = get_ext_regs_buf(smp_processor_id()); 2094 u64 sample_type = event->attr.sample_type; 2095 struct x86_perf_regs *perf_regs; 2096 u64 intr_mask = 0; 2097 u64 user_mask = 0; 2098 2099 if (WARN_ON_ONCE(!xsave) || !in_nmi()) 2100 return; 2101 2102 if ((sample_type & PERF_SAMPLE_REGS_INTR) && data->regs_intr.regs) { 2103 intr_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_INTR); 2104 intr_mask &= x86_pmu.ext_regs_mask; 2105 intr_mask = from_pebs ? 0 : intr_mask; 2106 } 2107 2108 if ((sample_type & PERF_SAMPLE_REGS_USER) && data->regs_user.regs) { 2109 user_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_USER); 2110 user_mask &= x86_pmu.ext_regs_mask; 2111 user_mask = x86_pmu_update_user_xregs(data, regs, 2112 user_mask, from_pebs); 2113 } 2114 2115 if (user_mask | intr_mask) { 2116 xsave->header.xfeatures = 0; 2117 xsaves_nmi(xsave, user_mask | intr_mask); 2118 } 2119 2120 if (intr_mask) { 2121 perf_regs = container_of(data->regs_intr.regs, 2122 struct x86_perf_regs, regs); 2123 update_perf_regs(perf_regs, xsave, intr_mask); 2124 } 2125 2126 if (user_mask) { 2127 perf_regs = container_of(data->regs_user.regs, 2128 struct x86_perf_regs, regs); 2129 update_perf_regs(perf_regs, xsave, user_mask); 2130 } 2131 } 2132 2133 void x86_pmu_update_perf_regs(struct perf_event *event, 2134 struct perf_sample_data *data, 2135 struct pt_regs *regs, 2136 bool from_pebs) 2137 { 2138 u64 sample_type = event->attr.sample_type; 2139 2140 if (!(sample_type & 2141 (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))) 2142 return; 2143 2144 if (!event_needs_xmm(event) && 2145 !event_has_simd_regs(event)) 2146 return; 2147 2148 if (sample_type & PERF_SAMPLE_REGS_INTR) { 2149 x86_pmu_update_regs_intr(event, data, regs, 2150 event->attr.exclude_kernel); 2151 } 2152 if (sample_type & PERF_SAMPLE_REGS_USER) 2153 x86_pmu_update_regs_user(event, data, regs); 2154 2155 x86_pmu_sample_xregs(event, data, regs, from_pebs); 2156 } 2157 2158 int x86_pmu_handle_irq(struct pt_regs *regs) 2159 { 2160 struct perf_sample_data data; 2161 struct cpu_hw_events *cpuc; 2162 struct perf_event *event; 2163 int idx, handled = 0; 2164 u64 last_period; 2165 u64 val; 2166 2167 cpuc = this_cpu_ptr(&cpu_hw_events); 2168 2169 /* 2170 * Some chipsets need to unmask the LVTPC in a particular spot 2171 * inside the nmi handler. As a result, the unmasking was pushed 2172 * into all the nmi handlers. 2173 * 2174 * This generic handler doesn't seem to have any issues where the 2175 * unmasking occurs so it was left at the top. 2176 */ 2177 apic_write(APIC_LVTPC, APIC_DM_NMI); 2178 2179 for_each_set_bit(idx, x86_pmu.cntr_mask, X86_PMC_IDX_MAX) { 2180 if (!test_bit(idx, cpuc->active_mask)) 2181 continue; 2182 2183 event = cpuc->events[idx]; 2184 last_period = event->hw.last_period; 2185 2186 val = static_call(x86_pmu_update)(event); 2187 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 2188 continue; 2189 2190 /* 2191 * event overflow 2192 */ 2193 handled++; 2194 2195 if (!static_call(x86_pmu_set_period)(event)) 2196 continue; 2197 2198 perf_sample_data_init(&data, 0, last_period); 2199 2200 perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL); 2201 2202 perf_event_overflow(event, &data, regs); 2203 } 2204 2205 if (handled) 2206 inc_perf_irq_stat(); 2207 2208 return handled; 2209 } 2210 2211 void perf_events_lapic_init(void) 2212 { 2213 if (!x86_pmu.apic || !x86_pmu_initialized()) 2214 return; 2215 2216 /* 2217 * Always use NMI for PMU 2218 */ 2219 apic_write(APIC_LVTPC, APIC_DM_NMI); 2220 } 2221 2222 #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU 2223 void perf_load_guest_lvtpc(u32 guest_lvtpc) 2224 { 2225 u32 masked = guest_lvtpc & APIC_LVT_MASKED; 2226 2227 apic_write(APIC_LVTPC, 2228 APIC_DM_FIXED | PERF_GUEST_MEDIATED_PMI_VECTOR | masked); 2229 this_cpu_write(guest_lvtpc_loaded, true); 2230 } 2231 EXPORT_SYMBOL_FOR_KVM(perf_load_guest_lvtpc); 2232 2233 void perf_put_guest_lvtpc(void) 2234 { 2235 this_cpu_write(guest_lvtpc_loaded, false); 2236 apic_write(APIC_LVTPC, APIC_DM_NMI); 2237 } 2238 EXPORT_SYMBOL_FOR_KVM(perf_put_guest_lvtpc); 2239 #endif /* CONFIG_PERF_GUEST_MEDIATED_PMU */ 2240 2241 static DEFINE_PER_CPU(struct x86_perf_regs, x86_intr_regs); 2242 static int 2243 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 2244 { 2245 struct x86_perf_regs *x86_regs = this_cpu_ptr(&x86_intr_regs); 2246 u64 start_clock; 2247 u64 finish_clock; 2248 int ret; 2249 2250 /* 2251 * Ignore all NMIs when the CPU's LVTPC is configured to route PMIs to 2252 * PERF_GUEST_MEDIATED_PMI_VECTOR, i.e. when an NMI time can't be due 2253 * to a PMI. Attempting to handle a PMI while the guest's context is 2254 * loaded will generate false positives and clobber guest state. Note, 2255 * the LVTPC is switched to/from the dedicated mediated PMI IRQ vector 2256 * while host events are quiesced. 2257 */ 2258 if (this_cpu_read(guest_lvtpc_loaded)) 2259 return NMI_DONE; 2260 2261 /* 2262 * All PMUs/events that share this PMI handler should make sure to 2263 * increment active_events for their events. 2264 */ 2265 if (!atomic_read(&active_events)) 2266 return NMI_DONE; 2267 2268 start_clock = sched_clock(); 2269 x86_regs->regs = *regs; 2270 ret = static_call(x86_pmu_handle_irq)(&x86_regs->regs); 2271 finish_clock = sched_clock(); 2272 2273 perf_sample_event_took(finish_clock - start_clock); 2274 2275 return ret; 2276 } 2277 NOKPROBE_SYMBOL(perf_event_nmi_handler); 2278 2279 struct event_constraint emptyconstraint; 2280 struct event_constraint unconstrained; 2281 2282 static int x86_pmu_prepare_cpu(unsigned int cpu) 2283 { 2284 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 2285 int i; 2286 2287 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 2288 cpuc->kfree_on_online[i] = NULL; 2289 if (x86_pmu.cpu_prepare) 2290 return x86_pmu.cpu_prepare(cpu); 2291 return 0; 2292 } 2293 2294 static int x86_pmu_dead_cpu(unsigned int cpu) 2295 { 2296 if (x86_pmu.cpu_dead) 2297 x86_pmu.cpu_dead(cpu); 2298 return 0; 2299 } 2300 2301 static int x86_pmu_online_cpu(unsigned int cpu) 2302 { 2303 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 2304 int i; 2305 2306 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 2307 kfree(cpuc->kfree_on_online[i]); 2308 cpuc->kfree_on_online[i] = NULL; 2309 } 2310 return 0; 2311 } 2312 2313 static int x86_pmu_starting_cpu(unsigned int cpu) 2314 { 2315 if (x86_pmu.cpu_starting) 2316 x86_pmu.cpu_starting(cpu); 2317 return 0; 2318 } 2319 2320 static int x86_pmu_dying_cpu(unsigned int cpu) 2321 { 2322 if (x86_pmu.cpu_dying) 2323 x86_pmu.cpu_dying(cpu); 2324 return 0; 2325 } 2326 2327 static void __init pmu_check_apic(void) 2328 { 2329 if (boot_cpu_has(X86_FEATURE_APIC)) 2330 return; 2331 2332 x86_pmu.apic = 0; 2333 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 2334 pr_info("no hardware sampling interrupt available.\n"); 2335 2336 /* 2337 * If we have a PMU initialized but no APIC 2338 * interrupts, we cannot sample hardware 2339 * events (user-space has to fall back and 2340 * sample via a hrtimer based software event): 2341 */ 2342 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 2343 2344 } 2345 2346 static struct attribute_group x86_pmu_format_group __ro_after_init = { 2347 .name = "format", 2348 .attrs = NULL, 2349 }; 2350 2351 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 2352 { 2353 struct perf_pmu_events_attr *pmu_attr = 2354 container_of(attr, struct perf_pmu_events_attr, attr); 2355 u64 config = 0; 2356 2357 if (pmu_attr->id < x86_pmu.max_events) 2358 config = x86_pmu.event_map(pmu_attr->id); 2359 2360 /* string trumps id */ 2361 if (pmu_attr->event_str) 2362 return sprintf(page, "%s\n", pmu_attr->event_str); 2363 2364 return x86_pmu.events_sysfs_show(page, config); 2365 } 2366 EXPORT_SYMBOL_GPL(events_sysfs_show); 2367 2368 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 2369 char *page) 2370 { 2371 struct perf_pmu_events_ht_attr *pmu_attr = 2372 container_of(attr, struct perf_pmu_events_ht_attr, attr); 2373 2374 /* 2375 * Report conditional events depending on Hyper-Threading. 2376 * 2377 * This is overly conservative as usually the HT special 2378 * handling is not needed if the other CPU thread is idle. 2379 * 2380 * Note this does not (and cannot) handle the case when thread 2381 * siblings are invisible, for example with virtualization 2382 * if they are owned by some other guest. The user tool 2383 * has to re-read when a thread sibling gets onlined later. 2384 */ 2385 return sprintf(page, "%s", 2386 topology_max_smt_threads() > 1 ? 2387 pmu_attr->event_str_ht : 2388 pmu_attr->event_str_noht); 2389 } 2390 2391 ssize_t events_hybrid_sysfs_show(struct device *dev, 2392 struct device_attribute *attr, 2393 char *page) 2394 { 2395 struct perf_pmu_events_hybrid_attr *pmu_attr = 2396 container_of(attr, struct perf_pmu_events_hybrid_attr, attr); 2397 struct x86_hybrid_pmu *pmu; 2398 const char *str, *next_str; 2399 int i; 2400 2401 if (hweight64(pmu_attr->pmu_type) == 1) 2402 return sprintf(page, "%s", pmu_attr->event_str); 2403 2404 /* 2405 * Hybrid PMUs may support the same event name, but with different 2406 * event encoding, e.g., the mem-loads event on an Atom PMU has 2407 * different event encoding from a Core PMU. 2408 * 2409 * The event_str includes all event encodings. Each event encoding 2410 * is divided by ";". The order of the event encodings must follow 2411 * the order of the hybrid PMU index. 2412 */ 2413 pmu = container_of(dev_get_drvdata(dev), struct x86_hybrid_pmu, pmu); 2414 2415 str = pmu_attr->event_str; 2416 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { 2417 if (!(x86_pmu.hybrid_pmu[i].pmu_type & pmu_attr->pmu_type)) 2418 continue; 2419 if (x86_pmu.hybrid_pmu[i].pmu_type & pmu->pmu_type) { 2420 next_str = strchr(str, ';'); 2421 if (next_str) 2422 return snprintf(page, next_str - str + 1, "%s", str); 2423 else 2424 return sprintf(page, "%s", str); 2425 } 2426 str = strchr(str, ';'); 2427 str++; 2428 } 2429 2430 return 0; 2431 } 2432 EXPORT_SYMBOL_GPL(events_hybrid_sysfs_show); 2433 2434 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 2435 EVENT_ATTR(instructions, INSTRUCTIONS ); 2436 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 2437 EVENT_ATTR(cache-misses, CACHE_MISSES ); 2438 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 2439 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 2440 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 2441 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 2442 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 2443 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 2444 2445 static struct attribute *empty_attrs; 2446 2447 static struct attribute *events_attr[] = { 2448 EVENT_PTR(CPU_CYCLES), 2449 EVENT_PTR(INSTRUCTIONS), 2450 EVENT_PTR(CACHE_REFERENCES), 2451 EVENT_PTR(CACHE_MISSES), 2452 EVENT_PTR(BRANCH_INSTRUCTIONS), 2453 EVENT_PTR(BRANCH_MISSES), 2454 EVENT_PTR(BUS_CYCLES), 2455 EVENT_PTR(STALLED_CYCLES_FRONTEND), 2456 EVENT_PTR(STALLED_CYCLES_BACKEND), 2457 EVENT_PTR(REF_CPU_CYCLES), 2458 NULL, 2459 }; 2460 2461 /* 2462 * Remove all undefined events (x86_pmu.event_map(id) == 0) 2463 * out of events_attr attributes. 2464 */ 2465 static umode_t 2466 is_visible(struct kobject *kobj, struct attribute *attr, int idx) 2467 { 2468 struct perf_pmu_events_attr *pmu_attr; 2469 2470 if (idx >= x86_pmu.max_events) 2471 return 0; 2472 2473 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr); 2474 /* str trumps id */ 2475 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0; 2476 } 2477 2478 static struct attribute_group x86_pmu_events_group __ro_after_init = { 2479 .name = "events", 2480 .attrs = events_attr, 2481 .is_visible = is_visible, 2482 }; 2483 2484 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 2485 { 2486 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 2487 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 2488 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 2489 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 2490 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 2491 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 2492 ssize_t ret; 2493 2494 /* 2495 * We have whole page size to spend and just little data 2496 * to write, so we can safely use sprintf. 2497 */ 2498 ret = sprintf(page, "event=0x%02llx", event); 2499 2500 if (umask) 2501 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 2502 2503 if (edge) 2504 ret += sprintf(page + ret, ",edge"); 2505 2506 if (pc) 2507 ret += sprintf(page + ret, ",pc"); 2508 2509 if (any) 2510 ret += sprintf(page + ret, ",any"); 2511 2512 if (inv) 2513 ret += sprintf(page + ret, ",inv"); 2514 2515 if (cmask) 2516 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 2517 2518 ret += sprintf(page + ret, "\n"); 2519 2520 return ret; 2521 } 2522 2523 static struct attribute_group x86_pmu_attr_group; 2524 static struct attribute_group x86_pmu_caps_group; 2525 2526 static void x86_pmu_static_call_update(void) 2527 { 2528 static_call_update(x86_pmu_handle_irq, x86_pmu.handle_irq); 2529 static_call_update(x86_pmu_disable_all, x86_pmu.disable_all); 2530 static_call_update(x86_pmu_enable_all, x86_pmu.enable_all); 2531 static_call_update(x86_pmu_enable, x86_pmu.enable); 2532 static_call_update(x86_pmu_disable, x86_pmu.disable); 2533 2534 static_call_update(x86_pmu_assign, x86_pmu.assign); 2535 2536 static_call_update(x86_pmu_add, x86_pmu.add); 2537 static_call_update(x86_pmu_del, x86_pmu.del); 2538 static_call_update(x86_pmu_read, x86_pmu.read); 2539 2540 static_call_update(x86_pmu_set_period, x86_pmu.set_period); 2541 static_call_update(x86_pmu_update, x86_pmu.update); 2542 static_call_update(x86_pmu_limit_period, x86_pmu.limit_period); 2543 2544 static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events); 2545 static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints); 2546 static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints); 2547 2548 static_call_update(x86_pmu_start_scheduling, x86_pmu.start_scheduling); 2549 static_call_update(x86_pmu_commit_scheduling, x86_pmu.commit_scheduling); 2550 static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling); 2551 2552 static_call_update(x86_pmu_sched_task, x86_pmu.sched_task); 2553 2554 static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs); 2555 static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases); 2556 2557 static_call_update(x86_pmu_guest_get_msrs, x86_pmu.guest_get_msrs); 2558 static_call_update(x86_pmu_filter, x86_pmu.filter); 2559 2560 static_call_update(x86_pmu_late_setup, x86_pmu.late_setup); 2561 2562 static_call_update(x86_pmu_pebs_enable, x86_pmu.pebs_enable); 2563 static_call_update(x86_pmu_pebs_disable, x86_pmu.pebs_disable); 2564 static_call_update(x86_pmu_pebs_enable_all, x86_pmu.pebs_enable_all); 2565 static_call_update(x86_pmu_pebs_disable_all, x86_pmu.pebs_disable_all); 2566 } 2567 2568 static void _x86_pmu_read(struct perf_event *event) 2569 { 2570 static_call(x86_pmu_update)(event); 2571 } 2572 2573 void x86_pmu_show_pmu_cap(struct pmu *pmu) 2574 { 2575 pr_info("... version: %d\n", x86_pmu.version); 2576 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 2577 pr_info("... generic counters: %d\n", x86_pmu_num_counters(pmu)); 2578 pr_info("... generic bitmap: %016llx\n", hybrid(pmu, cntr_mask64)); 2579 pr_info("... fixed-purpose counters: %d\n", x86_pmu_num_counters_fixed(pmu)); 2580 pr_info("... fixed-purpose bitmap: %016llx\n", hybrid(pmu, fixed_cntr_mask64)); 2581 pr_info("... value mask: %016llx\n", x86_pmu.cntval_mask); 2582 pr_info("... max period: %016llx\n", x86_pmu.max_period); 2583 pr_info("... global_ctrl mask: %016llx\n", hybrid(pmu, intel_ctrl)); 2584 } 2585 2586 static void x86_pmu_free_hybrid(void) 2587 { 2588 if (!x86_pmu.hybrid_pmu) 2589 return; 2590 2591 static_branch_disable(&perf_is_hybrid); 2592 kfree(x86_pmu.hybrid_pmu); 2593 x86_pmu.hybrid_pmu = NULL; 2594 x86_pmu.num_hybrid_pmus = 0; 2595 } 2596 2597 static int __init init_hw_perf_events(void) 2598 { 2599 struct x86_pmu_quirk *quirk; 2600 int err; 2601 2602 pr_info("Performance Events: "); 2603 2604 switch (boot_cpu_data.x86_vendor) { 2605 case X86_VENDOR_INTEL: 2606 err = intel_pmu_init(); 2607 break; 2608 case X86_VENDOR_AMD: 2609 err = amd_pmu_init(); 2610 break; 2611 case X86_VENDOR_HYGON: 2612 err = amd_pmu_init(); 2613 x86_pmu.name = "HYGON"; 2614 break; 2615 case X86_VENDOR_ZHAOXIN: 2616 case X86_VENDOR_CENTAUR: 2617 err = zhaoxin_pmu_init(); 2618 break; 2619 default: 2620 err = -ENOTSUPP; 2621 } 2622 if (err != 0) { 2623 pr_cont("no PMU driver, software events only.\n"); 2624 err = 0; 2625 goto out_bad_pmu; 2626 } 2627 2628 pmu_check_apic(); 2629 2630 /* sanity check that the hardware exists or is emulated */ 2631 if (!check_hw_exists(x86_pmu.cntr_mask, x86_pmu.fixed_cntr_mask)) 2632 goto out_bad_pmu; 2633 2634 pr_cont("%s PMU driver.\n", x86_pmu.name); 2635 2636 /* enable userspace RDPMC usage by default */ 2637 x86_pmu.attr_rdpmc = X86_USER_RDPMC_CONDITIONAL_ENABLE; 2638 2639 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 2640 quirk->func(); 2641 2642 if (!x86_pmu.intel_ctrl) 2643 x86_pmu.intel_ctrl = x86_pmu.cntr_mask64; 2644 2645 if (!x86_pmu.config_mask) 2646 x86_pmu.config_mask = X86_RAW_EVENT_MASK; 2647 2648 perf_events_lapic_init(); 2649 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 2650 2651 unconstrained = (struct event_constraint) 2652 __EVENT_CONSTRAINT(0, x86_pmu.cntr_mask64, 2653 0, x86_pmu_num_counters(NULL), 0, 0); 2654 2655 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 2656 2657 if (!x86_pmu.events_sysfs_show) 2658 x86_pmu_events_group.attrs = &empty_attrs; 2659 2660 pmu.attr_update = x86_pmu.attr_update; 2661 2662 if (!is_hybrid()) { 2663 x86_pmu_show_pmu_cap(NULL); 2664 } else { 2665 int i; 2666 2667 /* 2668 * Init default ops. 2669 * Must be called before registering x86_pmu_starting_cpu(), 2670 * otherwise some key PMU fields, e.g., capabilities 2671 * initialized in x86_pmu_starting_cpu(), would be overwritten. 2672 */ 2673 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) 2674 x86_pmu.hybrid_pmu[i].pmu = pmu; 2675 } 2676 2677 if (!x86_pmu.read) 2678 x86_pmu.read = _x86_pmu_read; 2679 2680 if (!x86_pmu.guest_get_msrs) 2681 x86_pmu.guest_get_msrs = (void *)&__static_call_return0; 2682 2683 if (!x86_pmu.set_period) 2684 x86_pmu.set_period = x86_perf_event_set_period; 2685 2686 if (!x86_pmu.update) 2687 x86_pmu.update = x86_perf_event_update; 2688 2689 x86_pmu_static_call_update(); 2690 2691 /* 2692 * Install callbacks. Core will call them for each online 2693 * cpu. 2694 */ 2695 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 2696 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 2697 if (err) 2698 goto pmi_unregister; 2699 2700 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 2701 "perf/x86:starting", x86_pmu_starting_cpu, 2702 x86_pmu_dying_cpu); 2703 if (err) 2704 goto out; 2705 2706 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 2707 x86_pmu_online_cpu, NULL); 2708 if (err) 2709 goto out1; 2710 2711 if (!is_hybrid()) { 2712 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 2713 if (err) 2714 goto out2; 2715 } else { 2716 struct x86_hybrid_pmu *hybrid_pmu; 2717 int i, j; 2718 2719 for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { 2720 hybrid_pmu = &x86_pmu.hybrid_pmu[i]; 2721 2722 hybrid_pmu->pmu.type = -1; 2723 hybrid_pmu->pmu.attr_update = x86_pmu.attr_update; 2724 hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE; 2725 2726 err = perf_pmu_register(&hybrid_pmu->pmu, hybrid_pmu->name, 2727 (hybrid_pmu->pmu_type == hybrid_big) ? PERF_TYPE_RAW : -1); 2728 if (err) 2729 break; 2730 } 2731 2732 if (i < x86_pmu.num_hybrid_pmus) { 2733 for (j = 0; j < i; j++) 2734 perf_pmu_unregister(&x86_pmu.hybrid_pmu[j].pmu); 2735 pr_warn("Failed to register hybrid PMUs\n"); 2736 goto out2; 2737 } 2738 } 2739 2740 return 0; 2741 2742 out2: 2743 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 2744 out1: 2745 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 2746 out: 2747 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 2748 pmi_unregister: 2749 unregister_nmi_handler(NMI_LOCAL, "PMI"); 2750 out_bad_pmu: 2751 x86_pmu_free_hybrid(); 2752 memset(&x86_pmu, 0, sizeof(x86_pmu)); 2753 return err; 2754 } 2755 early_initcall(init_hw_perf_events); 2756 2757 static void x86_pmu_read(struct perf_event *event) 2758 { 2759 static_call(x86_pmu_read)(event); 2760 } 2761 2762 /* 2763 * Start group events scheduling transaction 2764 * Set the flag to make pmu::enable() not perform the 2765 * schedulability test, it will be performed at commit time 2766 * 2767 * We only support PERF_PMU_TXN_ADD transactions. Save the 2768 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 2769 * transactions. 2770 */ 2771 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 2772 { 2773 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2774 2775 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 2776 2777 cpuc->txn_flags = txn_flags; 2778 if (txn_flags & ~PERF_PMU_TXN_ADD) 2779 return; 2780 2781 perf_pmu_disable(pmu); 2782 __this_cpu_write(cpu_hw_events.n_txn, 0); 2783 __this_cpu_write(cpu_hw_events.n_txn_pair, 0); 2784 __this_cpu_write(cpu_hw_events.n_txn_metric, 0); 2785 } 2786 2787 /* 2788 * Stop group events scheduling transaction 2789 * Clear the flag and pmu::enable() will perform the 2790 * schedulability test. 2791 */ 2792 static void x86_pmu_cancel_txn(struct pmu *pmu) 2793 { 2794 unsigned int txn_flags; 2795 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2796 2797 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 2798 2799 txn_flags = cpuc->txn_flags; 2800 cpuc->txn_flags = 0; 2801 if (txn_flags & ~PERF_PMU_TXN_ADD) 2802 return; 2803 2804 /* 2805 * Truncate collected array by the number of events added in this 2806 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 2807 */ 2808 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 2809 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 2810 __this_cpu_sub(cpu_hw_events.n_pair, __this_cpu_read(cpu_hw_events.n_txn_pair)); 2811 __this_cpu_sub(cpu_hw_events.n_metric, __this_cpu_read(cpu_hw_events.n_txn_metric)); 2812 perf_pmu_enable(pmu); 2813 } 2814 2815 /* 2816 * Commit group events scheduling transaction 2817 * Perform the group schedulability test as a whole 2818 * Return 0 if success 2819 * 2820 * Does not cancel the transaction on failure; expects the caller to do this. 2821 */ 2822 static int x86_pmu_commit_txn(struct pmu *pmu) 2823 { 2824 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 2825 int assign[X86_PMC_IDX_MAX]; 2826 int n, ret; 2827 2828 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 2829 2830 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 2831 cpuc->txn_flags = 0; 2832 return 0; 2833 } 2834 2835 n = cpuc->n_events; 2836 2837 if (!x86_pmu_initialized()) 2838 return -EAGAIN; 2839 2840 ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign); 2841 if (ret) 2842 return ret; 2843 2844 /* 2845 * copy new assignment, now we know it is possible 2846 * will be used by hw_perf_enable() 2847 */ 2848 memcpy(cpuc->assign, assign, n*sizeof(int)); 2849 2850 cpuc->txn_flags = 0; 2851 perf_pmu_enable(pmu); 2852 return 0; 2853 } 2854 /* 2855 * a fake_cpuc is used to validate event groups. Due to 2856 * the extra reg logic, we need to also allocate a fake 2857 * per_core and per_cpu structure. Otherwise, group events 2858 * using extra reg may conflict without the kernel being 2859 * able to catch this when the last event gets added to 2860 * the group. 2861 */ 2862 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 2863 { 2864 intel_cpuc_finish(cpuc); 2865 kfree(cpuc); 2866 } 2867 2868 static struct cpu_hw_events *allocate_fake_cpuc(struct pmu *event_pmu) 2869 { 2870 struct cpu_hw_events *cpuc; 2871 int cpu; 2872 2873 cpuc = kzalloc_obj(*cpuc); 2874 if (!cpuc) 2875 return ERR_PTR(-ENOMEM); 2876 cpuc->is_fake = 1; 2877 2878 if (is_hybrid()) { 2879 struct x86_hybrid_pmu *h_pmu; 2880 2881 h_pmu = hybrid_pmu(event_pmu); 2882 if (cpumask_empty(&h_pmu->supported_cpus)) 2883 goto error; 2884 cpu = cpumask_first(&h_pmu->supported_cpus); 2885 } else 2886 cpu = raw_smp_processor_id(); 2887 cpuc->pmu = event_pmu; 2888 2889 if (intel_cpuc_prepare(cpuc, cpu)) 2890 goto error; 2891 2892 return cpuc; 2893 error: 2894 free_fake_cpuc(cpuc); 2895 return ERR_PTR(-ENOMEM); 2896 } 2897 2898 /* 2899 * validate that we can schedule this event 2900 */ 2901 static int validate_event(struct perf_event *event) 2902 { 2903 struct cpu_hw_events *fake_cpuc; 2904 struct event_constraint *c; 2905 int ret = 0; 2906 2907 fake_cpuc = allocate_fake_cpuc(event->pmu); 2908 if (IS_ERR(fake_cpuc)) 2909 return PTR_ERR(fake_cpuc); 2910 2911 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event); 2912 2913 if (!c || !c->weight) 2914 ret = -EINVAL; 2915 2916 if (x86_pmu.put_event_constraints) 2917 x86_pmu.put_event_constraints(fake_cpuc, event); 2918 2919 free_fake_cpuc(fake_cpuc); 2920 2921 return ret; 2922 } 2923 2924 /* 2925 * validate a single event group 2926 * 2927 * validation include: 2928 * - check events are compatible which each other 2929 * - events do not compete for the same counter 2930 * - number of events <= number of counters 2931 * 2932 * validation ensures the group can be loaded onto the 2933 * PMU if it was the only group available. 2934 */ 2935 static int validate_group(struct perf_event *event) 2936 { 2937 struct perf_event *leader = event->group_leader; 2938 struct cpu_hw_events *fake_cpuc; 2939 int ret = -EINVAL, n; 2940 2941 /* 2942 * Reject events from different hybrid PMUs. 2943 */ 2944 if (is_hybrid()) { 2945 struct perf_event *sibling; 2946 struct pmu *pmu = NULL; 2947 2948 if (is_x86_event(leader)) 2949 pmu = leader->pmu; 2950 2951 for_each_sibling_event(sibling, leader) { 2952 if (!is_x86_event(sibling)) 2953 continue; 2954 if (!pmu) 2955 pmu = sibling->pmu; 2956 else if (pmu != sibling->pmu) 2957 return ret; 2958 } 2959 } 2960 2961 fake_cpuc = allocate_fake_cpuc(event->pmu); 2962 if (IS_ERR(fake_cpuc)) 2963 return PTR_ERR(fake_cpuc); 2964 /* 2965 * the event is not yet connected with its 2966 * siblings therefore we must first collect 2967 * existing siblings, then add the new event 2968 * before we can simulate the scheduling 2969 */ 2970 n = collect_events(fake_cpuc, leader, true); 2971 if (n < 0) 2972 goto out; 2973 2974 fake_cpuc->n_events = n; 2975 n = collect_events(fake_cpuc, event, false); 2976 if (n < 0) 2977 goto out; 2978 2979 fake_cpuc->n_events = 0; 2980 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2981 2982 out: 2983 free_fake_cpuc(fake_cpuc); 2984 return ret; 2985 } 2986 2987 static int x86_pmu_event_init(struct perf_event *event) 2988 { 2989 struct x86_hybrid_pmu *pmu = NULL; 2990 int err; 2991 2992 if ((event->attr.type != event->pmu->type) && 2993 (event->attr.type != PERF_TYPE_HARDWARE) && 2994 (event->attr.type != PERF_TYPE_HW_CACHE)) 2995 return -ENOENT; 2996 2997 if (is_hybrid() && (event->cpu != -1)) { 2998 pmu = hybrid_pmu(event->pmu); 2999 if (!cpumask_test_cpu(event->cpu, &pmu->supported_cpus)) 3000 return -ENOENT; 3001 } 3002 3003 err = __x86_pmu_event_init(event); 3004 if (!err) { 3005 if (event->group_leader != event) 3006 err = validate_group(event); 3007 else 3008 err = validate_event(event); 3009 } 3010 if (err) { 3011 if (event->destroy) 3012 event->destroy(event); 3013 event->destroy = NULL; 3014 } 3015 3016 if (READ_ONCE(x86_pmu.attr_rdpmc) && 3017 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS) && 3018 !(event->hw.config & ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE)) 3019 event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; 3020 3021 return err; 3022 } 3023 3024 void perf_clear_dirty_counters(void) 3025 { 3026 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 3027 int i; 3028 3029 /* Don't need to clear the assigned counter. */ 3030 for (i = 0; i < cpuc->n_events; i++) 3031 __clear_bit(cpuc->assign[i], cpuc->dirty); 3032 3033 if (bitmap_empty(cpuc->dirty, X86_PMC_IDX_MAX)) 3034 return; 3035 3036 for_each_set_bit(i, cpuc->dirty, X86_PMC_IDX_MAX) { 3037 if (i >= INTEL_PMC_IDX_FIXED) { 3038 /* Metrics and fake events don't have corresponding HW counters. */ 3039 if (!test_bit(i - INTEL_PMC_IDX_FIXED, hybrid(cpuc->pmu, fixed_cntr_mask))) 3040 continue; 3041 3042 wrmsrq(x86_pmu_fixed_ctr_addr(i - INTEL_PMC_IDX_FIXED), 0); 3043 } else { 3044 wrmsrq(x86_pmu_event_addr(i), 0); 3045 } 3046 } 3047 3048 bitmap_zero(cpuc->dirty, X86_PMC_IDX_MAX); 3049 } 3050 3051 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) 3052 { 3053 if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) 3054 return; 3055 3056 /* 3057 * This function relies on not being called concurrently in two 3058 * tasks in the same mm. Otherwise one task could observe 3059 * perf_rdpmc_allowed > 1 and return all the way back to 3060 * userspace with CR4.PCE clear while another task is still 3061 * doing on_each_cpu_mask() to propagate CR4.PCE. 3062 * 3063 * For now, this can't happen because all callers hold mmap_lock 3064 * for write. If this changes, we'll need a different solution. 3065 */ 3066 mmap_assert_write_locked(mm); 3067 3068 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) 3069 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1); 3070 } 3071 3072 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) 3073 { 3074 if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) 3075 return; 3076 3077 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) 3078 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1); 3079 } 3080 3081 static int x86_pmu_event_idx(struct perf_event *event) 3082 { 3083 struct hw_perf_event *hwc = &event->hw; 3084 3085 if (!(hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT)) 3086 return 0; 3087 3088 if (is_metric_idx(hwc->idx)) 3089 return INTEL_PMC_FIXED_RDPMC_METRICS + 1; 3090 else 3091 return hwc->event_base_rdpmc + 1; 3092 } 3093 3094 static ssize_t get_attr_rdpmc(struct device *cdev, 3095 struct device_attribute *attr, 3096 char *buf) 3097 { 3098 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 3099 } 3100 3101 /* 3102 * Behaviors of rdpmc value: 3103 * - rdpmc = 0 3104 * global user space rdpmc and counter level's user space rdpmc of all 3105 * counters are both disabled. 3106 * - rdpmc = 1 3107 * global user space rdpmc is enabled in mmap enabled time window and 3108 * counter level's user space rdpmc is enabled for only non system-wide 3109 * events. Counter level's user space rdpmc of system-wide events is 3110 * still disabled by default. This won't introduce counter data leak for 3111 * non system-wide events since their count data would be cleared when 3112 * context switches. 3113 * - rdpmc = 2 3114 * global user space rdpmc and counter level's user space rdpmc of all 3115 * counters are enabled unconditionally. 3116 * 3117 * Suppose the rdpmc value won't be changed frequently, don't dynamically 3118 * reschedule events to make the new rpdmc value take effect on active perf 3119 * events immediately, the new rdpmc value would only impact the new 3120 * activated perf events. This makes code simpler and cleaner. 3121 */ 3122 static ssize_t set_attr_rdpmc(struct device *cdev, 3123 struct device_attribute *attr, 3124 const char *buf, size_t count) 3125 { 3126 static DEFINE_MUTEX(rdpmc_mutex); 3127 unsigned long val; 3128 ssize_t ret; 3129 3130 ret = kstrtoul(buf, 0, &val); 3131 if (ret) 3132 return ret; 3133 3134 if (val > 2) 3135 return -EINVAL; 3136 3137 if (x86_pmu.attr_rdpmc_broken) 3138 return -ENOTSUPP; 3139 3140 guard(mutex)(&rdpmc_mutex); 3141 3142 if (val != x86_pmu.attr_rdpmc) { 3143 /* 3144 * Changing into or out of never available or always available, 3145 * aka perf-event-bypassing mode. This path is extremely slow, 3146 * but only root can trigger it, so it's okay. 3147 */ 3148 if (val == 0) 3149 static_branch_inc(&rdpmc_never_available_key); 3150 else if (x86_pmu.attr_rdpmc == X86_USER_RDPMC_NEVER_ENABLE) 3151 static_branch_dec(&rdpmc_never_available_key); 3152 3153 if (val == 2) 3154 static_branch_inc(&rdpmc_always_available_key); 3155 else if (x86_pmu.attr_rdpmc == X86_USER_RDPMC_ALWAYS_ENABLE) 3156 static_branch_dec(&rdpmc_always_available_key); 3157 3158 on_each_cpu(cr4_update_pce, NULL, 1); 3159 x86_pmu.attr_rdpmc = val; 3160 } 3161 3162 return count; 3163 } 3164 3165 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 3166 3167 static struct attribute *x86_pmu_attrs[] = { 3168 &dev_attr_rdpmc.attr, 3169 NULL, 3170 }; 3171 3172 static struct attribute_group x86_pmu_attr_group __ro_after_init = { 3173 .attrs = x86_pmu_attrs, 3174 }; 3175 3176 static ssize_t max_precise_show(struct device *cdev, 3177 struct device_attribute *attr, 3178 char *buf) 3179 { 3180 struct pmu *pmu = dev_get_drvdata(cdev); 3181 3182 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise(pmu)); 3183 } 3184 3185 static DEVICE_ATTR_RO(max_precise); 3186 3187 static struct attribute *x86_pmu_caps_attrs[] = { 3188 &dev_attr_max_precise.attr, 3189 NULL 3190 }; 3191 3192 static struct attribute_group x86_pmu_caps_group __ro_after_init = { 3193 .name = "caps", 3194 .attrs = x86_pmu_caps_attrs, 3195 }; 3196 3197 static const struct attribute_group *x86_pmu_attr_groups[] = { 3198 &x86_pmu_attr_group, 3199 &x86_pmu_format_group, 3200 &x86_pmu_events_group, 3201 &x86_pmu_caps_group, 3202 NULL, 3203 }; 3204 3205 static void x86_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, 3206 struct task_struct *task, bool sched_in) 3207 { 3208 static_call_cond(x86_pmu_sched_task)(pmu_ctx, task, sched_in); 3209 } 3210 3211 void perf_check_microcode(void) 3212 { 3213 if (x86_pmu.check_microcode) 3214 x86_pmu.check_microcode(); 3215 } 3216 3217 static int x86_pmu_check_period(struct perf_event *event, u64 value) 3218 { 3219 if (x86_pmu.check_period && x86_pmu.check_period(event, value)) 3220 return -EINVAL; 3221 3222 if (value && x86_pmu.limit_period) { 3223 s64 left = value; 3224 x86_pmu.limit_period(event, &left); 3225 if (left > value) 3226 return -EINVAL; 3227 } 3228 3229 return 0; 3230 } 3231 3232 static int x86_pmu_aux_output_match(struct perf_event *event) 3233 { 3234 if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT)) 3235 return 0; 3236 3237 if (x86_pmu.aux_output_match) 3238 return x86_pmu.aux_output_match(event); 3239 3240 return 0; 3241 } 3242 3243 static bool x86_pmu_filter(struct pmu *pmu, int cpu) 3244 { 3245 bool ret = false; 3246 3247 static_call_cond(x86_pmu_filter)(pmu, cpu, &ret); 3248 3249 return ret; 3250 } 3251 3252 static struct pmu pmu = { 3253 .pmu_enable = x86_pmu_enable, 3254 .pmu_disable = x86_pmu_disable, 3255 3256 .attr_groups = x86_pmu_attr_groups, 3257 3258 .event_init = x86_pmu_event_init, 3259 3260 .event_mapped = x86_pmu_event_mapped, 3261 .event_unmapped = x86_pmu_event_unmapped, 3262 3263 .add = x86_pmu_add, 3264 .del = x86_pmu_del, 3265 .start = x86_pmu_start, 3266 .stop = x86_pmu_stop, 3267 .read = x86_pmu_read, 3268 3269 .start_txn = x86_pmu_start_txn, 3270 .cancel_txn = x86_pmu_cancel_txn, 3271 .commit_txn = x86_pmu_commit_txn, 3272 3273 .event_idx = x86_pmu_event_idx, 3274 .sched_task = x86_pmu_sched_task, 3275 .check_period = x86_pmu_check_period, 3276 3277 .aux_output_match = x86_pmu_aux_output_match, 3278 3279 .filter = x86_pmu_filter, 3280 }; 3281 3282 void arch_perf_update_userpage(struct perf_event *event, 3283 struct perf_event_mmap_page *userpg, u64 now) 3284 { 3285 struct cyc2ns_data data; 3286 u64 offset; 3287 3288 userpg->cap_user_time = 0; 3289 userpg->cap_user_time_zero = 0; 3290 userpg->cap_user_rdpmc = 3291 !!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT); 3292 userpg->pmc_width = x86_pmu.cntval_bits; 3293 3294 if (!using_native_sched_clock() || !sched_clock_stable()) 3295 return; 3296 3297 cyc2ns_read_begin(&data); 3298 3299 offset = data.cyc2ns_offset + __sched_clock_offset; 3300 3301 /* 3302 * Internal timekeeping for enabled/running/stopped times 3303 * is always in the local_clock domain. 3304 */ 3305 userpg->cap_user_time = 1; 3306 userpg->time_mult = data.cyc2ns_mul; 3307 userpg->time_shift = data.cyc2ns_shift; 3308 userpg->time_offset = offset - now; 3309 3310 /* 3311 * cap_user_time_zero doesn't make sense when we're using a different 3312 * time base for the records. 3313 */ 3314 if (!event->attr.use_clockid) { 3315 userpg->cap_user_time_zero = 1; 3316 userpg->time_zero = offset; 3317 } 3318 3319 cyc2ns_read_end(); 3320 } 3321 3322 /* 3323 * Determine whether the regs were taken from an irq/exception handler rather 3324 * than from perf_arch_fetch_caller_regs(). 3325 */ 3326 static bool perf_hw_regs(struct pt_regs *regs) 3327 { 3328 return regs->flags & X86_EFLAGS_FIXED; 3329 } 3330 3331 void 3332 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 3333 { 3334 struct unwind_state state; 3335 unsigned long addr; 3336 3337 if (perf_guest_state()) { 3338 /* TODO: We don't support guest os callchain now */ 3339 return; 3340 } 3341 3342 if (perf_hw_regs(regs)) { 3343 if (perf_callchain_store(entry, regs->ip)) 3344 return; 3345 unwind_start(&state, current, regs, NULL); 3346 } else { 3347 unwind_start(&state, current, NULL, (void *)regs->sp); 3348 } 3349 3350 for (; !unwind_done(&state); unwind_next_frame(&state)) { 3351 addr = unwind_get_return_address(&state); 3352 if (!addr || perf_callchain_store(entry, addr)) 3353 return; 3354 } 3355 } 3356 3357 static unsigned long get_segment_base(unsigned int segment) 3358 { 3359 struct desc_struct *desc; 3360 unsigned int idx = segment >> 3; 3361 3362 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 3363 #ifdef CONFIG_MODIFY_LDT_SYSCALL 3364 struct ldt_struct *ldt; 3365 3366 /* 3367 * If we're not in a valid context with a real (not just lazy) 3368 * user mm, then don't even try. 3369 */ 3370 if (!nmi_uaccess_okay()) 3371 return 0; 3372 3373 /* IRQs are off, so this synchronizes with smp_store_release */ 3374 ldt = smp_load_acquire(¤t->mm->context.ldt); 3375 if (!ldt || idx >= ldt->nr_entries) 3376 return 0; 3377 3378 desc = &ldt->entries[idx]; 3379 #else 3380 return 0; 3381 #endif 3382 } else { 3383 if (idx >= GDT_ENTRIES) 3384 return 0; 3385 3386 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 3387 } 3388 3389 return get_desc_base(desc); 3390 } 3391 3392 #ifdef CONFIG_IA32_EMULATION 3393 3394 #include <linux/compat.h> 3395 3396 static inline int 3397 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 3398 { 3399 /* 32-bit process in 64-bit kernel. */ 3400 unsigned long ss_base, cs_base; 3401 struct stack_frame_ia32 frame; 3402 const struct stack_frame_ia32 __user *fp; 3403 u32 ret_addr; 3404 3405 if (user_64bit_mode(regs)) 3406 return 0; 3407 3408 cs_base = get_segment_base(regs->cs); 3409 ss_base = get_segment_base(regs->ss); 3410 3411 fp = compat_ptr(ss_base + regs->bp); 3412 pagefault_disable(); 3413 3414 /* see perf_callchain_user() below for why we do this */ 3415 if (is_uprobe_at_func_entry(regs) && 3416 !get_user(ret_addr, (const u32 __user *)regs->sp)) 3417 perf_callchain_store(entry, ret_addr); 3418 3419 while (entry->nr < entry->max_stack) { 3420 scoped_user_read_access(fp, out) { 3421 unsafe_get_user(frame.next_frame, &fp->next_frame, out); 3422 unsafe_get_user(frame.return_address, &fp->return_address, out); 3423 } 3424 3425 perf_callchain_store(entry, cs_base + frame.return_address); 3426 fp = compat_ptr(ss_base + frame.next_frame); 3427 } 3428 out: 3429 pagefault_enable(); 3430 return 1; 3431 } 3432 #else 3433 static inline int 3434 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 3435 { 3436 return 0; 3437 } 3438 #endif 3439 3440 void 3441 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 3442 { 3443 struct stack_frame frame; 3444 const struct stack_frame __user *fp; 3445 unsigned long ret_addr; 3446 3447 if (perf_guest_state()) { 3448 /* TODO: We don't support guest os callchain now */ 3449 return; 3450 } 3451 3452 /* 3453 * We don't know what to do with VM86 stacks.. ignore them for now. 3454 */ 3455 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 3456 return; 3457 3458 fp = (void __user *)regs->bp; 3459 3460 perf_callchain_store(entry, regs->ip); 3461 3462 if (!nmi_uaccess_okay()) 3463 return; 3464 3465 if (perf_callchain_user32(regs, entry)) 3466 return; 3467 3468 pagefault_disable(); 3469 3470 /* 3471 * If we are called from uprobe handler, and we are indeed at the very 3472 * entry to user function (which is normally a `push %rbp` instruction, 3473 * under assumption of application being compiled with frame pointers), 3474 * we should read return address from *regs->sp before proceeding 3475 * to follow frame pointers, otherwise we'll skip immediate caller 3476 * as %rbp is not yet setup. 3477 */ 3478 if (is_uprobe_at_func_entry(regs) && 3479 !get_user(ret_addr, (const unsigned long __user *)regs->sp)) 3480 perf_callchain_store(entry, ret_addr); 3481 3482 while (entry->nr < entry->max_stack) { 3483 scoped_user_read_access(fp, out) { 3484 unsafe_get_user(frame.next_frame, &fp->next_frame, out); 3485 unsafe_get_user(frame.return_address, &fp->return_address, out); 3486 } 3487 3488 perf_callchain_store(entry, frame.return_address); 3489 fp = (void __user *)frame.next_frame; 3490 } 3491 out: 3492 pagefault_enable(); 3493 } 3494 3495 /* 3496 * Deal with code segment offsets for the various execution modes: 3497 * 3498 * VM86 - the good olde 16 bit days, where the linear address is 3499 * 20 bits and we use regs->ip + 0x10 * regs->cs. 3500 * 3501 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 3502 * to figure out what the 32bit base address is. 3503 * 3504 * X32 - has TIF_X32 set, but is running in x86_64 3505 * 3506 * X86_64 - CS,DS,SS,ES are all zero based. 3507 */ 3508 static unsigned long code_segment_base(struct pt_regs *regs) 3509 { 3510 /* 3511 * For IA32 we look at the GDT/LDT segment base to convert the 3512 * effective IP to a linear address. 3513 */ 3514 3515 #ifdef CONFIG_X86_32 3516 /* 3517 * If we are in VM86 mode, add the segment offset to convert to a 3518 * linear address. 3519 */ 3520 if (regs->flags & X86_VM_MASK) 3521 return 0x10 * regs->cs; 3522 3523 if (user_mode(regs) && regs->cs != __USER_CS) 3524 return get_segment_base(regs->cs); 3525 #else 3526 if (user_mode(regs) && !user_64bit_mode(regs) && 3527 regs->cs != __USER32_CS) 3528 return get_segment_base(regs->cs); 3529 #endif 3530 return 0; 3531 } 3532 3533 unsigned long perf_arch_instruction_pointer(struct pt_regs *regs) 3534 { 3535 return regs->ip + code_segment_base(regs); 3536 } 3537 3538 static unsigned long common_misc_flags(struct pt_regs *regs) 3539 { 3540 if (regs->flags & PERF_EFLAGS_EXACT) 3541 return PERF_RECORD_MISC_EXACT_IP; 3542 3543 return 0; 3544 } 3545 3546 static unsigned long guest_misc_flags(struct pt_regs *regs) 3547 { 3548 unsigned long guest_state = perf_guest_state(); 3549 3550 if (!(guest_state & PERF_GUEST_ACTIVE)) 3551 return 0; 3552 3553 if (guest_state & PERF_GUEST_USER) 3554 return PERF_RECORD_MISC_GUEST_USER; 3555 else 3556 return PERF_RECORD_MISC_GUEST_KERNEL; 3557 3558 } 3559 3560 static unsigned long host_misc_flags(struct pt_regs *regs) 3561 { 3562 if (user_mode(regs)) 3563 return PERF_RECORD_MISC_USER; 3564 else 3565 return PERF_RECORD_MISC_KERNEL; 3566 } 3567 3568 unsigned long perf_arch_guest_misc_flags(struct pt_regs *regs) 3569 { 3570 unsigned long flags = common_misc_flags(regs); 3571 3572 flags |= guest_misc_flags(regs); 3573 3574 return flags; 3575 } 3576 3577 unsigned long perf_arch_misc_flags(struct pt_regs *regs) 3578 { 3579 unsigned long flags = common_misc_flags(regs); 3580 3581 flags |= host_misc_flags(regs); 3582 3583 return flags; 3584 } 3585 3586 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 3587 { 3588 /* This API doesn't currently support enumerating hybrid PMUs. */ 3589 if (WARN_ON_ONCE(cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) || 3590 !x86_pmu_initialized()) { 3591 memset(cap, 0, sizeof(*cap)); 3592 return; 3593 } 3594 3595 /* 3596 * Note, hybrid CPU models get tracked as having hybrid PMUs even when 3597 * all E-cores are disabled via BIOS. When E-cores are disabled, the 3598 * base PMU holds the correct number of counters for P-cores. 3599 */ 3600 cap->version = x86_pmu.version; 3601 cap->num_counters_gp = x86_pmu_num_counters(NULL); 3602 cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL); 3603 cap->bit_width_gp = cap->num_counters_gp ? x86_pmu.cntval_bits : 0; 3604 cap->bit_width_fixed = cap->num_counters_fixed ? x86_pmu.cntval_bits : 0; 3605 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 3606 cap->events_mask_len = x86_pmu.events_mask_len; 3607 cap->pebs_ept = x86_pmu.pebs_ept; 3608 cap->mediated = !!(pmu.capabilities & PERF_PMU_CAP_MEDIATED_VPMU); 3609 } 3610 EXPORT_SYMBOL_FOR_KVM(perf_get_x86_pmu_capability); 3611 3612 u64 perf_get_hw_event_config(int hw_event) 3613 { 3614 int max = x86_pmu.max_events; 3615 3616 if (hw_event < max) 3617 return x86_pmu.event_map(array_index_nospec(hw_event, max)); 3618 3619 return 0; 3620 } 3621 EXPORT_SYMBOL_FOR_KVM(perf_get_hw_event_config); 3622