1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine -- Performance Monitoring Unit support 4 * 5 * Copyright 2015 Red Hat, Inc. and/or its affiliates. 6 * 7 * Authors: 8 * Avi Kivity <avi@redhat.com> 9 * Gleb Natapov <gleb@redhat.com> 10 * Wei Huang <wei@redhat.com> 11 */ 12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 13 14 #include <linux/types.h> 15 #include <linux/kvm_host.h> 16 #include <linux/perf_event.h> 17 #include <linux/bsearch.h> 18 #include <linux/sort.h> 19 #include <linux/moduleparam.h> 20 #include <asm/perf_event.h> 21 #include <asm/cpu_device_id.h> 22 #include "x86.h" 23 #include "cpuid.h" 24 #include "lapic.h" 25 #include "pmu.h" 26 27 /* This is enough to filter the vast majority of currently defined events. */ 28 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300 29 30 /* Unadultered PMU capabilities of the host, i.e. of hardware. */ 31 static struct x86_pmu_capability __read_mostly kvm_host_pmu; 32 33 /* KVM's PMU capabilities, i.e. the intersection of KVM and hardware support. */ 34 struct x86_pmu_capability __read_mostly kvm_pmu_cap; 35 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_cap); 36 37 /* Enable/disable PMU virtualization */ 38 bool __read_mostly enable_pmu = true; 39 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_pmu); 40 module_param(enable_pmu, bool, 0444); 41 42 /* Enable/disabled mediated PMU virtualization. */ 43 bool __read_mostly enable_mediated_pmu; 44 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_mediated_pmu); 45 46 struct kvm_x86_pmu_event_filter { 47 __u32 action; 48 __u32 nevents; 49 __u32 fixed_counter_bitmap; 50 __u32 flags; 51 __u32 nr_includes; 52 __u32 nr_excludes; 53 __u64 *includes; 54 __u64 *excludes; 55 __u64 events[] __counted_by(nevents); 56 }; 57 58 struct kvm_pmu_emulated_event_selectors { 59 u64 INSTRUCTIONS_RETIRED; 60 u64 BRANCH_INSTRUCTIONS_RETIRED; 61 }; 62 static struct kvm_pmu_emulated_event_selectors __read_mostly kvm_pmu_eventsel; 63 64 /* Precise Distribution of Instructions Retired (PDIR) */ 65 static const struct x86_cpu_id vmx_pebs_pdir_cpu[] = { 66 X86_MATCH_VFM(INTEL_ICELAKE_D, NULL), 67 X86_MATCH_VFM(INTEL_ICELAKE_X, NULL), 68 /* Instruction-Accurate PDIR (PDIR++) */ 69 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL), 70 {} 71 }; 72 73 /* Precise Distribution (PDist) */ 74 static const struct x86_cpu_id vmx_pebs_pdist_cpu[] = { 75 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL), 76 {} 77 }; 78 79 /* NOTE: 80 * - Each perf counter is defined as "struct kvm_pmc"; 81 * - There are two types of perf counters: general purpose (gp) and fixed. 82 * gp counters are stored in gp_counters[] and fixed counters are stored 83 * in fixed_counters[] respectively. Both of them are part of "struct 84 * kvm_pmu"; 85 * - pmu.c understands the difference between gp counters and fixed counters. 86 * However AMD doesn't support fixed-counters; 87 * - There are three types of index to access perf counters (PMC): 88 * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD 89 * has MSR_K7_PERFCTRn and, for families 15H and later, 90 * MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are 91 * aliased to MSR_K7_PERFCTRn. 92 * 2. MSR Index (named idx): This normally is used by RDPMC instruction. 93 * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access 94 * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except 95 * that it also supports fixed counters. idx can be used to as index to 96 * gp and fixed counters. 97 * 3. Global PMC Index (named pmc): pmc is an index specific to PMU 98 * code. Each pmc, stored in kvm_pmc.idx field, is unique across 99 * all perf counters (both gp and fixed). The mapping relationship 100 * between pmc and perf counters is as the following: 101 * * Intel: [0 .. KVM_MAX_NR_INTEL_GP_COUNTERS-1] <=> gp counters 102 * [KVM_FIXED_PMC_BASE_IDX .. KVM_FIXED_PMC_BASE_IDX + 2] <=> fixed 103 * * AMD: [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H 104 * and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters 105 */ 106 107 static struct kvm_pmu_ops kvm_pmu_ops __read_mostly; 108 109 #define KVM_X86_PMU_OP(func) \ 110 DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ 111 *(((struct kvm_pmu_ops *)0)->func)); 112 #define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP 113 #define KVM_X86_PMU_OP_OPTIONAL_RET0 KVM_X86_PMU_OP 114 #include <asm/kvm-x86-pmu-ops.h> 115 EXPORT_STATIC_CALL_GPL(kvm_x86_pmu_pmc_is_disabled_in_current_mode); 116 117 void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) 118 { 119 memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops)); 120 121 #define __KVM_X86_PMU_OP(func) \ 122 static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func); 123 #define KVM_X86_PMU_OP(func) \ 124 WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func) 125 #define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP 126 #define KVM_X86_PMU_OP_OPTIONAL_RET0(func) \ 127 static_call_update(kvm_x86_pmu_##func, (void *)kvm_pmu_ops.func ? : \ 128 (void *)__static_call_return0); 129 #include <asm/kvm-x86-pmu-ops.h> 130 #undef __KVM_X86_PMU_OP 131 } 132 133 void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops) 134 { 135 bool is_intel = boot_cpu_data.x86_vendor == X86_VENDOR_INTEL; 136 int min_nr_gp_ctrs = pmu_ops->MIN_NR_GP_COUNTERS; 137 138 /* 139 * Hybrid PMUs don't play nice with virtualization without careful 140 * configuration by userspace, and KVM's APIs for reporting supported 141 * vPMU features do not account for hybrid PMUs. Disable vPMU support 142 * for hybrid PMUs until KVM gains a way to let userspace opt-in. 143 */ 144 if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) { 145 enable_pmu = false; 146 memset(&kvm_host_pmu, 0, sizeof(kvm_host_pmu)); 147 } else { 148 perf_get_x86_pmu_capability(&kvm_host_pmu); 149 } 150 151 if (enable_pmu) { 152 /* 153 * WARN if perf did NOT disable hardware PMU if the number of 154 * architecturally required GP counters aren't present, i.e. if 155 * there are a non-zero number of counters, but fewer than what 156 * is architecturally required. 157 */ 158 if (!kvm_host_pmu.num_counters_gp || 159 WARN_ON_ONCE(kvm_host_pmu.num_counters_gp < min_nr_gp_ctrs)) 160 enable_pmu = false; 161 else if (is_intel && !kvm_host_pmu.version) 162 enable_pmu = false; 163 } 164 165 if (!enable_pmu || !enable_mediated_pmu || !kvm_host_pmu.mediated || 166 !pmu_ops->is_mediated_pmu_supported(&kvm_host_pmu)) 167 enable_mediated_pmu = false; 168 169 if (!enable_mediated_pmu) 170 pmu_ops->write_global_ctrl = NULL; 171 172 if (!enable_pmu) { 173 memset(&kvm_pmu_cap, 0, sizeof(kvm_pmu_cap)); 174 return; 175 } 176 177 memcpy(&kvm_pmu_cap, &kvm_host_pmu, sizeof(kvm_host_pmu)); 178 kvm_pmu_cap.version = min(kvm_pmu_cap.version, 2); 179 kvm_pmu_cap.num_counters_gp = min(kvm_pmu_cap.num_counters_gp, 180 pmu_ops->MAX_NR_GP_COUNTERS); 181 kvm_pmu_cap.num_counters_fixed = min(kvm_pmu_cap.num_counters_fixed, 182 KVM_MAX_NR_FIXED_COUNTERS); 183 184 kvm_pmu_eventsel.INSTRUCTIONS_RETIRED = 185 perf_get_hw_event_config(PERF_COUNT_HW_INSTRUCTIONS); 186 kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED = 187 perf_get_hw_event_config(PERF_COUNT_HW_BRANCH_INSTRUCTIONS); 188 } 189 190 void kvm_handle_guest_mediated_pmi(void) 191 { 192 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 193 194 if (WARN_ON_ONCE(!vcpu || !kvm_vcpu_has_mediated_pmu(vcpu))) 195 return; 196 197 kvm_make_request(KVM_REQ_PMI, vcpu); 198 } 199 200 static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi) 201 { 202 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 203 bool skip_pmi = false; 204 205 if (pmc->perf_event && pmc->perf_event->attr.precise_ip) { 206 if (!in_pmi) { 207 /* 208 * TODO: KVM is currently _choosing_ to not generate records 209 * for emulated instructions, avoiding BUFFER_OVF PMI when 210 * there are no records. Strictly speaking, it should be done 211 * as well in the right context to improve sampling accuracy. 212 */ 213 skip_pmi = true; 214 } else { 215 /* Indicate PEBS overflow PMI to guest. */ 216 skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, 217 (unsigned long *)&pmu->global_status); 218 } 219 } else { 220 __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); 221 } 222 223 if (pmc->intr && !skip_pmi) 224 kvm_make_request(KVM_REQ_PMI, pmc->vcpu); 225 } 226 227 static void kvm_perf_overflow(struct perf_event *perf_event, 228 struct perf_sample_data *data, 229 struct pt_regs *regs) 230 { 231 struct kvm_pmc *pmc = perf_event->overflow_handler_context; 232 233 /* 234 * Ignore asynchronous overflow events for counters that are scheduled 235 * to be reprogrammed, e.g. if a PMI for the previous event races with 236 * KVM's handling of a related guest WRMSR. 237 */ 238 if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi)) 239 return; 240 241 __kvm_perf_overflow(pmc, true); 242 243 kvm_make_request(KVM_REQ_PMU, pmc->vcpu); 244 } 245 246 static u64 pmc_get_pebs_precise_level(struct kvm_pmc *pmc) 247 { 248 /* 249 * For some model specific pebs counters with special capabilities 250 * (PDIR, PDIR++, PDIST), KVM needs to raise the event precise 251 * level to the maximum value (currently 3, backwards compatible) 252 * so that the perf subsystem would assign specific hardware counter 253 * with that capability for vPMC. 254 */ 255 if ((pmc->idx == 0 && x86_match_cpu(vmx_pebs_pdist_cpu)) || 256 (pmc->idx == 32 && x86_match_cpu(vmx_pebs_pdir_cpu))) 257 return 3; 258 259 /* 260 * The non-zero precision level of guest event makes the ordinary 261 * guest event becomes a guest PEBS event and triggers the host 262 * PEBS PMI handler to determine whether the PEBS overflow PMI 263 * comes from the host counters or the guest. 264 */ 265 return 1; 266 } 267 268 static u64 get_sample_period(struct kvm_pmc *pmc, u64 counter_value) 269 { 270 u64 sample_period = (-counter_value) & pmc_bitmask(pmc); 271 272 if (!sample_period) 273 sample_period = pmc_bitmask(pmc) + 1; 274 return sample_period; 275 } 276 277 static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config, 278 bool exclude_user, bool exclude_kernel, 279 bool intr) 280 { 281 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 282 struct perf_event *event; 283 struct perf_event_attr attr = { 284 .type = type, 285 .size = sizeof(attr), 286 .pinned = true, 287 .exclude_idle = true, 288 .exclude_host = 1, 289 .exclude_user = exclude_user, 290 .exclude_kernel = exclude_kernel, 291 .config = config, 292 }; 293 bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable); 294 295 attr.sample_period = get_sample_period(pmc, pmc->counter); 296 297 if ((attr.config & HSW_IN_TX_CHECKPOINTED) && 298 (boot_cpu_has(X86_FEATURE_RTM) || boot_cpu_has(X86_FEATURE_HLE))) { 299 /* 300 * HSW_IN_TX_CHECKPOINTED is not supported with nonzero 301 * period. Just clear the sample period so at least 302 * allocating the counter doesn't fail. 303 */ 304 attr.sample_period = 0; 305 } 306 if (pebs) { 307 /* 308 * For most PEBS hardware events, the difference in the software 309 * precision levels of guest and host PEBS events will not affect 310 * the accuracy of the PEBS profiling result, because the "event IP" 311 * in the PEBS record is calibrated on the guest side. 312 */ 313 attr.precise_ip = pmc_get_pebs_precise_level(pmc); 314 } 315 316 event = perf_event_create_kernel_counter(&attr, -1, current, 317 kvm_perf_overflow, pmc); 318 if (IS_ERR(event)) { 319 pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n", 320 PTR_ERR(event), pmc->idx); 321 return PTR_ERR(event); 322 } 323 324 pmc->perf_event = event; 325 pmc_to_pmu(pmc)->event_count++; 326 pmc->is_paused = false; 327 pmc->intr = intr || pebs; 328 return 0; 329 } 330 331 static bool pmc_pause_counter(struct kvm_pmc *pmc) 332 { 333 u64 counter = pmc->counter; 334 u64 prev_counter; 335 336 /* update counter, reset event value to avoid redundant accumulation */ 337 if (pmc->perf_event && !pmc->is_paused) 338 counter += perf_event_pause(pmc->perf_event, true); 339 340 /* 341 * Snapshot the previous counter *after* accumulating state from perf. 342 * If overflow already happened, hardware (via perf) is responsible for 343 * generating a PMI. KVM just needs to detect overflow on emulated 344 * counter events that haven't yet been processed. 345 */ 346 prev_counter = counter & pmc_bitmask(pmc); 347 348 counter += pmc->emulated_counter; 349 pmc->counter = counter & pmc_bitmask(pmc); 350 351 pmc->emulated_counter = 0; 352 pmc->is_paused = true; 353 354 return pmc->counter < prev_counter; 355 } 356 357 static bool pmc_resume_counter(struct kvm_pmc *pmc) 358 { 359 if (!pmc->perf_event) 360 return false; 361 362 /* recalibrate sample period and check if it's accepted by perf core */ 363 if (is_sampling_event(pmc->perf_event) && 364 perf_event_period(pmc->perf_event, 365 get_sample_period(pmc, pmc->counter))) 366 return false; 367 368 if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) != 369 (!!pmc->perf_event->attr.precise_ip)) 370 return false; 371 372 /* reuse perf_event to serve as pmc_reprogram_counter() does*/ 373 perf_event_enable(pmc->perf_event); 374 pmc->is_paused = false; 375 376 return true; 377 } 378 379 static void pmc_release_perf_event(struct kvm_pmc *pmc) 380 { 381 if (pmc->perf_event) { 382 perf_event_release_kernel(pmc->perf_event); 383 pmc->perf_event = NULL; 384 pmc->current_config = 0; 385 pmc_to_pmu(pmc)->event_count--; 386 } 387 } 388 389 static void pmc_stop_counter(struct kvm_pmc *pmc) 390 { 391 if (pmc->perf_event) { 392 pmc->counter = pmc_read_counter(pmc); 393 pmc_release_perf_event(pmc); 394 } 395 } 396 397 static void pmc_update_sample_period(struct kvm_pmc *pmc) 398 { 399 if (!pmc->perf_event || pmc->is_paused || 400 !is_sampling_event(pmc->perf_event)) 401 return; 402 403 perf_event_period(pmc->perf_event, 404 get_sample_period(pmc, pmc->counter)); 405 } 406 407 void pmc_write_counter(struct kvm_pmc *pmc, u64 val) 408 { 409 if (kvm_vcpu_has_mediated_pmu(pmc->vcpu)) { 410 pmc->counter = val & pmc_bitmask(pmc); 411 return; 412 } 413 414 /* 415 * Drop any unconsumed accumulated counts, the WRMSR is a write, not a 416 * read-modify-write. Adjust the counter value so that its value is 417 * relative to the current count, as reading the current count from 418 * perf is faster than pausing and reprogramming the event in order to 419 * reset it to '0'. Note, this very sneakily offsets the accumulated 420 * emulated count too, by using pmc_read_counter()! 421 */ 422 pmc->emulated_counter = 0; 423 pmc->counter += val - pmc_read_counter(pmc); 424 pmc->counter &= pmc_bitmask(pmc); 425 pmc_update_sample_period(pmc); 426 } 427 EXPORT_SYMBOL_FOR_KVM_INTERNAL(pmc_write_counter); 428 429 static int filter_cmp(const void *pa, const void *pb, u64 mask) 430 { 431 u64 a = *(u64 *)pa & mask; 432 u64 b = *(u64 *)pb & mask; 433 434 return (a > b) - (a < b); 435 } 436 437 static int filter_sort_cmp(const void *pa, const void *pb) 438 { 439 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT | 440 KVM_PMU_MASKED_ENTRY_EXCLUDE)); 441 } 442 443 /* 444 * For the event filter, searching is done on the 'includes' list and 445 * 'excludes' list separately rather than on the 'events' list (which 446 * has both). As a result the exclude bit can be ignored. 447 */ 448 static int filter_event_cmp(const void *pa, const void *pb) 449 { 450 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT)); 451 } 452 453 static int find_filter_index(u64 *events, u64 nevents, u64 key) 454 { 455 u64 *fe = bsearch(&key, events, nevents, sizeof(events[0]), 456 filter_event_cmp); 457 458 if (!fe) 459 return -1; 460 461 return fe - events; 462 } 463 464 static bool is_filter_entry_match(u64 filter_event, u64 umask) 465 { 466 u64 mask = filter_event >> (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8); 467 u64 match = filter_event & KVM_PMU_MASKED_ENTRY_UMASK_MATCH; 468 469 BUILD_BUG_ON((KVM_PMU_ENCODE_MASKED_ENTRY(0, 0xff, 0, false) >> 470 (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8)) != 471 ARCH_PERFMON_EVENTSEL_UMASK); 472 473 return (umask & mask) == match; 474 } 475 476 static bool filter_contains_match(u64 *events, u64 nevents, u64 eventsel) 477 { 478 u64 event_select = eventsel & kvm_pmu_ops.EVENTSEL_EVENT; 479 u64 umask = eventsel & ARCH_PERFMON_EVENTSEL_UMASK; 480 int i, index; 481 482 index = find_filter_index(events, nevents, event_select); 483 if (index < 0) 484 return false; 485 486 /* 487 * Entries are sorted by the event select. Walk the list in both 488 * directions to process all entries with the targeted event select. 489 */ 490 for (i = index; i < nevents; i++) { 491 if (filter_event_cmp(&events[i], &event_select)) 492 break; 493 494 if (is_filter_entry_match(events[i], umask)) 495 return true; 496 } 497 498 for (i = index - 1; i >= 0; i--) { 499 if (filter_event_cmp(&events[i], &event_select)) 500 break; 501 502 if (is_filter_entry_match(events[i], umask)) 503 return true; 504 } 505 506 return false; 507 } 508 509 static bool is_gp_event_allowed(struct kvm_x86_pmu_event_filter *f, 510 u64 eventsel) 511 { 512 if (filter_contains_match(f->includes, f->nr_includes, eventsel) && 513 !filter_contains_match(f->excludes, f->nr_excludes, eventsel)) 514 return f->action == KVM_PMU_EVENT_ALLOW; 515 516 return f->action == KVM_PMU_EVENT_DENY; 517 } 518 519 static bool is_fixed_event_allowed(struct kvm_x86_pmu_event_filter *filter, 520 int idx) 521 { 522 int fixed_idx = idx - KVM_FIXED_PMC_BASE_IDX; 523 524 if (filter->action == KVM_PMU_EVENT_DENY && 525 test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) 526 return false; 527 if (filter->action == KVM_PMU_EVENT_ALLOW && 528 !test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) 529 return false; 530 531 return true; 532 } 533 534 static bool pmc_is_event_allowed(struct kvm_pmc *pmc) 535 { 536 struct kvm_x86_pmu_event_filter *filter; 537 struct kvm *kvm = pmc->vcpu->kvm; 538 539 filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu); 540 if (!filter) 541 return true; 542 543 if (pmc_is_gp(pmc)) 544 return is_gp_event_allowed(filter, pmc->eventsel); 545 546 return is_fixed_event_allowed(filter, pmc->idx); 547 } 548 549 static void kvm_mediated_pmu_refresh_event_filter(struct kvm_pmc *pmc) 550 { 551 bool allowed = pmc_is_locally_enabled(pmc) && pmc_is_event_allowed(pmc); 552 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 553 554 if (pmc_is_gp(pmc)) { 555 pmc->eventsel_hw &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 556 if (allowed) 557 pmc->eventsel_hw |= pmc->eventsel & 558 ARCH_PERFMON_EVENTSEL_ENABLE; 559 } else { 560 u64 mask = intel_fixed_bits_by_idx(pmc->idx - KVM_FIXED_PMC_BASE_IDX, 0xf); 561 562 pmu->fixed_ctr_ctrl_hw &= ~mask; 563 if (allowed) 564 pmu->fixed_ctr_ctrl_hw |= pmu->fixed_ctr_ctrl & mask; 565 } 566 } 567 568 static int reprogram_counter(struct kvm_pmc *pmc) 569 { 570 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 571 u64 eventsel = pmc->eventsel; 572 u64 new_config = eventsel; 573 bool emulate_overflow; 574 u8 fixed_ctr_ctrl; 575 576 if (kvm_vcpu_has_mediated_pmu(pmu_to_vcpu(pmu))) { 577 kvm_mediated_pmu_refresh_event_filter(pmc); 578 return 0; 579 } 580 581 emulate_overflow = pmc_pause_counter(pmc); 582 583 if (!pmc_is_globally_enabled(pmc) || !pmc_is_locally_enabled(pmc) || 584 !pmc_is_event_allowed(pmc)) 585 return 0; 586 587 if (emulate_overflow) 588 __kvm_perf_overflow(pmc, false); 589 590 if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) 591 printk_once("kvm pmu: pin control bit is ignored\n"); 592 593 if (pmc_is_fixed(pmc)) { 594 fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, 595 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 596 if (fixed_ctr_ctrl & INTEL_FIXED_0_KERNEL) 597 eventsel |= ARCH_PERFMON_EVENTSEL_OS; 598 if (fixed_ctr_ctrl & INTEL_FIXED_0_USER) 599 eventsel |= ARCH_PERFMON_EVENTSEL_USR; 600 if (fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI) 601 eventsel |= ARCH_PERFMON_EVENTSEL_INT; 602 new_config = (u64)fixed_ctr_ctrl; 603 } 604 605 if (pmc->current_config == new_config && pmc_resume_counter(pmc)) 606 return 0; 607 608 pmc_release_perf_event(pmc); 609 610 pmc->current_config = new_config; 611 612 return pmc_reprogram_counter(pmc, PERF_TYPE_RAW, 613 (eventsel & pmu->raw_event_mask), 614 !(eventsel & ARCH_PERFMON_EVENTSEL_USR), 615 !(eventsel & ARCH_PERFMON_EVENTSEL_OS), 616 eventsel & ARCH_PERFMON_EVENTSEL_INT); 617 } 618 619 static bool pmc_is_event_match(struct kvm_pmc *pmc, u64 eventsel) 620 { 621 /* 622 * Ignore checks for edge detect (all events currently emulated by KVM 623 * are always rising edges), pin control (unsupported by modern CPUs), 624 * and counter mask and its invert flag (KVM doesn't emulate multiple 625 * events in a single clock cycle). 626 * 627 * Note, the uppermost nibble of AMD's mask overlaps Intel's IN_TX (bit 628 * 32) and IN_TXCP (bit 33), as well as two reserved bits (bits 35:34). 629 * Checking the "in HLE/RTM transaction" flags is correct as the vCPU 630 * can't be in a transaction if KVM is emulating an instruction. 631 * 632 * Checking the reserved bits might be wrong if they are defined in the 633 * future, but so could ignoring them, so do the simple thing for now. 634 */ 635 return !((pmc->eventsel ^ eventsel) & AMD64_RAW_EVENT_MASK_NB); 636 } 637 638 void kvm_pmu_recalc_pmc_emulation(struct kvm_pmu *pmu, struct kvm_pmc *pmc) 639 { 640 bitmap_clear(pmu->pmc_counting_instructions, pmc->idx, 1); 641 bitmap_clear(pmu->pmc_counting_branches, pmc->idx, 1); 642 643 /* 644 * Do NOT consult the PMU event filters, as the filters must be checked 645 * at the time of emulation to ensure KVM uses fresh information, e.g. 646 * omitting a PMC from a bitmap could result in a missed event if the 647 * filter is changed to allow counting the event. 648 */ 649 if (!pmc_is_locally_enabled(pmc)) 650 return; 651 652 if (pmc_is_event_match(pmc, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED)) 653 bitmap_set(pmu->pmc_counting_instructions, pmc->idx, 1); 654 655 if (pmc_is_event_match(pmc, kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED)) 656 bitmap_set(pmu->pmc_counting_branches, pmc->idx, 1); 657 } 658 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_recalc_pmc_emulation); 659 660 void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) 661 { 662 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); 663 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 664 struct kvm_pmc *pmc; 665 int bit, r; 666 667 bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); 668 669 /* 670 * The reprogramming bitmap can be written asynchronously by something 671 * other than the task that holds vcpu->mutex, take care to clear only 672 * the bits that will actually be processed. 673 */ 674 BUILD_BUG_ON(sizeof(bitmap) != sizeof(atomic64_t)); 675 atomic64_andnot(*(s64 *)bitmap, &pmu->__reprogram_pmi); 676 677 kvm_for_each_pmc(pmu, pmc, bit, bitmap) { 678 /* 679 * If reprogramming fails on a transient condition, e.g. due to 680 * contention, re-set the reprogram bit, i.e. opportunistically 681 * try again on the next PMU refresh. Don't make a new request 682 * as doing so can stall the guest if reprogramming repeatedly 683 * fails. 684 */ 685 r = reprogram_counter(pmc); 686 if (r == -EBUSY || r == -ENOMEM) 687 set_bit(pmc->idx, pmu->reprogram_pmi); 688 } 689 690 /* 691 * Release unused perf_events if the corresponding guest MSRs weren't 692 * accessed during the last vCPU time slice (need_cleanup is set when 693 * the vCPU is scheduled back in). 694 */ 695 if (unlikely(pmu->need_cleanup)) 696 kvm_pmu_cleanup(vcpu); 697 698 kvm_for_each_pmc(pmu, pmc, bit, bitmap) 699 kvm_pmu_recalc_pmc_emulation(pmu, pmc); 700 } 701 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_handle_event); 702 703 int kvm_pmu_check_rdpmc_early(struct kvm_vcpu *vcpu, unsigned int idx) 704 { 705 /* 706 * On Intel, VMX interception has priority over RDPMC exceptions that 707 * aren't already handled by the emulator, i.e. there are no additional 708 * check needed for Intel PMUs. 709 * 710 * On AMD, _all_ exceptions on RDPMC have priority over SVM intercepts, 711 * i.e. an invalid PMC results in a #GP, not #VMEXIT. 712 */ 713 if (!kvm_pmu_ops.check_rdpmc_early) 714 return 0; 715 716 return kvm_pmu_call(check_rdpmc_early)(vcpu, idx); 717 } 718 719 bool is_vmware_backdoor_pmc(u32 pmc_idx) 720 { 721 switch (pmc_idx) { 722 case VMWARE_BACKDOOR_PMC_HOST_TSC: 723 case VMWARE_BACKDOOR_PMC_REAL_TIME: 724 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 725 return true; 726 } 727 return false; 728 } 729 730 static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 731 { 732 u64 ctr_val; 733 734 switch (idx) { 735 case VMWARE_BACKDOOR_PMC_HOST_TSC: 736 ctr_val = rdtsc(); 737 break; 738 case VMWARE_BACKDOOR_PMC_REAL_TIME: 739 ctr_val = ktime_get_boottime_ns(); 740 break; 741 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 742 ctr_val = ktime_get_boottime_ns() + 743 vcpu->kvm->arch.kvmclock_offset; 744 break; 745 default: 746 return 1; 747 } 748 749 *data = ctr_val; 750 return 0; 751 } 752 753 int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 754 { 755 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 756 struct kvm_pmc *pmc; 757 u64 mask = ~0ull; 758 759 if (!pmu->version) 760 return 1; 761 762 if (is_vmware_backdoor_pmc(idx)) 763 return kvm_pmu_rdpmc_vmware(vcpu, idx, data); 764 765 pmc = kvm_pmu_call(rdpmc_ecx_to_pmc)(vcpu, idx, &mask); 766 if (!pmc) 767 return 1; 768 769 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCE) && 770 (kvm_x86_call(get_cpl)(vcpu) != 0) && 771 kvm_is_cr0_bit_set(vcpu, X86_CR0_PE)) 772 return 1; 773 774 *data = pmc_read_counter(pmc) & mask; 775 return 0; 776 } 777 778 static bool kvm_need_any_pmc_intercept(struct kvm_vcpu *vcpu) 779 { 780 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 781 782 if (!kvm_vcpu_has_mediated_pmu(vcpu)) 783 return true; 784 785 /* 786 * Note! Check *host* PMU capabilities, not KVM's PMU capabilities, as 787 * KVM's capabilities are constrained based on KVM support, i.e. KVM's 788 * capabilities themselves may be a subset of hardware capabilities. 789 */ 790 return pmu->nr_arch_gp_counters != kvm_host_pmu.num_counters_gp || 791 pmu->nr_arch_fixed_counters != kvm_host_pmu.num_counters_fixed; 792 } 793 794 bool kvm_need_perf_global_ctrl_intercept(struct kvm_vcpu *vcpu) 795 { 796 return kvm_need_any_pmc_intercept(vcpu) || 797 !kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); 798 } 799 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_perf_global_ctrl_intercept); 800 801 bool kvm_need_rdpmc_intercept(struct kvm_vcpu *vcpu) 802 { 803 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 804 805 /* 806 * VMware allows access to these Pseduo-PMCs even when read via RDPMC 807 * in Ring3 when CR4.PCE=0. 808 */ 809 if (enable_vmware_backdoor) 810 return true; 811 812 return kvm_need_any_pmc_intercept(vcpu) || 813 pmu->counter_bitmask[KVM_PMC_GP] != (BIT_ULL(kvm_host_pmu.bit_width_gp) - 1) || 814 pmu->counter_bitmask[KVM_PMC_FIXED] != (BIT_ULL(kvm_host_pmu.bit_width_fixed) - 1); 815 } 816 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_rdpmc_intercept); 817 818 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) 819 { 820 if (lapic_in_kernel(vcpu)) { 821 kvm_pmu_call(deliver_pmi)(vcpu); 822 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); 823 } 824 } 825 826 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 827 { 828 return kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr) || 829 kvm_pmu_call(is_valid_msr)(vcpu, msr); 830 } 831 832 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) 833 { 834 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 835 struct kvm_pmc *pmc = kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr); 836 837 if (pmc) 838 __set_bit(pmc->idx, pmu->pmc_in_use); 839 } 840 841 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 842 { 843 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 844 u32 msr = msr_info->index; 845 846 switch (msr) { 847 case MSR_CORE_PERF_GLOBAL_STATUS: 848 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: 849 msr_info->data = pmu->global_status; 850 break; 851 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: 852 case MSR_CORE_PERF_GLOBAL_CTRL: 853 msr_info->data = pmu->global_ctrl; 854 break; 855 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: 856 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: 857 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 858 msr_info->data = 0; 859 break; 860 default: 861 return kvm_pmu_call(get_msr)(vcpu, msr_info); 862 } 863 864 return 0; 865 } 866 867 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 868 { 869 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 870 u32 msr = msr_info->index; 871 u64 data = msr_info->data; 872 u64 diff; 873 874 /* 875 * Note, AMD ignores writes to reserved bits and read-only PMU MSRs, 876 * whereas Intel generates #GP on attempts to write reserved/RO MSRs. 877 */ 878 switch (msr) { 879 case MSR_CORE_PERF_GLOBAL_STATUS: 880 if (!msr_info->host_initiated) 881 return 1; /* RO MSR */ 882 fallthrough; 883 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: 884 /* Per PPR, Read-only MSR. Writes are ignored. */ 885 if (!msr_info->host_initiated) 886 break; 887 888 if (data & pmu->global_status_rsvd) 889 return 1; 890 891 pmu->global_status = data; 892 break; 893 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: 894 data &= ~pmu->global_ctrl_rsvd; 895 fallthrough; 896 case MSR_CORE_PERF_GLOBAL_CTRL: 897 if (!kvm_valid_perf_global_ctrl(pmu, data)) 898 return 1; 899 900 if (pmu->global_ctrl != data) { 901 diff = pmu->global_ctrl ^ data; 902 pmu->global_ctrl = data; 903 kvm_pmu_request_counters_reprogram(pmu, diff); 904 } 905 /* 906 * Unconditionally forward writes to vendor code, i.e. to the 907 * VMC{B,S}, as pmu->global_ctrl is per-VCPU, not per-VMC{B,S}. 908 */ 909 if (kvm_vcpu_has_mediated_pmu(vcpu)) 910 kvm_pmu_call(write_global_ctrl)(data); 911 break; 912 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 913 /* 914 * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in 915 * GLOBAL_STATUS, and so the set of reserved bits is the same. 916 */ 917 if (data & pmu->global_status_rsvd) 918 return 1; 919 fallthrough; 920 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: 921 if (!msr_info->host_initiated) 922 pmu->global_status &= ~data; 923 break; 924 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: 925 if (!msr_info->host_initiated) 926 pmu->global_status |= data & ~pmu->global_status_rsvd; 927 break; 928 default: 929 kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); 930 return kvm_pmu_call(set_msr)(vcpu, msr_info); 931 } 932 933 return 0; 934 } 935 936 static void kvm_pmu_reset(struct kvm_vcpu *vcpu) 937 { 938 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 939 struct kvm_pmc *pmc; 940 int i; 941 942 pmu->need_cleanup = false; 943 944 bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX); 945 bitmap_zero(pmu->pmc_has_mode_specific_enables, X86_PMC_IDX_MAX); 946 947 kvm_for_each_pmc(pmu, pmc, i, pmu->all_valid_pmc_idx) { 948 pmc_stop_counter(pmc); 949 pmc->counter = 0; 950 pmc->emulated_counter = 0; 951 952 if (pmc_is_gp(pmc)) { 953 pmc->eventsel = 0; 954 pmc->eventsel_hw = 0; 955 } 956 } 957 958 pmu->fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw = 0; 959 pmu->global_ctrl = pmu->global_status = 0; 960 961 kvm_pmu_call(reset)(vcpu); 962 } 963 964 /* 965 * Refresh the PMU configuration for the vCPU, e.g. if userspace changes CPUID 966 * and/or PERF_CAPABILITIES. 967 */ 968 void kvm_pmu_refresh(struct kvm_vcpu *vcpu) 969 { 970 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 971 972 if (KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm)) 973 return; 974 975 /* 976 * Stop/release all existing counters/events before realizing the new 977 * vPMU model. 978 */ 979 kvm_pmu_reset(vcpu); 980 981 pmu->version = 0; 982 pmu->nr_arch_gp_counters = 0; 983 pmu->nr_arch_fixed_counters = 0; 984 pmu->counter_bitmask[KVM_PMC_GP] = 0; 985 pmu->counter_bitmask[KVM_PMC_FIXED] = 0; 986 pmu->reserved_bits = 0xffffffff00200000ull; 987 pmu->raw_event_mask = X86_RAW_EVENT_MASK; 988 pmu->global_ctrl_rsvd = ~0ull; 989 pmu->global_status_rsvd = ~0ull; 990 pmu->fixed_ctr_ctrl_rsvd = ~0ull; 991 pmu->pebs_enable_rsvd = ~0ull; 992 pmu->pebs_data_cfg_rsvd = ~0ull; 993 bitmap_zero(pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX); 994 995 if (!vcpu->kvm->arch.enable_pmu) 996 return; 997 998 kvm_pmu_call(refresh)(vcpu); 999 1000 /* 1001 * At RESET, both Intel and AMD CPUs set all enable bits for general 1002 * purpose counters in IA32_PERF_GLOBAL_CTRL (so that software that 1003 * was written for v1 PMUs don't unknowingly leave GP counters disabled 1004 * in the global controls). Emulate that behavior when refreshing the 1005 * PMU so that userspace doesn't need to manually set PERF_GLOBAL_CTRL. 1006 */ 1007 if (pmu->nr_arch_gp_counters && 1008 (kvm_pmu_has_perf_global_ctrl(pmu) || kvm_vcpu_has_mediated_pmu(vcpu))) 1009 pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0); 1010 1011 if (kvm_vcpu_has_mediated_pmu(vcpu)) 1012 kvm_pmu_call(write_global_ctrl)(pmu->global_ctrl); 1013 1014 bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters); 1015 bitmap_set(pmu->all_valid_pmc_idx, KVM_FIXED_PMC_BASE_IDX, 1016 pmu->nr_arch_fixed_counters); 1017 } 1018 1019 void kvm_pmu_init(struct kvm_vcpu *vcpu) 1020 { 1021 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1022 1023 memset(pmu, 0, sizeof(*pmu)); 1024 kvm_pmu_call(init)(vcpu); 1025 } 1026 1027 /* Release perf_events for vPMCs that have been unused for a full time slice. */ 1028 void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) 1029 { 1030 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1031 struct kvm_pmc *pmc = NULL; 1032 DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX); 1033 int i; 1034 1035 pmu->need_cleanup = false; 1036 1037 bitmap_andnot(bitmask, pmu->all_valid_pmc_idx, 1038 pmu->pmc_in_use, X86_PMC_IDX_MAX); 1039 1040 kvm_for_each_pmc(pmu, pmc, i, bitmask) { 1041 if (pmc->perf_event && !pmc_is_locally_enabled(pmc)) 1042 pmc_stop_counter(pmc); 1043 } 1044 1045 kvm_pmu_call(cleanup)(vcpu); 1046 1047 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); 1048 } 1049 1050 void kvm_pmu_destroy(struct kvm_vcpu *vcpu) 1051 { 1052 kvm_pmu_reset(vcpu); 1053 } 1054 1055 static bool pmc_is_pmi_enabled(struct kvm_pmc *pmc) 1056 { 1057 u8 fixed_ctr_ctrl; 1058 1059 if (pmc_is_gp(pmc)) 1060 return pmc->eventsel & ARCH_PERFMON_EVENTSEL_INT; 1061 1062 fixed_ctr_ctrl = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl, 1063 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 1064 return fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI; 1065 } 1066 1067 static void kvm_pmu_incr_counter(struct kvm_pmc *pmc) 1068 { 1069 struct kvm_vcpu *vcpu = pmc->vcpu; 1070 1071 /* 1072 * For perf-based PMUs, accumulate software-emulated events separately 1073 * from pmc->counter, as pmc->counter is offset by the count of the 1074 * associated perf event. Request reprogramming, which will consult 1075 * both emulated and hardware-generated events to detect overflow. 1076 */ 1077 if (!kvm_vcpu_has_mediated_pmu(vcpu)) { 1078 pmc->emulated_counter++; 1079 kvm_pmu_request_counter_reprogram(pmc); 1080 return; 1081 } 1082 1083 /* 1084 * For mediated PMUs, pmc->counter is updated when the vCPU's PMU is 1085 * put, and will be loaded into hardware when the PMU is loaded. Simply 1086 * increment the counter and signal overflow if it wraps to zero. 1087 */ 1088 pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc); 1089 if (!pmc->counter) { 1090 pmc_to_pmu(pmc)->global_status |= BIT_ULL(pmc->idx); 1091 if (pmc_is_pmi_enabled(pmc)) 1092 kvm_make_request(KVM_REQ_PMI, vcpu); 1093 } 1094 } 1095 1096 static inline bool cpl_is_matched(struct kvm_pmc *pmc) 1097 { 1098 bool select_os, select_user; 1099 u64 config; 1100 1101 if (pmc_is_gp(pmc)) { 1102 config = pmc->eventsel; 1103 select_os = config & ARCH_PERFMON_EVENTSEL_OS; 1104 select_user = config & ARCH_PERFMON_EVENTSEL_USR; 1105 } else { 1106 config = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl, 1107 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 1108 select_os = config & INTEL_FIXED_0_KERNEL; 1109 select_user = config & INTEL_FIXED_0_USER; 1110 } 1111 1112 /* 1113 * Skip the CPL lookup, which isn't free on Intel, if the result will 1114 * be the same regardless of the CPL. 1115 */ 1116 if (select_os == select_user) 1117 return select_os; 1118 1119 return (kvm_x86_call(get_cpl)(pmc->vcpu) == 0) ? select_os : 1120 select_user; 1121 } 1122 1123 static void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, 1124 const unsigned long *event_pmcs) 1125 { 1126 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); 1127 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1128 struct kvm_pmc *pmc; 1129 int i, idx; 1130 1131 BUILD_BUG_ON(sizeof(pmu->global_ctrl) * BITS_PER_BYTE != X86_PMC_IDX_MAX); 1132 1133 if (bitmap_empty(event_pmcs, X86_PMC_IDX_MAX)) 1134 return; 1135 1136 if (!kvm_pmu_has_perf_global_ctrl(pmu)) 1137 bitmap_copy(bitmap, event_pmcs, X86_PMC_IDX_MAX); 1138 else if (!bitmap_and(bitmap, event_pmcs, 1139 (unsigned long *)&pmu->global_ctrl, X86_PMC_IDX_MAX)) 1140 return; 1141 1142 idx = srcu_read_lock(&vcpu->kvm->srcu); 1143 kvm_for_each_pmc(pmu, pmc, i, bitmap) { 1144 if (!pmc_is_event_allowed(pmc) || !cpl_is_matched(pmc)) 1145 continue; 1146 1147 kvm_pmu_incr_counter(pmc); 1148 } 1149 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1150 } 1151 1152 void kvm_pmu_instruction_retired(struct kvm_vcpu *vcpu) 1153 { 1154 kvm_pmu_trigger_event(vcpu, vcpu_to_pmu(vcpu)->pmc_counting_instructions); 1155 } 1156 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_instruction_retired); 1157 1158 void kvm_pmu_branch_retired(struct kvm_vcpu *vcpu) 1159 { 1160 kvm_pmu_trigger_event(vcpu, vcpu_to_pmu(vcpu)->pmc_counting_branches); 1161 } 1162 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_branch_retired); 1163 1164 static bool is_masked_filter_valid(const struct kvm_x86_pmu_event_filter *filter) 1165 { 1166 u64 mask = kvm_pmu_ops.EVENTSEL_EVENT | 1167 KVM_PMU_MASKED_ENTRY_UMASK_MASK | 1168 KVM_PMU_MASKED_ENTRY_UMASK_MATCH | 1169 KVM_PMU_MASKED_ENTRY_EXCLUDE; 1170 int i; 1171 1172 for (i = 0; i < filter->nevents; i++) { 1173 if (filter->events[i] & ~mask) 1174 return false; 1175 } 1176 1177 return true; 1178 } 1179 1180 static void convert_to_masked_filter(struct kvm_x86_pmu_event_filter *filter) 1181 { 1182 int i, j; 1183 1184 for (i = 0, j = 0; i < filter->nevents; i++) { 1185 /* 1186 * Skip events that are impossible to match against a guest 1187 * event. When filtering, only the event select + unit mask 1188 * of the guest event is used. To maintain backwards 1189 * compatibility, impossible filters can't be rejected :-( 1190 */ 1191 if (filter->events[i] & ~(kvm_pmu_ops.EVENTSEL_EVENT | 1192 ARCH_PERFMON_EVENTSEL_UMASK)) 1193 continue; 1194 /* 1195 * Convert userspace events to a common in-kernel event so 1196 * only one code path is needed to support both events. For 1197 * the in-kernel events use masked events because they are 1198 * flexible enough to handle both cases. To convert to masked 1199 * events all that's needed is to add an "all ones" umask_mask, 1200 * (unmasked filter events don't support EXCLUDE). 1201 */ 1202 filter->events[j++] = filter->events[i] | 1203 (0xFFULL << KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT); 1204 } 1205 1206 filter->nevents = j; 1207 } 1208 1209 static int prepare_filter_lists(struct kvm_x86_pmu_event_filter *filter) 1210 { 1211 int i; 1212 1213 if (!(filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS)) 1214 convert_to_masked_filter(filter); 1215 else if (!is_masked_filter_valid(filter)) 1216 return -EINVAL; 1217 1218 /* 1219 * Sort entries by event select and includes vs. excludes so that all 1220 * entries for a given event select can be processed efficiently during 1221 * filtering. The EXCLUDE flag uses a more significant bit than the 1222 * event select, and so the sorted list is also effectively split into 1223 * includes and excludes sub-lists. 1224 */ 1225 sort(&filter->events, filter->nevents, sizeof(filter->events[0]), 1226 filter_sort_cmp, NULL); 1227 1228 i = filter->nevents; 1229 /* Find the first EXCLUDE event (only supported for masked events). */ 1230 if (filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS) { 1231 for (i = 0; i < filter->nevents; i++) { 1232 if (filter->events[i] & KVM_PMU_MASKED_ENTRY_EXCLUDE) 1233 break; 1234 } 1235 } 1236 1237 filter->nr_includes = i; 1238 filter->nr_excludes = filter->nevents - filter->nr_includes; 1239 filter->includes = filter->events; 1240 filter->excludes = filter->events + filter->nr_includes; 1241 1242 return 0; 1243 } 1244 1245 int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp) 1246 { 1247 struct kvm_pmu_event_filter __user *user_filter = argp; 1248 struct kvm_x86_pmu_event_filter *filter; 1249 struct kvm_pmu_event_filter tmp; 1250 struct kvm_vcpu *vcpu; 1251 unsigned long i; 1252 size_t size; 1253 int r; 1254 1255 if (copy_from_user(&tmp, user_filter, sizeof(tmp))) 1256 return -EFAULT; 1257 1258 if (tmp.action != KVM_PMU_EVENT_ALLOW && 1259 tmp.action != KVM_PMU_EVENT_DENY) 1260 return -EINVAL; 1261 1262 if (tmp.flags & ~KVM_PMU_EVENT_FLAGS_VALID_MASK) 1263 return -EINVAL; 1264 1265 if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS) 1266 return -E2BIG; 1267 1268 size = struct_size(filter, events, tmp.nevents); 1269 filter = kzalloc(size, GFP_KERNEL_ACCOUNT); 1270 if (!filter) 1271 return -ENOMEM; 1272 1273 filter->action = tmp.action; 1274 filter->nevents = tmp.nevents; 1275 filter->fixed_counter_bitmap = tmp.fixed_counter_bitmap; 1276 filter->flags = tmp.flags; 1277 1278 r = -EFAULT; 1279 if (copy_from_user(filter->events, user_filter->events, 1280 flex_array_size(filter, events, filter->nevents))) 1281 goto cleanup; 1282 1283 r = prepare_filter_lists(filter); 1284 if (r) 1285 goto cleanup; 1286 1287 mutex_lock(&kvm->lock); 1288 filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter, 1289 mutex_is_locked(&kvm->lock)); 1290 mutex_unlock(&kvm->lock); 1291 synchronize_srcu_expedited(&kvm->srcu); 1292 1293 BUILD_BUG_ON(sizeof(((struct kvm_pmu *)0)->reprogram_pmi) > 1294 sizeof(((struct kvm_pmu *)0)->__reprogram_pmi)); 1295 1296 kvm_for_each_vcpu(i, vcpu, kvm) 1297 atomic64_set(&vcpu_to_pmu(vcpu)->__reprogram_pmi, -1ull); 1298 1299 kvm_make_all_cpus_request(kvm, KVM_REQ_PMU); 1300 1301 r = 0; 1302 cleanup: 1303 kfree(filter); 1304 return r; 1305 } 1306 1307 static __always_inline u32 fixed_counter_msr(u32 idx) 1308 { 1309 return kvm_pmu_ops.FIXED_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1310 } 1311 1312 static __always_inline u32 gp_counter_msr(u32 idx) 1313 { 1314 return kvm_pmu_ops.GP_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1315 } 1316 1317 static __always_inline u32 gp_eventsel_msr(u32 idx) 1318 { 1319 return kvm_pmu_ops.GP_EVENTSEL_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1320 } 1321 1322 static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu) 1323 { 1324 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1325 struct kvm_pmc *pmc; 1326 u32 i; 1327 1328 /* 1329 * No need to zero out unexposed GP/fixed counters/selectors since RDPMC 1330 * is intercepted if hardware has counters that aren't visible to the 1331 * guest (KVM will inject #GP as appropriate). 1332 */ 1333 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 1334 pmc = &pmu->gp_counters[i]; 1335 1336 if (pmc->counter != rdpmc(i)) 1337 wrmsrq(gp_counter_msr(i), pmc->counter); 1338 wrmsrq(gp_eventsel_msr(i), pmc->eventsel_hw); 1339 } 1340 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 1341 pmc = &pmu->fixed_counters[i]; 1342 1343 if (pmc->counter != rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i)) 1344 wrmsrq(fixed_counter_msr(i), pmc->counter); 1345 } 1346 } 1347 1348 void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu) 1349 { 1350 if (!kvm_vcpu_has_mediated_pmu(vcpu) || 1351 KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 1352 return; 1353 1354 lockdep_assert_irqs_disabled(); 1355 1356 perf_load_guest_context(); 1357 1358 /* 1359 * Explicitly clear PERF_GLOBAL_CTRL, as "loading" the guest's context 1360 * disables all individual counters (if any were enabled), but doesn't 1361 * globally disable the entire PMU. Loading event selectors and PMCs 1362 * with guest values while PERF_GLOBAL_CTRL is non-zero will generate 1363 * unexpected events and PMIs. 1364 * 1365 * VMX will enable/disable counters at VM-Enter/VM-Exit by atomically 1366 * loading PERF_GLOBAL_CONTROL. SVM effectively performs the switch by 1367 * configuring all events to be GUEST_ONLY. Clear PERF_GLOBAL_CONTROL 1368 * even for SVM to minimize the damage if a perf event is left enabled, 1369 * and to ensure a consistent starting state. 1370 */ 1371 wrmsrq(kvm_pmu_ops.PERF_GLOBAL_CTRL, 0); 1372 1373 perf_load_guest_lvtpc(kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVTPC)); 1374 1375 kvm_pmu_load_guest_pmcs(vcpu); 1376 1377 kvm_pmu_call(mediated_load)(vcpu); 1378 } 1379 1380 static void kvm_pmu_put_guest_pmcs(struct kvm_vcpu *vcpu) 1381 { 1382 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1383 struct kvm_pmc *pmc; 1384 u32 i; 1385 1386 /* 1387 * Clear selectors and counters to ensure hardware doesn't count using 1388 * guest controls when the host (perf) restores its state. 1389 */ 1390 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 1391 pmc = &pmu->gp_counters[i]; 1392 1393 pmc->counter = rdpmc(i); 1394 if (pmc->counter) 1395 wrmsrq(gp_counter_msr(i), 0); 1396 if (pmc->eventsel_hw) 1397 wrmsrq(gp_eventsel_msr(i), 0); 1398 } 1399 1400 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 1401 pmc = &pmu->fixed_counters[i]; 1402 1403 pmc->counter = rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i); 1404 if (pmc->counter) 1405 wrmsrq(fixed_counter_msr(i), 0); 1406 } 1407 } 1408 1409 void kvm_mediated_pmu_put(struct kvm_vcpu *vcpu) 1410 { 1411 if (!kvm_vcpu_has_mediated_pmu(vcpu) || 1412 KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 1413 return; 1414 1415 lockdep_assert_irqs_disabled(); 1416 1417 /* 1418 * Defer handling of PERF_GLOBAL_CTRL to vendor code. On Intel, it's 1419 * atomically cleared on VM-Exit, i.e. doesn't need to be clear here. 1420 */ 1421 kvm_pmu_call(mediated_put)(vcpu); 1422 1423 kvm_pmu_put_guest_pmcs(vcpu); 1424 1425 perf_put_guest_lvtpc(); 1426 1427 perf_put_guest_context(); 1428 } 1429