1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine -- Performance Monitoring Unit support 4 * 5 * Copyright 2015 Red Hat, Inc. and/or its affiliates. 6 * 7 * Authors: 8 * Avi Kivity <avi@redhat.com> 9 * Gleb Natapov <gleb@redhat.com> 10 * Wei Huang <wei@redhat.com> 11 */ 12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 13 14 #include <linux/types.h> 15 #include <linux/kvm_host.h> 16 #include <linux/perf_event.h> 17 #include <linux/bsearch.h> 18 #include <linux/sort.h> 19 #include <linux/moduleparam.h> 20 #include <asm/perf_event.h> 21 #include <asm/cpu_device_id.h> 22 #include "x86.h" 23 #include "cpuid.h" 24 #include "lapic.h" 25 #include "pmu.h" 26 27 /* This is enough to filter the vast majority of currently defined events. */ 28 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300 29 30 /* Unadultered PMU capabilities of the host, i.e. of hardware. */ 31 static struct x86_pmu_capability __read_mostly kvm_host_pmu; 32 33 /* KVM's PMU capabilities, i.e. the intersection of KVM and hardware support. */ 34 struct x86_pmu_capability __read_mostly kvm_pmu_cap; 35 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_cap); 36 37 /* Enable/disable PMU virtualization */ 38 bool __read_mostly enable_pmu = true; 39 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_pmu); 40 module_param(enable_pmu, bool, 0444); 41 42 /* Enable/disabled mediated PMU virtualization. */ 43 bool __read_mostly enable_mediated_pmu; 44 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_mediated_pmu); 45 46 struct kvm_x86_pmu_event_filter { 47 __u32 action; 48 __u32 nevents; 49 __u32 fixed_counter_bitmap; 50 __u32 flags; 51 __u32 nr_includes; 52 __u32 nr_excludes; 53 __u64 *includes; 54 __u64 *excludes; 55 __u64 events[] __counted_by(nevents); 56 }; 57 58 struct kvm_pmu_emulated_event_selectors { 59 u64 INSTRUCTIONS_RETIRED; 60 u64 BRANCH_INSTRUCTIONS_RETIRED; 61 }; 62 static struct kvm_pmu_emulated_event_selectors __read_mostly kvm_pmu_eventsel; 63 64 /* Precise Distribution of Instructions Retired (PDIR) */ 65 static const struct x86_cpu_id vmx_pebs_pdir_cpu[] = { 66 X86_MATCH_VFM(INTEL_ICELAKE_D, NULL), 67 X86_MATCH_VFM(INTEL_ICELAKE_X, NULL), 68 /* Instruction-Accurate PDIR (PDIR++) */ 69 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL), 70 {} 71 }; 72 73 /* Precise Distribution (PDist) */ 74 static const struct x86_cpu_id vmx_pebs_pdist_cpu[] = { 75 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL), 76 {} 77 }; 78 79 /* NOTE: 80 * - Each perf counter is defined as "struct kvm_pmc"; 81 * - There are two types of perf counters: general purpose (gp) and fixed. 82 * gp counters are stored in gp_counters[] and fixed counters are stored 83 * in fixed_counters[] respectively. Both of them are part of "struct 84 * kvm_pmu"; 85 * - pmu.c understands the difference between gp counters and fixed counters. 86 * However AMD doesn't support fixed-counters; 87 * - There are three types of index to access perf counters (PMC): 88 * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD 89 * has MSR_K7_PERFCTRn and, for families 15H and later, 90 * MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are 91 * aliased to MSR_K7_PERFCTRn. 92 * 2. MSR Index (named idx): This normally is used by RDPMC instruction. 93 * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access 94 * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except 95 * that it also supports fixed counters. idx can be used to as index to 96 * gp and fixed counters. 97 * 3. Global PMC Index (named pmc): pmc is an index specific to PMU 98 * code. Each pmc, stored in kvm_pmc.idx field, is unique across 99 * all perf counters (both gp and fixed). The mapping relationship 100 * between pmc and perf counters is as the following: 101 * * Intel: [0 .. KVM_MAX_NR_INTEL_GP_COUNTERS-1] <=> gp counters 102 * [KVM_FIXED_PMC_BASE_IDX .. KVM_FIXED_PMC_BASE_IDX + 2] <=> fixed 103 * * AMD: [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H 104 * and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters 105 */ 106 107 static struct kvm_pmu_ops kvm_pmu_ops __read_mostly; 108 109 #define KVM_X86_PMU_OP(func) \ 110 DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ 111 *(((struct kvm_pmu_ops *)0)->func)); 112 #define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP 113 #define KVM_X86_PMU_OP_OPTIONAL_RET0 KVM_X86_PMU_OP 114 #include <asm/kvm-x86-pmu-ops.h> 115 EXPORT_STATIC_CALL_GPL(kvm_x86_pmu_pmc_is_disabled_in_current_mode); 116 117 void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) 118 { 119 memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops)); 120 121 #define __KVM_X86_PMU_OP(func) \ 122 static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func); 123 #define KVM_X86_PMU_OP(func) \ 124 WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func) 125 #define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP 126 #define KVM_X86_PMU_OP_OPTIONAL_RET0(func) \ 127 static_call_update(kvm_x86_pmu_##func, (void *)kvm_pmu_ops.func ? : \ 128 (void *)__static_call_return0); 129 #include <asm/kvm-x86-pmu-ops.h> 130 #undef __KVM_X86_PMU_OP 131 } 132 133 void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops) 134 { 135 bool is_intel = boot_cpu_data.x86_vendor == X86_VENDOR_INTEL; 136 int min_nr_gp_ctrs = pmu_ops->MIN_NR_GP_COUNTERS; 137 138 /* 139 * Hybrid PMUs don't play nice with virtualization without careful 140 * configuration by userspace, and KVM's APIs for reporting supported 141 * vPMU features do not account for hybrid PMUs. Disable vPMU support 142 * for hybrid PMUs until KVM gains a way to let userspace opt-in. 143 */ 144 if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) { 145 enable_pmu = false; 146 memset(&kvm_host_pmu, 0, sizeof(kvm_host_pmu)); 147 } else { 148 perf_get_x86_pmu_capability(&kvm_host_pmu); 149 } 150 151 if (enable_pmu) { 152 /* 153 * WARN if perf did NOT disable hardware PMU if the number of 154 * architecturally required GP counters aren't present, i.e. if 155 * there are a non-zero number of counters, but fewer than what 156 * is architecturally required. 157 */ 158 if (!kvm_host_pmu.num_counters_gp || 159 WARN_ON_ONCE(kvm_host_pmu.num_counters_gp < min_nr_gp_ctrs)) 160 enable_pmu = false; 161 else if (is_intel && !kvm_host_pmu.version) 162 enable_pmu = false; 163 } 164 165 if (!enable_pmu || !enable_mediated_pmu || !kvm_host_pmu.mediated || 166 !pmu_ops->is_mediated_pmu_supported(&kvm_host_pmu)) 167 enable_mediated_pmu = false; 168 169 if (!enable_mediated_pmu) 170 pmu_ops->write_global_ctrl = NULL; 171 172 if (!enable_pmu) { 173 memset(&kvm_pmu_cap, 0, sizeof(kvm_pmu_cap)); 174 return; 175 } 176 177 memcpy(&kvm_pmu_cap, &kvm_host_pmu, sizeof(kvm_host_pmu)); 178 kvm_pmu_cap.version = min(kvm_pmu_cap.version, 2); 179 kvm_pmu_cap.num_counters_gp = min(kvm_pmu_cap.num_counters_gp, 180 pmu_ops->MAX_NR_GP_COUNTERS); 181 kvm_pmu_cap.num_counters_fixed = min(kvm_pmu_cap.num_counters_fixed, 182 KVM_MAX_NR_FIXED_COUNTERS); 183 184 kvm_pmu_eventsel.INSTRUCTIONS_RETIRED = 185 perf_get_hw_event_config(PERF_COUNT_HW_INSTRUCTIONS); 186 kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED = 187 perf_get_hw_event_config(PERF_COUNT_HW_BRANCH_INSTRUCTIONS); 188 } 189 190 void kvm_handle_guest_mediated_pmi(void) 191 { 192 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 193 194 if (WARN_ON_ONCE(!vcpu || !kvm_vcpu_has_mediated_pmu(vcpu))) 195 return; 196 197 kvm_make_request(KVM_REQ_PMI, vcpu); 198 } 199 200 static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi) 201 { 202 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 203 bool skip_pmi = false; 204 205 if (pmc->perf_event && pmc->perf_event->attr.precise_ip) { 206 if (!in_pmi) { 207 /* 208 * TODO: KVM is currently _choosing_ to not generate records 209 * for emulated instructions, avoiding BUFFER_OVF PMI when 210 * there are no records. Strictly speaking, it should be done 211 * as well in the right context to improve sampling accuracy. 212 */ 213 skip_pmi = true; 214 } else { 215 /* Indicate PEBS overflow PMI to guest. */ 216 skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, 217 (unsigned long *)&pmu->global_status); 218 } 219 } else { 220 __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); 221 } 222 223 if (pmc->intr && !skip_pmi) 224 kvm_make_request(KVM_REQ_PMI, pmc->vcpu); 225 } 226 227 static void kvm_perf_overflow(struct perf_event *perf_event, 228 struct perf_sample_data *data, 229 struct pt_regs *regs) 230 { 231 struct kvm_pmc *pmc = perf_event->overflow_handler_context; 232 233 /* 234 * Ignore asynchronous overflow events for counters that are scheduled 235 * to be reprogrammed, e.g. if a PMI for the previous event races with 236 * KVM's handling of a related guest WRMSR. 237 */ 238 if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi)) 239 return; 240 241 __kvm_perf_overflow(pmc, true); 242 243 kvm_make_request(KVM_REQ_PMU, pmc->vcpu); 244 } 245 246 static u64 pmc_get_pebs_precise_level(struct kvm_pmc *pmc) 247 { 248 /* 249 * For some model specific pebs counters with special capabilities 250 * (PDIR, PDIR++, PDIST), KVM needs to raise the event precise 251 * level to the maximum value (currently 3, backwards compatible) 252 * so that the perf subsystem would assign specific hardware counter 253 * with that capability for vPMC. 254 */ 255 if ((pmc->idx == 0 && x86_match_cpu(vmx_pebs_pdist_cpu)) || 256 (pmc->idx == 32 && x86_match_cpu(vmx_pebs_pdir_cpu))) 257 return 3; 258 259 /* 260 * The non-zero precision level of guest event makes the ordinary 261 * guest event becomes a guest PEBS event and triggers the host 262 * PEBS PMI handler to determine whether the PEBS overflow PMI 263 * comes from the host counters or the guest. 264 */ 265 return 1; 266 } 267 268 static u64 get_sample_period(struct kvm_pmc *pmc, u64 counter_value) 269 { 270 u64 sample_period = (-counter_value) & pmc_bitmask(pmc); 271 272 if (!sample_period) 273 sample_period = pmc_bitmask(pmc) + 1; 274 return sample_period; 275 } 276 277 static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config, 278 bool exclude_user, bool exclude_kernel, 279 bool intr) 280 { 281 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 282 struct perf_event *event; 283 struct perf_event_attr attr = { 284 .type = type, 285 .size = sizeof(attr), 286 .pinned = true, 287 .exclude_idle = true, 288 .exclude_host = 1, 289 .exclude_user = exclude_user, 290 .exclude_kernel = exclude_kernel, 291 .config = config, 292 }; 293 bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable); 294 295 attr.sample_period = get_sample_period(pmc, pmc->counter); 296 297 if ((attr.config & HSW_IN_TX_CHECKPOINTED) && 298 (boot_cpu_has(X86_FEATURE_RTM) || boot_cpu_has(X86_FEATURE_HLE))) { 299 /* 300 * HSW_IN_TX_CHECKPOINTED is not supported with nonzero 301 * period. Just clear the sample period so at least 302 * allocating the counter doesn't fail. 303 */ 304 attr.sample_period = 0; 305 } 306 if (pebs) { 307 /* 308 * For most PEBS hardware events, the difference in the software 309 * precision levels of guest and host PEBS events will not affect 310 * the accuracy of the PEBS profiling result, because the "event IP" 311 * in the PEBS record is calibrated on the guest side. 312 */ 313 attr.precise_ip = pmc_get_pebs_precise_level(pmc); 314 } 315 316 event = perf_event_create_kernel_counter(&attr, -1, current, 317 kvm_perf_overflow, pmc); 318 if (IS_ERR(event)) { 319 pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n", 320 PTR_ERR(event), pmc->idx); 321 return PTR_ERR(event); 322 } 323 324 pmc->perf_event = event; 325 pmc_to_pmu(pmc)->event_count++; 326 pmc->is_paused = false; 327 pmc->intr = intr || pebs; 328 return 0; 329 } 330 331 static bool pmc_pause_counter(struct kvm_pmc *pmc) 332 { 333 u64 counter = pmc->counter; 334 u64 prev_counter; 335 336 /* update counter, reset event value to avoid redundant accumulation */ 337 if (pmc->perf_event && !pmc->is_paused) 338 counter += perf_event_pause(pmc->perf_event, true); 339 340 /* 341 * Snapshot the previous counter *after* accumulating state from perf. 342 * If overflow already happened, hardware (via perf) is responsible for 343 * generating a PMI. KVM just needs to detect overflow on emulated 344 * counter events that haven't yet been processed. 345 */ 346 prev_counter = counter & pmc_bitmask(pmc); 347 348 counter += pmc->emulated_counter; 349 pmc->counter = counter & pmc_bitmask(pmc); 350 351 pmc->emulated_counter = 0; 352 pmc->is_paused = true; 353 354 return pmc->counter < prev_counter; 355 } 356 357 static bool pmc_resume_counter(struct kvm_pmc *pmc) 358 { 359 if (!pmc->perf_event) 360 return false; 361 362 /* recalibrate sample period and check if it's accepted by perf core */ 363 if (is_sampling_event(pmc->perf_event) && 364 perf_event_period(pmc->perf_event, 365 get_sample_period(pmc, pmc->counter))) 366 return false; 367 368 if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) != 369 (!!pmc->perf_event->attr.precise_ip)) 370 return false; 371 372 /* reuse perf_event to serve as pmc_reprogram_counter() does*/ 373 perf_event_enable(pmc->perf_event); 374 pmc->is_paused = false; 375 376 return true; 377 } 378 379 static void pmc_release_perf_event(struct kvm_pmc *pmc) 380 { 381 if (pmc->perf_event) { 382 perf_event_release_kernel(pmc->perf_event); 383 pmc->perf_event = NULL; 384 pmc->current_config = 0; 385 pmc_to_pmu(pmc)->event_count--; 386 } 387 } 388 389 static void pmc_stop_counter(struct kvm_pmc *pmc) 390 { 391 if (pmc->perf_event) { 392 pmc->counter = pmc_read_counter(pmc); 393 pmc_release_perf_event(pmc); 394 } 395 } 396 397 static void pmc_update_sample_period(struct kvm_pmc *pmc) 398 { 399 if (!pmc->perf_event || pmc->is_paused || 400 !is_sampling_event(pmc->perf_event)) 401 return; 402 403 perf_event_period(pmc->perf_event, 404 get_sample_period(pmc, pmc->counter)); 405 } 406 407 void pmc_write_counter(struct kvm_pmc *pmc, u64 val) 408 { 409 if (kvm_vcpu_has_mediated_pmu(pmc->vcpu)) { 410 pmc->counter = val & pmc_bitmask(pmc); 411 return; 412 } 413 414 /* 415 * Drop any unconsumed accumulated counts, the WRMSR is a write, not a 416 * read-modify-write. Adjust the counter value so that its value is 417 * relative to the current count, as reading the current count from 418 * perf is faster than pausing and reprogramming the event in order to 419 * reset it to '0'. Note, this very sneakily offsets the accumulated 420 * emulated count too, by using pmc_read_counter()! 421 */ 422 pmc->emulated_counter = 0; 423 pmc->counter += val - pmc_read_counter(pmc); 424 pmc->counter &= pmc_bitmask(pmc); 425 pmc_update_sample_period(pmc); 426 } 427 EXPORT_SYMBOL_FOR_KVM_INTERNAL(pmc_write_counter); 428 429 static int filter_cmp(const void *pa, const void *pb, u64 mask) 430 { 431 u64 a = *(u64 *)pa & mask; 432 u64 b = *(u64 *)pb & mask; 433 434 return (a > b) - (a < b); 435 } 436 437 static int filter_sort_cmp(const void *pa, const void *pb) 438 { 439 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT | 440 KVM_PMU_MASKED_ENTRY_EXCLUDE)); 441 } 442 443 /* 444 * For the event filter, searching is done on the 'includes' list and 445 * 'excludes' list separately rather than on the 'events' list (which 446 * has both). As a result the exclude bit can be ignored. 447 */ 448 static int filter_event_cmp(const void *pa, const void *pb) 449 { 450 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT)); 451 } 452 453 static int find_filter_index(u64 *events, u64 nevents, u64 key) 454 { 455 u64 *fe = bsearch(&key, events, nevents, sizeof(events[0]), 456 filter_event_cmp); 457 458 if (!fe) 459 return -1; 460 461 return fe - events; 462 } 463 464 static bool is_filter_entry_match(u64 filter_event, u64 umask) 465 { 466 u64 mask = filter_event >> (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8); 467 u64 match = filter_event & KVM_PMU_MASKED_ENTRY_UMASK_MATCH; 468 469 BUILD_BUG_ON((KVM_PMU_ENCODE_MASKED_ENTRY(0, 0xff, 0, false) >> 470 (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8)) != 471 ARCH_PERFMON_EVENTSEL_UMASK); 472 473 return (umask & mask) == match; 474 } 475 476 static bool filter_contains_match(u64 *events, u64 nevents, u64 eventsel) 477 { 478 u64 event_select = eventsel & kvm_pmu_ops.EVENTSEL_EVENT; 479 u64 umask = eventsel & ARCH_PERFMON_EVENTSEL_UMASK; 480 int i, index; 481 482 index = find_filter_index(events, nevents, event_select); 483 if (index < 0) 484 return false; 485 486 /* 487 * Entries are sorted by the event select. Walk the list in both 488 * directions to process all entries with the targeted event select. 489 */ 490 for (i = index; i < nevents; i++) { 491 if (filter_event_cmp(&events[i], &event_select)) 492 break; 493 494 if (is_filter_entry_match(events[i], umask)) 495 return true; 496 } 497 498 for (i = index - 1; i >= 0; i--) { 499 if (filter_event_cmp(&events[i], &event_select)) 500 break; 501 502 if (is_filter_entry_match(events[i], umask)) 503 return true; 504 } 505 506 return false; 507 } 508 509 static bool is_gp_event_allowed(struct kvm_x86_pmu_event_filter *f, 510 u64 eventsel) 511 { 512 if (filter_contains_match(f->includes, f->nr_includes, eventsel) && 513 !filter_contains_match(f->excludes, f->nr_excludes, eventsel)) 514 return f->action == KVM_PMU_EVENT_ALLOW; 515 516 return f->action == KVM_PMU_EVENT_DENY; 517 } 518 519 static bool is_fixed_event_allowed(struct kvm_x86_pmu_event_filter *filter, 520 int idx) 521 { 522 int fixed_idx = idx - KVM_FIXED_PMC_BASE_IDX; 523 524 if (filter->action == KVM_PMU_EVENT_DENY && 525 test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) 526 return false; 527 if (filter->action == KVM_PMU_EVENT_ALLOW && 528 !test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) 529 return false; 530 531 return true; 532 } 533 534 static bool pmc_is_event_allowed(struct kvm_pmc *pmc) 535 { 536 struct kvm_x86_pmu_event_filter *filter; 537 struct kvm *kvm = pmc->vcpu->kvm; 538 539 filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu); 540 if (!filter) 541 return true; 542 543 if (pmc_is_gp(pmc)) 544 return is_gp_event_allowed(filter, pmc->eventsel); 545 546 return is_fixed_event_allowed(filter, pmc->idx); 547 } 548 549 static void kvm_mediated_pmu_refresh_event_filter(struct kvm_pmc *pmc) 550 { 551 bool allowed = pmc_is_locally_enabled(pmc) && pmc_is_event_allowed(pmc); 552 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 553 554 if (pmc_is_gp(pmc)) { 555 pmc->eventsel_hw &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 556 if (allowed) 557 pmc->eventsel_hw |= pmc->eventsel & 558 ARCH_PERFMON_EVENTSEL_ENABLE; 559 } else { 560 u64 mask = intel_fixed_bits_by_idx(pmc->idx - KVM_FIXED_PMC_BASE_IDX, 0xf); 561 562 pmu->fixed_ctr_ctrl_hw &= ~mask; 563 if (allowed) 564 pmu->fixed_ctr_ctrl_hw |= pmu->fixed_ctr_ctrl & mask; 565 } 566 } 567 568 static int reprogram_counter(struct kvm_pmc *pmc) 569 { 570 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 571 u64 eventsel = pmc->eventsel; 572 u64 new_config = eventsel; 573 bool emulate_overflow; 574 u8 fixed_ctr_ctrl; 575 576 if (kvm_vcpu_has_mediated_pmu(pmu_to_vcpu(pmu))) { 577 kvm_mediated_pmu_refresh_event_filter(pmc); 578 return 0; 579 } 580 581 emulate_overflow = pmc_pause_counter(pmc); 582 583 if (!pmc_is_globally_enabled(pmc) || !pmc_is_locally_enabled(pmc) || 584 !pmc_is_event_allowed(pmc)) 585 return 0; 586 587 if (emulate_overflow) 588 __kvm_perf_overflow(pmc, false); 589 590 if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) 591 printk_once("kvm pmu: pin control bit is ignored\n"); 592 593 if (pmc_is_fixed(pmc)) { 594 fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, 595 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 596 if (fixed_ctr_ctrl & INTEL_FIXED_0_KERNEL) 597 eventsel |= ARCH_PERFMON_EVENTSEL_OS; 598 if (fixed_ctr_ctrl & INTEL_FIXED_0_USER) 599 eventsel |= ARCH_PERFMON_EVENTSEL_USR; 600 if (fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI) 601 eventsel |= ARCH_PERFMON_EVENTSEL_INT; 602 new_config = (u64)fixed_ctr_ctrl; 603 } 604 605 if (pmc->current_config == new_config && pmc_resume_counter(pmc)) 606 return 0; 607 608 pmc_release_perf_event(pmc); 609 610 pmc->current_config = new_config; 611 612 return pmc_reprogram_counter(pmc, PERF_TYPE_RAW, 613 (eventsel & pmu->raw_event_mask), 614 !(eventsel & ARCH_PERFMON_EVENTSEL_USR), 615 !(eventsel & ARCH_PERFMON_EVENTSEL_OS), 616 eventsel & ARCH_PERFMON_EVENTSEL_INT); 617 } 618 619 static bool pmc_is_event_match(struct kvm_pmc *pmc, u64 eventsel) 620 { 621 /* 622 * Ignore checks for edge detect (all events currently emulated by KVM 623 * are always rising edges), pin control (unsupported by modern CPUs), 624 * and counter mask and its invert flag (KVM doesn't emulate multiple 625 * events in a single clock cycle). 626 * 627 * Note, the uppermost nibble of AMD's mask overlaps Intel's IN_TX (bit 628 * 32) and IN_TXCP (bit 33), as well as two reserved bits (bits 35:34). 629 * Checking the "in HLE/RTM transaction" flags is correct as the vCPU 630 * can't be in a transaction if KVM is emulating an instruction. 631 * 632 * Checking the reserved bits might be wrong if they are defined in the 633 * future, but so could ignoring them, so do the simple thing for now. 634 */ 635 return !((pmc->eventsel ^ eventsel) & AMD64_RAW_EVENT_MASK_NB); 636 } 637 638 void kvm_pmu_recalc_pmc_emulation(struct kvm_pmu *pmu, struct kvm_pmc *pmc) 639 { 640 bitmap_clear(pmu->pmc_counting_instructions, pmc->idx, 1); 641 bitmap_clear(pmu->pmc_counting_branches, pmc->idx, 1); 642 643 /* 644 * Do NOT consult the PMU event filters, as the filters must be checked 645 * at the time of emulation to ensure KVM uses fresh information, e.g. 646 * omitting a PMC from a bitmap could result in a missed event if the 647 * filter is changed to allow counting the event. 648 */ 649 if (!pmc_is_locally_enabled(pmc)) 650 return; 651 652 if (pmc_is_event_match(pmc, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED)) 653 bitmap_set(pmu->pmc_counting_instructions, pmc->idx, 1); 654 655 if (pmc_is_event_match(pmc, kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED)) 656 bitmap_set(pmu->pmc_counting_branches, pmc->idx, 1); 657 } 658 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_recalc_pmc_emulation); 659 660 void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) 661 { 662 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); 663 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 664 struct kvm_pmc *pmc; 665 int bit; 666 667 bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); 668 669 /* 670 * The reprogramming bitmap can be written asynchronously by something 671 * other than the task that holds vcpu->mutex, take care to clear only 672 * the bits that will actually be processed. 673 */ 674 BUILD_BUG_ON(sizeof(bitmap) != sizeof(atomic64_t)); 675 atomic64_andnot(*(s64 *)bitmap, &pmu->__reprogram_pmi); 676 677 kvm_for_each_pmc(pmu, pmc, bit, bitmap) { 678 /* 679 * If reprogramming fails, e.g. due to contention, re-set the 680 * reprogram bit, i.e. opportunistically try again on the next 681 * PMU refresh. Don't make a new request as doing so can stall 682 * the guest if reprogramming repeatedly fails. 683 */ 684 if (reprogram_counter(pmc)) 685 set_bit(pmc->idx, pmu->reprogram_pmi); 686 } 687 688 /* 689 * Release unused perf_events if the corresponding guest MSRs weren't 690 * accessed during the last vCPU time slice (need_cleanup is set when 691 * the vCPU is scheduled back in). 692 */ 693 if (unlikely(pmu->need_cleanup)) 694 kvm_pmu_cleanup(vcpu); 695 696 kvm_for_each_pmc(pmu, pmc, bit, bitmap) 697 kvm_pmu_recalc_pmc_emulation(pmu, pmc); 698 } 699 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_handle_event); 700 701 int kvm_pmu_check_rdpmc_early(struct kvm_vcpu *vcpu, unsigned int idx) 702 { 703 /* 704 * On Intel, VMX interception has priority over RDPMC exceptions that 705 * aren't already handled by the emulator, i.e. there are no additional 706 * check needed for Intel PMUs. 707 * 708 * On AMD, _all_ exceptions on RDPMC have priority over SVM intercepts, 709 * i.e. an invalid PMC results in a #GP, not #VMEXIT. 710 */ 711 if (!kvm_pmu_ops.check_rdpmc_early) 712 return 0; 713 714 return kvm_pmu_call(check_rdpmc_early)(vcpu, idx); 715 } 716 717 bool is_vmware_backdoor_pmc(u32 pmc_idx) 718 { 719 switch (pmc_idx) { 720 case VMWARE_BACKDOOR_PMC_HOST_TSC: 721 case VMWARE_BACKDOOR_PMC_REAL_TIME: 722 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 723 return true; 724 } 725 return false; 726 } 727 728 static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 729 { 730 u64 ctr_val; 731 732 switch (idx) { 733 case VMWARE_BACKDOOR_PMC_HOST_TSC: 734 ctr_val = rdtsc(); 735 break; 736 case VMWARE_BACKDOOR_PMC_REAL_TIME: 737 ctr_val = ktime_get_boottime_ns(); 738 break; 739 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 740 ctr_val = ktime_get_boottime_ns() + 741 vcpu->kvm->arch.kvmclock_offset; 742 break; 743 default: 744 return 1; 745 } 746 747 *data = ctr_val; 748 return 0; 749 } 750 751 int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 752 { 753 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 754 struct kvm_pmc *pmc; 755 u64 mask = ~0ull; 756 757 if (!pmu->version) 758 return 1; 759 760 if (is_vmware_backdoor_pmc(idx)) 761 return kvm_pmu_rdpmc_vmware(vcpu, idx, data); 762 763 pmc = kvm_pmu_call(rdpmc_ecx_to_pmc)(vcpu, idx, &mask); 764 if (!pmc) 765 return 1; 766 767 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCE) && 768 (kvm_x86_call(get_cpl)(vcpu) != 0) && 769 kvm_is_cr0_bit_set(vcpu, X86_CR0_PE)) 770 return 1; 771 772 *data = pmc_read_counter(pmc) & mask; 773 return 0; 774 } 775 776 static bool kvm_need_any_pmc_intercept(struct kvm_vcpu *vcpu) 777 { 778 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 779 780 if (!kvm_vcpu_has_mediated_pmu(vcpu)) 781 return true; 782 783 /* 784 * Note! Check *host* PMU capabilities, not KVM's PMU capabilities, as 785 * KVM's capabilities are constrained based on KVM support, i.e. KVM's 786 * capabilities themselves may be a subset of hardware capabilities. 787 */ 788 return pmu->nr_arch_gp_counters != kvm_host_pmu.num_counters_gp || 789 pmu->nr_arch_fixed_counters != kvm_host_pmu.num_counters_fixed; 790 } 791 792 bool kvm_need_perf_global_ctrl_intercept(struct kvm_vcpu *vcpu) 793 { 794 return kvm_need_any_pmc_intercept(vcpu) || 795 !kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); 796 } 797 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_perf_global_ctrl_intercept); 798 799 bool kvm_need_rdpmc_intercept(struct kvm_vcpu *vcpu) 800 { 801 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 802 803 /* 804 * VMware allows access to these Pseduo-PMCs even when read via RDPMC 805 * in Ring3 when CR4.PCE=0. 806 */ 807 if (enable_vmware_backdoor) 808 return true; 809 810 return kvm_need_any_pmc_intercept(vcpu) || 811 pmu->counter_bitmask[KVM_PMC_GP] != (BIT_ULL(kvm_host_pmu.bit_width_gp) - 1) || 812 pmu->counter_bitmask[KVM_PMC_FIXED] != (BIT_ULL(kvm_host_pmu.bit_width_fixed) - 1); 813 } 814 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_rdpmc_intercept); 815 816 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) 817 { 818 if (lapic_in_kernel(vcpu)) { 819 kvm_pmu_call(deliver_pmi)(vcpu); 820 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); 821 } 822 } 823 824 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 825 { 826 switch (msr) { 827 case MSR_CORE_PERF_GLOBAL_STATUS: 828 case MSR_CORE_PERF_GLOBAL_CTRL: 829 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 830 return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); 831 default: 832 break; 833 } 834 return kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr) || 835 kvm_pmu_call(is_valid_msr)(vcpu, msr); 836 } 837 838 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) 839 { 840 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 841 struct kvm_pmc *pmc = kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr); 842 843 if (pmc) 844 __set_bit(pmc->idx, pmu->pmc_in_use); 845 } 846 847 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 848 { 849 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 850 u32 msr = msr_info->index; 851 852 switch (msr) { 853 case MSR_CORE_PERF_GLOBAL_STATUS: 854 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: 855 msr_info->data = pmu->global_status; 856 break; 857 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: 858 case MSR_CORE_PERF_GLOBAL_CTRL: 859 msr_info->data = pmu->global_ctrl; 860 break; 861 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: 862 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: 863 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 864 msr_info->data = 0; 865 break; 866 default: 867 return kvm_pmu_call(get_msr)(vcpu, msr_info); 868 } 869 870 return 0; 871 } 872 873 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 874 { 875 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 876 u32 msr = msr_info->index; 877 u64 data = msr_info->data; 878 u64 diff; 879 880 /* 881 * Note, AMD ignores writes to reserved bits and read-only PMU MSRs, 882 * whereas Intel generates #GP on attempts to write reserved/RO MSRs. 883 */ 884 switch (msr) { 885 case MSR_CORE_PERF_GLOBAL_STATUS: 886 if (!msr_info->host_initiated) 887 return 1; /* RO MSR */ 888 fallthrough; 889 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: 890 /* Per PPR, Read-only MSR. Writes are ignored. */ 891 if (!msr_info->host_initiated) 892 break; 893 894 if (data & pmu->global_status_rsvd) 895 return 1; 896 897 pmu->global_status = data; 898 break; 899 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: 900 data &= ~pmu->global_ctrl_rsvd; 901 fallthrough; 902 case MSR_CORE_PERF_GLOBAL_CTRL: 903 if (!kvm_valid_perf_global_ctrl(pmu, data)) 904 return 1; 905 906 if (pmu->global_ctrl != data) { 907 diff = pmu->global_ctrl ^ data; 908 pmu->global_ctrl = data; 909 kvm_pmu_request_counters_reprogram(pmu, diff); 910 } 911 /* 912 * Unconditionally forward writes to vendor code, i.e. to the 913 * VMC{B,S}, as pmu->global_ctrl is per-VCPU, not per-VMC{B,S}. 914 */ 915 if (kvm_vcpu_has_mediated_pmu(vcpu)) 916 kvm_pmu_call(write_global_ctrl)(data); 917 break; 918 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 919 /* 920 * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in 921 * GLOBAL_STATUS, and so the set of reserved bits is the same. 922 */ 923 if (data & pmu->global_status_rsvd) 924 return 1; 925 fallthrough; 926 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: 927 if (!msr_info->host_initiated) 928 pmu->global_status &= ~data; 929 break; 930 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: 931 if (!msr_info->host_initiated) 932 pmu->global_status |= data & ~pmu->global_status_rsvd; 933 break; 934 default: 935 kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); 936 return kvm_pmu_call(set_msr)(vcpu, msr_info); 937 } 938 939 return 0; 940 } 941 942 static void kvm_pmu_reset(struct kvm_vcpu *vcpu) 943 { 944 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 945 struct kvm_pmc *pmc; 946 int i; 947 948 pmu->need_cleanup = false; 949 950 bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX); 951 bitmap_zero(pmu->pmc_has_mode_specific_enables, X86_PMC_IDX_MAX); 952 953 kvm_for_each_pmc(pmu, pmc, i, pmu->all_valid_pmc_idx) { 954 pmc_stop_counter(pmc); 955 pmc->counter = 0; 956 pmc->emulated_counter = 0; 957 958 if (pmc_is_gp(pmc)) { 959 pmc->eventsel = 0; 960 pmc->eventsel_hw = 0; 961 } 962 } 963 964 pmu->fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw = 0; 965 pmu->global_ctrl = pmu->global_status = 0; 966 967 kvm_pmu_call(reset)(vcpu); 968 } 969 970 /* 971 * Refresh the PMU configuration for the vCPU, e.g. if userspace changes CPUID 972 * and/or PERF_CAPABILITIES. 973 */ 974 void kvm_pmu_refresh(struct kvm_vcpu *vcpu) 975 { 976 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 977 978 if (KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm)) 979 return; 980 981 /* 982 * Stop/release all existing counters/events before realizing the new 983 * vPMU model. 984 */ 985 kvm_pmu_reset(vcpu); 986 987 pmu->version = 0; 988 pmu->nr_arch_gp_counters = 0; 989 pmu->nr_arch_fixed_counters = 0; 990 pmu->counter_bitmask[KVM_PMC_GP] = 0; 991 pmu->counter_bitmask[KVM_PMC_FIXED] = 0; 992 pmu->reserved_bits = 0xffffffff00200000ull; 993 pmu->raw_event_mask = X86_RAW_EVENT_MASK; 994 pmu->global_ctrl_rsvd = ~0ull; 995 pmu->global_status_rsvd = ~0ull; 996 pmu->fixed_ctr_ctrl_rsvd = ~0ull; 997 pmu->pebs_enable_rsvd = ~0ull; 998 pmu->pebs_data_cfg_rsvd = ~0ull; 999 bitmap_zero(pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX); 1000 1001 if (!vcpu->kvm->arch.enable_pmu) 1002 return; 1003 1004 kvm_pmu_call(refresh)(vcpu); 1005 1006 /* 1007 * At RESET, both Intel and AMD CPUs set all enable bits for general 1008 * purpose counters in IA32_PERF_GLOBAL_CTRL (so that software that 1009 * was written for v1 PMUs don't unknowingly leave GP counters disabled 1010 * in the global controls). Emulate that behavior when refreshing the 1011 * PMU so that userspace doesn't need to manually set PERF_GLOBAL_CTRL. 1012 */ 1013 if (pmu->nr_arch_gp_counters && 1014 (kvm_pmu_has_perf_global_ctrl(pmu) || kvm_vcpu_has_mediated_pmu(vcpu))) 1015 pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0); 1016 1017 if (kvm_vcpu_has_mediated_pmu(vcpu)) 1018 kvm_pmu_call(write_global_ctrl)(pmu->global_ctrl); 1019 1020 bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters); 1021 bitmap_set(pmu->all_valid_pmc_idx, KVM_FIXED_PMC_BASE_IDX, 1022 pmu->nr_arch_fixed_counters); 1023 } 1024 1025 void kvm_pmu_init(struct kvm_vcpu *vcpu) 1026 { 1027 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1028 1029 memset(pmu, 0, sizeof(*pmu)); 1030 kvm_pmu_call(init)(vcpu); 1031 } 1032 1033 /* Release perf_events for vPMCs that have been unused for a full time slice. */ 1034 void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) 1035 { 1036 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1037 struct kvm_pmc *pmc = NULL; 1038 DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX); 1039 int i; 1040 1041 pmu->need_cleanup = false; 1042 1043 bitmap_andnot(bitmask, pmu->all_valid_pmc_idx, 1044 pmu->pmc_in_use, X86_PMC_IDX_MAX); 1045 1046 kvm_for_each_pmc(pmu, pmc, i, bitmask) { 1047 if (pmc->perf_event && !pmc_is_locally_enabled(pmc)) 1048 pmc_stop_counter(pmc); 1049 } 1050 1051 kvm_pmu_call(cleanup)(vcpu); 1052 1053 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); 1054 } 1055 1056 void kvm_pmu_destroy(struct kvm_vcpu *vcpu) 1057 { 1058 kvm_pmu_reset(vcpu); 1059 } 1060 1061 static bool pmc_is_pmi_enabled(struct kvm_pmc *pmc) 1062 { 1063 u8 fixed_ctr_ctrl; 1064 1065 if (pmc_is_gp(pmc)) 1066 return pmc->eventsel & ARCH_PERFMON_EVENTSEL_INT; 1067 1068 fixed_ctr_ctrl = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl, 1069 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 1070 return fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI; 1071 } 1072 1073 static void kvm_pmu_incr_counter(struct kvm_pmc *pmc) 1074 { 1075 struct kvm_vcpu *vcpu = pmc->vcpu; 1076 1077 /* 1078 * For perf-based PMUs, accumulate software-emulated events separately 1079 * from pmc->counter, as pmc->counter is offset by the count of the 1080 * associated perf event. Request reprogramming, which will consult 1081 * both emulated and hardware-generated events to detect overflow. 1082 */ 1083 if (!kvm_vcpu_has_mediated_pmu(vcpu)) { 1084 pmc->emulated_counter++; 1085 kvm_pmu_request_counter_reprogram(pmc); 1086 return; 1087 } 1088 1089 /* 1090 * For mediated PMUs, pmc->counter is updated when the vCPU's PMU is 1091 * put, and will be loaded into hardware when the PMU is loaded. Simply 1092 * increment the counter and signal overflow if it wraps to zero. 1093 */ 1094 pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc); 1095 if (!pmc->counter) { 1096 pmc_to_pmu(pmc)->global_status |= BIT_ULL(pmc->idx); 1097 if (pmc_is_pmi_enabled(pmc)) 1098 kvm_make_request(KVM_REQ_PMI, vcpu); 1099 } 1100 } 1101 1102 static inline bool cpl_is_matched(struct kvm_pmc *pmc) 1103 { 1104 bool select_os, select_user; 1105 u64 config; 1106 1107 if (pmc_is_gp(pmc)) { 1108 config = pmc->eventsel; 1109 select_os = config & ARCH_PERFMON_EVENTSEL_OS; 1110 select_user = config & ARCH_PERFMON_EVENTSEL_USR; 1111 } else { 1112 config = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl, 1113 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 1114 select_os = config & INTEL_FIXED_0_KERNEL; 1115 select_user = config & INTEL_FIXED_0_USER; 1116 } 1117 1118 /* 1119 * Skip the CPL lookup, which isn't free on Intel, if the result will 1120 * be the same regardless of the CPL. 1121 */ 1122 if (select_os == select_user) 1123 return select_os; 1124 1125 return (kvm_x86_call(get_cpl)(pmc->vcpu) == 0) ? select_os : 1126 select_user; 1127 } 1128 1129 static void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, 1130 const unsigned long *event_pmcs) 1131 { 1132 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); 1133 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1134 struct kvm_pmc *pmc; 1135 int i, idx; 1136 1137 BUILD_BUG_ON(sizeof(pmu->global_ctrl) * BITS_PER_BYTE != X86_PMC_IDX_MAX); 1138 1139 if (bitmap_empty(event_pmcs, X86_PMC_IDX_MAX)) 1140 return; 1141 1142 if (!kvm_pmu_has_perf_global_ctrl(pmu)) 1143 bitmap_copy(bitmap, event_pmcs, X86_PMC_IDX_MAX); 1144 else if (!bitmap_and(bitmap, event_pmcs, 1145 (unsigned long *)&pmu->global_ctrl, X86_PMC_IDX_MAX)) 1146 return; 1147 1148 idx = srcu_read_lock(&vcpu->kvm->srcu); 1149 kvm_for_each_pmc(pmu, pmc, i, bitmap) { 1150 if (!pmc_is_event_allowed(pmc) || !cpl_is_matched(pmc)) 1151 continue; 1152 1153 kvm_pmu_incr_counter(pmc); 1154 } 1155 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1156 } 1157 1158 void kvm_pmu_instruction_retired(struct kvm_vcpu *vcpu) 1159 { 1160 kvm_pmu_trigger_event(vcpu, vcpu_to_pmu(vcpu)->pmc_counting_instructions); 1161 } 1162 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_instruction_retired); 1163 1164 void kvm_pmu_branch_retired(struct kvm_vcpu *vcpu) 1165 { 1166 kvm_pmu_trigger_event(vcpu, vcpu_to_pmu(vcpu)->pmc_counting_branches); 1167 } 1168 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_branch_retired); 1169 1170 static bool is_masked_filter_valid(const struct kvm_x86_pmu_event_filter *filter) 1171 { 1172 u64 mask = kvm_pmu_ops.EVENTSEL_EVENT | 1173 KVM_PMU_MASKED_ENTRY_UMASK_MASK | 1174 KVM_PMU_MASKED_ENTRY_UMASK_MATCH | 1175 KVM_PMU_MASKED_ENTRY_EXCLUDE; 1176 int i; 1177 1178 for (i = 0; i < filter->nevents; i++) { 1179 if (filter->events[i] & ~mask) 1180 return false; 1181 } 1182 1183 return true; 1184 } 1185 1186 static void convert_to_masked_filter(struct kvm_x86_pmu_event_filter *filter) 1187 { 1188 int i, j; 1189 1190 for (i = 0, j = 0; i < filter->nevents; i++) { 1191 /* 1192 * Skip events that are impossible to match against a guest 1193 * event. When filtering, only the event select + unit mask 1194 * of the guest event is used. To maintain backwards 1195 * compatibility, impossible filters can't be rejected :-( 1196 */ 1197 if (filter->events[i] & ~(kvm_pmu_ops.EVENTSEL_EVENT | 1198 ARCH_PERFMON_EVENTSEL_UMASK)) 1199 continue; 1200 /* 1201 * Convert userspace events to a common in-kernel event so 1202 * only one code path is needed to support both events. For 1203 * the in-kernel events use masked events because they are 1204 * flexible enough to handle both cases. To convert to masked 1205 * events all that's needed is to add an "all ones" umask_mask, 1206 * (unmasked filter events don't support EXCLUDE). 1207 */ 1208 filter->events[j++] = filter->events[i] | 1209 (0xFFULL << KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT); 1210 } 1211 1212 filter->nevents = j; 1213 } 1214 1215 static int prepare_filter_lists(struct kvm_x86_pmu_event_filter *filter) 1216 { 1217 int i; 1218 1219 if (!(filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS)) 1220 convert_to_masked_filter(filter); 1221 else if (!is_masked_filter_valid(filter)) 1222 return -EINVAL; 1223 1224 /* 1225 * Sort entries by event select and includes vs. excludes so that all 1226 * entries for a given event select can be processed efficiently during 1227 * filtering. The EXCLUDE flag uses a more significant bit than the 1228 * event select, and so the sorted list is also effectively split into 1229 * includes and excludes sub-lists. 1230 */ 1231 sort(&filter->events, filter->nevents, sizeof(filter->events[0]), 1232 filter_sort_cmp, NULL); 1233 1234 i = filter->nevents; 1235 /* Find the first EXCLUDE event (only supported for masked events). */ 1236 if (filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS) { 1237 for (i = 0; i < filter->nevents; i++) { 1238 if (filter->events[i] & KVM_PMU_MASKED_ENTRY_EXCLUDE) 1239 break; 1240 } 1241 } 1242 1243 filter->nr_includes = i; 1244 filter->nr_excludes = filter->nevents - filter->nr_includes; 1245 filter->includes = filter->events; 1246 filter->excludes = filter->events + filter->nr_includes; 1247 1248 return 0; 1249 } 1250 1251 int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp) 1252 { 1253 struct kvm_pmu_event_filter __user *user_filter = argp; 1254 struct kvm_x86_pmu_event_filter *filter; 1255 struct kvm_pmu_event_filter tmp; 1256 struct kvm_vcpu *vcpu; 1257 unsigned long i; 1258 size_t size; 1259 int r; 1260 1261 if (copy_from_user(&tmp, user_filter, sizeof(tmp))) 1262 return -EFAULT; 1263 1264 if (tmp.action != KVM_PMU_EVENT_ALLOW && 1265 tmp.action != KVM_PMU_EVENT_DENY) 1266 return -EINVAL; 1267 1268 if (tmp.flags & ~KVM_PMU_EVENT_FLAGS_VALID_MASK) 1269 return -EINVAL; 1270 1271 if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS) 1272 return -E2BIG; 1273 1274 size = struct_size(filter, events, tmp.nevents); 1275 filter = kzalloc(size, GFP_KERNEL_ACCOUNT); 1276 if (!filter) 1277 return -ENOMEM; 1278 1279 filter->action = tmp.action; 1280 filter->nevents = tmp.nevents; 1281 filter->fixed_counter_bitmap = tmp.fixed_counter_bitmap; 1282 filter->flags = tmp.flags; 1283 1284 r = -EFAULT; 1285 if (copy_from_user(filter->events, user_filter->events, 1286 flex_array_size(filter, events, filter->nevents))) 1287 goto cleanup; 1288 1289 r = prepare_filter_lists(filter); 1290 if (r) 1291 goto cleanup; 1292 1293 mutex_lock(&kvm->lock); 1294 filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter, 1295 mutex_is_locked(&kvm->lock)); 1296 mutex_unlock(&kvm->lock); 1297 synchronize_srcu_expedited(&kvm->srcu); 1298 1299 BUILD_BUG_ON(sizeof(((struct kvm_pmu *)0)->reprogram_pmi) > 1300 sizeof(((struct kvm_pmu *)0)->__reprogram_pmi)); 1301 1302 kvm_for_each_vcpu(i, vcpu, kvm) 1303 atomic64_set(&vcpu_to_pmu(vcpu)->__reprogram_pmi, -1ull); 1304 1305 kvm_make_all_cpus_request(kvm, KVM_REQ_PMU); 1306 1307 r = 0; 1308 cleanup: 1309 kfree(filter); 1310 return r; 1311 } 1312 1313 static __always_inline u32 fixed_counter_msr(u32 idx) 1314 { 1315 return kvm_pmu_ops.FIXED_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1316 } 1317 1318 static __always_inline u32 gp_counter_msr(u32 idx) 1319 { 1320 return kvm_pmu_ops.GP_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1321 } 1322 1323 static __always_inline u32 gp_eventsel_msr(u32 idx) 1324 { 1325 return kvm_pmu_ops.GP_EVENTSEL_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1326 } 1327 1328 static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu) 1329 { 1330 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1331 struct kvm_pmc *pmc; 1332 u32 i; 1333 1334 /* 1335 * No need to zero out unexposed GP/fixed counters/selectors since RDPMC 1336 * is intercepted if hardware has counters that aren't visible to the 1337 * guest (KVM will inject #GP as appropriate). 1338 */ 1339 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 1340 pmc = &pmu->gp_counters[i]; 1341 1342 if (pmc->counter != rdpmc(i)) 1343 wrmsrq(gp_counter_msr(i), pmc->counter); 1344 wrmsrq(gp_eventsel_msr(i), pmc->eventsel_hw); 1345 } 1346 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 1347 pmc = &pmu->fixed_counters[i]; 1348 1349 if (pmc->counter != rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i)) 1350 wrmsrq(fixed_counter_msr(i), pmc->counter); 1351 } 1352 } 1353 1354 void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu) 1355 { 1356 if (!kvm_vcpu_has_mediated_pmu(vcpu) || 1357 KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 1358 return; 1359 1360 lockdep_assert_irqs_disabled(); 1361 1362 perf_load_guest_context(); 1363 1364 /* 1365 * Explicitly clear PERF_GLOBAL_CTRL, as "loading" the guest's context 1366 * disables all individual counters (if any were enabled), but doesn't 1367 * globally disable the entire PMU. Loading event selectors and PMCs 1368 * with guest values while PERF_GLOBAL_CTRL is non-zero will generate 1369 * unexpected events and PMIs. 1370 * 1371 * VMX will enable/disable counters at VM-Enter/VM-Exit by atomically 1372 * loading PERF_GLOBAL_CONTROL. SVM effectively performs the switch by 1373 * configuring all events to be GUEST_ONLY. Clear PERF_GLOBAL_CONTROL 1374 * even for SVM to minimize the damage if a perf event is left enabled, 1375 * and to ensure a consistent starting state. 1376 */ 1377 wrmsrq(kvm_pmu_ops.PERF_GLOBAL_CTRL, 0); 1378 1379 perf_load_guest_lvtpc(kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVTPC)); 1380 1381 kvm_pmu_load_guest_pmcs(vcpu); 1382 1383 kvm_pmu_call(mediated_load)(vcpu); 1384 } 1385 1386 static void kvm_pmu_put_guest_pmcs(struct kvm_vcpu *vcpu) 1387 { 1388 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1389 struct kvm_pmc *pmc; 1390 u32 i; 1391 1392 /* 1393 * Clear selectors and counters to ensure hardware doesn't count using 1394 * guest controls when the host (perf) restores its state. 1395 */ 1396 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 1397 pmc = &pmu->gp_counters[i]; 1398 1399 pmc->counter = rdpmc(i); 1400 if (pmc->counter) 1401 wrmsrq(gp_counter_msr(i), 0); 1402 if (pmc->eventsel_hw) 1403 wrmsrq(gp_eventsel_msr(i), 0); 1404 } 1405 1406 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 1407 pmc = &pmu->fixed_counters[i]; 1408 1409 pmc->counter = rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i); 1410 if (pmc->counter) 1411 wrmsrq(fixed_counter_msr(i), 0); 1412 } 1413 } 1414 1415 void kvm_mediated_pmu_put(struct kvm_vcpu *vcpu) 1416 { 1417 if (!kvm_vcpu_has_mediated_pmu(vcpu) || 1418 KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 1419 return; 1420 1421 lockdep_assert_irqs_disabled(); 1422 1423 /* 1424 * Defer handling of PERF_GLOBAL_CTRL to vendor code. On Intel, it's 1425 * atomically cleared on VM-Exit, i.e. doesn't need to be clear here. 1426 */ 1427 kvm_pmu_call(mediated_put)(vcpu); 1428 1429 kvm_pmu_put_guest_pmcs(vcpu); 1430 1431 perf_put_guest_lvtpc(); 1432 1433 perf_put_guest_context(); 1434 } 1435