1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine -- Performance Monitoring Unit support 4 * 5 * Copyright 2015 Red Hat, Inc. and/or its affiliates. 6 * 7 * Authors: 8 * Avi Kivity <avi@redhat.com> 9 * Gleb Natapov <gleb@redhat.com> 10 * Wei Huang <wei@redhat.com> 11 */ 12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 13 14 #include <linux/types.h> 15 #include <linux/kvm_host.h> 16 #include <linux/perf_event.h> 17 #include <linux/bsearch.h> 18 #include <linux/sort.h> 19 #include <linux/moduleparam.h> 20 #include <asm/perf_event.h> 21 #include <asm/cpu_device_id.h> 22 #include "x86.h" 23 #include "cpuid.h" 24 #include "lapic.h" 25 #include "pmu.h" 26 27 /* This is enough to filter the vast majority of currently defined events. */ 28 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300 29 30 /* Unadultered PMU capabilities of the host, i.e. of hardware. */ 31 static struct x86_pmu_capability __read_mostly kvm_host_pmu; 32 33 /* KVM's PMU capabilities, i.e. the intersection of KVM and hardware support. */ 34 struct x86_pmu_capability __read_mostly kvm_pmu_cap; 35 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_cap); 36 37 /* Enable/disable PMU virtualization */ 38 bool __read_mostly enable_pmu = true; 39 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_pmu); 40 module_param(enable_pmu, bool, 0444); 41 42 /* Enable/disabled mediated PMU virtualization. */ 43 bool __read_mostly enable_mediated_pmu; 44 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_mediated_pmu); 45 46 struct kvm_pmu_emulated_event_selectors { 47 u64 INSTRUCTIONS_RETIRED; 48 u64 BRANCH_INSTRUCTIONS_RETIRED; 49 }; 50 static struct kvm_pmu_emulated_event_selectors __read_mostly kvm_pmu_eventsel; 51 52 /* Precise Distribution of Instructions Retired (PDIR) */ 53 static const struct x86_cpu_id vmx_pebs_pdir_cpu[] = { 54 X86_MATCH_VFM(INTEL_ICELAKE_D, NULL), 55 X86_MATCH_VFM(INTEL_ICELAKE_X, NULL), 56 /* Instruction-Accurate PDIR (PDIR++) */ 57 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL), 58 {} 59 }; 60 61 /* Precise Distribution (PDist) */ 62 static const struct x86_cpu_id vmx_pebs_pdist_cpu[] = { 63 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL), 64 {} 65 }; 66 67 /* NOTE: 68 * - Each perf counter is defined as "struct kvm_pmc"; 69 * - There are two types of perf counters: general purpose (gp) and fixed. 70 * gp counters are stored in gp_counters[] and fixed counters are stored 71 * in fixed_counters[] respectively. Both of them are part of "struct 72 * kvm_pmu"; 73 * - pmu.c understands the difference between gp counters and fixed counters. 74 * However AMD doesn't support fixed-counters; 75 * - There are three types of index to access perf counters (PMC): 76 * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD 77 * has MSR_K7_PERFCTRn and, for families 15H and later, 78 * MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are 79 * aliased to MSR_K7_PERFCTRn. 80 * 2. MSR Index (named idx): This normally is used by RDPMC instruction. 81 * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access 82 * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except 83 * that it also supports fixed counters. idx can be used to as index to 84 * gp and fixed counters. 85 * 3. Global PMC Index (named pmc): pmc is an index specific to PMU 86 * code. Each pmc, stored in kvm_pmc.idx field, is unique across 87 * all perf counters (both gp and fixed). The mapping relationship 88 * between pmc and perf counters is as the following: 89 * * Intel: [0 .. KVM_MAX_NR_INTEL_GP_COUNTERS-1] <=> gp counters 90 * [KVM_FIXED_PMC_BASE_IDX .. KVM_FIXED_PMC_BASE_IDX + 2] <=> fixed 91 * * AMD: [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H 92 * and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters 93 */ 94 95 static struct kvm_pmu_ops kvm_pmu_ops __read_mostly; 96 97 #define KVM_X86_PMU_OP(func) \ 98 DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ 99 *(((struct kvm_pmu_ops *)0)->func)); 100 #define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP 101 #define KVM_X86_PMU_OP_OPTIONAL_RET0 KVM_X86_PMU_OP 102 #include <asm/kvm-x86-pmu-ops.h> 103 EXPORT_STATIC_CALL_GPL(kvm_x86_pmu_pmc_is_disabled_in_current_mode); 104 105 void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) 106 { 107 memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops)); 108 109 #define __KVM_X86_PMU_OP(func) \ 110 static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func); 111 #define KVM_X86_PMU_OP(func) \ 112 WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func) 113 #define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP 114 #define KVM_X86_PMU_OP_OPTIONAL_RET0(func) \ 115 static_call_update(kvm_x86_pmu_##func, (void *)kvm_pmu_ops.func ? : \ 116 (void *)__static_call_return0); 117 #include <asm/kvm-x86-pmu-ops.h> 118 #undef __KVM_X86_PMU_OP 119 } 120 121 void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops) 122 { 123 bool is_intel = boot_cpu_data.x86_vendor == X86_VENDOR_INTEL; 124 int min_nr_gp_ctrs = pmu_ops->MIN_NR_GP_COUNTERS; 125 126 /* 127 * Hybrid PMUs don't play nice with virtualization without careful 128 * configuration by userspace, and KVM's APIs for reporting supported 129 * vPMU features do not account for hybrid PMUs. Disable vPMU support 130 * for hybrid PMUs until KVM gains a way to let userspace opt-in. 131 */ 132 if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) { 133 enable_pmu = false; 134 memset(&kvm_host_pmu, 0, sizeof(kvm_host_pmu)); 135 } else { 136 perf_get_x86_pmu_capability(&kvm_host_pmu); 137 } 138 139 if (enable_pmu) { 140 /* 141 * WARN if perf did NOT disable hardware PMU if the number of 142 * architecturally required GP counters aren't present, i.e. if 143 * there are a non-zero number of counters, but fewer than what 144 * is architecturally required. 145 */ 146 if (!kvm_host_pmu.num_counters_gp || 147 WARN_ON_ONCE(kvm_host_pmu.num_counters_gp < min_nr_gp_ctrs)) 148 enable_pmu = false; 149 else if (is_intel && !kvm_host_pmu.version) 150 enable_pmu = false; 151 } 152 153 if (!enable_pmu || !enable_mediated_pmu || !kvm_host_pmu.mediated || 154 !pmu_ops->is_mediated_pmu_supported(&kvm_host_pmu)) 155 enable_mediated_pmu = false; 156 157 if (!enable_mediated_pmu) 158 pmu_ops->write_global_ctrl = NULL; 159 160 if (!enable_pmu) { 161 memset(&kvm_pmu_cap, 0, sizeof(kvm_pmu_cap)); 162 return; 163 } 164 165 memcpy(&kvm_pmu_cap, &kvm_host_pmu, sizeof(kvm_host_pmu)); 166 kvm_pmu_cap.version = min(kvm_pmu_cap.version, 2); 167 kvm_pmu_cap.num_counters_gp = min(kvm_pmu_cap.num_counters_gp, 168 pmu_ops->MAX_NR_GP_COUNTERS); 169 kvm_pmu_cap.num_counters_fixed = min(kvm_pmu_cap.num_counters_fixed, 170 KVM_MAX_NR_FIXED_COUNTERS); 171 172 kvm_pmu_eventsel.INSTRUCTIONS_RETIRED = 173 perf_get_hw_event_config(PERF_COUNT_HW_INSTRUCTIONS); 174 kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED = 175 perf_get_hw_event_config(PERF_COUNT_HW_BRANCH_INSTRUCTIONS); 176 } 177 178 void kvm_handle_guest_mediated_pmi(void) 179 { 180 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 181 182 if (WARN_ON_ONCE(!vcpu || !kvm_vcpu_has_mediated_pmu(vcpu))) 183 return; 184 185 kvm_make_request(KVM_REQ_PMI, vcpu); 186 } 187 188 static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi) 189 { 190 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 191 bool skip_pmi = false; 192 193 if (pmc->perf_event && pmc->perf_event->attr.precise_ip) { 194 if (!in_pmi) { 195 /* 196 * TODO: KVM is currently _choosing_ to not generate records 197 * for emulated instructions, avoiding BUFFER_OVF PMI when 198 * there are no records. Strictly speaking, it should be done 199 * as well in the right context to improve sampling accuracy. 200 */ 201 skip_pmi = true; 202 } else { 203 /* Indicate PEBS overflow PMI to guest. */ 204 skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, 205 (unsigned long *)&pmu->global_status); 206 } 207 } else { 208 __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); 209 } 210 211 if (pmc->intr && !skip_pmi) 212 kvm_make_request(KVM_REQ_PMI, pmc->vcpu); 213 } 214 215 static void kvm_perf_overflow(struct perf_event *perf_event, 216 struct perf_sample_data *data, 217 struct pt_regs *regs) 218 { 219 struct kvm_pmc *pmc = perf_event->overflow_handler_context; 220 221 /* 222 * Ignore asynchronous overflow events for counters that are scheduled 223 * to be reprogrammed, e.g. if a PMI for the previous event races with 224 * KVM's handling of a related guest WRMSR. 225 */ 226 if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi)) 227 return; 228 229 __kvm_perf_overflow(pmc, true); 230 231 kvm_make_request(KVM_REQ_PMU, pmc->vcpu); 232 } 233 234 static u64 pmc_get_pebs_precise_level(struct kvm_pmc *pmc) 235 { 236 /* 237 * For some model specific pebs counters with special capabilities 238 * (PDIR, PDIR++, PDIST), KVM needs to raise the event precise 239 * level to the maximum value (currently 3, backwards compatible) 240 * so that the perf subsystem would assign specific hardware counter 241 * with that capability for vPMC. 242 */ 243 if ((pmc->idx == 0 && x86_match_cpu(vmx_pebs_pdist_cpu)) || 244 (pmc->idx == 32 && x86_match_cpu(vmx_pebs_pdir_cpu))) 245 return 3; 246 247 /* 248 * The non-zero precision level of guest event makes the ordinary 249 * guest event becomes a guest PEBS event and triggers the host 250 * PEBS PMI handler to determine whether the PEBS overflow PMI 251 * comes from the host counters or the guest. 252 */ 253 return 1; 254 } 255 256 static u64 get_sample_period(struct kvm_pmc *pmc, u64 counter_value) 257 { 258 u64 sample_period = (-counter_value) & pmc_bitmask(pmc); 259 260 if (!sample_period) 261 sample_period = pmc_bitmask(pmc) + 1; 262 return sample_period; 263 } 264 265 static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config, 266 bool exclude_user, bool exclude_kernel, 267 bool intr) 268 { 269 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 270 struct perf_event *event; 271 struct perf_event_attr attr = { 272 .type = type, 273 .size = sizeof(attr), 274 .pinned = true, 275 .exclude_idle = true, 276 .exclude_host = 1, 277 .exclude_user = exclude_user, 278 .exclude_kernel = exclude_kernel, 279 .config = config, 280 }; 281 bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable); 282 283 attr.sample_period = get_sample_period(pmc, pmc->counter); 284 285 if ((attr.config & HSW_IN_TX_CHECKPOINTED) && 286 (boot_cpu_has(X86_FEATURE_RTM) || boot_cpu_has(X86_FEATURE_HLE))) { 287 /* 288 * HSW_IN_TX_CHECKPOINTED is not supported with nonzero 289 * period. Just clear the sample period so at least 290 * allocating the counter doesn't fail. 291 */ 292 attr.sample_period = 0; 293 } 294 if (pebs) { 295 /* 296 * For most PEBS hardware events, the difference in the software 297 * precision levels of guest and host PEBS events will not affect 298 * the accuracy of the PEBS profiling result, because the "event IP" 299 * in the PEBS record is calibrated on the guest side. 300 */ 301 attr.precise_ip = pmc_get_pebs_precise_level(pmc); 302 } 303 304 event = perf_event_create_kernel_counter(&attr, -1, current, 305 kvm_perf_overflow, pmc); 306 if (IS_ERR(event)) { 307 pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n", 308 PTR_ERR(event), pmc->idx); 309 return PTR_ERR(event); 310 } 311 312 pmc->perf_event = event; 313 pmc_to_pmu(pmc)->event_count++; 314 pmc->is_paused = false; 315 pmc->intr = intr || pebs; 316 return 0; 317 } 318 319 static bool pmc_pause_counter(struct kvm_pmc *pmc) 320 { 321 u64 counter = pmc->counter; 322 u64 prev_counter; 323 324 /* update counter, reset event value to avoid redundant accumulation */ 325 if (pmc->perf_event && !pmc->is_paused) 326 counter += perf_event_pause(pmc->perf_event, true); 327 328 /* 329 * Snapshot the previous counter *after* accumulating state from perf. 330 * If overflow already happened, hardware (via perf) is responsible for 331 * generating a PMI. KVM just needs to detect overflow on emulated 332 * counter events that haven't yet been processed. 333 */ 334 prev_counter = counter & pmc_bitmask(pmc); 335 336 counter += pmc->emulated_counter; 337 pmc->counter = counter & pmc_bitmask(pmc); 338 339 pmc->emulated_counter = 0; 340 pmc->is_paused = true; 341 342 return pmc->counter < prev_counter; 343 } 344 345 static bool pmc_resume_counter(struct kvm_pmc *pmc) 346 { 347 if (!pmc->perf_event) 348 return false; 349 350 /* recalibrate sample period and check if it's accepted by perf core */ 351 if (is_sampling_event(pmc->perf_event) && 352 perf_event_period(pmc->perf_event, 353 get_sample_period(pmc, pmc->counter))) 354 return false; 355 356 if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) != 357 (!!pmc->perf_event->attr.precise_ip)) 358 return false; 359 360 /* reuse perf_event to serve as pmc_reprogram_counter() does*/ 361 perf_event_enable(pmc->perf_event); 362 pmc->is_paused = false; 363 364 return true; 365 } 366 367 static void pmc_release_perf_event(struct kvm_pmc *pmc) 368 { 369 if (pmc->perf_event) { 370 perf_event_release_kernel(pmc->perf_event); 371 pmc->perf_event = NULL; 372 pmc->current_config = 0; 373 pmc_to_pmu(pmc)->event_count--; 374 } 375 } 376 377 static void pmc_stop_counter(struct kvm_pmc *pmc) 378 { 379 if (pmc->perf_event) { 380 pmc->counter = pmc_read_counter(pmc); 381 pmc_release_perf_event(pmc); 382 } 383 } 384 385 static void pmc_update_sample_period(struct kvm_pmc *pmc) 386 { 387 if (!pmc->perf_event || pmc->is_paused || 388 !is_sampling_event(pmc->perf_event)) 389 return; 390 391 perf_event_period(pmc->perf_event, 392 get_sample_period(pmc, pmc->counter)); 393 } 394 395 void pmc_write_counter(struct kvm_pmc *pmc, u64 val) 396 { 397 if (kvm_vcpu_has_mediated_pmu(pmc->vcpu)) { 398 pmc->counter = val & pmc_bitmask(pmc); 399 return; 400 } 401 402 /* 403 * Drop any unconsumed accumulated counts, the WRMSR is a write, not a 404 * read-modify-write. Adjust the counter value so that its value is 405 * relative to the current count, as reading the current count from 406 * perf is faster than pausing and repgrogramming the event in order to 407 * reset it to '0'. Note, this very sneakily offsets the accumulated 408 * emulated count too, by using pmc_read_counter()! 409 */ 410 pmc->emulated_counter = 0; 411 pmc->counter += val - pmc_read_counter(pmc); 412 pmc->counter &= pmc_bitmask(pmc); 413 pmc_update_sample_period(pmc); 414 } 415 EXPORT_SYMBOL_FOR_KVM_INTERNAL(pmc_write_counter); 416 417 static int filter_cmp(const void *pa, const void *pb, u64 mask) 418 { 419 u64 a = *(u64 *)pa & mask; 420 u64 b = *(u64 *)pb & mask; 421 422 return (a > b) - (a < b); 423 } 424 425 426 static int filter_sort_cmp(const void *pa, const void *pb) 427 { 428 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT | 429 KVM_PMU_MASKED_ENTRY_EXCLUDE)); 430 } 431 432 /* 433 * For the event filter, searching is done on the 'includes' list and 434 * 'excludes' list separately rather than on the 'events' list (which 435 * has both). As a result the exclude bit can be ignored. 436 */ 437 static int filter_event_cmp(const void *pa, const void *pb) 438 { 439 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT)); 440 } 441 442 static int find_filter_index(u64 *events, u64 nevents, u64 key) 443 { 444 u64 *fe = bsearch(&key, events, nevents, sizeof(events[0]), 445 filter_event_cmp); 446 447 if (!fe) 448 return -1; 449 450 return fe - events; 451 } 452 453 static bool is_filter_entry_match(u64 filter_event, u64 umask) 454 { 455 u64 mask = filter_event >> (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8); 456 u64 match = filter_event & KVM_PMU_MASKED_ENTRY_UMASK_MATCH; 457 458 BUILD_BUG_ON((KVM_PMU_ENCODE_MASKED_ENTRY(0, 0xff, 0, false) >> 459 (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8)) != 460 ARCH_PERFMON_EVENTSEL_UMASK); 461 462 return (umask & mask) == match; 463 } 464 465 static bool filter_contains_match(u64 *events, u64 nevents, u64 eventsel) 466 { 467 u64 event_select = eventsel & kvm_pmu_ops.EVENTSEL_EVENT; 468 u64 umask = eventsel & ARCH_PERFMON_EVENTSEL_UMASK; 469 int i, index; 470 471 index = find_filter_index(events, nevents, event_select); 472 if (index < 0) 473 return false; 474 475 /* 476 * Entries are sorted by the event select. Walk the list in both 477 * directions to process all entries with the targeted event select. 478 */ 479 for (i = index; i < nevents; i++) { 480 if (filter_event_cmp(&events[i], &event_select)) 481 break; 482 483 if (is_filter_entry_match(events[i], umask)) 484 return true; 485 } 486 487 for (i = index - 1; i >= 0; i--) { 488 if (filter_event_cmp(&events[i], &event_select)) 489 break; 490 491 if (is_filter_entry_match(events[i], umask)) 492 return true; 493 } 494 495 return false; 496 } 497 498 static bool is_gp_event_allowed(struct kvm_x86_pmu_event_filter *f, 499 u64 eventsel) 500 { 501 if (filter_contains_match(f->includes, f->nr_includes, eventsel) && 502 !filter_contains_match(f->excludes, f->nr_excludes, eventsel)) 503 return f->action == KVM_PMU_EVENT_ALLOW; 504 505 return f->action == KVM_PMU_EVENT_DENY; 506 } 507 508 static bool is_fixed_event_allowed(struct kvm_x86_pmu_event_filter *filter, 509 int idx) 510 { 511 int fixed_idx = idx - KVM_FIXED_PMC_BASE_IDX; 512 513 if (filter->action == KVM_PMU_EVENT_DENY && 514 test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) 515 return false; 516 if (filter->action == KVM_PMU_EVENT_ALLOW && 517 !test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) 518 return false; 519 520 return true; 521 } 522 523 static bool pmc_is_event_allowed(struct kvm_pmc *pmc) 524 { 525 struct kvm_x86_pmu_event_filter *filter; 526 struct kvm *kvm = pmc->vcpu->kvm; 527 528 filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu); 529 if (!filter) 530 return true; 531 532 if (pmc_is_gp(pmc)) 533 return is_gp_event_allowed(filter, pmc->eventsel); 534 535 return is_fixed_event_allowed(filter, pmc->idx); 536 } 537 538 static void kvm_mediated_pmu_refresh_event_filter(struct kvm_pmc *pmc) 539 { 540 bool allowed = pmc_is_locally_enabled(pmc) && pmc_is_event_allowed(pmc); 541 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 542 543 if (pmc_is_gp(pmc)) { 544 pmc->eventsel_hw &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 545 if (allowed) 546 pmc->eventsel_hw |= pmc->eventsel & 547 ARCH_PERFMON_EVENTSEL_ENABLE; 548 } else { 549 u64 mask = intel_fixed_bits_by_idx(pmc->idx - KVM_FIXED_PMC_BASE_IDX, 0xf); 550 551 pmu->fixed_ctr_ctrl_hw &= ~mask; 552 if (allowed) 553 pmu->fixed_ctr_ctrl_hw |= pmu->fixed_ctr_ctrl & mask; 554 } 555 } 556 557 static int reprogram_counter(struct kvm_pmc *pmc) 558 { 559 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 560 u64 eventsel = pmc->eventsel; 561 u64 new_config = eventsel; 562 bool emulate_overflow; 563 u8 fixed_ctr_ctrl; 564 565 if (kvm_vcpu_has_mediated_pmu(pmu_to_vcpu(pmu))) { 566 kvm_mediated_pmu_refresh_event_filter(pmc); 567 return 0; 568 } 569 570 emulate_overflow = pmc_pause_counter(pmc); 571 572 if (!pmc_is_globally_enabled(pmc) || !pmc_is_locally_enabled(pmc) || 573 !pmc_is_event_allowed(pmc)) 574 return 0; 575 576 if (emulate_overflow) 577 __kvm_perf_overflow(pmc, false); 578 579 if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) 580 printk_once("kvm pmu: pin control bit is ignored\n"); 581 582 if (pmc_is_fixed(pmc)) { 583 fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, 584 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 585 if (fixed_ctr_ctrl & INTEL_FIXED_0_KERNEL) 586 eventsel |= ARCH_PERFMON_EVENTSEL_OS; 587 if (fixed_ctr_ctrl & INTEL_FIXED_0_USER) 588 eventsel |= ARCH_PERFMON_EVENTSEL_USR; 589 if (fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI) 590 eventsel |= ARCH_PERFMON_EVENTSEL_INT; 591 new_config = (u64)fixed_ctr_ctrl; 592 } 593 594 if (pmc->current_config == new_config && pmc_resume_counter(pmc)) 595 return 0; 596 597 pmc_release_perf_event(pmc); 598 599 pmc->current_config = new_config; 600 601 return pmc_reprogram_counter(pmc, PERF_TYPE_RAW, 602 (eventsel & pmu->raw_event_mask), 603 !(eventsel & ARCH_PERFMON_EVENTSEL_USR), 604 !(eventsel & ARCH_PERFMON_EVENTSEL_OS), 605 eventsel & ARCH_PERFMON_EVENTSEL_INT); 606 } 607 608 static bool pmc_is_event_match(struct kvm_pmc *pmc, u64 eventsel) 609 { 610 /* 611 * Ignore checks for edge detect (all events currently emulated by KVM 612 * are always rising edges), pin control (unsupported by modern CPUs), 613 * and counter mask and its invert flag (KVM doesn't emulate multiple 614 * events in a single clock cycle). 615 * 616 * Note, the uppermost nibble of AMD's mask overlaps Intel's IN_TX (bit 617 * 32) and IN_TXCP (bit 33), as well as two reserved bits (bits 35:34). 618 * Checking the "in HLE/RTM transaction" flags is correct as the vCPU 619 * can't be in a transaction if KVM is emulating an instruction. 620 * 621 * Checking the reserved bits might be wrong if they are defined in the 622 * future, but so could ignoring them, so do the simple thing for now. 623 */ 624 return !((pmc->eventsel ^ eventsel) & AMD64_RAW_EVENT_MASK_NB); 625 } 626 627 void kvm_pmu_recalc_pmc_emulation(struct kvm_pmu *pmu, struct kvm_pmc *pmc) 628 { 629 bitmap_clear(pmu->pmc_counting_instructions, pmc->idx, 1); 630 bitmap_clear(pmu->pmc_counting_branches, pmc->idx, 1); 631 632 /* 633 * Do NOT consult the PMU event filters, as the filters must be checked 634 * at the time of emulation to ensure KVM uses fresh information, e.g. 635 * omitting a PMC from a bitmap could result in a missed event if the 636 * filter is changed to allow counting the event. 637 */ 638 if (!pmc_is_locally_enabled(pmc)) 639 return; 640 641 if (pmc_is_event_match(pmc, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED)) 642 bitmap_set(pmu->pmc_counting_instructions, pmc->idx, 1); 643 644 if (pmc_is_event_match(pmc, kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED)) 645 bitmap_set(pmu->pmc_counting_branches, pmc->idx, 1); 646 } 647 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_recalc_pmc_emulation); 648 649 void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) 650 { 651 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); 652 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 653 struct kvm_pmc *pmc; 654 int bit; 655 656 bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); 657 658 /* 659 * The reprogramming bitmap can be written asynchronously by something 660 * other than the task that holds vcpu->mutex, take care to clear only 661 * the bits that will actually processed. 662 */ 663 BUILD_BUG_ON(sizeof(bitmap) != sizeof(atomic64_t)); 664 atomic64_andnot(*(s64 *)bitmap, &pmu->__reprogram_pmi); 665 666 kvm_for_each_pmc(pmu, pmc, bit, bitmap) { 667 /* 668 * If reprogramming fails, e.g. due to contention, re-set the 669 * regprogram bit set, i.e. opportunistically try again on the 670 * next PMU refresh. Don't make a new request as doing so can 671 * stall the guest if reprogramming repeatedly fails. 672 */ 673 if (reprogram_counter(pmc)) 674 set_bit(pmc->idx, pmu->reprogram_pmi); 675 } 676 677 /* 678 * Release unused perf_events if the corresponding guest MSRs weren't 679 * accessed during the last vCPU time slice (need_cleanup is set when 680 * the vCPU is scheduled back in). 681 */ 682 if (unlikely(pmu->need_cleanup)) 683 kvm_pmu_cleanup(vcpu); 684 685 kvm_for_each_pmc(pmu, pmc, bit, bitmap) 686 kvm_pmu_recalc_pmc_emulation(pmu, pmc); 687 } 688 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_handle_event); 689 690 int kvm_pmu_check_rdpmc_early(struct kvm_vcpu *vcpu, unsigned int idx) 691 { 692 /* 693 * On Intel, VMX interception has priority over RDPMC exceptions that 694 * aren't already handled by the emulator, i.e. there are no additional 695 * check needed for Intel PMUs. 696 * 697 * On AMD, _all_ exceptions on RDPMC have priority over SVM intercepts, 698 * i.e. an invalid PMC results in a #GP, not #VMEXIT. 699 */ 700 if (!kvm_pmu_ops.check_rdpmc_early) 701 return 0; 702 703 return kvm_pmu_call(check_rdpmc_early)(vcpu, idx); 704 } 705 706 bool is_vmware_backdoor_pmc(u32 pmc_idx) 707 { 708 switch (pmc_idx) { 709 case VMWARE_BACKDOOR_PMC_HOST_TSC: 710 case VMWARE_BACKDOOR_PMC_REAL_TIME: 711 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 712 return true; 713 } 714 return false; 715 } 716 717 static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 718 { 719 u64 ctr_val; 720 721 switch (idx) { 722 case VMWARE_BACKDOOR_PMC_HOST_TSC: 723 ctr_val = rdtsc(); 724 break; 725 case VMWARE_BACKDOOR_PMC_REAL_TIME: 726 ctr_val = ktime_get_boottime_ns(); 727 break; 728 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 729 ctr_val = ktime_get_boottime_ns() + 730 vcpu->kvm->arch.kvmclock_offset; 731 break; 732 default: 733 return 1; 734 } 735 736 *data = ctr_val; 737 return 0; 738 } 739 740 int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 741 { 742 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 743 struct kvm_pmc *pmc; 744 u64 mask = ~0ull; 745 746 if (!pmu->version) 747 return 1; 748 749 if (is_vmware_backdoor_pmc(idx)) 750 return kvm_pmu_rdpmc_vmware(vcpu, idx, data); 751 752 pmc = kvm_pmu_call(rdpmc_ecx_to_pmc)(vcpu, idx, &mask); 753 if (!pmc) 754 return 1; 755 756 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCE) && 757 (kvm_x86_call(get_cpl)(vcpu) != 0) && 758 kvm_is_cr0_bit_set(vcpu, X86_CR0_PE)) 759 return 1; 760 761 *data = pmc_read_counter(pmc) & mask; 762 return 0; 763 } 764 765 static bool kvm_need_any_pmc_intercept(struct kvm_vcpu *vcpu) 766 { 767 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 768 769 if (!kvm_vcpu_has_mediated_pmu(vcpu)) 770 return true; 771 772 /* 773 * Note! Check *host* PMU capabilities, not KVM's PMU capabilities, as 774 * KVM's capabilities are constrained based on KVM support, i.e. KVM's 775 * capabilities themselves may be a subset of hardware capabilities. 776 */ 777 return pmu->nr_arch_gp_counters != kvm_host_pmu.num_counters_gp || 778 pmu->nr_arch_fixed_counters != kvm_host_pmu.num_counters_fixed; 779 } 780 781 bool kvm_need_perf_global_ctrl_intercept(struct kvm_vcpu *vcpu) 782 { 783 return kvm_need_any_pmc_intercept(vcpu) || 784 !kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); 785 } 786 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_perf_global_ctrl_intercept); 787 788 bool kvm_need_rdpmc_intercept(struct kvm_vcpu *vcpu) 789 { 790 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 791 792 /* 793 * VMware allows access to these Pseduo-PMCs even when read via RDPMC 794 * in Ring3 when CR4.PCE=0. 795 */ 796 if (enable_vmware_backdoor) 797 return true; 798 799 return kvm_need_any_pmc_intercept(vcpu) || 800 pmu->counter_bitmask[KVM_PMC_GP] != (BIT_ULL(kvm_host_pmu.bit_width_gp) - 1) || 801 pmu->counter_bitmask[KVM_PMC_FIXED] != (BIT_ULL(kvm_host_pmu.bit_width_fixed) - 1); 802 } 803 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_rdpmc_intercept); 804 805 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) 806 { 807 if (lapic_in_kernel(vcpu)) { 808 kvm_pmu_call(deliver_pmi)(vcpu); 809 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); 810 } 811 } 812 813 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 814 { 815 switch (msr) { 816 case MSR_CORE_PERF_GLOBAL_STATUS: 817 case MSR_CORE_PERF_GLOBAL_CTRL: 818 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 819 return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); 820 default: 821 break; 822 } 823 return kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr) || 824 kvm_pmu_call(is_valid_msr)(vcpu, msr); 825 } 826 827 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) 828 { 829 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 830 struct kvm_pmc *pmc = kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr); 831 832 if (pmc) 833 __set_bit(pmc->idx, pmu->pmc_in_use); 834 } 835 836 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 837 { 838 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 839 u32 msr = msr_info->index; 840 841 switch (msr) { 842 case MSR_CORE_PERF_GLOBAL_STATUS: 843 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: 844 msr_info->data = pmu->global_status; 845 break; 846 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: 847 case MSR_CORE_PERF_GLOBAL_CTRL: 848 msr_info->data = pmu->global_ctrl; 849 break; 850 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: 851 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: 852 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 853 msr_info->data = 0; 854 break; 855 default: 856 return kvm_pmu_call(get_msr)(vcpu, msr_info); 857 } 858 859 return 0; 860 } 861 862 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 863 { 864 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 865 u32 msr = msr_info->index; 866 u64 data = msr_info->data; 867 u64 diff; 868 869 /* 870 * Note, AMD ignores writes to reserved bits and read-only PMU MSRs, 871 * whereas Intel generates #GP on attempts to write reserved/RO MSRs. 872 */ 873 switch (msr) { 874 case MSR_CORE_PERF_GLOBAL_STATUS: 875 if (!msr_info->host_initiated) 876 return 1; /* RO MSR */ 877 fallthrough; 878 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: 879 /* Per PPR, Read-only MSR. Writes are ignored. */ 880 if (!msr_info->host_initiated) 881 break; 882 883 if (data & pmu->global_status_rsvd) 884 return 1; 885 886 pmu->global_status = data; 887 break; 888 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: 889 data &= ~pmu->global_ctrl_rsvd; 890 fallthrough; 891 case MSR_CORE_PERF_GLOBAL_CTRL: 892 if (!kvm_valid_perf_global_ctrl(pmu, data)) 893 return 1; 894 895 if (pmu->global_ctrl != data) { 896 diff = pmu->global_ctrl ^ data; 897 pmu->global_ctrl = data; 898 kvm_pmu_request_counters_reprogram(pmu, diff); 899 } 900 /* 901 * Unconditionally forward writes to vendor code, i.e. to the 902 * VMC{B,S}, as pmu->global_ctrl is per-VCPU, not per-VMC{B,S}. 903 */ 904 if (kvm_vcpu_has_mediated_pmu(vcpu)) 905 kvm_pmu_call(write_global_ctrl)(data); 906 break; 907 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 908 /* 909 * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in 910 * GLOBAL_STATUS, and so the set of reserved bits is the same. 911 */ 912 if (data & pmu->global_status_rsvd) 913 return 1; 914 fallthrough; 915 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: 916 if (!msr_info->host_initiated) 917 pmu->global_status &= ~data; 918 break; 919 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: 920 if (!msr_info->host_initiated) 921 pmu->global_status |= data & ~pmu->global_status_rsvd; 922 break; 923 default: 924 kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); 925 return kvm_pmu_call(set_msr)(vcpu, msr_info); 926 } 927 928 return 0; 929 } 930 931 static void kvm_pmu_reset(struct kvm_vcpu *vcpu) 932 { 933 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 934 struct kvm_pmc *pmc; 935 int i; 936 937 pmu->need_cleanup = false; 938 939 bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX); 940 bitmap_zero(pmu->pmc_has_mode_specific_enables, X86_PMC_IDX_MAX); 941 942 kvm_for_each_pmc(pmu, pmc, i, pmu->all_valid_pmc_idx) { 943 pmc_stop_counter(pmc); 944 pmc->counter = 0; 945 pmc->emulated_counter = 0; 946 947 if (pmc_is_gp(pmc)) { 948 pmc->eventsel = 0; 949 pmc->eventsel_hw = 0; 950 } 951 } 952 953 pmu->fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw = 0; 954 pmu->global_ctrl = pmu->global_status = 0; 955 956 kvm_pmu_call(reset)(vcpu); 957 } 958 959 960 /* 961 * Refresh the PMU configuration for the vCPU, e.g. if userspace changes CPUID 962 * and/or PERF_CAPABILITIES. 963 */ 964 void kvm_pmu_refresh(struct kvm_vcpu *vcpu) 965 { 966 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 967 968 if (KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm)) 969 return; 970 971 /* 972 * Stop/release all existing counters/events before realizing the new 973 * vPMU model. 974 */ 975 kvm_pmu_reset(vcpu); 976 977 pmu->version = 0; 978 pmu->nr_arch_gp_counters = 0; 979 pmu->nr_arch_fixed_counters = 0; 980 pmu->counter_bitmask[KVM_PMC_GP] = 0; 981 pmu->counter_bitmask[KVM_PMC_FIXED] = 0; 982 pmu->reserved_bits = 0xffffffff00200000ull; 983 pmu->raw_event_mask = X86_RAW_EVENT_MASK; 984 pmu->global_ctrl_rsvd = ~0ull; 985 pmu->global_status_rsvd = ~0ull; 986 pmu->fixed_ctr_ctrl_rsvd = ~0ull; 987 pmu->pebs_enable_rsvd = ~0ull; 988 pmu->pebs_data_cfg_rsvd = ~0ull; 989 bitmap_zero(pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX); 990 991 if (!vcpu->kvm->arch.enable_pmu) 992 return; 993 994 kvm_pmu_call(refresh)(vcpu); 995 996 /* 997 * At RESET, both Intel and AMD CPUs set all enable bits for general 998 * purpose counters in IA32_PERF_GLOBAL_CTRL (so that software that 999 * was written for v1 PMUs don't unknowingly leave GP counters disabled 1000 * in the global controls). Emulate that behavior when refreshing the 1001 * PMU so that userspace doesn't need to manually set PERF_GLOBAL_CTRL. 1002 */ 1003 if (pmu->nr_arch_gp_counters && 1004 (kvm_pmu_has_perf_global_ctrl(pmu) || kvm_vcpu_has_mediated_pmu(vcpu))) 1005 pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0); 1006 1007 if (kvm_vcpu_has_mediated_pmu(vcpu)) 1008 kvm_pmu_call(write_global_ctrl)(pmu->global_ctrl); 1009 1010 bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters); 1011 bitmap_set(pmu->all_valid_pmc_idx, KVM_FIXED_PMC_BASE_IDX, 1012 pmu->nr_arch_fixed_counters); 1013 } 1014 1015 void kvm_pmu_init(struct kvm_vcpu *vcpu) 1016 { 1017 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1018 1019 memset(pmu, 0, sizeof(*pmu)); 1020 kvm_pmu_call(init)(vcpu); 1021 } 1022 1023 /* Release perf_events for vPMCs that have been unused for a full time slice. */ 1024 void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) 1025 { 1026 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1027 struct kvm_pmc *pmc = NULL; 1028 DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX); 1029 int i; 1030 1031 pmu->need_cleanup = false; 1032 1033 bitmap_andnot(bitmask, pmu->all_valid_pmc_idx, 1034 pmu->pmc_in_use, X86_PMC_IDX_MAX); 1035 1036 kvm_for_each_pmc(pmu, pmc, i, bitmask) { 1037 if (pmc->perf_event && !pmc_is_locally_enabled(pmc)) 1038 pmc_stop_counter(pmc); 1039 } 1040 1041 kvm_pmu_call(cleanup)(vcpu); 1042 1043 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); 1044 } 1045 1046 void kvm_pmu_destroy(struct kvm_vcpu *vcpu) 1047 { 1048 kvm_pmu_reset(vcpu); 1049 } 1050 1051 static bool pmc_is_pmi_enabled(struct kvm_pmc *pmc) 1052 { 1053 u8 fixed_ctr_ctrl; 1054 1055 if (pmc_is_gp(pmc)) 1056 return pmc->eventsel & ARCH_PERFMON_EVENTSEL_INT; 1057 1058 fixed_ctr_ctrl = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl, 1059 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 1060 return fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI; 1061 } 1062 1063 static void kvm_pmu_incr_counter(struct kvm_pmc *pmc) 1064 { 1065 struct kvm_vcpu *vcpu = pmc->vcpu; 1066 1067 /* 1068 * For perf-based PMUs, accumulate software-emulated events separately 1069 * from pmc->counter, as pmc->counter is offset by the count of the 1070 * associated perf event. Request reprogramming, which will consult 1071 * both emulated and hardware-generated events to detect overflow. 1072 */ 1073 if (!kvm_vcpu_has_mediated_pmu(vcpu)) { 1074 pmc->emulated_counter++; 1075 kvm_pmu_request_counter_reprogram(pmc); 1076 return; 1077 } 1078 1079 /* 1080 * For mediated PMUs, pmc->counter is updated when the vCPU's PMU is 1081 * put, and will be loaded into hardware when the PMU is loaded. Simply 1082 * increment the counter and signal overflow if it wraps to zero. 1083 */ 1084 pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc); 1085 if (!pmc->counter) { 1086 pmc_to_pmu(pmc)->global_status |= BIT_ULL(pmc->idx); 1087 if (pmc_is_pmi_enabled(pmc)) 1088 kvm_make_request(KVM_REQ_PMI, vcpu); 1089 } 1090 } 1091 1092 static inline bool cpl_is_matched(struct kvm_pmc *pmc) 1093 { 1094 bool select_os, select_user; 1095 u64 config; 1096 1097 if (pmc_is_gp(pmc)) { 1098 config = pmc->eventsel; 1099 select_os = config & ARCH_PERFMON_EVENTSEL_OS; 1100 select_user = config & ARCH_PERFMON_EVENTSEL_USR; 1101 } else { 1102 config = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl, 1103 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 1104 select_os = config & INTEL_FIXED_0_KERNEL; 1105 select_user = config & INTEL_FIXED_0_USER; 1106 } 1107 1108 /* 1109 * Skip the CPL lookup, which isn't free on Intel, if the result will 1110 * be the same regardless of the CPL. 1111 */ 1112 if (select_os == select_user) 1113 return select_os; 1114 1115 return (kvm_x86_call(get_cpl)(pmc->vcpu) == 0) ? select_os : 1116 select_user; 1117 } 1118 1119 static void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, 1120 const unsigned long *event_pmcs) 1121 { 1122 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); 1123 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1124 struct kvm_pmc *pmc; 1125 int i, idx; 1126 1127 BUILD_BUG_ON(sizeof(pmu->global_ctrl) * BITS_PER_BYTE != X86_PMC_IDX_MAX); 1128 1129 if (bitmap_empty(event_pmcs, X86_PMC_IDX_MAX)) 1130 return; 1131 1132 if (!kvm_pmu_has_perf_global_ctrl(pmu)) 1133 bitmap_copy(bitmap, event_pmcs, X86_PMC_IDX_MAX); 1134 else if (!bitmap_and(bitmap, event_pmcs, 1135 (unsigned long *)&pmu->global_ctrl, X86_PMC_IDX_MAX)) 1136 return; 1137 1138 idx = srcu_read_lock(&vcpu->kvm->srcu); 1139 kvm_for_each_pmc(pmu, pmc, i, bitmap) { 1140 if (!pmc_is_event_allowed(pmc) || !cpl_is_matched(pmc)) 1141 continue; 1142 1143 kvm_pmu_incr_counter(pmc); 1144 } 1145 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1146 } 1147 1148 void kvm_pmu_instruction_retired(struct kvm_vcpu *vcpu) 1149 { 1150 kvm_pmu_trigger_event(vcpu, vcpu_to_pmu(vcpu)->pmc_counting_instructions); 1151 } 1152 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_instruction_retired); 1153 1154 void kvm_pmu_branch_retired(struct kvm_vcpu *vcpu) 1155 { 1156 kvm_pmu_trigger_event(vcpu, vcpu_to_pmu(vcpu)->pmc_counting_branches); 1157 } 1158 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_branch_retired); 1159 1160 static bool is_masked_filter_valid(const struct kvm_x86_pmu_event_filter *filter) 1161 { 1162 u64 mask = kvm_pmu_ops.EVENTSEL_EVENT | 1163 KVM_PMU_MASKED_ENTRY_UMASK_MASK | 1164 KVM_PMU_MASKED_ENTRY_UMASK_MATCH | 1165 KVM_PMU_MASKED_ENTRY_EXCLUDE; 1166 int i; 1167 1168 for (i = 0; i < filter->nevents; i++) { 1169 if (filter->events[i] & ~mask) 1170 return false; 1171 } 1172 1173 return true; 1174 } 1175 1176 static void convert_to_masked_filter(struct kvm_x86_pmu_event_filter *filter) 1177 { 1178 int i, j; 1179 1180 for (i = 0, j = 0; i < filter->nevents; i++) { 1181 /* 1182 * Skip events that are impossible to match against a guest 1183 * event. When filtering, only the event select + unit mask 1184 * of the guest event is used. To maintain backwards 1185 * compatibility, impossible filters can't be rejected :-( 1186 */ 1187 if (filter->events[i] & ~(kvm_pmu_ops.EVENTSEL_EVENT | 1188 ARCH_PERFMON_EVENTSEL_UMASK)) 1189 continue; 1190 /* 1191 * Convert userspace events to a common in-kernel event so 1192 * only one code path is needed to support both events. For 1193 * the in-kernel events use masked events because they are 1194 * flexible enough to handle both cases. To convert to masked 1195 * events all that's needed is to add an "all ones" umask_mask, 1196 * (unmasked filter events don't support EXCLUDE). 1197 */ 1198 filter->events[j++] = filter->events[i] | 1199 (0xFFULL << KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT); 1200 } 1201 1202 filter->nevents = j; 1203 } 1204 1205 static int prepare_filter_lists(struct kvm_x86_pmu_event_filter *filter) 1206 { 1207 int i; 1208 1209 if (!(filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS)) 1210 convert_to_masked_filter(filter); 1211 else if (!is_masked_filter_valid(filter)) 1212 return -EINVAL; 1213 1214 /* 1215 * Sort entries by event select and includes vs. excludes so that all 1216 * entries for a given event select can be processed efficiently during 1217 * filtering. The EXCLUDE flag uses a more significant bit than the 1218 * event select, and so the sorted list is also effectively split into 1219 * includes and excludes sub-lists. 1220 */ 1221 sort(&filter->events, filter->nevents, sizeof(filter->events[0]), 1222 filter_sort_cmp, NULL); 1223 1224 i = filter->nevents; 1225 /* Find the first EXCLUDE event (only supported for masked events). */ 1226 if (filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS) { 1227 for (i = 0; i < filter->nevents; i++) { 1228 if (filter->events[i] & KVM_PMU_MASKED_ENTRY_EXCLUDE) 1229 break; 1230 } 1231 } 1232 1233 filter->nr_includes = i; 1234 filter->nr_excludes = filter->nevents - filter->nr_includes; 1235 filter->includes = filter->events; 1236 filter->excludes = filter->events + filter->nr_includes; 1237 1238 return 0; 1239 } 1240 1241 int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp) 1242 { 1243 struct kvm_pmu_event_filter __user *user_filter = argp; 1244 struct kvm_x86_pmu_event_filter *filter; 1245 struct kvm_pmu_event_filter tmp; 1246 struct kvm_vcpu *vcpu; 1247 unsigned long i; 1248 size_t size; 1249 int r; 1250 1251 if (copy_from_user(&tmp, user_filter, sizeof(tmp))) 1252 return -EFAULT; 1253 1254 if (tmp.action != KVM_PMU_EVENT_ALLOW && 1255 tmp.action != KVM_PMU_EVENT_DENY) 1256 return -EINVAL; 1257 1258 if (tmp.flags & ~KVM_PMU_EVENT_FLAGS_VALID_MASK) 1259 return -EINVAL; 1260 1261 if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS) 1262 return -E2BIG; 1263 1264 size = struct_size(filter, events, tmp.nevents); 1265 filter = kzalloc(size, GFP_KERNEL_ACCOUNT); 1266 if (!filter) 1267 return -ENOMEM; 1268 1269 filter->action = tmp.action; 1270 filter->nevents = tmp.nevents; 1271 filter->fixed_counter_bitmap = tmp.fixed_counter_bitmap; 1272 filter->flags = tmp.flags; 1273 1274 r = -EFAULT; 1275 if (copy_from_user(filter->events, user_filter->events, 1276 flex_array_size(filter, events, filter->nevents))) 1277 goto cleanup; 1278 1279 r = prepare_filter_lists(filter); 1280 if (r) 1281 goto cleanup; 1282 1283 mutex_lock(&kvm->lock); 1284 filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter, 1285 mutex_is_locked(&kvm->lock)); 1286 mutex_unlock(&kvm->lock); 1287 synchronize_srcu_expedited(&kvm->srcu); 1288 1289 BUILD_BUG_ON(sizeof(((struct kvm_pmu *)0)->reprogram_pmi) > 1290 sizeof(((struct kvm_pmu *)0)->__reprogram_pmi)); 1291 1292 kvm_for_each_vcpu(i, vcpu, kvm) 1293 atomic64_set(&vcpu_to_pmu(vcpu)->__reprogram_pmi, -1ull); 1294 1295 kvm_make_all_cpus_request(kvm, KVM_REQ_PMU); 1296 1297 r = 0; 1298 cleanup: 1299 kfree(filter); 1300 return r; 1301 } 1302 1303 static __always_inline u32 fixed_counter_msr(u32 idx) 1304 { 1305 return kvm_pmu_ops.FIXED_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1306 } 1307 1308 static __always_inline u32 gp_counter_msr(u32 idx) 1309 { 1310 return kvm_pmu_ops.GP_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1311 } 1312 1313 static __always_inline u32 gp_eventsel_msr(u32 idx) 1314 { 1315 return kvm_pmu_ops.GP_EVENTSEL_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1316 } 1317 1318 static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu) 1319 { 1320 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1321 struct kvm_pmc *pmc; 1322 u32 i; 1323 1324 /* 1325 * No need to zero out unexposed GP/fixed counters/selectors since RDPMC 1326 * is intercepted if hardware has counters that aren't visible to the 1327 * guest (KVM will inject #GP as appropriate). 1328 */ 1329 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 1330 pmc = &pmu->gp_counters[i]; 1331 1332 if (pmc->counter != rdpmc(i)) 1333 wrmsrq(gp_counter_msr(i), pmc->counter); 1334 wrmsrq(gp_eventsel_msr(i), pmc->eventsel_hw); 1335 } 1336 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 1337 pmc = &pmu->fixed_counters[i]; 1338 1339 if (pmc->counter != rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i)) 1340 wrmsrq(fixed_counter_msr(i), pmc->counter); 1341 } 1342 } 1343 1344 void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu) 1345 { 1346 if (!kvm_vcpu_has_mediated_pmu(vcpu) || 1347 KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 1348 return; 1349 1350 lockdep_assert_irqs_disabled(); 1351 1352 perf_load_guest_context(); 1353 1354 /* 1355 * Explicitly clear PERF_GLOBAL_CTRL, as "loading" the guest's context 1356 * disables all individual counters (if any were enabled), but doesn't 1357 * globally disable the entire PMU. Loading event selectors and PMCs 1358 * with guest values while PERF_GLOBAL_CTRL is non-zero will generate 1359 * unexpected events and PMIs. 1360 * 1361 * VMX will enable/disable counters at VM-Enter/VM-Exit by atomically 1362 * loading PERF_GLOBAL_CONTROL. SVM effectively performs the switch by 1363 * configuring all events to be GUEST_ONLY. Clear PERF_GLOBAL_CONTROL 1364 * even for SVM to minimize the damage if a perf event is left enabled, 1365 * and to ensure a consistent starting state. 1366 */ 1367 wrmsrq(kvm_pmu_ops.PERF_GLOBAL_CTRL, 0); 1368 1369 perf_load_guest_lvtpc(kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVTPC)); 1370 1371 kvm_pmu_load_guest_pmcs(vcpu); 1372 1373 kvm_pmu_call(mediated_load)(vcpu); 1374 } 1375 1376 static void kvm_pmu_put_guest_pmcs(struct kvm_vcpu *vcpu) 1377 { 1378 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1379 struct kvm_pmc *pmc; 1380 u32 i; 1381 1382 /* 1383 * Clear selectors and counters to ensure hardware doesn't count using 1384 * guest controls when the host (perf) restores its state. 1385 */ 1386 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 1387 pmc = &pmu->gp_counters[i]; 1388 1389 pmc->counter = rdpmc(i); 1390 if (pmc->counter) 1391 wrmsrq(gp_counter_msr(i), 0); 1392 if (pmc->eventsel_hw) 1393 wrmsrq(gp_eventsel_msr(i), 0); 1394 } 1395 1396 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 1397 pmc = &pmu->fixed_counters[i]; 1398 1399 pmc->counter = rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i); 1400 if (pmc->counter) 1401 wrmsrq(fixed_counter_msr(i), 0); 1402 } 1403 } 1404 1405 void kvm_mediated_pmu_put(struct kvm_vcpu *vcpu) 1406 { 1407 if (!kvm_vcpu_has_mediated_pmu(vcpu) || 1408 KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 1409 return; 1410 1411 lockdep_assert_irqs_disabled(); 1412 1413 /* 1414 * Defer handling of PERF_GLOBAL_CTRL to vendor code. On Intel, it's 1415 * atomically cleared on VM-Exit, i.e. doesn't need to be clear here. 1416 */ 1417 kvm_pmu_call(mediated_put)(vcpu); 1418 1419 kvm_pmu_put_guest_pmcs(vcpu); 1420 1421 perf_put_guest_lvtpc(); 1422 1423 perf_put_guest_context(); 1424 } 1425