1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine -- Performance Monitoring Unit support 4 * 5 * Copyright 2015 Red Hat, Inc. and/or its affiliates. 6 * 7 * Authors: 8 * Avi Kivity <avi@redhat.com> 9 * Gleb Natapov <gleb@redhat.com> 10 * Wei Huang <wei@redhat.com> 11 */ 12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 13 14 #include <linux/types.h> 15 #include <linux/kvm_host.h> 16 #include <linux/perf_event.h> 17 #include <linux/bsearch.h> 18 #include <linux/sort.h> 19 #include <asm/perf_event.h> 20 #include <asm/cpu_device_id.h> 21 #include "x86.h" 22 #include "cpuid.h" 23 #include "lapic.h" 24 #include "pmu.h" 25 26 /* This is enough to filter the vast majority of currently defined events. */ 27 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300 28 29 /* Unadultered PMU capabilities of the host, i.e. of hardware. */ 30 static struct x86_pmu_capability __read_mostly kvm_host_pmu; 31 32 /* KVM's PMU capabilities, i.e. the intersection of KVM and hardware support. */ 33 struct x86_pmu_capability __read_mostly kvm_pmu_cap; 34 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_cap); 35 36 struct kvm_pmu_emulated_event_selectors { 37 u64 INSTRUCTIONS_RETIRED; 38 u64 BRANCH_INSTRUCTIONS_RETIRED; 39 }; 40 static struct kvm_pmu_emulated_event_selectors __read_mostly kvm_pmu_eventsel; 41 42 /* Precise Distribution of Instructions Retired (PDIR) */ 43 static const struct x86_cpu_id vmx_pebs_pdir_cpu[] = { 44 X86_MATCH_VFM(INTEL_ICELAKE_D, NULL), 45 X86_MATCH_VFM(INTEL_ICELAKE_X, NULL), 46 /* Instruction-Accurate PDIR (PDIR++) */ 47 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL), 48 {} 49 }; 50 51 /* Precise Distribution (PDist) */ 52 static const struct x86_cpu_id vmx_pebs_pdist_cpu[] = { 53 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL), 54 {} 55 }; 56 57 /* NOTE: 58 * - Each perf counter is defined as "struct kvm_pmc"; 59 * - There are two types of perf counters: general purpose (gp) and fixed. 60 * gp counters are stored in gp_counters[] and fixed counters are stored 61 * in fixed_counters[] respectively. Both of them are part of "struct 62 * kvm_pmu"; 63 * - pmu.c understands the difference between gp counters and fixed counters. 64 * However AMD doesn't support fixed-counters; 65 * - There are three types of index to access perf counters (PMC): 66 * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD 67 * has MSR_K7_PERFCTRn and, for families 15H and later, 68 * MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are 69 * aliased to MSR_K7_PERFCTRn. 70 * 2. MSR Index (named idx): This normally is used by RDPMC instruction. 71 * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access 72 * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except 73 * that it also supports fixed counters. idx can be used to as index to 74 * gp and fixed counters. 75 * 3. Global PMC Index (named pmc): pmc is an index specific to PMU 76 * code. Each pmc, stored in kvm_pmc.idx field, is unique across 77 * all perf counters (both gp and fixed). The mapping relationship 78 * between pmc and perf counters is as the following: 79 * * Intel: [0 .. KVM_MAX_NR_INTEL_GP_COUNTERS-1] <=> gp counters 80 * [KVM_FIXED_PMC_BASE_IDX .. KVM_FIXED_PMC_BASE_IDX + 2] <=> fixed 81 * * AMD: [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H 82 * and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters 83 */ 84 85 static struct kvm_pmu_ops kvm_pmu_ops __read_mostly; 86 87 #define KVM_X86_PMU_OP(func) \ 88 DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ 89 *(((struct kvm_pmu_ops *)0)->func)); 90 #define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP 91 #include <asm/kvm-x86-pmu-ops.h> 92 93 void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) 94 { 95 memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops)); 96 97 #define __KVM_X86_PMU_OP(func) \ 98 static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func); 99 #define KVM_X86_PMU_OP(func) \ 100 WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func) 101 #define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP 102 #include <asm/kvm-x86-pmu-ops.h> 103 #undef __KVM_X86_PMU_OP 104 } 105 106 void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops) 107 { 108 bool is_intel = boot_cpu_data.x86_vendor == X86_VENDOR_INTEL; 109 int min_nr_gp_ctrs = pmu_ops->MIN_NR_GP_COUNTERS; 110 111 /* 112 * Hybrid PMUs don't play nice with virtualization without careful 113 * configuration by userspace, and KVM's APIs for reporting supported 114 * vPMU features do not account for hybrid PMUs. Disable vPMU support 115 * for hybrid PMUs until KVM gains a way to let userspace opt-in. 116 */ 117 if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) { 118 enable_pmu = false; 119 memset(&kvm_host_pmu, 0, sizeof(kvm_host_pmu)); 120 } else { 121 perf_get_x86_pmu_capability(&kvm_host_pmu); 122 } 123 124 if (enable_pmu) { 125 /* 126 * WARN if perf did NOT disable hardware PMU if the number of 127 * architecturally required GP counters aren't present, i.e. if 128 * there are a non-zero number of counters, but fewer than what 129 * is architecturally required. 130 */ 131 if (!kvm_host_pmu.num_counters_gp || 132 WARN_ON_ONCE(kvm_host_pmu.num_counters_gp < min_nr_gp_ctrs)) 133 enable_pmu = false; 134 else if (is_intel && !kvm_host_pmu.version) 135 enable_pmu = false; 136 } 137 138 if (!enable_pmu || !enable_mediated_pmu || !kvm_host_pmu.mediated || 139 !pmu_ops->is_mediated_pmu_supported(&kvm_host_pmu)) 140 enable_mediated_pmu = false; 141 142 if (!enable_mediated_pmu) 143 pmu_ops->write_global_ctrl = NULL; 144 145 if (!enable_pmu) { 146 memset(&kvm_pmu_cap, 0, sizeof(kvm_pmu_cap)); 147 return; 148 } 149 150 memcpy(&kvm_pmu_cap, &kvm_host_pmu, sizeof(kvm_host_pmu)); 151 kvm_pmu_cap.version = min(kvm_pmu_cap.version, 2); 152 kvm_pmu_cap.num_counters_gp = min(kvm_pmu_cap.num_counters_gp, 153 pmu_ops->MAX_NR_GP_COUNTERS); 154 kvm_pmu_cap.num_counters_fixed = min(kvm_pmu_cap.num_counters_fixed, 155 KVM_MAX_NR_FIXED_COUNTERS); 156 157 kvm_pmu_eventsel.INSTRUCTIONS_RETIRED = 158 perf_get_hw_event_config(PERF_COUNT_HW_INSTRUCTIONS); 159 kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED = 160 perf_get_hw_event_config(PERF_COUNT_HW_BRANCH_INSTRUCTIONS); 161 } 162 163 void kvm_handle_guest_mediated_pmi(void) 164 { 165 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 166 167 if (WARN_ON_ONCE(!vcpu || !kvm_vcpu_has_mediated_pmu(vcpu))) 168 return; 169 170 kvm_make_request(KVM_REQ_PMI, vcpu); 171 } 172 173 static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi) 174 { 175 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 176 bool skip_pmi = false; 177 178 if (pmc->perf_event && pmc->perf_event->attr.precise_ip) { 179 if (!in_pmi) { 180 /* 181 * TODO: KVM is currently _choosing_ to not generate records 182 * for emulated instructions, avoiding BUFFER_OVF PMI when 183 * there are no records. Strictly speaking, it should be done 184 * as well in the right context to improve sampling accuracy. 185 */ 186 skip_pmi = true; 187 } else { 188 /* Indicate PEBS overflow PMI to guest. */ 189 skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, 190 (unsigned long *)&pmu->global_status); 191 } 192 } else { 193 __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); 194 } 195 196 if (pmc->intr && !skip_pmi) 197 kvm_make_request(KVM_REQ_PMI, pmc->vcpu); 198 } 199 200 static void kvm_perf_overflow(struct perf_event *perf_event, 201 struct perf_sample_data *data, 202 struct pt_regs *regs) 203 { 204 struct kvm_pmc *pmc = perf_event->overflow_handler_context; 205 206 /* 207 * Ignore asynchronous overflow events for counters that are scheduled 208 * to be reprogrammed, e.g. if a PMI for the previous event races with 209 * KVM's handling of a related guest WRMSR. 210 */ 211 if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi)) 212 return; 213 214 __kvm_perf_overflow(pmc, true); 215 216 kvm_make_request(KVM_REQ_PMU, pmc->vcpu); 217 } 218 219 static u64 pmc_get_pebs_precise_level(struct kvm_pmc *pmc) 220 { 221 /* 222 * For some model specific pebs counters with special capabilities 223 * (PDIR, PDIR++, PDIST), KVM needs to raise the event precise 224 * level to the maximum value (currently 3, backwards compatible) 225 * so that the perf subsystem would assign specific hardware counter 226 * with that capability for vPMC. 227 */ 228 if ((pmc->idx == 0 && x86_match_cpu(vmx_pebs_pdist_cpu)) || 229 (pmc->idx == 32 && x86_match_cpu(vmx_pebs_pdir_cpu))) 230 return 3; 231 232 /* 233 * The non-zero precision level of guest event makes the ordinary 234 * guest event becomes a guest PEBS event and triggers the host 235 * PEBS PMI handler to determine whether the PEBS overflow PMI 236 * comes from the host counters or the guest. 237 */ 238 return 1; 239 } 240 241 static u64 get_sample_period(struct kvm_pmc *pmc, u64 counter_value) 242 { 243 u64 sample_period = (-counter_value) & pmc_bitmask(pmc); 244 245 if (!sample_period) 246 sample_period = pmc_bitmask(pmc) + 1; 247 return sample_period; 248 } 249 250 static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config, 251 bool exclude_user, bool exclude_kernel, 252 bool intr) 253 { 254 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 255 struct perf_event *event; 256 struct perf_event_attr attr = { 257 .type = type, 258 .size = sizeof(attr), 259 .pinned = true, 260 .exclude_idle = true, 261 .exclude_host = 1, 262 .exclude_user = exclude_user, 263 .exclude_kernel = exclude_kernel, 264 .config = config, 265 }; 266 bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable); 267 268 attr.sample_period = get_sample_period(pmc, pmc->counter); 269 270 if ((attr.config & HSW_IN_TX_CHECKPOINTED) && 271 (boot_cpu_has(X86_FEATURE_RTM) || boot_cpu_has(X86_FEATURE_HLE))) { 272 /* 273 * HSW_IN_TX_CHECKPOINTED is not supported with nonzero 274 * period. Just clear the sample period so at least 275 * allocating the counter doesn't fail. 276 */ 277 attr.sample_period = 0; 278 } 279 if (pebs) { 280 /* 281 * For most PEBS hardware events, the difference in the software 282 * precision levels of guest and host PEBS events will not affect 283 * the accuracy of the PEBS profiling result, because the "event IP" 284 * in the PEBS record is calibrated on the guest side. 285 */ 286 attr.precise_ip = pmc_get_pebs_precise_level(pmc); 287 } 288 289 event = perf_event_create_kernel_counter(&attr, -1, current, 290 kvm_perf_overflow, pmc); 291 if (IS_ERR(event)) { 292 pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n", 293 PTR_ERR(event), pmc->idx); 294 return PTR_ERR(event); 295 } 296 297 pmc->perf_event = event; 298 pmc_to_pmu(pmc)->event_count++; 299 pmc->is_paused = false; 300 pmc->intr = intr || pebs; 301 return 0; 302 } 303 304 static bool pmc_pause_counter(struct kvm_pmc *pmc) 305 { 306 u64 counter = pmc->counter; 307 u64 prev_counter; 308 309 /* update counter, reset event value to avoid redundant accumulation */ 310 if (pmc->perf_event && !pmc->is_paused) 311 counter += perf_event_pause(pmc->perf_event, true); 312 313 /* 314 * Snapshot the previous counter *after* accumulating state from perf. 315 * If overflow already happened, hardware (via perf) is responsible for 316 * generating a PMI. KVM just needs to detect overflow on emulated 317 * counter events that haven't yet been processed. 318 */ 319 prev_counter = counter & pmc_bitmask(pmc); 320 321 counter += pmc->emulated_counter; 322 pmc->counter = counter & pmc_bitmask(pmc); 323 324 pmc->emulated_counter = 0; 325 pmc->is_paused = true; 326 327 return pmc->counter < prev_counter; 328 } 329 330 static bool pmc_resume_counter(struct kvm_pmc *pmc) 331 { 332 if (!pmc->perf_event) 333 return false; 334 335 /* recalibrate sample period and check if it's accepted by perf core */ 336 if (is_sampling_event(pmc->perf_event) && 337 perf_event_period(pmc->perf_event, 338 get_sample_period(pmc, pmc->counter))) 339 return false; 340 341 if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) != 342 (!!pmc->perf_event->attr.precise_ip)) 343 return false; 344 345 /* reuse perf_event to serve as pmc_reprogram_counter() does*/ 346 perf_event_enable(pmc->perf_event); 347 pmc->is_paused = false; 348 349 return true; 350 } 351 352 static void pmc_release_perf_event(struct kvm_pmc *pmc) 353 { 354 if (pmc->perf_event) { 355 perf_event_release_kernel(pmc->perf_event); 356 pmc->perf_event = NULL; 357 pmc->current_config = 0; 358 pmc_to_pmu(pmc)->event_count--; 359 } 360 } 361 362 static void pmc_stop_counter(struct kvm_pmc *pmc) 363 { 364 if (pmc->perf_event) { 365 pmc->counter = pmc_read_counter(pmc); 366 pmc_release_perf_event(pmc); 367 } 368 } 369 370 static void pmc_update_sample_period(struct kvm_pmc *pmc) 371 { 372 if (!pmc->perf_event || pmc->is_paused || 373 !is_sampling_event(pmc->perf_event)) 374 return; 375 376 perf_event_period(pmc->perf_event, 377 get_sample_period(pmc, pmc->counter)); 378 } 379 380 void pmc_write_counter(struct kvm_pmc *pmc, u64 val) 381 { 382 if (kvm_vcpu_has_mediated_pmu(pmc->vcpu)) { 383 pmc->counter = val & pmc_bitmask(pmc); 384 return; 385 } 386 387 /* 388 * Drop any unconsumed accumulated counts, the WRMSR is a write, not a 389 * read-modify-write. Adjust the counter value so that its value is 390 * relative to the current count, as reading the current count from 391 * perf is faster than pausing and repgrogramming the event in order to 392 * reset it to '0'. Note, this very sneakily offsets the accumulated 393 * emulated count too, by using pmc_read_counter()! 394 */ 395 pmc->emulated_counter = 0; 396 pmc->counter += val - pmc_read_counter(pmc); 397 pmc->counter &= pmc_bitmask(pmc); 398 pmc_update_sample_period(pmc); 399 } 400 EXPORT_SYMBOL_FOR_KVM_INTERNAL(pmc_write_counter); 401 402 static int filter_cmp(const void *pa, const void *pb, u64 mask) 403 { 404 u64 a = *(u64 *)pa & mask; 405 u64 b = *(u64 *)pb & mask; 406 407 return (a > b) - (a < b); 408 } 409 410 411 static int filter_sort_cmp(const void *pa, const void *pb) 412 { 413 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT | 414 KVM_PMU_MASKED_ENTRY_EXCLUDE)); 415 } 416 417 /* 418 * For the event filter, searching is done on the 'includes' list and 419 * 'excludes' list separately rather than on the 'events' list (which 420 * has both). As a result the exclude bit can be ignored. 421 */ 422 static int filter_event_cmp(const void *pa, const void *pb) 423 { 424 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT)); 425 } 426 427 static int find_filter_index(u64 *events, u64 nevents, u64 key) 428 { 429 u64 *fe = bsearch(&key, events, nevents, sizeof(events[0]), 430 filter_event_cmp); 431 432 if (!fe) 433 return -1; 434 435 return fe - events; 436 } 437 438 static bool is_filter_entry_match(u64 filter_event, u64 umask) 439 { 440 u64 mask = filter_event >> (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8); 441 u64 match = filter_event & KVM_PMU_MASKED_ENTRY_UMASK_MATCH; 442 443 BUILD_BUG_ON((KVM_PMU_ENCODE_MASKED_ENTRY(0, 0xff, 0, false) >> 444 (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8)) != 445 ARCH_PERFMON_EVENTSEL_UMASK); 446 447 return (umask & mask) == match; 448 } 449 450 static bool filter_contains_match(u64 *events, u64 nevents, u64 eventsel) 451 { 452 u64 event_select = eventsel & kvm_pmu_ops.EVENTSEL_EVENT; 453 u64 umask = eventsel & ARCH_PERFMON_EVENTSEL_UMASK; 454 int i, index; 455 456 index = find_filter_index(events, nevents, event_select); 457 if (index < 0) 458 return false; 459 460 /* 461 * Entries are sorted by the event select. Walk the list in both 462 * directions to process all entries with the targeted event select. 463 */ 464 for (i = index; i < nevents; i++) { 465 if (filter_event_cmp(&events[i], &event_select)) 466 break; 467 468 if (is_filter_entry_match(events[i], umask)) 469 return true; 470 } 471 472 for (i = index - 1; i >= 0; i--) { 473 if (filter_event_cmp(&events[i], &event_select)) 474 break; 475 476 if (is_filter_entry_match(events[i], umask)) 477 return true; 478 } 479 480 return false; 481 } 482 483 static bool is_gp_event_allowed(struct kvm_x86_pmu_event_filter *f, 484 u64 eventsel) 485 { 486 if (filter_contains_match(f->includes, f->nr_includes, eventsel) && 487 !filter_contains_match(f->excludes, f->nr_excludes, eventsel)) 488 return f->action == KVM_PMU_EVENT_ALLOW; 489 490 return f->action == KVM_PMU_EVENT_DENY; 491 } 492 493 static bool is_fixed_event_allowed(struct kvm_x86_pmu_event_filter *filter, 494 int idx) 495 { 496 int fixed_idx = idx - KVM_FIXED_PMC_BASE_IDX; 497 498 if (filter->action == KVM_PMU_EVENT_DENY && 499 test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) 500 return false; 501 if (filter->action == KVM_PMU_EVENT_ALLOW && 502 !test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap)) 503 return false; 504 505 return true; 506 } 507 508 static bool pmc_is_event_allowed(struct kvm_pmc *pmc) 509 { 510 struct kvm_x86_pmu_event_filter *filter; 511 struct kvm *kvm = pmc->vcpu->kvm; 512 513 filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu); 514 if (!filter) 515 return true; 516 517 if (pmc_is_gp(pmc)) 518 return is_gp_event_allowed(filter, pmc->eventsel); 519 520 return is_fixed_event_allowed(filter, pmc->idx); 521 } 522 523 static void kvm_mediated_pmu_refresh_event_filter(struct kvm_pmc *pmc) 524 { 525 bool allowed = pmc_is_event_allowed(pmc); 526 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 527 528 if (pmc_is_gp(pmc)) { 529 pmc->eventsel_hw &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 530 if (allowed) 531 pmc->eventsel_hw |= pmc->eventsel & 532 ARCH_PERFMON_EVENTSEL_ENABLE; 533 } else { 534 u64 mask = intel_fixed_bits_by_idx(pmc->idx - KVM_FIXED_PMC_BASE_IDX, 0xf); 535 536 pmu->fixed_ctr_ctrl_hw &= ~mask; 537 if (allowed) 538 pmu->fixed_ctr_ctrl_hw |= pmu->fixed_ctr_ctrl & mask; 539 } 540 } 541 542 static int reprogram_counter(struct kvm_pmc *pmc) 543 { 544 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 545 u64 eventsel = pmc->eventsel; 546 u64 new_config = eventsel; 547 bool emulate_overflow; 548 u8 fixed_ctr_ctrl; 549 550 if (kvm_vcpu_has_mediated_pmu(pmu_to_vcpu(pmu))) { 551 kvm_mediated_pmu_refresh_event_filter(pmc); 552 return 0; 553 } 554 555 emulate_overflow = pmc_pause_counter(pmc); 556 557 if (!pmc_is_globally_enabled(pmc) || !pmc_is_locally_enabled(pmc) || 558 !pmc_is_event_allowed(pmc)) 559 return 0; 560 561 if (emulate_overflow) 562 __kvm_perf_overflow(pmc, false); 563 564 if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) 565 printk_once("kvm pmu: pin control bit is ignored\n"); 566 567 if (pmc_is_fixed(pmc)) { 568 fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, 569 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 570 if (fixed_ctr_ctrl & INTEL_FIXED_0_KERNEL) 571 eventsel |= ARCH_PERFMON_EVENTSEL_OS; 572 if (fixed_ctr_ctrl & INTEL_FIXED_0_USER) 573 eventsel |= ARCH_PERFMON_EVENTSEL_USR; 574 if (fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI) 575 eventsel |= ARCH_PERFMON_EVENTSEL_INT; 576 new_config = (u64)fixed_ctr_ctrl; 577 } 578 579 if (pmc->current_config == new_config && pmc_resume_counter(pmc)) 580 return 0; 581 582 pmc_release_perf_event(pmc); 583 584 pmc->current_config = new_config; 585 586 return pmc_reprogram_counter(pmc, PERF_TYPE_RAW, 587 (eventsel & pmu->raw_event_mask), 588 !(eventsel & ARCH_PERFMON_EVENTSEL_USR), 589 !(eventsel & ARCH_PERFMON_EVENTSEL_OS), 590 eventsel & ARCH_PERFMON_EVENTSEL_INT); 591 } 592 593 static bool pmc_is_event_match(struct kvm_pmc *pmc, u64 eventsel) 594 { 595 /* 596 * Ignore checks for edge detect (all events currently emulated by KVM 597 * are always rising edges), pin control (unsupported by modern CPUs), 598 * and counter mask and its invert flag (KVM doesn't emulate multiple 599 * events in a single clock cycle). 600 * 601 * Note, the uppermost nibble of AMD's mask overlaps Intel's IN_TX (bit 602 * 32) and IN_TXCP (bit 33), as well as two reserved bits (bits 35:34). 603 * Checking the "in HLE/RTM transaction" flags is correct as the vCPU 604 * can't be in a transaction if KVM is emulating an instruction. 605 * 606 * Checking the reserved bits might be wrong if they are defined in the 607 * future, but so could ignoring them, so do the simple thing for now. 608 */ 609 return !((pmc->eventsel ^ eventsel) & AMD64_RAW_EVENT_MASK_NB); 610 } 611 612 void kvm_pmu_recalc_pmc_emulation(struct kvm_pmu *pmu, struct kvm_pmc *pmc) 613 { 614 bitmap_clear(pmu->pmc_counting_instructions, pmc->idx, 1); 615 bitmap_clear(pmu->pmc_counting_branches, pmc->idx, 1); 616 617 /* 618 * Do NOT consult the PMU event filters, as the filters must be checked 619 * at the time of emulation to ensure KVM uses fresh information, e.g. 620 * omitting a PMC from a bitmap could result in a missed event if the 621 * filter is changed to allow counting the event. 622 */ 623 if (!pmc_is_locally_enabled(pmc)) 624 return; 625 626 if (pmc_is_event_match(pmc, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED)) 627 bitmap_set(pmu->pmc_counting_instructions, pmc->idx, 1); 628 629 if (pmc_is_event_match(pmc, kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED)) 630 bitmap_set(pmu->pmc_counting_branches, pmc->idx, 1); 631 } 632 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_recalc_pmc_emulation); 633 634 void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) 635 { 636 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); 637 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 638 struct kvm_pmc *pmc; 639 int bit; 640 641 bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); 642 643 /* 644 * The reprogramming bitmap can be written asynchronously by something 645 * other than the task that holds vcpu->mutex, take care to clear only 646 * the bits that will actually processed. 647 */ 648 BUILD_BUG_ON(sizeof(bitmap) != sizeof(atomic64_t)); 649 atomic64_andnot(*(s64 *)bitmap, &pmu->__reprogram_pmi); 650 651 kvm_for_each_pmc(pmu, pmc, bit, bitmap) { 652 /* 653 * If reprogramming fails, e.g. due to contention, re-set the 654 * regprogram bit set, i.e. opportunistically try again on the 655 * next PMU refresh. Don't make a new request as doing so can 656 * stall the guest if reprogramming repeatedly fails. 657 */ 658 if (reprogram_counter(pmc)) 659 set_bit(pmc->idx, pmu->reprogram_pmi); 660 } 661 662 /* 663 * Release unused perf_events if the corresponding guest MSRs weren't 664 * accessed during the last vCPU time slice (need_cleanup is set when 665 * the vCPU is scheduled back in). 666 */ 667 if (unlikely(pmu->need_cleanup)) 668 kvm_pmu_cleanup(vcpu); 669 670 kvm_for_each_pmc(pmu, pmc, bit, bitmap) 671 kvm_pmu_recalc_pmc_emulation(pmu, pmc); 672 } 673 674 int kvm_pmu_check_rdpmc_early(struct kvm_vcpu *vcpu, unsigned int idx) 675 { 676 /* 677 * On Intel, VMX interception has priority over RDPMC exceptions that 678 * aren't already handled by the emulator, i.e. there are no additional 679 * check needed for Intel PMUs. 680 * 681 * On AMD, _all_ exceptions on RDPMC have priority over SVM intercepts, 682 * i.e. an invalid PMC results in a #GP, not #VMEXIT. 683 */ 684 if (!kvm_pmu_ops.check_rdpmc_early) 685 return 0; 686 687 return kvm_pmu_call(check_rdpmc_early)(vcpu, idx); 688 } 689 690 bool is_vmware_backdoor_pmc(u32 pmc_idx) 691 { 692 switch (pmc_idx) { 693 case VMWARE_BACKDOOR_PMC_HOST_TSC: 694 case VMWARE_BACKDOOR_PMC_REAL_TIME: 695 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 696 return true; 697 } 698 return false; 699 } 700 701 static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 702 { 703 u64 ctr_val; 704 705 switch (idx) { 706 case VMWARE_BACKDOOR_PMC_HOST_TSC: 707 ctr_val = rdtsc(); 708 break; 709 case VMWARE_BACKDOOR_PMC_REAL_TIME: 710 ctr_val = ktime_get_boottime_ns(); 711 break; 712 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 713 ctr_val = ktime_get_boottime_ns() + 714 vcpu->kvm->arch.kvmclock_offset; 715 break; 716 default: 717 return 1; 718 } 719 720 *data = ctr_val; 721 return 0; 722 } 723 724 int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 725 { 726 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 727 struct kvm_pmc *pmc; 728 u64 mask = ~0ull; 729 730 if (!pmu->version) 731 return 1; 732 733 if (is_vmware_backdoor_pmc(idx)) 734 return kvm_pmu_rdpmc_vmware(vcpu, idx, data); 735 736 pmc = kvm_pmu_call(rdpmc_ecx_to_pmc)(vcpu, idx, &mask); 737 if (!pmc) 738 return 1; 739 740 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCE) && 741 (kvm_x86_call(get_cpl)(vcpu) != 0) && 742 kvm_is_cr0_bit_set(vcpu, X86_CR0_PE)) 743 return 1; 744 745 *data = pmc_read_counter(pmc) & mask; 746 return 0; 747 } 748 749 static bool kvm_need_any_pmc_intercept(struct kvm_vcpu *vcpu) 750 { 751 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 752 753 if (!kvm_vcpu_has_mediated_pmu(vcpu)) 754 return true; 755 756 /* 757 * Note! Check *host* PMU capabilities, not KVM's PMU capabilities, as 758 * KVM's capabilities are constrained based on KVM support, i.e. KVM's 759 * capabilities themselves may be a subset of hardware capabilities. 760 */ 761 return pmu->nr_arch_gp_counters != kvm_host_pmu.num_counters_gp || 762 pmu->nr_arch_fixed_counters != kvm_host_pmu.num_counters_fixed; 763 } 764 765 bool kvm_need_perf_global_ctrl_intercept(struct kvm_vcpu *vcpu) 766 { 767 return kvm_need_any_pmc_intercept(vcpu) || 768 !kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); 769 } 770 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_perf_global_ctrl_intercept); 771 772 bool kvm_need_rdpmc_intercept(struct kvm_vcpu *vcpu) 773 { 774 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 775 776 /* 777 * VMware allows access to these Pseduo-PMCs even when read via RDPMC 778 * in Ring3 when CR4.PCE=0. 779 */ 780 if (enable_vmware_backdoor) 781 return true; 782 783 return kvm_need_any_pmc_intercept(vcpu) || 784 pmu->counter_bitmask[KVM_PMC_GP] != (BIT_ULL(kvm_host_pmu.bit_width_gp) - 1) || 785 pmu->counter_bitmask[KVM_PMC_FIXED] != (BIT_ULL(kvm_host_pmu.bit_width_fixed) - 1); 786 } 787 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_need_rdpmc_intercept); 788 789 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) 790 { 791 if (lapic_in_kernel(vcpu)) { 792 kvm_pmu_call(deliver_pmi)(vcpu); 793 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); 794 } 795 } 796 797 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 798 { 799 switch (msr) { 800 case MSR_CORE_PERF_GLOBAL_STATUS: 801 case MSR_CORE_PERF_GLOBAL_CTRL: 802 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 803 return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); 804 default: 805 break; 806 } 807 return kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr) || 808 kvm_pmu_call(is_valid_msr)(vcpu, msr); 809 } 810 811 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) 812 { 813 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 814 struct kvm_pmc *pmc = kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr); 815 816 if (pmc) 817 __set_bit(pmc->idx, pmu->pmc_in_use); 818 } 819 820 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 821 { 822 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 823 u32 msr = msr_info->index; 824 825 switch (msr) { 826 case MSR_CORE_PERF_GLOBAL_STATUS: 827 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: 828 msr_info->data = pmu->global_status; 829 break; 830 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: 831 case MSR_CORE_PERF_GLOBAL_CTRL: 832 msr_info->data = pmu->global_ctrl; 833 break; 834 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: 835 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: 836 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 837 msr_info->data = 0; 838 break; 839 default: 840 return kvm_pmu_call(get_msr)(vcpu, msr_info); 841 } 842 843 return 0; 844 } 845 846 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 847 { 848 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 849 u32 msr = msr_info->index; 850 u64 data = msr_info->data; 851 u64 diff; 852 853 /* 854 * Note, AMD ignores writes to reserved bits and read-only PMU MSRs, 855 * whereas Intel generates #GP on attempts to write reserved/RO MSRs. 856 */ 857 switch (msr) { 858 case MSR_CORE_PERF_GLOBAL_STATUS: 859 if (!msr_info->host_initiated) 860 return 1; /* RO MSR */ 861 fallthrough; 862 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: 863 /* Per PPR, Read-only MSR. Writes are ignored. */ 864 if (!msr_info->host_initiated) 865 break; 866 867 if (data & pmu->global_status_rsvd) 868 return 1; 869 870 pmu->global_status = data; 871 break; 872 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: 873 data &= ~pmu->global_ctrl_rsvd; 874 fallthrough; 875 case MSR_CORE_PERF_GLOBAL_CTRL: 876 if (!kvm_valid_perf_global_ctrl(pmu, data)) 877 return 1; 878 879 if (pmu->global_ctrl != data) { 880 diff = pmu->global_ctrl ^ data; 881 pmu->global_ctrl = data; 882 reprogram_counters(pmu, diff); 883 } 884 /* 885 * Unconditionally forward writes to vendor code, i.e. to the 886 * VMC{B,S}, as pmu->global_ctrl is per-VCPU, not per-VMC{B,S}. 887 */ 888 if (kvm_vcpu_has_mediated_pmu(vcpu)) 889 kvm_pmu_call(write_global_ctrl)(data); 890 break; 891 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 892 /* 893 * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in 894 * GLOBAL_STATUS, and so the set of reserved bits is the same. 895 */ 896 if (data & pmu->global_status_rsvd) 897 return 1; 898 fallthrough; 899 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: 900 if (!msr_info->host_initiated) 901 pmu->global_status &= ~data; 902 break; 903 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: 904 if (!msr_info->host_initiated) 905 pmu->global_status |= data & ~pmu->global_status_rsvd; 906 break; 907 default: 908 kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); 909 return kvm_pmu_call(set_msr)(vcpu, msr_info); 910 } 911 912 return 0; 913 } 914 915 static void kvm_pmu_reset(struct kvm_vcpu *vcpu) 916 { 917 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 918 struct kvm_pmc *pmc; 919 int i; 920 921 pmu->need_cleanup = false; 922 923 bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX); 924 925 kvm_for_each_pmc(pmu, pmc, i, pmu->all_valid_pmc_idx) { 926 pmc_stop_counter(pmc); 927 pmc->counter = 0; 928 pmc->emulated_counter = 0; 929 930 if (pmc_is_gp(pmc)) { 931 pmc->eventsel = 0; 932 pmc->eventsel_hw = 0; 933 } 934 } 935 936 pmu->fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw = 0; 937 pmu->global_ctrl = pmu->global_status = 0; 938 939 kvm_pmu_call(reset)(vcpu); 940 } 941 942 943 /* 944 * Refresh the PMU configuration for the vCPU, e.g. if userspace changes CPUID 945 * and/or PERF_CAPABILITIES. 946 */ 947 void kvm_pmu_refresh(struct kvm_vcpu *vcpu) 948 { 949 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 950 951 if (KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm)) 952 return; 953 954 /* 955 * Stop/release all existing counters/events before realizing the new 956 * vPMU model. 957 */ 958 kvm_pmu_reset(vcpu); 959 960 pmu->version = 0; 961 pmu->nr_arch_gp_counters = 0; 962 pmu->nr_arch_fixed_counters = 0; 963 pmu->counter_bitmask[KVM_PMC_GP] = 0; 964 pmu->counter_bitmask[KVM_PMC_FIXED] = 0; 965 pmu->reserved_bits = 0xffffffff00200000ull; 966 pmu->raw_event_mask = X86_RAW_EVENT_MASK; 967 pmu->global_ctrl_rsvd = ~0ull; 968 pmu->global_status_rsvd = ~0ull; 969 pmu->fixed_ctr_ctrl_rsvd = ~0ull; 970 pmu->pebs_enable_rsvd = ~0ull; 971 pmu->pebs_data_cfg_rsvd = ~0ull; 972 bitmap_zero(pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX); 973 974 if (!vcpu->kvm->arch.enable_pmu) 975 return; 976 977 kvm_pmu_call(refresh)(vcpu); 978 979 /* 980 * At RESET, both Intel and AMD CPUs set all enable bits for general 981 * purpose counters in IA32_PERF_GLOBAL_CTRL (so that software that 982 * was written for v1 PMUs don't unknowingly leave GP counters disabled 983 * in the global controls). Emulate that behavior when refreshing the 984 * PMU so that userspace doesn't need to manually set PERF_GLOBAL_CTRL. 985 */ 986 if (pmu->nr_arch_gp_counters && 987 (kvm_pmu_has_perf_global_ctrl(pmu) || kvm_vcpu_has_mediated_pmu(vcpu))) 988 pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0); 989 990 if (kvm_vcpu_has_mediated_pmu(vcpu)) 991 kvm_pmu_call(write_global_ctrl)(pmu->global_ctrl); 992 993 bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters); 994 bitmap_set(pmu->all_valid_pmc_idx, KVM_FIXED_PMC_BASE_IDX, 995 pmu->nr_arch_fixed_counters); 996 } 997 998 void kvm_pmu_init(struct kvm_vcpu *vcpu) 999 { 1000 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1001 1002 memset(pmu, 0, sizeof(*pmu)); 1003 kvm_pmu_call(init)(vcpu); 1004 } 1005 1006 /* Release perf_events for vPMCs that have been unused for a full time slice. */ 1007 void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) 1008 { 1009 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1010 struct kvm_pmc *pmc = NULL; 1011 DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX); 1012 int i; 1013 1014 pmu->need_cleanup = false; 1015 1016 bitmap_andnot(bitmask, pmu->all_valid_pmc_idx, 1017 pmu->pmc_in_use, X86_PMC_IDX_MAX); 1018 1019 kvm_for_each_pmc(pmu, pmc, i, bitmask) { 1020 if (pmc->perf_event && !pmc_is_locally_enabled(pmc)) 1021 pmc_stop_counter(pmc); 1022 } 1023 1024 kvm_pmu_call(cleanup)(vcpu); 1025 1026 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); 1027 } 1028 1029 void kvm_pmu_destroy(struct kvm_vcpu *vcpu) 1030 { 1031 kvm_pmu_reset(vcpu); 1032 } 1033 1034 static bool pmc_is_pmi_enabled(struct kvm_pmc *pmc) 1035 { 1036 u8 fixed_ctr_ctrl; 1037 1038 if (pmc_is_gp(pmc)) 1039 return pmc->eventsel & ARCH_PERFMON_EVENTSEL_INT; 1040 1041 fixed_ctr_ctrl = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl, 1042 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 1043 return fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI; 1044 } 1045 1046 static void kvm_pmu_incr_counter(struct kvm_pmc *pmc) 1047 { 1048 struct kvm_vcpu *vcpu = pmc->vcpu; 1049 1050 /* 1051 * For perf-based PMUs, accumulate software-emulated events separately 1052 * from pmc->counter, as pmc->counter is offset by the count of the 1053 * associated perf event. Request reprogramming, which will consult 1054 * both emulated and hardware-generated events to detect overflow. 1055 */ 1056 if (!kvm_vcpu_has_mediated_pmu(vcpu)) { 1057 pmc->emulated_counter++; 1058 kvm_pmu_request_counter_reprogram(pmc); 1059 return; 1060 } 1061 1062 /* 1063 * For mediated PMUs, pmc->counter is updated when the vCPU's PMU is 1064 * put, and will be loaded into hardware when the PMU is loaded. Simply 1065 * increment the counter and signal overflow if it wraps to zero. 1066 */ 1067 pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc); 1068 if (!pmc->counter) { 1069 pmc_to_pmu(pmc)->global_status |= BIT_ULL(pmc->idx); 1070 if (pmc_is_pmi_enabled(pmc)) 1071 kvm_make_request(KVM_REQ_PMI, vcpu); 1072 } 1073 } 1074 1075 static inline bool cpl_is_matched(struct kvm_pmc *pmc) 1076 { 1077 bool select_os, select_user; 1078 u64 config; 1079 1080 if (pmc_is_gp(pmc)) { 1081 config = pmc->eventsel; 1082 select_os = config & ARCH_PERFMON_EVENTSEL_OS; 1083 select_user = config & ARCH_PERFMON_EVENTSEL_USR; 1084 } else { 1085 config = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl, 1086 pmc->idx - KVM_FIXED_PMC_BASE_IDX); 1087 select_os = config & INTEL_FIXED_0_KERNEL; 1088 select_user = config & INTEL_FIXED_0_USER; 1089 } 1090 1091 /* 1092 * Skip the CPL lookup, which isn't free on Intel, if the result will 1093 * be the same regardless of the CPL. 1094 */ 1095 if (select_os == select_user) 1096 return select_os; 1097 1098 return (kvm_x86_call(get_cpl)(pmc->vcpu) == 0) ? select_os : 1099 select_user; 1100 } 1101 1102 static void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, 1103 const unsigned long *event_pmcs) 1104 { 1105 DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); 1106 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1107 struct kvm_pmc *pmc; 1108 int i, idx; 1109 1110 BUILD_BUG_ON(sizeof(pmu->global_ctrl) * BITS_PER_BYTE != X86_PMC_IDX_MAX); 1111 1112 if (bitmap_empty(event_pmcs, X86_PMC_IDX_MAX)) 1113 return; 1114 1115 if (!kvm_pmu_has_perf_global_ctrl(pmu)) 1116 bitmap_copy(bitmap, event_pmcs, X86_PMC_IDX_MAX); 1117 else if (!bitmap_and(bitmap, event_pmcs, 1118 (unsigned long *)&pmu->global_ctrl, X86_PMC_IDX_MAX)) 1119 return; 1120 1121 idx = srcu_read_lock(&vcpu->kvm->srcu); 1122 kvm_for_each_pmc(pmu, pmc, i, bitmap) { 1123 if (!pmc_is_event_allowed(pmc) || !cpl_is_matched(pmc)) 1124 continue; 1125 1126 kvm_pmu_incr_counter(pmc); 1127 } 1128 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1129 } 1130 1131 void kvm_pmu_instruction_retired(struct kvm_vcpu *vcpu) 1132 { 1133 kvm_pmu_trigger_event(vcpu, vcpu_to_pmu(vcpu)->pmc_counting_instructions); 1134 } 1135 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_instruction_retired); 1136 1137 void kvm_pmu_branch_retired(struct kvm_vcpu *vcpu) 1138 { 1139 kvm_pmu_trigger_event(vcpu, vcpu_to_pmu(vcpu)->pmc_counting_branches); 1140 } 1141 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_branch_retired); 1142 1143 static bool is_masked_filter_valid(const struct kvm_x86_pmu_event_filter *filter) 1144 { 1145 u64 mask = kvm_pmu_ops.EVENTSEL_EVENT | 1146 KVM_PMU_MASKED_ENTRY_UMASK_MASK | 1147 KVM_PMU_MASKED_ENTRY_UMASK_MATCH | 1148 KVM_PMU_MASKED_ENTRY_EXCLUDE; 1149 int i; 1150 1151 for (i = 0; i < filter->nevents; i++) { 1152 if (filter->events[i] & ~mask) 1153 return false; 1154 } 1155 1156 return true; 1157 } 1158 1159 static void convert_to_masked_filter(struct kvm_x86_pmu_event_filter *filter) 1160 { 1161 int i, j; 1162 1163 for (i = 0, j = 0; i < filter->nevents; i++) { 1164 /* 1165 * Skip events that are impossible to match against a guest 1166 * event. When filtering, only the event select + unit mask 1167 * of the guest event is used. To maintain backwards 1168 * compatibility, impossible filters can't be rejected :-( 1169 */ 1170 if (filter->events[i] & ~(kvm_pmu_ops.EVENTSEL_EVENT | 1171 ARCH_PERFMON_EVENTSEL_UMASK)) 1172 continue; 1173 /* 1174 * Convert userspace events to a common in-kernel event so 1175 * only one code path is needed to support both events. For 1176 * the in-kernel events use masked events because they are 1177 * flexible enough to handle both cases. To convert to masked 1178 * events all that's needed is to add an "all ones" umask_mask, 1179 * (unmasked filter events don't support EXCLUDE). 1180 */ 1181 filter->events[j++] = filter->events[i] | 1182 (0xFFULL << KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT); 1183 } 1184 1185 filter->nevents = j; 1186 } 1187 1188 static int prepare_filter_lists(struct kvm_x86_pmu_event_filter *filter) 1189 { 1190 int i; 1191 1192 if (!(filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS)) 1193 convert_to_masked_filter(filter); 1194 else if (!is_masked_filter_valid(filter)) 1195 return -EINVAL; 1196 1197 /* 1198 * Sort entries by event select and includes vs. excludes so that all 1199 * entries for a given event select can be processed efficiently during 1200 * filtering. The EXCLUDE flag uses a more significant bit than the 1201 * event select, and so the sorted list is also effectively split into 1202 * includes and excludes sub-lists. 1203 */ 1204 sort(&filter->events, filter->nevents, sizeof(filter->events[0]), 1205 filter_sort_cmp, NULL); 1206 1207 i = filter->nevents; 1208 /* Find the first EXCLUDE event (only supported for masked events). */ 1209 if (filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS) { 1210 for (i = 0; i < filter->nevents; i++) { 1211 if (filter->events[i] & KVM_PMU_MASKED_ENTRY_EXCLUDE) 1212 break; 1213 } 1214 } 1215 1216 filter->nr_includes = i; 1217 filter->nr_excludes = filter->nevents - filter->nr_includes; 1218 filter->includes = filter->events; 1219 filter->excludes = filter->events + filter->nr_includes; 1220 1221 return 0; 1222 } 1223 1224 int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp) 1225 { 1226 struct kvm_pmu_event_filter __user *user_filter = argp; 1227 struct kvm_x86_pmu_event_filter *filter; 1228 struct kvm_pmu_event_filter tmp; 1229 struct kvm_vcpu *vcpu; 1230 unsigned long i; 1231 size_t size; 1232 int r; 1233 1234 if (copy_from_user(&tmp, user_filter, sizeof(tmp))) 1235 return -EFAULT; 1236 1237 if (tmp.action != KVM_PMU_EVENT_ALLOW && 1238 tmp.action != KVM_PMU_EVENT_DENY) 1239 return -EINVAL; 1240 1241 if (tmp.flags & ~KVM_PMU_EVENT_FLAGS_VALID_MASK) 1242 return -EINVAL; 1243 1244 if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS) 1245 return -E2BIG; 1246 1247 size = struct_size(filter, events, tmp.nevents); 1248 filter = kzalloc(size, GFP_KERNEL_ACCOUNT); 1249 if (!filter) 1250 return -ENOMEM; 1251 1252 filter->action = tmp.action; 1253 filter->nevents = tmp.nevents; 1254 filter->fixed_counter_bitmap = tmp.fixed_counter_bitmap; 1255 filter->flags = tmp.flags; 1256 1257 r = -EFAULT; 1258 if (copy_from_user(filter->events, user_filter->events, 1259 sizeof(filter->events[0]) * filter->nevents)) 1260 goto cleanup; 1261 1262 r = prepare_filter_lists(filter); 1263 if (r) 1264 goto cleanup; 1265 1266 mutex_lock(&kvm->lock); 1267 filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter, 1268 mutex_is_locked(&kvm->lock)); 1269 mutex_unlock(&kvm->lock); 1270 synchronize_srcu_expedited(&kvm->srcu); 1271 1272 BUILD_BUG_ON(sizeof(((struct kvm_pmu *)0)->reprogram_pmi) > 1273 sizeof(((struct kvm_pmu *)0)->__reprogram_pmi)); 1274 1275 kvm_for_each_vcpu(i, vcpu, kvm) 1276 atomic64_set(&vcpu_to_pmu(vcpu)->__reprogram_pmi, -1ull); 1277 1278 kvm_make_all_cpus_request(kvm, KVM_REQ_PMU); 1279 1280 r = 0; 1281 cleanup: 1282 kfree(filter); 1283 return r; 1284 } 1285 1286 static __always_inline u32 fixed_counter_msr(u32 idx) 1287 { 1288 return kvm_pmu_ops.FIXED_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1289 } 1290 1291 static __always_inline u32 gp_counter_msr(u32 idx) 1292 { 1293 return kvm_pmu_ops.GP_COUNTER_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1294 } 1295 1296 static __always_inline u32 gp_eventsel_msr(u32 idx) 1297 { 1298 return kvm_pmu_ops.GP_EVENTSEL_BASE + idx * kvm_pmu_ops.MSR_STRIDE; 1299 } 1300 1301 static void kvm_pmu_load_guest_pmcs(struct kvm_vcpu *vcpu) 1302 { 1303 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1304 struct kvm_pmc *pmc; 1305 u32 i; 1306 1307 /* 1308 * No need to zero out unexposed GP/fixed counters/selectors since RDPMC 1309 * is intercepted if hardware has counters that aren't visible to the 1310 * guest (KVM will inject #GP as appropriate). 1311 */ 1312 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 1313 pmc = &pmu->gp_counters[i]; 1314 1315 if (pmc->counter != rdpmc(i)) 1316 wrmsrl(gp_counter_msr(i), pmc->counter); 1317 wrmsrl(gp_eventsel_msr(i), pmc->eventsel_hw); 1318 } 1319 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 1320 pmc = &pmu->fixed_counters[i]; 1321 1322 if (pmc->counter != rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i)) 1323 wrmsrl(fixed_counter_msr(i), pmc->counter); 1324 } 1325 } 1326 1327 void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu) 1328 { 1329 if (!kvm_vcpu_has_mediated_pmu(vcpu) || 1330 KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 1331 return; 1332 1333 lockdep_assert_irqs_disabled(); 1334 1335 perf_load_guest_context(); 1336 1337 /* 1338 * Explicitly clear PERF_GLOBAL_CTRL, as "loading" the guest's context 1339 * disables all individual counters (if any were enabled), but doesn't 1340 * globally disable the entire PMU. Loading event selectors and PMCs 1341 * with guest values while PERF_GLOBAL_CTRL is non-zero will generate 1342 * unexpected events and PMIs. 1343 * 1344 * VMX will enable/disable counters at VM-Enter/VM-Exit by atomically 1345 * loading PERF_GLOBAL_CONTROL. SVM effectively performs the switch by 1346 * configuring all events to be GUEST_ONLY. Clear PERF_GLOBAL_CONTROL 1347 * even for SVM to minimize the damage if a perf event is left enabled, 1348 * and to ensure a consistent starting state. 1349 */ 1350 wrmsrq(kvm_pmu_ops.PERF_GLOBAL_CTRL, 0); 1351 1352 perf_load_guest_lvtpc(kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVTPC)); 1353 1354 kvm_pmu_load_guest_pmcs(vcpu); 1355 1356 kvm_pmu_call(mediated_load)(vcpu); 1357 } 1358 1359 static void kvm_pmu_put_guest_pmcs(struct kvm_vcpu *vcpu) 1360 { 1361 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 1362 struct kvm_pmc *pmc; 1363 u32 i; 1364 1365 /* 1366 * Clear selectors and counters to ensure hardware doesn't count using 1367 * guest controls when the host (perf) restores its state. 1368 */ 1369 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 1370 pmc = &pmu->gp_counters[i]; 1371 1372 pmc->counter = rdpmc(i); 1373 if (pmc->counter) 1374 wrmsrq(gp_counter_msr(i), 0); 1375 if (pmc->eventsel_hw) 1376 wrmsrq(gp_eventsel_msr(i), 0); 1377 } 1378 1379 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 1380 pmc = &pmu->fixed_counters[i]; 1381 1382 pmc->counter = rdpmc(INTEL_PMC_FIXED_RDPMC_BASE | i); 1383 if (pmc->counter) 1384 wrmsrq(fixed_counter_msr(i), 0); 1385 } 1386 } 1387 1388 void kvm_mediated_pmu_put(struct kvm_vcpu *vcpu) 1389 { 1390 if (!kvm_vcpu_has_mediated_pmu(vcpu) || 1391 KVM_BUG_ON(!lapic_in_kernel(vcpu), vcpu->kvm)) 1392 return; 1393 1394 lockdep_assert_irqs_disabled(); 1395 1396 /* 1397 * Defer handling of PERF_GLOBAL_CTRL to vendor code. On Intel, it's 1398 * atomically cleared on VM-Exit, i.e. doesn't need to be clear here. 1399 */ 1400 kvm_pmu_call(mediated_put)(vcpu); 1401 1402 kvm_pmu_put_guest_pmcs(vcpu); 1403 1404 perf_put_guest_lvtpc(); 1405 1406 perf_put_guest_context(); 1407 } 1408