1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KVM PMU support for Intel CPUs 4 * 5 * Copyright 2011 Red Hat, Inc. and/or its affiliates. 6 * 7 * Authors: 8 * Avi Kivity <avi@redhat.com> 9 * Gleb Natapov <gleb@redhat.com> 10 */ 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/types.h> 14 #include <linux/kvm_host.h> 15 #include <linux/perf_event.h> 16 #include <asm/msr.h> 17 #include <asm/perf_event.h> 18 #include <asm/cpuid/api.h> 19 #include "x86.h" 20 #include "cpuid.h" 21 #include "lapic.h" 22 #include "nested.h" 23 #include "pmu.h" 24 #include "tdx.h" 25 26 /* 27 * Perf's "BASE" is wildly misleading, architectural PMUs use bits 31:16 of ECX 28 * to encode the "type" of counter to read, i.e. this is not a "base". And to 29 * further confuse things, non-architectural PMUs use bit 31 as a flag for 30 * "fast" reads, whereas the "type" is an explicit value. 31 */ 32 #define INTEL_RDPMC_GP 0 33 #define INTEL_RDPMC_FIXED INTEL_PMC_FIXED_RDPMC_BASE 34 35 #define INTEL_RDPMC_TYPE_MASK GENMASK(31, 16) 36 #define INTEL_RDPMC_INDEX_MASK GENMASK(15, 0) 37 38 #define MSR_PMC_FULL_WIDTH_BIT (MSR_IA32_PMC0 - MSR_IA32_PERFCTR0) 39 40 static struct lbr_desc *vcpu_to_lbr_desc(struct kvm_vcpu *vcpu) 41 { 42 if (is_td_vcpu(vcpu)) 43 return NULL; 44 45 return &to_vmx(vcpu)->lbr_desc; 46 } 47 48 static struct x86_pmu_lbr *vcpu_to_lbr_records(struct kvm_vcpu *vcpu) 49 { 50 if (is_td_vcpu(vcpu)) 51 return NULL; 52 53 return &to_vmx(vcpu)->lbr_desc.records; 54 } 55 56 #pragma GCC poison to_vmx 57 58 static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data) 59 { 60 /* 61 * Compare against the value the mediated PMU shoves into hardware, not 62 * the guest's desired value. For the emulated PMU (proxied via perf), 63 * they are one and the same (fixed_ctr_ctrl_hw isn't used other than 64 * here). For the mediated PMU, KVM needs to reprogram the actual MSR, 65 * and so needs to react to potential changes in the value shoved into 66 * hardware, e.g. to ensure the event filter is enforced. 67 */ 68 u64 old_fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw; 69 struct kvm_pmc *pmc; 70 int i; 71 72 pmu->fixed_ctr_ctrl = data; 73 pmu->fixed_ctr_ctrl_hw = data; 74 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 75 u8 new_ctrl = fixed_ctrl_field(data, i); 76 u8 old_ctrl = fixed_ctrl_field(old_fixed_ctr_ctrl, i); 77 78 if (old_ctrl == new_ctrl) 79 continue; 80 81 pmc = get_fixed_pmc(pmu, MSR_CORE_PERF_FIXED_CTR0 + i); 82 83 __set_bit(KVM_FIXED_PMC_BASE_IDX + i, pmu->pmc_in_use); 84 kvm_pmu_request_counter_reprogram(pmc); 85 } 86 } 87 88 static struct kvm_pmc *intel_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu, 89 unsigned int idx, u64 *mask) 90 { 91 unsigned int type = idx & INTEL_RDPMC_TYPE_MASK; 92 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 93 struct kvm_pmc *counters; 94 unsigned int num_counters; 95 u64 bitmask; 96 97 /* 98 * The encoding of ECX for RDPMC is different for architectural versus 99 * non-architecturals PMUs (PMUs with version '0'). For architectural 100 * PMUs, bits 31:16 specify the PMC type and bits 15:0 specify the PMC 101 * index. For non-architectural PMUs, bit 31 is a "fast" flag, and 102 * bits 30:0 specify the PMC index. 103 * 104 * Yell and reject attempts to read PMCs for a non-architectural PMU, 105 * as KVM doesn't support such PMUs. 106 */ 107 if (WARN_ON_ONCE(!pmu->version)) 108 return NULL; 109 110 /* 111 * General Purpose (GP) PMCs are supported on all PMUs, and fixed PMCs 112 * are supported on all architectural PMUs, i.e. on all virtual PMUs 113 * supported by KVM. Note, KVM only emulates fixed PMCs for PMU v2+, 114 * but the type itself is still valid, i.e. let RDPMC fail due to 115 * accessing a non-existent counter. Reject attempts to read all other 116 * types, which are unknown/unsupported. 117 */ 118 switch (type) { 119 case INTEL_RDPMC_FIXED: 120 counters = pmu->fixed_counters; 121 num_counters = pmu->nr_arch_fixed_counters; 122 bitmask = pmu->counter_bitmask[KVM_PMC_FIXED]; 123 break; 124 case INTEL_RDPMC_GP: 125 counters = pmu->gp_counters; 126 num_counters = pmu->nr_arch_gp_counters; 127 bitmask = pmu->counter_bitmask[KVM_PMC_GP]; 128 break; 129 default: 130 return NULL; 131 } 132 133 idx &= INTEL_RDPMC_INDEX_MASK; 134 if (idx >= num_counters) 135 return NULL; 136 137 *mask &= bitmask; 138 return &counters[array_index_nospec(idx, num_counters)]; 139 } 140 141 static inline struct kvm_pmc *get_fw_gp_pmc(struct kvm_pmu *pmu, u32 msr) 142 { 143 if (!fw_writes_is_enabled(pmu_to_vcpu(pmu))) 144 return NULL; 145 146 return get_gp_pmc(pmu, msr, MSR_IA32_PMC0); 147 } 148 149 static bool intel_pmu_lbr_is_compatible(struct kvm_vcpu *vcpu) 150 { 151 if (is_td_vcpu(vcpu)) 152 return false; 153 154 return cpuid_model_is_consistent(vcpu); 155 } 156 157 bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu) 158 { 159 if (is_td_vcpu(vcpu)) 160 return false; 161 162 return !!vcpu_to_lbr_records(vcpu)->nr; 163 } 164 165 static bool intel_pmu_is_valid_lbr_msr(struct kvm_vcpu *vcpu, u32 index) 166 { 167 struct x86_pmu_lbr *records = vcpu_to_lbr_records(vcpu); 168 bool ret = false; 169 170 if (!intel_pmu_lbr_is_enabled(vcpu)) 171 return ret; 172 173 ret = (index == MSR_LBR_SELECT) || (index == MSR_LBR_TOS) || 174 (index >= records->from && index < records->from + records->nr) || 175 (index >= records->to && index < records->to + records->nr); 176 177 if (!ret && records->info) 178 ret = (index >= records->info && index < records->info + records->nr); 179 180 return ret; 181 } 182 183 static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 184 { 185 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 186 u64 perf_capabilities; 187 int ret; 188 189 switch (msr) { 190 case MSR_CORE_PERF_FIXED_CTR_CTRL: 191 return kvm_pmu_has_perf_global_ctrl(pmu); 192 case MSR_IA32_PEBS_ENABLE: 193 ret = vcpu_get_perf_capabilities(vcpu) & PERF_CAP_PEBS_FORMAT; 194 break; 195 case MSR_IA32_DS_AREA: 196 ret = guest_cpu_cap_has(vcpu, X86_FEATURE_DS); 197 break; 198 case MSR_PEBS_DATA_CFG: 199 perf_capabilities = vcpu_get_perf_capabilities(vcpu); 200 ret = (perf_capabilities & PERF_CAP_PEBS_BASELINE) && 201 ((perf_capabilities & PERF_CAP_PEBS_FORMAT) > 3); 202 break; 203 default: 204 ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) || 205 get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) || 206 get_fixed_pmc(pmu, msr) || get_fw_gp_pmc(pmu, msr) || 207 intel_pmu_is_valid_lbr_msr(vcpu, msr); 208 break; 209 } 210 211 return ret; 212 } 213 214 static struct kvm_pmc *intel_msr_idx_to_pmc(struct kvm_vcpu *vcpu, u32 msr) 215 { 216 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 217 struct kvm_pmc *pmc; 218 219 pmc = get_fixed_pmc(pmu, msr); 220 pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0); 221 pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0); 222 223 return pmc; 224 } 225 226 static inline void intel_pmu_release_guest_lbr_event(struct kvm_vcpu *vcpu) 227 { 228 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 229 230 if (!lbr_desc) 231 return; 232 233 if (lbr_desc->event) { 234 perf_event_release_kernel(lbr_desc->event); 235 lbr_desc->event = NULL; 236 vcpu_to_pmu(vcpu)->event_count--; 237 } 238 } 239 240 int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu) 241 { 242 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 243 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 244 struct perf_event *event; 245 246 /* 247 * The perf_event_attr is constructed in the minimum efficient way: 248 * - set 'pinned = true' to make it task pinned so that if another 249 * cpu pinned event reclaims LBR, the event->oncpu will be set to -1; 250 * - set '.exclude_host = true' to record guest branches behavior; 251 * 252 * - set '.config = INTEL_FIXED_VLBR_EVENT' to indicates host perf 253 * schedule the event without a real HW counter but a fake one; 254 * check is_guest_lbr_event() and __intel_get_event_constraints(); 255 * 256 * - set 'sample_type = PERF_SAMPLE_BRANCH_STACK' and 257 * 'branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK | 258 * PERF_SAMPLE_BRANCH_USER' to configure it as a LBR callstack 259 * event, which helps KVM to save/restore guest LBR records 260 * during host context switches and reduces quite a lot overhead, 261 * check branch_user_callstack() and intel_pmu_lbr_sched_task(); 262 */ 263 struct perf_event_attr attr = { 264 .type = PERF_TYPE_RAW, 265 .size = sizeof(attr), 266 .config = INTEL_FIXED_VLBR_EVENT, 267 .sample_type = PERF_SAMPLE_BRANCH_STACK, 268 .pinned = true, 269 .exclude_host = true, 270 .branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK | 271 PERF_SAMPLE_BRANCH_USER, 272 }; 273 274 if (WARN_ON_ONCE(!lbr_desc)) 275 return 0; 276 277 if (unlikely(lbr_desc->event)) { 278 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use); 279 return 0; 280 } 281 282 event = perf_event_create_kernel_counter(&attr, -1, 283 current, NULL, NULL); 284 if (IS_ERR(event)) { 285 pr_debug_ratelimited("%s: failed %ld\n", 286 __func__, PTR_ERR(event)); 287 return PTR_ERR(event); 288 } 289 lbr_desc->event = event; 290 pmu->event_count++; 291 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use); 292 return 0; 293 } 294 295 /* 296 * It's safe to access LBR msrs from guest when they have not 297 * been passthrough since the host would help restore or reset 298 * the LBR msrs records when the guest LBR event is scheduled in. 299 */ 300 static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu, 301 struct msr_data *msr_info, bool read) 302 { 303 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 304 u32 index = msr_info->index; 305 306 if (!intel_pmu_is_valid_lbr_msr(vcpu, index)) 307 return false; 308 309 if (!lbr_desc->event && intel_pmu_create_guest_lbr_event(vcpu) < 0) 310 goto dummy; 311 312 /* 313 * Disable irq to ensure the LBR feature doesn't get reclaimed by the 314 * host at the time the value is read from the msr, and this avoids the 315 * host LBR value to be leaked to the guest. If LBR has been reclaimed, 316 * return 0 on guest reads. 317 */ 318 local_irq_disable(); 319 if (lbr_desc->event->state == PERF_EVENT_STATE_ACTIVE) { 320 int err = 0; 321 322 if (read) 323 rdmsrq(index, msr_info->data); 324 else 325 err = wrmsrq_safe(index, msr_info->data); 326 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); 327 local_irq_enable(); 328 return !err; 329 } 330 clear_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); 331 local_irq_enable(); 332 333 dummy: 334 if (read) 335 msr_info->data = 0; 336 return true; 337 } 338 339 static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 340 { 341 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 342 struct kvm_pmc *pmc; 343 u32 msr = msr_info->index; 344 345 switch (msr) { 346 case MSR_CORE_PERF_FIXED_CTR_CTRL: 347 msr_info->data = pmu->fixed_ctr_ctrl; 348 break; 349 case MSR_IA32_PEBS_ENABLE: 350 msr_info->data = pmu->pebs_enable; 351 break; 352 case MSR_IA32_DS_AREA: 353 msr_info->data = pmu->ds_area; 354 break; 355 case MSR_PEBS_DATA_CFG: 356 msr_info->data = pmu->pebs_data_cfg; 357 break; 358 default: 359 if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || 360 (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) { 361 u64 val = pmc_read_counter(pmc); 362 msr_info->data = 363 val & pmu->counter_bitmask[KVM_PMC_GP]; 364 break; 365 } else if ((pmc = get_fixed_pmc(pmu, msr))) { 366 u64 val = pmc_read_counter(pmc); 367 msr_info->data = 368 val & pmu->counter_bitmask[KVM_PMC_FIXED]; 369 break; 370 } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { 371 msr_info->data = pmc->eventsel; 372 break; 373 } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, true)) { 374 break; 375 } 376 return 1; 377 } 378 379 return 0; 380 } 381 382 static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 383 { 384 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 385 struct kvm_pmc *pmc; 386 u32 msr = msr_info->index; 387 u64 data = msr_info->data; 388 u64 reserved_bits, diff; 389 390 switch (msr) { 391 case MSR_CORE_PERF_FIXED_CTR_CTRL: 392 if (data & pmu->fixed_ctr_ctrl_rsvd) 393 return 1; 394 395 if (pmu->fixed_ctr_ctrl != data) 396 reprogram_fixed_counters(pmu, data); 397 break; 398 case MSR_IA32_PEBS_ENABLE: 399 if (data & pmu->pebs_enable_rsvd) 400 return 1; 401 402 if (pmu->pebs_enable != data) { 403 diff = pmu->pebs_enable ^ data; 404 pmu->pebs_enable = data; 405 kvm_pmu_request_counters_reprogram(pmu, diff); 406 } 407 break; 408 case MSR_IA32_DS_AREA: 409 if (is_noncanonical_msr_address(data, vcpu)) 410 return 1; 411 412 pmu->ds_area = data; 413 break; 414 case MSR_PEBS_DATA_CFG: 415 if (data & pmu->pebs_data_cfg_rsvd) 416 return 1; 417 418 pmu->pebs_data_cfg = data; 419 break; 420 default: 421 if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || 422 (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) { 423 if ((msr & MSR_PMC_FULL_WIDTH_BIT) && 424 (data & ~pmu->counter_bitmask[KVM_PMC_GP])) 425 return 1; 426 427 if (!msr_info->host_initiated && 428 !(msr & MSR_PMC_FULL_WIDTH_BIT)) 429 data = (s64)(s32)data; 430 pmc_write_counter(pmc, data); 431 break; 432 } else if ((pmc = get_fixed_pmc(pmu, msr))) { 433 pmc_write_counter(pmc, data); 434 break; 435 } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { 436 reserved_bits = pmu->reserved_bits; 437 if ((pmc->idx == 2) && 438 (pmu->raw_event_mask & HSW_IN_TX_CHECKPOINTED)) 439 reserved_bits ^= HSW_IN_TX_CHECKPOINTED; 440 if (data & reserved_bits) 441 return 1; 442 443 if (data != pmc->eventsel) { 444 pmc->eventsel = data; 445 pmc->eventsel_hw = data; 446 kvm_pmu_request_counter_reprogram(pmc); 447 } 448 break; 449 } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, false)) { 450 break; 451 } 452 /* Not a known PMU MSR. */ 453 return 1; 454 } 455 456 return 0; 457 } 458 459 /* 460 * Map fixed counter events to architectural general purpose event encodings. 461 * Perf doesn't provide APIs to allow KVM to directly program a fixed counter, 462 * and so KVM instead programs the architectural event to effectively request 463 * the fixed counter. Perf isn't guaranteed to use a fixed counter and may 464 * instead program the encoding into a general purpose counter, e.g. if a 465 * different perf_event is already utilizing the requested counter, but the end 466 * result is the same (ignoring the fact that using a general purpose counter 467 * will likely exacerbate counter contention). 468 * 469 * Forcibly inlined to allow asserting on @index at build time, and there should 470 * never be more than one user. 471 */ 472 static __always_inline u64 intel_get_fixed_pmc_eventsel(unsigned int index) 473 { 474 const enum perf_hw_id fixed_pmc_perf_ids[] = { 475 [0] = PERF_COUNT_HW_INSTRUCTIONS, 476 [1] = PERF_COUNT_HW_CPU_CYCLES, 477 [2] = PERF_COUNT_HW_REF_CPU_CYCLES, 478 }; 479 u64 eventsel; 480 481 BUILD_BUG_ON(ARRAY_SIZE(fixed_pmc_perf_ids) != KVM_MAX_NR_INTEL_FIXED_COUNTERS); 482 BUILD_BUG_ON(index >= KVM_MAX_NR_INTEL_FIXED_COUNTERS); 483 484 /* 485 * Yell if perf reports support for a fixed counter but perf doesn't 486 * have a known encoding for the associated general purpose event. 487 */ 488 eventsel = perf_get_hw_event_config(fixed_pmc_perf_ids[index]); 489 WARN_ON_ONCE(!eventsel && index < kvm_pmu_cap.num_counters_fixed); 490 return eventsel; 491 } 492 493 static void intel_pmu_enable_fixed_counter_bits(struct kvm_pmu *pmu, u64 bits) 494 { 495 int i; 496 497 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) 498 pmu->fixed_ctr_ctrl_rsvd &= ~intel_fixed_bits_by_idx(i, bits); 499 } 500 501 static void intel_pmu_refresh(struct kvm_vcpu *vcpu) 502 { 503 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 504 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 505 struct kvm_cpuid_entry2 *entry; 506 union cpuid10_eax eax; 507 union cpuid10_edx edx; 508 u64 perf_capabilities; 509 u64 counter_rsvd; 510 511 if (!lbr_desc) 512 return; 513 514 memset(&lbr_desc->records, 0, sizeof(lbr_desc->records)); 515 516 /* 517 * Setting passthrough of LBR MSRs is done only in the VM-Entry loop, 518 * and PMU refresh is disallowed after the vCPU has run, i.e. this code 519 * should never be reached while KVM is passing through MSRs. 520 */ 521 if (KVM_BUG_ON(lbr_desc->msr_passthrough, vcpu->kvm)) 522 return; 523 524 entry = kvm_find_cpuid_entry(vcpu, 0xa); 525 if (!entry) 526 return; 527 528 eax.full = entry->eax; 529 edx.full = entry->edx; 530 531 pmu->version = eax.split.version_id; 532 if (!pmu->version) 533 return; 534 535 pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters, 536 kvm_pmu_cap.num_counters_gp); 537 eax.split.bit_width = min_t(int, eax.split.bit_width, 538 kvm_pmu_cap.bit_width_gp); 539 pmu->counter_bitmask[KVM_PMC_GP] = BIT_ULL(eax.split.bit_width) - 1; 540 eax.split.mask_length = min_t(int, eax.split.mask_length, 541 kvm_pmu_cap.events_mask_len); 542 pmu->available_event_types = ~entry->ebx & (BIT_ULL(eax.split.mask_length) - 1); 543 544 entry = kvm_find_cpuid_entry_index(vcpu, 7, 0); 545 if (entry && 546 (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) && 547 (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) { 548 pmu->reserved_bits ^= HSW_IN_TX; 549 pmu->raw_event_mask |= (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED); 550 } 551 552 perf_capabilities = vcpu_get_perf_capabilities(vcpu); 553 if (intel_pmu_lbr_is_compatible(vcpu) && 554 (perf_capabilities & PERF_CAP_LBR_FMT)) 555 memcpy(&lbr_desc->records, &vmx_lbr_caps, sizeof(vmx_lbr_caps)); 556 else 557 lbr_desc->records.nr = 0; 558 559 if (lbr_desc->records.nr) 560 bitmap_set(pmu->all_valid_pmc_idx, INTEL_PMC_IDX_FIXED_VLBR, 1); 561 562 if (pmu->version == 1) 563 return; 564 565 pmu->nr_arch_fixed_counters = min_t(int, edx.split.num_counters_fixed, 566 kvm_pmu_cap.num_counters_fixed); 567 edx.split.bit_width_fixed = min_t(int, edx.split.bit_width_fixed, 568 kvm_pmu_cap.bit_width_fixed); 569 pmu->counter_bitmask[KVM_PMC_FIXED] = BIT_ULL(edx.split.bit_width_fixed) - 1; 570 571 intel_pmu_enable_fixed_counter_bits(pmu, INTEL_FIXED_0_KERNEL | 572 INTEL_FIXED_0_USER | 573 INTEL_FIXED_0_ENABLE_PMI); 574 575 counter_rsvd = ~((BIT_ULL(pmu->nr_arch_gp_counters) - 1) | 576 ((BIT_ULL(pmu->nr_arch_fixed_counters) - 1) << KVM_FIXED_PMC_BASE_IDX)); 577 pmu->global_ctrl_rsvd = counter_rsvd; 578 579 /* 580 * GLOBAL_STATUS and GLOBAL_OVF_CONTROL (a.k.a. GLOBAL_STATUS_RESET) 581 * share reserved bit definitions. The kernel just happens to use 582 * OVF_CTRL for the names. 583 */ 584 pmu->global_status_rsvd = pmu->global_ctrl_rsvd 585 & ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF | 586 MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD); 587 if (vmx_pt_mode_is_host_guest()) 588 pmu->global_status_rsvd &= 589 ~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI; 590 591 if (perf_capabilities & PERF_CAP_PEBS_FORMAT) { 592 if (perf_capabilities & PERF_CAP_PEBS_BASELINE) { 593 pmu->pebs_enable_rsvd = counter_rsvd; 594 pmu->reserved_bits &= ~ICL_EVENTSEL_ADAPTIVE; 595 pmu->pebs_data_cfg_rsvd = ~0xff00000full; 596 intel_pmu_enable_fixed_counter_bits(pmu, ICL_FIXED_0_ADAPTIVE); 597 } else { 598 pmu->pebs_enable_rsvd = ~(BIT_ULL(pmu->nr_arch_gp_counters) - 1); 599 } 600 } 601 } 602 603 static void intel_pmu_init(struct kvm_vcpu *vcpu) 604 { 605 int i; 606 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 607 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 608 609 if (!lbr_desc) 610 return; 611 612 for (i = 0; i < KVM_MAX_NR_INTEL_GP_COUNTERS; i++) { 613 pmu->gp_counters[i].type = KVM_PMC_GP; 614 pmu->gp_counters[i].vcpu = vcpu; 615 pmu->gp_counters[i].idx = i; 616 pmu->gp_counters[i].current_config = 0; 617 } 618 619 for (i = 0; i < KVM_MAX_NR_INTEL_FIXED_COUNTERS; i++) { 620 pmu->fixed_counters[i].type = KVM_PMC_FIXED; 621 pmu->fixed_counters[i].vcpu = vcpu; 622 pmu->fixed_counters[i].idx = i + KVM_FIXED_PMC_BASE_IDX; 623 pmu->fixed_counters[i].current_config = 0; 624 pmu->fixed_counters[i].eventsel = intel_get_fixed_pmc_eventsel(i); 625 } 626 627 lbr_desc->records.nr = 0; 628 lbr_desc->event = NULL; 629 lbr_desc->msr_passthrough = false; 630 } 631 632 static void intel_pmu_reset(struct kvm_vcpu *vcpu) 633 { 634 intel_pmu_release_guest_lbr_event(vcpu); 635 } 636 637 /* 638 * Emulate LBR_On_PMI behavior for 1 < pmu.version < 4. 639 * 640 * If Freeze_LBR_On_PMI = 1, the LBR is frozen on PMI and 641 * the KVM emulates to clear the LBR bit (bit 0) in IA32_DEBUGCTL. 642 * 643 * Guest needs to re-enable LBR to resume branches recording. 644 */ 645 static void intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu *vcpu) 646 { 647 u64 data = vmx_guest_debugctl_read(); 648 649 if (data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) { 650 data &= ~DEBUGCTLMSR_LBR; 651 vmx_guest_debugctl_write(vcpu, data); 652 } 653 } 654 655 static void intel_pmu_deliver_pmi(struct kvm_vcpu *vcpu) 656 { 657 u8 version = vcpu_to_pmu(vcpu)->version; 658 659 if (!intel_pmu_lbr_is_enabled(vcpu)) 660 return; 661 662 if (version > 1 && version < 4) 663 intel_pmu_legacy_freezing_lbrs_on_pmi(vcpu); 664 } 665 666 static void vmx_update_intercept_for_lbr_msrs(struct kvm_vcpu *vcpu, bool set) 667 { 668 struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu); 669 int i; 670 671 for (i = 0; i < lbr->nr; i++) { 672 vmx_set_intercept_for_msr(vcpu, lbr->from + i, MSR_TYPE_RW, set); 673 vmx_set_intercept_for_msr(vcpu, lbr->to + i, MSR_TYPE_RW, set); 674 if (lbr->info) 675 vmx_set_intercept_for_msr(vcpu, lbr->info + i, MSR_TYPE_RW, set); 676 } 677 678 vmx_set_intercept_for_msr(vcpu, MSR_LBR_SELECT, MSR_TYPE_RW, set); 679 vmx_set_intercept_for_msr(vcpu, MSR_LBR_TOS, MSR_TYPE_RW, set); 680 } 681 682 static inline void vmx_disable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu) 683 { 684 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 685 686 if (!lbr_desc->msr_passthrough) 687 return; 688 689 vmx_update_intercept_for_lbr_msrs(vcpu, true); 690 lbr_desc->msr_passthrough = false; 691 } 692 693 static inline void vmx_enable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu) 694 { 695 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 696 697 if (lbr_desc->msr_passthrough) 698 return; 699 700 vmx_update_intercept_for_lbr_msrs(vcpu, false); 701 lbr_desc->msr_passthrough = true; 702 } 703 704 /* 705 * Higher priority host perf events (e.g. cpu pinned) could reclaim the 706 * pmu resources (e.g. LBR) that were assigned to the guest. This is 707 * usually done via ipi calls (more details in perf_install_in_context). 708 * 709 * Before entering the non-root mode (with irq disabled here), double 710 * confirm that the pmu features enabled to the guest are not reclaimed 711 * by higher priority host events. Otherwise, disallow vcpu's access to 712 * the reclaimed features. 713 */ 714 void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu) 715 { 716 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 717 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 718 719 if (WARN_ON_ONCE(!lbr_desc)) 720 return; 721 722 if (!lbr_desc->event) { 723 vmx_disable_lbr_msrs_passthrough(vcpu); 724 if (vmx_guest_debugctl_read() & DEBUGCTLMSR_LBR) 725 goto warn; 726 if (test_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use)) 727 goto warn; 728 return; 729 } 730 731 if (lbr_desc->event->state < PERF_EVENT_STATE_ACTIVE) { 732 vmx_disable_lbr_msrs_passthrough(vcpu); 733 __clear_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use); 734 goto warn; 735 } else 736 vmx_enable_lbr_msrs_passthrough(vcpu); 737 738 return; 739 740 warn: 741 pr_warn_ratelimited("vcpu-%d: fail to passthrough LBR.\n", vcpu->vcpu_id); 742 } 743 744 static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) 745 { 746 if (!(vmx_guest_debugctl_read() & DEBUGCTLMSR_LBR)) 747 intel_pmu_release_guest_lbr_event(vcpu); 748 } 749 750 void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu) 751 { 752 struct kvm_pmc *pmc = NULL; 753 int bit, hw_idx; 754 755 kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&pmu->global_ctrl) { 756 if (!pmc_is_locally_enabled(pmc) || 757 !pmc_is_globally_enabled(pmc) || !pmc->perf_event) 758 continue; 759 760 /* 761 * A negative index indicates the event isn't mapped to a 762 * physical counter in the host, e.g. due to contention. 763 */ 764 hw_idx = pmc->perf_event->hw.idx; 765 if (hw_idx != pmc->idx && hw_idx > -1) 766 pmu->host_cross_mapped_mask |= BIT_ULL(hw_idx); 767 } 768 } 769 770 static bool intel_pmu_is_mediated_pmu_supported(struct x86_pmu_capability *host_pmu) 771 { 772 u64 host_perf_cap = 0; 773 774 if (boot_cpu_has(X86_FEATURE_PDCM)) 775 rdmsrq(MSR_IA32_PERF_CAPABILITIES, host_perf_cap); 776 777 /* 778 * Require v4+ for MSR_CORE_PERF_GLOBAL_STATUS_SET, and full-width 779 * writes so that KVM can precisely load guest counter values. 780 */ 781 if (host_pmu->version < 4 || !(host_perf_cap & PERF_CAP_FW_WRITES)) 782 return false; 783 784 /* 785 * All CPUs that support a mediated PMU are expected to support loading 786 * PERF_GLOBAL_CTRL via dedicated VMCS fields. 787 */ 788 if (WARN_ON_ONCE(!cpu_has_load_perf_global_ctrl())) 789 return false; 790 791 return true; 792 } 793 794 static void intel_pmu_write_global_ctrl(u64 global_ctrl) 795 { 796 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL, global_ctrl); 797 } 798 799 static void intel_mediated_pmu_load(struct kvm_vcpu *vcpu) 800 { 801 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 802 u64 global_status, toggle; 803 804 rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, global_status); 805 toggle = pmu->global_status ^ global_status; 806 if (global_status & toggle) 807 wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, global_status & toggle); 808 if (pmu->global_status & toggle) 809 wrmsrq(MSR_CORE_PERF_GLOBAL_STATUS_SET, pmu->global_status & toggle); 810 811 wrmsrq(MSR_CORE_PERF_FIXED_CTR_CTRL, pmu->fixed_ctr_ctrl_hw); 812 } 813 814 static void intel_mediated_pmu_put(struct kvm_vcpu *vcpu) 815 { 816 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 817 818 /* MSR_CORE_PERF_GLOBAL_CTRL is already saved at VM-exit. */ 819 rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, pmu->global_status); 820 821 /* Clear hardware MSR_CORE_PERF_GLOBAL_STATUS MSR, if non-zero. */ 822 if (pmu->global_status) 823 wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, pmu->global_status); 824 825 /* 826 * Clear hardware FIXED_CTR_CTRL MSR to avoid information leakage and 827 * also to avoid accidentally enabling fixed counters (based on guest 828 * state) while running in the host, e.g. when setting global ctrl. 829 */ 830 if (pmu->fixed_ctr_ctrl_hw) 831 wrmsrq(MSR_CORE_PERF_FIXED_CTR_CTRL, 0); 832 } 833 834 struct kvm_pmu_ops intel_pmu_ops __initdata = { 835 .rdpmc_ecx_to_pmc = intel_rdpmc_ecx_to_pmc, 836 .msr_idx_to_pmc = intel_msr_idx_to_pmc, 837 .is_valid_msr = intel_is_valid_msr, 838 .get_msr = intel_pmu_get_msr, 839 .set_msr = intel_pmu_set_msr, 840 .refresh = intel_pmu_refresh, 841 .init = intel_pmu_init, 842 .reset = intel_pmu_reset, 843 .deliver_pmi = intel_pmu_deliver_pmi, 844 .cleanup = intel_pmu_cleanup, 845 846 .is_mediated_pmu_supported = intel_pmu_is_mediated_pmu_supported, 847 .mediated_load = intel_mediated_pmu_load, 848 .mediated_put = intel_mediated_pmu_put, 849 .write_global_ctrl = intel_pmu_write_global_ctrl, 850 851 .EVENTSEL_EVENT = ARCH_PERFMON_EVENTSEL_EVENT, 852 .MAX_NR_GP_COUNTERS = KVM_MAX_NR_INTEL_GP_COUNTERS, 853 .MIN_NR_GP_COUNTERS = 1, 854 855 .PERF_GLOBAL_CTRL = MSR_CORE_PERF_GLOBAL_CTRL, 856 .GP_EVENTSEL_BASE = MSR_P6_EVNTSEL0, 857 .GP_COUNTER_BASE = MSR_IA32_PMC0, 858 .FIXED_COUNTER_BASE = MSR_CORE_PERF_FIXED_CTR0, 859 .MSR_STRIDE = 1, 860 }; 861