1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KVM PMU support for Intel CPUs 4 * 5 * Copyright 2011 Red Hat, Inc. and/or its affiliates. 6 * 7 * Authors: 8 * Avi Kivity <avi@redhat.com> 9 * Gleb Natapov <gleb@redhat.com> 10 */ 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13 #include <linux/types.h> 14 #include <linux/kvm_host.h> 15 #include <linux/perf_event.h> 16 #include <asm/msr.h> 17 #include <asm/perf_event.h> 18 #include <asm/cpuid/api.h> 19 #include "x86.h" 20 #include "cpuid.h" 21 #include "lapic.h" 22 #include "nested.h" 23 #include "pmu.h" 24 #include "tdx.h" 25 26 /* 27 * Perf's "BASE" is wildly misleading, architectural PMUs use bits 31:16 of ECX 28 * to encode the "type" of counter to read, i.e. this is not a "base". And to 29 * further confuse things, non-architectural PMUs use bit 31 as a flag for 30 * "fast" reads, whereas the "type" is an explicit value. 31 */ 32 #define INTEL_RDPMC_GP 0 33 #define INTEL_RDPMC_FIXED INTEL_PMC_FIXED_RDPMC_BASE 34 35 #define INTEL_RDPMC_TYPE_MASK GENMASK(31, 16) 36 #define INTEL_RDPMC_INDEX_MASK GENMASK(15, 0) 37 38 #define MSR_PMC_FULL_WIDTH_BIT (MSR_IA32_PMC0 - MSR_IA32_PERFCTR0) 39 40 static struct lbr_desc *vcpu_to_lbr_desc(struct kvm_vcpu *vcpu) 41 { 42 if (is_td_vcpu(vcpu)) 43 return NULL; 44 45 return &to_vmx(vcpu)->lbr_desc; 46 } 47 48 static struct x86_pmu_lbr *vcpu_to_lbr_records(struct kvm_vcpu *vcpu) 49 { 50 if (is_td_vcpu(vcpu)) 51 return NULL; 52 53 return &to_vmx(vcpu)->lbr_desc.records; 54 } 55 56 #pragma GCC poison to_vmx 57 58 static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data) 59 { 60 struct kvm_pmc *pmc; 61 u64 old_fixed_ctr_ctrl = pmu->fixed_ctr_ctrl; 62 int i; 63 64 pmu->fixed_ctr_ctrl = data; 65 pmu->fixed_ctr_ctrl_hw = data; 66 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 67 u8 new_ctrl = fixed_ctrl_field(data, i); 68 u8 old_ctrl = fixed_ctrl_field(old_fixed_ctr_ctrl, i); 69 70 if (old_ctrl == new_ctrl) 71 continue; 72 73 pmc = get_fixed_pmc(pmu, MSR_CORE_PERF_FIXED_CTR0 + i); 74 75 __set_bit(KVM_FIXED_PMC_BASE_IDX + i, pmu->pmc_in_use); 76 kvm_pmu_request_counter_reprogram(pmc); 77 } 78 } 79 80 static struct kvm_pmc *intel_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu, 81 unsigned int idx, u64 *mask) 82 { 83 unsigned int type = idx & INTEL_RDPMC_TYPE_MASK; 84 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 85 struct kvm_pmc *counters; 86 unsigned int num_counters; 87 u64 bitmask; 88 89 /* 90 * The encoding of ECX for RDPMC is different for architectural versus 91 * non-architecturals PMUs (PMUs with version '0'). For architectural 92 * PMUs, bits 31:16 specify the PMC type and bits 15:0 specify the PMC 93 * index. For non-architectural PMUs, bit 31 is a "fast" flag, and 94 * bits 30:0 specify the PMC index. 95 * 96 * Yell and reject attempts to read PMCs for a non-architectural PMU, 97 * as KVM doesn't support such PMUs. 98 */ 99 if (WARN_ON_ONCE(!pmu->version)) 100 return NULL; 101 102 /* 103 * General Purpose (GP) PMCs are supported on all PMUs, and fixed PMCs 104 * are supported on all architectural PMUs, i.e. on all virtual PMUs 105 * supported by KVM. Note, KVM only emulates fixed PMCs for PMU v2+, 106 * but the type itself is still valid, i.e. let RDPMC fail due to 107 * accessing a non-existent counter. Reject attempts to read all other 108 * types, which are unknown/unsupported. 109 */ 110 switch (type) { 111 case INTEL_RDPMC_FIXED: 112 counters = pmu->fixed_counters; 113 num_counters = pmu->nr_arch_fixed_counters; 114 bitmask = pmu->counter_bitmask[KVM_PMC_FIXED]; 115 break; 116 case INTEL_RDPMC_GP: 117 counters = pmu->gp_counters; 118 num_counters = pmu->nr_arch_gp_counters; 119 bitmask = pmu->counter_bitmask[KVM_PMC_GP]; 120 break; 121 default: 122 return NULL; 123 } 124 125 idx &= INTEL_RDPMC_INDEX_MASK; 126 if (idx >= num_counters) 127 return NULL; 128 129 *mask &= bitmask; 130 return &counters[array_index_nospec(idx, num_counters)]; 131 } 132 133 static inline struct kvm_pmc *get_fw_gp_pmc(struct kvm_pmu *pmu, u32 msr) 134 { 135 if (!fw_writes_is_enabled(pmu_to_vcpu(pmu))) 136 return NULL; 137 138 return get_gp_pmc(pmu, msr, MSR_IA32_PMC0); 139 } 140 141 static bool intel_pmu_lbr_is_compatible(struct kvm_vcpu *vcpu) 142 { 143 if (is_td_vcpu(vcpu)) 144 return false; 145 146 return cpuid_model_is_consistent(vcpu); 147 } 148 149 bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu) 150 { 151 if (is_td_vcpu(vcpu)) 152 return false; 153 154 return !!vcpu_to_lbr_records(vcpu)->nr; 155 } 156 157 static bool intel_pmu_is_valid_lbr_msr(struct kvm_vcpu *vcpu, u32 index) 158 { 159 struct x86_pmu_lbr *records = vcpu_to_lbr_records(vcpu); 160 bool ret = false; 161 162 if (!intel_pmu_lbr_is_enabled(vcpu)) 163 return ret; 164 165 ret = (index == MSR_LBR_SELECT) || (index == MSR_LBR_TOS) || 166 (index >= records->from && index < records->from + records->nr) || 167 (index >= records->to && index < records->to + records->nr); 168 169 if (!ret && records->info) 170 ret = (index >= records->info && index < records->info + records->nr); 171 172 return ret; 173 } 174 175 static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 176 { 177 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 178 u64 perf_capabilities; 179 int ret; 180 181 switch (msr) { 182 case MSR_CORE_PERF_FIXED_CTR_CTRL: 183 return kvm_pmu_has_perf_global_ctrl(pmu); 184 case MSR_IA32_PEBS_ENABLE: 185 ret = vcpu_get_perf_capabilities(vcpu) & PERF_CAP_PEBS_FORMAT; 186 break; 187 case MSR_IA32_DS_AREA: 188 ret = guest_cpu_cap_has(vcpu, X86_FEATURE_DS); 189 break; 190 case MSR_PEBS_DATA_CFG: 191 perf_capabilities = vcpu_get_perf_capabilities(vcpu); 192 ret = (perf_capabilities & PERF_CAP_PEBS_BASELINE) && 193 ((perf_capabilities & PERF_CAP_PEBS_FORMAT) > 3); 194 break; 195 default: 196 ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) || 197 get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) || 198 get_fixed_pmc(pmu, msr) || get_fw_gp_pmc(pmu, msr) || 199 intel_pmu_is_valid_lbr_msr(vcpu, msr); 200 break; 201 } 202 203 return ret; 204 } 205 206 static struct kvm_pmc *intel_msr_idx_to_pmc(struct kvm_vcpu *vcpu, u32 msr) 207 { 208 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 209 struct kvm_pmc *pmc; 210 211 pmc = get_fixed_pmc(pmu, msr); 212 pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0); 213 pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0); 214 215 return pmc; 216 } 217 218 static inline void intel_pmu_release_guest_lbr_event(struct kvm_vcpu *vcpu) 219 { 220 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 221 222 if (!lbr_desc) 223 return; 224 225 if (lbr_desc->event) { 226 perf_event_release_kernel(lbr_desc->event); 227 lbr_desc->event = NULL; 228 vcpu_to_pmu(vcpu)->event_count--; 229 } 230 } 231 232 int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu) 233 { 234 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 235 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 236 struct perf_event *event; 237 238 /* 239 * The perf_event_attr is constructed in the minimum efficient way: 240 * - set 'pinned = true' to make it task pinned so that if another 241 * cpu pinned event reclaims LBR, the event->oncpu will be set to -1; 242 * - set '.exclude_host = true' to record guest branches behavior; 243 * 244 * - set '.config = INTEL_FIXED_VLBR_EVENT' to indicates host perf 245 * schedule the event without a real HW counter but a fake one; 246 * check is_guest_lbr_event() and __intel_get_event_constraints(); 247 * 248 * - set 'sample_type = PERF_SAMPLE_BRANCH_STACK' and 249 * 'branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK | 250 * PERF_SAMPLE_BRANCH_USER' to configure it as a LBR callstack 251 * event, which helps KVM to save/restore guest LBR records 252 * during host context switches and reduces quite a lot overhead, 253 * check branch_user_callstack() and intel_pmu_lbr_sched_task(); 254 */ 255 struct perf_event_attr attr = { 256 .type = PERF_TYPE_RAW, 257 .size = sizeof(attr), 258 .config = INTEL_FIXED_VLBR_EVENT, 259 .sample_type = PERF_SAMPLE_BRANCH_STACK, 260 .pinned = true, 261 .exclude_host = true, 262 .branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK | 263 PERF_SAMPLE_BRANCH_USER, 264 }; 265 266 if (WARN_ON_ONCE(!lbr_desc)) 267 return 0; 268 269 if (unlikely(lbr_desc->event)) { 270 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use); 271 return 0; 272 } 273 274 event = perf_event_create_kernel_counter(&attr, -1, 275 current, NULL, NULL); 276 if (IS_ERR(event)) { 277 pr_debug_ratelimited("%s: failed %ld\n", 278 __func__, PTR_ERR(event)); 279 return PTR_ERR(event); 280 } 281 lbr_desc->event = event; 282 pmu->event_count++; 283 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use); 284 return 0; 285 } 286 287 /* 288 * It's safe to access LBR msrs from guest when they have not 289 * been passthrough since the host would help restore or reset 290 * the LBR msrs records when the guest LBR event is scheduled in. 291 */ 292 static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu, 293 struct msr_data *msr_info, bool read) 294 { 295 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 296 u32 index = msr_info->index; 297 298 if (!intel_pmu_is_valid_lbr_msr(vcpu, index)) 299 return false; 300 301 if (!lbr_desc->event && intel_pmu_create_guest_lbr_event(vcpu) < 0) 302 goto dummy; 303 304 /* 305 * Disable irq to ensure the LBR feature doesn't get reclaimed by the 306 * host at the time the value is read from the msr, and this avoids the 307 * host LBR value to be leaked to the guest. If LBR has been reclaimed, 308 * return 0 on guest reads. 309 */ 310 local_irq_disable(); 311 if (lbr_desc->event->state == PERF_EVENT_STATE_ACTIVE) { 312 if (read) 313 rdmsrq(index, msr_info->data); 314 else 315 wrmsrq(index, msr_info->data); 316 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); 317 local_irq_enable(); 318 return true; 319 } 320 clear_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); 321 local_irq_enable(); 322 323 dummy: 324 if (read) 325 msr_info->data = 0; 326 return true; 327 } 328 329 static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 330 { 331 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 332 struct kvm_pmc *pmc; 333 u32 msr = msr_info->index; 334 335 switch (msr) { 336 case MSR_CORE_PERF_FIXED_CTR_CTRL: 337 msr_info->data = pmu->fixed_ctr_ctrl; 338 break; 339 case MSR_IA32_PEBS_ENABLE: 340 msr_info->data = pmu->pebs_enable; 341 break; 342 case MSR_IA32_DS_AREA: 343 msr_info->data = pmu->ds_area; 344 break; 345 case MSR_PEBS_DATA_CFG: 346 msr_info->data = pmu->pebs_data_cfg; 347 break; 348 default: 349 if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || 350 (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) { 351 u64 val = pmc_read_counter(pmc); 352 msr_info->data = 353 val & pmu->counter_bitmask[KVM_PMC_GP]; 354 break; 355 } else if ((pmc = get_fixed_pmc(pmu, msr))) { 356 u64 val = pmc_read_counter(pmc); 357 msr_info->data = 358 val & pmu->counter_bitmask[KVM_PMC_FIXED]; 359 break; 360 } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { 361 msr_info->data = pmc->eventsel; 362 break; 363 } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, true)) { 364 break; 365 } 366 return 1; 367 } 368 369 return 0; 370 } 371 372 static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 373 { 374 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 375 struct kvm_pmc *pmc; 376 u32 msr = msr_info->index; 377 u64 data = msr_info->data; 378 u64 reserved_bits, diff; 379 380 switch (msr) { 381 case MSR_CORE_PERF_FIXED_CTR_CTRL: 382 if (data & pmu->fixed_ctr_ctrl_rsvd) 383 return 1; 384 385 if (pmu->fixed_ctr_ctrl != data) 386 reprogram_fixed_counters(pmu, data); 387 break; 388 case MSR_IA32_PEBS_ENABLE: 389 if (data & pmu->pebs_enable_rsvd) 390 return 1; 391 392 if (pmu->pebs_enable != data) { 393 diff = pmu->pebs_enable ^ data; 394 pmu->pebs_enable = data; 395 reprogram_counters(pmu, diff); 396 } 397 break; 398 case MSR_IA32_DS_AREA: 399 if (is_noncanonical_msr_address(data, vcpu)) 400 return 1; 401 402 pmu->ds_area = data; 403 break; 404 case MSR_PEBS_DATA_CFG: 405 if (data & pmu->pebs_data_cfg_rsvd) 406 return 1; 407 408 pmu->pebs_data_cfg = data; 409 break; 410 default: 411 if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || 412 (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) { 413 if ((msr & MSR_PMC_FULL_WIDTH_BIT) && 414 (data & ~pmu->counter_bitmask[KVM_PMC_GP])) 415 return 1; 416 417 if (!msr_info->host_initiated && 418 !(msr & MSR_PMC_FULL_WIDTH_BIT)) 419 data = (s64)(s32)data; 420 pmc_write_counter(pmc, data); 421 break; 422 } else if ((pmc = get_fixed_pmc(pmu, msr))) { 423 pmc_write_counter(pmc, data); 424 break; 425 } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { 426 reserved_bits = pmu->reserved_bits; 427 if ((pmc->idx == 2) && 428 (pmu->raw_event_mask & HSW_IN_TX_CHECKPOINTED)) 429 reserved_bits ^= HSW_IN_TX_CHECKPOINTED; 430 if (data & reserved_bits) 431 return 1; 432 433 if (data != pmc->eventsel) { 434 pmc->eventsel = data; 435 pmc->eventsel_hw = data; 436 kvm_pmu_request_counter_reprogram(pmc); 437 } 438 break; 439 } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, false)) { 440 break; 441 } 442 /* Not a known PMU MSR. */ 443 return 1; 444 } 445 446 return 0; 447 } 448 449 /* 450 * Map fixed counter events to architectural general purpose event encodings. 451 * Perf doesn't provide APIs to allow KVM to directly program a fixed counter, 452 * and so KVM instead programs the architectural event to effectively request 453 * the fixed counter. Perf isn't guaranteed to use a fixed counter and may 454 * instead program the encoding into a general purpose counter, e.g. if a 455 * different perf_event is already utilizing the requested counter, but the end 456 * result is the same (ignoring the fact that using a general purpose counter 457 * will likely exacerbate counter contention). 458 * 459 * Forcibly inlined to allow asserting on @index at build time, and there should 460 * never be more than one user. 461 */ 462 static __always_inline u64 intel_get_fixed_pmc_eventsel(unsigned int index) 463 { 464 const enum perf_hw_id fixed_pmc_perf_ids[] = { 465 [0] = PERF_COUNT_HW_INSTRUCTIONS, 466 [1] = PERF_COUNT_HW_CPU_CYCLES, 467 [2] = PERF_COUNT_HW_REF_CPU_CYCLES, 468 }; 469 u64 eventsel; 470 471 BUILD_BUG_ON(ARRAY_SIZE(fixed_pmc_perf_ids) != KVM_MAX_NR_INTEL_FIXED_COUNTERS); 472 BUILD_BUG_ON(index >= KVM_MAX_NR_INTEL_FIXED_COUNTERS); 473 474 /* 475 * Yell if perf reports support for a fixed counter but perf doesn't 476 * have a known encoding for the associated general purpose event. 477 */ 478 eventsel = perf_get_hw_event_config(fixed_pmc_perf_ids[index]); 479 WARN_ON_ONCE(!eventsel && index < kvm_pmu_cap.num_counters_fixed); 480 return eventsel; 481 } 482 483 static void intel_pmu_enable_fixed_counter_bits(struct kvm_pmu *pmu, u64 bits) 484 { 485 int i; 486 487 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) 488 pmu->fixed_ctr_ctrl_rsvd &= ~intel_fixed_bits_by_idx(i, bits); 489 } 490 491 static void intel_pmu_refresh(struct kvm_vcpu *vcpu) 492 { 493 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 494 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 495 struct kvm_cpuid_entry2 *entry; 496 union cpuid10_eax eax; 497 union cpuid10_edx edx; 498 u64 perf_capabilities; 499 u64 counter_rsvd; 500 501 if (!lbr_desc) 502 return; 503 504 memset(&lbr_desc->records, 0, sizeof(lbr_desc->records)); 505 506 /* 507 * Setting passthrough of LBR MSRs is done only in the VM-Entry loop, 508 * and PMU refresh is disallowed after the vCPU has run, i.e. this code 509 * should never be reached while KVM is passing through MSRs. 510 */ 511 if (KVM_BUG_ON(lbr_desc->msr_passthrough, vcpu->kvm)) 512 return; 513 514 entry = kvm_find_cpuid_entry(vcpu, 0xa); 515 if (!entry) 516 return; 517 518 eax.full = entry->eax; 519 edx.full = entry->edx; 520 521 pmu->version = eax.split.version_id; 522 if (!pmu->version) 523 return; 524 525 pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters, 526 kvm_pmu_cap.num_counters_gp); 527 eax.split.bit_width = min_t(int, eax.split.bit_width, 528 kvm_pmu_cap.bit_width_gp); 529 pmu->counter_bitmask[KVM_PMC_GP] = BIT_ULL(eax.split.bit_width) - 1; 530 eax.split.mask_length = min_t(int, eax.split.mask_length, 531 kvm_pmu_cap.events_mask_len); 532 pmu->available_event_types = ~entry->ebx & (BIT_ULL(eax.split.mask_length) - 1); 533 534 entry = kvm_find_cpuid_entry_index(vcpu, 7, 0); 535 if (entry && 536 (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) && 537 (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) { 538 pmu->reserved_bits ^= HSW_IN_TX; 539 pmu->raw_event_mask |= (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED); 540 } 541 542 perf_capabilities = vcpu_get_perf_capabilities(vcpu); 543 if (intel_pmu_lbr_is_compatible(vcpu) && 544 (perf_capabilities & PERF_CAP_LBR_FMT)) 545 memcpy(&lbr_desc->records, &vmx_lbr_caps, sizeof(vmx_lbr_caps)); 546 else 547 lbr_desc->records.nr = 0; 548 549 if (lbr_desc->records.nr) 550 bitmap_set(pmu->all_valid_pmc_idx, INTEL_PMC_IDX_FIXED_VLBR, 1); 551 552 if (pmu->version == 1) 553 return; 554 555 pmu->nr_arch_fixed_counters = min_t(int, edx.split.num_counters_fixed, 556 kvm_pmu_cap.num_counters_fixed); 557 edx.split.bit_width_fixed = min_t(int, edx.split.bit_width_fixed, 558 kvm_pmu_cap.bit_width_fixed); 559 pmu->counter_bitmask[KVM_PMC_FIXED] = BIT_ULL(edx.split.bit_width_fixed) - 1; 560 561 intel_pmu_enable_fixed_counter_bits(pmu, INTEL_FIXED_0_KERNEL | 562 INTEL_FIXED_0_USER | 563 INTEL_FIXED_0_ENABLE_PMI); 564 565 counter_rsvd = ~((BIT_ULL(pmu->nr_arch_gp_counters) - 1) | 566 ((BIT_ULL(pmu->nr_arch_fixed_counters) - 1) << KVM_FIXED_PMC_BASE_IDX)); 567 pmu->global_ctrl_rsvd = counter_rsvd; 568 569 /* 570 * GLOBAL_STATUS and GLOBAL_OVF_CONTROL (a.k.a. GLOBAL_STATUS_RESET) 571 * share reserved bit definitions. The kernel just happens to use 572 * OVF_CTRL for the names. 573 */ 574 pmu->global_status_rsvd = pmu->global_ctrl_rsvd 575 & ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF | 576 MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD); 577 if (vmx_pt_mode_is_host_guest()) 578 pmu->global_status_rsvd &= 579 ~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI; 580 581 if (perf_capabilities & PERF_CAP_PEBS_FORMAT) { 582 if (perf_capabilities & PERF_CAP_PEBS_BASELINE) { 583 pmu->pebs_enable_rsvd = counter_rsvd; 584 pmu->reserved_bits &= ~ICL_EVENTSEL_ADAPTIVE; 585 pmu->pebs_data_cfg_rsvd = ~0xff00000full; 586 intel_pmu_enable_fixed_counter_bits(pmu, ICL_FIXED_0_ADAPTIVE); 587 } else { 588 pmu->pebs_enable_rsvd = ~(BIT_ULL(pmu->nr_arch_gp_counters) - 1); 589 } 590 } 591 } 592 593 static void intel_pmu_init(struct kvm_vcpu *vcpu) 594 { 595 int i; 596 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 597 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 598 599 if (!lbr_desc) 600 return; 601 602 for (i = 0; i < KVM_MAX_NR_INTEL_GP_COUNTERS; i++) { 603 pmu->gp_counters[i].type = KVM_PMC_GP; 604 pmu->gp_counters[i].vcpu = vcpu; 605 pmu->gp_counters[i].idx = i; 606 pmu->gp_counters[i].current_config = 0; 607 } 608 609 for (i = 0; i < KVM_MAX_NR_INTEL_FIXED_COUNTERS; i++) { 610 pmu->fixed_counters[i].type = KVM_PMC_FIXED; 611 pmu->fixed_counters[i].vcpu = vcpu; 612 pmu->fixed_counters[i].idx = i + KVM_FIXED_PMC_BASE_IDX; 613 pmu->fixed_counters[i].current_config = 0; 614 pmu->fixed_counters[i].eventsel = intel_get_fixed_pmc_eventsel(i); 615 } 616 617 lbr_desc->records.nr = 0; 618 lbr_desc->event = NULL; 619 lbr_desc->msr_passthrough = false; 620 } 621 622 static void intel_pmu_reset(struct kvm_vcpu *vcpu) 623 { 624 intel_pmu_release_guest_lbr_event(vcpu); 625 } 626 627 /* 628 * Emulate LBR_On_PMI behavior for 1 < pmu.version < 4. 629 * 630 * If Freeze_LBR_On_PMI = 1, the LBR is frozen on PMI and 631 * the KVM emulates to clear the LBR bit (bit 0) in IA32_DEBUGCTL. 632 * 633 * Guest needs to re-enable LBR to resume branches recording. 634 */ 635 static void intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu *vcpu) 636 { 637 u64 data = vmx_guest_debugctl_read(); 638 639 if (data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) { 640 data &= ~DEBUGCTLMSR_LBR; 641 vmx_guest_debugctl_write(vcpu, data); 642 } 643 } 644 645 static void intel_pmu_deliver_pmi(struct kvm_vcpu *vcpu) 646 { 647 u8 version = vcpu_to_pmu(vcpu)->version; 648 649 if (!intel_pmu_lbr_is_enabled(vcpu)) 650 return; 651 652 if (version > 1 && version < 4) 653 intel_pmu_legacy_freezing_lbrs_on_pmi(vcpu); 654 } 655 656 static void vmx_update_intercept_for_lbr_msrs(struct kvm_vcpu *vcpu, bool set) 657 { 658 struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu); 659 int i; 660 661 for (i = 0; i < lbr->nr; i++) { 662 vmx_set_intercept_for_msr(vcpu, lbr->from + i, MSR_TYPE_RW, set); 663 vmx_set_intercept_for_msr(vcpu, lbr->to + i, MSR_TYPE_RW, set); 664 if (lbr->info) 665 vmx_set_intercept_for_msr(vcpu, lbr->info + i, MSR_TYPE_RW, set); 666 } 667 668 vmx_set_intercept_for_msr(vcpu, MSR_LBR_SELECT, MSR_TYPE_RW, set); 669 vmx_set_intercept_for_msr(vcpu, MSR_LBR_TOS, MSR_TYPE_RW, set); 670 } 671 672 static inline void vmx_disable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu) 673 { 674 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 675 676 if (!lbr_desc->msr_passthrough) 677 return; 678 679 vmx_update_intercept_for_lbr_msrs(vcpu, true); 680 lbr_desc->msr_passthrough = false; 681 } 682 683 static inline void vmx_enable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu) 684 { 685 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 686 687 if (lbr_desc->msr_passthrough) 688 return; 689 690 vmx_update_intercept_for_lbr_msrs(vcpu, false); 691 lbr_desc->msr_passthrough = true; 692 } 693 694 /* 695 * Higher priority host perf events (e.g. cpu pinned) could reclaim the 696 * pmu resources (e.g. LBR) that were assigned to the guest. This is 697 * usually done via ipi calls (more details in perf_install_in_context). 698 * 699 * Before entering the non-root mode (with irq disabled here), double 700 * confirm that the pmu features enabled to the guest are not reclaimed 701 * by higher priority host events. Otherwise, disallow vcpu's access to 702 * the reclaimed features. 703 */ 704 void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu) 705 { 706 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 707 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 708 709 if (WARN_ON_ONCE(!lbr_desc)) 710 return; 711 712 if (!lbr_desc->event) { 713 vmx_disable_lbr_msrs_passthrough(vcpu); 714 if (vmx_guest_debugctl_read() & DEBUGCTLMSR_LBR) 715 goto warn; 716 if (test_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use)) 717 goto warn; 718 return; 719 } 720 721 if (lbr_desc->event->state < PERF_EVENT_STATE_ACTIVE) { 722 vmx_disable_lbr_msrs_passthrough(vcpu); 723 __clear_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use); 724 goto warn; 725 } else 726 vmx_enable_lbr_msrs_passthrough(vcpu); 727 728 return; 729 730 warn: 731 pr_warn_ratelimited("vcpu-%d: fail to passthrough LBR.\n", vcpu->vcpu_id); 732 } 733 734 static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) 735 { 736 if (!(vmx_guest_debugctl_read() & DEBUGCTLMSR_LBR)) 737 intel_pmu_release_guest_lbr_event(vcpu); 738 } 739 740 void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu) 741 { 742 struct kvm_pmc *pmc = NULL; 743 int bit, hw_idx; 744 745 kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&pmu->global_ctrl) { 746 if (!pmc_is_locally_enabled(pmc) || 747 !pmc_is_globally_enabled(pmc) || !pmc->perf_event) 748 continue; 749 750 /* 751 * A negative index indicates the event isn't mapped to a 752 * physical counter in the host, e.g. due to contention. 753 */ 754 hw_idx = pmc->perf_event->hw.idx; 755 if (hw_idx != pmc->idx && hw_idx > -1) 756 pmu->host_cross_mapped_mask |= BIT_ULL(hw_idx); 757 } 758 } 759 760 static bool intel_pmu_is_mediated_pmu_supported(struct x86_pmu_capability *host_pmu) 761 { 762 u64 host_perf_cap = 0; 763 764 if (boot_cpu_has(X86_FEATURE_PDCM)) 765 rdmsrq(MSR_IA32_PERF_CAPABILITIES, host_perf_cap); 766 767 /* 768 * Require v4+ for MSR_CORE_PERF_GLOBAL_STATUS_SET, and full-width 769 * writes so that KVM can precisely load guest counter values. 770 */ 771 if (host_pmu->version < 4 || !(host_perf_cap & PERF_CAP_FW_WRITES)) 772 return false; 773 774 /* 775 * All CPUs that support a mediated PMU are expected to support loading 776 * PERF_GLOBAL_CTRL via dedicated VMCS fields. 777 */ 778 if (WARN_ON_ONCE(!cpu_has_load_perf_global_ctrl())) 779 return false; 780 781 return true; 782 } 783 784 static void intel_pmu_write_global_ctrl(u64 global_ctrl) 785 { 786 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL, global_ctrl); 787 } 788 789 790 static void intel_mediated_pmu_load(struct kvm_vcpu *vcpu) 791 { 792 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 793 u64 global_status, toggle; 794 795 rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, global_status); 796 toggle = pmu->global_status ^ global_status; 797 if (global_status & toggle) 798 wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, global_status & toggle); 799 if (pmu->global_status & toggle) 800 wrmsrq(MSR_CORE_PERF_GLOBAL_STATUS_SET, pmu->global_status & toggle); 801 802 wrmsrq(MSR_CORE_PERF_FIXED_CTR_CTRL, pmu->fixed_ctr_ctrl_hw); 803 } 804 805 static void intel_mediated_pmu_put(struct kvm_vcpu *vcpu) 806 { 807 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 808 809 /* MSR_CORE_PERF_GLOBAL_CTRL is already saved at VM-exit. */ 810 rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, pmu->global_status); 811 812 /* Clear hardware MSR_CORE_PERF_GLOBAL_STATUS MSR, if non-zero. */ 813 if (pmu->global_status) 814 wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, pmu->global_status); 815 816 /* 817 * Clear hardware FIXED_CTR_CTRL MSR to avoid information leakage and 818 * also to avoid accidentally enabling fixed counters (based on guest 819 * state) while running in the host, e.g. when setting global ctrl. 820 */ 821 if (pmu->fixed_ctr_ctrl_hw) 822 wrmsrq(MSR_CORE_PERF_FIXED_CTR_CTRL, 0); 823 } 824 825 struct kvm_pmu_ops intel_pmu_ops __initdata = { 826 .rdpmc_ecx_to_pmc = intel_rdpmc_ecx_to_pmc, 827 .msr_idx_to_pmc = intel_msr_idx_to_pmc, 828 .is_valid_msr = intel_is_valid_msr, 829 .get_msr = intel_pmu_get_msr, 830 .set_msr = intel_pmu_set_msr, 831 .refresh = intel_pmu_refresh, 832 .init = intel_pmu_init, 833 .reset = intel_pmu_reset, 834 .deliver_pmi = intel_pmu_deliver_pmi, 835 .cleanup = intel_pmu_cleanup, 836 837 .is_mediated_pmu_supported = intel_pmu_is_mediated_pmu_supported, 838 .mediated_load = intel_mediated_pmu_load, 839 .mediated_put = intel_mediated_pmu_put, 840 .write_global_ctrl = intel_pmu_write_global_ctrl, 841 842 .EVENTSEL_EVENT = ARCH_PERFMON_EVENTSEL_EVENT, 843 .MAX_NR_GP_COUNTERS = KVM_MAX_NR_INTEL_GP_COUNTERS, 844 .MIN_NR_GP_COUNTERS = 1, 845 846 .PERF_GLOBAL_CTRL = MSR_CORE_PERF_GLOBAL_CTRL, 847 .GP_EVENTSEL_BASE = MSR_P6_EVNTSEL0, 848 .GP_COUNTER_BASE = MSR_IA32_PMC0, 849 .FIXED_COUNTER_BASE = MSR_CORE_PERF_FIXED_CTR0, 850 .MSR_STRIDE = 1, 851 }; 852