1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023, Tencent, Inc. 4 */ 5 #include <x86intrin.h> 6 7 #include "pmu.h" 8 #include "processor.h" 9 10 /* Number of iterations of the loop for the guest measurement payload. */ 11 #define NUM_LOOPS 10 12 13 /* Each iteration of the loop retires one branch instruction. */ 14 #define NUM_BRANCH_INSNS_RETIRED (NUM_LOOPS) 15 16 /* 17 * Number of instructions in each loop. 1 ENTER, 1 CLFLUSH/CLFLUSHOPT/NOP, 18 * 1 MFENCE, 1 MOV, 1 LEAVE, 1 LOOP. 19 */ 20 #define NUM_INSNS_PER_LOOP 6 21 22 /* 23 * Number of "extra" instructions that will be counted, i.e. the number of 24 * instructions that are needed to set up the loop and then disable the 25 * counter. 2 MOV, 2 XOR, 1 WRMSR. 26 */ 27 #define NUM_EXTRA_INSNS 5 28 29 /* Total number of instructions retired within the measured section. */ 30 #define NUM_INSNS_RETIRED (NUM_LOOPS * NUM_INSNS_PER_LOOP + NUM_EXTRA_INSNS) 31 32 /* Track which architectural events are supported by hardware. */ 33 static u32 hardware_pmu_arch_events; 34 35 static u8 kvm_pmu_version; 36 static bool kvm_has_perf_caps; 37 38 #define X86_PMU_FEATURE_NULL \ 39 ({ \ 40 struct kvm_x86_pmu_feature feature = {}; \ 41 \ 42 feature; \ 43 }) 44 45 static bool pmu_is_null_feature(struct kvm_x86_pmu_feature event) 46 { 47 return !(*(u64 *)&event); 48 } 49 50 struct kvm_intel_pmu_event { 51 struct kvm_x86_pmu_feature gp_event; 52 struct kvm_x86_pmu_feature fixed_event; 53 }; 54 55 /* 56 * Wrap the array to appease the compiler, as the macros used to construct each 57 * kvm_x86_pmu_feature use syntax that's only valid in function scope, and the 58 * compiler often thinks the feature definitions aren't compile-time constants. 59 */ 60 static struct kvm_intel_pmu_event intel_event_to_feature(u8 idx) 61 { 62 const struct kvm_intel_pmu_event __intel_event_to_feature[] = { 63 [INTEL_ARCH_CPU_CYCLES_INDEX] = { X86_PMU_FEATURE_CPU_CYCLES, X86_PMU_FEATURE_CPU_CYCLES_FIXED }, 64 [INTEL_ARCH_INSTRUCTIONS_RETIRED_INDEX] = { X86_PMU_FEATURE_INSNS_RETIRED, X86_PMU_FEATURE_INSNS_RETIRED_FIXED }, 65 /* 66 * Note, the fixed counter for reference cycles is NOT the same as the 67 * general purpose architectural event. The fixed counter explicitly 68 * counts at the same frequency as the TSC, whereas the GP event counts 69 * at a fixed, but uarch specific, frequency. Bundle them here for 70 * simplicity. 71 */ 72 [INTEL_ARCH_REFERENCE_CYCLES_INDEX] = { X86_PMU_FEATURE_REFERENCE_CYCLES, X86_PMU_FEATURE_REFERENCE_TSC_CYCLES_FIXED }, 73 [INTEL_ARCH_LLC_REFERENCES_INDEX] = { X86_PMU_FEATURE_LLC_REFERENCES, X86_PMU_FEATURE_NULL }, 74 [INTEL_ARCH_LLC_MISSES_INDEX] = { X86_PMU_FEATURE_LLC_MISSES, X86_PMU_FEATURE_NULL }, 75 [INTEL_ARCH_BRANCHES_RETIRED_INDEX] = { X86_PMU_FEATURE_BRANCH_INSNS_RETIRED, X86_PMU_FEATURE_NULL }, 76 [INTEL_ARCH_BRANCHES_MISPREDICTED_INDEX] = { X86_PMU_FEATURE_BRANCHES_MISPREDICTED, X86_PMU_FEATURE_NULL }, 77 [INTEL_ARCH_TOPDOWN_SLOTS_INDEX] = { X86_PMU_FEATURE_TOPDOWN_SLOTS, X86_PMU_FEATURE_TOPDOWN_SLOTS_FIXED }, 78 [INTEL_ARCH_TOPDOWN_BE_BOUND_INDEX] = { X86_PMU_FEATURE_TOPDOWN_BE_BOUND, X86_PMU_FEATURE_NULL }, 79 [INTEL_ARCH_TOPDOWN_BAD_SPEC_INDEX] = { X86_PMU_FEATURE_TOPDOWN_BAD_SPEC, X86_PMU_FEATURE_NULL }, 80 [INTEL_ARCH_TOPDOWN_FE_BOUND_INDEX] = { X86_PMU_FEATURE_TOPDOWN_FE_BOUND, X86_PMU_FEATURE_NULL }, 81 [INTEL_ARCH_TOPDOWN_RETIRING_INDEX] = { X86_PMU_FEATURE_TOPDOWN_RETIRING, X86_PMU_FEATURE_NULL }, 82 [INTEL_ARCH_LBR_INSERTS_INDEX] = { X86_PMU_FEATURE_LBR_INSERTS, X86_PMU_FEATURE_NULL }, 83 }; 84 85 kvm_static_assert(ARRAY_SIZE(__intel_event_to_feature) == NR_INTEL_ARCH_EVENTS); 86 87 return __intel_event_to_feature[idx]; 88 } 89 90 static struct kvm_vm *pmu_vm_create_with_vcpus(u32 nr_vcpus, void *guest_code, 91 u8 pmu_version, 92 u64 perf_capabilities, 93 struct kvm_vcpu **__vcpus[]) 94 { 95 struct kvm_vcpu **vcpus = calloc(nr_vcpus, sizeof(*vcpus)); 96 struct kvm_vm *vm; 97 int i; 98 99 *__vcpus = vcpus; 100 101 vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); 102 sync_global_to_guest(vm, kvm_pmu_version); 103 sync_global_to_guest(vm, hardware_pmu_arch_events); 104 105 /* 106 * Set PERF_CAPABILITIES before PMU version as KVM disallows enabling 107 * features via PERF_CAPABILITIES if the guest doesn't have a vPMU. 108 */ 109 for (i = 0; i < nr_vcpus; i++) { 110 if (kvm_has_perf_caps) 111 vcpu_set_msr(vcpus[i], MSR_IA32_PERF_CAPABILITIES, perf_capabilities); 112 113 vcpu_set_cpuid_property(vcpus[i], X86_PROPERTY_PMU_VERSION, pmu_version); 114 } 115 116 return vm; 117 } 118 119 static void pmu_vm_free(struct kvm_vm *vm, struct kvm_vcpu **vcpus) 120 { 121 kvm_vm_free(vm); 122 free(vcpus); 123 } 124 125 static void run_vcpu(struct kvm_vcpu *vcpu) 126 { 127 struct ucall uc; 128 129 do { 130 vcpu_run(vcpu); 131 switch (get_ucall(vcpu, &uc)) { 132 case UCALL_SYNC: 133 break; 134 case UCALL_ABORT: 135 REPORT_GUEST_ASSERT(uc); 136 break; 137 case UCALL_PRINTF: 138 pr_info("%s", uc.buffer); 139 break; 140 case UCALL_DONE: 141 break; 142 default: 143 TEST_FAIL("Unexpected ucall: %lu", uc.cmd); 144 } 145 } while (uc.cmd != UCALL_DONE); 146 } 147 148 static u8 guest_get_pmu_version(void) 149 { 150 /* 151 * Return the effective PMU version, i.e. the minimum between what KVM 152 * supports and what is enumerated to the guest. The host deliberately 153 * advertises a PMU version to the guest beyond what is actually 154 * supported by KVM to verify KVM doesn't freak out and do something 155 * bizarre with an architecturally valid, but unsupported, version. 156 */ 157 return min_t(u8, kvm_pmu_version, this_cpu_property(X86_PROPERTY_PMU_VERSION)); 158 } 159 160 /* 161 * If an architectural event is supported and guaranteed to generate at least 162 * one "hit", assert that its count is non-zero. If an event isn't supported 163 * or the test can't guarantee the associated action will occur, then all bets 164 * are off regarding the count, i.e. no checks can be done. 165 * 166 * Sanity check that in all cases, the event doesn't count when it's disabled, 167 * and that KVM correctly emulates the write of an arbitrary value. 168 */ 169 static void guest_assert_event_count(u8 idx, u32 pmc, u32 pmc_msr) 170 { 171 u64 count; 172 173 count = _rdpmc(pmc); 174 if (!(hardware_pmu_arch_events & BIT(idx))) 175 goto sanity_checks; 176 177 switch (idx) { 178 case INTEL_ARCH_INSTRUCTIONS_RETIRED_INDEX: 179 /* Relax precise count check due to VM-EXIT/VM-ENTRY overcount issue */ 180 if (this_pmu_has_errata(INSTRUCTIONS_RETIRED_OVERCOUNT)) 181 GUEST_ASSERT(count >= NUM_INSNS_RETIRED); 182 else 183 GUEST_ASSERT_EQ(count, NUM_INSNS_RETIRED); 184 break; 185 case INTEL_ARCH_BRANCHES_RETIRED_INDEX: 186 /* Relax precise count check due to VM-EXIT/VM-ENTRY overcount issue */ 187 if (this_pmu_has_errata(BRANCHES_RETIRED_OVERCOUNT)) 188 GUEST_ASSERT(count >= NUM_BRANCH_INSNS_RETIRED); 189 else 190 GUEST_ASSERT_EQ(count, NUM_BRANCH_INSNS_RETIRED); 191 break; 192 case INTEL_ARCH_LLC_REFERENCES_INDEX: 193 case INTEL_ARCH_LLC_MISSES_INDEX: 194 if (!this_cpu_has(X86_FEATURE_CLFLUSHOPT) && 195 !this_cpu_has(X86_FEATURE_CLFLUSH)) 196 break; 197 fallthrough; 198 case INTEL_ARCH_CPU_CYCLES_INDEX: 199 case INTEL_ARCH_REFERENCE_CYCLES_INDEX: 200 case INTEL_ARCH_TOPDOWN_BE_BOUND_INDEX: 201 case INTEL_ARCH_TOPDOWN_FE_BOUND_INDEX: 202 GUEST_ASSERT_NE(count, 0); 203 break; 204 case INTEL_ARCH_TOPDOWN_SLOTS_INDEX: 205 case INTEL_ARCH_TOPDOWN_RETIRING_INDEX: 206 __GUEST_ASSERT(count >= NUM_INSNS_RETIRED, 207 "Expected top-down slots >= %u, got count = %lu", 208 NUM_INSNS_RETIRED, count); 209 break; 210 default: 211 break; 212 } 213 214 sanity_checks: 215 __asm__ __volatile__("loop ." : "+c"((int){NUM_LOOPS})); 216 GUEST_ASSERT_EQ(_rdpmc(pmc), count); 217 218 wrmsr(pmc_msr, 0xdead); 219 GUEST_ASSERT_EQ(_rdpmc(pmc), 0xdead); 220 } 221 222 /* 223 * Enable and disable the PMC in a monolithic asm blob to ensure that the 224 * compiler can't insert _any_ code into the measured sequence. Note, ECX 225 * doesn't need to be clobbered as the input value, @pmc_msr, is restored 226 * before the end of the sequence. 227 * 228 * If CLFUSH{,OPT} is supported, flush the cacheline containing (at least) the 229 * CLFUSH{,OPT} instruction on each loop iteration to force LLC references and 230 * misses, i.e. to allow testing that those events actually count. 231 * 232 * If forced emulation is enabled (and specified), force emulation on a subset 233 * of the measured code to verify that KVM correctly emulates instructions and 234 * branches retired events in conjunction with hardware also counting said 235 * events. 236 */ 237 #define GUEST_MEASURE_EVENT(_msr, _value, clflush, FEP) \ 238 do { \ 239 __asm__ __volatile__("wrmsr\n\t" \ 240 " mov $" __stringify(NUM_LOOPS) ", %%ecx\n\t" \ 241 "1:\n\t" \ 242 FEP "enter $0, $0\n\t" \ 243 clflush "\n\t" \ 244 "mfence\n\t" \ 245 "mov %[m], %%eax\n\t" \ 246 FEP "leave\n\t" \ 247 FEP "loop 1b\n\t" \ 248 FEP "mov %%edi, %%ecx\n\t" \ 249 FEP "xor %%eax, %%eax\n\t" \ 250 FEP "xor %%edx, %%edx\n\t" \ 251 "wrmsr\n\t" \ 252 :: "a"((u32)_value), "d"(_value >> 32), \ 253 "c"(_msr), "D"(_msr), [m]"m"(kvm_pmu_version) \ 254 ); \ 255 } while (0) 256 257 #define GUEST_TEST_EVENT(_idx, _pmc, _pmc_msr, _ctrl_msr, _value, FEP) \ 258 do { \ 259 wrmsr(_pmc_msr, 0); \ 260 \ 261 if (this_cpu_has(X86_FEATURE_CLFLUSHOPT)) \ 262 GUEST_MEASURE_EVENT(_ctrl_msr, _value, "clflushopt %[m]", FEP); \ 263 else if (this_cpu_has(X86_FEATURE_CLFLUSH)) \ 264 GUEST_MEASURE_EVENT(_ctrl_msr, _value, "clflush %[m]", FEP); \ 265 else \ 266 GUEST_MEASURE_EVENT(_ctrl_msr, _value, "nop", FEP); \ 267 \ 268 guest_assert_event_count(_idx, _pmc, _pmc_msr); \ 269 } while (0) 270 271 static void __guest_test_arch_event(u8 idx, u32 pmc, u32 pmc_msr, 272 u32 ctrl_msr, u64 ctrl_msr_value) 273 { 274 GUEST_TEST_EVENT(idx, pmc, pmc_msr, ctrl_msr, ctrl_msr_value, ""); 275 276 if (is_forced_emulation_enabled) 277 GUEST_TEST_EVENT(idx, pmc, pmc_msr, ctrl_msr, ctrl_msr_value, KVM_FEP); 278 } 279 280 static void guest_test_arch_event(u8 idx) 281 { 282 u32 nr_gp_counters = this_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS); 283 u32 pmu_version = guest_get_pmu_version(); 284 /* PERF_GLOBAL_CTRL exists only for Architectural PMU Version 2+. */ 285 bool guest_has_perf_global_ctrl = pmu_version >= 2; 286 struct kvm_x86_pmu_feature gp_event, fixed_event; 287 u32 base_pmc_msr; 288 unsigned int i; 289 u64 eventsel; 290 291 /* The host side shouldn't invoke this without a guest PMU. */ 292 GUEST_ASSERT(pmu_version); 293 294 if (this_cpu_has(X86_FEATURE_PDCM) && 295 rdmsr(MSR_IA32_PERF_CAPABILITIES) & PMU_CAP_FW_WRITES) 296 base_pmc_msr = MSR_IA32_PMC0; 297 else 298 base_pmc_msr = MSR_IA32_PERFCTR0; 299 300 gp_event = intel_event_to_feature(idx).gp_event; 301 GUEST_ASSERT_EQ(idx, gp_event.f.bit); 302 303 GUEST_ASSERT(nr_gp_counters); 304 i = kvm_random_u32_in_range(&kvm_rng, 0, nr_gp_counters - 1); 305 306 eventsel = ARCH_PERFMON_EVENTSEL_OS | ARCH_PERFMON_EVENTSEL_ENABLE | 307 intel_pmu_arch_events[idx]; 308 309 wrmsr(MSR_P6_EVNTSEL0 + i, 0); 310 if (guest_has_perf_global_ctrl) 311 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, BIT_ULL(i)); 312 313 __guest_test_arch_event(idx, i, base_pmc_msr + i, MSR_P6_EVNTSEL0 + i, eventsel); 314 315 if (!guest_has_perf_global_ctrl) 316 return; 317 318 fixed_event = intel_event_to_feature(idx).fixed_event; 319 if (pmu_is_null_feature(fixed_event) || !this_pmu_has(fixed_event)) 320 return; 321 322 i = fixed_event.f.bit; 323 324 wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(i, FIXED_PMC_KERNEL)); 325 326 __guest_test_arch_event(idx, i | INTEL_RDPMC_FIXED, 327 MSR_CORE_PERF_FIXED_CTR0 + i, 328 MSR_CORE_PERF_GLOBAL_CTRL, 329 FIXED_PMC_GLOBAL_CTRL_ENABLE(i)); 330 } 331 332 static void guest_test_arch_events(void) 333 { 334 u8 i; 335 336 for (i = 0; i < NR_INTEL_ARCH_EVENTS; i++) 337 guest_test_arch_event(i); 338 339 GUEST_DONE(); 340 } 341 342 static void __test_arch_events(struct kvm_vcpu *vcpu, u8 length, u32 unavailable_mask) 343 { 344 unavailable_mask &= GENMASK(X86_PROPERTY_PMU_EVENTS_MASK.hi_bit, 345 X86_PROPERTY_PMU_EVENTS_MASK.lo_bit); 346 347 vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH, 348 length); 349 vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_EVENTS_MASK, 350 unavailable_mask); 351 352 run_vcpu(vcpu); 353 } 354 355 static void test_arch_events(u8 pmu_version, u64 perf_capabilities) 356 { 357 struct kvm_vcpu **vcpus; 358 struct kvm_vm *vm; 359 int i = 0; 360 u32 k; 361 u8 j; 362 363 /* 364 * To keep the total runtime reasonable, test only a handful of select, 365 * semi-arbitrary values for the mask of unavailable PMU events. Test 366 * 0 (all events available) and all ones (no events available) as well 367 * as alternating bit sequences, e.g. to detect if KVM is checking the 368 * wrong bit(s). 369 */ 370 const u32 unavailable_masks[] = { 371 0x0, 372 0xffffffffu, 373 0xaaaaaaaau, 374 0x55555555u, 375 0xf0f0f0f0u, 376 0x0f0f0f0fu, 377 0xa0a0a0a0u, 378 0x0a0a0a0au, 379 0x50505050u, 380 0x05050505u, 381 }; 382 383 pr_info("Testing arch events, PMU version %u, perf_caps = %lx\n", 384 pmu_version, perf_capabilities); 385 386 /* Testing arch events requires a vPMU (there are no negative tests). */ 387 if (!pmu_version) 388 return; 389 390 vm = pmu_vm_create_with_vcpus((NR_INTEL_ARCH_EVENTS + 2) * (ARRAY_SIZE(unavailable_masks) - 1), 391 guest_test_arch_events, pmu_version, 392 perf_capabilities, &vcpus); 393 394 /* 395 * Test single bits for all PMU version and lengths up the number of 396 * events +1 (to verify KVM doesn't do weird things if the guest length 397 * is greater than the host length). Explicitly test a mask of '0' and 398 * all ones i.e. all events being available and unavailable. 399 */ 400 for (j = 0; j <= NR_INTEL_ARCH_EVENTS + 1; j++) { 401 for (k = 1; k < ARRAY_SIZE(unavailable_masks); k++) 402 __test_arch_events(vcpus[i++], j, unavailable_masks[k]); 403 } 404 405 pmu_vm_free(vm, vcpus); 406 } 407 408 /* 409 * Limit testing to MSRs that are actually defined by Intel (in the SDM). MSRs 410 * that aren't defined counter MSRs *probably* don't exist, but there's no 411 * guarantee that currently undefined MSR indices won't be used for something 412 * other than PMCs in the future. 413 */ 414 #define MAX_NR_GP_COUNTERS 8 415 #define MAX_NR_FIXED_COUNTERS 3 416 417 #define GUEST_ASSERT_PMC_MSR_ACCESS(insn, msr, expect_gp, vector) \ 418 __GUEST_ASSERT(expect_gp ? vector == GP_VECTOR : !vector, \ 419 "Expected %s on " #insn "(0x%x), got %s", \ 420 expect_gp ? "#GP" : "no fault", msr, ex_str(vector)) \ 421 422 #define GUEST_ASSERT_PMC_VALUE(insn, msr, val, expected) \ 423 __GUEST_ASSERT(val == expected, \ 424 "Expected " #insn "(0x%x) to yield 0x%lx, got 0x%lx", \ 425 msr, expected, val); 426 427 static void guest_test_rdpmc(u32 rdpmc_idx, bool expect_success, 428 u64 expected_val) 429 { 430 u8 vector; 431 u64 val; 432 433 vector = rdpmc_safe(rdpmc_idx, &val); 434 GUEST_ASSERT_PMC_MSR_ACCESS(RDPMC, rdpmc_idx, !expect_success, vector); 435 if (expect_success) 436 GUEST_ASSERT_PMC_VALUE(RDPMC, rdpmc_idx, val, expected_val); 437 438 if (!is_forced_emulation_enabled) 439 return; 440 441 vector = rdpmc_safe_fep(rdpmc_idx, &val); 442 GUEST_ASSERT_PMC_MSR_ACCESS(RDPMC, rdpmc_idx, !expect_success, vector); 443 if (expect_success) 444 GUEST_ASSERT_PMC_VALUE(RDPMC, rdpmc_idx, val, expected_val); 445 } 446 447 static void guest_rd_wr_counters(u32 base_msr, u8 nr_possible_counters, 448 u8 nr_counters, u32 or_mask) 449 { 450 const bool pmu_has_fast_mode = !guest_get_pmu_version(); 451 u8 i; 452 453 for (i = 0; i < nr_possible_counters; i++) { 454 /* 455 * TODO: Test a value that validates full-width writes and the 456 * width of the counters. 457 */ 458 const u64 test_val = 0xffff; 459 const u32 msr = base_msr + i; 460 461 /* 462 * Fixed counters are supported if the counter is less than the 463 * number of enumerated contiguous counters *or* the counter is 464 * explicitly enumerated in the supported counters mask. 465 */ 466 const bool expect_success = i < nr_counters || (or_mask & BIT(i)); 467 468 /* 469 * KVM drops writes to MSR_P6_PERFCTR[0|1] if the counters are 470 * unsupported, i.e. doesn't #GP and reads back '0'. 471 */ 472 const u64 expected_val = expect_success ? test_val : 0; 473 const bool expect_gp = !expect_success && msr != MSR_P6_PERFCTR0 && 474 msr != MSR_P6_PERFCTR1; 475 u32 rdpmc_idx; 476 u8 vector; 477 u64 val; 478 479 vector = wrmsr_safe(msr, test_val); 480 GUEST_ASSERT_PMC_MSR_ACCESS(WRMSR, msr, expect_gp, vector); 481 482 vector = rdmsr_safe(msr, &val); 483 GUEST_ASSERT_PMC_MSR_ACCESS(RDMSR, msr, expect_gp, vector); 484 485 /* On #GP, the result of RDMSR is undefined. */ 486 if (!expect_gp) 487 GUEST_ASSERT_PMC_VALUE(RDMSR, msr, val, expected_val); 488 489 /* 490 * Redo the read tests with RDPMC, which has different indexing 491 * semantics and additional capabilities. 492 */ 493 rdpmc_idx = i; 494 if (base_msr == MSR_CORE_PERF_FIXED_CTR0) 495 rdpmc_idx |= INTEL_RDPMC_FIXED; 496 497 guest_test_rdpmc(rdpmc_idx, expect_success, expected_val); 498 499 /* 500 * KVM doesn't support non-architectural PMUs, i.e. it should 501 * impossible to have fast mode RDPMC. Verify that attempting 502 * to use fast RDPMC always #GPs. 503 */ 504 GUEST_ASSERT(!expect_success || !pmu_has_fast_mode); 505 rdpmc_idx |= INTEL_RDPMC_FAST; 506 guest_test_rdpmc(rdpmc_idx, false, -1ull); 507 508 vector = wrmsr_safe(msr, 0); 509 GUEST_ASSERT_PMC_MSR_ACCESS(WRMSR, msr, expect_gp, vector); 510 } 511 } 512 513 static void guest_test_gp_counters(void) 514 { 515 u8 pmu_version = guest_get_pmu_version(); 516 u8 nr_gp_counters = 0; 517 u32 base_msr; 518 519 if (pmu_version) 520 nr_gp_counters = this_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS); 521 522 /* 523 * For v2+ PMUs, PERF_GLOBAL_CTRL's architectural post-RESET value is 524 * "Sets bits n-1:0 and clears the upper bits", where 'n' is the number 525 * of GP counters. If there are no GP counters, require KVM to leave 526 * PERF_GLOBAL_CTRL '0'. This edge case isn't covered by the SDM, but 527 * follow the spirit of the architecture and only globally enable GP 528 * counters, of which there are none. 529 */ 530 if (pmu_version > 1) { 531 u64 global_ctrl = rdmsr(MSR_CORE_PERF_GLOBAL_CTRL); 532 533 if (nr_gp_counters) 534 GUEST_ASSERT_EQ(global_ctrl, GENMASK_ULL(nr_gp_counters - 1, 0)); 535 else 536 GUEST_ASSERT_EQ(global_ctrl, 0); 537 } 538 539 if (this_cpu_has(X86_FEATURE_PDCM) && 540 rdmsr(MSR_IA32_PERF_CAPABILITIES) & PMU_CAP_FW_WRITES) 541 base_msr = MSR_IA32_PMC0; 542 else 543 base_msr = MSR_IA32_PERFCTR0; 544 545 guest_rd_wr_counters(base_msr, MAX_NR_GP_COUNTERS, nr_gp_counters, 0); 546 GUEST_DONE(); 547 } 548 549 static void test_gp_counters(u8 pmu_version, u64 perf_capabilities) 550 { 551 u8 nr_gp_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS); 552 struct kvm_vcpu **vcpus; 553 struct kvm_vm *vm; 554 u8 j; 555 556 pr_info("Testing %u GP counters, PMU version %u, perf_caps = %lx\n", 557 nr_gp_counters, pmu_version, perf_capabilities); 558 559 vm = pmu_vm_create_with_vcpus(nr_gp_counters + 1, guest_test_gp_counters, 560 pmu_version, perf_capabilities, &vcpus); 561 562 for (j = 0; j <= nr_gp_counters; j++) { 563 vcpu_set_cpuid_property(vcpus[j], X86_PROPERTY_PMU_NR_GP_COUNTERS, j); 564 565 run_vcpu(vcpus[j]); 566 } 567 568 pmu_vm_free(vm, vcpus); 569 } 570 571 static void guest_test_fixed_counters(void) 572 { 573 u64 supported_bitmask = 0; 574 u8 nr_fixed_counters = 0; 575 u8 i; 576 577 /* Fixed counters require Architectural vPMU Version 2+. */ 578 if (guest_get_pmu_version() >= 2) 579 nr_fixed_counters = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); 580 581 /* 582 * The supported bitmask for fixed counters was introduced in PMU 583 * version 5. 584 */ 585 if (guest_get_pmu_version() >= 5) 586 supported_bitmask = this_cpu_property(X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK); 587 588 guest_rd_wr_counters(MSR_CORE_PERF_FIXED_CTR0, MAX_NR_FIXED_COUNTERS, 589 nr_fixed_counters, supported_bitmask); 590 591 for (i = 0; i < MAX_NR_FIXED_COUNTERS; i++) { 592 u8 vector; 593 u64 val; 594 595 if (i >= nr_fixed_counters && !(supported_bitmask & BIT_ULL(i))) { 596 vector = wrmsr_safe(MSR_CORE_PERF_FIXED_CTR_CTRL, 597 FIXED_PMC_CTRL(i, FIXED_PMC_KERNEL)); 598 __GUEST_ASSERT(vector == GP_VECTOR, 599 "Expected #GP for counter %u in FIXED_CTR_CTRL", i); 600 601 vector = wrmsr_safe(MSR_CORE_PERF_GLOBAL_CTRL, 602 FIXED_PMC_GLOBAL_CTRL_ENABLE(i)); 603 __GUEST_ASSERT(vector == GP_VECTOR, 604 "Expected #GP for counter %u in PERF_GLOBAL_CTRL", i); 605 continue; 606 } 607 608 wrmsr(MSR_CORE_PERF_FIXED_CTR0 + i, 0); 609 wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(i, FIXED_PMC_KERNEL)); 610 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, FIXED_PMC_GLOBAL_CTRL_ENABLE(i)); 611 __asm__ __volatile__("loop ." : "+c"((int){NUM_LOOPS})); 612 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0); 613 val = rdmsr(MSR_CORE_PERF_FIXED_CTR0 + i); 614 615 GUEST_ASSERT_NE(val, 0); 616 } 617 GUEST_DONE(); 618 } 619 620 static void __test_fixed_counters(struct kvm_vcpu *vcpu, u8 nr_fixed_counters, 621 u32 supported_bitmask) 622 { 623 vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK, 624 supported_bitmask); 625 vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_NR_FIXED_COUNTERS, 626 nr_fixed_counters); 627 628 run_vcpu(vcpu); 629 } 630 631 static void test_fixed_counters(u8 pmu_version, u64 perf_capabilities) 632 { 633 u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); 634 struct kvm_vcpu **vcpus; 635 struct kvm_vm *vm; 636 int i = 0; 637 u32 k; 638 u8 j; 639 640 pr_info("Testing %u fixed counters, PMU version %u, perf_caps = %lx\n", 641 nr_fixed_counters, pmu_version, perf_capabilities); 642 643 644 vm = pmu_vm_create_with_vcpus((nr_fixed_counters + 1) * BIT(nr_fixed_counters), 645 guest_test_fixed_counters, 646 pmu_version, perf_capabilities, &vcpus); 647 648 for (j = 0; j <= nr_fixed_counters; j++) { 649 for (k = 0; k <= (BIT(nr_fixed_counters) - 1); k++) 650 __test_fixed_counters(vcpus[i++], j, k); 651 } 652 653 pmu_vm_free(vm, vcpus); 654 } 655 656 static void test_intel_counters(void) 657 { 658 u8 pmu_version = kvm_cpu_property(X86_PROPERTY_PMU_VERSION); 659 unsigned int i; 660 u8 v; 661 662 const u64 perf_caps[] = { 663 0, 664 PMU_CAP_FW_WRITES, 665 }; 666 667 /* 668 * Test up to PMU v5, which is the current maximum version defined by 669 * Intel, i.e. is the last version that is guaranteed to be backwards 670 * compatible with KVM's existing behavior. 671 */ 672 u8 max_pmu_version = max_t(typeof(pmu_version), pmu_version, 5); 673 674 /* 675 * Detect the existence of events that aren't supported by selftests. 676 * This will (obviously) fail any time hardware adds support for a new 677 * event, but it's worth paying that price to keep the test fresh. 678 */ 679 TEST_ASSERT(this_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH) <= NR_INTEL_ARCH_EVENTS, 680 "New architectural event(s) detected; please update this test (length = %u, mask = %x)", 681 this_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH), 682 this_cpu_property(X86_PROPERTY_PMU_EVENTS_MASK)); 683 684 /* 685 * Iterate over known arch events irrespective of KVM/hardware support 686 * to verify that KVM doesn't reject programming of events just because 687 * the *architectural* encoding is unsupported. Track which events are 688 * supported in hardware; the guest side will validate supported events 689 * count correctly, even if *enumeration* of the event is unsupported 690 * by KVM and/or isn't exposed to the guest. 691 */ 692 for (i = 0; i < NR_INTEL_ARCH_EVENTS; i++) { 693 if (this_pmu_has(intel_event_to_feature(i).gp_event)) 694 hardware_pmu_arch_events |= BIT(i); 695 } 696 697 for (v = 0; v <= max_pmu_version; v++) { 698 for (i = 0; i < ARRAY_SIZE(perf_caps); i++) { 699 if (!kvm_has_perf_caps && perf_caps[i]) 700 continue; 701 702 test_arch_events(v, perf_caps[i]); 703 test_gp_counters(v, perf_caps[i]); 704 test_fixed_counters(v, perf_caps[i]); 705 } 706 } 707 } 708 709 int main(int argc, char *argv[]) 710 { 711 TEST_REQUIRE(kvm_is_pmu_enabled()); 712 713 TEST_REQUIRE(host_cpu_is_intel); 714 TEST_REQUIRE(kvm_cpu_has_p(X86_PROPERTY_PMU_VERSION)); 715 TEST_REQUIRE(kvm_cpu_property(X86_PROPERTY_PMU_VERSION) > 0); 716 717 kvm_pmu_version = kvm_cpu_property(X86_PROPERTY_PMU_VERSION); 718 kvm_has_perf_caps = kvm_cpu_has(X86_FEATURE_PDCM); 719 720 test_intel_counters(); 721 722 return 0; 723 } 724