1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023, Tencent, Inc. 4 */ 5 #include <x86intrin.h> 6 7 #include "pmu.h" 8 #include "processor.h" 9 10 /* Number of iterations of the loop for the guest measurement payload. */ 11 #define NUM_LOOPS 10 12 13 /* Each iteration of the loop retires one branch instruction. */ 14 #define NUM_BRANCH_INSNS_RETIRED (NUM_LOOPS) 15 16 /* 17 * Number of instructions in each loop. 1 ENTER, 1 CLFLUSH/CLFLUSHOPT/NOP, 18 * 1 MFENCE, 1 MOV, 1 LEAVE, 1 LOOP. 19 */ 20 #define NUM_INSNS_PER_LOOP 6 21 22 /* 23 * Number of "extra" instructions that will be counted, i.e. the number of 24 * instructions that are needed to set up the loop and then disable the 25 * counter. 2 MOV, 2 XOR, 1 WRMSR. 26 */ 27 #define NUM_EXTRA_INSNS 5 28 29 /* Total number of instructions retired within the measured section. */ 30 #define NUM_INSNS_RETIRED (NUM_LOOPS * NUM_INSNS_PER_LOOP + NUM_EXTRA_INSNS) 31 32 /* Track which architectural events are supported by hardware. */ 33 static uint32_t hardware_pmu_arch_events; 34 35 static uint8_t kvm_pmu_version; 36 static bool kvm_has_perf_caps; 37 38 #define X86_PMU_FEATURE_NULL \ 39 ({ \ 40 struct kvm_x86_pmu_feature feature = {}; \ 41 \ 42 feature; \ 43 }) 44 45 static bool pmu_is_null_feature(struct kvm_x86_pmu_feature event) 46 { 47 return !(*(u64 *)&event); 48 } 49 50 struct kvm_intel_pmu_event { 51 struct kvm_x86_pmu_feature gp_event; 52 struct kvm_x86_pmu_feature fixed_event; 53 }; 54 55 /* 56 * Wrap the array to appease the compiler, as the macros used to construct each 57 * kvm_x86_pmu_feature use syntax that's only valid in function scope, and the 58 * compiler often thinks the feature definitions aren't compile-time constants. 59 */ 60 static struct kvm_intel_pmu_event intel_event_to_feature(uint8_t idx) 61 { 62 const struct kvm_intel_pmu_event __intel_event_to_feature[] = { 63 [INTEL_ARCH_CPU_CYCLES_INDEX] = { X86_PMU_FEATURE_CPU_CYCLES, X86_PMU_FEATURE_CPU_CYCLES_FIXED }, 64 [INTEL_ARCH_INSTRUCTIONS_RETIRED_INDEX] = { X86_PMU_FEATURE_INSNS_RETIRED, X86_PMU_FEATURE_INSNS_RETIRED_FIXED }, 65 /* 66 * Note, the fixed counter for reference cycles is NOT the same as the 67 * general purpose architectural event. The fixed counter explicitly 68 * counts at the same frequency as the TSC, whereas the GP event counts 69 * at a fixed, but uarch specific, frequency. Bundle them here for 70 * simplicity. 71 */ 72 [INTEL_ARCH_REFERENCE_CYCLES_INDEX] = { X86_PMU_FEATURE_REFERENCE_CYCLES, X86_PMU_FEATURE_REFERENCE_TSC_CYCLES_FIXED }, 73 [INTEL_ARCH_LLC_REFERENCES_INDEX] = { X86_PMU_FEATURE_LLC_REFERENCES, X86_PMU_FEATURE_NULL }, 74 [INTEL_ARCH_LLC_MISSES_INDEX] = { X86_PMU_FEATURE_LLC_MISSES, X86_PMU_FEATURE_NULL }, 75 [INTEL_ARCH_BRANCHES_RETIRED_INDEX] = { X86_PMU_FEATURE_BRANCH_INSNS_RETIRED, X86_PMU_FEATURE_NULL }, 76 [INTEL_ARCH_BRANCHES_MISPREDICTED_INDEX] = { X86_PMU_FEATURE_BRANCHES_MISPREDICTED, X86_PMU_FEATURE_NULL }, 77 [INTEL_ARCH_TOPDOWN_SLOTS_INDEX] = { X86_PMU_FEATURE_TOPDOWN_SLOTS, X86_PMU_FEATURE_TOPDOWN_SLOTS_FIXED }, 78 }; 79 80 kvm_static_assert(ARRAY_SIZE(__intel_event_to_feature) == NR_INTEL_ARCH_EVENTS); 81 82 return __intel_event_to_feature[idx]; 83 } 84 85 static struct kvm_vm *pmu_vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 86 void *guest_code, 87 uint8_t pmu_version, 88 uint64_t perf_capabilities) 89 { 90 struct kvm_vm *vm; 91 92 vm = vm_create_with_one_vcpu(vcpu, guest_code); 93 sync_global_to_guest(vm, kvm_pmu_version); 94 sync_global_to_guest(vm, hardware_pmu_arch_events); 95 96 /* 97 * Set PERF_CAPABILITIES before PMU version as KVM disallows enabling 98 * features via PERF_CAPABILITIES if the guest doesn't have a vPMU. 99 */ 100 if (kvm_has_perf_caps) 101 vcpu_set_msr(*vcpu, MSR_IA32_PERF_CAPABILITIES, perf_capabilities); 102 103 vcpu_set_cpuid_property(*vcpu, X86_PROPERTY_PMU_VERSION, pmu_version); 104 return vm; 105 } 106 107 static void run_vcpu(struct kvm_vcpu *vcpu) 108 { 109 struct ucall uc; 110 111 do { 112 vcpu_run(vcpu); 113 switch (get_ucall(vcpu, &uc)) { 114 case UCALL_SYNC: 115 break; 116 case UCALL_ABORT: 117 REPORT_GUEST_ASSERT(uc); 118 break; 119 case UCALL_PRINTF: 120 pr_info("%s", uc.buffer); 121 break; 122 case UCALL_DONE: 123 break; 124 default: 125 TEST_FAIL("Unexpected ucall: %lu", uc.cmd); 126 } 127 } while (uc.cmd != UCALL_DONE); 128 } 129 130 static uint8_t guest_get_pmu_version(void) 131 { 132 /* 133 * Return the effective PMU version, i.e. the minimum between what KVM 134 * supports and what is enumerated to the guest. The host deliberately 135 * advertises a PMU version to the guest beyond what is actually 136 * supported by KVM to verify KVM doesn't freak out and do something 137 * bizarre with an architecturally valid, but unsupported, version. 138 */ 139 return min_t(uint8_t, kvm_pmu_version, this_cpu_property(X86_PROPERTY_PMU_VERSION)); 140 } 141 142 /* 143 * If an architectural event is supported and guaranteed to generate at least 144 * one "hit, assert that its count is non-zero. If an event isn't supported or 145 * the test can't guarantee the associated action will occur, then all bets are 146 * off regarding the count, i.e. no checks can be done. 147 * 148 * Sanity check that in all cases, the event doesn't count when it's disabled, 149 * and that KVM correctly emulates the write of an arbitrary value. 150 */ 151 static void guest_assert_event_count(uint8_t idx, uint32_t pmc, uint32_t pmc_msr) 152 { 153 uint64_t count; 154 155 count = _rdpmc(pmc); 156 if (!(hardware_pmu_arch_events & BIT(idx))) 157 goto sanity_checks; 158 159 switch (idx) { 160 case INTEL_ARCH_INSTRUCTIONS_RETIRED_INDEX: 161 GUEST_ASSERT_EQ(count, NUM_INSNS_RETIRED); 162 break; 163 case INTEL_ARCH_BRANCHES_RETIRED_INDEX: 164 GUEST_ASSERT_EQ(count, NUM_BRANCH_INSNS_RETIRED); 165 break; 166 case INTEL_ARCH_LLC_REFERENCES_INDEX: 167 case INTEL_ARCH_LLC_MISSES_INDEX: 168 if (!this_cpu_has(X86_FEATURE_CLFLUSHOPT) && 169 !this_cpu_has(X86_FEATURE_CLFLUSH)) 170 break; 171 fallthrough; 172 case INTEL_ARCH_CPU_CYCLES_INDEX: 173 case INTEL_ARCH_REFERENCE_CYCLES_INDEX: 174 GUEST_ASSERT_NE(count, 0); 175 break; 176 case INTEL_ARCH_TOPDOWN_SLOTS_INDEX: 177 __GUEST_ASSERT(count >= NUM_INSNS_RETIRED, 178 "Expected top-down slots >= %u, got count = %lu", 179 NUM_INSNS_RETIRED, count); 180 break; 181 default: 182 break; 183 } 184 185 sanity_checks: 186 __asm__ __volatile__("loop ." : "+c"((int){NUM_LOOPS})); 187 GUEST_ASSERT_EQ(_rdpmc(pmc), count); 188 189 wrmsr(pmc_msr, 0xdead); 190 GUEST_ASSERT_EQ(_rdpmc(pmc), 0xdead); 191 } 192 193 /* 194 * Enable and disable the PMC in a monolithic asm blob to ensure that the 195 * compiler can't insert _any_ code into the measured sequence. Note, ECX 196 * doesn't need to be clobbered as the input value, @pmc_msr, is restored 197 * before the end of the sequence. 198 * 199 * If CLFUSH{,OPT} is supported, flush the cacheline containing (at least) the 200 * CLFUSH{,OPT} instruction on each loop iteration to force LLC references and 201 * misses, i.e. to allow testing that those events actually count. 202 * 203 * If forced emulation is enabled (and specified), force emulation on a subset 204 * of the measured code to verify that KVM correctly emulates instructions and 205 * branches retired events in conjunction with hardware also counting said 206 * events. 207 */ 208 #define GUEST_MEASURE_EVENT(_msr, _value, clflush, FEP) \ 209 do { \ 210 __asm__ __volatile__("wrmsr\n\t" \ 211 " mov $" __stringify(NUM_LOOPS) ", %%ecx\n\t" \ 212 "1:\n\t" \ 213 FEP "enter $0, $0\n\t" \ 214 clflush "\n\t" \ 215 "mfence\n\t" \ 216 "mov %[m], %%eax\n\t" \ 217 FEP "leave\n\t" \ 218 FEP "loop 1b\n\t" \ 219 FEP "mov %%edi, %%ecx\n\t" \ 220 FEP "xor %%eax, %%eax\n\t" \ 221 FEP "xor %%edx, %%edx\n\t" \ 222 "wrmsr\n\t" \ 223 :: "a"((uint32_t)_value), "d"(_value >> 32), \ 224 "c"(_msr), "D"(_msr), [m]"m"(kvm_pmu_version) \ 225 ); \ 226 } while (0) 227 228 #define GUEST_TEST_EVENT(_idx, _pmc, _pmc_msr, _ctrl_msr, _value, FEP) \ 229 do { \ 230 wrmsr(_pmc_msr, 0); \ 231 \ 232 if (this_cpu_has(X86_FEATURE_CLFLUSHOPT)) \ 233 GUEST_MEASURE_EVENT(_ctrl_msr, _value, "clflushopt %[m]", FEP); \ 234 else if (this_cpu_has(X86_FEATURE_CLFLUSH)) \ 235 GUEST_MEASURE_EVENT(_ctrl_msr, _value, "clflush %[m]", FEP); \ 236 else \ 237 GUEST_MEASURE_EVENT(_ctrl_msr, _value, "nop", FEP); \ 238 \ 239 guest_assert_event_count(_idx, _pmc, _pmc_msr); \ 240 } while (0) 241 242 static void __guest_test_arch_event(uint8_t idx, uint32_t pmc, uint32_t pmc_msr, 243 uint32_t ctrl_msr, uint64_t ctrl_msr_value) 244 { 245 GUEST_TEST_EVENT(idx, pmc, pmc_msr, ctrl_msr, ctrl_msr_value, ""); 246 247 if (is_forced_emulation_enabled) 248 GUEST_TEST_EVENT(idx, pmc, pmc_msr, ctrl_msr, ctrl_msr_value, KVM_FEP); 249 } 250 251 static void guest_test_arch_event(uint8_t idx) 252 { 253 uint32_t nr_gp_counters = this_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS); 254 uint32_t pmu_version = guest_get_pmu_version(); 255 /* PERF_GLOBAL_CTRL exists only for Architectural PMU Version 2+. */ 256 bool guest_has_perf_global_ctrl = pmu_version >= 2; 257 struct kvm_x86_pmu_feature gp_event, fixed_event; 258 uint32_t base_pmc_msr; 259 unsigned int i; 260 261 /* The host side shouldn't invoke this without a guest PMU. */ 262 GUEST_ASSERT(pmu_version); 263 264 if (this_cpu_has(X86_FEATURE_PDCM) && 265 rdmsr(MSR_IA32_PERF_CAPABILITIES) & PMU_CAP_FW_WRITES) 266 base_pmc_msr = MSR_IA32_PMC0; 267 else 268 base_pmc_msr = MSR_IA32_PERFCTR0; 269 270 gp_event = intel_event_to_feature(idx).gp_event; 271 GUEST_ASSERT_EQ(idx, gp_event.f.bit); 272 273 GUEST_ASSERT(nr_gp_counters); 274 275 for (i = 0; i < nr_gp_counters; i++) { 276 uint64_t eventsel = ARCH_PERFMON_EVENTSEL_OS | 277 ARCH_PERFMON_EVENTSEL_ENABLE | 278 intel_pmu_arch_events[idx]; 279 280 wrmsr(MSR_P6_EVNTSEL0 + i, 0); 281 if (guest_has_perf_global_ctrl) 282 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, BIT_ULL(i)); 283 284 __guest_test_arch_event(idx, i, base_pmc_msr + i, 285 MSR_P6_EVNTSEL0 + i, eventsel); 286 } 287 288 if (!guest_has_perf_global_ctrl) 289 return; 290 291 fixed_event = intel_event_to_feature(idx).fixed_event; 292 if (pmu_is_null_feature(fixed_event) || !this_pmu_has(fixed_event)) 293 return; 294 295 i = fixed_event.f.bit; 296 297 wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(i, FIXED_PMC_KERNEL)); 298 299 __guest_test_arch_event(idx, i | INTEL_RDPMC_FIXED, 300 MSR_CORE_PERF_FIXED_CTR0 + i, 301 MSR_CORE_PERF_GLOBAL_CTRL, 302 FIXED_PMC_GLOBAL_CTRL_ENABLE(i)); 303 } 304 305 static void guest_test_arch_events(void) 306 { 307 uint8_t i; 308 309 for (i = 0; i < NR_INTEL_ARCH_EVENTS; i++) 310 guest_test_arch_event(i); 311 312 GUEST_DONE(); 313 } 314 315 static void test_arch_events(uint8_t pmu_version, uint64_t perf_capabilities, 316 uint8_t length, uint8_t unavailable_mask) 317 { 318 struct kvm_vcpu *vcpu; 319 struct kvm_vm *vm; 320 321 /* Testing arch events requires a vPMU (there are no negative tests). */ 322 if (!pmu_version) 323 return; 324 325 vm = pmu_vm_create_with_one_vcpu(&vcpu, guest_test_arch_events, 326 pmu_version, perf_capabilities); 327 328 vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH, 329 length); 330 vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_EVENTS_MASK, 331 unavailable_mask); 332 333 run_vcpu(vcpu); 334 335 kvm_vm_free(vm); 336 } 337 338 /* 339 * Limit testing to MSRs that are actually defined by Intel (in the SDM). MSRs 340 * that aren't defined counter MSRs *probably* don't exist, but there's no 341 * guarantee that currently undefined MSR indices won't be used for something 342 * other than PMCs in the future. 343 */ 344 #define MAX_NR_GP_COUNTERS 8 345 #define MAX_NR_FIXED_COUNTERS 3 346 347 #define GUEST_ASSERT_PMC_MSR_ACCESS(insn, msr, expect_gp, vector) \ 348 __GUEST_ASSERT(expect_gp ? vector == GP_VECTOR : !vector, \ 349 "Expected %s on " #insn "(0x%x), got vector %u", \ 350 expect_gp ? "#GP" : "no fault", msr, vector) \ 351 352 #define GUEST_ASSERT_PMC_VALUE(insn, msr, val, expected) \ 353 __GUEST_ASSERT(val == expected, \ 354 "Expected " #insn "(0x%x) to yield 0x%lx, got 0x%lx", \ 355 msr, expected, val); 356 357 static void guest_test_rdpmc(uint32_t rdpmc_idx, bool expect_success, 358 uint64_t expected_val) 359 { 360 uint8_t vector; 361 uint64_t val; 362 363 vector = rdpmc_safe(rdpmc_idx, &val); 364 GUEST_ASSERT_PMC_MSR_ACCESS(RDPMC, rdpmc_idx, !expect_success, vector); 365 if (expect_success) 366 GUEST_ASSERT_PMC_VALUE(RDPMC, rdpmc_idx, val, expected_val); 367 368 if (!is_forced_emulation_enabled) 369 return; 370 371 vector = rdpmc_safe_fep(rdpmc_idx, &val); 372 GUEST_ASSERT_PMC_MSR_ACCESS(RDPMC, rdpmc_idx, !expect_success, vector); 373 if (expect_success) 374 GUEST_ASSERT_PMC_VALUE(RDPMC, rdpmc_idx, val, expected_val); 375 } 376 377 static void guest_rd_wr_counters(uint32_t base_msr, uint8_t nr_possible_counters, 378 uint8_t nr_counters, uint32_t or_mask) 379 { 380 const bool pmu_has_fast_mode = !guest_get_pmu_version(); 381 uint8_t i; 382 383 for (i = 0; i < nr_possible_counters; i++) { 384 /* 385 * TODO: Test a value that validates full-width writes and the 386 * width of the counters. 387 */ 388 const uint64_t test_val = 0xffff; 389 const uint32_t msr = base_msr + i; 390 391 /* 392 * Fixed counters are supported if the counter is less than the 393 * number of enumerated contiguous counters *or* the counter is 394 * explicitly enumerated in the supported counters mask. 395 */ 396 const bool expect_success = i < nr_counters || (or_mask & BIT(i)); 397 398 /* 399 * KVM drops writes to MSR_P6_PERFCTR[0|1] if the counters are 400 * unsupported, i.e. doesn't #GP and reads back '0'. 401 */ 402 const uint64_t expected_val = expect_success ? test_val : 0; 403 const bool expect_gp = !expect_success && msr != MSR_P6_PERFCTR0 && 404 msr != MSR_P6_PERFCTR1; 405 uint32_t rdpmc_idx; 406 uint8_t vector; 407 uint64_t val; 408 409 vector = wrmsr_safe(msr, test_val); 410 GUEST_ASSERT_PMC_MSR_ACCESS(WRMSR, msr, expect_gp, vector); 411 412 vector = rdmsr_safe(msr, &val); 413 GUEST_ASSERT_PMC_MSR_ACCESS(RDMSR, msr, expect_gp, vector); 414 415 /* On #GP, the result of RDMSR is undefined. */ 416 if (!expect_gp) 417 GUEST_ASSERT_PMC_VALUE(RDMSR, msr, val, expected_val); 418 419 /* 420 * Redo the read tests with RDPMC, which has different indexing 421 * semantics and additional capabilities. 422 */ 423 rdpmc_idx = i; 424 if (base_msr == MSR_CORE_PERF_FIXED_CTR0) 425 rdpmc_idx |= INTEL_RDPMC_FIXED; 426 427 guest_test_rdpmc(rdpmc_idx, expect_success, expected_val); 428 429 /* 430 * KVM doesn't support non-architectural PMUs, i.e. it should 431 * impossible to have fast mode RDPMC. Verify that attempting 432 * to use fast RDPMC always #GPs. 433 */ 434 GUEST_ASSERT(!expect_success || !pmu_has_fast_mode); 435 rdpmc_idx |= INTEL_RDPMC_FAST; 436 guest_test_rdpmc(rdpmc_idx, false, -1ull); 437 438 vector = wrmsr_safe(msr, 0); 439 GUEST_ASSERT_PMC_MSR_ACCESS(WRMSR, msr, expect_gp, vector); 440 } 441 } 442 443 static void guest_test_gp_counters(void) 444 { 445 uint8_t pmu_version = guest_get_pmu_version(); 446 uint8_t nr_gp_counters = 0; 447 uint32_t base_msr; 448 449 if (pmu_version) 450 nr_gp_counters = this_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS); 451 452 /* 453 * For v2+ PMUs, PERF_GLOBAL_CTRL's architectural post-RESET value is 454 * "Sets bits n-1:0 and clears the upper bits", where 'n' is the number 455 * of GP counters. If there are no GP counters, require KVM to leave 456 * PERF_GLOBAL_CTRL '0'. This edge case isn't covered by the SDM, but 457 * follow the spirit of the architecture and only globally enable GP 458 * counters, of which there are none. 459 */ 460 if (pmu_version > 1) { 461 uint64_t global_ctrl = rdmsr(MSR_CORE_PERF_GLOBAL_CTRL); 462 463 if (nr_gp_counters) 464 GUEST_ASSERT_EQ(global_ctrl, GENMASK_ULL(nr_gp_counters - 1, 0)); 465 else 466 GUEST_ASSERT_EQ(global_ctrl, 0); 467 } 468 469 if (this_cpu_has(X86_FEATURE_PDCM) && 470 rdmsr(MSR_IA32_PERF_CAPABILITIES) & PMU_CAP_FW_WRITES) 471 base_msr = MSR_IA32_PMC0; 472 else 473 base_msr = MSR_IA32_PERFCTR0; 474 475 guest_rd_wr_counters(base_msr, MAX_NR_GP_COUNTERS, nr_gp_counters, 0); 476 GUEST_DONE(); 477 } 478 479 static void test_gp_counters(uint8_t pmu_version, uint64_t perf_capabilities, 480 uint8_t nr_gp_counters) 481 { 482 struct kvm_vcpu *vcpu; 483 struct kvm_vm *vm; 484 485 vm = pmu_vm_create_with_one_vcpu(&vcpu, guest_test_gp_counters, 486 pmu_version, perf_capabilities); 487 488 vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_NR_GP_COUNTERS, 489 nr_gp_counters); 490 491 run_vcpu(vcpu); 492 493 kvm_vm_free(vm); 494 } 495 496 static void guest_test_fixed_counters(void) 497 { 498 uint64_t supported_bitmask = 0; 499 uint8_t nr_fixed_counters = 0; 500 uint8_t i; 501 502 /* Fixed counters require Architectural vPMU Version 2+. */ 503 if (guest_get_pmu_version() >= 2) 504 nr_fixed_counters = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); 505 506 /* 507 * The supported bitmask for fixed counters was introduced in PMU 508 * version 5. 509 */ 510 if (guest_get_pmu_version() >= 5) 511 supported_bitmask = this_cpu_property(X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK); 512 513 guest_rd_wr_counters(MSR_CORE_PERF_FIXED_CTR0, MAX_NR_FIXED_COUNTERS, 514 nr_fixed_counters, supported_bitmask); 515 516 for (i = 0; i < MAX_NR_FIXED_COUNTERS; i++) { 517 uint8_t vector; 518 uint64_t val; 519 520 if (i >= nr_fixed_counters && !(supported_bitmask & BIT_ULL(i))) { 521 vector = wrmsr_safe(MSR_CORE_PERF_FIXED_CTR_CTRL, 522 FIXED_PMC_CTRL(i, FIXED_PMC_KERNEL)); 523 __GUEST_ASSERT(vector == GP_VECTOR, 524 "Expected #GP for counter %u in FIXED_CTR_CTRL", i); 525 526 vector = wrmsr_safe(MSR_CORE_PERF_GLOBAL_CTRL, 527 FIXED_PMC_GLOBAL_CTRL_ENABLE(i)); 528 __GUEST_ASSERT(vector == GP_VECTOR, 529 "Expected #GP for counter %u in PERF_GLOBAL_CTRL", i); 530 continue; 531 } 532 533 wrmsr(MSR_CORE_PERF_FIXED_CTR0 + i, 0); 534 wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(i, FIXED_PMC_KERNEL)); 535 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, FIXED_PMC_GLOBAL_CTRL_ENABLE(i)); 536 __asm__ __volatile__("loop ." : "+c"((int){NUM_LOOPS})); 537 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0); 538 val = rdmsr(MSR_CORE_PERF_FIXED_CTR0 + i); 539 540 GUEST_ASSERT_NE(val, 0); 541 } 542 GUEST_DONE(); 543 } 544 545 static void test_fixed_counters(uint8_t pmu_version, uint64_t perf_capabilities, 546 uint8_t nr_fixed_counters, 547 uint32_t supported_bitmask) 548 { 549 struct kvm_vcpu *vcpu; 550 struct kvm_vm *vm; 551 552 vm = pmu_vm_create_with_one_vcpu(&vcpu, guest_test_fixed_counters, 553 pmu_version, perf_capabilities); 554 555 vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK, 556 supported_bitmask); 557 vcpu_set_cpuid_property(vcpu, X86_PROPERTY_PMU_NR_FIXED_COUNTERS, 558 nr_fixed_counters); 559 560 run_vcpu(vcpu); 561 562 kvm_vm_free(vm); 563 } 564 565 static void test_intel_counters(void) 566 { 567 uint8_t nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); 568 uint8_t nr_gp_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS); 569 uint8_t pmu_version = kvm_cpu_property(X86_PROPERTY_PMU_VERSION); 570 unsigned int i; 571 uint8_t v, j; 572 uint32_t k; 573 574 const uint64_t perf_caps[] = { 575 0, 576 PMU_CAP_FW_WRITES, 577 }; 578 579 /* 580 * Test up to PMU v5, which is the current maximum version defined by 581 * Intel, i.e. is the last version that is guaranteed to be backwards 582 * compatible with KVM's existing behavior. 583 */ 584 uint8_t max_pmu_version = max_t(typeof(pmu_version), pmu_version, 5); 585 586 /* 587 * Detect the existence of events that aren't supported by selftests. 588 * This will (obviously) fail any time hardware adds support for a new 589 * event, but it's worth paying that price to keep the test fresh. 590 */ 591 TEST_ASSERT(this_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH) <= NR_INTEL_ARCH_EVENTS, 592 "New architectural event(s) detected; please update this test (length = %u, mask = %x)", 593 this_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH), 594 this_cpu_property(X86_PROPERTY_PMU_EVENTS_MASK)); 595 596 /* 597 * Iterate over known arch events irrespective of KVM/hardware support 598 * to verify that KVM doesn't reject programming of events just because 599 * the *architectural* encoding is unsupported. Track which events are 600 * supported in hardware; the guest side will validate supported events 601 * count correctly, even if *enumeration* of the event is unsupported 602 * by KVM and/or isn't exposed to the guest. 603 */ 604 for (i = 0; i < NR_INTEL_ARCH_EVENTS; i++) { 605 if (this_pmu_has(intel_event_to_feature(i).gp_event)) 606 hardware_pmu_arch_events |= BIT(i); 607 } 608 609 for (v = 0; v <= max_pmu_version; v++) { 610 for (i = 0; i < ARRAY_SIZE(perf_caps); i++) { 611 if (!kvm_has_perf_caps && perf_caps[i]) 612 continue; 613 614 pr_info("Testing arch events, PMU version %u, perf_caps = %lx\n", 615 v, perf_caps[i]); 616 /* 617 * To keep the total runtime reasonable, test every 618 * possible non-zero, non-reserved bitmap combination 619 * only with the native PMU version and the full bit 620 * vector length. 621 */ 622 if (v == pmu_version) { 623 for (k = 1; k < (BIT(NR_INTEL_ARCH_EVENTS) - 1); k++) 624 test_arch_events(v, perf_caps[i], NR_INTEL_ARCH_EVENTS, k); 625 } 626 /* 627 * Test single bits for all PMU version and lengths up 628 * the number of events +1 (to verify KVM doesn't do 629 * weird things if the guest length is greater than the 630 * host length). Explicitly test a mask of '0' and all 631 * ones i.e. all events being available and unavailable. 632 */ 633 for (j = 0; j <= NR_INTEL_ARCH_EVENTS + 1; j++) { 634 test_arch_events(v, perf_caps[i], j, 0); 635 test_arch_events(v, perf_caps[i], j, 0xff); 636 637 for (k = 0; k < NR_INTEL_ARCH_EVENTS; k++) 638 test_arch_events(v, perf_caps[i], j, BIT(k)); 639 } 640 641 pr_info("Testing GP counters, PMU version %u, perf_caps = %lx\n", 642 v, perf_caps[i]); 643 for (j = 0; j <= nr_gp_counters; j++) 644 test_gp_counters(v, perf_caps[i], j); 645 646 pr_info("Testing fixed counters, PMU version %u, perf_caps = %lx\n", 647 v, perf_caps[i]); 648 for (j = 0; j <= nr_fixed_counters; j++) { 649 for (k = 0; k <= (BIT(nr_fixed_counters) - 1); k++) 650 test_fixed_counters(v, perf_caps[i], j, k); 651 } 652 } 653 } 654 } 655 656 int main(int argc, char *argv[]) 657 { 658 TEST_REQUIRE(kvm_is_pmu_enabled()); 659 660 TEST_REQUIRE(host_cpu_is_intel); 661 TEST_REQUIRE(kvm_cpu_has_p(X86_PROPERTY_PMU_VERSION)); 662 TEST_REQUIRE(kvm_cpu_property(X86_PROPERTY_PMU_VERSION) > 0); 663 664 kvm_pmu_version = kvm_cpu_property(X86_PROPERTY_PMU_VERSION); 665 kvm_has_perf_caps = kvm_cpu_has(X86_FEATURE_PDCM); 666 667 test_intel_counters(); 668 669 return 0; 670 } 671