1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Test for x86 KVM_SET_PMU_EVENT_FILTER. 4 * 5 * Copyright (C) 2022, Google LLC. 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2. 8 * 9 * Verifies the expected behavior of allow lists and deny lists for 10 * virtual PMU events. 11 */ 12 #include "kvm_util.h" 13 #include "pmu.h" 14 #include "processor.h" 15 #include "test_util.h" 16 17 #define NUM_BRANCHES 42 18 #define MAX_TEST_EVENTS 10 19 20 #define PMU_EVENT_FILTER_INVALID_ACTION (KVM_PMU_EVENT_DENY + 1) 21 #define PMU_EVENT_FILTER_INVALID_FLAGS (KVM_PMU_EVENT_FLAGS_VALID_MASK << 1) 22 #define PMU_EVENT_FILTER_INVALID_NEVENTS (KVM_PMU_EVENT_FILTER_MAX_EVENTS + 1) 23 24 struct __kvm_pmu_event_filter { 25 __u32 action; 26 __u32 nevents; 27 __u32 fixed_counter_bitmap; 28 __u32 flags; 29 __u32 pad[4]; 30 __u64 events[KVM_PMU_EVENT_FILTER_MAX_EVENTS]; 31 }; 32 33 /* 34 * This event list comprises Intel's known architectural events, plus AMD's 35 * Branch Instructions Retired for Zen CPUs. Note, AMD and Intel use the 36 * same encoding for Instructions Retired. 37 */ 38 kvm_static_assert(INTEL_ARCH_INSTRUCTIONS_RETIRED == AMD_ZEN_INSTRUCTIONS_RETIRED); 39 40 static const struct __kvm_pmu_event_filter base_event_filter = { 41 .nevents = ARRAY_SIZE(base_event_filter.events), 42 .events = { 43 INTEL_ARCH_CPU_CYCLES, 44 INTEL_ARCH_INSTRUCTIONS_RETIRED, 45 INTEL_ARCH_REFERENCE_CYCLES, 46 INTEL_ARCH_LLC_REFERENCES, 47 INTEL_ARCH_LLC_MISSES, 48 INTEL_ARCH_BRANCHES_RETIRED, 49 INTEL_ARCH_BRANCHES_MISPREDICTED, 50 INTEL_ARCH_TOPDOWN_SLOTS, 51 AMD_ZEN_BRANCHES_RETIRED, 52 }, 53 }; 54 55 struct { 56 u64 loads; 57 u64 stores; 58 u64 loads_stores; 59 u64 branches_retired; 60 u64 instructions_retired; 61 } pmc_results; 62 63 /* 64 * If we encounter a #GP during the guest PMU sanity check, then the guest 65 * PMU is not functional. Inform the hypervisor via GUEST_SYNC(0). 66 */ 67 static void guest_gp_handler(struct ex_regs *regs) 68 { 69 GUEST_SYNC(-EFAULT); 70 } 71 72 /* 73 * Check that we can write a new value to the given MSR and read it back. 74 * The caller should provide a non-empty set of bits that are safe to flip. 75 * 76 * Return on success. GUEST_SYNC(0) on error. 77 */ 78 static void check_msr(u32 msr, u64 bits_to_flip) 79 { 80 u64 v = rdmsr(msr) ^ bits_to_flip; 81 82 wrmsr(msr, v); 83 if (rdmsr(msr) != v) 84 GUEST_SYNC(-EIO); 85 86 v ^= bits_to_flip; 87 wrmsr(msr, v); 88 if (rdmsr(msr) != v) 89 GUEST_SYNC(-EIO); 90 } 91 92 static void run_and_measure_loop(u32 msr_base) 93 { 94 const u64 branches_retired = rdmsr(msr_base + 0); 95 const u64 insn_retired = rdmsr(msr_base + 1); 96 97 __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES})); 98 99 pmc_results.branches_retired = rdmsr(msr_base + 0) - branches_retired; 100 pmc_results.instructions_retired = rdmsr(msr_base + 1) - insn_retired; 101 } 102 103 static void intel_guest_code(void) 104 { 105 check_msr(MSR_CORE_PERF_GLOBAL_CTRL, 1); 106 check_msr(MSR_P6_EVNTSEL0, 0xffff); 107 check_msr(MSR_IA32_PMC0, 0xffff); 108 GUEST_SYNC(0); 109 110 for (;;) { 111 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0); 112 wrmsr(MSR_P6_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE | 113 ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_BRANCHES_RETIRED); 114 wrmsr(MSR_P6_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE | 115 ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_INSTRUCTIONS_RETIRED); 116 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x3); 117 118 run_and_measure_loop(MSR_IA32_PMC0); 119 GUEST_SYNC(0); 120 } 121 } 122 123 /* 124 * To avoid needing a check for CPUID.80000001:ECX.PerfCtrExtCore[bit 23], 125 * this code uses the always-available, legacy K7 PMU MSRs, which alias to 126 * the first four of the six extended core PMU MSRs. 127 */ 128 static void amd_guest_code(void) 129 { 130 check_msr(MSR_K7_EVNTSEL0, 0xffff); 131 check_msr(MSR_K7_PERFCTR0, 0xffff); 132 GUEST_SYNC(0); 133 134 for (;;) { 135 wrmsr(MSR_K7_EVNTSEL0, 0); 136 wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE | 137 ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_BRANCHES_RETIRED); 138 wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE | 139 ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_INSTRUCTIONS_RETIRED); 140 141 run_and_measure_loop(MSR_K7_PERFCTR0); 142 GUEST_SYNC(0); 143 } 144 } 145 146 /* 147 * Run the VM to the next GUEST_SYNC(value), and return the value passed 148 * to the sync. Any other exit from the guest is fatal. 149 */ 150 static u64 run_vcpu_to_sync(struct kvm_vcpu *vcpu) 151 { 152 struct ucall uc; 153 154 vcpu_run(vcpu); 155 TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); 156 get_ucall(vcpu, &uc); 157 TEST_ASSERT(uc.cmd == UCALL_SYNC, 158 "Received ucall other than UCALL_SYNC: %lu", uc.cmd); 159 return uc.args[1]; 160 } 161 162 static void run_vcpu_and_sync_pmc_results(struct kvm_vcpu *vcpu) 163 { 164 u64 r; 165 166 memset(&pmc_results, 0, sizeof(pmc_results)); 167 sync_global_to_guest(vcpu->vm, pmc_results); 168 169 r = run_vcpu_to_sync(vcpu); 170 TEST_ASSERT(!r, "Unexpected sync value: 0x%lx", r); 171 172 sync_global_from_guest(vcpu->vm, pmc_results); 173 } 174 175 /* 176 * In a nested environment or if the vPMU is disabled, the guest PMU 177 * might not work as architected (accessing the PMU MSRs may raise 178 * #GP, or writes could simply be discarded). In those situations, 179 * there is no point in running these tests. The guest code will perform 180 * a sanity check and then GUEST_SYNC(success). In the case of failure, 181 * the behavior of the guest on resumption is undefined. 182 */ 183 static bool sanity_check_pmu(struct kvm_vcpu *vcpu) 184 { 185 u64 r; 186 187 vm_install_exception_handler(vcpu->vm, GP_VECTOR, guest_gp_handler); 188 r = run_vcpu_to_sync(vcpu); 189 vm_install_exception_handler(vcpu->vm, GP_VECTOR, NULL); 190 191 return !r; 192 } 193 194 /* 195 * Remove the first occurrence of 'event' (if any) from the filter's 196 * event list. 197 */ 198 static void remove_event(struct __kvm_pmu_event_filter *f, u64 event) 199 { 200 bool found = false; 201 int i; 202 203 for (i = 0; i < f->nevents; i++) { 204 if (found) 205 f->events[i - 1] = f->events[i]; 206 else 207 found = f->events[i] == event; 208 } 209 if (found) 210 f->nevents--; 211 } 212 213 #define ASSERT_PMC_COUNTING_INSTRUCTIONS() \ 214 do { \ 215 u64 br = pmc_results.branches_retired; \ 216 u64 ir = pmc_results.instructions_retired; \ 217 bool br_matched = this_pmu_has_errata(BRANCHES_RETIRED_OVERCOUNT) ? \ 218 br >= NUM_BRANCHES : br == NUM_BRANCHES; \ 219 \ 220 if (br && !br_matched) \ 221 pr_info("%s: Branch instructions retired = %lu (expected %u)\n", \ 222 __func__, br, NUM_BRANCHES); \ 223 TEST_ASSERT(br, "%s: Branch instructions retired = %lu (expected > 0)", \ 224 __func__, br); \ 225 TEST_ASSERT(ir, "%s: Instructions retired = %lu (expected > 0)", \ 226 __func__, ir); \ 227 } while (0) 228 229 #define ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS() \ 230 do { \ 231 u64 br = pmc_results.branches_retired; \ 232 u64 ir = pmc_results.instructions_retired; \ 233 \ 234 TEST_ASSERT(!br, "%s: Branch instructions retired = %lu (expected 0)", \ 235 __func__, br); \ 236 TEST_ASSERT(!ir, "%s: Instructions retired = %lu (expected 0)", \ 237 __func__, ir); \ 238 } while (0) 239 240 static void test_without_filter(struct kvm_vcpu *vcpu) 241 { 242 run_vcpu_and_sync_pmc_results(vcpu); 243 244 ASSERT_PMC_COUNTING_INSTRUCTIONS(); 245 } 246 247 static void test_with_filter(struct kvm_vcpu *vcpu, 248 struct __kvm_pmu_event_filter *__f) 249 { 250 struct kvm_pmu_event_filter *f = (void *)__f; 251 252 vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f); 253 run_vcpu_and_sync_pmc_results(vcpu); 254 } 255 256 static void test_amd_deny_list(struct kvm_vcpu *vcpu) 257 { 258 struct __kvm_pmu_event_filter f = { 259 .action = KVM_PMU_EVENT_DENY, 260 .nevents = 1, 261 .events = { 262 RAW_EVENT(0x1C2, 0), 263 }, 264 }; 265 266 test_with_filter(vcpu, &f); 267 268 ASSERT_PMC_COUNTING_INSTRUCTIONS(); 269 } 270 271 static void test_member_deny_list(struct kvm_vcpu *vcpu) 272 { 273 struct __kvm_pmu_event_filter f = base_event_filter; 274 275 f.action = KVM_PMU_EVENT_DENY; 276 test_with_filter(vcpu, &f); 277 278 ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS(); 279 } 280 281 static void test_member_allow_list(struct kvm_vcpu *vcpu) 282 { 283 struct __kvm_pmu_event_filter f = base_event_filter; 284 285 f.action = KVM_PMU_EVENT_ALLOW; 286 test_with_filter(vcpu, &f); 287 288 ASSERT_PMC_COUNTING_INSTRUCTIONS(); 289 } 290 291 static void test_not_member_deny_list(struct kvm_vcpu *vcpu) 292 { 293 struct __kvm_pmu_event_filter f = base_event_filter; 294 295 f.action = KVM_PMU_EVENT_DENY; 296 297 remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED); 298 remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED); 299 remove_event(&f, AMD_ZEN_BRANCHES_RETIRED); 300 test_with_filter(vcpu, &f); 301 302 ASSERT_PMC_COUNTING_INSTRUCTIONS(); 303 } 304 305 static void test_not_member_allow_list(struct kvm_vcpu *vcpu) 306 { 307 struct __kvm_pmu_event_filter f = base_event_filter; 308 309 f.action = KVM_PMU_EVENT_ALLOW; 310 311 remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED); 312 remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED); 313 remove_event(&f, AMD_ZEN_BRANCHES_RETIRED); 314 test_with_filter(vcpu, &f); 315 316 ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS(); 317 } 318 319 /* 320 * Verify that setting KVM_PMU_CAP_DISABLE prevents the use of the PMU. 321 * 322 * Note that KVM_CAP_PMU_CAPABILITY must be invoked prior to creating VCPUs. 323 */ 324 static void test_pmu_config_disable(void (*guest_code)(void)) 325 { 326 struct kvm_vcpu *vcpu; 327 int r; 328 struct kvm_vm *vm; 329 330 r = kvm_check_cap(KVM_CAP_PMU_CAPABILITY); 331 if (!(r & KVM_PMU_CAP_DISABLE)) 332 return; 333 334 vm = vm_create(1); 335 336 vm_enable_cap(vm, KVM_CAP_PMU_CAPABILITY, KVM_PMU_CAP_DISABLE); 337 338 vcpu = vm_vcpu_add(vm, 0, guest_code); 339 TEST_ASSERT(!sanity_check_pmu(vcpu), 340 "Guest should not be able to use disabled PMU."); 341 342 kvm_vm_free(vm); 343 } 344 345 /* 346 * On Intel, check for a non-zero PMU version, at least one general-purpose 347 * counter per logical processor, and support for counting the number of branch 348 * instructions retired. 349 */ 350 static bool use_intel_pmu(void) 351 { 352 return host_cpu_is_intel && 353 kvm_cpu_property(X86_PROPERTY_PMU_VERSION) && 354 kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) && 355 kvm_pmu_has(X86_PMU_FEATURE_BRANCH_INSNS_RETIRED); 356 } 357 358 /* 359 * On AMD, all Family 17h+ CPUs (Zen and its successors) use event encoding 360 * 0xc2,0 for Branch Instructions Retired. 361 */ 362 static bool use_amd_pmu(void) 363 { 364 return (host_cpu_is_amd && kvm_cpu_family() >= 0x17) || 365 host_cpu_is_hygon; 366 } 367 368 /* 369 * "MEM_INST_RETIRED.ALL_LOADS", "MEM_INST_RETIRED.ALL_STORES", and 370 * "MEM_INST_RETIRED.ANY" from https://perfmon-events.intel.com/ 371 * supported on Intel Xeon processors: 372 * - Sapphire Rapids, Ice Lake, Cascade Lake, Skylake. 373 */ 374 #define MEM_INST_RETIRED 0xD0 375 #define MEM_INST_RETIRED_LOAD RAW_EVENT(MEM_INST_RETIRED, 0x81) 376 #define MEM_INST_RETIRED_STORE RAW_EVENT(MEM_INST_RETIRED, 0x82) 377 #define MEM_INST_RETIRED_LOAD_STORE RAW_EVENT(MEM_INST_RETIRED, 0x83) 378 379 static bool supports_event_mem_inst_retired(void) 380 { 381 u32 eax, ebx, ecx, edx; 382 383 cpuid(1, &eax, &ebx, &ecx, &edx); 384 if (x86_family(eax) == 0x6) { 385 switch (x86_model(eax)) { 386 /* Sapphire Rapids */ 387 case 0x8F: 388 /* Ice Lake */ 389 case 0x6A: 390 /* Skylake */ 391 /* Cascade Lake */ 392 case 0x55: 393 return true; 394 } 395 } 396 397 return false; 398 } 399 400 /* 401 * "LS Dispatch", from Processor Programming Reference 402 * (PPR) for AMD Family 17h Model 01h, Revision B1 Processors, 403 * Preliminary Processor Programming Reference (PPR) for AMD Family 404 * 17h Model 31h, Revision B0 Processors, and Preliminary Processor 405 * Programming Reference (PPR) for AMD Family 19h Model 01h, Revision 406 * B1 Processors Volume 1 of 2. 407 */ 408 #define LS_DISPATCH 0x29 409 #define LS_DISPATCH_LOAD RAW_EVENT(LS_DISPATCH, BIT(0)) 410 #define LS_DISPATCH_STORE RAW_EVENT(LS_DISPATCH, BIT(1)) 411 #define LS_DISPATCH_LOAD_STORE RAW_EVENT(LS_DISPATCH, BIT(2)) 412 413 #define INCLUDE_MASKED_ENTRY(event_select, mask, match) \ 414 KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, false) 415 #define EXCLUDE_MASKED_ENTRY(event_select, mask, match) \ 416 KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, true) 417 418 static void masked_events_guest_test(u32 msr_base) 419 { 420 /* 421 * The actual value of the counters don't determine the outcome of 422 * the test. Only that they are zero or non-zero. 423 */ 424 const u64 loads = rdmsr(msr_base + 0); 425 const u64 stores = rdmsr(msr_base + 1); 426 const u64 loads_stores = rdmsr(msr_base + 2); 427 int val; 428 429 430 __asm__ __volatile__("movl $0, %[v];" 431 "movl %[v], %%eax;" 432 "incl %[v];" 433 : [v]"+m"(val) :: "eax"); 434 435 pmc_results.loads = rdmsr(msr_base + 0) - loads; 436 pmc_results.stores = rdmsr(msr_base + 1) - stores; 437 pmc_results.loads_stores = rdmsr(msr_base + 2) - loads_stores; 438 } 439 440 static void intel_masked_events_guest_code(void) 441 { 442 for (;;) { 443 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0); 444 445 wrmsr(MSR_P6_EVNTSEL0 + 0, ARCH_PERFMON_EVENTSEL_ENABLE | 446 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD); 447 wrmsr(MSR_P6_EVNTSEL0 + 1, ARCH_PERFMON_EVENTSEL_ENABLE | 448 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_STORE); 449 wrmsr(MSR_P6_EVNTSEL0 + 2, ARCH_PERFMON_EVENTSEL_ENABLE | 450 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD_STORE); 451 452 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x7); 453 454 masked_events_guest_test(MSR_IA32_PMC0); 455 GUEST_SYNC(0); 456 } 457 } 458 459 static void amd_masked_events_guest_code(void) 460 { 461 for (;;) { 462 wrmsr(MSR_K7_EVNTSEL0, 0); 463 wrmsr(MSR_K7_EVNTSEL1, 0); 464 wrmsr(MSR_K7_EVNTSEL2, 0); 465 466 wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE | 467 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD); 468 wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE | 469 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_STORE); 470 wrmsr(MSR_K7_EVNTSEL2, ARCH_PERFMON_EVENTSEL_ENABLE | 471 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD_STORE); 472 473 masked_events_guest_test(MSR_K7_PERFCTR0); 474 GUEST_SYNC(0); 475 } 476 } 477 478 static void run_masked_events_test(struct kvm_vcpu *vcpu, 479 const u64 masked_events[], 480 const int nmasked_events) 481 { 482 struct __kvm_pmu_event_filter f = { 483 .nevents = nmasked_events, 484 .action = KVM_PMU_EVENT_ALLOW, 485 .flags = KVM_PMU_EVENT_FLAG_MASKED_EVENTS, 486 }; 487 488 memcpy(f.events, masked_events, sizeof(u64) * nmasked_events); 489 test_with_filter(vcpu, &f); 490 } 491 492 #define ALLOW_LOADS BIT(0) 493 #define ALLOW_STORES BIT(1) 494 #define ALLOW_LOADS_STORES BIT(2) 495 496 struct masked_events_test { 497 u64 intel_events[MAX_TEST_EVENTS]; 498 u64 intel_event_end; 499 u64 amd_events[MAX_TEST_EVENTS]; 500 u64 amd_event_end; 501 const char *msg; 502 u32 flags; 503 }; 504 505 /* 506 * These are the test cases for the masked events tests. 507 * 508 * For each test, the guest enables 3 PMU counters (loads, stores, 509 * loads + stores). The filter is then set in KVM with the masked events 510 * provided. The test then verifies that the counters agree with which 511 * ones should be counting and which ones should be filtered. 512 */ 513 const struct masked_events_test test_cases[] = { 514 { 515 .intel_events = { 516 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x81), 517 }, 518 .amd_events = { 519 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)), 520 }, 521 .msg = "Only allow loads.", 522 .flags = ALLOW_LOADS, 523 }, { 524 .intel_events = { 525 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82), 526 }, 527 .amd_events = { 528 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)), 529 }, 530 .msg = "Only allow stores.", 531 .flags = ALLOW_STORES, 532 }, { 533 .intel_events = { 534 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83), 535 }, 536 .amd_events = { 537 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(2)), 538 }, 539 .msg = "Only allow loads + stores.", 540 .flags = ALLOW_LOADS_STORES, 541 }, { 542 .intel_events = { 543 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0), 544 EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83), 545 }, 546 .amd_events = { 547 INCLUDE_MASKED_ENTRY(LS_DISPATCH, ~(BIT(0) | BIT(1)), 0), 548 }, 549 .msg = "Only allow loads and stores.", 550 .flags = ALLOW_LOADS | ALLOW_STORES, 551 }, { 552 .intel_events = { 553 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0), 554 EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82), 555 }, 556 .amd_events = { 557 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0), 558 EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)), 559 }, 560 .msg = "Only allow loads and loads + stores.", 561 .flags = ALLOW_LOADS | ALLOW_LOADS_STORES 562 }, { 563 .intel_events = { 564 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFE, 0x82), 565 }, 566 .amd_events = { 567 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0), 568 EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)), 569 }, 570 .msg = "Only allow stores and loads + stores.", 571 .flags = ALLOW_STORES | ALLOW_LOADS_STORES 572 }, { 573 .intel_events = { 574 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0), 575 }, 576 .amd_events = { 577 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0), 578 }, 579 .msg = "Only allow loads, stores, and loads + stores.", 580 .flags = ALLOW_LOADS | ALLOW_STORES | ALLOW_LOADS_STORES 581 }, 582 }; 583 584 static int append_test_events(const struct masked_events_test *test, 585 u64 *events, int nevents) 586 { 587 const u64 *evts; 588 int i; 589 590 evts = use_intel_pmu() ? test->intel_events : test->amd_events; 591 for (i = 0; i < MAX_TEST_EVENTS; i++) { 592 if (evts[i] == 0) 593 break; 594 595 events[nevents + i] = evts[i]; 596 } 597 598 return nevents + i; 599 } 600 601 static bool bool_eq(bool a, bool b) 602 { 603 return a == b; 604 } 605 606 static void run_masked_events_tests(struct kvm_vcpu *vcpu, u64 *events, 607 int nevents) 608 { 609 int ntests = ARRAY_SIZE(test_cases); 610 int i, n; 611 612 for (i = 0; i < ntests; i++) { 613 const struct masked_events_test *test = &test_cases[i]; 614 615 /* Do any test case events overflow MAX_TEST_EVENTS? */ 616 assert(test->intel_event_end == 0); 617 assert(test->amd_event_end == 0); 618 619 n = append_test_events(test, events, nevents); 620 621 run_masked_events_test(vcpu, events, n); 622 623 TEST_ASSERT(bool_eq(pmc_results.loads, test->flags & ALLOW_LOADS) && 624 bool_eq(pmc_results.stores, test->flags & ALLOW_STORES) && 625 bool_eq(pmc_results.loads_stores, 626 test->flags & ALLOW_LOADS_STORES), 627 "%s loads: %lu, stores: %lu, loads + stores: %lu", 628 test->msg, pmc_results.loads, pmc_results.stores, 629 pmc_results.loads_stores); 630 } 631 } 632 633 static void add_dummy_events(u64 *events, int nevents) 634 { 635 int i; 636 637 for (i = 0; i < nevents; i++) { 638 int event_select = i % 0xFF; 639 bool exclude = ((i % 4) == 0); 640 641 if (event_select == MEM_INST_RETIRED || 642 event_select == LS_DISPATCH) 643 event_select++; 644 645 events[i] = KVM_PMU_ENCODE_MASKED_ENTRY(event_select, 0, 646 0, exclude); 647 } 648 } 649 650 static void test_masked_events(struct kvm_vcpu *vcpu) 651 { 652 int nevents = KVM_PMU_EVENT_FILTER_MAX_EVENTS - MAX_TEST_EVENTS; 653 u64 events[KVM_PMU_EVENT_FILTER_MAX_EVENTS]; 654 655 /* Run the test cases against a sparse PMU event filter. */ 656 run_masked_events_tests(vcpu, events, 0); 657 658 /* Run the test cases against a dense PMU event filter. */ 659 add_dummy_events(events, KVM_PMU_EVENT_FILTER_MAX_EVENTS); 660 run_masked_events_tests(vcpu, events, nevents); 661 } 662 663 static int set_pmu_event_filter(struct kvm_vcpu *vcpu, 664 struct __kvm_pmu_event_filter *__f) 665 { 666 struct kvm_pmu_event_filter *f = (void *)__f; 667 668 return __vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f); 669 } 670 671 static int set_pmu_single_event_filter(struct kvm_vcpu *vcpu, u64 event, 672 u32 flags, u32 action) 673 { 674 struct __kvm_pmu_event_filter f = { 675 .nevents = 1, 676 .flags = flags, 677 .action = action, 678 .events = { 679 event, 680 }, 681 }; 682 683 return set_pmu_event_filter(vcpu, &f); 684 } 685 686 static void test_filter_ioctl(struct kvm_vcpu *vcpu) 687 { 688 u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); 689 struct __kvm_pmu_event_filter f; 690 u64 e = ~0ul; 691 int r; 692 693 /* 694 * Unfortunately having invalid bits set in event data is expected to 695 * pass when flags == 0 (bits other than eventsel+umask). 696 */ 697 r = set_pmu_single_event_filter(vcpu, e, 0, KVM_PMU_EVENT_ALLOW); 698 TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing"); 699 700 r = set_pmu_single_event_filter(vcpu, e, 701 KVM_PMU_EVENT_FLAG_MASKED_EVENTS, 702 KVM_PMU_EVENT_ALLOW); 703 TEST_ASSERT(r != 0, "Invalid PMU Event Filter is expected to fail"); 704 705 e = KVM_PMU_ENCODE_MASKED_ENTRY(0xff, 0xff, 0xff, 0xf); 706 r = set_pmu_single_event_filter(vcpu, e, 707 KVM_PMU_EVENT_FLAG_MASKED_EVENTS, 708 KVM_PMU_EVENT_ALLOW); 709 TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing"); 710 711 f = base_event_filter; 712 f.action = PMU_EVENT_FILTER_INVALID_ACTION; 713 r = set_pmu_event_filter(vcpu, &f); 714 TEST_ASSERT(r, "Set invalid action is expected to fail"); 715 716 f = base_event_filter; 717 f.flags = PMU_EVENT_FILTER_INVALID_FLAGS; 718 r = set_pmu_event_filter(vcpu, &f); 719 TEST_ASSERT(r, "Set invalid flags is expected to fail"); 720 721 f = base_event_filter; 722 f.nevents = PMU_EVENT_FILTER_INVALID_NEVENTS; 723 r = set_pmu_event_filter(vcpu, &f); 724 TEST_ASSERT(r, "Exceeding the max number of filter events should fail"); 725 726 f = base_event_filter; 727 f.fixed_counter_bitmap = ~GENMASK_ULL(nr_fixed_counters, 0); 728 r = set_pmu_event_filter(vcpu, &f); 729 TEST_ASSERT(!r, "Masking non-existent fixed counters should be allowed"); 730 } 731 732 static void intel_run_fixed_counter_guest_code(u8 idx) 733 { 734 u8 nr_fixed_counters = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); 735 736 for (;;) { 737 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0); 738 wrmsr(MSR_CORE_PERF_FIXED_CTR0 + idx, 0); 739 740 /* Only OS_EN bit is enabled for fixed counter[idx]. */ 741 wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL)); 742 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, FIXED_PMC_GLOBAL_CTRL_ENABLE(idx)); 743 if (nr_fixed_counters > 1) 744 wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, 745 FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL) | 746 FIXED_PMC_CTRL((idx + 1) % nr_fixed_counters, FIXED_PMC_KERNEL)); 747 __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES})); 748 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0); 749 750 GUEST_SYNC(rdmsr(MSR_CORE_PERF_FIXED_CTR0 + idx)); 751 } 752 } 753 754 static u64 test_with_fixed_counter_filter(struct kvm_vcpu *vcpu, 755 u32 action, u32 bitmap) 756 { 757 struct __kvm_pmu_event_filter f = { 758 .action = action, 759 .fixed_counter_bitmap = bitmap, 760 }; 761 set_pmu_event_filter(vcpu, &f); 762 763 return run_vcpu_to_sync(vcpu); 764 } 765 766 static u64 test_set_gp_and_fixed_event_filter(struct kvm_vcpu *vcpu, 767 u32 action, 768 u32 bitmap) 769 { 770 struct __kvm_pmu_event_filter f = base_event_filter; 771 772 f.action = action; 773 f.fixed_counter_bitmap = bitmap; 774 set_pmu_event_filter(vcpu, &f); 775 776 return run_vcpu_to_sync(vcpu); 777 } 778 779 static void __test_fixed_counter_bitmap(struct kvm_vcpu *vcpu, u8 idx, 780 u8 nr_fixed_counters) 781 { 782 unsigned int i; 783 u32 bitmap; 784 u64 count; 785 786 TEST_ASSERT(nr_fixed_counters < sizeof(bitmap) * 8, 787 "Invalid nr_fixed_counters"); 788 789 /* 790 * Check the fixed performance counter can count normally when KVM 791 * userspace doesn't set any pmu filter. 792 */ 793 count = run_vcpu_to_sync(vcpu); 794 TEST_ASSERT(count, "Unexpected count value: %ld", count); 795 796 for (i = 0; i < BIT(nr_fixed_counters); i++) { 797 bitmap = BIT(i); 798 count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_ALLOW, 799 bitmap); 800 TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx))); 801 802 count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_DENY, 803 bitmap); 804 TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx))); 805 806 /* 807 * Check that fixed_counter_bitmap has higher priority than 808 * events[] when both are set. 809 */ 810 count = test_set_gp_and_fixed_event_filter(vcpu, 811 KVM_PMU_EVENT_ALLOW, 812 bitmap); 813 TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx))); 814 815 count = test_set_gp_and_fixed_event_filter(vcpu, 816 KVM_PMU_EVENT_DENY, 817 bitmap); 818 TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx))); 819 } 820 } 821 822 static void test_fixed_counter_bitmap(void) 823 { 824 u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS); 825 struct kvm_vm *vm; 826 struct kvm_vcpu *vcpu; 827 u8 idx; 828 829 /* 830 * Check that pmu_event_filter works as expected when it's applied to 831 * fixed performance counters. 832 */ 833 for (idx = 0; idx < nr_fixed_counters; idx++) { 834 vm = vm_create_with_one_vcpu(&vcpu, 835 intel_run_fixed_counter_guest_code); 836 vcpu_args_set(vcpu, 1, idx); 837 __test_fixed_counter_bitmap(vcpu, idx, nr_fixed_counters); 838 kvm_vm_free(vm); 839 } 840 } 841 842 int main(int argc, char *argv[]) 843 { 844 void (*guest_code)(void); 845 struct kvm_vcpu *vcpu, *vcpu2 = NULL; 846 struct kvm_vm *vm; 847 848 TEST_REQUIRE(kvm_is_pmu_enabled()); 849 TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_FILTER)); 850 TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_MASKED_EVENTS)); 851 852 TEST_REQUIRE(use_intel_pmu() || use_amd_pmu()); 853 guest_code = use_intel_pmu() ? intel_guest_code : amd_guest_code; 854 855 vm = vm_create_with_one_vcpu(&vcpu, guest_code); 856 857 TEST_REQUIRE(sanity_check_pmu(vcpu)); 858 859 if (use_amd_pmu()) 860 test_amd_deny_list(vcpu); 861 862 test_without_filter(vcpu); 863 test_member_deny_list(vcpu); 864 test_member_allow_list(vcpu); 865 test_not_member_deny_list(vcpu); 866 test_not_member_allow_list(vcpu); 867 868 if (use_intel_pmu() && 869 supports_event_mem_inst_retired() && 870 kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) >= 3) 871 vcpu2 = vm_vcpu_add(vm, 2, intel_masked_events_guest_code); 872 else if (use_amd_pmu()) 873 vcpu2 = vm_vcpu_add(vm, 2, amd_masked_events_guest_code); 874 875 if (vcpu2) 876 test_masked_events(vcpu2); 877 test_filter_ioctl(vcpu); 878 879 kvm_vm_free(vm); 880 881 test_pmu_config_disable(guest_code); 882 test_fixed_counter_bitmap(); 883 884 return 0; 885 } 886