1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Test for x86 KVM_SET_PMU_EVENT_FILTER.
4 *
5 * Copyright (C) 2022, Google LLC.
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2.
8 *
9 * Verifies the expected behavior of allow lists and deny lists for
10 * virtual PMU events.
11 */
12 #include "kvm_util.h"
13 #include "pmu.h"
14 #include "processor.h"
15 #include "test_util.h"
16
17 #define NUM_BRANCHES 42
18 #define MAX_TEST_EVENTS 10
19
20 #define PMU_EVENT_FILTER_INVALID_ACTION (KVM_PMU_EVENT_DENY + 1)
21 #define PMU_EVENT_FILTER_INVALID_FLAGS (KVM_PMU_EVENT_FLAGS_VALID_MASK << 1)
22 #define PMU_EVENT_FILTER_INVALID_NEVENTS (KVM_PMU_EVENT_FILTER_MAX_EVENTS + 1)
23
24 struct __kvm_pmu_event_filter {
25 __u32 action;
26 __u32 nevents;
27 __u32 fixed_counter_bitmap;
28 __u32 flags;
29 __u32 pad[4];
30 __u64 events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];
31 };
32
33 /*
34 * This event list comprises Intel's known architectural events, plus AMD's
35 * Branch Instructions Retired for Zen CPUs. Note, AMD and Intel use the
36 * same encoding for Instructions Retired.
37 */
38 kvm_static_assert(INTEL_ARCH_INSTRUCTIONS_RETIRED == AMD_ZEN_INSTRUCTIONS_RETIRED);
39
40 static const struct __kvm_pmu_event_filter base_event_filter = {
41 .nevents = ARRAY_SIZE(base_event_filter.events),
42 .events = {
43 INTEL_ARCH_CPU_CYCLES,
44 INTEL_ARCH_INSTRUCTIONS_RETIRED,
45 INTEL_ARCH_REFERENCE_CYCLES,
46 INTEL_ARCH_LLC_REFERENCES,
47 INTEL_ARCH_LLC_MISSES,
48 INTEL_ARCH_BRANCHES_RETIRED,
49 INTEL_ARCH_BRANCHES_MISPREDICTED,
50 INTEL_ARCH_TOPDOWN_SLOTS,
51 AMD_ZEN_BRANCHES_RETIRED,
52 },
53 };
54
55 struct {
56 uint64_t loads;
57 uint64_t stores;
58 uint64_t loads_stores;
59 uint64_t branches_retired;
60 uint64_t instructions_retired;
61 } pmc_results;
62
63 /*
64 * If we encounter a #GP during the guest PMU sanity check, then the guest
65 * PMU is not functional. Inform the hypervisor via GUEST_SYNC(0).
66 */
guest_gp_handler(struct ex_regs * regs)67 static void guest_gp_handler(struct ex_regs *regs)
68 {
69 GUEST_SYNC(-EFAULT);
70 }
71
72 /*
73 * Check that we can write a new value to the given MSR and read it back.
74 * The caller should provide a non-empty set of bits that are safe to flip.
75 *
76 * Return on success. GUEST_SYNC(0) on error.
77 */
check_msr(uint32_t msr,uint64_t bits_to_flip)78 static void check_msr(uint32_t msr, uint64_t bits_to_flip)
79 {
80 uint64_t v = rdmsr(msr) ^ bits_to_flip;
81
82 wrmsr(msr, v);
83 if (rdmsr(msr) != v)
84 GUEST_SYNC(-EIO);
85
86 v ^= bits_to_flip;
87 wrmsr(msr, v);
88 if (rdmsr(msr) != v)
89 GUEST_SYNC(-EIO);
90 }
91
run_and_measure_loop(uint32_t msr_base)92 static void run_and_measure_loop(uint32_t msr_base)
93 {
94 const uint64_t branches_retired = rdmsr(msr_base + 0);
95 const uint64_t insn_retired = rdmsr(msr_base + 1);
96
97 __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
98
99 pmc_results.branches_retired = rdmsr(msr_base + 0) - branches_retired;
100 pmc_results.instructions_retired = rdmsr(msr_base + 1) - insn_retired;
101 }
102
intel_guest_code(void)103 static void intel_guest_code(void)
104 {
105 check_msr(MSR_CORE_PERF_GLOBAL_CTRL, 1);
106 check_msr(MSR_P6_EVNTSEL0, 0xffff);
107 check_msr(MSR_IA32_PMC0, 0xffff);
108 GUEST_SYNC(0);
109
110 for (;;) {
111 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
112 wrmsr(MSR_P6_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
113 ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_BRANCHES_RETIRED);
114 wrmsr(MSR_P6_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
115 ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_INSTRUCTIONS_RETIRED);
116 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x3);
117
118 run_and_measure_loop(MSR_IA32_PMC0);
119 GUEST_SYNC(0);
120 }
121 }
122
123 /*
124 * To avoid needing a check for CPUID.80000001:ECX.PerfCtrExtCore[bit 23],
125 * this code uses the always-available, legacy K7 PMU MSRs, which alias to
126 * the first four of the six extended core PMU MSRs.
127 */
amd_guest_code(void)128 static void amd_guest_code(void)
129 {
130 check_msr(MSR_K7_EVNTSEL0, 0xffff);
131 check_msr(MSR_K7_PERFCTR0, 0xffff);
132 GUEST_SYNC(0);
133
134 for (;;) {
135 wrmsr(MSR_K7_EVNTSEL0, 0);
136 wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
137 ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_BRANCHES_RETIRED);
138 wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
139 ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_INSTRUCTIONS_RETIRED);
140
141 run_and_measure_loop(MSR_K7_PERFCTR0);
142 GUEST_SYNC(0);
143 }
144 }
145
146 /*
147 * Run the VM to the next GUEST_SYNC(value), and return the value passed
148 * to the sync. Any other exit from the guest is fatal.
149 */
run_vcpu_to_sync(struct kvm_vcpu * vcpu)150 static uint64_t run_vcpu_to_sync(struct kvm_vcpu *vcpu)
151 {
152 struct ucall uc;
153
154 vcpu_run(vcpu);
155 TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
156 get_ucall(vcpu, &uc);
157 TEST_ASSERT(uc.cmd == UCALL_SYNC,
158 "Received ucall other than UCALL_SYNC: %lu", uc.cmd);
159 return uc.args[1];
160 }
161
run_vcpu_and_sync_pmc_results(struct kvm_vcpu * vcpu)162 static void run_vcpu_and_sync_pmc_results(struct kvm_vcpu *vcpu)
163 {
164 uint64_t r;
165
166 memset(&pmc_results, 0, sizeof(pmc_results));
167 sync_global_to_guest(vcpu->vm, pmc_results);
168
169 r = run_vcpu_to_sync(vcpu);
170 TEST_ASSERT(!r, "Unexpected sync value: 0x%lx", r);
171
172 sync_global_from_guest(vcpu->vm, pmc_results);
173 }
174
175 /*
176 * In a nested environment or if the vPMU is disabled, the guest PMU
177 * might not work as architected (accessing the PMU MSRs may raise
178 * #GP, or writes could simply be discarded). In those situations,
179 * there is no point in running these tests. The guest code will perform
180 * a sanity check and then GUEST_SYNC(success). In the case of failure,
181 * the behavior of the guest on resumption is undefined.
182 */
sanity_check_pmu(struct kvm_vcpu * vcpu)183 static bool sanity_check_pmu(struct kvm_vcpu *vcpu)
184 {
185 uint64_t r;
186
187 vm_install_exception_handler(vcpu->vm, GP_VECTOR, guest_gp_handler);
188 r = run_vcpu_to_sync(vcpu);
189 vm_install_exception_handler(vcpu->vm, GP_VECTOR, NULL);
190
191 return !r;
192 }
193
194 /*
195 * Remove the first occurrence of 'event' (if any) from the filter's
196 * event list.
197 */
remove_event(struct __kvm_pmu_event_filter * f,uint64_t event)198 static void remove_event(struct __kvm_pmu_event_filter *f, uint64_t event)
199 {
200 bool found = false;
201 int i;
202
203 for (i = 0; i < f->nevents; i++) {
204 if (found)
205 f->events[i - 1] = f->events[i];
206 else
207 found = f->events[i] == event;
208 }
209 if (found)
210 f->nevents--;
211 }
212
213 #define ASSERT_PMC_COUNTING_INSTRUCTIONS() \
214 do { \
215 uint64_t br = pmc_results.branches_retired; \
216 uint64_t ir = pmc_results.instructions_retired; \
217 \
218 if (br && br != NUM_BRANCHES) \
219 pr_info("%s: Branch instructions retired = %lu (expected %u)\n", \
220 __func__, br, NUM_BRANCHES); \
221 TEST_ASSERT(br, "%s: Branch instructions retired = %lu (expected > 0)", \
222 __func__, br); \
223 TEST_ASSERT(ir, "%s: Instructions retired = %lu (expected > 0)", \
224 __func__, ir); \
225 } while (0)
226
227 #define ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS() \
228 do { \
229 uint64_t br = pmc_results.branches_retired; \
230 uint64_t ir = pmc_results.instructions_retired; \
231 \
232 TEST_ASSERT(!br, "%s: Branch instructions retired = %lu (expected 0)", \
233 __func__, br); \
234 TEST_ASSERT(!ir, "%s: Instructions retired = %lu (expected 0)", \
235 __func__, ir); \
236 } while (0)
237
test_without_filter(struct kvm_vcpu * vcpu)238 static void test_without_filter(struct kvm_vcpu *vcpu)
239 {
240 run_vcpu_and_sync_pmc_results(vcpu);
241
242 ASSERT_PMC_COUNTING_INSTRUCTIONS();
243 }
244
test_with_filter(struct kvm_vcpu * vcpu,struct __kvm_pmu_event_filter * __f)245 static void test_with_filter(struct kvm_vcpu *vcpu,
246 struct __kvm_pmu_event_filter *__f)
247 {
248 struct kvm_pmu_event_filter *f = (void *)__f;
249
250 vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f);
251 run_vcpu_and_sync_pmc_results(vcpu);
252 }
253
test_amd_deny_list(struct kvm_vcpu * vcpu)254 static void test_amd_deny_list(struct kvm_vcpu *vcpu)
255 {
256 struct __kvm_pmu_event_filter f = {
257 .action = KVM_PMU_EVENT_DENY,
258 .nevents = 1,
259 .events = {
260 RAW_EVENT(0x1C2, 0),
261 },
262 };
263
264 test_with_filter(vcpu, &f);
265
266 ASSERT_PMC_COUNTING_INSTRUCTIONS();
267 }
268
test_member_deny_list(struct kvm_vcpu * vcpu)269 static void test_member_deny_list(struct kvm_vcpu *vcpu)
270 {
271 struct __kvm_pmu_event_filter f = base_event_filter;
272
273 f.action = KVM_PMU_EVENT_DENY;
274 test_with_filter(vcpu, &f);
275
276 ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS();
277 }
278
test_member_allow_list(struct kvm_vcpu * vcpu)279 static void test_member_allow_list(struct kvm_vcpu *vcpu)
280 {
281 struct __kvm_pmu_event_filter f = base_event_filter;
282
283 f.action = KVM_PMU_EVENT_ALLOW;
284 test_with_filter(vcpu, &f);
285
286 ASSERT_PMC_COUNTING_INSTRUCTIONS();
287 }
288
test_not_member_deny_list(struct kvm_vcpu * vcpu)289 static void test_not_member_deny_list(struct kvm_vcpu *vcpu)
290 {
291 struct __kvm_pmu_event_filter f = base_event_filter;
292
293 f.action = KVM_PMU_EVENT_DENY;
294
295 remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED);
296 remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED);
297 remove_event(&f, AMD_ZEN_BRANCHES_RETIRED);
298 test_with_filter(vcpu, &f);
299
300 ASSERT_PMC_COUNTING_INSTRUCTIONS();
301 }
302
test_not_member_allow_list(struct kvm_vcpu * vcpu)303 static void test_not_member_allow_list(struct kvm_vcpu *vcpu)
304 {
305 struct __kvm_pmu_event_filter f = base_event_filter;
306
307 f.action = KVM_PMU_EVENT_ALLOW;
308
309 remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED);
310 remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED);
311 remove_event(&f, AMD_ZEN_BRANCHES_RETIRED);
312 test_with_filter(vcpu, &f);
313
314 ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS();
315 }
316
317 /*
318 * Verify that setting KVM_PMU_CAP_DISABLE prevents the use of the PMU.
319 *
320 * Note that KVM_CAP_PMU_CAPABILITY must be invoked prior to creating VCPUs.
321 */
test_pmu_config_disable(void (* guest_code)(void))322 static void test_pmu_config_disable(void (*guest_code)(void))
323 {
324 struct kvm_vcpu *vcpu;
325 int r;
326 struct kvm_vm *vm;
327
328 r = kvm_check_cap(KVM_CAP_PMU_CAPABILITY);
329 if (!(r & KVM_PMU_CAP_DISABLE))
330 return;
331
332 vm = vm_create(1);
333
334 vm_enable_cap(vm, KVM_CAP_PMU_CAPABILITY, KVM_PMU_CAP_DISABLE);
335
336 vcpu = vm_vcpu_add(vm, 0, guest_code);
337 TEST_ASSERT(!sanity_check_pmu(vcpu),
338 "Guest should not be able to use disabled PMU.");
339
340 kvm_vm_free(vm);
341 }
342
343 /*
344 * On Intel, check for a non-zero PMU version, at least one general-purpose
345 * counter per logical processor, and support for counting the number of branch
346 * instructions retired.
347 */
use_intel_pmu(void)348 static bool use_intel_pmu(void)
349 {
350 return host_cpu_is_intel &&
351 kvm_cpu_property(X86_PROPERTY_PMU_VERSION) &&
352 kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) &&
353 kvm_pmu_has(X86_PMU_FEATURE_BRANCH_INSNS_RETIRED);
354 }
355
356 /*
357 * On AMD, all Family 17h+ CPUs (Zen and its successors) use event encoding
358 * 0xc2,0 for Branch Instructions Retired.
359 */
use_amd_pmu(void)360 static bool use_amd_pmu(void)
361 {
362 return host_cpu_is_amd && kvm_cpu_family() >= 0x17;
363 }
364
365 /*
366 * "MEM_INST_RETIRED.ALL_LOADS", "MEM_INST_RETIRED.ALL_STORES", and
367 * "MEM_INST_RETIRED.ANY" from https://perfmon-events.intel.com/
368 * supported on Intel Xeon processors:
369 * - Sapphire Rapids, Ice Lake, Cascade Lake, Skylake.
370 */
371 #define MEM_INST_RETIRED 0xD0
372 #define MEM_INST_RETIRED_LOAD RAW_EVENT(MEM_INST_RETIRED, 0x81)
373 #define MEM_INST_RETIRED_STORE RAW_EVENT(MEM_INST_RETIRED, 0x82)
374 #define MEM_INST_RETIRED_LOAD_STORE RAW_EVENT(MEM_INST_RETIRED, 0x83)
375
supports_event_mem_inst_retired(void)376 static bool supports_event_mem_inst_retired(void)
377 {
378 uint32_t eax, ebx, ecx, edx;
379
380 cpuid(1, &eax, &ebx, &ecx, &edx);
381 if (x86_family(eax) == 0x6) {
382 switch (x86_model(eax)) {
383 /* Sapphire Rapids */
384 case 0x8F:
385 /* Ice Lake */
386 case 0x6A:
387 /* Skylake */
388 /* Cascade Lake */
389 case 0x55:
390 return true;
391 }
392 }
393
394 return false;
395 }
396
397 /*
398 * "LS Dispatch", from Processor Programming Reference
399 * (PPR) for AMD Family 17h Model 01h, Revision B1 Processors,
400 * Preliminary Processor Programming Reference (PPR) for AMD Family
401 * 17h Model 31h, Revision B0 Processors, and Preliminary Processor
402 * Programming Reference (PPR) for AMD Family 19h Model 01h, Revision
403 * B1 Processors Volume 1 of 2.
404 */
405 #define LS_DISPATCH 0x29
406 #define LS_DISPATCH_LOAD RAW_EVENT(LS_DISPATCH, BIT(0))
407 #define LS_DISPATCH_STORE RAW_EVENT(LS_DISPATCH, BIT(1))
408 #define LS_DISPATCH_LOAD_STORE RAW_EVENT(LS_DISPATCH, BIT(2))
409
410 #define INCLUDE_MASKED_ENTRY(event_select, mask, match) \
411 KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, false)
412 #define EXCLUDE_MASKED_ENTRY(event_select, mask, match) \
413 KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, true)
414
masked_events_guest_test(uint32_t msr_base)415 static void masked_events_guest_test(uint32_t msr_base)
416 {
417 /*
418 * The actual value of the counters don't determine the outcome of
419 * the test. Only that they are zero or non-zero.
420 */
421 const uint64_t loads = rdmsr(msr_base + 0);
422 const uint64_t stores = rdmsr(msr_base + 1);
423 const uint64_t loads_stores = rdmsr(msr_base + 2);
424 int val;
425
426
427 __asm__ __volatile__("movl $0, %[v];"
428 "movl %[v], %%eax;"
429 "incl %[v];"
430 : [v]"+m"(val) :: "eax");
431
432 pmc_results.loads = rdmsr(msr_base + 0) - loads;
433 pmc_results.stores = rdmsr(msr_base + 1) - stores;
434 pmc_results.loads_stores = rdmsr(msr_base + 2) - loads_stores;
435 }
436
intel_masked_events_guest_code(void)437 static void intel_masked_events_guest_code(void)
438 {
439 for (;;) {
440 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
441
442 wrmsr(MSR_P6_EVNTSEL0 + 0, ARCH_PERFMON_EVENTSEL_ENABLE |
443 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD);
444 wrmsr(MSR_P6_EVNTSEL0 + 1, ARCH_PERFMON_EVENTSEL_ENABLE |
445 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_STORE);
446 wrmsr(MSR_P6_EVNTSEL0 + 2, ARCH_PERFMON_EVENTSEL_ENABLE |
447 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD_STORE);
448
449 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x7);
450
451 masked_events_guest_test(MSR_IA32_PMC0);
452 GUEST_SYNC(0);
453 }
454 }
455
amd_masked_events_guest_code(void)456 static void amd_masked_events_guest_code(void)
457 {
458 for (;;) {
459 wrmsr(MSR_K7_EVNTSEL0, 0);
460 wrmsr(MSR_K7_EVNTSEL1, 0);
461 wrmsr(MSR_K7_EVNTSEL2, 0);
462
463 wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
464 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD);
465 wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
466 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_STORE);
467 wrmsr(MSR_K7_EVNTSEL2, ARCH_PERFMON_EVENTSEL_ENABLE |
468 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD_STORE);
469
470 masked_events_guest_test(MSR_K7_PERFCTR0);
471 GUEST_SYNC(0);
472 }
473 }
474
run_masked_events_test(struct kvm_vcpu * vcpu,const uint64_t masked_events[],const int nmasked_events)475 static void run_masked_events_test(struct kvm_vcpu *vcpu,
476 const uint64_t masked_events[],
477 const int nmasked_events)
478 {
479 struct __kvm_pmu_event_filter f = {
480 .nevents = nmasked_events,
481 .action = KVM_PMU_EVENT_ALLOW,
482 .flags = KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
483 };
484
485 memcpy(f.events, masked_events, sizeof(uint64_t) * nmasked_events);
486 test_with_filter(vcpu, &f);
487 }
488
489 #define ALLOW_LOADS BIT(0)
490 #define ALLOW_STORES BIT(1)
491 #define ALLOW_LOADS_STORES BIT(2)
492
493 struct masked_events_test {
494 uint64_t intel_events[MAX_TEST_EVENTS];
495 uint64_t intel_event_end;
496 uint64_t amd_events[MAX_TEST_EVENTS];
497 uint64_t amd_event_end;
498 const char *msg;
499 uint32_t flags;
500 };
501
502 /*
503 * These are the test cases for the masked events tests.
504 *
505 * For each test, the guest enables 3 PMU counters (loads, stores,
506 * loads + stores). The filter is then set in KVM with the masked events
507 * provided. The test then verifies that the counters agree with which
508 * ones should be counting and which ones should be filtered.
509 */
510 const struct masked_events_test test_cases[] = {
511 {
512 .intel_events = {
513 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x81),
514 },
515 .amd_events = {
516 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)),
517 },
518 .msg = "Only allow loads.",
519 .flags = ALLOW_LOADS,
520 }, {
521 .intel_events = {
522 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82),
523 },
524 .amd_events = {
525 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)),
526 },
527 .msg = "Only allow stores.",
528 .flags = ALLOW_STORES,
529 }, {
530 .intel_events = {
531 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83),
532 },
533 .amd_events = {
534 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(2)),
535 },
536 .msg = "Only allow loads + stores.",
537 .flags = ALLOW_LOADS_STORES,
538 }, {
539 .intel_events = {
540 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
541 EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83),
542 },
543 .amd_events = {
544 INCLUDE_MASKED_ENTRY(LS_DISPATCH, ~(BIT(0) | BIT(1)), 0),
545 },
546 .msg = "Only allow loads and stores.",
547 .flags = ALLOW_LOADS | ALLOW_STORES,
548 }, {
549 .intel_events = {
550 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
551 EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82),
552 },
553 .amd_events = {
554 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
555 EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)),
556 },
557 .msg = "Only allow loads and loads + stores.",
558 .flags = ALLOW_LOADS | ALLOW_LOADS_STORES
559 }, {
560 .intel_events = {
561 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFE, 0x82),
562 },
563 .amd_events = {
564 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
565 EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)),
566 },
567 .msg = "Only allow stores and loads + stores.",
568 .flags = ALLOW_STORES | ALLOW_LOADS_STORES
569 }, {
570 .intel_events = {
571 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
572 },
573 .amd_events = {
574 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
575 },
576 .msg = "Only allow loads, stores, and loads + stores.",
577 .flags = ALLOW_LOADS | ALLOW_STORES | ALLOW_LOADS_STORES
578 },
579 };
580
append_test_events(const struct masked_events_test * test,uint64_t * events,int nevents)581 static int append_test_events(const struct masked_events_test *test,
582 uint64_t *events, int nevents)
583 {
584 const uint64_t *evts;
585 int i;
586
587 evts = use_intel_pmu() ? test->intel_events : test->amd_events;
588 for (i = 0; i < MAX_TEST_EVENTS; i++) {
589 if (evts[i] == 0)
590 break;
591
592 events[nevents + i] = evts[i];
593 }
594
595 return nevents + i;
596 }
597
bool_eq(bool a,bool b)598 static bool bool_eq(bool a, bool b)
599 {
600 return a == b;
601 }
602
run_masked_events_tests(struct kvm_vcpu * vcpu,uint64_t * events,int nevents)603 static void run_masked_events_tests(struct kvm_vcpu *vcpu, uint64_t *events,
604 int nevents)
605 {
606 int ntests = ARRAY_SIZE(test_cases);
607 int i, n;
608
609 for (i = 0; i < ntests; i++) {
610 const struct masked_events_test *test = &test_cases[i];
611
612 /* Do any test case events overflow MAX_TEST_EVENTS? */
613 assert(test->intel_event_end == 0);
614 assert(test->amd_event_end == 0);
615
616 n = append_test_events(test, events, nevents);
617
618 run_masked_events_test(vcpu, events, n);
619
620 TEST_ASSERT(bool_eq(pmc_results.loads, test->flags & ALLOW_LOADS) &&
621 bool_eq(pmc_results.stores, test->flags & ALLOW_STORES) &&
622 bool_eq(pmc_results.loads_stores,
623 test->flags & ALLOW_LOADS_STORES),
624 "%s loads: %lu, stores: %lu, loads + stores: %lu",
625 test->msg, pmc_results.loads, pmc_results.stores,
626 pmc_results.loads_stores);
627 }
628 }
629
add_dummy_events(uint64_t * events,int nevents)630 static void add_dummy_events(uint64_t *events, int nevents)
631 {
632 int i;
633
634 for (i = 0; i < nevents; i++) {
635 int event_select = i % 0xFF;
636 bool exclude = ((i % 4) == 0);
637
638 if (event_select == MEM_INST_RETIRED ||
639 event_select == LS_DISPATCH)
640 event_select++;
641
642 events[i] = KVM_PMU_ENCODE_MASKED_ENTRY(event_select, 0,
643 0, exclude);
644 }
645 }
646
test_masked_events(struct kvm_vcpu * vcpu)647 static void test_masked_events(struct kvm_vcpu *vcpu)
648 {
649 int nevents = KVM_PMU_EVENT_FILTER_MAX_EVENTS - MAX_TEST_EVENTS;
650 uint64_t events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];
651
652 /* Run the test cases against a sparse PMU event filter. */
653 run_masked_events_tests(vcpu, events, 0);
654
655 /* Run the test cases against a dense PMU event filter. */
656 add_dummy_events(events, KVM_PMU_EVENT_FILTER_MAX_EVENTS);
657 run_masked_events_tests(vcpu, events, nevents);
658 }
659
set_pmu_event_filter(struct kvm_vcpu * vcpu,struct __kvm_pmu_event_filter * __f)660 static int set_pmu_event_filter(struct kvm_vcpu *vcpu,
661 struct __kvm_pmu_event_filter *__f)
662 {
663 struct kvm_pmu_event_filter *f = (void *)__f;
664
665 return __vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f);
666 }
667
set_pmu_single_event_filter(struct kvm_vcpu * vcpu,uint64_t event,uint32_t flags,uint32_t action)668 static int set_pmu_single_event_filter(struct kvm_vcpu *vcpu, uint64_t event,
669 uint32_t flags, uint32_t action)
670 {
671 struct __kvm_pmu_event_filter f = {
672 .nevents = 1,
673 .flags = flags,
674 .action = action,
675 .events = {
676 event,
677 },
678 };
679
680 return set_pmu_event_filter(vcpu, &f);
681 }
682
test_filter_ioctl(struct kvm_vcpu * vcpu)683 static void test_filter_ioctl(struct kvm_vcpu *vcpu)
684 {
685 uint8_t nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
686 struct __kvm_pmu_event_filter f;
687 uint64_t e = ~0ul;
688 int r;
689
690 /*
691 * Unfortunately having invalid bits set in event data is expected to
692 * pass when flags == 0 (bits other than eventsel+umask).
693 */
694 r = set_pmu_single_event_filter(vcpu, e, 0, KVM_PMU_EVENT_ALLOW);
695 TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing");
696
697 r = set_pmu_single_event_filter(vcpu, e,
698 KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
699 KVM_PMU_EVENT_ALLOW);
700 TEST_ASSERT(r != 0, "Invalid PMU Event Filter is expected to fail");
701
702 e = KVM_PMU_ENCODE_MASKED_ENTRY(0xff, 0xff, 0xff, 0xf);
703 r = set_pmu_single_event_filter(vcpu, e,
704 KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
705 KVM_PMU_EVENT_ALLOW);
706 TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing");
707
708 f = base_event_filter;
709 f.action = PMU_EVENT_FILTER_INVALID_ACTION;
710 r = set_pmu_event_filter(vcpu, &f);
711 TEST_ASSERT(r, "Set invalid action is expected to fail");
712
713 f = base_event_filter;
714 f.flags = PMU_EVENT_FILTER_INVALID_FLAGS;
715 r = set_pmu_event_filter(vcpu, &f);
716 TEST_ASSERT(r, "Set invalid flags is expected to fail");
717
718 f = base_event_filter;
719 f.nevents = PMU_EVENT_FILTER_INVALID_NEVENTS;
720 r = set_pmu_event_filter(vcpu, &f);
721 TEST_ASSERT(r, "Exceeding the max number of filter events should fail");
722
723 f = base_event_filter;
724 f.fixed_counter_bitmap = ~GENMASK_ULL(nr_fixed_counters, 0);
725 r = set_pmu_event_filter(vcpu, &f);
726 TEST_ASSERT(!r, "Masking non-existent fixed counters should be allowed");
727 }
728
intel_run_fixed_counter_guest_code(uint8_t idx)729 static void intel_run_fixed_counter_guest_code(uint8_t idx)
730 {
731 for (;;) {
732 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
733 wrmsr(MSR_CORE_PERF_FIXED_CTR0 + idx, 0);
734
735 /* Only OS_EN bit is enabled for fixed counter[idx]. */
736 wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL));
737 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, FIXED_PMC_GLOBAL_CTRL_ENABLE(idx));
738 __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
739 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
740
741 GUEST_SYNC(rdmsr(MSR_CORE_PERF_FIXED_CTR0 + idx));
742 }
743 }
744
test_with_fixed_counter_filter(struct kvm_vcpu * vcpu,uint32_t action,uint32_t bitmap)745 static uint64_t test_with_fixed_counter_filter(struct kvm_vcpu *vcpu,
746 uint32_t action, uint32_t bitmap)
747 {
748 struct __kvm_pmu_event_filter f = {
749 .action = action,
750 .fixed_counter_bitmap = bitmap,
751 };
752 set_pmu_event_filter(vcpu, &f);
753
754 return run_vcpu_to_sync(vcpu);
755 }
756
test_set_gp_and_fixed_event_filter(struct kvm_vcpu * vcpu,uint32_t action,uint32_t bitmap)757 static uint64_t test_set_gp_and_fixed_event_filter(struct kvm_vcpu *vcpu,
758 uint32_t action,
759 uint32_t bitmap)
760 {
761 struct __kvm_pmu_event_filter f = base_event_filter;
762
763 f.action = action;
764 f.fixed_counter_bitmap = bitmap;
765 set_pmu_event_filter(vcpu, &f);
766
767 return run_vcpu_to_sync(vcpu);
768 }
769
__test_fixed_counter_bitmap(struct kvm_vcpu * vcpu,uint8_t idx,uint8_t nr_fixed_counters)770 static void __test_fixed_counter_bitmap(struct kvm_vcpu *vcpu, uint8_t idx,
771 uint8_t nr_fixed_counters)
772 {
773 unsigned int i;
774 uint32_t bitmap;
775 uint64_t count;
776
777 TEST_ASSERT(nr_fixed_counters < sizeof(bitmap) * 8,
778 "Invalid nr_fixed_counters");
779
780 /*
781 * Check the fixed performance counter can count normally when KVM
782 * userspace doesn't set any pmu filter.
783 */
784 count = run_vcpu_to_sync(vcpu);
785 TEST_ASSERT(count, "Unexpected count value: %ld", count);
786
787 for (i = 0; i < BIT(nr_fixed_counters); i++) {
788 bitmap = BIT(i);
789 count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_ALLOW,
790 bitmap);
791 TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx)));
792
793 count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_DENY,
794 bitmap);
795 TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx)));
796
797 /*
798 * Check that fixed_counter_bitmap has higher priority than
799 * events[] when both are set.
800 */
801 count = test_set_gp_and_fixed_event_filter(vcpu,
802 KVM_PMU_EVENT_ALLOW,
803 bitmap);
804 TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx)));
805
806 count = test_set_gp_and_fixed_event_filter(vcpu,
807 KVM_PMU_EVENT_DENY,
808 bitmap);
809 TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx)));
810 }
811 }
812
test_fixed_counter_bitmap(void)813 static void test_fixed_counter_bitmap(void)
814 {
815 uint8_t nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
816 struct kvm_vm *vm;
817 struct kvm_vcpu *vcpu;
818 uint8_t idx;
819
820 /*
821 * Check that pmu_event_filter works as expected when it's applied to
822 * fixed performance counters.
823 */
824 for (idx = 0; idx < nr_fixed_counters; idx++) {
825 vm = vm_create_with_one_vcpu(&vcpu,
826 intel_run_fixed_counter_guest_code);
827 vcpu_args_set(vcpu, 1, idx);
828 __test_fixed_counter_bitmap(vcpu, idx, nr_fixed_counters);
829 kvm_vm_free(vm);
830 }
831 }
832
main(int argc,char * argv[])833 int main(int argc, char *argv[])
834 {
835 void (*guest_code)(void);
836 struct kvm_vcpu *vcpu, *vcpu2 = NULL;
837 struct kvm_vm *vm;
838
839 TEST_REQUIRE(kvm_is_pmu_enabled());
840 TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_FILTER));
841 TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_MASKED_EVENTS));
842
843 TEST_REQUIRE(use_intel_pmu() || use_amd_pmu());
844 guest_code = use_intel_pmu() ? intel_guest_code : amd_guest_code;
845
846 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
847
848 TEST_REQUIRE(sanity_check_pmu(vcpu));
849
850 if (use_amd_pmu())
851 test_amd_deny_list(vcpu);
852
853 test_without_filter(vcpu);
854 test_member_deny_list(vcpu);
855 test_member_allow_list(vcpu);
856 test_not_member_deny_list(vcpu);
857 test_not_member_allow_list(vcpu);
858
859 if (use_intel_pmu() &&
860 supports_event_mem_inst_retired() &&
861 kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) >= 3)
862 vcpu2 = vm_vcpu_add(vm, 2, intel_masked_events_guest_code);
863 else if (use_amd_pmu())
864 vcpu2 = vm_vcpu_add(vm, 2, amd_masked_events_guest_code);
865
866 if (vcpu2)
867 test_masked_events(vcpu2);
868 test_filter_ioctl(vcpu);
869
870 kvm_vm_free(vm);
871
872 test_pmu_config_disable(guest_code);
873 test_fixed_counter_bitmap();
874
875 return 0;
876 }
877