1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Test for x86 KVM_SET_PMU_EVENT_FILTER.
4 *
5 * Copyright (C) 2022, Google LLC.
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2.
8 *
9 * Verifies the expected behavior of allow lists and deny lists for
10 * virtual PMU events.
11 */
12 #include "kvm_util.h"
13 #include "pmu.h"
14 #include "processor.h"
15 #include "test_util.h"
16
17 #define NUM_BRANCHES 42
18 #define MAX_TEST_EVENTS 10
19
20 #define PMU_EVENT_FILTER_INVALID_ACTION (KVM_PMU_EVENT_DENY + 1)
21 #define PMU_EVENT_FILTER_INVALID_FLAGS (KVM_PMU_EVENT_FLAGS_VALID_MASK << 1)
22 #define PMU_EVENT_FILTER_INVALID_NEVENTS (KVM_PMU_EVENT_FILTER_MAX_EVENTS + 1)
23
24 struct __kvm_pmu_event_filter {
25 __u32 action;
26 __u32 nevents;
27 __u32 fixed_counter_bitmap;
28 __u32 flags;
29 __u32 pad[4];
30 __u64 events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];
31 };
32
33 /*
34 * This event list comprises Intel's known architectural events, plus AMD's
35 * Branch Instructions Retired for Zen CPUs. Note, AMD and Intel use the
36 * same encoding for Instructions Retired.
37 */
38 kvm_static_assert(INTEL_ARCH_INSTRUCTIONS_RETIRED == AMD_ZEN_INSTRUCTIONS_RETIRED);
39
40 static const struct __kvm_pmu_event_filter base_event_filter = {
41 .nevents = ARRAY_SIZE(base_event_filter.events),
42 .events = {
43 INTEL_ARCH_CPU_CYCLES,
44 INTEL_ARCH_INSTRUCTIONS_RETIRED,
45 INTEL_ARCH_REFERENCE_CYCLES,
46 INTEL_ARCH_LLC_REFERENCES,
47 INTEL_ARCH_LLC_MISSES,
48 INTEL_ARCH_BRANCHES_RETIRED,
49 INTEL_ARCH_BRANCHES_MISPREDICTED,
50 INTEL_ARCH_TOPDOWN_SLOTS,
51 AMD_ZEN_BRANCHES_RETIRED,
52 },
53 };
54
55 struct {
56 uint64_t loads;
57 uint64_t stores;
58 uint64_t loads_stores;
59 uint64_t branches_retired;
60 uint64_t instructions_retired;
61 } pmc_results;
62
63 /*
64 * If we encounter a #GP during the guest PMU sanity check, then the guest
65 * PMU is not functional. Inform the hypervisor via GUEST_SYNC(0).
66 */
guest_gp_handler(struct ex_regs * regs)67 static void guest_gp_handler(struct ex_regs *regs)
68 {
69 GUEST_SYNC(-EFAULT);
70 }
71
72 /*
73 * Check that we can write a new value to the given MSR and read it back.
74 * The caller should provide a non-empty set of bits that are safe to flip.
75 *
76 * Return on success. GUEST_SYNC(0) on error.
77 */
check_msr(uint32_t msr,uint64_t bits_to_flip)78 static void check_msr(uint32_t msr, uint64_t bits_to_flip)
79 {
80 uint64_t v = rdmsr(msr) ^ bits_to_flip;
81
82 wrmsr(msr, v);
83 if (rdmsr(msr) != v)
84 GUEST_SYNC(-EIO);
85
86 v ^= bits_to_flip;
87 wrmsr(msr, v);
88 if (rdmsr(msr) != v)
89 GUEST_SYNC(-EIO);
90 }
91
run_and_measure_loop(uint32_t msr_base)92 static void run_and_measure_loop(uint32_t msr_base)
93 {
94 const uint64_t branches_retired = rdmsr(msr_base + 0);
95 const uint64_t insn_retired = rdmsr(msr_base + 1);
96
97 __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
98
99 pmc_results.branches_retired = rdmsr(msr_base + 0) - branches_retired;
100 pmc_results.instructions_retired = rdmsr(msr_base + 1) - insn_retired;
101 }
102
intel_guest_code(void)103 static void intel_guest_code(void)
104 {
105 check_msr(MSR_CORE_PERF_GLOBAL_CTRL, 1);
106 check_msr(MSR_P6_EVNTSEL0, 0xffff);
107 check_msr(MSR_IA32_PMC0, 0xffff);
108 GUEST_SYNC(0);
109
110 for (;;) {
111 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
112 wrmsr(MSR_P6_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
113 ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_BRANCHES_RETIRED);
114 wrmsr(MSR_P6_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
115 ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_INSTRUCTIONS_RETIRED);
116 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x3);
117
118 run_and_measure_loop(MSR_IA32_PMC0);
119 GUEST_SYNC(0);
120 }
121 }
122
123 /*
124 * To avoid needing a check for CPUID.80000001:ECX.PerfCtrExtCore[bit 23],
125 * this code uses the always-available, legacy K7 PMU MSRs, which alias to
126 * the first four of the six extended core PMU MSRs.
127 */
amd_guest_code(void)128 static void amd_guest_code(void)
129 {
130 check_msr(MSR_K7_EVNTSEL0, 0xffff);
131 check_msr(MSR_K7_PERFCTR0, 0xffff);
132 GUEST_SYNC(0);
133
134 for (;;) {
135 wrmsr(MSR_K7_EVNTSEL0, 0);
136 wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
137 ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_BRANCHES_RETIRED);
138 wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
139 ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_INSTRUCTIONS_RETIRED);
140
141 run_and_measure_loop(MSR_K7_PERFCTR0);
142 GUEST_SYNC(0);
143 }
144 }
145
146 /*
147 * Run the VM to the next GUEST_SYNC(value), and return the value passed
148 * to the sync. Any other exit from the guest is fatal.
149 */
run_vcpu_to_sync(struct kvm_vcpu * vcpu)150 static uint64_t run_vcpu_to_sync(struct kvm_vcpu *vcpu)
151 {
152 struct ucall uc;
153
154 vcpu_run(vcpu);
155 TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
156 get_ucall(vcpu, &uc);
157 TEST_ASSERT(uc.cmd == UCALL_SYNC,
158 "Received ucall other than UCALL_SYNC: %lu", uc.cmd);
159 return uc.args[1];
160 }
161
run_vcpu_and_sync_pmc_results(struct kvm_vcpu * vcpu)162 static void run_vcpu_and_sync_pmc_results(struct kvm_vcpu *vcpu)
163 {
164 uint64_t r;
165
166 memset(&pmc_results, 0, sizeof(pmc_results));
167 sync_global_to_guest(vcpu->vm, pmc_results);
168
169 r = run_vcpu_to_sync(vcpu);
170 TEST_ASSERT(!r, "Unexpected sync value: 0x%lx", r);
171
172 sync_global_from_guest(vcpu->vm, pmc_results);
173 }
174
175 /*
176 * In a nested environment or if the vPMU is disabled, the guest PMU
177 * might not work as architected (accessing the PMU MSRs may raise
178 * #GP, or writes could simply be discarded). In those situations,
179 * there is no point in running these tests. The guest code will perform
180 * a sanity check and then GUEST_SYNC(success). In the case of failure,
181 * the behavior of the guest on resumption is undefined.
182 */
sanity_check_pmu(struct kvm_vcpu * vcpu)183 static bool sanity_check_pmu(struct kvm_vcpu *vcpu)
184 {
185 uint64_t r;
186
187 vm_install_exception_handler(vcpu->vm, GP_VECTOR, guest_gp_handler);
188 r = run_vcpu_to_sync(vcpu);
189 vm_install_exception_handler(vcpu->vm, GP_VECTOR, NULL);
190
191 return !r;
192 }
193
194 /*
195 * Remove the first occurrence of 'event' (if any) from the filter's
196 * event list.
197 */
remove_event(struct __kvm_pmu_event_filter * f,uint64_t event)198 static void remove_event(struct __kvm_pmu_event_filter *f, uint64_t event)
199 {
200 bool found = false;
201 int i;
202
203 for (i = 0; i < f->nevents; i++) {
204 if (found)
205 f->events[i - 1] = f->events[i];
206 else
207 found = f->events[i] == event;
208 }
209 if (found)
210 f->nevents--;
211 }
212
213 #define ASSERT_PMC_COUNTING_INSTRUCTIONS() \
214 do { \
215 uint64_t br = pmc_results.branches_retired; \
216 uint64_t ir = pmc_results.instructions_retired; \
217 bool br_matched = this_pmu_has_errata(BRANCHES_RETIRED_OVERCOUNT) ? \
218 br >= NUM_BRANCHES : br == NUM_BRANCHES; \
219 \
220 if (br && !br_matched) \
221 pr_info("%s: Branch instructions retired = %lu (expected %u)\n", \
222 __func__, br, NUM_BRANCHES); \
223 TEST_ASSERT(br, "%s: Branch instructions retired = %lu (expected > 0)", \
224 __func__, br); \
225 TEST_ASSERT(ir, "%s: Instructions retired = %lu (expected > 0)", \
226 __func__, ir); \
227 } while (0)
228
229 #define ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS() \
230 do { \
231 uint64_t br = pmc_results.branches_retired; \
232 uint64_t ir = pmc_results.instructions_retired; \
233 \
234 TEST_ASSERT(!br, "%s: Branch instructions retired = %lu (expected 0)", \
235 __func__, br); \
236 TEST_ASSERT(!ir, "%s: Instructions retired = %lu (expected 0)", \
237 __func__, ir); \
238 } while (0)
239
test_without_filter(struct kvm_vcpu * vcpu)240 static void test_without_filter(struct kvm_vcpu *vcpu)
241 {
242 run_vcpu_and_sync_pmc_results(vcpu);
243
244 ASSERT_PMC_COUNTING_INSTRUCTIONS();
245 }
246
test_with_filter(struct kvm_vcpu * vcpu,struct __kvm_pmu_event_filter * __f)247 static void test_with_filter(struct kvm_vcpu *vcpu,
248 struct __kvm_pmu_event_filter *__f)
249 {
250 struct kvm_pmu_event_filter *f = (void *)__f;
251
252 vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f);
253 run_vcpu_and_sync_pmc_results(vcpu);
254 }
255
test_amd_deny_list(struct kvm_vcpu * vcpu)256 static void test_amd_deny_list(struct kvm_vcpu *vcpu)
257 {
258 struct __kvm_pmu_event_filter f = {
259 .action = KVM_PMU_EVENT_DENY,
260 .nevents = 1,
261 .events = {
262 RAW_EVENT(0x1C2, 0),
263 },
264 };
265
266 test_with_filter(vcpu, &f);
267
268 ASSERT_PMC_COUNTING_INSTRUCTIONS();
269 }
270
test_member_deny_list(struct kvm_vcpu * vcpu)271 static void test_member_deny_list(struct kvm_vcpu *vcpu)
272 {
273 struct __kvm_pmu_event_filter f = base_event_filter;
274
275 f.action = KVM_PMU_EVENT_DENY;
276 test_with_filter(vcpu, &f);
277
278 ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS();
279 }
280
test_member_allow_list(struct kvm_vcpu * vcpu)281 static void test_member_allow_list(struct kvm_vcpu *vcpu)
282 {
283 struct __kvm_pmu_event_filter f = base_event_filter;
284
285 f.action = KVM_PMU_EVENT_ALLOW;
286 test_with_filter(vcpu, &f);
287
288 ASSERT_PMC_COUNTING_INSTRUCTIONS();
289 }
290
test_not_member_deny_list(struct kvm_vcpu * vcpu)291 static void test_not_member_deny_list(struct kvm_vcpu *vcpu)
292 {
293 struct __kvm_pmu_event_filter f = base_event_filter;
294
295 f.action = KVM_PMU_EVENT_DENY;
296
297 remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED);
298 remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED);
299 remove_event(&f, AMD_ZEN_BRANCHES_RETIRED);
300 test_with_filter(vcpu, &f);
301
302 ASSERT_PMC_COUNTING_INSTRUCTIONS();
303 }
304
test_not_member_allow_list(struct kvm_vcpu * vcpu)305 static void test_not_member_allow_list(struct kvm_vcpu *vcpu)
306 {
307 struct __kvm_pmu_event_filter f = base_event_filter;
308
309 f.action = KVM_PMU_EVENT_ALLOW;
310
311 remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED);
312 remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED);
313 remove_event(&f, AMD_ZEN_BRANCHES_RETIRED);
314 test_with_filter(vcpu, &f);
315
316 ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS();
317 }
318
319 /*
320 * Verify that setting KVM_PMU_CAP_DISABLE prevents the use of the PMU.
321 *
322 * Note that KVM_CAP_PMU_CAPABILITY must be invoked prior to creating VCPUs.
323 */
test_pmu_config_disable(void (* guest_code)(void))324 static void test_pmu_config_disable(void (*guest_code)(void))
325 {
326 struct kvm_vcpu *vcpu;
327 int r;
328 struct kvm_vm *vm;
329
330 r = kvm_check_cap(KVM_CAP_PMU_CAPABILITY);
331 if (!(r & KVM_PMU_CAP_DISABLE))
332 return;
333
334 vm = vm_create(1);
335
336 vm_enable_cap(vm, KVM_CAP_PMU_CAPABILITY, KVM_PMU_CAP_DISABLE);
337
338 vcpu = vm_vcpu_add(vm, 0, guest_code);
339 TEST_ASSERT(!sanity_check_pmu(vcpu),
340 "Guest should not be able to use disabled PMU.");
341
342 kvm_vm_free(vm);
343 }
344
345 /*
346 * On Intel, check for a non-zero PMU version, at least one general-purpose
347 * counter per logical processor, and support for counting the number of branch
348 * instructions retired.
349 */
use_intel_pmu(void)350 static bool use_intel_pmu(void)
351 {
352 return host_cpu_is_intel &&
353 kvm_cpu_property(X86_PROPERTY_PMU_VERSION) &&
354 kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) &&
355 kvm_pmu_has(X86_PMU_FEATURE_BRANCH_INSNS_RETIRED);
356 }
357
358 /*
359 * On AMD, all Family 17h+ CPUs (Zen and its successors) use event encoding
360 * 0xc2,0 for Branch Instructions Retired.
361 */
use_amd_pmu(void)362 static bool use_amd_pmu(void)
363 {
364 return host_cpu_is_amd && kvm_cpu_family() >= 0x17;
365 }
366
367 /*
368 * "MEM_INST_RETIRED.ALL_LOADS", "MEM_INST_RETIRED.ALL_STORES", and
369 * "MEM_INST_RETIRED.ANY" from https://perfmon-events.intel.com/
370 * supported on Intel Xeon processors:
371 * - Sapphire Rapids, Ice Lake, Cascade Lake, Skylake.
372 */
373 #define MEM_INST_RETIRED 0xD0
374 #define MEM_INST_RETIRED_LOAD RAW_EVENT(MEM_INST_RETIRED, 0x81)
375 #define MEM_INST_RETIRED_STORE RAW_EVENT(MEM_INST_RETIRED, 0x82)
376 #define MEM_INST_RETIRED_LOAD_STORE RAW_EVENT(MEM_INST_RETIRED, 0x83)
377
supports_event_mem_inst_retired(void)378 static bool supports_event_mem_inst_retired(void)
379 {
380 uint32_t eax, ebx, ecx, edx;
381
382 cpuid(1, &eax, &ebx, &ecx, &edx);
383 if (x86_family(eax) == 0x6) {
384 switch (x86_model(eax)) {
385 /* Sapphire Rapids */
386 case 0x8F:
387 /* Ice Lake */
388 case 0x6A:
389 /* Skylake */
390 /* Cascade Lake */
391 case 0x55:
392 return true;
393 }
394 }
395
396 return false;
397 }
398
399 /*
400 * "LS Dispatch", from Processor Programming Reference
401 * (PPR) for AMD Family 17h Model 01h, Revision B1 Processors,
402 * Preliminary Processor Programming Reference (PPR) for AMD Family
403 * 17h Model 31h, Revision B0 Processors, and Preliminary Processor
404 * Programming Reference (PPR) for AMD Family 19h Model 01h, Revision
405 * B1 Processors Volume 1 of 2.
406 */
407 #define LS_DISPATCH 0x29
408 #define LS_DISPATCH_LOAD RAW_EVENT(LS_DISPATCH, BIT(0))
409 #define LS_DISPATCH_STORE RAW_EVENT(LS_DISPATCH, BIT(1))
410 #define LS_DISPATCH_LOAD_STORE RAW_EVENT(LS_DISPATCH, BIT(2))
411
412 #define INCLUDE_MASKED_ENTRY(event_select, mask, match) \
413 KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, false)
414 #define EXCLUDE_MASKED_ENTRY(event_select, mask, match) \
415 KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, true)
416
masked_events_guest_test(uint32_t msr_base)417 static void masked_events_guest_test(uint32_t msr_base)
418 {
419 /*
420 * The actual value of the counters don't determine the outcome of
421 * the test. Only that they are zero or non-zero.
422 */
423 const uint64_t loads = rdmsr(msr_base + 0);
424 const uint64_t stores = rdmsr(msr_base + 1);
425 const uint64_t loads_stores = rdmsr(msr_base + 2);
426 int val;
427
428
429 __asm__ __volatile__("movl $0, %[v];"
430 "movl %[v], %%eax;"
431 "incl %[v];"
432 : [v]"+m"(val) :: "eax");
433
434 pmc_results.loads = rdmsr(msr_base + 0) - loads;
435 pmc_results.stores = rdmsr(msr_base + 1) - stores;
436 pmc_results.loads_stores = rdmsr(msr_base + 2) - loads_stores;
437 }
438
intel_masked_events_guest_code(void)439 static void intel_masked_events_guest_code(void)
440 {
441 for (;;) {
442 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
443
444 wrmsr(MSR_P6_EVNTSEL0 + 0, ARCH_PERFMON_EVENTSEL_ENABLE |
445 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD);
446 wrmsr(MSR_P6_EVNTSEL0 + 1, ARCH_PERFMON_EVENTSEL_ENABLE |
447 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_STORE);
448 wrmsr(MSR_P6_EVNTSEL0 + 2, ARCH_PERFMON_EVENTSEL_ENABLE |
449 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD_STORE);
450
451 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x7);
452
453 masked_events_guest_test(MSR_IA32_PMC0);
454 GUEST_SYNC(0);
455 }
456 }
457
amd_masked_events_guest_code(void)458 static void amd_masked_events_guest_code(void)
459 {
460 for (;;) {
461 wrmsr(MSR_K7_EVNTSEL0, 0);
462 wrmsr(MSR_K7_EVNTSEL1, 0);
463 wrmsr(MSR_K7_EVNTSEL2, 0);
464
465 wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
466 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD);
467 wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
468 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_STORE);
469 wrmsr(MSR_K7_EVNTSEL2, ARCH_PERFMON_EVENTSEL_ENABLE |
470 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD_STORE);
471
472 masked_events_guest_test(MSR_K7_PERFCTR0);
473 GUEST_SYNC(0);
474 }
475 }
476
run_masked_events_test(struct kvm_vcpu * vcpu,const uint64_t masked_events[],const int nmasked_events)477 static void run_masked_events_test(struct kvm_vcpu *vcpu,
478 const uint64_t masked_events[],
479 const int nmasked_events)
480 {
481 struct __kvm_pmu_event_filter f = {
482 .nevents = nmasked_events,
483 .action = KVM_PMU_EVENT_ALLOW,
484 .flags = KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
485 };
486
487 memcpy(f.events, masked_events, sizeof(uint64_t) * nmasked_events);
488 test_with_filter(vcpu, &f);
489 }
490
491 #define ALLOW_LOADS BIT(0)
492 #define ALLOW_STORES BIT(1)
493 #define ALLOW_LOADS_STORES BIT(2)
494
495 struct masked_events_test {
496 uint64_t intel_events[MAX_TEST_EVENTS];
497 uint64_t intel_event_end;
498 uint64_t amd_events[MAX_TEST_EVENTS];
499 uint64_t amd_event_end;
500 const char *msg;
501 uint32_t flags;
502 };
503
504 /*
505 * These are the test cases for the masked events tests.
506 *
507 * For each test, the guest enables 3 PMU counters (loads, stores,
508 * loads + stores). The filter is then set in KVM with the masked events
509 * provided. The test then verifies that the counters agree with which
510 * ones should be counting and which ones should be filtered.
511 */
512 const struct masked_events_test test_cases[] = {
513 {
514 .intel_events = {
515 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x81),
516 },
517 .amd_events = {
518 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)),
519 },
520 .msg = "Only allow loads.",
521 .flags = ALLOW_LOADS,
522 }, {
523 .intel_events = {
524 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82),
525 },
526 .amd_events = {
527 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)),
528 },
529 .msg = "Only allow stores.",
530 .flags = ALLOW_STORES,
531 }, {
532 .intel_events = {
533 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83),
534 },
535 .amd_events = {
536 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(2)),
537 },
538 .msg = "Only allow loads + stores.",
539 .flags = ALLOW_LOADS_STORES,
540 }, {
541 .intel_events = {
542 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
543 EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83),
544 },
545 .amd_events = {
546 INCLUDE_MASKED_ENTRY(LS_DISPATCH, ~(BIT(0) | BIT(1)), 0),
547 },
548 .msg = "Only allow loads and stores.",
549 .flags = ALLOW_LOADS | ALLOW_STORES,
550 }, {
551 .intel_events = {
552 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
553 EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82),
554 },
555 .amd_events = {
556 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
557 EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)),
558 },
559 .msg = "Only allow loads and loads + stores.",
560 .flags = ALLOW_LOADS | ALLOW_LOADS_STORES
561 }, {
562 .intel_events = {
563 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFE, 0x82),
564 },
565 .amd_events = {
566 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
567 EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)),
568 },
569 .msg = "Only allow stores and loads + stores.",
570 .flags = ALLOW_STORES | ALLOW_LOADS_STORES
571 }, {
572 .intel_events = {
573 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
574 },
575 .amd_events = {
576 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
577 },
578 .msg = "Only allow loads, stores, and loads + stores.",
579 .flags = ALLOW_LOADS | ALLOW_STORES | ALLOW_LOADS_STORES
580 },
581 };
582
append_test_events(const struct masked_events_test * test,uint64_t * events,int nevents)583 static int append_test_events(const struct masked_events_test *test,
584 uint64_t *events, int nevents)
585 {
586 const uint64_t *evts;
587 int i;
588
589 evts = use_intel_pmu() ? test->intel_events : test->amd_events;
590 for (i = 0; i < MAX_TEST_EVENTS; i++) {
591 if (evts[i] == 0)
592 break;
593
594 events[nevents + i] = evts[i];
595 }
596
597 return nevents + i;
598 }
599
bool_eq(bool a,bool b)600 static bool bool_eq(bool a, bool b)
601 {
602 return a == b;
603 }
604
run_masked_events_tests(struct kvm_vcpu * vcpu,uint64_t * events,int nevents)605 static void run_masked_events_tests(struct kvm_vcpu *vcpu, uint64_t *events,
606 int nevents)
607 {
608 int ntests = ARRAY_SIZE(test_cases);
609 int i, n;
610
611 for (i = 0; i < ntests; i++) {
612 const struct masked_events_test *test = &test_cases[i];
613
614 /* Do any test case events overflow MAX_TEST_EVENTS? */
615 assert(test->intel_event_end == 0);
616 assert(test->amd_event_end == 0);
617
618 n = append_test_events(test, events, nevents);
619
620 run_masked_events_test(vcpu, events, n);
621
622 TEST_ASSERT(bool_eq(pmc_results.loads, test->flags & ALLOW_LOADS) &&
623 bool_eq(pmc_results.stores, test->flags & ALLOW_STORES) &&
624 bool_eq(pmc_results.loads_stores,
625 test->flags & ALLOW_LOADS_STORES),
626 "%s loads: %lu, stores: %lu, loads + stores: %lu",
627 test->msg, pmc_results.loads, pmc_results.stores,
628 pmc_results.loads_stores);
629 }
630 }
631
add_dummy_events(uint64_t * events,int nevents)632 static void add_dummy_events(uint64_t *events, int nevents)
633 {
634 int i;
635
636 for (i = 0; i < nevents; i++) {
637 int event_select = i % 0xFF;
638 bool exclude = ((i % 4) == 0);
639
640 if (event_select == MEM_INST_RETIRED ||
641 event_select == LS_DISPATCH)
642 event_select++;
643
644 events[i] = KVM_PMU_ENCODE_MASKED_ENTRY(event_select, 0,
645 0, exclude);
646 }
647 }
648
test_masked_events(struct kvm_vcpu * vcpu)649 static void test_masked_events(struct kvm_vcpu *vcpu)
650 {
651 int nevents = KVM_PMU_EVENT_FILTER_MAX_EVENTS - MAX_TEST_EVENTS;
652 uint64_t events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];
653
654 /* Run the test cases against a sparse PMU event filter. */
655 run_masked_events_tests(vcpu, events, 0);
656
657 /* Run the test cases against a dense PMU event filter. */
658 add_dummy_events(events, KVM_PMU_EVENT_FILTER_MAX_EVENTS);
659 run_masked_events_tests(vcpu, events, nevents);
660 }
661
set_pmu_event_filter(struct kvm_vcpu * vcpu,struct __kvm_pmu_event_filter * __f)662 static int set_pmu_event_filter(struct kvm_vcpu *vcpu,
663 struct __kvm_pmu_event_filter *__f)
664 {
665 struct kvm_pmu_event_filter *f = (void *)__f;
666
667 return __vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f);
668 }
669
set_pmu_single_event_filter(struct kvm_vcpu * vcpu,uint64_t event,uint32_t flags,uint32_t action)670 static int set_pmu_single_event_filter(struct kvm_vcpu *vcpu, uint64_t event,
671 uint32_t flags, uint32_t action)
672 {
673 struct __kvm_pmu_event_filter f = {
674 .nevents = 1,
675 .flags = flags,
676 .action = action,
677 .events = {
678 event,
679 },
680 };
681
682 return set_pmu_event_filter(vcpu, &f);
683 }
684
test_filter_ioctl(struct kvm_vcpu * vcpu)685 static void test_filter_ioctl(struct kvm_vcpu *vcpu)
686 {
687 uint8_t nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
688 struct __kvm_pmu_event_filter f;
689 uint64_t e = ~0ul;
690 int r;
691
692 /*
693 * Unfortunately having invalid bits set in event data is expected to
694 * pass when flags == 0 (bits other than eventsel+umask).
695 */
696 r = set_pmu_single_event_filter(vcpu, e, 0, KVM_PMU_EVENT_ALLOW);
697 TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing");
698
699 r = set_pmu_single_event_filter(vcpu, e,
700 KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
701 KVM_PMU_EVENT_ALLOW);
702 TEST_ASSERT(r != 0, "Invalid PMU Event Filter is expected to fail");
703
704 e = KVM_PMU_ENCODE_MASKED_ENTRY(0xff, 0xff, 0xff, 0xf);
705 r = set_pmu_single_event_filter(vcpu, e,
706 KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
707 KVM_PMU_EVENT_ALLOW);
708 TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing");
709
710 f = base_event_filter;
711 f.action = PMU_EVENT_FILTER_INVALID_ACTION;
712 r = set_pmu_event_filter(vcpu, &f);
713 TEST_ASSERT(r, "Set invalid action is expected to fail");
714
715 f = base_event_filter;
716 f.flags = PMU_EVENT_FILTER_INVALID_FLAGS;
717 r = set_pmu_event_filter(vcpu, &f);
718 TEST_ASSERT(r, "Set invalid flags is expected to fail");
719
720 f = base_event_filter;
721 f.nevents = PMU_EVENT_FILTER_INVALID_NEVENTS;
722 r = set_pmu_event_filter(vcpu, &f);
723 TEST_ASSERT(r, "Exceeding the max number of filter events should fail");
724
725 f = base_event_filter;
726 f.fixed_counter_bitmap = ~GENMASK_ULL(nr_fixed_counters, 0);
727 r = set_pmu_event_filter(vcpu, &f);
728 TEST_ASSERT(!r, "Masking non-existent fixed counters should be allowed");
729 }
730
intel_run_fixed_counter_guest_code(uint8_t idx)731 static void intel_run_fixed_counter_guest_code(uint8_t idx)
732 {
733 for (;;) {
734 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
735 wrmsr(MSR_CORE_PERF_FIXED_CTR0 + idx, 0);
736
737 /* Only OS_EN bit is enabled for fixed counter[idx]. */
738 wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL));
739 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, FIXED_PMC_GLOBAL_CTRL_ENABLE(idx));
740 __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
741 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
742
743 GUEST_SYNC(rdmsr(MSR_CORE_PERF_FIXED_CTR0 + idx));
744 }
745 }
746
test_with_fixed_counter_filter(struct kvm_vcpu * vcpu,uint32_t action,uint32_t bitmap)747 static uint64_t test_with_fixed_counter_filter(struct kvm_vcpu *vcpu,
748 uint32_t action, uint32_t bitmap)
749 {
750 struct __kvm_pmu_event_filter f = {
751 .action = action,
752 .fixed_counter_bitmap = bitmap,
753 };
754 set_pmu_event_filter(vcpu, &f);
755
756 return run_vcpu_to_sync(vcpu);
757 }
758
test_set_gp_and_fixed_event_filter(struct kvm_vcpu * vcpu,uint32_t action,uint32_t bitmap)759 static uint64_t test_set_gp_and_fixed_event_filter(struct kvm_vcpu *vcpu,
760 uint32_t action,
761 uint32_t bitmap)
762 {
763 struct __kvm_pmu_event_filter f = base_event_filter;
764
765 f.action = action;
766 f.fixed_counter_bitmap = bitmap;
767 set_pmu_event_filter(vcpu, &f);
768
769 return run_vcpu_to_sync(vcpu);
770 }
771
__test_fixed_counter_bitmap(struct kvm_vcpu * vcpu,uint8_t idx,uint8_t nr_fixed_counters)772 static void __test_fixed_counter_bitmap(struct kvm_vcpu *vcpu, uint8_t idx,
773 uint8_t nr_fixed_counters)
774 {
775 unsigned int i;
776 uint32_t bitmap;
777 uint64_t count;
778
779 TEST_ASSERT(nr_fixed_counters < sizeof(bitmap) * 8,
780 "Invalid nr_fixed_counters");
781
782 /*
783 * Check the fixed performance counter can count normally when KVM
784 * userspace doesn't set any pmu filter.
785 */
786 count = run_vcpu_to_sync(vcpu);
787 TEST_ASSERT(count, "Unexpected count value: %ld", count);
788
789 for (i = 0; i < BIT(nr_fixed_counters); i++) {
790 bitmap = BIT(i);
791 count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_ALLOW,
792 bitmap);
793 TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx)));
794
795 count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_DENY,
796 bitmap);
797 TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx)));
798
799 /*
800 * Check that fixed_counter_bitmap has higher priority than
801 * events[] when both are set.
802 */
803 count = test_set_gp_and_fixed_event_filter(vcpu,
804 KVM_PMU_EVENT_ALLOW,
805 bitmap);
806 TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx)));
807
808 count = test_set_gp_and_fixed_event_filter(vcpu,
809 KVM_PMU_EVENT_DENY,
810 bitmap);
811 TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx)));
812 }
813 }
814
test_fixed_counter_bitmap(void)815 static void test_fixed_counter_bitmap(void)
816 {
817 uint8_t nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
818 struct kvm_vm *vm;
819 struct kvm_vcpu *vcpu;
820 uint8_t idx;
821
822 /*
823 * Check that pmu_event_filter works as expected when it's applied to
824 * fixed performance counters.
825 */
826 for (idx = 0; idx < nr_fixed_counters; idx++) {
827 vm = vm_create_with_one_vcpu(&vcpu,
828 intel_run_fixed_counter_guest_code);
829 vcpu_args_set(vcpu, 1, idx);
830 __test_fixed_counter_bitmap(vcpu, idx, nr_fixed_counters);
831 kvm_vm_free(vm);
832 }
833 }
834
main(int argc,char * argv[])835 int main(int argc, char *argv[])
836 {
837 void (*guest_code)(void);
838 struct kvm_vcpu *vcpu, *vcpu2 = NULL;
839 struct kvm_vm *vm;
840
841 TEST_REQUIRE(kvm_is_pmu_enabled());
842 TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_FILTER));
843 TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_MASKED_EVENTS));
844
845 TEST_REQUIRE(use_intel_pmu() || use_amd_pmu());
846 guest_code = use_intel_pmu() ? intel_guest_code : amd_guest_code;
847
848 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
849
850 TEST_REQUIRE(sanity_check_pmu(vcpu));
851
852 if (use_amd_pmu())
853 test_amd_deny_list(vcpu);
854
855 test_without_filter(vcpu);
856 test_member_deny_list(vcpu);
857 test_member_allow_list(vcpu);
858 test_not_member_deny_list(vcpu);
859 test_not_member_allow_list(vcpu);
860
861 if (use_intel_pmu() &&
862 supports_event_mem_inst_retired() &&
863 kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) >= 3)
864 vcpu2 = vm_vcpu_add(vm, 2, intel_masked_events_guest_code);
865 else if (use_amd_pmu())
866 vcpu2 = vm_vcpu_add(vm, 2, amd_masked_events_guest_code);
867
868 if (vcpu2)
869 test_masked_events(vcpu2);
870 test_filter_ioctl(vcpu);
871
872 kvm_vm_free(vm);
873
874 test_pmu_config_disable(guest_code);
875 test_fixed_counter_bitmap();
876
877 return 0;
878 }
879