1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Test for x86 KVM_SET_PMU_EVENT_FILTER.
4 *
5 * Copyright (C) 2022, Google LLC.
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2.
8 *
9 * Verifies the expected behavior of allow lists and deny lists for
10 * virtual PMU events.
11 */
12 #include "kvm_util.h"
13 #include "pmu.h"
14 #include "processor.h"
15 #include "test_util.h"
16
17 #define NUM_BRANCHES 42
18 #define MAX_TEST_EVENTS 10
19
20 #define PMU_EVENT_FILTER_INVALID_ACTION (KVM_PMU_EVENT_DENY + 1)
21 #define PMU_EVENT_FILTER_INVALID_FLAGS (KVM_PMU_EVENT_FLAGS_VALID_MASK << 1)
22 #define PMU_EVENT_FILTER_INVALID_NEVENTS (KVM_PMU_EVENT_FILTER_MAX_EVENTS + 1)
23
24 struct __kvm_pmu_event_filter {
25 __u32 action;
26 __u32 nevents;
27 __u32 fixed_counter_bitmap;
28 __u32 flags;
29 __u32 pad[4];
30 __u64 events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];
31 };
32
33 /*
34 * This event list comprises Intel's known architectural events, plus AMD's
35 * Branch Instructions Retired for Zen CPUs. Note, AMD and Intel use the
36 * same encoding for Instructions Retired.
37 */
38 kvm_static_assert(INTEL_ARCH_INSTRUCTIONS_RETIRED == AMD_ZEN_INSTRUCTIONS_RETIRED);
39
40 static const struct __kvm_pmu_event_filter base_event_filter = {
41 .nevents = ARRAY_SIZE(base_event_filter.events),
42 .events = {
43 INTEL_ARCH_CPU_CYCLES,
44 INTEL_ARCH_INSTRUCTIONS_RETIRED,
45 INTEL_ARCH_REFERENCE_CYCLES,
46 INTEL_ARCH_LLC_REFERENCES,
47 INTEL_ARCH_LLC_MISSES,
48 INTEL_ARCH_BRANCHES_RETIRED,
49 INTEL_ARCH_BRANCHES_MISPREDICTED,
50 INTEL_ARCH_TOPDOWN_SLOTS,
51 AMD_ZEN_BRANCHES_RETIRED,
52 },
53 };
54
55 struct {
56 u64 loads;
57 u64 stores;
58 u64 loads_stores;
59 u64 branches_retired;
60 u64 instructions_retired;
61 } pmc_results;
62
63 /*
64 * If we encounter a #GP during the guest PMU sanity check, then the guest
65 * PMU is not functional. Inform the hypervisor via GUEST_SYNC(0).
66 */
guest_gp_handler(struct ex_regs * regs)67 static void guest_gp_handler(struct ex_regs *regs)
68 {
69 GUEST_SYNC(-EFAULT);
70 }
71
72 /*
73 * Check that we can write a new value to the given MSR and read it back.
74 * The caller should provide a non-empty set of bits that are safe to flip.
75 *
76 * Return on success. GUEST_SYNC(0) on error.
77 */
check_msr(u32 msr,u64 bits_to_flip)78 static void check_msr(u32 msr, u64 bits_to_flip)
79 {
80 u64 v = rdmsr(msr) ^ bits_to_flip;
81
82 wrmsr(msr, v);
83 if (rdmsr(msr) != v)
84 GUEST_SYNC(-EIO);
85
86 v ^= bits_to_flip;
87 wrmsr(msr, v);
88 if (rdmsr(msr) != v)
89 GUEST_SYNC(-EIO);
90 }
91
run_and_measure_loop(u32 msr_base)92 static void run_and_measure_loop(u32 msr_base)
93 {
94 const u64 branches_retired = rdmsr(msr_base + 0);
95 const u64 insn_retired = rdmsr(msr_base + 1);
96
97 __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
98
99 pmc_results.branches_retired = rdmsr(msr_base + 0) - branches_retired;
100 pmc_results.instructions_retired = rdmsr(msr_base + 1) - insn_retired;
101 }
102
intel_guest_code(void)103 static void intel_guest_code(void)
104 {
105 check_msr(MSR_CORE_PERF_GLOBAL_CTRL, 1);
106 check_msr(MSR_P6_EVNTSEL0, 0xffff);
107 check_msr(MSR_IA32_PMC0, 0xffff);
108 GUEST_SYNC(0);
109
110 for (;;) {
111 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
112 wrmsr(MSR_P6_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
113 ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_BRANCHES_RETIRED);
114 wrmsr(MSR_P6_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
115 ARCH_PERFMON_EVENTSEL_OS | INTEL_ARCH_INSTRUCTIONS_RETIRED);
116 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x3);
117
118 run_and_measure_loop(MSR_IA32_PMC0);
119 GUEST_SYNC(0);
120 }
121 }
122
123 /*
124 * To avoid needing a check for CPUID.80000001:ECX.PerfCtrExtCore[bit 23],
125 * this code uses the always-available, legacy K7 PMU MSRs, which alias to
126 * the first four of the six extended core PMU MSRs.
127 */
amd_guest_code(void)128 static void amd_guest_code(void)
129 {
130 check_msr(MSR_K7_EVNTSEL0, 0xffff);
131 check_msr(MSR_K7_PERFCTR0, 0xffff);
132 GUEST_SYNC(0);
133
134 for (;;) {
135 wrmsr(MSR_K7_EVNTSEL0, 0);
136 wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
137 ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_BRANCHES_RETIRED);
138 wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
139 ARCH_PERFMON_EVENTSEL_OS | AMD_ZEN_INSTRUCTIONS_RETIRED);
140
141 run_and_measure_loop(MSR_K7_PERFCTR0);
142 GUEST_SYNC(0);
143 }
144 }
145
146 /*
147 * Run the VM to the next GUEST_SYNC(value), and return the value passed
148 * to the sync. Any other exit from the guest is fatal.
149 */
run_vcpu_to_sync(struct kvm_vcpu * vcpu)150 static u64 run_vcpu_to_sync(struct kvm_vcpu *vcpu)
151 {
152 struct ucall uc;
153
154 vcpu_run(vcpu);
155 TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
156 get_ucall(vcpu, &uc);
157 TEST_ASSERT(uc.cmd == UCALL_SYNC,
158 "Received ucall other than UCALL_SYNC: %lu", uc.cmd);
159 return uc.args[1];
160 }
161
run_vcpu_and_sync_pmc_results(struct kvm_vcpu * vcpu)162 static void run_vcpu_and_sync_pmc_results(struct kvm_vcpu *vcpu)
163 {
164 u64 r;
165
166 memset(&pmc_results, 0, sizeof(pmc_results));
167 sync_global_to_guest(vcpu->vm, pmc_results);
168
169 r = run_vcpu_to_sync(vcpu);
170 TEST_ASSERT(!r, "Unexpected sync value: 0x%lx", r);
171
172 sync_global_from_guest(vcpu->vm, pmc_results);
173 }
174
175 /*
176 * In a nested environment or if the vPMU is disabled, the guest PMU
177 * might not work as architected (accessing the PMU MSRs may raise
178 * #GP, or writes could simply be discarded). In those situations,
179 * there is no point in running these tests. The guest code will perform
180 * a sanity check and then GUEST_SYNC(success). In the case of failure,
181 * the behavior of the guest on resumption is undefined.
182 */
sanity_check_pmu(struct kvm_vcpu * vcpu)183 static bool sanity_check_pmu(struct kvm_vcpu *vcpu)
184 {
185 u64 r;
186
187 vm_install_exception_handler(vcpu->vm, GP_VECTOR, guest_gp_handler);
188 r = run_vcpu_to_sync(vcpu);
189 vm_install_exception_handler(vcpu->vm, GP_VECTOR, NULL);
190
191 return !r;
192 }
193
194 /*
195 * Remove the first occurrence of 'event' (if any) from the filter's
196 * event list.
197 */
remove_event(struct __kvm_pmu_event_filter * f,u64 event)198 static void remove_event(struct __kvm_pmu_event_filter *f, u64 event)
199 {
200 bool found = false;
201 int i;
202
203 for (i = 0; i < f->nevents; i++) {
204 if (found)
205 f->events[i - 1] = f->events[i];
206 else
207 found = f->events[i] == event;
208 }
209 if (found)
210 f->nevents--;
211 }
212
213 #define ASSERT_PMC_COUNTING_INSTRUCTIONS() \
214 do { \
215 u64 br = pmc_results.branches_retired; \
216 u64 ir = pmc_results.instructions_retired; \
217 bool br_matched = this_pmu_has_errata(BRANCHES_RETIRED_OVERCOUNT) ? \
218 br >= NUM_BRANCHES : br == NUM_BRANCHES; \
219 \
220 if (br && !br_matched) \
221 pr_info("%s: Branch instructions retired = %lu (expected %u)\n", \
222 __func__, br, NUM_BRANCHES); \
223 TEST_ASSERT(br, "%s: Branch instructions retired = %lu (expected > 0)", \
224 __func__, br); \
225 TEST_ASSERT(ir, "%s: Instructions retired = %lu (expected > 0)", \
226 __func__, ir); \
227 } while (0)
228
229 #define ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS() \
230 do { \
231 u64 br = pmc_results.branches_retired; \
232 u64 ir = pmc_results.instructions_retired; \
233 \
234 TEST_ASSERT(!br, "%s: Branch instructions retired = %lu (expected 0)", \
235 __func__, br); \
236 TEST_ASSERT(!ir, "%s: Instructions retired = %lu (expected 0)", \
237 __func__, ir); \
238 } while (0)
239
test_without_filter(struct kvm_vcpu * vcpu)240 static void test_without_filter(struct kvm_vcpu *vcpu)
241 {
242 run_vcpu_and_sync_pmc_results(vcpu);
243
244 ASSERT_PMC_COUNTING_INSTRUCTIONS();
245 }
246
test_with_filter(struct kvm_vcpu * vcpu,struct __kvm_pmu_event_filter * __f)247 static void test_with_filter(struct kvm_vcpu *vcpu,
248 struct __kvm_pmu_event_filter *__f)
249 {
250 struct kvm_pmu_event_filter *f = (void *)__f;
251
252 vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f);
253 run_vcpu_and_sync_pmc_results(vcpu);
254 }
255
test_amd_deny_list(struct kvm_vcpu * vcpu)256 static void test_amd_deny_list(struct kvm_vcpu *vcpu)
257 {
258 struct __kvm_pmu_event_filter f = {
259 .action = KVM_PMU_EVENT_DENY,
260 .nevents = 1,
261 .events = {
262 RAW_EVENT(0x1C2, 0),
263 },
264 };
265
266 test_with_filter(vcpu, &f);
267
268 ASSERT_PMC_COUNTING_INSTRUCTIONS();
269 }
270
test_member_deny_list(struct kvm_vcpu * vcpu)271 static void test_member_deny_list(struct kvm_vcpu *vcpu)
272 {
273 struct __kvm_pmu_event_filter f = base_event_filter;
274
275 f.action = KVM_PMU_EVENT_DENY;
276 test_with_filter(vcpu, &f);
277
278 ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS();
279 }
280
test_member_allow_list(struct kvm_vcpu * vcpu)281 static void test_member_allow_list(struct kvm_vcpu *vcpu)
282 {
283 struct __kvm_pmu_event_filter f = base_event_filter;
284
285 f.action = KVM_PMU_EVENT_ALLOW;
286 test_with_filter(vcpu, &f);
287
288 ASSERT_PMC_COUNTING_INSTRUCTIONS();
289 }
290
test_not_member_deny_list(struct kvm_vcpu * vcpu)291 static void test_not_member_deny_list(struct kvm_vcpu *vcpu)
292 {
293 struct __kvm_pmu_event_filter f = base_event_filter;
294
295 f.action = KVM_PMU_EVENT_DENY;
296
297 remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED);
298 remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED);
299 remove_event(&f, AMD_ZEN_BRANCHES_RETIRED);
300 test_with_filter(vcpu, &f);
301
302 ASSERT_PMC_COUNTING_INSTRUCTIONS();
303 }
304
test_not_member_allow_list(struct kvm_vcpu * vcpu)305 static void test_not_member_allow_list(struct kvm_vcpu *vcpu)
306 {
307 struct __kvm_pmu_event_filter f = base_event_filter;
308
309 f.action = KVM_PMU_EVENT_ALLOW;
310
311 remove_event(&f, INTEL_ARCH_INSTRUCTIONS_RETIRED);
312 remove_event(&f, INTEL_ARCH_BRANCHES_RETIRED);
313 remove_event(&f, AMD_ZEN_BRANCHES_RETIRED);
314 test_with_filter(vcpu, &f);
315
316 ASSERT_PMC_NOT_COUNTING_INSTRUCTIONS();
317 }
318
319 /*
320 * Verify that setting KVM_PMU_CAP_DISABLE prevents the use of the PMU.
321 *
322 * Note that KVM_CAP_PMU_CAPABILITY must be invoked prior to creating VCPUs.
323 */
test_pmu_config_disable(void (* guest_code)(void))324 static void test_pmu_config_disable(void (*guest_code)(void))
325 {
326 struct kvm_vcpu *vcpu;
327 int r;
328 struct kvm_vm *vm;
329
330 r = kvm_check_cap(KVM_CAP_PMU_CAPABILITY);
331 if (!(r & KVM_PMU_CAP_DISABLE))
332 return;
333
334 vm = vm_create(1);
335
336 vm_enable_cap(vm, KVM_CAP_PMU_CAPABILITY, KVM_PMU_CAP_DISABLE);
337
338 vcpu = vm_vcpu_add(vm, 0, guest_code);
339 TEST_ASSERT(!sanity_check_pmu(vcpu),
340 "Guest should not be able to use disabled PMU.");
341
342 kvm_vm_free(vm);
343 }
344
345 /*
346 * On Intel, check for a non-zero PMU version, at least one general-purpose
347 * counter per logical processor, and support for counting the number of branch
348 * instructions retired.
349 */
use_intel_pmu(void)350 static bool use_intel_pmu(void)
351 {
352 return host_cpu_is_intel &&
353 kvm_cpu_property(X86_PROPERTY_PMU_VERSION) &&
354 kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) &&
355 kvm_pmu_has(X86_PMU_FEATURE_BRANCH_INSNS_RETIRED);
356 }
357
358 /*
359 * On AMD, all Family 17h+ CPUs (Zen and its successors) use event encoding
360 * 0xc2,0 for Branch Instructions Retired.
361 */
use_amd_pmu(void)362 static bool use_amd_pmu(void)
363 {
364 return (host_cpu_is_amd && kvm_cpu_family() >= 0x17) ||
365 host_cpu_is_hygon;
366 }
367
368 /*
369 * "MEM_INST_RETIRED.ALL_LOADS", "MEM_INST_RETIRED.ALL_STORES", and
370 * "MEM_INST_RETIRED.ANY" from https://perfmon-events.intel.com/
371 * supported on Intel Xeon processors:
372 * - Sapphire Rapids, Ice Lake, Cascade Lake, Skylake.
373 */
374 #define MEM_INST_RETIRED 0xD0
375 #define MEM_INST_RETIRED_LOAD RAW_EVENT(MEM_INST_RETIRED, 0x81)
376 #define MEM_INST_RETIRED_STORE RAW_EVENT(MEM_INST_RETIRED, 0x82)
377 #define MEM_INST_RETIRED_LOAD_STORE RAW_EVENT(MEM_INST_RETIRED, 0x83)
378
supports_event_mem_inst_retired(void)379 static bool supports_event_mem_inst_retired(void)
380 {
381 u32 eax, ebx, ecx, edx;
382
383 cpuid(1, &eax, &ebx, &ecx, &edx);
384 if (x86_family(eax) == 0x6) {
385 switch (x86_model(eax)) {
386 /* Sapphire Rapids */
387 case 0x8F:
388 /* Ice Lake */
389 case 0x6A:
390 /* Skylake */
391 /* Cascade Lake */
392 case 0x55:
393 return true;
394 }
395 }
396
397 return false;
398 }
399
400 /*
401 * "LS Dispatch", from Processor Programming Reference
402 * (PPR) for AMD Family 17h Model 01h, Revision B1 Processors,
403 * Preliminary Processor Programming Reference (PPR) for AMD Family
404 * 17h Model 31h, Revision B0 Processors, and Preliminary Processor
405 * Programming Reference (PPR) for AMD Family 19h Model 01h, Revision
406 * B1 Processors Volume 1 of 2.
407 */
408 #define LS_DISPATCH 0x29
409 #define LS_DISPATCH_LOAD RAW_EVENT(LS_DISPATCH, BIT(0))
410 #define LS_DISPATCH_STORE RAW_EVENT(LS_DISPATCH, BIT(1))
411 #define LS_DISPATCH_LOAD_STORE RAW_EVENT(LS_DISPATCH, BIT(2))
412
413 #define INCLUDE_MASKED_ENTRY(event_select, mask, match) \
414 KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, false)
415 #define EXCLUDE_MASKED_ENTRY(event_select, mask, match) \
416 KVM_PMU_ENCODE_MASKED_ENTRY(event_select, mask, match, true)
417
masked_events_guest_test(u32 msr_base)418 static void masked_events_guest_test(u32 msr_base)
419 {
420 /*
421 * The actual value of the counters don't determine the outcome of
422 * the test. Only that they are zero or non-zero.
423 */
424 const u64 loads = rdmsr(msr_base + 0);
425 const u64 stores = rdmsr(msr_base + 1);
426 const u64 loads_stores = rdmsr(msr_base + 2);
427 int val;
428
429
430 __asm__ __volatile__("movl $0, %[v];"
431 "movl %[v], %%eax;"
432 "incl %[v];"
433 : [v]"+m"(val) :: "eax");
434
435 pmc_results.loads = rdmsr(msr_base + 0) - loads;
436 pmc_results.stores = rdmsr(msr_base + 1) - stores;
437 pmc_results.loads_stores = rdmsr(msr_base + 2) - loads_stores;
438 }
439
intel_masked_events_guest_code(void)440 static void intel_masked_events_guest_code(void)
441 {
442 for (;;) {
443 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
444
445 wrmsr(MSR_P6_EVNTSEL0 + 0, ARCH_PERFMON_EVENTSEL_ENABLE |
446 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD);
447 wrmsr(MSR_P6_EVNTSEL0 + 1, ARCH_PERFMON_EVENTSEL_ENABLE |
448 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_STORE);
449 wrmsr(MSR_P6_EVNTSEL0 + 2, ARCH_PERFMON_EVENTSEL_ENABLE |
450 ARCH_PERFMON_EVENTSEL_OS | MEM_INST_RETIRED_LOAD_STORE);
451
452 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0x7);
453
454 masked_events_guest_test(MSR_IA32_PMC0);
455 GUEST_SYNC(0);
456 }
457 }
458
amd_masked_events_guest_code(void)459 static void amd_masked_events_guest_code(void)
460 {
461 for (;;) {
462 wrmsr(MSR_K7_EVNTSEL0, 0);
463 wrmsr(MSR_K7_EVNTSEL1, 0);
464 wrmsr(MSR_K7_EVNTSEL2, 0);
465
466 wrmsr(MSR_K7_EVNTSEL0, ARCH_PERFMON_EVENTSEL_ENABLE |
467 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD);
468 wrmsr(MSR_K7_EVNTSEL1, ARCH_PERFMON_EVENTSEL_ENABLE |
469 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_STORE);
470 wrmsr(MSR_K7_EVNTSEL2, ARCH_PERFMON_EVENTSEL_ENABLE |
471 ARCH_PERFMON_EVENTSEL_OS | LS_DISPATCH_LOAD_STORE);
472
473 masked_events_guest_test(MSR_K7_PERFCTR0);
474 GUEST_SYNC(0);
475 }
476 }
477
run_masked_events_test(struct kvm_vcpu * vcpu,const u64 masked_events[],const int nmasked_events)478 static void run_masked_events_test(struct kvm_vcpu *vcpu,
479 const u64 masked_events[],
480 const int nmasked_events)
481 {
482 struct __kvm_pmu_event_filter f = {
483 .nevents = nmasked_events,
484 .action = KVM_PMU_EVENT_ALLOW,
485 .flags = KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
486 };
487
488 memcpy(f.events, masked_events, sizeof(u64) * nmasked_events);
489 test_with_filter(vcpu, &f);
490 }
491
492 #define ALLOW_LOADS BIT(0)
493 #define ALLOW_STORES BIT(1)
494 #define ALLOW_LOADS_STORES BIT(2)
495
496 struct masked_events_test {
497 u64 intel_events[MAX_TEST_EVENTS];
498 u64 intel_event_end;
499 u64 amd_events[MAX_TEST_EVENTS];
500 u64 amd_event_end;
501 const char *msg;
502 u32 flags;
503 };
504
505 /*
506 * These are the test cases for the masked events tests.
507 *
508 * For each test, the guest enables 3 PMU counters (loads, stores,
509 * loads + stores). The filter is then set in KVM with the masked events
510 * provided. The test then verifies that the counters agree with which
511 * ones should be counting and which ones should be filtered.
512 */
513 const struct masked_events_test test_cases[] = {
514 {
515 .intel_events = {
516 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x81),
517 },
518 .amd_events = {
519 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)),
520 },
521 .msg = "Only allow loads.",
522 .flags = ALLOW_LOADS,
523 }, {
524 .intel_events = {
525 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82),
526 },
527 .amd_events = {
528 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)),
529 },
530 .msg = "Only allow stores.",
531 .flags = ALLOW_STORES,
532 }, {
533 .intel_events = {
534 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83),
535 },
536 .amd_events = {
537 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(2)),
538 },
539 .msg = "Only allow loads + stores.",
540 .flags = ALLOW_LOADS_STORES,
541 }, {
542 .intel_events = {
543 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
544 EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x83),
545 },
546 .amd_events = {
547 INCLUDE_MASKED_ENTRY(LS_DISPATCH, ~(BIT(0) | BIT(1)), 0),
548 },
549 .msg = "Only allow loads and stores.",
550 .flags = ALLOW_LOADS | ALLOW_STORES,
551 }, {
552 .intel_events = {
553 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
554 EXCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFF, 0x82),
555 },
556 .amd_events = {
557 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
558 EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(1)),
559 },
560 .msg = "Only allow loads and loads + stores.",
561 .flags = ALLOW_LOADS | ALLOW_LOADS_STORES
562 }, {
563 .intel_events = {
564 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0xFE, 0x82),
565 },
566 .amd_events = {
567 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
568 EXCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xFF, BIT(0)),
569 },
570 .msg = "Only allow stores and loads + stores.",
571 .flags = ALLOW_STORES | ALLOW_LOADS_STORES
572 }, {
573 .intel_events = {
574 INCLUDE_MASKED_ENTRY(MEM_INST_RETIRED, 0x7C, 0),
575 },
576 .amd_events = {
577 INCLUDE_MASKED_ENTRY(LS_DISPATCH, 0xF8, 0),
578 },
579 .msg = "Only allow loads, stores, and loads + stores.",
580 .flags = ALLOW_LOADS | ALLOW_STORES | ALLOW_LOADS_STORES
581 },
582 };
583
append_test_events(const struct masked_events_test * test,u64 * events,int nevents)584 static int append_test_events(const struct masked_events_test *test,
585 u64 *events, int nevents)
586 {
587 const u64 *evts;
588 int i;
589
590 evts = use_intel_pmu() ? test->intel_events : test->amd_events;
591 for (i = 0; i < MAX_TEST_EVENTS; i++) {
592 if (evts[i] == 0)
593 break;
594
595 events[nevents + i] = evts[i];
596 }
597
598 return nevents + i;
599 }
600
bool_eq(bool a,bool b)601 static bool bool_eq(bool a, bool b)
602 {
603 return a == b;
604 }
605
run_masked_events_tests(struct kvm_vcpu * vcpu,u64 * events,int nevents)606 static void run_masked_events_tests(struct kvm_vcpu *vcpu, u64 *events,
607 int nevents)
608 {
609 int ntests = ARRAY_SIZE(test_cases);
610 int i, n;
611
612 for (i = 0; i < ntests; i++) {
613 const struct masked_events_test *test = &test_cases[i];
614
615 /* Do any test case events overflow MAX_TEST_EVENTS? */
616 assert(test->intel_event_end == 0);
617 assert(test->amd_event_end == 0);
618
619 n = append_test_events(test, events, nevents);
620
621 run_masked_events_test(vcpu, events, n);
622
623 TEST_ASSERT(bool_eq(pmc_results.loads, test->flags & ALLOW_LOADS) &&
624 bool_eq(pmc_results.stores, test->flags & ALLOW_STORES) &&
625 bool_eq(pmc_results.loads_stores,
626 test->flags & ALLOW_LOADS_STORES),
627 "%s loads: %lu, stores: %lu, loads + stores: %lu",
628 test->msg, pmc_results.loads, pmc_results.stores,
629 pmc_results.loads_stores);
630 }
631 }
632
add_dummy_events(u64 * events,int nevents)633 static void add_dummy_events(u64 *events, int nevents)
634 {
635 int i;
636
637 for (i = 0; i < nevents; i++) {
638 int event_select = i % 0xFF;
639 bool exclude = ((i % 4) == 0);
640
641 if (event_select == MEM_INST_RETIRED ||
642 event_select == LS_DISPATCH)
643 event_select++;
644
645 events[i] = KVM_PMU_ENCODE_MASKED_ENTRY(event_select, 0,
646 0, exclude);
647 }
648 }
649
test_masked_events(struct kvm_vcpu * vcpu)650 static void test_masked_events(struct kvm_vcpu *vcpu)
651 {
652 int nevents = KVM_PMU_EVENT_FILTER_MAX_EVENTS - MAX_TEST_EVENTS;
653 u64 events[KVM_PMU_EVENT_FILTER_MAX_EVENTS];
654
655 /* Run the test cases against a sparse PMU event filter. */
656 run_masked_events_tests(vcpu, events, 0);
657
658 /* Run the test cases against a dense PMU event filter. */
659 add_dummy_events(events, KVM_PMU_EVENT_FILTER_MAX_EVENTS);
660 run_masked_events_tests(vcpu, events, nevents);
661 }
662
set_pmu_event_filter(struct kvm_vcpu * vcpu,struct __kvm_pmu_event_filter * __f)663 static int set_pmu_event_filter(struct kvm_vcpu *vcpu,
664 struct __kvm_pmu_event_filter *__f)
665 {
666 struct kvm_pmu_event_filter *f = (void *)__f;
667
668 return __vm_ioctl(vcpu->vm, KVM_SET_PMU_EVENT_FILTER, f);
669 }
670
set_pmu_single_event_filter(struct kvm_vcpu * vcpu,u64 event,u32 flags,u32 action)671 static int set_pmu_single_event_filter(struct kvm_vcpu *vcpu, u64 event,
672 u32 flags, u32 action)
673 {
674 struct __kvm_pmu_event_filter f = {
675 .nevents = 1,
676 .flags = flags,
677 .action = action,
678 .events = {
679 event,
680 },
681 };
682
683 return set_pmu_event_filter(vcpu, &f);
684 }
685
test_filter_ioctl(struct kvm_vcpu * vcpu)686 static void test_filter_ioctl(struct kvm_vcpu *vcpu)
687 {
688 u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
689 struct __kvm_pmu_event_filter f;
690 u64 e = ~0ul;
691 int r;
692
693 /*
694 * Unfortunately having invalid bits set in event data is expected to
695 * pass when flags == 0 (bits other than eventsel+umask).
696 */
697 r = set_pmu_single_event_filter(vcpu, e, 0, KVM_PMU_EVENT_ALLOW);
698 TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing");
699
700 r = set_pmu_single_event_filter(vcpu, e,
701 KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
702 KVM_PMU_EVENT_ALLOW);
703 TEST_ASSERT(r != 0, "Invalid PMU Event Filter is expected to fail");
704
705 e = KVM_PMU_ENCODE_MASKED_ENTRY(0xff, 0xff, 0xff, 0xf);
706 r = set_pmu_single_event_filter(vcpu, e,
707 KVM_PMU_EVENT_FLAG_MASKED_EVENTS,
708 KVM_PMU_EVENT_ALLOW);
709 TEST_ASSERT(r == 0, "Valid PMU Event Filter is failing");
710
711 f = base_event_filter;
712 f.action = PMU_EVENT_FILTER_INVALID_ACTION;
713 r = set_pmu_event_filter(vcpu, &f);
714 TEST_ASSERT(r, "Set invalid action is expected to fail");
715
716 f = base_event_filter;
717 f.flags = PMU_EVENT_FILTER_INVALID_FLAGS;
718 r = set_pmu_event_filter(vcpu, &f);
719 TEST_ASSERT(r, "Set invalid flags is expected to fail");
720
721 f = base_event_filter;
722 f.nevents = PMU_EVENT_FILTER_INVALID_NEVENTS;
723 r = set_pmu_event_filter(vcpu, &f);
724 TEST_ASSERT(r, "Exceeding the max number of filter events should fail");
725
726 f = base_event_filter;
727 f.fixed_counter_bitmap = ~GENMASK_ULL(nr_fixed_counters, 0);
728 r = set_pmu_event_filter(vcpu, &f);
729 TEST_ASSERT(!r, "Masking non-existent fixed counters should be allowed");
730 }
731
intel_run_fixed_counter_guest_code(u8 idx)732 static void intel_run_fixed_counter_guest_code(u8 idx)
733 {
734 for (;;) {
735 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
736 wrmsr(MSR_CORE_PERF_FIXED_CTR0 + idx, 0);
737
738 /* Only OS_EN bit is enabled for fixed counter[idx]. */
739 wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, FIXED_PMC_CTRL(idx, FIXED_PMC_KERNEL));
740 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, FIXED_PMC_GLOBAL_CTRL_ENABLE(idx));
741 __asm__ __volatile__("loop ." : "+c"((int){NUM_BRANCHES}));
742 wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
743
744 GUEST_SYNC(rdmsr(MSR_CORE_PERF_FIXED_CTR0 + idx));
745 }
746 }
747
test_with_fixed_counter_filter(struct kvm_vcpu * vcpu,u32 action,u32 bitmap)748 static u64 test_with_fixed_counter_filter(struct kvm_vcpu *vcpu,
749 u32 action, u32 bitmap)
750 {
751 struct __kvm_pmu_event_filter f = {
752 .action = action,
753 .fixed_counter_bitmap = bitmap,
754 };
755 set_pmu_event_filter(vcpu, &f);
756
757 return run_vcpu_to_sync(vcpu);
758 }
759
test_set_gp_and_fixed_event_filter(struct kvm_vcpu * vcpu,u32 action,u32 bitmap)760 static u64 test_set_gp_and_fixed_event_filter(struct kvm_vcpu *vcpu,
761 u32 action,
762 u32 bitmap)
763 {
764 struct __kvm_pmu_event_filter f = base_event_filter;
765
766 f.action = action;
767 f.fixed_counter_bitmap = bitmap;
768 set_pmu_event_filter(vcpu, &f);
769
770 return run_vcpu_to_sync(vcpu);
771 }
772
__test_fixed_counter_bitmap(struct kvm_vcpu * vcpu,u8 idx,u8 nr_fixed_counters)773 static void __test_fixed_counter_bitmap(struct kvm_vcpu *vcpu, u8 idx,
774 u8 nr_fixed_counters)
775 {
776 unsigned int i;
777 u32 bitmap;
778 u64 count;
779
780 TEST_ASSERT(nr_fixed_counters < sizeof(bitmap) * 8,
781 "Invalid nr_fixed_counters");
782
783 /*
784 * Check the fixed performance counter can count normally when KVM
785 * userspace doesn't set any pmu filter.
786 */
787 count = run_vcpu_to_sync(vcpu);
788 TEST_ASSERT(count, "Unexpected count value: %ld", count);
789
790 for (i = 0; i < BIT(nr_fixed_counters); i++) {
791 bitmap = BIT(i);
792 count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_ALLOW,
793 bitmap);
794 TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx)));
795
796 count = test_with_fixed_counter_filter(vcpu, KVM_PMU_EVENT_DENY,
797 bitmap);
798 TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx)));
799
800 /*
801 * Check that fixed_counter_bitmap has higher priority than
802 * events[] when both are set.
803 */
804 count = test_set_gp_and_fixed_event_filter(vcpu,
805 KVM_PMU_EVENT_ALLOW,
806 bitmap);
807 TEST_ASSERT_EQ(!!count, !!(bitmap & BIT(idx)));
808
809 count = test_set_gp_and_fixed_event_filter(vcpu,
810 KVM_PMU_EVENT_DENY,
811 bitmap);
812 TEST_ASSERT_EQ(!!count, !(bitmap & BIT(idx)));
813 }
814 }
815
test_fixed_counter_bitmap(void)816 static void test_fixed_counter_bitmap(void)
817 {
818 u8 nr_fixed_counters = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
819 struct kvm_vm *vm;
820 struct kvm_vcpu *vcpu;
821 u8 idx;
822
823 /*
824 * Check that pmu_event_filter works as expected when it's applied to
825 * fixed performance counters.
826 */
827 for (idx = 0; idx < nr_fixed_counters; idx++) {
828 vm = vm_create_with_one_vcpu(&vcpu,
829 intel_run_fixed_counter_guest_code);
830 vcpu_args_set(vcpu, 1, idx);
831 __test_fixed_counter_bitmap(vcpu, idx, nr_fixed_counters);
832 kvm_vm_free(vm);
833 }
834 }
835
main(int argc,char * argv[])836 int main(int argc, char *argv[])
837 {
838 void (*guest_code)(void);
839 struct kvm_vcpu *vcpu, *vcpu2 = NULL;
840 struct kvm_vm *vm;
841
842 TEST_REQUIRE(kvm_is_pmu_enabled());
843 TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_FILTER));
844 TEST_REQUIRE(kvm_has_cap(KVM_CAP_PMU_EVENT_MASKED_EVENTS));
845
846 TEST_REQUIRE(use_intel_pmu() || use_amd_pmu());
847 guest_code = use_intel_pmu() ? intel_guest_code : amd_guest_code;
848
849 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
850
851 TEST_REQUIRE(sanity_check_pmu(vcpu));
852
853 if (use_amd_pmu())
854 test_amd_deny_list(vcpu);
855
856 test_without_filter(vcpu);
857 test_member_deny_list(vcpu);
858 test_member_allow_list(vcpu);
859 test_not_member_deny_list(vcpu);
860 test_not_member_allow_list(vcpu);
861
862 if (use_intel_pmu() &&
863 supports_event_mem_inst_retired() &&
864 kvm_cpu_property(X86_PROPERTY_PMU_NR_GP_COUNTERS) >= 3)
865 vcpu2 = vm_vcpu_add(vm, 2, intel_masked_events_guest_code);
866 else if (use_amd_pmu())
867 vcpu2 = vm_vcpu_add(vm, 2, amd_masked_events_guest_code);
868
869 if (vcpu2)
870 test_masked_events(vcpu2);
871 test_filter_ioctl(vcpu);
872
873 kvm_vm_free(vm);
874
875 test_pmu_config_disable(guest_code);
876 test_fixed_counter_bitmap();
877
878 return 0;
879 }
880