1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2023 Rivos Inc
4 *
5 * Authors:
6 * Atish Patra <atishp@rivosinc.com>
7 */
8
9 #define pr_fmt(fmt) "riscv-kvm-pmu: " fmt
10 #include <linux/bitops.h>
11 #include <linux/errno.h>
12 #include <linux/err.h>
13 #include <linux/kvm_host.h>
14 #include <linux/nospec.h>
15 #include <linux/overflow.h>
16 #include <linux/perf/riscv_pmu.h>
17 #include <linux/slab.h>
18 #include <asm/csr.h>
19 #include <asm/kvm_isa.h>
20 #include <asm/kvm_vcpu_sbi.h>
21 #include <asm/kvm_vcpu_pmu.h>
22 #include <asm/sbi.h>
23
24 #define kvm_pmu_num_counters(pmu) ((pmu)->num_hw_ctrs + (pmu)->num_fw_ctrs)
25 #define get_event_type(x) (((x) & SBI_PMU_EVENT_IDX_TYPE_MASK) >> 16)
26 #define get_event_code(x) ((x) & SBI_PMU_EVENT_IDX_CODE_MASK)
27
28 static enum perf_hw_id hw_event_perf_map[SBI_PMU_HW_GENERAL_MAX] = {
29 [SBI_PMU_HW_CPU_CYCLES] = PERF_COUNT_HW_CPU_CYCLES,
30 [SBI_PMU_HW_INSTRUCTIONS] = PERF_COUNT_HW_INSTRUCTIONS,
31 [SBI_PMU_HW_CACHE_REFERENCES] = PERF_COUNT_HW_CACHE_REFERENCES,
32 [SBI_PMU_HW_CACHE_MISSES] = PERF_COUNT_HW_CACHE_MISSES,
33 [SBI_PMU_HW_BRANCH_INSTRUCTIONS] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS,
34 [SBI_PMU_HW_BRANCH_MISSES] = PERF_COUNT_HW_BRANCH_MISSES,
35 [SBI_PMU_HW_BUS_CYCLES] = PERF_COUNT_HW_BUS_CYCLES,
36 [SBI_PMU_HW_STALLED_CYCLES_FRONTEND] = PERF_COUNT_HW_STALLED_CYCLES_FRONTEND,
37 [SBI_PMU_HW_STALLED_CYCLES_BACKEND] = PERF_COUNT_HW_STALLED_CYCLES_BACKEND,
38 [SBI_PMU_HW_REF_CPU_CYCLES] = PERF_COUNT_HW_REF_CPU_CYCLES,
39 };
40
kvm_pmu_get_sample_period(struct kvm_pmc * pmc)41 static u64 kvm_pmu_get_sample_period(struct kvm_pmc *pmc)
42 {
43 u64 counter_val_mask = GENMASK(pmc->cinfo.width, 0);
44 u64 sample_period;
45
46 if (!pmc->counter_val)
47 sample_period = counter_val_mask;
48 else
49 sample_period = (-pmc->counter_val) & counter_val_mask;
50
51 return sample_period;
52 }
53
kvm_pmu_get_perf_event_type(unsigned long eidx)54 static u32 kvm_pmu_get_perf_event_type(unsigned long eidx)
55 {
56 enum sbi_pmu_event_type etype = get_event_type(eidx);
57 u32 type = PERF_TYPE_MAX;
58
59 switch (etype) {
60 case SBI_PMU_EVENT_TYPE_HW:
61 type = PERF_TYPE_HARDWARE;
62 break;
63 case SBI_PMU_EVENT_TYPE_CACHE:
64 type = PERF_TYPE_HW_CACHE;
65 break;
66 case SBI_PMU_EVENT_TYPE_RAW:
67 case SBI_PMU_EVENT_TYPE_RAW_V2:
68 case SBI_PMU_EVENT_TYPE_FW:
69 type = PERF_TYPE_RAW;
70 break;
71 default:
72 break;
73 }
74
75 return type;
76 }
77
kvm_pmu_is_fw_event(unsigned long eidx)78 static bool kvm_pmu_is_fw_event(unsigned long eidx)
79 {
80 return get_event_type(eidx) == SBI_PMU_EVENT_TYPE_FW;
81 }
82
kvm_pmu_release_perf_event(struct kvm_pmc * pmc)83 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)
84 {
85 if (pmc->perf_event) {
86 perf_event_disable(pmc->perf_event);
87 perf_event_release_kernel(pmc->perf_event);
88 pmc->perf_event = NULL;
89 }
90 }
91
kvm_pmu_get_perf_event_hw_config(u32 sbi_event_code)92 static u64 kvm_pmu_get_perf_event_hw_config(u32 sbi_event_code)
93 {
94 return hw_event_perf_map[array_index_nospec(sbi_event_code,
95 SBI_PMU_HW_GENERAL_MAX)];
96 }
97
kvm_pmu_get_perf_event_cache_config(u32 sbi_event_code)98 static u64 kvm_pmu_get_perf_event_cache_config(u32 sbi_event_code)
99 {
100 u64 config = U64_MAX;
101 unsigned int cache_type, cache_op, cache_result;
102
103 /* All the cache event masks lie within 0xFF. No separate masking is necessary */
104 cache_type = (sbi_event_code & SBI_PMU_EVENT_CACHE_ID_CODE_MASK) >>
105 SBI_PMU_EVENT_CACHE_ID_SHIFT;
106 cache_op = (sbi_event_code & SBI_PMU_EVENT_CACHE_OP_ID_CODE_MASK) >>
107 SBI_PMU_EVENT_CACHE_OP_SHIFT;
108 cache_result = sbi_event_code & SBI_PMU_EVENT_CACHE_RESULT_ID_CODE_MASK;
109
110 if (cache_type >= PERF_COUNT_HW_CACHE_MAX ||
111 cache_op >= PERF_COUNT_HW_CACHE_OP_MAX ||
112 cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
113 return config;
114
115 config = cache_type | (cache_op << 8) | (cache_result << 16);
116
117 return config;
118 }
119
kvm_pmu_get_perf_event_config(unsigned long eidx,uint64_t evt_data)120 static u64 kvm_pmu_get_perf_event_config(unsigned long eidx, uint64_t evt_data)
121 {
122 enum sbi_pmu_event_type etype = get_event_type(eidx);
123 u32 ecode = get_event_code(eidx);
124 u64 config = U64_MAX;
125
126 switch (etype) {
127 case SBI_PMU_EVENT_TYPE_HW:
128 if (ecode < SBI_PMU_HW_GENERAL_MAX)
129 config = kvm_pmu_get_perf_event_hw_config(ecode);
130 break;
131 case SBI_PMU_EVENT_TYPE_CACHE:
132 config = kvm_pmu_get_perf_event_cache_config(ecode);
133 break;
134 case SBI_PMU_EVENT_TYPE_RAW:
135 config = evt_data & RISCV_PMU_RAW_EVENT_MASK;
136 break;
137 case SBI_PMU_EVENT_TYPE_RAW_V2:
138 config = evt_data & RISCV_PMU_RAW_EVENT_V2_MASK;
139 break;
140 case SBI_PMU_EVENT_TYPE_FW:
141 if (ecode < SBI_PMU_FW_MAX)
142 config = (1ULL << 63) | ecode;
143 break;
144 default:
145 break;
146 }
147
148 return config;
149 }
150
kvm_pmu_get_fixed_pmc_index(unsigned long eidx)151 static int kvm_pmu_get_fixed_pmc_index(unsigned long eidx)
152 {
153 u32 etype = kvm_pmu_get_perf_event_type(eidx);
154 u32 ecode = get_event_code(eidx);
155
156 if (etype != SBI_PMU_EVENT_TYPE_HW)
157 return -EINVAL;
158
159 if (ecode == SBI_PMU_HW_CPU_CYCLES)
160 return 0;
161 else if (ecode == SBI_PMU_HW_INSTRUCTIONS)
162 return 2;
163 else
164 return -EINVAL;
165 }
166
kvm_pmu_get_programmable_pmc_index(struct kvm_pmu * kvpmu,unsigned long eidx,unsigned long cbase,unsigned long cmask)167 static int kvm_pmu_get_programmable_pmc_index(struct kvm_pmu *kvpmu, unsigned long eidx,
168 unsigned long cbase, unsigned long cmask)
169 {
170 int ctr_idx = -1;
171 int i, pmc_idx;
172 int min, max;
173
174 if (kvm_pmu_is_fw_event(eidx)) {
175 /* Firmware counters are mapped 1:1 starting from num_hw_ctrs for simplicity */
176 min = kvpmu->num_hw_ctrs;
177 max = min + kvpmu->num_fw_ctrs;
178 } else {
179 /* First 3 counters are reserved for fixed counters */
180 min = 3;
181 max = kvpmu->num_hw_ctrs;
182 }
183
184 for_each_set_bit(i, &cmask, BITS_PER_LONG) {
185 pmc_idx = i + cbase;
186 if ((pmc_idx >= min && pmc_idx < max) &&
187 !test_bit(pmc_idx, kvpmu->pmc_in_use)) {
188 ctr_idx = pmc_idx;
189 break;
190 }
191 }
192
193 return ctr_idx;
194 }
195
pmu_get_pmc_index(struct kvm_pmu * pmu,unsigned long eidx,unsigned long cbase,unsigned long cmask)196 static int pmu_get_pmc_index(struct kvm_pmu *pmu, unsigned long eidx,
197 unsigned long cbase, unsigned long cmask)
198 {
199 int ret;
200
201 /* Fixed counters need to be have fixed mapping as they have different width */
202 ret = kvm_pmu_get_fixed_pmc_index(eidx);
203 if (ret >= 0)
204 return ret;
205
206 return kvm_pmu_get_programmable_pmc_index(pmu, eidx, cbase, cmask);
207 }
208
pmu_fw_ctr_read_hi(struct kvm_vcpu * vcpu,unsigned long cidx,unsigned long * out_val)209 static int pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx,
210 unsigned long *out_val)
211 {
212 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
213 struct kvm_pmc *pmc;
214 int fevent_code;
215
216 if (!IS_ENABLED(CONFIG_32BIT)) {
217 pr_warn("%s: should be invoked for only RV32\n", __func__);
218 return -EINVAL;
219 }
220
221 if (cidx >= kvm_pmu_num_counters(kvpmu) || cidx == 1) {
222 pr_warn("Invalid counter id [%ld]during read\n", cidx);
223 return -EINVAL;
224 }
225
226 cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS);
227 pmc = &kvpmu->pmc[cidx];
228
229 if (pmc->cinfo.type != SBI_PMU_CTR_TYPE_FW)
230 return -EINVAL;
231
232 if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID)
233 return -EINVAL;
234
235 fevent_code = get_event_code(pmc->event_idx);
236 if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX,
237 "Invalid firmware event code: %d\n", fevent_code))
238 return -EINVAL;
239
240 pmc->counter_val = kvpmu->fw_event[fevent_code].value;
241
242 *out_val = pmc->counter_val >> 32;
243
244 return 0;
245 }
246
pmu_ctr_read(struct kvm_vcpu * vcpu,unsigned long cidx,unsigned long * out_val)247 static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx,
248 unsigned long *out_val)
249 {
250 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
251 struct kvm_pmc *pmc;
252 u64 enabled, running;
253 int fevent_code;
254
255 if (cidx >= kvm_pmu_num_counters(kvpmu) || cidx == 1) {
256 pr_warn("Invalid counter id [%ld] during read\n", cidx);
257 return -EINVAL;
258 }
259
260 cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS);
261 pmc = &kvpmu->pmc[cidx];
262
263 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) {
264 if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID)
265 return -EINVAL;
266
267 fevent_code = get_event_code(pmc->event_idx);
268 if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX,
269 "Invalid firmware event code: %d\n", fevent_code))
270 return -EINVAL;
271
272 pmc->counter_val = kvpmu->fw_event[fevent_code].value;
273 *out_val = pmc->counter_val;
274 } else if (pmc->perf_event) {
275 *out_val = pmc->counter_val +
276 perf_event_read_value(pmc->perf_event, &enabled, &running);
277 } else {
278 return -EINVAL;
279 }
280
281 return 0;
282 }
283
kvm_pmu_validate_counter_mask(struct kvm_pmu * kvpmu,unsigned long ctr_base,unsigned long ctr_mask)284 static int kvm_pmu_validate_counter_mask(struct kvm_pmu *kvpmu, unsigned long ctr_base,
285 unsigned long ctr_mask)
286 {
287 unsigned long num_ctrs = kvm_pmu_num_counters(kvpmu);
288
289 /* Make sure we have a valid counter mask requested from the caller */
290 if (!ctr_mask || ctr_base >= num_ctrs || (ctr_base + __fls(ctr_mask) >= num_ctrs))
291 return -EINVAL;
292
293 return 0;
294 }
295
kvm_riscv_pmu_overflow(struct perf_event * perf_event,struct perf_sample_data * data,struct pt_regs * regs)296 static void kvm_riscv_pmu_overflow(struct perf_event *perf_event,
297 struct perf_sample_data *data,
298 struct pt_regs *regs)
299 {
300 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
301 struct kvm_vcpu *vcpu = pmc->vcpu;
302 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
303 struct riscv_pmu *rpmu = to_riscv_pmu(perf_event->pmu);
304 u64 period;
305
306 /*
307 * Stop the event counting by directly accessing the perf_event.
308 * Otherwise, this needs to deferred via a workqueue.
309 * That will introduce skew in the counter value because the actual
310 * physical counter would start after returning from this function.
311 * It will be stopped again once the workqueue is scheduled
312 */
313 rpmu->pmu.stop(perf_event, PERF_EF_UPDATE);
314
315 /*
316 * The hw counter would start automatically when this function returns.
317 * Thus, the host may continue to interrupt and inject it to the guest
318 * even without the guest configuring the next event. Depending on the hardware
319 * the host may have some sluggishness only if privilege mode filtering is not
320 * available. In an ideal world, where qemu is not the only capable hardware,
321 * this can be removed.
322 * FYI: ARM64 does this way while x86 doesn't do anything as such.
323 * TODO: Should we keep it for RISC-V ?
324 */
325 period = -(local64_read(&perf_event->count));
326
327 local64_set(&perf_event->hw.period_left, 0);
328 perf_event->attr.sample_period = period;
329 perf_event->hw.sample_period = period;
330
331 set_bit(pmc->idx, kvpmu->pmc_overflown);
332 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_PMU_OVF);
333
334 rpmu->pmu.start(perf_event, PERF_EF_RELOAD);
335 }
336
kvm_pmu_create_perf_event(struct kvm_pmc * pmc,struct perf_event_attr * attr,unsigned long flags,unsigned long eidx,unsigned long evtdata)337 static long kvm_pmu_create_perf_event(struct kvm_pmc *pmc, struct perf_event_attr *attr,
338 unsigned long flags, unsigned long eidx,
339 unsigned long evtdata)
340 {
341 struct perf_event *event;
342
343 kvm_pmu_release_perf_event(pmc);
344 attr->config = kvm_pmu_get_perf_event_config(eidx, evtdata);
345 if (flags & SBI_PMU_CFG_FLAG_CLEAR_VALUE) {
346 //TODO: Do we really want to clear the value in hardware counter
347 pmc->counter_val = 0;
348 }
349
350 /*
351 * Set the default sample_period for now. The guest specified value
352 * will be updated in the start call.
353 */
354 attr->sample_period = kvm_pmu_get_sample_period(pmc);
355
356 event = perf_event_create_kernel_counter(attr, -1, current, kvm_riscv_pmu_overflow, pmc);
357 if (IS_ERR(event)) {
358 pr_debug("kvm pmu event creation failed for eidx %lx: %ld\n", eidx, PTR_ERR(event));
359 return PTR_ERR(event);
360 }
361
362 pmc->perf_event = event;
363 if (flags & SBI_PMU_CFG_FLAG_AUTO_START)
364 perf_event_enable(pmc->perf_event);
365
366 return 0;
367 }
368
kvm_riscv_vcpu_pmu_incr_fw(struct kvm_vcpu * vcpu,unsigned long fid)369 int kvm_riscv_vcpu_pmu_incr_fw(struct kvm_vcpu *vcpu, unsigned long fid)
370 {
371 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
372 struct kvm_fw_event *fevent;
373
374 if (!kvpmu || fid >= SBI_PMU_FW_MAX)
375 return -EINVAL;
376
377 fevent = &kvpmu->fw_event[fid];
378 if (fevent->started)
379 fevent->value++;
380
381 return 0;
382 }
383
kvm_riscv_vcpu_pmu_read_hpm(struct kvm_vcpu * vcpu,unsigned int csr_num,unsigned long * val,unsigned long new_val,unsigned long wr_mask)384 int kvm_riscv_vcpu_pmu_read_hpm(struct kvm_vcpu *vcpu, unsigned int csr_num,
385 unsigned long *val, unsigned long new_val,
386 unsigned long wr_mask)
387 {
388 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
389 int cidx, ret = KVM_INSN_CONTINUE_NEXT_SEPC;
390
391 if (!kvpmu || !kvpmu->init_done) {
392 /*
393 * In absence of sscofpmf in the platform, the guest OS may use
394 * the legacy PMU driver to read cycle/instret. In that case,
395 * just return 0 to avoid any illegal trap. However, any other
396 * hpmcounter access should result in illegal trap as they must
397 * be access through SBI PMU only.
398 */
399 if (csr_num == CSR_CYCLE || csr_num == CSR_INSTRET) {
400 *val = 0;
401 return ret;
402 } else {
403 return KVM_INSN_ILLEGAL_TRAP;
404 }
405 }
406
407 /* The counter CSR are read only. Thus, any write should result in illegal traps */
408 if (wr_mask)
409 return KVM_INSN_ILLEGAL_TRAP;
410
411 cidx = csr_num - CSR_CYCLE;
412
413 if (pmu_ctr_read(vcpu, cidx, val) < 0)
414 return KVM_INSN_ILLEGAL_TRAP;
415
416 return ret;
417 }
418
kvm_pmu_clear_snapshot_area(struct kvm_vcpu * vcpu)419 static void kvm_pmu_clear_snapshot_area(struct kvm_vcpu *vcpu)
420 {
421 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
422
423 kfree(kvpmu->sdata);
424 kvpmu->sdata = NULL;
425 kvpmu->snapshot_addr = INVALID_GPA;
426 }
427
kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu * vcpu,unsigned long saddr_low,unsigned long saddr_high,unsigned long flags,struct kvm_vcpu_sbi_return * retdata)428 int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long saddr_low,
429 unsigned long saddr_high, unsigned long flags,
430 struct kvm_vcpu_sbi_return *retdata)
431 {
432 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
433 int snapshot_area_size = sizeof(struct riscv_pmu_snapshot_data);
434 int sbiret = 0;
435 gpa_t saddr;
436
437 if (!kvpmu || flags) {
438 sbiret = SBI_ERR_INVALID_PARAM;
439 goto out;
440 }
441
442 if (saddr_low == SBI_SHMEM_DISABLE && saddr_high == SBI_SHMEM_DISABLE) {
443 kvm_pmu_clear_snapshot_area(vcpu);
444 return 0;
445 }
446
447 saddr = saddr_low;
448
449 if (saddr_high != 0) {
450 if (IS_ENABLED(CONFIG_32BIT)) {
451 saddr |= ((gpa_t)saddr_high << 32);
452 } else {
453 sbiret = SBI_ERR_INVALID_ADDRESS;
454 goto out;
455 }
456 }
457
458 /*
459 * Clear any previously installed snapshot area to avoid leaking
460 * the old sdata and to keep sdata/snapshot_addr consistent if
461 * the re-install fails below.
462 */
463 if (kvpmu->snapshot_addr != INVALID_GPA)
464 kvm_pmu_clear_snapshot_area(vcpu);
465
466 kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC | __GFP_ACCOUNT);
467 if (!kvpmu->sdata) {
468 sbiret = SBI_ERR_FAILURE;
469 goto out;
470 }
471
472 /* No need to check writable slot explicitly as kvm_vcpu_write_guest does it internally */
473 if (kvm_vcpu_write_guest(vcpu, saddr, kvpmu->sdata, snapshot_area_size)) {
474 kfree(kvpmu->sdata);
475 kvpmu->sdata = NULL;
476 sbiret = SBI_ERR_INVALID_ADDRESS;
477 goto out;
478 }
479
480 kvpmu->snapshot_addr = saddr;
481
482 out:
483 retdata->err_val = sbiret;
484
485 return 0;
486 }
487
kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu * vcpu,unsigned long saddr_low,unsigned long saddr_high,unsigned long num_events,unsigned long flags,struct kvm_vcpu_sbi_return * retdata)488 int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low,
489 unsigned long saddr_high, unsigned long num_events,
490 unsigned long flags, struct kvm_vcpu_sbi_return *retdata)
491 {
492 struct riscv_pmu_event_info *einfo = NULL;
493 size_t shmem_size;
494 gpa_t shmem;
495 u32 eidx, etype;
496 u64 econfig;
497 int ret;
498
499 if (flags != 0 || (saddr_low & (SZ_16 - 1)) || num_events == 0 ||
500 check_mul_overflow(num_events, sizeof(*einfo), &shmem_size)) {
501 ret = SBI_ERR_INVALID_PARAM;
502 goto out;
503 }
504
505 shmem = saddr_low;
506 if (saddr_high != 0) {
507 if (IS_ENABLED(CONFIG_32BIT)) {
508 shmem |= ((gpa_t)saddr_high << 32);
509 } else {
510 ret = SBI_ERR_INVALID_ADDRESS;
511 goto out;
512 }
513 }
514
515 einfo = kvzalloc_objs(*einfo, num_events,
516 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
517 if (!einfo) {
518 ret = SBI_ERR_FAILURE;
519 goto out;
520 }
521
522 ret = kvm_vcpu_read_guest(vcpu, shmem, einfo, shmem_size);
523 if (ret) {
524 ret = SBI_ERR_FAILURE;
525 goto free_mem;
526 }
527
528 for (unsigned long i = 0; i < num_events; i++) {
529 eidx = einfo[i].event_idx;
530 etype = kvm_pmu_get_perf_event_type(eidx);
531 econfig = kvm_pmu_get_perf_event_config(eidx, einfo[i].event_data);
532 ret = riscv_pmu_get_event_info(etype, econfig, NULL);
533 einfo[i].output = (ret > 0) ? 1 : 0;
534 }
535
536 ret = kvm_vcpu_write_guest(vcpu, shmem, einfo, shmem_size);
537 if (ret)
538 ret = SBI_ERR_INVALID_ADDRESS;
539
540 free_mem:
541 kvfree(einfo);
542 out:
543 retdata->err_val = ret;
544
545 return 0;
546 }
547
kvm_riscv_vcpu_pmu_num_ctrs(struct kvm_vcpu * vcpu,struct kvm_vcpu_sbi_return * retdata)548 int kvm_riscv_vcpu_pmu_num_ctrs(struct kvm_vcpu *vcpu,
549 struct kvm_vcpu_sbi_return *retdata)
550 {
551 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
552
553 retdata->out_val = kvm_pmu_num_counters(kvpmu);
554
555 return 0;
556 }
557
kvm_riscv_vcpu_pmu_ctr_info(struct kvm_vcpu * vcpu,unsigned long cidx,struct kvm_vcpu_sbi_return * retdata)558 int kvm_riscv_vcpu_pmu_ctr_info(struct kvm_vcpu *vcpu, unsigned long cidx,
559 struct kvm_vcpu_sbi_return *retdata)
560 {
561 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
562
563 if (cidx >= RISCV_KVM_MAX_COUNTERS || cidx == 1) {
564 retdata->err_val = SBI_ERR_INVALID_PARAM;
565 return 0;
566 }
567
568 cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS);
569 retdata->out_val = kvpmu->pmc[cidx].cinfo.value;
570
571 return 0;
572 }
573
kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu * vcpu,unsigned long ctr_base,unsigned long ctr_mask,unsigned long flags,u64 ival,struct kvm_vcpu_sbi_return * retdata)574 int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base,
575 unsigned long ctr_mask, unsigned long flags, u64 ival,
576 struct kvm_vcpu_sbi_return *retdata)
577 {
578 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
579 int i, pmc_index, sbiret = 0;
580 struct kvm_pmc *pmc;
581 int fevent_code;
582 bool snap_flag_set = flags & SBI_PMU_START_FLAG_INIT_SNAPSHOT;
583
584 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) {
585 sbiret = SBI_ERR_INVALID_PARAM;
586 goto out;
587 }
588
589 if (snap_flag_set) {
590 if (kvpmu->snapshot_addr == INVALID_GPA) {
591 sbiret = SBI_ERR_NO_SHMEM;
592 goto out;
593 }
594 if (kvm_vcpu_read_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata,
595 sizeof(struct riscv_pmu_snapshot_data))) {
596 pr_warn("Unable to read snapshot shared memory while starting counters\n");
597 sbiret = SBI_ERR_FAILURE;
598 goto out;
599 }
600 }
601 /* Start the counters that have been configured and requested by the guest */
602 for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) {
603 pmc_index = array_index_nospec(i + ctr_base,
604 RISCV_KVM_MAX_COUNTERS);
605 if (!test_bit(pmc_index, kvpmu->pmc_in_use))
606 continue;
607 /* The guest started the counter again. Reset the overflow status */
608 clear_bit(pmc_index, kvpmu->pmc_overflown);
609 pmc = &kvpmu->pmc[pmc_index];
610 if (flags & SBI_PMU_START_FLAG_SET_INIT_VALUE) {
611 pmc->counter_val = ival;
612 } else if (snap_flag_set) {
613 /* The counter index in the snapshot are relative to the counter base */
614 pmc->counter_val = kvpmu->sdata->ctr_values[i];
615 }
616
617 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) {
618 fevent_code = get_event_code(pmc->event_idx);
619 if (fevent_code >= SBI_PMU_FW_MAX) {
620 sbiret = SBI_ERR_INVALID_PARAM;
621 goto out;
622 }
623
624 /* Check if the counter was already started for some reason */
625 if (kvpmu->fw_event[fevent_code].started) {
626 sbiret = SBI_ERR_ALREADY_STARTED;
627 continue;
628 }
629
630 kvpmu->fw_event[fevent_code].started = true;
631 kvpmu->fw_event[fevent_code].value = pmc->counter_val;
632 } else if (pmc->perf_event) {
633 if (unlikely(pmc->started)) {
634 sbiret = SBI_ERR_ALREADY_STARTED;
635 continue;
636 }
637 perf_event_period(pmc->perf_event, kvm_pmu_get_sample_period(pmc));
638 perf_event_enable(pmc->perf_event);
639 pmc->started = true;
640 } else {
641 sbiret = SBI_ERR_INVALID_PARAM;
642 }
643 }
644
645 out:
646 retdata->err_val = sbiret;
647
648 return 0;
649 }
650
kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu * vcpu,unsigned long ctr_base,unsigned long ctr_mask,unsigned long flags,struct kvm_vcpu_sbi_return * retdata)651 int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
652 unsigned long ctr_mask, unsigned long flags,
653 struct kvm_vcpu_sbi_return *retdata)
654 {
655 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
656 int i, pmc_index, sbiret = 0;
657 struct kvm_pmc *pmc;
658 int fevent_code;
659 bool snap_flag_set = flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
660 bool shmem_needs_update = false;
661
662 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) {
663 sbiret = SBI_ERR_INVALID_PARAM;
664 goto out;
665 }
666
667 if (snap_flag_set && kvpmu->snapshot_addr == INVALID_GPA) {
668 sbiret = SBI_ERR_NO_SHMEM;
669 goto out;
670 }
671
672 /* Stop the counters that have been configured and requested by the guest */
673 for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) {
674 pmc_index = array_index_nospec(i + ctr_base,
675 RISCV_KVM_MAX_COUNTERS);
676 if (!test_bit(pmc_index, kvpmu->pmc_in_use))
677 continue;
678 pmc = &kvpmu->pmc[pmc_index];
679 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) {
680 fevent_code = get_event_code(pmc->event_idx);
681 if (fevent_code >= SBI_PMU_FW_MAX) {
682 sbiret = SBI_ERR_INVALID_PARAM;
683 goto out;
684 }
685
686 if (!kvpmu->fw_event[fevent_code].started) {
687 sbiret = SBI_ERR_ALREADY_STOPPED;
688 } else {
689 kvpmu->fw_event[fevent_code].started = false;
690 pmc->counter_val = kvpmu->fw_event[fevent_code].value;
691 }
692 } else if (pmc->perf_event) {
693 if (pmc->started) {
694 /*
695 * Stop the counter and fold the live count into counter_val.
696 * Reset the event value to avoid redundant accumulation.
697 */
698 pmc->counter_val += perf_event_pause(pmc->perf_event, true);
699 pmc->started = false;
700 } else {
701 sbiret = SBI_ERR_ALREADY_STOPPED;
702 }
703
704 if (flags & SBI_PMU_STOP_FLAG_RESET)
705 /* Release the counter if this is a reset request */
706 kvm_pmu_release_perf_event(pmc);
707 } else {
708 sbiret = SBI_ERR_INVALID_PARAM;
709 }
710
711 if (snap_flag_set && !sbiret) {
712 /*
713 * The counter and overflow indices in the snapshot region are w.r.to
714 * cbase. Modify the set bit in the counter mask instead of the pmc_index
715 * which indicates the absolute counter index.
716 */
717 if (test_bit(pmc_index, kvpmu->pmc_overflown))
718 kvpmu->sdata->ctr_overflow_mask |= BIT(i);
719 kvpmu->sdata->ctr_values[i] = pmc->counter_val;
720 shmem_needs_update = true;
721 }
722
723 if (flags & SBI_PMU_STOP_FLAG_RESET) {
724 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID;
725 clear_bit(pmc_index, kvpmu->pmc_in_use);
726 clear_bit(pmc_index, kvpmu->pmc_overflown);
727 if (snap_flag_set) {
728 /*
729 * Only clear the given counter as the caller is responsible to
730 * validate both the overflow mask and configured counters.
731 */
732 kvpmu->sdata->ctr_overflow_mask &= ~BIT(i);
733 shmem_needs_update = true;
734 }
735 }
736 }
737
738 if (shmem_needs_update &&
739 kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata,
740 sizeof(struct riscv_pmu_snapshot_data)))
741 sbiret = SBI_ERR_FAILURE;
742
743 out:
744 retdata->err_val = sbiret;
745
746 return 0;
747 }
748
kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu * vcpu,unsigned long ctr_base,unsigned long ctr_mask,unsigned long flags,unsigned long eidx,u64 evtdata,struct kvm_vcpu_sbi_return * retdata)749 int kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu *vcpu, unsigned long ctr_base,
750 unsigned long ctr_mask, unsigned long flags,
751 unsigned long eidx, u64 evtdata,
752 struct kvm_vcpu_sbi_return *retdata)
753 {
754 int ctr_idx, sbiret = 0;
755 long ret;
756 bool is_fevent;
757 unsigned long event_code;
758 u32 etype = kvm_pmu_get_perf_event_type(eidx);
759 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
760 struct kvm_pmc *pmc = NULL;
761 struct perf_event_attr attr = {
762 .type = etype,
763 .size = sizeof(struct perf_event_attr),
764 .pinned = true,
765 .disabled = true,
766 /*
767 * It should never reach here if the platform doesn't support the sscofpmf
768 * extension as mode filtering won't work without it.
769 */
770 .exclude_host = true,
771 .exclude_hv = true,
772 .exclude_user = !!(flags & SBI_PMU_CFG_FLAG_SET_UINH),
773 .exclude_kernel = !!(flags & SBI_PMU_CFG_FLAG_SET_SINH),
774 .config1 = RISCV_PMU_CONFIG1_GUEST_EVENTS,
775 };
776
777 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) {
778 sbiret = SBI_ERR_INVALID_PARAM;
779 goto out;
780 }
781
782 event_code = get_event_code(eidx);
783 is_fevent = kvm_pmu_is_fw_event(eidx);
784 if (is_fevent && event_code >= SBI_PMU_FW_MAX) {
785 sbiret = SBI_ERR_NOT_SUPPORTED;
786 goto out;
787 }
788
789 /*
790 * SKIP_MATCH flag indicates the caller is aware of the assigned counter
791 * for this event. Just do a sanity check if it already marked used.
792 */
793 if (flags & SBI_PMU_CFG_FLAG_SKIP_MATCH) {
794 if (!test_bit(ctr_base + __ffs(ctr_mask), kvpmu->pmc_in_use)) {
795 sbiret = SBI_ERR_FAILURE;
796 goto out;
797 }
798 ctr_idx = ctr_base + __ffs(ctr_mask);
799 } else {
800 ctr_idx = pmu_get_pmc_index(kvpmu, eidx, ctr_base, ctr_mask);
801 if (ctr_idx < 0) {
802 sbiret = SBI_ERR_NOT_SUPPORTED;
803 goto out;
804 }
805 }
806
807 ctr_idx = array_index_nospec(ctr_idx, RISCV_KVM_MAX_COUNTERS);
808 pmc = &kvpmu->pmc[ctr_idx];
809 pmc->idx = ctr_idx;
810
811 if (is_fevent) {
812 if (flags & SBI_PMU_CFG_FLAG_AUTO_START)
813 kvpmu->fw_event[event_code].started = true;
814 } else {
815 ret = kvm_pmu_create_perf_event(pmc, &attr, flags, eidx, evtdata);
816 if (ret) {
817 sbiret = SBI_ERR_NOT_SUPPORTED;
818 goto out;
819 }
820 }
821
822 set_bit(ctr_idx, kvpmu->pmc_in_use);
823 pmc->event_idx = eidx;
824 retdata->out_val = ctr_idx;
825 out:
826 retdata->err_val = sbiret;
827
828 return 0;
829 }
830
kvm_riscv_vcpu_pmu_fw_ctr_read_hi(struct kvm_vcpu * vcpu,unsigned long cidx,struct kvm_vcpu_sbi_return * retdata)831 int kvm_riscv_vcpu_pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx,
832 struct kvm_vcpu_sbi_return *retdata)
833 {
834 int ret;
835
836 ret = pmu_fw_ctr_read_hi(vcpu, cidx, &retdata->out_val);
837 if (ret == -EINVAL)
838 retdata->err_val = SBI_ERR_INVALID_PARAM;
839
840 return 0;
841 }
842
kvm_riscv_vcpu_pmu_fw_ctr_read(struct kvm_vcpu * vcpu,unsigned long cidx,struct kvm_vcpu_sbi_return * retdata)843 int kvm_riscv_vcpu_pmu_fw_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx,
844 struct kvm_vcpu_sbi_return *retdata)
845 {
846 int ret;
847
848 ret = pmu_ctr_read(vcpu, cidx, &retdata->out_val);
849 if (ret == -EINVAL)
850 retdata->err_val = SBI_ERR_INVALID_PARAM;
851
852 return 0;
853 }
854
kvm_riscv_vcpu_pmu_init(struct kvm_vcpu * vcpu)855 void kvm_riscv_vcpu_pmu_init(struct kvm_vcpu *vcpu)
856 {
857 int i = 0, ret, num_hw_ctrs = 0, hpm_width = 0;
858 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
859 struct kvm_pmc *pmc;
860
861 /*
862 * PMU functionality should be only available to guests if privilege mode
863 * filtering is available in the host. Otherwise, guest will always count
864 * events while the execution is in hypervisor mode.
865 */
866 if (kvm_riscv_isa_check_host(SSCOFPMF))
867 return;
868
869 ret = riscv_pmu_get_hpm_info(&hpm_width, &num_hw_ctrs);
870 if (ret < 0 || !hpm_width || !num_hw_ctrs)
871 return;
872
873 /*
874 * Increase the number of hardware counters to offset the time counter.
875 */
876 kvpmu->num_hw_ctrs = num_hw_ctrs + 1;
877 kvpmu->num_fw_ctrs = SBI_PMU_FW_MAX;
878 memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event));
879 kvpmu->snapshot_addr = INVALID_GPA;
880
881 if (kvpmu->num_hw_ctrs > RISCV_KVM_MAX_HW_CTRS) {
882 pr_warn_once("Limiting the hardware counters to 32 as specified by the ISA");
883 kvpmu->num_hw_ctrs = RISCV_KVM_MAX_HW_CTRS;
884 }
885
886 /*
887 * There is no correlation between the logical hardware counter and virtual counters.
888 * However, we need to encode a hpmcounter CSR in the counter info field so that
889 * KVM can trap n emulate the read. This works well in the migration use case as
890 * KVM doesn't care if the actual hpmcounter is available in the hardware or not.
891 */
892 for (i = 0; i < kvm_pmu_num_counters(kvpmu); i++) {
893 /* TIME CSR shouldn't be read from perf interface */
894 if (i == 1)
895 continue;
896 pmc = &kvpmu->pmc[i];
897 pmc->idx = i;
898 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID;
899 pmc->vcpu = vcpu;
900 if (i < kvpmu->num_hw_ctrs) {
901 pmc->cinfo.type = SBI_PMU_CTR_TYPE_HW;
902 if (i < 3)
903 /* CY, IR counters */
904 pmc->cinfo.width = 63;
905 else
906 pmc->cinfo.width = hpm_width;
907 /*
908 * The CSR number doesn't have any relation with the logical
909 * hardware counters. The CSR numbers are encoded sequentially
910 * to avoid maintaining a map between the virtual counter
911 * and CSR number.
912 */
913 pmc->cinfo.csr = CSR_CYCLE + i;
914 } else {
915 pmc->cinfo.type = SBI_PMU_CTR_TYPE_FW;
916 pmc->cinfo.width = 63;
917 }
918 }
919
920 kvpmu->init_done = true;
921 }
922
kvm_riscv_vcpu_pmu_deinit(struct kvm_vcpu * vcpu)923 void kvm_riscv_vcpu_pmu_deinit(struct kvm_vcpu *vcpu)
924 {
925 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
926 struct kvm_pmc *pmc;
927 int i;
928
929 if (!kvpmu)
930 return;
931
932 for_each_set_bit(i, kvpmu->pmc_in_use, RISCV_KVM_MAX_COUNTERS) {
933 pmc = &kvpmu->pmc[i];
934 pmc->counter_val = 0;
935 kvm_pmu_release_perf_event(pmc);
936 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID;
937 }
938 bitmap_zero(kvpmu->pmc_in_use, RISCV_KVM_MAX_COUNTERS);
939 bitmap_zero(kvpmu->pmc_overflown, RISCV_KVM_MAX_COUNTERS);
940 memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event));
941 kvm_pmu_clear_snapshot_area(vcpu);
942 }
943
kvm_riscv_vcpu_pmu_reset(struct kvm_vcpu * vcpu)944 void kvm_riscv_vcpu_pmu_reset(struct kvm_vcpu *vcpu)
945 {
946 kvm_riscv_vcpu_pmu_deinit(vcpu);
947 }
948