1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2023 Rivos Inc
4 *
5 * Authors:
6 * Atish Patra <atishp@rivosinc.com>
7 */
8
9 #define pr_fmt(fmt) "riscv-kvm-pmu: " fmt
10 #include <linux/bitops.h>
11 #include <linux/errno.h>
12 #include <linux/err.h>
13 #include <linux/kvm_host.h>
14 #include <linux/nospec.h>
15 #include <linux/overflow.h>
16 #include <linux/perf/riscv_pmu.h>
17 #include <linux/slab.h>
18 #include <asm/csr.h>
19 #include <asm/kvm_isa.h>
20 #include <asm/kvm_vcpu_sbi.h>
21 #include <asm/kvm_vcpu_pmu.h>
22 #include <asm/sbi.h>
23
24 #define kvm_pmu_num_counters(pmu) ((pmu)->num_hw_ctrs + (pmu)->num_fw_ctrs)
25 #define get_event_type(x) (((x) & SBI_PMU_EVENT_IDX_TYPE_MASK) >> 16)
26 #define get_event_code(x) ((x) & SBI_PMU_EVENT_IDX_CODE_MASK)
27
28 static enum perf_hw_id hw_event_perf_map[SBI_PMU_HW_GENERAL_MAX] = {
29 [SBI_PMU_HW_CPU_CYCLES] = PERF_COUNT_HW_CPU_CYCLES,
30 [SBI_PMU_HW_INSTRUCTIONS] = PERF_COUNT_HW_INSTRUCTIONS,
31 [SBI_PMU_HW_CACHE_REFERENCES] = PERF_COUNT_HW_CACHE_REFERENCES,
32 [SBI_PMU_HW_CACHE_MISSES] = PERF_COUNT_HW_CACHE_MISSES,
33 [SBI_PMU_HW_BRANCH_INSTRUCTIONS] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS,
34 [SBI_PMU_HW_BRANCH_MISSES] = PERF_COUNT_HW_BRANCH_MISSES,
35 [SBI_PMU_HW_BUS_CYCLES] = PERF_COUNT_HW_BUS_CYCLES,
36 [SBI_PMU_HW_STALLED_CYCLES_FRONTEND] = PERF_COUNT_HW_STALLED_CYCLES_FRONTEND,
37 [SBI_PMU_HW_STALLED_CYCLES_BACKEND] = PERF_COUNT_HW_STALLED_CYCLES_BACKEND,
38 [SBI_PMU_HW_REF_CPU_CYCLES] = PERF_COUNT_HW_REF_CPU_CYCLES,
39 };
40
kvm_pmu_get_sample_period(struct kvm_pmc * pmc)41 static u64 kvm_pmu_get_sample_period(struct kvm_pmc *pmc)
42 {
43 u64 counter_val_mask = GENMASK(pmc->cinfo.width, 0);
44 u64 sample_period;
45
46 if (!pmc->counter_val)
47 sample_period = counter_val_mask;
48 else
49 sample_period = (-pmc->counter_val) & counter_val_mask;
50
51 return sample_period;
52 }
53
kvm_pmu_get_perf_event_type(unsigned long eidx)54 static u32 kvm_pmu_get_perf_event_type(unsigned long eidx)
55 {
56 enum sbi_pmu_event_type etype = get_event_type(eidx);
57 u32 type = PERF_TYPE_MAX;
58
59 switch (etype) {
60 case SBI_PMU_EVENT_TYPE_HW:
61 type = PERF_TYPE_HARDWARE;
62 break;
63 case SBI_PMU_EVENT_TYPE_CACHE:
64 type = PERF_TYPE_HW_CACHE;
65 break;
66 case SBI_PMU_EVENT_TYPE_RAW:
67 case SBI_PMU_EVENT_TYPE_RAW_V2:
68 case SBI_PMU_EVENT_TYPE_FW:
69 type = PERF_TYPE_RAW;
70 break;
71 default:
72 break;
73 }
74
75 return type;
76 }
77
kvm_pmu_is_fw_event(unsigned long eidx)78 static bool kvm_pmu_is_fw_event(unsigned long eidx)
79 {
80 return get_event_type(eidx) == SBI_PMU_EVENT_TYPE_FW;
81 }
82
kvm_pmu_release_perf_event(struct kvm_pmc * pmc)83 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)
84 {
85 if (pmc->perf_event) {
86 perf_event_disable(pmc->perf_event);
87 perf_event_release_kernel(pmc->perf_event);
88 pmc->perf_event = NULL;
89 }
90 }
91
kvm_pmu_get_perf_event_hw_config(u32 sbi_event_code)92 static u64 kvm_pmu_get_perf_event_hw_config(u32 sbi_event_code)
93 {
94 return hw_event_perf_map[array_index_nospec(sbi_event_code,
95 SBI_PMU_HW_GENERAL_MAX)];
96 }
97
kvm_pmu_get_perf_event_cache_config(u32 sbi_event_code)98 static u64 kvm_pmu_get_perf_event_cache_config(u32 sbi_event_code)
99 {
100 u64 config = U64_MAX;
101 unsigned int cache_type, cache_op, cache_result;
102
103 /* All the cache event masks lie within 0xFF. No separate masking is necessary */
104 cache_type = (sbi_event_code & SBI_PMU_EVENT_CACHE_ID_CODE_MASK) >>
105 SBI_PMU_EVENT_CACHE_ID_SHIFT;
106 cache_op = (sbi_event_code & SBI_PMU_EVENT_CACHE_OP_ID_CODE_MASK) >>
107 SBI_PMU_EVENT_CACHE_OP_SHIFT;
108 cache_result = sbi_event_code & SBI_PMU_EVENT_CACHE_RESULT_ID_CODE_MASK;
109
110 if (cache_type >= PERF_COUNT_HW_CACHE_MAX ||
111 cache_op >= PERF_COUNT_HW_CACHE_OP_MAX ||
112 cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
113 return config;
114
115 config = cache_type | (cache_op << 8) | (cache_result << 16);
116
117 return config;
118 }
119
kvm_pmu_get_perf_event_config(unsigned long eidx,uint64_t evt_data)120 static u64 kvm_pmu_get_perf_event_config(unsigned long eidx, uint64_t evt_data)
121 {
122 enum sbi_pmu_event_type etype = get_event_type(eidx);
123 u32 ecode = get_event_code(eidx);
124 u64 config = U64_MAX;
125
126 switch (etype) {
127 case SBI_PMU_EVENT_TYPE_HW:
128 if (ecode < SBI_PMU_HW_GENERAL_MAX)
129 config = kvm_pmu_get_perf_event_hw_config(ecode);
130 break;
131 case SBI_PMU_EVENT_TYPE_CACHE:
132 config = kvm_pmu_get_perf_event_cache_config(ecode);
133 break;
134 case SBI_PMU_EVENT_TYPE_RAW:
135 config = evt_data & RISCV_PMU_RAW_EVENT_MASK;
136 break;
137 case SBI_PMU_EVENT_TYPE_RAW_V2:
138 config = evt_data & RISCV_PMU_RAW_EVENT_V2_MASK;
139 break;
140 case SBI_PMU_EVENT_TYPE_FW:
141 if (ecode < SBI_PMU_FW_MAX)
142 config = (1ULL << 63) | ecode;
143 break;
144 default:
145 break;
146 }
147
148 return config;
149 }
150
kvm_pmu_get_fixed_pmc_index(unsigned long eidx)151 static int kvm_pmu_get_fixed_pmc_index(unsigned long eidx)
152 {
153 u32 etype = kvm_pmu_get_perf_event_type(eidx);
154 u32 ecode = get_event_code(eidx);
155
156 if (etype != SBI_PMU_EVENT_TYPE_HW)
157 return -EINVAL;
158
159 if (ecode == SBI_PMU_HW_CPU_CYCLES)
160 return 0;
161 else if (ecode == SBI_PMU_HW_INSTRUCTIONS)
162 return 2;
163 else
164 return -EINVAL;
165 }
166
kvm_pmu_get_programmable_pmc_index(struct kvm_pmu * kvpmu,unsigned long eidx,unsigned long cbase,unsigned long cmask)167 static int kvm_pmu_get_programmable_pmc_index(struct kvm_pmu *kvpmu, unsigned long eidx,
168 unsigned long cbase, unsigned long cmask)
169 {
170 int ctr_idx = -1;
171 int i, pmc_idx;
172 int min, max;
173
174 if (kvm_pmu_is_fw_event(eidx)) {
175 /* Firmware counters are mapped 1:1 starting from num_hw_ctrs for simplicity */
176 min = kvpmu->num_hw_ctrs;
177 max = min + kvpmu->num_fw_ctrs;
178 } else {
179 /* First 3 counters are reserved for fixed counters */
180 min = 3;
181 max = kvpmu->num_hw_ctrs;
182 }
183
184 for_each_set_bit(i, &cmask, BITS_PER_LONG) {
185 pmc_idx = i + cbase;
186 if ((pmc_idx >= min && pmc_idx < max) &&
187 !test_bit(pmc_idx, kvpmu->pmc_in_use)) {
188 ctr_idx = pmc_idx;
189 break;
190 }
191 }
192
193 return ctr_idx;
194 }
195
pmu_get_pmc_index(struct kvm_pmu * pmu,unsigned long eidx,unsigned long cbase,unsigned long cmask)196 static int pmu_get_pmc_index(struct kvm_pmu *pmu, unsigned long eidx,
197 unsigned long cbase, unsigned long cmask)
198 {
199 int ret;
200
201 /* Fixed counters need to be have fixed mapping as they have different width */
202 ret = kvm_pmu_get_fixed_pmc_index(eidx);
203 if (ret >= 0)
204 return ret;
205
206 return kvm_pmu_get_programmable_pmc_index(pmu, eidx, cbase, cmask);
207 }
208
pmu_fw_ctr_read_hi(struct kvm_vcpu * vcpu,unsigned long cidx,unsigned long * out_val)209 static int pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx,
210 unsigned long *out_val)
211 {
212 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
213 struct kvm_pmc *pmc;
214 int fevent_code;
215
216 if (!IS_ENABLED(CONFIG_32BIT)) {
217 pr_warn("%s: should be invoked for only RV32\n", __func__);
218 return -EINVAL;
219 }
220
221 if (cidx >= kvm_pmu_num_counters(kvpmu) || cidx == 1) {
222 pr_warn("Invalid counter id [%ld]during read\n", cidx);
223 return -EINVAL;
224 }
225
226 cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS);
227 pmc = &kvpmu->pmc[cidx];
228
229 if (pmc->cinfo.type != SBI_PMU_CTR_TYPE_FW)
230 return -EINVAL;
231
232 if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID)
233 return -EINVAL;
234
235 fevent_code = get_event_code(pmc->event_idx);
236 if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX,
237 "Invalid firmware event code: %d\n", fevent_code))
238 return -EINVAL;
239
240 pmc->counter_val = kvpmu->fw_event[fevent_code].value;
241
242 *out_val = pmc->counter_val >> 32;
243
244 return 0;
245 }
246
pmu_ctr_read(struct kvm_vcpu * vcpu,unsigned long cidx,unsigned long * out_val)247 static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx,
248 unsigned long *out_val)
249 {
250 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
251 struct kvm_pmc *pmc;
252 u64 enabled, running;
253 int fevent_code;
254
255 if (cidx >= kvm_pmu_num_counters(kvpmu) || cidx == 1) {
256 pr_warn("Invalid counter id [%ld] during read\n", cidx);
257 return -EINVAL;
258 }
259
260 cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS);
261 pmc = &kvpmu->pmc[cidx];
262
263 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) {
264 if (pmc->event_idx == SBI_PMU_EVENT_IDX_INVALID)
265 return -EINVAL;
266
267 fevent_code = get_event_code(pmc->event_idx);
268 if (WARN_ONCE(fevent_code >= SBI_PMU_FW_MAX,
269 "Invalid firmware event code: %d\n", fevent_code))
270 return -EINVAL;
271
272 pmc->counter_val = kvpmu->fw_event[fevent_code].value;
273 } else if (pmc->perf_event) {
274 pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running);
275 } else {
276 return -EINVAL;
277 }
278 *out_val = pmc->counter_val;
279
280 return 0;
281 }
282
kvm_pmu_validate_counter_mask(struct kvm_pmu * kvpmu,unsigned long ctr_base,unsigned long ctr_mask)283 static int kvm_pmu_validate_counter_mask(struct kvm_pmu *kvpmu, unsigned long ctr_base,
284 unsigned long ctr_mask)
285 {
286 unsigned long num_ctrs = kvm_pmu_num_counters(kvpmu);
287
288 /* Make sure we have a valid counter mask requested from the caller */
289 if (!ctr_mask || ctr_base >= num_ctrs || (ctr_base + __fls(ctr_mask) >= num_ctrs))
290 return -EINVAL;
291
292 return 0;
293 }
294
kvm_riscv_pmu_overflow(struct perf_event * perf_event,struct perf_sample_data * data,struct pt_regs * regs)295 static void kvm_riscv_pmu_overflow(struct perf_event *perf_event,
296 struct perf_sample_data *data,
297 struct pt_regs *regs)
298 {
299 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
300 struct kvm_vcpu *vcpu = pmc->vcpu;
301 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
302 struct riscv_pmu *rpmu = to_riscv_pmu(perf_event->pmu);
303 u64 period;
304
305 /*
306 * Stop the event counting by directly accessing the perf_event.
307 * Otherwise, this needs to deferred via a workqueue.
308 * That will introduce skew in the counter value because the actual
309 * physical counter would start after returning from this function.
310 * It will be stopped again once the workqueue is scheduled
311 */
312 rpmu->pmu.stop(perf_event, PERF_EF_UPDATE);
313
314 /*
315 * The hw counter would start automatically when this function returns.
316 * Thus, the host may continue to interrupt and inject it to the guest
317 * even without the guest configuring the next event. Depending on the hardware
318 * the host may have some sluggishness only if privilege mode filtering is not
319 * available. In an ideal world, where qemu is not the only capable hardware,
320 * this can be removed.
321 * FYI: ARM64 does this way while x86 doesn't do anything as such.
322 * TODO: Should we keep it for RISC-V ?
323 */
324 period = -(local64_read(&perf_event->count));
325
326 local64_set(&perf_event->hw.period_left, 0);
327 perf_event->attr.sample_period = period;
328 perf_event->hw.sample_period = period;
329
330 set_bit(pmc->idx, kvpmu->pmc_overflown);
331 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_PMU_OVF);
332
333 rpmu->pmu.start(perf_event, PERF_EF_RELOAD);
334 }
335
kvm_pmu_create_perf_event(struct kvm_pmc * pmc,struct perf_event_attr * attr,unsigned long flags,unsigned long eidx,unsigned long evtdata)336 static long kvm_pmu_create_perf_event(struct kvm_pmc *pmc, struct perf_event_attr *attr,
337 unsigned long flags, unsigned long eidx,
338 unsigned long evtdata)
339 {
340 struct perf_event *event;
341
342 kvm_pmu_release_perf_event(pmc);
343 attr->config = kvm_pmu_get_perf_event_config(eidx, evtdata);
344 if (flags & SBI_PMU_CFG_FLAG_CLEAR_VALUE) {
345 //TODO: Do we really want to clear the value in hardware counter
346 pmc->counter_val = 0;
347 }
348
349 /*
350 * Set the default sample_period for now. The guest specified value
351 * will be updated in the start call.
352 */
353 attr->sample_period = kvm_pmu_get_sample_period(pmc);
354
355 event = perf_event_create_kernel_counter(attr, -1, current, kvm_riscv_pmu_overflow, pmc);
356 if (IS_ERR(event)) {
357 pr_debug("kvm pmu event creation failed for eidx %lx: %ld\n", eidx, PTR_ERR(event));
358 return PTR_ERR(event);
359 }
360
361 pmc->perf_event = event;
362 if (flags & SBI_PMU_CFG_FLAG_AUTO_START)
363 perf_event_enable(pmc->perf_event);
364
365 return 0;
366 }
367
kvm_riscv_vcpu_pmu_incr_fw(struct kvm_vcpu * vcpu,unsigned long fid)368 int kvm_riscv_vcpu_pmu_incr_fw(struct kvm_vcpu *vcpu, unsigned long fid)
369 {
370 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
371 struct kvm_fw_event *fevent;
372
373 if (!kvpmu || fid >= SBI_PMU_FW_MAX)
374 return -EINVAL;
375
376 fevent = &kvpmu->fw_event[fid];
377 if (fevent->started)
378 fevent->value++;
379
380 return 0;
381 }
382
kvm_riscv_vcpu_pmu_read_hpm(struct kvm_vcpu * vcpu,unsigned int csr_num,unsigned long * val,unsigned long new_val,unsigned long wr_mask)383 int kvm_riscv_vcpu_pmu_read_hpm(struct kvm_vcpu *vcpu, unsigned int csr_num,
384 unsigned long *val, unsigned long new_val,
385 unsigned long wr_mask)
386 {
387 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
388 int cidx, ret = KVM_INSN_CONTINUE_NEXT_SEPC;
389
390 if (!kvpmu || !kvpmu->init_done) {
391 /*
392 * In absence of sscofpmf in the platform, the guest OS may use
393 * the legacy PMU driver to read cycle/instret. In that case,
394 * just return 0 to avoid any illegal trap. However, any other
395 * hpmcounter access should result in illegal trap as they must
396 * be access through SBI PMU only.
397 */
398 if (csr_num == CSR_CYCLE || csr_num == CSR_INSTRET) {
399 *val = 0;
400 return ret;
401 } else {
402 return KVM_INSN_ILLEGAL_TRAP;
403 }
404 }
405
406 /* The counter CSR are read only. Thus, any write should result in illegal traps */
407 if (wr_mask)
408 return KVM_INSN_ILLEGAL_TRAP;
409
410 cidx = csr_num - CSR_CYCLE;
411
412 if (pmu_ctr_read(vcpu, cidx, val) < 0)
413 return KVM_INSN_ILLEGAL_TRAP;
414
415 return ret;
416 }
417
kvm_pmu_clear_snapshot_area(struct kvm_vcpu * vcpu)418 static void kvm_pmu_clear_snapshot_area(struct kvm_vcpu *vcpu)
419 {
420 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
421
422 kfree(kvpmu->sdata);
423 kvpmu->sdata = NULL;
424 kvpmu->snapshot_addr = INVALID_GPA;
425 }
426
kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu * vcpu,unsigned long saddr_low,unsigned long saddr_high,unsigned long flags,struct kvm_vcpu_sbi_return * retdata)427 int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long saddr_low,
428 unsigned long saddr_high, unsigned long flags,
429 struct kvm_vcpu_sbi_return *retdata)
430 {
431 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
432 int snapshot_area_size = sizeof(struct riscv_pmu_snapshot_data);
433 int sbiret = 0;
434 gpa_t saddr;
435
436 if (!kvpmu || flags) {
437 sbiret = SBI_ERR_INVALID_PARAM;
438 goto out;
439 }
440
441 if (saddr_low == SBI_SHMEM_DISABLE && saddr_high == SBI_SHMEM_DISABLE) {
442 kvm_pmu_clear_snapshot_area(vcpu);
443 return 0;
444 }
445
446 saddr = saddr_low;
447
448 if (saddr_high != 0) {
449 if (IS_ENABLED(CONFIG_32BIT)) {
450 saddr |= ((gpa_t)saddr_high << 32);
451 } else {
452 sbiret = SBI_ERR_INVALID_ADDRESS;
453 goto out;
454 }
455 }
456
457 kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC | __GFP_ACCOUNT);
458 if (!kvpmu->sdata) {
459 sbiret = SBI_ERR_FAILURE;
460 goto out;
461 }
462
463 /* No need to check writable slot explicitly as kvm_vcpu_write_guest does it internally */
464 if (kvm_vcpu_write_guest(vcpu, saddr, kvpmu->sdata, snapshot_area_size)) {
465 kfree(kvpmu->sdata);
466 kvpmu->sdata = NULL;
467 sbiret = SBI_ERR_INVALID_ADDRESS;
468 goto out;
469 }
470
471 kvpmu->snapshot_addr = saddr;
472
473 out:
474 retdata->err_val = sbiret;
475
476 return 0;
477 }
478
kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu * vcpu,unsigned long saddr_low,unsigned long saddr_high,unsigned long num_events,unsigned long flags,struct kvm_vcpu_sbi_return * retdata)479 int kvm_riscv_vcpu_pmu_event_info(struct kvm_vcpu *vcpu, unsigned long saddr_low,
480 unsigned long saddr_high, unsigned long num_events,
481 unsigned long flags, struct kvm_vcpu_sbi_return *retdata)
482 {
483 struct riscv_pmu_event_info *einfo = NULL;
484 size_t shmem_size;
485 gpa_t shmem;
486 u32 eidx, etype;
487 u64 econfig;
488 int ret;
489
490 if (flags != 0 || (saddr_low & (SZ_16 - 1)) || num_events == 0 ||
491 check_mul_overflow(num_events, sizeof(*einfo), &shmem_size)) {
492 ret = SBI_ERR_INVALID_PARAM;
493 goto out;
494 }
495
496 shmem = saddr_low;
497 if (saddr_high != 0) {
498 if (IS_ENABLED(CONFIG_32BIT)) {
499 shmem |= ((gpa_t)saddr_high << 32);
500 } else {
501 ret = SBI_ERR_INVALID_ADDRESS;
502 goto out;
503 }
504 }
505
506 einfo = kvzalloc_objs(*einfo, num_events,
507 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
508 if (!einfo) {
509 ret = SBI_ERR_FAILURE;
510 goto out;
511 }
512
513 ret = kvm_vcpu_read_guest(vcpu, shmem, einfo, shmem_size);
514 if (ret) {
515 ret = SBI_ERR_FAILURE;
516 goto free_mem;
517 }
518
519 for (unsigned long i = 0; i < num_events; i++) {
520 eidx = einfo[i].event_idx;
521 etype = kvm_pmu_get_perf_event_type(eidx);
522 econfig = kvm_pmu_get_perf_event_config(eidx, einfo[i].event_data);
523 ret = riscv_pmu_get_event_info(etype, econfig, NULL);
524 einfo[i].output = (ret > 0) ? 1 : 0;
525 }
526
527 ret = kvm_vcpu_write_guest(vcpu, shmem, einfo, shmem_size);
528 if (ret)
529 ret = SBI_ERR_INVALID_ADDRESS;
530
531 free_mem:
532 kvfree(einfo);
533 out:
534 retdata->err_val = ret;
535
536 return 0;
537 }
538
kvm_riscv_vcpu_pmu_num_ctrs(struct kvm_vcpu * vcpu,struct kvm_vcpu_sbi_return * retdata)539 int kvm_riscv_vcpu_pmu_num_ctrs(struct kvm_vcpu *vcpu,
540 struct kvm_vcpu_sbi_return *retdata)
541 {
542 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
543
544 retdata->out_val = kvm_pmu_num_counters(kvpmu);
545
546 return 0;
547 }
548
kvm_riscv_vcpu_pmu_ctr_info(struct kvm_vcpu * vcpu,unsigned long cidx,struct kvm_vcpu_sbi_return * retdata)549 int kvm_riscv_vcpu_pmu_ctr_info(struct kvm_vcpu *vcpu, unsigned long cidx,
550 struct kvm_vcpu_sbi_return *retdata)
551 {
552 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
553
554 if (cidx >= RISCV_KVM_MAX_COUNTERS || cidx == 1) {
555 retdata->err_val = SBI_ERR_INVALID_PARAM;
556 return 0;
557 }
558
559 cidx = array_index_nospec(cidx, RISCV_KVM_MAX_COUNTERS);
560 retdata->out_val = kvpmu->pmc[cidx].cinfo.value;
561
562 return 0;
563 }
564
kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu * vcpu,unsigned long ctr_base,unsigned long ctr_mask,unsigned long flags,u64 ival,struct kvm_vcpu_sbi_return * retdata)565 int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base,
566 unsigned long ctr_mask, unsigned long flags, u64 ival,
567 struct kvm_vcpu_sbi_return *retdata)
568 {
569 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
570 int i, pmc_index, sbiret = 0;
571 struct kvm_pmc *pmc;
572 int fevent_code;
573 bool snap_flag_set = flags & SBI_PMU_START_FLAG_INIT_SNAPSHOT;
574
575 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) {
576 sbiret = SBI_ERR_INVALID_PARAM;
577 goto out;
578 }
579
580 if (snap_flag_set) {
581 if (kvpmu->snapshot_addr == INVALID_GPA) {
582 sbiret = SBI_ERR_NO_SHMEM;
583 goto out;
584 }
585 if (kvm_vcpu_read_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata,
586 sizeof(struct riscv_pmu_snapshot_data))) {
587 pr_warn("Unable to read snapshot shared memory while starting counters\n");
588 sbiret = SBI_ERR_FAILURE;
589 goto out;
590 }
591 }
592 /* Start the counters that have been configured and requested by the guest */
593 for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) {
594 pmc_index = array_index_nospec(i + ctr_base,
595 RISCV_KVM_MAX_COUNTERS);
596 if (!test_bit(pmc_index, kvpmu->pmc_in_use))
597 continue;
598 /* The guest started the counter again. Reset the overflow status */
599 clear_bit(pmc_index, kvpmu->pmc_overflown);
600 pmc = &kvpmu->pmc[pmc_index];
601 if (flags & SBI_PMU_START_FLAG_SET_INIT_VALUE) {
602 pmc->counter_val = ival;
603 } else if (snap_flag_set) {
604 /* The counter index in the snapshot are relative to the counter base */
605 pmc->counter_val = kvpmu->sdata->ctr_values[i];
606 }
607
608 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) {
609 fevent_code = get_event_code(pmc->event_idx);
610 if (fevent_code >= SBI_PMU_FW_MAX) {
611 sbiret = SBI_ERR_INVALID_PARAM;
612 goto out;
613 }
614
615 /* Check if the counter was already started for some reason */
616 if (kvpmu->fw_event[fevent_code].started) {
617 sbiret = SBI_ERR_ALREADY_STARTED;
618 continue;
619 }
620
621 kvpmu->fw_event[fevent_code].started = true;
622 kvpmu->fw_event[fevent_code].value = pmc->counter_val;
623 } else if (pmc->perf_event) {
624 if (unlikely(pmc->started)) {
625 sbiret = SBI_ERR_ALREADY_STARTED;
626 continue;
627 }
628 perf_event_period(pmc->perf_event, kvm_pmu_get_sample_period(pmc));
629 perf_event_enable(pmc->perf_event);
630 pmc->started = true;
631 } else {
632 sbiret = SBI_ERR_INVALID_PARAM;
633 }
634 }
635
636 out:
637 retdata->err_val = sbiret;
638
639 return 0;
640 }
641
kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu * vcpu,unsigned long ctr_base,unsigned long ctr_mask,unsigned long flags,struct kvm_vcpu_sbi_return * retdata)642 int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
643 unsigned long ctr_mask, unsigned long flags,
644 struct kvm_vcpu_sbi_return *retdata)
645 {
646 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
647 int i, pmc_index, sbiret = 0;
648 u64 enabled, running;
649 struct kvm_pmc *pmc;
650 int fevent_code;
651 bool snap_flag_set = flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
652 bool shmem_needs_update = false;
653
654 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) {
655 sbiret = SBI_ERR_INVALID_PARAM;
656 goto out;
657 }
658
659 if (snap_flag_set && kvpmu->snapshot_addr == INVALID_GPA) {
660 sbiret = SBI_ERR_NO_SHMEM;
661 goto out;
662 }
663
664 /* Stop the counters that have been configured and requested by the guest */
665 for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) {
666 pmc_index = array_index_nospec(i + ctr_base,
667 RISCV_KVM_MAX_COUNTERS);
668 if (!test_bit(pmc_index, kvpmu->pmc_in_use))
669 continue;
670 pmc = &kvpmu->pmc[pmc_index];
671 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) {
672 fevent_code = get_event_code(pmc->event_idx);
673 if (fevent_code >= SBI_PMU_FW_MAX) {
674 sbiret = SBI_ERR_INVALID_PARAM;
675 goto out;
676 }
677
678 if (!kvpmu->fw_event[fevent_code].started)
679 sbiret = SBI_ERR_ALREADY_STOPPED;
680
681 kvpmu->fw_event[fevent_code].started = false;
682 } else if (pmc->perf_event) {
683 if (pmc->started) {
684 /* Stop counting the counter */
685 perf_event_disable(pmc->perf_event);
686 pmc->started = false;
687 } else {
688 sbiret = SBI_ERR_ALREADY_STOPPED;
689 }
690
691 if (flags & SBI_PMU_STOP_FLAG_RESET)
692 /* Release the counter if this is a reset request */
693 kvm_pmu_release_perf_event(pmc);
694 } else {
695 sbiret = SBI_ERR_INVALID_PARAM;
696 }
697
698 if (snap_flag_set && !sbiret) {
699 if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW)
700 pmc->counter_val = kvpmu->fw_event[fevent_code].value;
701 else if (pmc->perf_event)
702 pmc->counter_val += perf_event_read_value(pmc->perf_event,
703 &enabled, &running);
704 /*
705 * The counter and overflow indices in the snapshot region are w.r.to
706 * cbase. Modify the set bit in the counter mask instead of the pmc_index
707 * which indicates the absolute counter index.
708 */
709 if (test_bit(pmc_index, kvpmu->pmc_overflown))
710 kvpmu->sdata->ctr_overflow_mask |= BIT(i);
711 kvpmu->sdata->ctr_values[i] = pmc->counter_val;
712 shmem_needs_update = true;
713 }
714
715 if (flags & SBI_PMU_STOP_FLAG_RESET) {
716 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID;
717 clear_bit(pmc_index, kvpmu->pmc_in_use);
718 clear_bit(pmc_index, kvpmu->pmc_overflown);
719 if (snap_flag_set) {
720 /*
721 * Only clear the given counter as the caller is responsible to
722 * validate both the overflow mask and configured counters.
723 */
724 kvpmu->sdata->ctr_overflow_mask &= ~BIT(i);
725 shmem_needs_update = true;
726 }
727 }
728 }
729
730 if (shmem_needs_update)
731 kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata,
732 sizeof(struct riscv_pmu_snapshot_data));
733
734 out:
735 retdata->err_val = sbiret;
736
737 return 0;
738 }
739
kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu * vcpu,unsigned long ctr_base,unsigned long ctr_mask,unsigned long flags,unsigned long eidx,u64 evtdata,struct kvm_vcpu_sbi_return * retdata)740 int kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu *vcpu, unsigned long ctr_base,
741 unsigned long ctr_mask, unsigned long flags,
742 unsigned long eidx, u64 evtdata,
743 struct kvm_vcpu_sbi_return *retdata)
744 {
745 int ctr_idx, sbiret = 0;
746 long ret;
747 bool is_fevent;
748 unsigned long event_code;
749 u32 etype = kvm_pmu_get_perf_event_type(eidx);
750 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
751 struct kvm_pmc *pmc = NULL;
752 struct perf_event_attr attr = {
753 .type = etype,
754 .size = sizeof(struct perf_event_attr),
755 .pinned = true,
756 .disabled = true,
757 /*
758 * It should never reach here if the platform doesn't support the sscofpmf
759 * extension as mode filtering won't work without it.
760 */
761 .exclude_host = true,
762 .exclude_hv = true,
763 .exclude_user = !!(flags & SBI_PMU_CFG_FLAG_SET_UINH),
764 .exclude_kernel = !!(flags & SBI_PMU_CFG_FLAG_SET_SINH),
765 .config1 = RISCV_PMU_CONFIG1_GUEST_EVENTS,
766 };
767
768 if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) {
769 sbiret = SBI_ERR_INVALID_PARAM;
770 goto out;
771 }
772
773 event_code = get_event_code(eidx);
774 is_fevent = kvm_pmu_is_fw_event(eidx);
775 if (is_fevent && event_code >= SBI_PMU_FW_MAX) {
776 sbiret = SBI_ERR_NOT_SUPPORTED;
777 goto out;
778 }
779
780 /*
781 * SKIP_MATCH flag indicates the caller is aware of the assigned counter
782 * for this event. Just do a sanity check if it already marked used.
783 */
784 if (flags & SBI_PMU_CFG_FLAG_SKIP_MATCH) {
785 if (!test_bit(ctr_base + __ffs(ctr_mask), kvpmu->pmc_in_use)) {
786 sbiret = SBI_ERR_FAILURE;
787 goto out;
788 }
789 ctr_idx = ctr_base + __ffs(ctr_mask);
790 } else {
791 ctr_idx = pmu_get_pmc_index(kvpmu, eidx, ctr_base, ctr_mask);
792 if (ctr_idx < 0) {
793 sbiret = SBI_ERR_NOT_SUPPORTED;
794 goto out;
795 }
796 }
797
798 ctr_idx = array_index_nospec(ctr_idx, RISCV_KVM_MAX_COUNTERS);
799 pmc = &kvpmu->pmc[ctr_idx];
800 pmc->idx = ctr_idx;
801
802 if (is_fevent) {
803 if (flags & SBI_PMU_CFG_FLAG_AUTO_START)
804 kvpmu->fw_event[event_code].started = true;
805 } else {
806 ret = kvm_pmu_create_perf_event(pmc, &attr, flags, eidx, evtdata);
807 if (ret) {
808 sbiret = SBI_ERR_NOT_SUPPORTED;
809 goto out;
810 }
811 }
812
813 set_bit(ctr_idx, kvpmu->pmc_in_use);
814 pmc->event_idx = eidx;
815 retdata->out_val = ctr_idx;
816 out:
817 retdata->err_val = sbiret;
818
819 return 0;
820 }
821
kvm_riscv_vcpu_pmu_fw_ctr_read_hi(struct kvm_vcpu * vcpu,unsigned long cidx,struct kvm_vcpu_sbi_return * retdata)822 int kvm_riscv_vcpu_pmu_fw_ctr_read_hi(struct kvm_vcpu *vcpu, unsigned long cidx,
823 struct kvm_vcpu_sbi_return *retdata)
824 {
825 int ret;
826
827 ret = pmu_fw_ctr_read_hi(vcpu, cidx, &retdata->out_val);
828 if (ret == -EINVAL)
829 retdata->err_val = SBI_ERR_INVALID_PARAM;
830
831 return 0;
832 }
833
kvm_riscv_vcpu_pmu_fw_ctr_read(struct kvm_vcpu * vcpu,unsigned long cidx,struct kvm_vcpu_sbi_return * retdata)834 int kvm_riscv_vcpu_pmu_fw_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx,
835 struct kvm_vcpu_sbi_return *retdata)
836 {
837 int ret;
838
839 ret = pmu_ctr_read(vcpu, cidx, &retdata->out_val);
840 if (ret == -EINVAL)
841 retdata->err_val = SBI_ERR_INVALID_PARAM;
842
843 return 0;
844 }
845
kvm_riscv_vcpu_pmu_init(struct kvm_vcpu * vcpu)846 void kvm_riscv_vcpu_pmu_init(struct kvm_vcpu *vcpu)
847 {
848 int i = 0, ret, num_hw_ctrs = 0, hpm_width = 0;
849 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
850 struct kvm_pmc *pmc;
851
852 /*
853 * PMU functionality should be only available to guests if privilege mode
854 * filtering is available in the host. Otherwise, guest will always count
855 * events while the execution is in hypervisor mode.
856 */
857 if (kvm_riscv_isa_check_host(SSCOFPMF))
858 return;
859
860 ret = riscv_pmu_get_hpm_info(&hpm_width, &num_hw_ctrs);
861 if (ret < 0 || !hpm_width || !num_hw_ctrs)
862 return;
863
864 /*
865 * Increase the number of hardware counters to offset the time counter.
866 */
867 kvpmu->num_hw_ctrs = num_hw_ctrs + 1;
868 kvpmu->num_fw_ctrs = SBI_PMU_FW_MAX;
869 memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event));
870 kvpmu->snapshot_addr = INVALID_GPA;
871
872 if (kvpmu->num_hw_ctrs > RISCV_KVM_MAX_HW_CTRS) {
873 pr_warn_once("Limiting the hardware counters to 32 as specified by the ISA");
874 kvpmu->num_hw_ctrs = RISCV_KVM_MAX_HW_CTRS;
875 }
876
877 /*
878 * There is no correlation between the logical hardware counter and virtual counters.
879 * However, we need to encode a hpmcounter CSR in the counter info field so that
880 * KVM can trap n emulate the read. This works well in the migration use case as
881 * KVM doesn't care if the actual hpmcounter is available in the hardware or not.
882 */
883 for (i = 0; i < kvm_pmu_num_counters(kvpmu); i++) {
884 /* TIME CSR shouldn't be read from perf interface */
885 if (i == 1)
886 continue;
887 pmc = &kvpmu->pmc[i];
888 pmc->idx = i;
889 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID;
890 pmc->vcpu = vcpu;
891 if (i < kvpmu->num_hw_ctrs) {
892 pmc->cinfo.type = SBI_PMU_CTR_TYPE_HW;
893 if (i < 3)
894 /* CY, IR counters */
895 pmc->cinfo.width = 63;
896 else
897 pmc->cinfo.width = hpm_width;
898 /*
899 * The CSR number doesn't have any relation with the logical
900 * hardware counters. The CSR numbers are encoded sequentially
901 * to avoid maintaining a map between the virtual counter
902 * and CSR number.
903 */
904 pmc->cinfo.csr = CSR_CYCLE + i;
905 } else {
906 pmc->cinfo.type = SBI_PMU_CTR_TYPE_FW;
907 pmc->cinfo.width = 63;
908 }
909 }
910
911 kvpmu->init_done = true;
912 }
913
kvm_riscv_vcpu_pmu_deinit(struct kvm_vcpu * vcpu)914 void kvm_riscv_vcpu_pmu_deinit(struct kvm_vcpu *vcpu)
915 {
916 struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
917 struct kvm_pmc *pmc;
918 int i;
919
920 if (!kvpmu)
921 return;
922
923 for_each_set_bit(i, kvpmu->pmc_in_use, RISCV_KVM_MAX_COUNTERS) {
924 pmc = &kvpmu->pmc[i];
925 pmc->counter_val = 0;
926 kvm_pmu_release_perf_event(pmc);
927 pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID;
928 }
929 bitmap_zero(kvpmu->pmc_in_use, RISCV_KVM_MAX_COUNTERS);
930 bitmap_zero(kvpmu->pmc_overflown, RISCV_KVM_MAX_COUNTERS);
931 memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event));
932 kvm_pmu_clear_snapshot_area(vcpu);
933 }
934
kvm_riscv_vcpu_pmu_reset(struct kvm_vcpu * vcpu)935 void kvm_riscv_vcpu_pmu_reset(struct kvm_vcpu *vcpu)
936 {
937 kvm_riscv_vcpu_pmu_deinit(vcpu);
938 }
939