1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4 *
5 * Authors:
6 * Anup Patel <anup.patel@wdc.com>
7 */
8
9 #include <linux/bitops.h>
10 #include <linux/errno.h>
11 #include <linux/err.h>
12 #include <linux/kdebug.h>
13 #include <linux/module.h>
14 #include <linux/percpu.h>
15 #include <linux/vmalloc.h>
16 #include <linux/sched/signal.h>
17 #include <linux/fs.h>
18 #include <linux/kvm_host.h>
19 #include <asm/cacheflush.h>
20 #include <asm/kvm_mmu.h>
21 #include <asm/kvm_nacl.h>
22 #include <asm/kvm_vcpu_vector.h>
23
24 #define CREATE_TRACE_POINTS
25 #include "trace.h"
26
27 static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_former_vcpu);
28
kvm_riscv_clear_former_vcpu(void)29 void kvm_riscv_clear_former_vcpu(void)
30 {
31 /*
32 * Clear the per-CPU former VCPU pointer because hypervisor CSR state
33 * will be lost. This ensures that the next VCPU entry will properly
34 * restore all CSRs instead of incorrectly skipping CSR restoration
35 * via the fast-path optimization.
36 */
37 __this_cpu_write(kvm_former_vcpu, NULL);
38 }
39
40 const struct kvm_stats_desc kvm_vcpu_stats_desc[] = {
41 KVM_GENERIC_VCPU_STATS(),
42 STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
43 STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
44 STATS_DESC_COUNTER(VCPU, wrs_exit_stat),
45 STATS_DESC_COUNTER(VCPU, mmio_exit_user),
46 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
47 STATS_DESC_COUNTER(VCPU, csr_exit_user),
48 STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
49 STATS_DESC_COUNTER(VCPU, signal_exits),
50 STATS_DESC_COUNTER(VCPU, exits),
51 STATS_DESC_COUNTER(VCPU, instr_illegal_exits),
52 STATS_DESC_COUNTER(VCPU, load_misaligned_exits),
53 STATS_DESC_COUNTER(VCPU, store_misaligned_exits),
54 STATS_DESC_COUNTER(VCPU, load_access_exits),
55 STATS_DESC_COUNTER(VCPU, store_access_exits),
56 };
57
58 const struct kvm_stats_header kvm_vcpu_stats_header = {
59 .name_size = KVM_STATS_NAME_SIZE,
60 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
61 .id_offset = sizeof(struct kvm_stats_header),
62 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
63 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
64 sizeof(kvm_vcpu_stats_desc),
65 };
66
kvm_riscv_vcpu_context_reset(struct kvm_vcpu * vcpu,bool kvm_sbi_reset)67 static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu,
68 bool kvm_sbi_reset)
69 {
70 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
71 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
72 void *vector_datap = cntx->vector.datap;
73
74 memset(cntx, 0, sizeof(*cntx));
75 memset(csr, 0, sizeof(*csr));
76 memset(&vcpu->arch.smstateen_csr, 0, sizeof(vcpu->arch.smstateen_csr));
77 memset(&vcpu->arch.zicfiss_csr, 0, sizeof(vcpu->arch.zicfiss_csr));
78
79 /* Restore datap as it's not a part of the guest context. */
80 cntx->vector.datap = vector_datap;
81
82 if (kvm_sbi_reset)
83 kvm_riscv_vcpu_sbi_load_reset_state(vcpu);
84
85 /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */
86 cntx->sstatus = SR_SPP | SR_SPIE;
87
88 cntx->hstatus |= HSTATUS_VTW;
89 cntx->hstatus |= HSTATUS_SPVP;
90 cntx->hstatus |= HSTATUS_SPV;
91 }
92
kvm_riscv_reset_vcpu(struct kvm_vcpu * vcpu,bool kvm_sbi_reset)93 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset)
94 {
95 unsigned long flags;
96 bool loaded;
97
98 /**
99 * The preemption should be disabled here because it races with
100 * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which
101 * also calls vcpu_load/put.
102 */
103 get_cpu();
104 loaded = (vcpu->cpu != -1);
105 if (loaded)
106 kvm_arch_vcpu_put(vcpu);
107
108 vcpu->arch.last_exit_cpu = -1;
109
110 kvm_riscv_vcpu_context_reset(vcpu, kvm_sbi_reset);
111
112 kvm_riscv_vcpu_fp_reset(vcpu);
113
114 kvm_riscv_vcpu_vector_reset(vcpu);
115
116 kvm_riscv_vcpu_timer_reset(vcpu);
117
118 kvm_riscv_vcpu_aia_reset(vcpu);
119
120 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
121 bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS);
122 bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS);
123 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
124
125 kvm_riscv_vcpu_pmu_reset(vcpu);
126
127 vcpu->arch.hfence_head = 0;
128 vcpu->arch.hfence_tail = 0;
129 memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue));
130
131 kvm_riscv_vcpu_sbi_reset(vcpu);
132
133 /* Reset the guest CSRs for hotplug usecase */
134 if (loaded)
135 kvm_arch_vcpu_load(vcpu, smp_processor_id());
136 put_cpu();
137 }
138
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)139 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
140 {
141 return 0;
142 }
143
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)144 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
145 {
146 int rc;
147
148 spin_lock_init(&vcpu->arch.mp_state_lock);
149
150 /* Mark this VCPU never ran */
151 vcpu->arch.ran_atleast_once = false;
152
153 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
154 bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX);
155
156 /* Setup VCPU config */
157 kvm_riscv_vcpu_config_init(vcpu);
158
159 /* Setup ISA features available to VCPU */
160 kvm_riscv_vcpu_setup_isa(vcpu);
161
162 /* Setup vendor, arch, and implementation details */
163 vcpu->arch.mvendorid = sbi_get_mvendorid();
164 vcpu->arch.marchid = sbi_get_marchid();
165 vcpu->arch.mimpid = sbi_get_mimpid();
166
167 /* Setup VCPU hfence queue */
168 spin_lock_init(&vcpu->arch.hfence_lock);
169 raw_spin_lock_init(&vcpu->arch.irqs_pending_lock);
170
171 spin_lock_init(&vcpu->arch.reset_state.lock);
172
173 rc = kvm_riscv_vcpu_alloc_vector_context(vcpu);
174 if (rc)
175 return rc;
176
177 /* Setup VCPU timer */
178 kvm_riscv_vcpu_timer_init(vcpu);
179
180 /* setup performance monitoring */
181 kvm_riscv_vcpu_pmu_init(vcpu);
182
183 /* Setup VCPU AIA */
184 kvm_riscv_vcpu_aia_init(vcpu);
185
186 /*
187 * Setup SBI extensions
188 * NOTE: This must be the last thing to be initialized.
189 */
190 kvm_riscv_vcpu_sbi_init(vcpu);
191
192 /* Reset VCPU */
193 kvm_riscv_reset_vcpu(vcpu, false);
194
195 return 0;
196 }
197
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)198 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
199 {
200 /**
201 * vcpu with id 0 is the designated boot cpu.
202 * Keep all vcpus with non-zero id in power-off state so that
203 * they can be brought up using SBI HSM extension.
204 */
205 if (vcpu->vcpu_idx != 0)
206 kvm_riscv_vcpu_power_off(vcpu);
207 }
208
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)209 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
210 {
211 kvm_riscv_vcpu_sbi_deinit(vcpu);
212
213 /* Cleanup VCPU AIA context */
214 kvm_riscv_vcpu_aia_deinit(vcpu);
215
216 /* Cleanup VCPU timer */
217 kvm_riscv_vcpu_timer_deinit(vcpu);
218
219 kvm_riscv_vcpu_pmu_deinit(vcpu);
220
221 /* Free unused pages pre-allocated for G-stage page table mappings */
222 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
223
224 /* Free vector context space for host and guest kernel */
225 kvm_riscv_vcpu_free_vector_context(vcpu);
226 }
227
kvm_cpu_has_pending_timer(struct kvm_vcpu * vcpu)228 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
229 {
230 return kvm_riscv_vcpu_timer_pending(vcpu);
231 }
232
kvm_arch_vcpu_runnable(struct kvm_vcpu * vcpu)233 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
234 {
235 return (kvm_riscv_vcpu_has_interrupts(vcpu, -1ULL) &&
236 !kvm_riscv_vcpu_stopped(vcpu) && !vcpu->arch.pause);
237 }
238
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)239 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
240 {
241 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
242 }
243
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)244 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
245 {
246 return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false;
247 }
248
249 #ifdef CONFIG_GUEST_PERF_EVENTS
kvm_arch_vcpu_get_ip(struct kvm_vcpu * vcpu)250 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
251 {
252 return vcpu->arch.guest_context.sepc;
253 }
254 #endif
255
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)256 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
257 {
258 return VM_FAULT_SIGBUS;
259 }
260
kvm_arch_vcpu_unlocked_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)261 long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
262 unsigned long arg)
263 {
264 struct kvm_vcpu *vcpu = filp->private_data;
265 void __user *argp = (void __user *)arg;
266
267 if (ioctl == KVM_INTERRUPT) {
268 struct kvm_interrupt irq;
269
270 if (copy_from_user(&irq, argp, sizeof(irq)))
271 return -EFAULT;
272
273 if (irq.irq == KVM_INTERRUPT_SET)
274 return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);
275 else
276 return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
277 }
278
279 return -ENOIOCTLCMD;
280 }
281
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)282 long kvm_arch_vcpu_ioctl(struct file *filp,
283 unsigned int ioctl, unsigned long arg)
284 {
285 struct kvm_vcpu *vcpu = filp->private_data;
286 void __user *argp = (void __user *)arg;
287 long r = -EINVAL;
288
289 switch (ioctl) {
290 case KVM_SET_ONE_REG:
291 case KVM_GET_ONE_REG: {
292 struct kvm_one_reg reg;
293
294 r = -EFAULT;
295 if (copy_from_user(®, argp, sizeof(reg)))
296 break;
297
298 if (ioctl == KVM_SET_ONE_REG)
299 r = kvm_riscv_vcpu_set_reg(vcpu, ®);
300 else
301 r = kvm_riscv_vcpu_get_reg(vcpu, ®);
302 break;
303 }
304 case KVM_GET_REG_LIST: {
305 struct kvm_reg_list __user *user_list = argp;
306 struct kvm_reg_list reg_list;
307 unsigned int n;
308
309 r = -EFAULT;
310 if (copy_from_user(®_list, user_list, sizeof(reg_list)))
311 break;
312 n = reg_list.n;
313 reg_list.n = kvm_riscv_vcpu_num_regs(vcpu);
314 if (copy_to_user(user_list, ®_list, sizeof(reg_list)))
315 break;
316 r = -E2BIG;
317 if (n < reg_list.n)
318 break;
319 r = kvm_riscv_vcpu_copy_reg_indices(vcpu, user_list->reg);
320 break;
321 }
322 default:
323 break;
324 }
325
326 return r;
327 }
328
kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)329 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
330 struct kvm_sregs *sregs)
331 {
332 return -EINVAL;
333 }
334
kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)335 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
336 struct kvm_sregs *sregs)
337 {
338 return -EINVAL;
339 }
340
kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)341 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
342 {
343 return -EINVAL;
344 }
345
kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)346 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
347 {
348 return -EINVAL;
349 }
350
kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu * vcpu,struct kvm_translation * tr)351 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
352 struct kvm_translation *tr)
353 {
354 return -EINVAL;
355 }
356
kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)357 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
358 {
359 return -EINVAL;
360 }
361
kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)362 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
363 {
364 return -EINVAL;
365 }
366
kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu * vcpu)367 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
368 {
369 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
370 unsigned long mask, val;
371 unsigned long flags;
372
373 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
374
375 mask = vcpu->arch.irqs_pending_mask[0];
376 if (mask) {
377 vcpu->arch.irqs_pending_mask[0] = 0;
378 val = vcpu->arch.irqs_pending[0] & mask;
379
380 csr->hvip &= ~mask;
381 csr->hvip |= val;
382 }
383
384 /* Flush AIA high interrupts */
385 kvm_riscv_vcpu_aia_flush_interrupts(vcpu);
386
387 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
388 }
389
kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu * vcpu)390 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
391 {
392 unsigned long hvip;
393 unsigned long flags;
394 struct kvm_vcpu_arch *v = &vcpu->arch;
395 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
396
397 /* Read current HVIP and VSIE CSRs */
398 csr->vsie = ncsr_read(CSR_VSIE);
399
400 /* Sync-up HVIP.VSSIP bit changes does by Guest */
401 hvip = ncsr_read(CSR_HVIP);
402
403 raw_spin_lock_irqsave(&v->irqs_pending_lock, flags);
404
405 if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) {
406 if (hvip & (1UL << IRQ_VS_SOFT)) {
407 if (!__test_and_set_bit(IRQ_VS_SOFT,
408 v->irqs_pending_mask))
409 __set_bit(IRQ_VS_SOFT, v->irqs_pending);
410 } else {
411 if (!__test_and_set_bit(IRQ_VS_SOFT,
412 v->irqs_pending_mask))
413 __clear_bit(IRQ_VS_SOFT, v->irqs_pending);
414 }
415 }
416
417 /* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */
418 if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) {
419 if (!(hvip & (1UL << IRQ_PMU_OVF)) &&
420 !__test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask))
421 __clear_bit(IRQ_PMU_OVF, v->irqs_pending);
422 }
423
424 /* Sync-up AIA high interrupts */
425 kvm_riscv_vcpu_aia_sync_interrupts(vcpu);
426
427 raw_spin_unlock_irqrestore(&v->irqs_pending_lock, flags);
428
429 /* Sync-up timer CSRs */
430 kvm_riscv_vcpu_timer_sync(vcpu);
431 }
432
kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu * vcpu,unsigned int irq)433 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
434 {
435 unsigned long flags;
436
437 /*
438 * We only allow VS-mode software, timer, and external
439 * interrupts when irq is one of the local interrupts
440 * defined by RISC-V privilege specification.
441 */
442 if (irq < IRQ_LOCAL_MAX &&
443 irq != IRQ_VS_SOFT &&
444 irq != IRQ_VS_TIMER &&
445 irq != IRQ_VS_EXT &&
446 irq != IRQ_PMU_OVF)
447 return -EINVAL;
448
449 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
450 __set_bit(irq, vcpu->arch.irqs_pending);
451 __set_bit(irq, vcpu->arch.irqs_pending_mask);
452 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
453
454 trace_kvm_vcpu_irq(vcpu->vcpu_id, irq, 1);
455
456 kvm_vcpu_kick(vcpu);
457
458 return 0;
459 }
460
kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu * vcpu,unsigned int irq)461 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
462 {
463 unsigned long flags;
464
465 /*
466 * We only allow VS-mode software, timer, counter overflow and external
467 * interrupts when irq is one of the local interrupts
468 * defined by RISC-V privilege specification.
469 */
470 if (irq < IRQ_LOCAL_MAX &&
471 irq != IRQ_VS_SOFT &&
472 irq != IRQ_VS_TIMER &&
473 irq != IRQ_VS_EXT &&
474 irq != IRQ_PMU_OVF)
475 return -EINVAL;
476
477 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
478 __clear_bit(irq, vcpu->arch.irqs_pending);
479 __set_bit(irq, vcpu->arch.irqs_pending_mask);
480 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
481
482 trace_kvm_vcpu_irq(vcpu->vcpu_id, irq, 0);
483
484 return 0;
485 }
486
kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu * vcpu,u64 mask)487 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
488 {
489 unsigned long flags;
490 unsigned long ie;
491 bool ret;
492
493 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
494 ie = vsip_to_hvip(vcpu->arch.guest_csr.vsie) & (unsigned long)mask;
495 ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK &
496 (unsigned long)mask;
497 ret = vcpu->arch.irqs_pending[0] & ie;
498 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
499
500 /* Check AIA high interrupts */
501 if (!ret)
502 ret = kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);
503
504 return ret;
505 }
506
__kvm_riscv_vcpu_power_off(struct kvm_vcpu * vcpu)507 void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
508 {
509 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
510 kvm_make_request(KVM_REQ_SLEEP, vcpu);
511 kvm_vcpu_kick(vcpu);
512 }
513
kvm_riscv_vcpu_power_off(struct kvm_vcpu * vcpu)514 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
515 {
516 spin_lock(&vcpu->arch.mp_state_lock);
517 __kvm_riscv_vcpu_power_off(vcpu);
518 spin_unlock(&vcpu->arch.mp_state_lock);
519 }
520
__kvm_riscv_vcpu_power_on(struct kvm_vcpu * vcpu)521 void __kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu)
522 {
523 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
524 kvm_vcpu_wake_up(vcpu);
525 }
526
kvm_riscv_vcpu_power_on(struct kvm_vcpu * vcpu)527 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu)
528 {
529 spin_lock(&vcpu->arch.mp_state_lock);
530 __kvm_riscv_vcpu_power_on(vcpu);
531 spin_unlock(&vcpu->arch.mp_state_lock);
532 }
533
kvm_riscv_vcpu_stopped(struct kvm_vcpu * vcpu)534 bool kvm_riscv_vcpu_stopped(struct kvm_vcpu *vcpu)
535 {
536 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
537 }
538
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)539 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
540 struct kvm_mp_state *mp_state)
541 {
542 *mp_state = READ_ONCE(vcpu->arch.mp_state);
543
544 return 0;
545 }
546
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)547 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
548 struct kvm_mp_state *mp_state)
549 {
550 int ret = 0;
551
552 spin_lock(&vcpu->arch.mp_state_lock);
553
554 switch (mp_state->mp_state) {
555 case KVM_MP_STATE_RUNNABLE:
556 WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
557 break;
558 case KVM_MP_STATE_STOPPED:
559 __kvm_riscv_vcpu_power_off(vcpu);
560 break;
561 case KVM_MP_STATE_INIT_RECEIVED:
562 if (vcpu->kvm->arch.mp_state_reset)
563 kvm_riscv_reset_vcpu(vcpu, false);
564 else
565 ret = -EINVAL;
566 break;
567 default:
568 ret = -EINVAL;
569 }
570
571 spin_unlock(&vcpu->arch.mp_state_lock);
572
573 return ret;
574 }
575
kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu * vcpu,struct kvm_guest_debug * dbg)576 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
577 struct kvm_guest_debug *dbg)
578 {
579 if (dbg->control & KVM_GUESTDBG_ENABLE)
580 vcpu->guest_debug = dbg->control;
581 else
582 vcpu->guest_debug = 0;
583
584 kvm_riscv_vcpu_config_guest_debug(vcpu);
585 return 0;
586 }
587
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)588 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
589 {
590 void *nsh;
591 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
592
593 /*
594 * If VCPU is being reloaded on the same physical CPU and no
595 * other KVM VCPU has run on this CPU since it was last put,
596 * we can skip the expensive CSR and HGATP writes.
597 *
598 * Note: If a new CSR is added to this fast-path skip block,
599 * make sure that 'csr_dirty' is set to true in any
600 * ioctl (e.g., KVM_SET_ONE_REG) that modifies it.
601 */
602 if (vcpu != __this_cpu_read(kvm_former_vcpu))
603 __this_cpu_write(kvm_former_vcpu, vcpu);
604 else if (vcpu->arch.last_exit_cpu == cpu && !vcpu->arch.csr_dirty)
605 goto csr_restore_done;
606
607 vcpu->arch.csr_dirty = false;
608
609 /*
610 * Load VCPU config CSRs before other CSRs because
611 * the read/write behaviour of certain CSRs change
612 * based on VCPU config CSRs.
613 */
614 kvm_riscv_vcpu_config_load(vcpu);
615
616 if (kvm_riscv_nacl_sync_csr_available()) {
617 nsh = nacl_shmem();
618 nacl_csr_write(nsh, CSR_VSSTATUS, csr->vsstatus);
619 nacl_csr_write(nsh, CSR_VSIE, csr->vsie);
620 nacl_csr_write(nsh, CSR_VSTVEC, csr->vstvec);
621 nacl_csr_write(nsh, CSR_VSSCRATCH, csr->vsscratch);
622 nacl_csr_write(nsh, CSR_VSEPC, csr->vsepc);
623 nacl_csr_write(nsh, CSR_VSCAUSE, csr->vscause);
624 nacl_csr_write(nsh, CSR_VSTVAL, csr->vstval);
625 nacl_csr_write(nsh, CSR_HVIP, csr->hvip);
626 nacl_csr_write(nsh, CSR_VSATP, csr->vsatp);
627 } else {
628 csr_write(CSR_VSSTATUS, csr->vsstatus);
629 csr_write(CSR_VSIE, csr->vsie);
630 csr_write(CSR_VSTVEC, csr->vstvec);
631 csr_write(CSR_VSSCRATCH, csr->vsscratch);
632 csr_write(CSR_VSEPC, csr->vsepc);
633 csr_write(CSR_VSCAUSE, csr->vscause);
634 csr_write(CSR_VSTVAL, csr->vstval);
635 csr_write(CSR_HVIP, csr->hvip);
636 csr_write(CSR_VSATP, csr->vsatp);
637 }
638
639 kvm_riscv_mmu_update_hgatp(vcpu);
640
641 kvm_riscv_vcpu_aia_load(vcpu, cpu);
642
643 csr_restore_done:
644 kvm_riscv_vcpu_timer_restore(vcpu);
645
646 kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context);
647 kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context,
648 vcpu->arch.isa);
649 get_cpu_vector_context();
650 kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context);
651 kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context,
652 vcpu->arch.isa);
653 put_cpu_vector_context();
654
655 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
656
657 vcpu->cpu = cpu;
658 }
659
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)660 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
661 {
662 void *nsh;
663 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
664
665 vcpu->cpu = -1;
666
667 kvm_riscv_vcpu_aia_put(vcpu);
668
669 kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context,
670 vcpu->arch.isa);
671 kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context);
672
673 kvm_riscv_vcpu_timer_save(vcpu);
674 get_cpu_vector_context();
675 kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context,
676 vcpu->arch.isa);
677 kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context);
678 put_cpu_vector_context();
679
680 if (kvm_riscv_nacl_available()) {
681 nsh = nacl_shmem();
682 csr->vsstatus = nacl_csr_read(nsh, CSR_VSSTATUS);
683 csr->vsie = nacl_csr_read(nsh, CSR_VSIE);
684 csr->vstvec = nacl_csr_read(nsh, CSR_VSTVEC);
685 csr->vsscratch = nacl_csr_read(nsh, CSR_VSSCRATCH);
686 csr->vsepc = nacl_csr_read(nsh, CSR_VSEPC);
687 csr->vscause = nacl_csr_read(nsh, CSR_VSCAUSE);
688 csr->vstval = nacl_csr_read(nsh, CSR_VSTVAL);
689 csr->hvip = nacl_csr_read(nsh, CSR_HVIP);
690 csr->vsatp = nacl_csr_read(nsh, CSR_VSATP);
691 } else {
692 csr->vsstatus = csr_read(CSR_VSSTATUS);
693 csr->vsie = csr_read(CSR_VSIE);
694 csr->vstvec = csr_read(CSR_VSTVEC);
695 csr->vsscratch = csr_read(CSR_VSSCRATCH);
696 csr->vsepc = csr_read(CSR_VSEPC);
697 csr->vscause = csr_read(CSR_VSCAUSE);
698 csr->vstval = csr_read(CSR_VSTVAL);
699 csr->hvip = csr_read(CSR_HVIP);
700 csr->vsatp = csr_read(CSR_VSATP);
701 }
702 }
703
704 /**
705 * kvm_riscv_check_vcpu_requests - check and handle pending vCPU requests
706 * @vcpu: the VCPU pointer
707 *
708 * Return: 1 if we should enter the guest
709 * 0 if we should exit to userspace
710 */
kvm_riscv_check_vcpu_requests(struct kvm_vcpu * vcpu)711 static int kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
712 {
713 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
714
715 if (kvm_request_pending(vcpu)) {
716 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) {
717 kvm_vcpu_srcu_read_unlock(vcpu);
718 rcuwait_wait_event(wait,
719 (!kvm_riscv_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
720 TASK_INTERRUPTIBLE);
721 kvm_vcpu_srcu_read_lock(vcpu);
722
723 if (kvm_riscv_vcpu_stopped(vcpu) || vcpu->arch.pause) {
724 /*
725 * Awaken to handle a signal, request to
726 * sleep again later.
727 */
728 kvm_make_request(KVM_REQ_SLEEP, vcpu);
729 }
730 }
731
732 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
733 kvm_riscv_reset_vcpu(vcpu, true);
734
735 if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))
736 kvm_riscv_mmu_update_hgatp(vcpu);
737
738 if (kvm_check_request(KVM_REQ_FENCE_I, vcpu))
739 kvm_riscv_fence_i_process(vcpu);
740
741 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
742 kvm_riscv_tlb_flush_process(vcpu);
743
744 if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu))
745 kvm_riscv_hfence_vvma_all_process(vcpu);
746
747 if (kvm_check_request(KVM_REQ_HFENCE, vcpu))
748 kvm_riscv_hfence_process(vcpu);
749
750 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
751 kvm_riscv_vcpu_record_steal_time(vcpu);
752
753 if (kvm_dirty_ring_check_request(vcpu))
754 return 0;
755 }
756
757 return 1;
758 }
759
kvm_riscv_update_hvip(struct kvm_vcpu * vcpu)760 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu)
761 {
762 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
763
764 ncsr_write(CSR_HVIP, csr->hvip);
765 kvm_riscv_vcpu_aia_update_hvip(vcpu);
766 }
767
kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu * vcpu)768 static __always_inline void kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu *vcpu)
769 {
770 struct kvm_vcpu_zicfiss_csr *zicficsr = &vcpu->arch.zicfiss_csr;
771 struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr;
772 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
773
774 vcpu->arch.host_scounteren = csr_swap(CSR_SCOUNTEREN, csr->scounteren);
775 vcpu->arch.host_senvcfg = csr_swap(CSR_SENVCFG, csr->senvcfg);
776 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN))
777 vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0, smcsr->sstateen0);
778 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFISS))
779 csr_write(CSR_SSP, zicficsr->ssp);
780 }
781
kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu * vcpu)782 static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *vcpu)
783 {
784 struct kvm_vcpu_zicfiss_csr *zicficsr = &vcpu->arch.zicfiss_csr;
785 struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr;
786 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
787
788 csr->scounteren = csr_swap(CSR_SCOUNTEREN, vcpu->arch.host_scounteren);
789 csr->senvcfg = csr_swap(CSR_SENVCFG, vcpu->arch.host_senvcfg);
790 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN))
791 smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0, vcpu->arch.host_sstateen0);
792 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFISS))
793 zicficsr->ssp = csr_swap(CSR_SSP, 0);
794 }
795
796 /*
797 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
798 * the vCPU is running.
799 *
800 * This must be noinstr as instrumentation may make use of RCU, and this is not
801 * safe during the EQS.
802 */
kvm_riscv_vcpu_enter_exit(struct kvm_vcpu * vcpu,struct kvm_cpu_trap * trap)803 static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu,
804 struct kvm_cpu_trap *trap)
805 {
806 void *nsh;
807 struct kvm_cpu_context *gcntx = &vcpu->arch.guest_context;
808 struct kvm_cpu_context *hcntx = &vcpu->arch.host_context;
809
810 /*
811 * We save trap CSRs (such as SEPC, SCAUSE, STVAL, HTVAL, and
812 * HTINST) here because we do local_irq_enable() after this
813 * function in kvm_arch_vcpu_ioctl_run() which can result in
814 * an interrupt immediately after local_irq_enable() and can
815 * potentially change trap CSRs.
816 */
817
818 kvm_riscv_vcpu_swap_in_guest_state(vcpu);
819 guest_state_enter_irqoff();
820
821 /* sstatus.VS != SR_VS_OFF is guaranteed when NEED_RESTORE is set */
822 if (current->thread.riscv_v_flags & RISCV_V_VCPU_NEED_RESTORE) {
823 current->thread.riscv_v_flags &= ~RISCV_V_VCPU_NEED_RESTORE;
824 current->thread.riscv_v_flags |= RISCV_V_VCPU_CTX;
825 __kvm_riscv_vector_restore(gcntx);
826 gcntx->sstatus = (gcntx->sstatus & ~SR_VS) | SR_VS_CLEAN;
827 }
828
829 if (kvm_riscv_nacl_sync_sret_available()) {
830 nsh = nacl_shmem();
831
832 if (kvm_riscv_nacl_autoswap_csr_available()) {
833 hcntx->hstatus =
834 nacl_csr_read(nsh, CSR_HSTATUS);
835 nacl_scratch_write_long(nsh,
836 SBI_NACL_SHMEM_AUTOSWAP_OFFSET +
837 SBI_NACL_SHMEM_AUTOSWAP_HSTATUS,
838 gcntx->hstatus);
839 nacl_scratch_write_long(nsh,
840 SBI_NACL_SHMEM_AUTOSWAP_OFFSET,
841 SBI_NACL_SHMEM_AUTOSWAP_FLAG_HSTATUS);
842 } else if (kvm_riscv_nacl_sync_csr_available()) {
843 hcntx->hstatus = nacl_csr_swap(nsh,
844 CSR_HSTATUS, gcntx->hstatus);
845 } else {
846 hcntx->hstatus = csr_swap(CSR_HSTATUS, gcntx->hstatus);
847 }
848
849 nacl_scratch_write_longs(nsh,
850 SBI_NACL_SHMEM_SRET_OFFSET +
851 SBI_NACL_SHMEM_SRET_X(1),
852 &gcntx->ra,
853 SBI_NACL_SHMEM_SRET_X_LAST);
854
855 __kvm_riscv_nacl_switch_to(&vcpu->arch, SBI_EXT_NACL,
856 SBI_EXT_NACL_SYNC_SRET);
857
858 if (kvm_riscv_nacl_autoswap_csr_available()) {
859 nacl_scratch_write_long(nsh,
860 SBI_NACL_SHMEM_AUTOSWAP_OFFSET,
861 0);
862 gcntx->hstatus = nacl_scratch_read_long(nsh,
863 SBI_NACL_SHMEM_AUTOSWAP_OFFSET +
864 SBI_NACL_SHMEM_AUTOSWAP_HSTATUS);
865 } else {
866 gcntx->hstatus = csr_swap(CSR_HSTATUS, hcntx->hstatus);
867 }
868
869 trap->htval = nacl_csr_read(nsh, CSR_HTVAL);
870 trap->htinst = nacl_csr_read(nsh, CSR_HTINST);
871 } else {
872 hcntx->hstatus = csr_swap(CSR_HSTATUS, gcntx->hstatus);
873
874 __kvm_riscv_switch_to(&vcpu->arch);
875
876 gcntx->hstatus = csr_swap(CSR_HSTATUS, hcntx->hstatus);
877
878 trap->htval = csr_read(CSR_HTVAL);
879 trap->htinst = csr_read(CSR_HTINST);
880 }
881
882 trap->sepc = gcntx->sepc;
883 trap->scause = csr_read(CSR_SCAUSE);
884 trap->stval = csr_read(CSR_STVAL);
885
886 vcpu->arch.last_exit_cpu = vcpu->cpu;
887 guest_state_exit_irqoff();
888 kvm_riscv_vcpu_swap_in_host_state(vcpu);
889 }
890
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)891 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
892 {
893 int ret;
894 struct kvm_cpu_trap trap;
895 struct kvm_run *run = vcpu->run;
896
897 if (!vcpu->arch.ran_atleast_once)
898 kvm_riscv_vcpu_config_ran_once(vcpu);
899
900 /* Mark this VCPU ran at least once */
901 vcpu->arch.ran_atleast_once = true;
902
903 kvm_vcpu_srcu_read_lock(vcpu);
904
905 switch (run->exit_reason) {
906 case KVM_EXIT_MMIO:
907 /* Process MMIO value returned from user-space */
908 ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run);
909 break;
910 case KVM_EXIT_RISCV_SBI:
911 /* Process SBI value returned from user-space */
912 ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run);
913 break;
914 case KVM_EXIT_RISCV_CSR:
915 /* Process CSR value returned from user-space */
916 ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run);
917 break;
918 default:
919 ret = 0;
920 break;
921 }
922 if (ret) {
923 kvm_vcpu_srcu_read_unlock(vcpu);
924 return ret;
925 }
926
927 if (!vcpu->wants_to_run) {
928 kvm_vcpu_srcu_read_unlock(vcpu);
929 return -EINTR;
930 }
931
932 vcpu_load(vcpu);
933
934 kvm_sigset_activate(vcpu);
935
936 ret = 1;
937 run->exit_reason = KVM_EXIT_UNKNOWN;
938 while (ret > 0) {
939 /* Check conditions before entering the guest */
940 ret = kvm_xfer_to_guest_mode_handle_work(vcpu);
941 if (ret)
942 continue;
943 ret = 1;
944
945 kvm_riscv_gstage_vmid_update(vcpu);
946
947 ret = kvm_riscv_check_vcpu_requests(vcpu);
948 if (ret <= 0)
949 continue;
950
951 preempt_disable();
952
953 /* Update AIA HW state before entering guest */
954 ret = kvm_riscv_vcpu_aia_update(vcpu);
955 if (ret <= 0) {
956 preempt_enable();
957 continue;
958 }
959
960 local_irq_disable();
961
962 /*
963 * Ensure we set mode to IN_GUEST_MODE after we disable
964 * interrupts and before the final VCPU requests check.
965 * See the comment in kvm_vcpu_exiting_guest_mode() and
966 * Documentation/virt/kvm/vcpu-requests.rst
967 */
968 vcpu->mode = IN_GUEST_MODE;
969
970 kvm_vcpu_srcu_read_unlock(vcpu);
971 smp_mb__after_srcu_read_unlock();
972
973 /*
974 * We might have got VCPU interrupts updated asynchronously
975 * so update it in HW.
976 */
977 kvm_riscv_vcpu_flush_interrupts(vcpu);
978
979 /* Update HVIP CSR for current CPU */
980 kvm_riscv_update_hvip(vcpu);
981
982 if (kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
983 kvm_request_pending(vcpu) ||
984 xfer_to_guest_mode_work_pending()) {
985 vcpu->mode = OUTSIDE_GUEST_MODE;
986 local_irq_enable();
987 preempt_enable();
988 kvm_vcpu_srcu_read_lock(vcpu);
989 continue;
990 }
991
992 /*
993 * Sanitize VMID mappings cached (TLB) on current CPU
994 *
995 * Note: This should be done after G-stage VMID has been
996 * updated using kvm_riscv_gstage_vmid_ver_changed()
997 */
998 kvm_riscv_local_tlb_sanitize(vcpu);
999
1000 trace_kvm_entry(vcpu);
1001
1002 guest_timing_enter_irqoff();
1003
1004 kvm_riscv_vcpu_enter_exit(vcpu, &trap);
1005
1006 vcpu->mode = OUTSIDE_GUEST_MODE;
1007 vcpu->stat.exits++;
1008
1009 /* Syncup interrupts state with HW */
1010 kvm_riscv_vcpu_sync_interrupts(vcpu);
1011
1012 /*
1013 * We must ensure that any pending interrupts are taken before
1014 * we exit guest timing so that timer ticks are accounted as
1015 * guest time. Transiently unmask interrupts so that any
1016 * pending interrupts are taken.
1017 *
1018 * There's no barrier which ensures that pending interrupts are
1019 * recognised, so we just hope that the CPU takes any pending
1020 * interrupts between the enable and disable.
1021 */
1022 local_irq_enable();
1023 local_irq_disable();
1024
1025 guest_timing_exit_irqoff();
1026
1027 local_irq_enable();
1028
1029 trace_kvm_exit(&trap);
1030
1031 preempt_enable();
1032
1033 kvm_vcpu_srcu_read_lock(vcpu);
1034
1035 ret = kvm_riscv_vcpu_exit(vcpu, run, &trap);
1036 }
1037
1038 kvm_sigset_deactivate(vcpu);
1039
1040 vcpu_put(vcpu);
1041
1042 kvm_vcpu_srcu_read_unlock(vcpu);
1043
1044 return ret;
1045 }
1046