1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4 * Copyright (c) 2022 Ventana Micro Systems Inc.
5 */
6
7 #include <linux/bitops.h>
8 #include <linux/kvm_host.h>
9
10 #include <asm/cpufeature.h>
11 #include <asm/insn.h>
12
13 struct insn_func {
14 unsigned long mask;
15 unsigned long match;
16 /*
17 * Possible return values are as follows:
18 * 1) Returns < 0 for error case
19 * 2) Returns 0 for exit to user-space
20 * 3) Returns 1 to continue with next sepc
21 * 4) Returns 2 to continue with same sepc
22 * 5) Returns 3 to inject illegal instruction trap and continue
23 * 6) Returns 4 to inject virtual instruction trap and continue
24 *
25 * Use enum kvm_insn_return for return values
26 */
27 int (*func)(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn);
28 };
29
truly_illegal_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)30 static int truly_illegal_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
31 ulong insn)
32 {
33 struct kvm_cpu_trap utrap = { 0 };
34
35 /* Redirect trap to Guest VCPU */
36 utrap.sepc = vcpu->arch.guest_context.sepc;
37 utrap.scause = EXC_INST_ILLEGAL;
38 utrap.stval = insn;
39 utrap.htval = 0;
40 utrap.htinst = 0;
41 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
42
43 return 1;
44 }
45
truly_virtual_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)46 static int truly_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
47 ulong insn)
48 {
49 struct kvm_cpu_trap utrap = { 0 };
50
51 /* Redirect trap to Guest VCPU */
52 utrap.sepc = vcpu->arch.guest_context.sepc;
53 utrap.scause = EXC_VIRTUAL_INST_FAULT;
54 utrap.stval = insn;
55 utrap.htval = 0;
56 utrap.htinst = 0;
57 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
58
59 return 1;
60 }
61
62 /**
63 * kvm_riscv_vcpu_wfi -- Emulate wait for interrupt (WFI) behaviour
64 *
65 * @vcpu: The VCPU pointer
66 */
kvm_riscv_vcpu_wfi(struct kvm_vcpu * vcpu)67 void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu)
68 {
69 if (!kvm_arch_vcpu_runnable(vcpu)) {
70 kvm_vcpu_srcu_read_unlock(vcpu);
71 kvm_vcpu_halt(vcpu);
72 kvm_vcpu_srcu_read_lock(vcpu);
73 }
74 }
75
wfi_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)76 static int wfi_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
77 {
78 vcpu->stat.wfi_exit_stat++;
79 kvm_riscv_vcpu_wfi(vcpu);
80 return KVM_INSN_CONTINUE_NEXT_SEPC;
81 }
82
wrs_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)83 static int wrs_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
84 {
85 vcpu->stat.wrs_exit_stat++;
86 kvm_vcpu_on_spin(vcpu, vcpu->arch.guest_context.sstatus & SR_SPP);
87 return KVM_INSN_CONTINUE_NEXT_SEPC;
88 }
89
90 struct csr_func {
91 unsigned int base;
92 unsigned int count;
93 /*
94 * Possible return values are as same as "func" callback in
95 * "struct insn_func".
96 */
97 int (*func)(struct kvm_vcpu *vcpu, unsigned int csr_num,
98 unsigned long *val, unsigned long new_val,
99 unsigned long wr_mask);
100 };
101
seed_csr_rmw(struct kvm_vcpu * vcpu,unsigned int csr_num,unsigned long * val,unsigned long new_val,unsigned long wr_mask)102 static int seed_csr_rmw(struct kvm_vcpu *vcpu, unsigned int csr_num,
103 unsigned long *val, unsigned long new_val,
104 unsigned long wr_mask)
105 {
106 if (!riscv_isa_extension_available(vcpu->arch.isa, ZKR))
107 return KVM_INSN_ILLEGAL_TRAP;
108
109 return KVM_INSN_EXIT_TO_USER_SPACE;
110 }
111
112 static const struct csr_func csr_funcs[] = {
113 KVM_RISCV_VCPU_AIA_CSR_FUNCS
114 KVM_RISCV_VCPU_HPMCOUNTER_CSR_FUNCS
115 { .base = CSR_SEED, .count = 1, .func = seed_csr_rmw },
116 };
117
118 /**
119 * kvm_riscv_vcpu_csr_return -- Handle CSR read/write after user space
120 * emulation or in-kernel emulation
121 *
122 * @vcpu: The VCPU pointer
123 * @run: The VCPU run struct containing the CSR data
124 *
125 * Returns > 0 upon failure and 0 upon success
126 */
kvm_riscv_vcpu_csr_return(struct kvm_vcpu * vcpu,struct kvm_run * run)127 int kvm_riscv_vcpu_csr_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
128 {
129 ulong insn;
130
131 if (vcpu->arch.csr_decode.return_handled)
132 return 0;
133 vcpu->arch.csr_decode.return_handled = 1;
134
135 /* Update destination register for CSR reads */
136 insn = vcpu->arch.csr_decode.insn;
137 if ((insn >> SH_RD) & MASK_RX)
138 SET_RD(insn, &vcpu->arch.guest_context,
139 run->riscv_csr.ret_value);
140
141 /* Move to next instruction */
142 vcpu->arch.guest_context.sepc += INSN_LEN(insn);
143
144 return 0;
145 }
146
csr_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)147 static int csr_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
148 {
149 int i, rc = KVM_INSN_ILLEGAL_TRAP;
150 unsigned int csr_num = insn >> SH_RS2;
151 unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
152 ulong rs1_val = GET_RS1(insn, &vcpu->arch.guest_context);
153 const struct csr_func *tcfn, *cfn = NULL;
154 ulong val = 0, wr_mask = 0, new_val = 0;
155
156 /* Decode the CSR instruction */
157 switch (GET_FUNCT3(insn)) {
158 case GET_FUNCT3(INSN_MATCH_CSRRW):
159 wr_mask = -1UL;
160 new_val = rs1_val;
161 break;
162 case GET_FUNCT3(INSN_MATCH_CSRRS):
163 wr_mask = rs1_val;
164 new_val = -1UL;
165 break;
166 case GET_FUNCT3(INSN_MATCH_CSRRC):
167 wr_mask = rs1_val;
168 new_val = 0;
169 break;
170 case GET_FUNCT3(INSN_MATCH_CSRRWI):
171 wr_mask = -1UL;
172 new_val = rs1_num;
173 break;
174 case GET_FUNCT3(INSN_MATCH_CSRRSI):
175 wr_mask = rs1_num;
176 new_val = -1UL;
177 break;
178 case GET_FUNCT3(INSN_MATCH_CSRRCI):
179 wr_mask = rs1_num;
180 new_val = 0;
181 break;
182 default:
183 return rc;
184 }
185
186 /* Save instruction decode info */
187 vcpu->arch.csr_decode.insn = insn;
188 vcpu->arch.csr_decode.return_handled = 0;
189
190 /* Update CSR details in kvm_run struct */
191 run->riscv_csr.csr_num = csr_num;
192 run->riscv_csr.new_value = new_val;
193 run->riscv_csr.write_mask = wr_mask;
194 run->riscv_csr.ret_value = 0;
195
196 /* Find in-kernel CSR function */
197 for (i = 0; i < ARRAY_SIZE(csr_funcs); i++) {
198 tcfn = &csr_funcs[i];
199 if ((tcfn->base <= csr_num) &&
200 (csr_num < (tcfn->base + tcfn->count))) {
201 cfn = tcfn;
202 break;
203 }
204 }
205
206 /* First try in-kernel CSR emulation */
207 if (cfn && cfn->func) {
208 rc = cfn->func(vcpu, csr_num, &val, new_val, wr_mask);
209 if (rc > KVM_INSN_EXIT_TO_USER_SPACE) {
210 if (rc == KVM_INSN_CONTINUE_NEXT_SEPC) {
211 run->riscv_csr.ret_value = val;
212 vcpu->stat.csr_exit_kernel++;
213 kvm_riscv_vcpu_csr_return(vcpu, run);
214 rc = KVM_INSN_CONTINUE_SAME_SEPC;
215 }
216 return rc;
217 }
218 }
219
220 /* Exit to user-space for CSR emulation */
221 if (rc <= KVM_INSN_EXIT_TO_USER_SPACE) {
222 vcpu->stat.csr_exit_user++;
223 run->exit_reason = KVM_EXIT_RISCV_CSR;
224 }
225
226 return rc;
227 }
228
229 static const struct insn_func system_opcode_funcs[] = {
230 {
231 .mask = INSN_MASK_CSRRW,
232 .match = INSN_MATCH_CSRRW,
233 .func = csr_insn,
234 },
235 {
236 .mask = INSN_MASK_CSRRS,
237 .match = INSN_MATCH_CSRRS,
238 .func = csr_insn,
239 },
240 {
241 .mask = INSN_MASK_CSRRC,
242 .match = INSN_MATCH_CSRRC,
243 .func = csr_insn,
244 },
245 {
246 .mask = INSN_MASK_CSRRWI,
247 .match = INSN_MATCH_CSRRWI,
248 .func = csr_insn,
249 },
250 {
251 .mask = INSN_MASK_CSRRSI,
252 .match = INSN_MATCH_CSRRSI,
253 .func = csr_insn,
254 },
255 {
256 .mask = INSN_MASK_CSRRCI,
257 .match = INSN_MATCH_CSRRCI,
258 .func = csr_insn,
259 },
260 {
261 .mask = INSN_MASK_WFI,
262 .match = INSN_MATCH_WFI,
263 .func = wfi_insn,
264 },
265 {
266 .mask = INSN_MASK_WRS,
267 .match = INSN_MATCH_WRS,
268 .func = wrs_insn,
269 },
270 };
271
system_opcode_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)272 static int system_opcode_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
273 ulong insn)
274 {
275 int i, rc = KVM_INSN_ILLEGAL_TRAP;
276 const struct insn_func *ifn;
277
278 for (i = 0; i < ARRAY_SIZE(system_opcode_funcs); i++) {
279 ifn = &system_opcode_funcs[i];
280 if ((insn & ifn->mask) == ifn->match) {
281 rc = ifn->func(vcpu, run, insn);
282 break;
283 }
284 }
285
286 switch (rc) {
287 case KVM_INSN_ILLEGAL_TRAP:
288 return truly_illegal_insn(vcpu, run, insn);
289 case KVM_INSN_VIRTUAL_TRAP:
290 return truly_virtual_insn(vcpu, run, insn);
291 case KVM_INSN_CONTINUE_NEXT_SEPC:
292 vcpu->arch.guest_context.sepc += INSN_LEN(insn);
293 break;
294 default:
295 break;
296 }
297
298 return (rc <= 0) ? rc : 1;
299 }
300
is_load_guest_page_fault(unsigned long scause)301 static bool is_load_guest_page_fault(unsigned long scause)
302 {
303 /**
304 * If a g-stage page fault occurs, the direct approach
305 * is to let the g-stage page fault handler handle it
306 * naturally, however, calling the g-stage page fault
307 * handler here seems rather strange.
308 * Considering this is a corner case, we can directly
309 * return to the guest and re-execute the same PC, this
310 * will trigger a g-stage page fault again and then the
311 * regular g-stage page fault handler will populate
312 * g-stage page table.
313 */
314 return (scause == EXC_LOAD_GUEST_PAGE_FAULT);
315 }
316
317 /**
318 * kvm_riscv_vcpu_virtual_insn -- Handle virtual instruction trap
319 *
320 * @vcpu: The VCPU pointer
321 * @run: The VCPU run struct containing the mmio data
322 * @trap: Trap details
323 *
324 * Returns > 0 to continue run-loop
325 * Returns 0 to exit run-loop and handle in user-space.
326 * Returns < 0 to report failure and exit run-loop
327 */
kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,struct kvm_cpu_trap * trap)328 int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
329 struct kvm_cpu_trap *trap)
330 {
331 unsigned long insn = trap->stval;
332 struct kvm_cpu_trap utrap = { 0 };
333 struct kvm_cpu_context *ct;
334
335 if (unlikely(INSN_IS_16BIT(insn))) {
336 if (insn == 0) {
337 ct = &vcpu->arch.guest_context;
338 insn = kvm_riscv_vcpu_unpriv_read(vcpu, true,
339 ct->sepc,
340 &utrap);
341 if (utrap.scause) {
342 if (is_load_guest_page_fault(utrap.scause))
343 return 1;
344 utrap.sepc = ct->sepc;
345 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
346 return 1;
347 }
348 }
349 if (INSN_IS_16BIT(insn))
350 return truly_illegal_insn(vcpu, run, insn);
351 }
352
353 switch ((insn & INSN_OPCODE_MASK) >> INSN_OPCODE_SHIFT) {
354 case INSN_OPCODE_SYSTEM:
355 return system_opcode_insn(vcpu, run, insn);
356 default:
357 return truly_illegal_insn(vcpu, run, insn);
358 }
359 }
360
361 /**
362 * kvm_riscv_vcpu_mmio_load -- Emulate MMIO load instruction
363 *
364 * @vcpu: The VCPU pointer
365 * @run: The VCPU run struct containing the mmio data
366 * @fault_addr: Guest physical address to load
367 * @htinst: Transformed encoding of the load instruction
368 *
369 * Returns > 0 to continue run-loop
370 * Returns 0 to exit run-loop and handle in user-space.
371 * Returns < 0 to report failure and exit run-loop
372 */
kvm_riscv_vcpu_mmio_load(struct kvm_vcpu * vcpu,struct kvm_run * run,unsigned long fault_addr,unsigned long htinst)373 int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run,
374 unsigned long fault_addr,
375 unsigned long htinst)
376 {
377 u8 data_buf[8];
378 unsigned long insn;
379 int shift = 0, len = 0, insn_len = 0;
380 struct kvm_cpu_trap utrap = { 0 };
381 struct kvm_cpu_context *ct = &vcpu->arch.guest_context;
382
383 /* Determine trapped instruction */
384 if (htinst & 0x1) {
385 /*
386 * Bit[0] == 1 implies trapped instruction value is
387 * transformed instruction or custom instruction.
388 */
389 insn = htinst | INSN_16BIT_MASK;
390 insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
391 } else {
392 /*
393 * Bit[0] == 0 implies trapped instruction value is
394 * zero or special value.
395 */
396 insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
397 &utrap);
398 if (utrap.scause) {
399 if (is_load_guest_page_fault(utrap.scause))
400 return 1;
401 /* Redirect trap if we failed to read instruction */
402 utrap.sepc = ct->sepc;
403 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
404 return 1;
405 }
406 insn_len = INSN_LEN(insn);
407 }
408
409 /* Decode length of MMIO and shift */
410 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
411 len = 4;
412 shift = 8 * (sizeof(ulong) - len);
413 } else if ((insn & INSN_MASK_LB) == INSN_MATCH_LB) {
414 len = 1;
415 shift = 8 * (sizeof(ulong) - len);
416 } else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) {
417 len = 1;
418 #ifdef CONFIG_64BIT
419 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
420 len = 8;
421 shift = 8 * (sizeof(ulong) - len);
422 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
423 len = 4;
424 #endif
425 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
426 len = 2;
427 shift = 8 * (sizeof(ulong) - len);
428 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
429 len = 2;
430 #ifdef CONFIG_64BIT
431 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
432 len = 8;
433 shift = 8 * (sizeof(ulong) - len);
434 insn = RVC_RS2S(insn) << SH_RD;
435 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
436 ((insn >> SH_RD) & 0x1f)) {
437 len = 8;
438 shift = 8 * (sizeof(ulong) - len);
439 #endif
440 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
441 len = 4;
442 shift = 8 * (sizeof(ulong) - len);
443 insn = RVC_RS2S(insn) << SH_RD;
444 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
445 ((insn >> SH_RD) & 0x1f)) {
446 len = 4;
447 shift = 8 * (sizeof(ulong) - len);
448 } else {
449 return -EOPNOTSUPP;
450 }
451
452 /* Fault address should be aligned to length of MMIO */
453 if (fault_addr & (len - 1))
454 return -EIO;
455
456 /* Save instruction decode info */
457 vcpu->arch.mmio_decode.insn = insn;
458 vcpu->arch.mmio_decode.insn_len = insn_len;
459 vcpu->arch.mmio_decode.shift = shift;
460 vcpu->arch.mmio_decode.len = len;
461 vcpu->arch.mmio_decode.return_handled = 0;
462
463 /* Update MMIO details in kvm_run struct */
464 run->mmio.is_write = false;
465 run->mmio.phys_addr = fault_addr;
466 run->mmio.len = len;
467
468 /* Try to handle MMIO access in the kernel */
469 if (!kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_addr, len, data_buf)) {
470 /* Successfully handled MMIO access in the kernel so resume */
471 memcpy(run->mmio.data, data_buf, len);
472 vcpu->stat.mmio_exit_kernel++;
473 kvm_riscv_vcpu_mmio_return(vcpu, run);
474 return 1;
475 }
476
477 /* Exit to userspace for MMIO emulation */
478 vcpu->stat.mmio_exit_user++;
479 run->exit_reason = KVM_EXIT_MMIO;
480
481 return 0;
482 }
483
484 /**
485 * kvm_riscv_vcpu_mmio_store -- Emulate MMIO store instruction
486 *
487 * @vcpu: The VCPU pointer
488 * @run: The VCPU run struct containing the mmio data
489 * @fault_addr: Guest physical address to store
490 * @htinst: Transformed encoding of the store instruction
491 *
492 * Returns > 0 to continue run-loop
493 * Returns 0 to exit run-loop and handle in user-space.
494 * Returns < 0 to report failure and exit run-loop
495 */
kvm_riscv_vcpu_mmio_store(struct kvm_vcpu * vcpu,struct kvm_run * run,unsigned long fault_addr,unsigned long htinst)496 int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run,
497 unsigned long fault_addr,
498 unsigned long htinst)
499 {
500 u8 data8;
501 u16 data16;
502 u32 data32;
503 u64 data64;
504 ulong data;
505 unsigned long insn;
506 int len = 0, insn_len = 0;
507 struct kvm_cpu_trap utrap = { 0 };
508 struct kvm_cpu_context *ct = &vcpu->arch.guest_context;
509
510 /* Determine trapped instruction */
511 if (htinst & 0x1) {
512 /*
513 * Bit[0] == 1 implies trapped instruction value is
514 * transformed instruction or custom instruction.
515 */
516 insn = htinst | INSN_16BIT_MASK;
517 insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
518 } else {
519 /*
520 * Bit[0] == 0 implies trapped instruction value is
521 * zero or special value.
522 */
523 insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
524 &utrap);
525 if (utrap.scause) {
526 if (is_load_guest_page_fault(utrap.scause))
527 return 1;
528 /* Redirect trap if we failed to read instruction */
529 utrap.sepc = ct->sepc;
530 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
531 return 1;
532 }
533 insn_len = INSN_LEN(insn);
534 }
535
536 data = GET_RS2(insn, &vcpu->arch.guest_context);
537 data8 = data16 = data32 = data64 = data;
538
539 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
540 len = 4;
541 } else if ((insn & INSN_MASK_SB) == INSN_MATCH_SB) {
542 len = 1;
543 #ifdef CONFIG_64BIT
544 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
545 len = 8;
546 #endif
547 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
548 len = 2;
549 #ifdef CONFIG_64BIT
550 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
551 len = 8;
552 data64 = GET_RS2S(insn, &vcpu->arch.guest_context);
553 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP &&
554 ((insn >> SH_RD) & 0x1f)) {
555 len = 8;
556 data64 = GET_RS2C(insn, &vcpu->arch.guest_context);
557 #endif
558 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
559 len = 4;
560 data32 = GET_RS2S(insn, &vcpu->arch.guest_context);
561 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP &&
562 ((insn >> SH_RD) & 0x1f)) {
563 len = 4;
564 data32 = GET_RS2C(insn, &vcpu->arch.guest_context);
565 } else {
566 return -EOPNOTSUPP;
567 }
568
569 /* Fault address should be aligned to length of MMIO */
570 if (fault_addr & (len - 1))
571 return -EIO;
572
573 /* Save instruction decode info */
574 vcpu->arch.mmio_decode.insn = insn;
575 vcpu->arch.mmio_decode.insn_len = insn_len;
576 vcpu->arch.mmio_decode.shift = 0;
577 vcpu->arch.mmio_decode.len = len;
578 vcpu->arch.mmio_decode.return_handled = 0;
579
580 /* Copy data to kvm_run instance */
581 switch (len) {
582 case 1:
583 *((u8 *)run->mmio.data) = data8;
584 break;
585 case 2:
586 *((u16 *)run->mmio.data) = data16;
587 break;
588 case 4:
589 *((u32 *)run->mmio.data) = data32;
590 break;
591 case 8:
592 *((u64 *)run->mmio.data) = data64;
593 break;
594 default:
595 return -EOPNOTSUPP;
596 }
597
598 /* Update MMIO details in kvm_run struct */
599 run->mmio.is_write = true;
600 run->mmio.phys_addr = fault_addr;
601 run->mmio.len = len;
602
603 /* Try to handle MMIO access in the kernel */
604 if (!kvm_io_bus_write(vcpu, KVM_MMIO_BUS,
605 fault_addr, len, run->mmio.data)) {
606 /* Successfully handled MMIO access in the kernel so resume */
607 vcpu->stat.mmio_exit_kernel++;
608 kvm_riscv_vcpu_mmio_return(vcpu, run);
609 return 1;
610 }
611
612 /* Exit to userspace for MMIO emulation */
613 vcpu->stat.mmio_exit_user++;
614 run->exit_reason = KVM_EXIT_MMIO;
615
616 return 0;
617 }
618
619 /**
620 * kvm_riscv_vcpu_mmio_return -- Handle MMIO loads after user space emulation
621 * or in-kernel IO emulation
622 *
623 * @vcpu: The VCPU pointer
624 * @run: The VCPU run struct containing the mmio data
625 */
kvm_riscv_vcpu_mmio_return(struct kvm_vcpu * vcpu,struct kvm_run * run)626 int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
627 {
628 u8 data8;
629 u16 data16;
630 u32 data32;
631 u64 data64;
632 ulong insn;
633 int len, shift;
634
635 if (vcpu->arch.mmio_decode.return_handled)
636 return 0;
637
638 vcpu->arch.mmio_decode.return_handled = 1;
639 insn = vcpu->arch.mmio_decode.insn;
640
641 if (run->mmio.is_write)
642 goto done;
643
644 len = vcpu->arch.mmio_decode.len;
645 shift = vcpu->arch.mmio_decode.shift;
646
647 switch (len) {
648 case 1:
649 data8 = *((u8 *)run->mmio.data);
650 SET_RD(insn, &vcpu->arch.guest_context,
651 (long)((ulong)data8 << shift) >> shift);
652 break;
653 case 2:
654 data16 = *((u16 *)run->mmio.data);
655 SET_RD(insn, &vcpu->arch.guest_context,
656 (long)((ulong)data16 << shift) >> shift);
657 break;
658 case 4:
659 data32 = *((u32 *)run->mmio.data);
660 SET_RD(insn, &vcpu->arch.guest_context,
661 (long)((ulong)data32 << shift) >> shift);
662 break;
663 case 8:
664 data64 = *((u64 *)run->mmio.data);
665 SET_RD(insn, &vcpu->arch.guest_context,
666 (long)((ulong)data64 << shift) >> shift);
667 break;
668 default:
669 return -EOPNOTSUPP;
670 }
671
672 done:
673 /* Move to next instruction */
674 vcpu->arch.guest_context.sepc += vcpu->arch.mmio_decode.insn_len;
675
676 return 0;
677 }
678