1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4 * Copyright (c) 2022 Ventana Micro Systems Inc.
5 */
6
7 #include <linux/bitops.h>
8 #include <linux/kvm_host.h>
9
10 #include <asm/cpufeature.h>
11
12 #define INSN_OPCODE_MASK 0x007c
13 #define INSN_OPCODE_SHIFT 2
14 #define INSN_OPCODE_SYSTEM 28
15
16 #define INSN_MASK_WFI 0xffffffff
17 #define INSN_MATCH_WFI 0x10500073
18
19 #define INSN_MASK_WRS 0xffffffff
20 #define INSN_MATCH_WRS 0x00d00073
21
22 #define INSN_MATCH_CSRRW 0x1073
23 #define INSN_MASK_CSRRW 0x707f
24 #define INSN_MATCH_CSRRS 0x2073
25 #define INSN_MASK_CSRRS 0x707f
26 #define INSN_MATCH_CSRRC 0x3073
27 #define INSN_MASK_CSRRC 0x707f
28 #define INSN_MATCH_CSRRWI 0x5073
29 #define INSN_MASK_CSRRWI 0x707f
30 #define INSN_MATCH_CSRRSI 0x6073
31 #define INSN_MASK_CSRRSI 0x707f
32 #define INSN_MATCH_CSRRCI 0x7073
33 #define INSN_MASK_CSRRCI 0x707f
34
35 #define INSN_MATCH_LB 0x3
36 #define INSN_MASK_LB 0x707f
37 #define INSN_MATCH_LH 0x1003
38 #define INSN_MASK_LH 0x707f
39 #define INSN_MATCH_LW 0x2003
40 #define INSN_MASK_LW 0x707f
41 #define INSN_MATCH_LD 0x3003
42 #define INSN_MASK_LD 0x707f
43 #define INSN_MATCH_LBU 0x4003
44 #define INSN_MASK_LBU 0x707f
45 #define INSN_MATCH_LHU 0x5003
46 #define INSN_MASK_LHU 0x707f
47 #define INSN_MATCH_LWU 0x6003
48 #define INSN_MASK_LWU 0x707f
49 #define INSN_MATCH_SB 0x23
50 #define INSN_MASK_SB 0x707f
51 #define INSN_MATCH_SH 0x1023
52 #define INSN_MASK_SH 0x707f
53 #define INSN_MATCH_SW 0x2023
54 #define INSN_MASK_SW 0x707f
55 #define INSN_MATCH_SD 0x3023
56 #define INSN_MASK_SD 0x707f
57
58 #define INSN_MATCH_C_LD 0x6000
59 #define INSN_MASK_C_LD 0xe003
60 #define INSN_MATCH_C_SD 0xe000
61 #define INSN_MASK_C_SD 0xe003
62 #define INSN_MATCH_C_LW 0x4000
63 #define INSN_MASK_C_LW 0xe003
64 #define INSN_MATCH_C_SW 0xc000
65 #define INSN_MASK_C_SW 0xe003
66 #define INSN_MATCH_C_LDSP 0x6002
67 #define INSN_MASK_C_LDSP 0xe003
68 #define INSN_MATCH_C_SDSP 0xe002
69 #define INSN_MASK_C_SDSP 0xe003
70 #define INSN_MATCH_C_LWSP 0x4002
71 #define INSN_MASK_C_LWSP 0xe003
72 #define INSN_MATCH_C_SWSP 0xc002
73 #define INSN_MASK_C_SWSP 0xe003
74
75 #define INSN_16BIT_MASK 0x3
76
77 #define INSN_IS_16BIT(insn) (((insn) & INSN_16BIT_MASK) != INSN_16BIT_MASK)
78
79 #define INSN_LEN(insn) (INSN_IS_16BIT(insn) ? 2 : 4)
80
81 #ifdef CONFIG_64BIT
82 #define LOG_REGBYTES 3
83 #else
84 #define LOG_REGBYTES 2
85 #endif
86 #define REGBYTES (1 << LOG_REGBYTES)
87
88 #define SH_RD 7
89 #define SH_RS1 15
90 #define SH_RS2 20
91 #define SH_RS2C 2
92 #define MASK_RX 0x1f
93
94 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
95 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
96 (RV_X(x, 10, 3) << 3) | \
97 (RV_X(x, 5, 1) << 6))
98 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
99 (RV_X(x, 5, 2) << 6))
100 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
101 (RV_X(x, 12, 1) << 5) | \
102 (RV_X(x, 2, 2) << 6))
103 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
104 (RV_X(x, 12, 1) << 5) | \
105 (RV_X(x, 2, 3) << 6))
106 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
107 (RV_X(x, 7, 2) << 6))
108 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
109 (RV_X(x, 7, 3) << 6))
110 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
111 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
112 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
113
114 #define SHIFT_RIGHT(x, y) \
115 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
116
117 #define REG_MASK \
118 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
119
120 #define REG_OFFSET(insn, pos) \
121 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
122
123 #define REG_PTR(insn, pos, regs) \
124 ((ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)))
125
126 #define GET_FUNCT3(insn) (((insn) >> 12) & 7)
127
128 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
129 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
130 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
131 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
132 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
133 #define GET_SP(regs) (*REG_PTR(2, 0, regs))
134 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
135 #define IMM_I(insn) ((s32)(insn) >> 20)
136 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
137 (s32)(((insn) >> 7) & 0x1f))
138
139 struct insn_func {
140 unsigned long mask;
141 unsigned long match;
142 /*
143 * Possible return values are as follows:
144 * 1) Returns < 0 for error case
145 * 2) Returns 0 for exit to user-space
146 * 3) Returns 1 to continue with next sepc
147 * 4) Returns 2 to continue with same sepc
148 * 5) Returns 3 to inject illegal instruction trap and continue
149 * 6) Returns 4 to inject virtual instruction trap and continue
150 *
151 * Use enum kvm_insn_return for return values
152 */
153 int (*func)(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn);
154 };
155
truly_illegal_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)156 static int truly_illegal_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
157 ulong insn)
158 {
159 struct kvm_cpu_trap utrap = { 0 };
160
161 /* Redirect trap to Guest VCPU */
162 utrap.sepc = vcpu->arch.guest_context.sepc;
163 utrap.scause = EXC_INST_ILLEGAL;
164 utrap.stval = insn;
165 utrap.htval = 0;
166 utrap.htinst = 0;
167 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
168
169 return 1;
170 }
171
truly_virtual_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)172 static int truly_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
173 ulong insn)
174 {
175 struct kvm_cpu_trap utrap = { 0 };
176
177 /* Redirect trap to Guest VCPU */
178 utrap.sepc = vcpu->arch.guest_context.sepc;
179 utrap.scause = EXC_VIRTUAL_INST_FAULT;
180 utrap.stval = insn;
181 utrap.htval = 0;
182 utrap.htinst = 0;
183 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
184
185 return 1;
186 }
187
188 /**
189 * kvm_riscv_vcpu_wfi -- Emulate wait for interrupt (WFI) behaviour
190 *
191 * @vcpu: The VCPU pointer
192 */
kvm_riscv_vcpu_wfi(struct kvm_vcpu * vcpu)193 void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu)
194 {
195 if (!kvm_arch_vcpu_runnable(vcpu)) {
196 kvm_vcpu_srcu_read_unlock(vcpu);
197 kvm_vcpu_halt(vcpu);
198 kvm_vcpu_srcu_read_lock(vcpu);
199 }
200 }
201
wfi_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)202 static int wfi_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
203 {
204 vcpu->stat.wfi_exit_stat++;
205 kvm_riscv_vcpu_wfi(vcpu);
206 return KVM_INSN_CONTINUE_NEXT_SEPC;
207 }
208
wrs_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)209 static int wrs_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
210 {
211 vcpu->stat.wrs_exit_stat++;
212 kvm_vcpu_on_spin(vcpu, vcpu->arch.guest_context.sstatus & SR_SPP);
213 return KVM_INSN_CONTINUE_NEXT_SEPC;
214 }
215
216 struct csr_func {
217 unsigned int base;
218 unsigned int count;
219 /*
220 * Possible return values are as same as "func" callback in
221 * "struct insn_func".
222 */
223 int (*func)(struct kvm_vcpu *vcpu, unsigned int csr_num,
224 unsigned long *val, unsigned long new_val,
225 unsigned long wr_mask);
226 };
227
seed_csr_rmw(struct kvm_vcpu * vcpu,unsigned int csr_num,unsigned long * val,unsigned long new_val,unsigned long wr_mask)228 static int seed_csr_rmw(struct kvm_vcpu *vcpu, unsigned int csr_num,
229 unsigned long *val, unsigned long new_val,
230 unsigned long wr_mask)
231 {
232 if (!riscv_isa_extension_available(vcpu->arch.isa, ZKR))
233 return KVM_INSN_ILLEGAL_TRAP;
234
235 return KVM_INSN_EXIT_TO_USER_SPACE;
236 }
237
238 static const struct csr_func csr_funcs[] = {
239 KVM_RISCV_VCPU_AIA_CSR_FUNCS
240 KVM_RISCV_VCPU_HPMCOUNTER_CSR_FUNCS
241 { .base = CSR_SEED, .count = 1, .func = seed_csr_rmw },
242 };
243
244 /**
245 * kvm_riscv_vcpu_csr_return -- Handle CSR read/write after user space
246 * emulation or in-kernel emulation
247 *
248 * @vcpu: The VCPU pointer
249 * @run: The VCPU run struct containing the CSR data
250 *
251 * Returns > 0 upon failure and 0 upon success
252 */
kvm_riscv_vcpu_csr_return(struct kvm_vcpu * vcpu,struct kvm_run * run)253 int kvm_riscv_vcpu_csr_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
254 {
255 ulong insn;
256
257 if (vcpu->arch.csr_decode.return_handled)
258 return 0;
259 vcpu->arch.csr_decode.return_handled = 1;
260
261 /* Update destination register for CSR reads */
262 insn = vcpu->arch.csr_decode.insn;
263 if ((insn >> SH_RD) & MASK_RX)
264 SET_RD(insn, &vcpu->arch.guest_context,
265 run->riscv_csr.ret_value);
266
267 /* Move to next instruction */
268 vcpu->arch.guest_context.sepc += INSN_LEN(insn);
269
270 return 0;
271 }
272
csr_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)273 static int csr_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
274 {
275 int i, rc = KVM_INSN_ILLEGAL_TRAP;
276 unsigned int csr_num = insn >> SH_RS2;
277 unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
278 ulong rs1_val = GET_RS1(insn, &vcpu->arch.guest_context);
279 const struct csr_func *tcfn, *cfn = NULL;
280 ulong val = 0, wr_mask = 0, new_val = 0;
281
282 /* Decode the CSR instruction */
283 switch (GET_FUNCT3(insn)) {
284 case GET_FUNCT3(INSN_MATCH_CSRRW):
285 wr_mask = -1UL;
286 new_val = rs1_val;
287 break;
288 case GET_FUNCT3(INSN_MATCH_CSRRS):
289 wr_mask = rs1_val;
290 new_val = -1UL;
291 break;
292 case GET_FUNCT3(INSN_MATCH_CSRRC):
293 wr_mask = rs1_val;
294 new_val = 0;
295 break;
296 case GET_FUNCT3(INSN_MATCH_CSRRWI):
297 wr_mask = -1UL;
298 new_val = rs1_num;
299 break;
300 case GET_FUNCT3(INSN_MATCH_CSRRSI):
301 wr_mask = rs1_num;
302 new_val = -1UL;
303 break;
304 case GET_FUNCT3(INSN_MATCH_CSRRCI):
305 wr_mask = rs1_num;
306 new_val = 0;
307 break;
308 default:
309 return rc;
310 }
311
312 /* Save instruction decode info */
313 vcpu->arch.csr_decode.insn = insn;
314 vcpu->arch.csr_decode.return_handled = 0;
315
316 /* Update CSR details in kvm_run struct */
317 run->riscv_csr.csr_num = csr_num;
318 run->riscv_csr.new_value = new_val;
319 run->riscv_csr.write_mask = wr_mask;
320 run->riscv_csr.ret_value = 0;
321
322 /* Find in-kernel CSR function */
323 for (i = 0; i < ARRAY_SIZE(csr_funcs); i++) {
324 tcfn = &csr_funcs[i];
325 if ((tcfn->base <= csr_num) &&
326 (csr_num < (tcfn->base + tcfn->count))) {
327 cfn = tcfn;
328 break;
329 }
330 }
331
332 /* First try in-kernel CSR emulation */
333 if (cfn && cfn->func) {
334 rc = cfn->func(vcpu, csr_num, &val, new_val, wr_mask);
335 if (rc > KVM_INSN_EXIT_TO_USER_SPACE) {
336 if (rc == KVM_INSN_CONTINUE_NEXT_SEPC) {
337 run->riscv_csr.ret_value = val;
338 vcpu->stat.csr_exit_kernel++;
339 kvm_riscv_vcpu_csr_return(vcpu, run);
340 rc = KVM_INSN_CONTINUE_SAME_SEPC;
341 }
342 return rc;
343 }
344 }
345
346 /* Exit to user-space for CSR emulation */
347 if (rc <= KVM_INSN_EXIT_TO_USER_SPACE) {
348 vcpu->stat.csr_exit_user++;
349 run->exit_reason = KVM_EXIT_RISCV_CSR;
350 }
351
352 return rc;
353 }
354
355 static const struct insn_func system_opcode_funcs[] = {
356 {
357 .mask = INSN_MASK_CSRRW,
358 .match = INSN_MATCH_CSRRW,
359 .func = csr_insn,
360 },
361 {
362 .mask = INSN_MASK_CSRRS,
363 .match = INSN_MATCH_CSRRS,
364 .func = csr_insn,
365 },
366 {
367 .mask = INSN_MASK_CSRRC,
368 .match = INSN_MATCH_CSRRC,
369 .func = csr_insn,
370 },
371 {
372 .mask = INSN_MASK_CSRRWI,
373 .match = INSN_MATCH_CSRRWI,
374 .func = csr_insn,
375 },
376 {
377 .mask = INSN_MASK_CSRRSI,
378 .match = INSN_MATCH_CSRRSI,
379 .func = csr_insn,
380 },
381 {
382 .mask = INSN_MASK_CSRRCI,
383 .match = INSN_MATCH_CSRRCI,
384 .func = csr_insn,
385 },
386 {
387 .mask = INSN_MASK_WFI,
388 .match = INSN_MATCH_WFI,
389 .func = wfi_insn,
390 },
391 {
392 .mask = INSN_MASK_WRS,
393 .match = INSN_MATCH_WRS,
394 .func = wrs_insn,
395 },
396 };
397
system_opcode_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,ulong insn)398 static int system_opcode_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
399 ulong insn)
400 {
401 int i, rc = KVM_INSN_ILLEGAL_TRAP;
402 const struct insn_func *ifn;
403
404 for (i = 0; i < ARRAY_SIZE(system_opcode_funcs); i++) {
405 ifn = &system_opcode_funcs[i];
406 if ((insn & ifn->mask) == ifn->match) {
407 rc = ifn->func(vcpu, run, insn);
408 break;
409 }
410 }
411
412 switch (rc) {
413 case KVM_INSN_ILLEGAL_TRAP:
414 return truly_illegal_insn(vcpu, run, insn);
415 case KVM_INSN_VIRTUAL_TRAP:
416 return truly_virtual_insn(vcpu, run, insn);
417 case KVM_INSN_CONTINUE_NEXT_SEPC:
418 vcpu->arch.guest_context.sepc += INSN_LEN(insn);
419 break;
420 default:
421 break;
422 }
423
424 return (rc <= 0) ? rc : 1;
425 }
426
427 /**
428 * kvm_riscv_vcpu_virtual_insn -- Handle virtual instruction trap
429 *
430 * @vcpu: The VCPU pointer
431 * @run: The VCPU run struct containing the mmio data
432 * @trap: Trap details
433 *
434 * Returns > 0 to continue run-loop
435 * Returns 0 to exit run-loop and handle in user-space.
436 * Returns < 0 to report failure and exit run-loop
437 */
kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu * vcpu,struct kvm_run * run,struct kvm_cpu_trap * trap)438 int kvm_riscv_vcpu_virtual_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
439 struct kvm_cpu_trap *trap)
440 {
441 unsigned long insn = trap->stval;
442 struct kvm_cpu_trap utrap = { 0 };
443 struct kvm_cpu_context *ct;
444
445 if (unlikely(INSN_IS_16BIT(insn))) {
446 if (insn == 0) {
447 ct = &vcpu->arch.guest_context;
448 insn = kvm_riscv_vcpu_unpriv_read(vcpu, true,
449 ct->sepc,
450 &utrap);
451 if (utrap.scause) {
452 utrap.sepc = ct->sepc;
453 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
454 return 1;
455 }
456 }
457 if (INSN_IS_16BIT(insn))
458 return truly_illegal_insn(vcpu, run, insn);
459 }
460
461 switch ((insn & INSN_OPCODE_MASK) >> INSN_OPCODE_SHIFT) {
462 case INSN_OPCODE_SYSTEM:
463 return system_opcode_insn(vcpu, run, insn);
464 default:
465 return truly_illegal_insn(vcpu, run, insn);
466 }
467 }
468
469 /**
470 * kvm_riscv_vcpu_mmio_load -- Emulate MMIO load instruction
471 *
472 * @vcpu: The VCPU pointer
473 * @run: The VCPU run struct containing the mmio data
474 * @fault_addr: Guest physical address to load
475 * @htinst: Transformed encoding of the load instruction
476 *
477 * Returns > 0 to continue run-loop
478 * Returns 0 to exit run-loop and handle in user-space.
479 * Returns < 0 to report failure and exit run-loop
480 */
kvm_riscv_vcpu_mmio_load(struct kvm_vcpu * vcpu,struct kvm_run * run,unsigned long fault_addr,unsigned long htinst)481 int kvm_riscv_vcpu_mmio_load(struct kvm_vcpu *vcpu, struct kvm_run *run,
482 unsigned long fault_addr,
483 unsigned long htinst)
484 {
485 u8 data_buf[8];
486 unsigned long insn;
487 int shift = 0, len = 0, insn_len = 0;
488 struct kvm_cpu_trap utrap = { 0 };
489 struct kvm_cpu_context *ct = &vcpu->arch.guest_context;
490
491 /* Determine trapped instruction */
492 if (htinst & 0x1) {
493 /*
494 * Bit[0] == 1 implies trapped instruction value is
495 * transformed instruction or custom instruction.
496 */
497 insn = htinst | INSN_16BIT_MASK;
498 insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
499 } else {
500 /*
501 * Bit[0] == 0 implies trapped instruction value is
502 * zero or special value.
503 */
504 insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
505 &utrap);
506 if (utrap.scause) {
507 /* Redirect trap if we failed to read instruction */
508 utrap.sepc = ct->sepc;
509 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
510 return 1;
511 }
512 insn_len = INSN_LEN(insn);
513 }
514
515 /* Decode length of MMIO and shift */
516 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
517 len = 4;
518 shift = 8 * (sizeof(ulong) - len);
519 } else if ((insn & INSN_MASK_LB) == INSN_MATCH_LB) {
520 len = 1;
521 shift = 8 * (sizeof(ulong) - len);
522 } else if ((insn & INSN_MASK_LBU) == INSN_MATCH_LBU) {
523 len = 1;
524 shift = 8 * (sizeof(ulong) - len);
525 #ifdef CONFIG_64BIT
526 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
527 len = 8;
528 shift = 8 * (sizeof(ulong) - len);
529 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
530 len = 4;
531 #endif
532 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
533 len = 2;
534 shift = 8 * (sizeof(ulong) - len);
535 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
536 len = 2;
537 #ifdef CONFIG_64BIT
538 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
539 len = 8;
540 shift = 8 * (sizeof(ulong) - len);
541 insn = RVC_RS2S(insn) << SH_RD;
542 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
543 ((insn >> SH_RD) & 0x1f)) {
544 len = 8;
545 shift = 8 * (sizeof(ulong) - len);
546 #endif
547 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
548 len = 4;
549 shift = 8 * (sizeof(ulong) - len);
550 insn = RVC_RS2S(insn) << SH_RD;
551 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
552 ((insn >> SH_RD) & 0x1f)) {
553 len = 4;
554 shift = 8 * (sizeof(ulong) - len);
555 } else {
556 return -EOPNOTSUPP;
557 }
558
559 /* Fault address should be aligned to length of MMIO */
560 if (fault_addr & (len - 1))
561 return -EIO;
562
563 /* Save instruction decode info */
564 vcpu->arch.mmio_decode.insn = insn;
565 vcpu->arch.mmio_decode.insn_len = insn_len;
566 vcpu->arch.mmio_decode.shift = shift;
567 vcpu->arch.mmio_decode.len = len;
568 vcpu->arch.mmio_decode.return_handled = 0;
569
570 /* Update MMIO details in kvm_run struct */
571 run->mmio.is_write = false;
572 run->mmio.phys_addr = fault_addr;
573 run->mmio.len = len;
574
575 /* Try to handle MMIO access in the kernel */
576 if (!kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_addr, len, data_buf)) {
577 /* Successfully handled MMIO access in the kernel so resume */
578 memcpy(run->mmio.data, data_buf, len);
579 vcpu->stat.mmio_exit_kernel++;
580 kvm_riscv_vcpu_mmio_return(vcpu, run);
581 return 1;
582 }
583
584 /* Exit to userspace for MMIO emulation */
585 vcpu->stat.mmio_exit_user++;
586 run->exit_reason = KVM_EXIT_MMIO;
587
588 return 0;
589 }
590
591 /**
592 * kvm_riscv_vcpu_mmio_store -- Emulate MMIO store instruction
593 *
594 * @vcpu: The VCPU pointer
595 * @run: The VCPU run struct containing the mmio data
596 * @fault_addr: Guest physical address to store
597 * @htinst: Transformed encoding of the store instruction
598 *
599 * Returns > 0 to continue run-loop
600 * Returns 0 to exit run-loop and handle in user-space.
601 * Returns < 0 to report failure and exit run-loop
602 */
kvm_riscv_vcpu_mmio_store(struct kvm_vcpu * vcpu,struct kvm_run * run,unsigned long fault_addr,unsigned long htinst)603 int kvm_riscv_vcpu_mmio_store(struct kvm_vcpu *vcpu, struct kvm_run *run,
604 unsigned long fault_addr,
605 unsigned long htinst)
606 {
607 u8 data8;
608 u16 data16;
609 u32 data32;
610 u64 data64;
611 ulong data;
612 unsigned long insn;
613 int len = 0, insn_len = 0;
614 struct kvm_cpu_trap utrap = { 0 };
615 struct kvm_cpu_context *ct = &vcpu->arch.guest_context;
616
617 /* Determine trapped instruction */
618 if (htinst & 0x1) {
619 /*
620 * Bit[0] == 1 implies trapped instruction value is
621 * transformed instruction or custom instruction.
622 */
623 insn = htinst | INSN_16BIT_MASK;
624 insn_len = (htinst & BIT(1)) ? INSN_LEN(insn) : 2;
625 } else {
626 /*
627 * Bit[0] == 0 implies trapped instruction value is
628 * zero or special value.
629 */
630 insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
631 &utrap);
632 if (utrap.scause) {
633 /* Redirect trap if we failed to read instruction */
634 utrap.sepc = ct->sepc;
635 kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
636 return 1;
637 }
638 insn_len = INSN_LEN(insn);
639 }
640
641 data = GET_RS2(insn, &vcpu->arch.guest_context);
642 data8 = data16 = data32 = data64 = data;
643
644 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
645 len = 4;
646 } else if ((insn & INSN_MASK_SB) == INSN_MATCH_SB) {
647 len = 1;
648 #ifdef CONFIG_64BIT
649 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
650 len = 8;
651 #endif
652 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
653 len = 2;
654 #ifdef CONFIG_64BIT
655 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
656 len = 8;
657 data64 = GET_RS2S(insn, &vcpu->arch.guest_context);
658 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP &&
659 ((insn >> SH_RD) & 0x1f)) {
660 len = 8;
661 data64 = GET_RS2C(insn, &vcpu->arch.guest_context);
662 #endif
663 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
664 len = 4;
665 data32 = GET_RS2S(insn, &vcpu->arch.guest_context);
666 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP &&
667 ((insn >> SH_RD) & 0x1f)) {
668 len = 4;
669 data32 = GET_RS2C(insn, &vcpu->arch.guest_context);
670 } else {
671 return -EOPNOTSUPP;
672 }
673
674 /* Fault address should be aligned to length of MMIO */
675 if (fault_addr & (len - 1))
676 return -EIO;
677
678 /* Save instruction decode info */
679 vcpu->arch.mmio_decode.insn = insn;
680 vcpu->arch.mmio_decode.insn_len = insn_len;
681 vcpu->arch.mmio_decode.shift = 0;
682 vcpu->arch.mmio_decode.len = len;
683 vcpu->arch.mmio_decode.return_handled = 0;
684
685 /* Copy data to kvm_run instance */
686 switch (len) {
687 case 1:
688 *((u8 *)run->mmio.data) = data8;
689 break;
690 case 2:
691 *((u16 *)run->mmio.data) = data16;
692 break;
693 case 4:
694 *((u32 *)run->mmio.data) = data32;
695 break;
696 case 8:
697 *((u64 *)run->mmio.data) = data64;
698 break;
699 default:
700 return -EOPNOTSUPP;
701 }
702
703 /* Update MMIO details in kvm_run struct */
704 run->mmio.is_write = true;
705 run->mmio.phys_addr = fault_addr;
706 run->mmio.len = len;
707
708 /* Try to handle MMIO access in the kernel */
709 if (!kvm_io_bus_write(vcpu, KVM_MMIO_BUS,
710 fault_addr, len, run->mmio.data)) {
711 /* Successfully handled MMIO access in the kernel so resume */
712 vcpu->stat.mmio_exit_kernel++;
713 kvm_riscv_vcpu_mmio_return(vcpu, run);
714 return 1;
715 }
716
717 /* Exit to userspace for MMIO emulation */
718 vcpu->stat.mmio_exit_user++;
719 run->exit_reason = KVM_EXIT_MMIO;
720
721 return 0;
722 }
723
724 /**
725 * kvm_riscv_vcpu_mmio_return -- Handle MMIO loads after user space emulation
726 * or in-kernel IO emulation
727 *
728 * @vcpu: The VCPU pointer
729 * @run: The VCPU run struct containing the mmio data
730 */
kvm_riscv_vcpu_mmio_return(struct kvm_vcpu * vcpu,struct kvm_run * run)731 int kvm_riscv_vcpu_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
732 {
733 u8 data8;
734 u16 data16;
735 u32 data32;
736 u64 data64;
737 ulong insn;
738 int len, shift;
739
740 if (vcpu->arch.mmio_decode.return_handled)
741 return 0;
742
743 vcpu->arch.mmio_decode.return_handled = 1;
744 insn = vcpu->arch.mmio_decode.insn;
745
746 if (run->mmio.is_write)
747 goto done;
748
749 len = vcpu->arch.mmio_decode.len;
750 shift = vcpu->arch.mmio_decode.shift;
751
752 switch (len) {
753 case 1:
754 data8 = *((u8 *)run->mmio.data);
755 SET_RD(insn, &vcpu->arch.guest_context,
756 (ulong)data8 << shift >> shift);
757 break;
758 case 2:
759 data16 = *((u16 *)run->mmio.data);
760 SET_RD(insn, &vcpu->arch.guest_context,
761 (ulong)data16 << shift >> shift);
762 break;
763 case 4:
764 data32 = *((u32 *)run->mmio.data);
765 SET_RD(insn, &vcpu->arch.guest_context,
766 (ulong)data32 << shift >> shift);
767 break;
768 case 8:
769 data64 = *((u64 *)run->mmio.data);
770 SET_RD(insn, &vcpu->arch.guest_context,
771 (ulong)data64 << shift >> shift);
772 break;
773 default:
774 return -EOPNOTSUPP;
775 }
776
777 done:
778 /* Move to next instruction */
779 vcpu->arch.guest_context.sepc += vcpu->arch.mmio_decode.insn_len;
780
781 return 0;
782 }
783