1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4 *
5 * Authors:
6 * Anup Patel <anup.patel@wdc.com>
7 */
8
9 #include <linux/kvm_host.h>
10 #include <asm/csr.h>
11 #include <asm/insn-def.h>
12 #include <asm/kvm_mmu.h>
13 #include <asm/kvm_nacl.h>
14 #include "trace.h"
15
gstage_page_fault(struct kvm_vcpu * vcpu,struct kvm_run * run,struct kvm_cpu_trap * trap)16 static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run,
17 struct kvm_cpu_trap *trap)
18 {
19 struct kvm_gstage_mapping host_map;
20 struct kvm_memory_slot *memslot;
21 unsigned long hva;
22 gpa_t fault_addr;
23 bool writable;
24 gfn_t gfn;
25 int ret;
26
27 fault_addr = ((gpa_t)trap->htval << 2) | (trap->stval & 0x3);
28 gfn = fault_addr >> PAGE_SHIFT;
29 memslot = gfn_to_memslot(vcpu->kvm, gfn);
30 hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
31
32 if (kvm_is_error_hva(hva) ||
33 (trap->scause == EXC_STORE_GUEST_PAGE_FAULT && !writable)) {
34 switch (trap->scause) {
35 case EXC_LOAD_GUEST_PAGE_FAULT:
36 return kvm_riscv_vcpu_mmio_load(vcpu, run,
37 fault_addr,
38 trap->htinst);
39 case EXC_STORE_GUEST_PAGE_FAULT:
40 return kvm_riscv_vcpu_mmio_store(vcpu, run,
41 fault_addr,
42 trap->htinst);
43 case EXC_INST_GUEST_PAGE_FAULT: {
44 /*
45 * No memslot backs this GPA and an instruction fetch
46 * cannot be emulated as MMIO. On bare metal a fetch
47 * from an unbacked physical address raises an
48 * instruction access fault, so reflect that back to
49 * the guest.
50 */
51 struct kvm_cpu_trap inst_trap = {
52 .sepc = trap->sepc,
53 .scause = EXC_INST_ACCESS,
54 .stval = trap->stval,
55 .htval = 0,
56 .htinst = 0,
57 };
58
59 kvm_riscv_vcpu_trap_redirect(vcpu, &inst_trap);
60 return 1;
61 }
62 default:
63 return -EOPNOTSUPP;
64 };
65 }
66
67 ret = kvm_riscv_mmu_map(vcpu, memslot, fault_addr, hva,
68 (trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false,
69 &host_map);
70 if (ret < 0)
71 return ret;
72
73 return 1;
74 }
75
76 /**
77 * kvm_riscv_vcpu_unpriv_read -- Read machine word from Guest memory
78 *
79 * @vcpu: The VCPU pointer
80 * @read_insn: Flag representing whether we are reading instruction
81 * @guest_addr: Guest address to read
82 * @trap: Output pointer to trap details
83 */
kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu * vcpu,bool read_insn,unsigned long guest_addr,struct kvm_cpu_trap * trap)84 unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu,
85 bool read_insn,
86 unsigned long guest_addr,
87 struct kvm_cpu_trap *trap)
88 {
89 register unsigned long taddr asm("a0") = (unsigned long)trap;
90 register unsigned long ttmp asm("a1");
91 unsigned long flags, val, tmp, old_stvec, old_hstatus;
92
93 local_irq_save(flags);
94
95 old_hstatus = csr_swap(CSR_HSTATUS, vcpu->arch.guest_context.hstatus);
96 old_stvec = csr_swap(CSR_STVEC, (ulong)&__kvm_riscv_unpriv_trap);
97
98 if (read_insn) {
99 /*
100 * HLVX.HU instruction
101 * 0110010 00011 rs1 100 rd 1110011
102 */
103 asm volatile ("\n"
104 ".option push\n"
105 ".option norvc\n"
106 "add %[ttmp], %[taddr], 0\n"
107 HLVX_HU(%[val], %[addr])
108 "andi %[tmp], %[val], 3\n"
109 "addi %[tmp], %[tmp], -3\n"
110 "bne %[tmp], zero, 2f\n"
111 "addi %[addr], %[addr], 2\n"
112 HLVX_HU(%[tmp], %[addr])
113 "sll %[tmp], %[tmp], 16\n"
114 "add %[val], %[val], %[tmp]\n"
115 "2:\n"
116 ".option pop"
117 : [val] "=&r" (val), [tmp] "=&r" (tmp),
118 [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp),
119 [addr] "+&r" (guest_addr) : : "memory");
120
121 if (trap->scause == EXC_LOAD_PAGE_FAULT)
122 trap->scause = EXC_INST_PAGE_FAULT;
123 } else {
124 /*
125 * HLV.D instruction
126 * 0110110 00000 rs1 100 rd 1110011
127 *
128 * HLV.W instruction
129 * 0110100 00000 rs1 100 rd 1110011
130 */
131 asm volatile ("\n"
132 ".option push\n"
133 ".option norvc\n"
134 "add %[ttmp], %[taddr], 0\n"
135 #ifdef CONFIG_64BIT
136 HLV_D(%[val], %[addr])
137 #else
138 HLV_W(%[val], %[addr])
139 #endif
140 ".option pop"
141 : [val] "=&r" (val),
142 [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp)
143 : [addr] "r" (guest_addr) : "memory");
144 }
145
146 csr_write(CSR_STVEC, old_stvec);
147 csr_write(CSR_HSTATUS, old_hstatus);
148
149 local_irq_restore(flags);
150
151 return val;
152 }
153
154 /**
155 * kvm_riscv_vcpu_trap_redirect -- Redirect trap to Guest
156 *
157 * @vcpu: The VCPU pointer
158 * @trap: Trap details
159 */
kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu * vcpu,struct kvm_cpu_trap * trap)160 void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu,
161 struct kvm_cpu_trap *trap)
162 {
163 unsigned long vsstatus = ncsr_read(CSR_VSSTATUS);
164
165 /* Change Guest SSTATUS.SPP bit */
166 vsstatus &= ~SR_SPP;
167 if (vcpu->arch.guest_context.sstatus & SR_SPP)
168 vsstatus |= SR_SPP;
169
170 /* Change Guest SSTATUS.SPIE bit */
171 vsstatus &= ~SR_SPIE;
172 if (vsstatus & SR_SIE)
173 vsstatus |= SR_SPIE;
174
175 /* Clear Guest SSTATUS.SIE bit */
176 vsstatus &= ~SR_SIE;
177
178 /* Change Guest SSTATUS.SPELP bit */
179 if (vcpu->arch.cfg.henvcfg & ENVCFG_LPE) {
180 vsstatus &= ~SR_SPELP;
181 vsstatus |= vcpu->arch.guest_context.sstatus & SR_SPELP;
182 vcpu->arch.guest_context.sstatus &= ~SR_SPELP;
183 }
184
185 /* Update Guest SSTATUS */
186 ncsr_write(CSR_VSSTATUS, vsstatus);
187
188 /* Update Guest SCAUSE, STVAL, and SEPC */
189 ncsr_write(CSR_VSCAUSE, trap->scause);
190 ncsr_write(CSR_VSTVAL, trap->stval);
191 ncsr_write(CSR_VSEPC, trap->sepc);
192
193 /* Set Guest PC to Guest exception vector */
194 vcpu->arch.guest_context.sepc = ncsr_read(CSR_VSTVEC);
195
196 /* Set Guest privilege mode to supervisor */
197 vcpu->arch.guest_context.sstatus |= SR_SPP;
198 }
199
vcpu_redirect(struct kvm_vcpu * vcpu,struct kvm_cpu_trap * trap)200 static inline int vcpu_redirect(struct kvm_vcpu *vcpu, struct kvm_cpu_trap *trap)
201 {
202 int ret = -EFAULT;
203
204 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) {
205 kvm_riscv_vcpu_trap_redirect(vcpu, trap);
206 ret = 1;
207 }
208 return ret;
209 }
210
211 /*
212 * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on
213 * proper exit to userspace.
214 */
kvm_riscv_vcpu_exit(struct kvm_vcpu * vcpu,struct kvm_run * run,struct kvm_cpu_trap * trap)215 int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
216 struct kvm_cpu_trap *trap)
217 {
218 int ret;
219
220 /* If we got host interrupt then do nothing */
221 if (trap->scause & CAUSE_IRQ_FLAG)
222 return 1;
223
224 trace_kvm_vcpu_exit(vcpu->vcpu_id, trap->sepc, trap->scause,
225 trap->stval, trap->htval, trap->htinst);
226
227 /* Handle guest traps */
228 ret = -EFAULT;
229 run->exit_reason = KVM_EXIT_UNKNOWN;
230 switch (trap->scause) {
231 case EXC_INST_ILLEGAL:
232 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ILLEGAL_INSN);
233 vcpu->stat.instr_illegal_exits++;
234 ret = vcpu_redirect(vcpu, trap);
235 break;
236 case EXC_LOAD_MISALIGNED:
237 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_MISALIGNED_LOAD);
238 vcpu->stat.load_misaligned_exits++;
239 ret = vcpu_redirect(vcpu, trap);
240 break;
241 case EXC_STORE_MISALIGNED:
242 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_MISALIGNED_STORE);
243 vcpu->stat.store_misaligned_exits++;
244 ret = vcpu_redirect(vcpu, trap);
245 break;
246 case EXC_LOAD_ACCESS:
247 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ACCESS_LOAD);
248 vcpu->stat.load_access_exits++;
249 ret = vcpu_redirect(vcpu, trap);
250 break;
251 case EXC_STORE_ACCESS:
252 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ACCESS_STORE);
253 vcpu->stat.store_access_exits++;
254 ret = vcpu_redirect(vcpu, trap);
255 break;
256 case EXC_INST_ACCESS:
257 ret = vcpu_redirect(vcpu, trap);
258 break;
259 case EXC_VIRTUAL_INST_FAULT:
260 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV)
261 ret = kvm_riscv_vcpu_virtual_insn(vcpu, run, trap);
262 break;
263 case EXC_INST_GUEST_PAGE_FAULT:
264 case EXC_LOAD_GUEST_PAGE_FAULT:
265 case EXC_STORE_GUEST_PAGE_FAULT:
266 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV)
267 ret = gstage_page_fault(vcpu, run, trap);
268 break;
269 case EXC_SUPERVISOR_SYSCALL:
270 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV)
271 ret = kvm_riscv_vcpu_sbi_ecall(vcpu, run);
272 break;
273 case EXC_BREAKPOINT:
274 run->exit_reason = KVM_EXIT_DEBUG;
275 ret = 0;
276 break;
277 case EXC_SOFTWARE_CHECK:
278 if (vcpu->arch.cfg.henvcfg & (ENVCFG_LPE | ENVCFG_SSE))
279 ret = vcpu_redirect(vcpu, trap);
280 break;
281 default:
282 break;
283 }
284
285 /* Print details in-case of error */
286 if (ret < 0 && ret != -EINTR) {
287 kvm_err("VCPU exit error %d\n", ret);
288 kvm_err("SEPC=0x%lx SSTATUS=0x%lx HSTATUS=0x%lx\n",
289 vcpu->arch.guest_context.sepc,
290 vcpu->arch.guest_context.sstatus,
291 vcpu->arch.guest_context.hstatus);
292 kvm_err("SCAUSE=0x%lx STVAL=0x%lx HTVAL=0x%lx HTINST=0x%lx\n",
293 trap->scause, trap->stval, trap->htval, trap->htinst);
294 }
295
296 return ret;
297 }
298