1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4 *
5 * Authors:
6 * Anup Patel <anup.patel@wdc.com>
7 */
8
9 #include <linux/kvm_host.h>
10 #include <asm/csr.h>
11 #include <asm/insn-def.h>
12 #include <asm/kvm_mmu.h>
13 #include <asm/kvm_nacl.h>
14
gstage_page_fault(struct kvm_vcpu * vcpu,struct kvm_run * run,struct kvm_cpu_trap * trap)15 static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run,
16 struct kvm_cpu_trap *trap)
17 {
18 struct kvm_gstage_mapping host_map;
19 struct kvm_memory_slot *memslot;
20 unsigned long hva, fault_addr;
21 bool writable;
22 gfn_t gfn;
23 int ret;
24
25 fault_addr = (trap->htval << 2) | (trap->stval & 0x3);
26 gfn = fault_addr >> PAGE_SHIFT;
27 memslot = gfn_to_memslot(vcpu->kvm, gfn);
28 hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
29
30 if (kvm_is_error_hva(hva) ||
31 (trap->scause == EXC_STORE_GUEST_PAGE_FAULT && !writable)) {
32 switch (trap->scause) {
33 case EXC_LOAD_GUEST_PAGE_FAULT:
34 return kvm_riscv_vcpu_mmio_load(vcpu, run,
35 fault_addr,
36 trap->htinst);
37 case EXC_STORE_GUEST_PAGE_FAULT:
38 return kvm_riscv_vcpu_mmio_store(vcpu, run,
39 fault_addr,
40 trap->htinst);
41 case EXC_INST_GUEST_PAGE_FAULT: {
42 /*
43 * No memslot backs this GPA and an instruction fetch
44 * cannot be emulated as MMIO. On bare metal a fetch
45 * from an unbacked physical address raises an
46 * instruction access fault, so reflect that back to
47 * the guest.
48 */
49 struct kvm_cpu_trap inst_trap = {
50 .sepc = trap->sepc,
51 .scause = EXC_INST_ACCESS,
52 .stval = trap->stval,
53 .htval = 0,
54 .htinst = 0,
55 };
56
57 kvm_riscv_vcpu_trap_redirect(vcpu, &inst_trap);
58 return 1;
59 }
60 default:
61 return -EOPNOTSUPP;
62 };
63 }
64
65 ret = kvm_riscv_mmu_map(vcpu, memslot, fault_addr, hva,
66 (trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false,
67 &host_map);
68 if (ret < 0)
69 return ret;
70
71 return 1;
72 }
73
74 /**
75 * kvm_riscv_vcpu_unpriv_read -- Read machine word from Guest memory
76 *
77 * @vcpu: The VCPU pointer
78 * @read_insn: Flag representing whether we are reading instruction
79 * @guest_addr: Guest address to read
80 * @trap: Output pointer to trap details
81 */
kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu * vcpu,bool read_insn,unsigned long guest_addr,struct kvm_cpu_trap * trap)82 unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu,
83 bool read_insn,
84 unsigned long guest_addr,
85 struct kvm_cpu_trap *trap)
86 {
87 register unsigned long taddr asm("a0") = (unsigned long)trap;
88 register unsigned long ttmp asm("a1");
89 unsigned long flags, val, tmp, old_stvec, old_hstatus;
90
91 local_irq_save(flags);
92
93 old_hstatus = csr_swap(CSR_HSTATUS, vcpu->arch.guest_context.hstatus);
94 old_stvec = csr_swap(CSR_STVEC, (ulong)&__kvm_riscv_unpriv_trap);
95
96 if (read_insn) {
97 /*
98 * HLVX.HU instruction
99 * 0110010 00011 rs1 100 rd 1110011
100 */
101 asm volatile ("\n"
102 ".option push\n"
103 ".option norvc\n"
104 "add %[ttmp], %[taddr], 0\n"
105 HLVX_HU(%[val], %[addr])
106 "andi %[tmp], %[val], 3\n"
107 "addi %[tmp], %[tmp], -3\n"
108 "bne %[tmp], zero, 2f\n"
109 "addi %[addr], %[addr], 2\n"
110 HLVX_HU(%[tmp], %[addr])
111 "sll %[tmp], %[tmp], 16\n"
112 "add %[val], %[val], %[tmp]\n"
113 "2:\n"
114 ".option pop"
115 : [val] "=&r" (val), [tmp] "=&r" (tmp),
116 [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp),
117 [addr] "+&r" (guest_addr) : : "memory");
118
119 if (trap->scause == EXC_LOAD_PAGE_FAULT)
120 trap->scause = EXC_INST_PAGE_FAULT;
121 } else {
122 /*
123 * HLV.D instruction
124 * 0110110 00000 rs1 100 rd 1110011
125 *
126 * HLV.W instruction
127 * 0110100 00000 rs1 100 rd 1110011
128 */
129 asm volatile ("\n"
130 ".option push\n"
131 ".option norvc\n"
132 "add %[ttmp], %[taddr], 0\n"
133 #ifdef CONFIG_64BIT
134 HLV_D(%[val], %[addr])
135 #else
136 HLV_W(%[val], %[addr])
137 #endif
138 ".option pop"
139 : [val] "=&r" (val),
140 [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp)
141 : [addr] "r" (guest_addr) : "memory");
142 }
143
144 csr_write(CSR_STVEC, old_stvec);
145 csr_write(CSR_HSTATUS, old_hstatus);
146
147 local_irq_restore(flags);
148
149 return val;
150 }
151
152 /**
153 * kvm_riscv_vcpu_trap_redirect -- Redirect trap to Guest
154 *
155 * @vcpu: The VCPU pointer
156 * @trap: Trap details
157 */
kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu * vcpu,struct kvm_cpu_trap * trap)158 void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu,
159 struct kvm_cpu_trap *trap)
160 {
161 unsigned long vsstatus = ncsr_read(CSR_VSSTATUS);
162
163 /* Change Guest SSTATUS.SPP bit */
164 vsstatus &= ~SR_SPP;
165 if (vcpu->arch.guest_context.sstatus & SR_SPP)
166 vsstatus |= SR_SPP;
167
168 /* Change Guest SSTATUS.SPIE bit */
169 vsstatus &= ~SR_SPIE;
170 if (vsstatus & SR_SIE)
171 vsstatus |= SR_SPIE;
172
173 /* Clear Guest SSTATUS.SIE bit */
174 vsstatus &= ~SR_SIE;
175
176 /* Update Guest SSTATUS */
177 ncsr_write(CSR_VSSTATUS, vsstatus);
178
179 /* Update Guest SCAUSE, STVAL, and SEPC */
180 ncsr_write(CSR_VSCAUSE, trap->scause);
181 ncsr_write(CSR_VSTVAL, trap->stval);
182 ncsr_write(CSR_VSEPC, trap->sepc);
183
184 /* Set Guest PC to Guest exception vector */
185 vcpu->arch.guest_context.sepc = ncsr_read(CSR_VSTVEC);
186
187 /* Set Guest privilege mode to supervisor */
188 vcpu->arch.guest_context.sstatus |= SR_SPP;
189 }
190
vcpu_redirect(struct kvm_vcpu * vcpu,struct kvm_cpu_trap * trap)191 static inline int vcpu_redirect(struct kvm_vcpu *vcpu, struct kvm_cpu_trap *trap)
192 {
193 int ret = -EFAULT;
194
195 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) {
196 kvm_riscv_vcpu_trap_redirect(vcpu, trap);
197 ret = 1;
198 }
199 return ret;
200 }
201
202 /*
203 * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on
204 * proper exit to userspace.
205 */
kvm_riscv_vcpu_exit(struct kvm_vcpu * vcpu,struct kvm_run * run,struct kvm_cpu_trap * trap)206 int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
207 struct kvm_cpu_trap *trap)
208 {
209 int ret;
210
211 /* If we got host interrupt then do nothing */
212 if (trap->scause & CAUSE_IRQ_FLAG)
213 return 1;
214
215 /* Handle guest traps */
216 ret = -EFAULT;
217 run->exit_reason = KVM_EXIT_UNKNOWN;
218 switch (trap->scause) {
219 case EXC_INST_ILLEGAL:
220 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ILLEGAL_INSN);
221 vcpu->stat.instr_illegal_exits++;
222 ret = vcpu_redirect(vcpu, trap);
223 break;
224 case EXC_LOAD_MISALIGNED:
225 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_MISALIGNED_LOAD);
226 vcpu->stat.load_misaligned_exits++;
227 ret = vcpu_redirect(vcpu, trap);
228 break;
229 case EXC_STORE_MISALIGNED:
230 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_MISALIGNED_STORE);
231 vcpu->stat.store_misaligned_exits++;
232 ret = vcpu_redirect(vcpu, trap);
233 break;
234 case EXC_LOAD_ACCESS:
235 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ACCESS_LOAD);
236 vcpu->stat.load_access_exits++;
237 ret = vcpu_redirect(vcpu, trap);
238 break;
239 case EXC_STORE_ACCESS:
240 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ACCESS_STORE);
241 vcpu->stat.store_access_exits++;
242 ret = vcpu_redirect(vcpu, trap);
243 break;
244 case EXC_INST_ACCESS:
245 ret = vcpu_redirect(vcpu, trap);
246 break;
247 case EXC_VIRTUAL_INST_FAULT:
248 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV)
249 ret = kvm_riscv_vcpu_virtual_insn(vcpu, run, trap);
250 break;
251 case EXC_INST_GUEST_PAGE_FAULT:
252 case EXC_LOAD_GUEST_PAGE_FAULT:
253 case EXC_STORE_GUEST_PAGE_FAULT:
254 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV)
255 ret = gstage_page_fault(vcpu, run, trap);
256 break;
257 case EXC_SUPERVISOR_SYSCALL:
258 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV)
259 ret = kvm_riscv_vcpu_sbi_ecall(vcpu, run);
260 break;
261 case EXC_BREAKPOINT:
262 run->exit_reason = KVM_EXIT_DEBUG;
263 ret = 0;
264 break;
265 default:
266 break;
267 }
268
269 /* Print details in-case of error */
270 if (ret < 0) {
271 kvm_err("VCPU exit error %d\n", ret);
272 kvm_err("SEPC=0x%lx SSTATUS=0x%lx HSTATUS=0x%lx\n",
273 vcpu->arch.guest_context.sepc,
274 vcpu->arch.guest_context.sstatus,
275 vcpu->arch.guest_context.hstatus);
276 kvm_err("SCAUSE=0x%lx STVAL=0x%lx HTVAL=0x%lx HTINST=0x%lx\n",
277 trap->scause, trap->stval, trap->htval, trap->htinst);
278 }
279
280 return ret;
281 }
282