1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/kvm_host.h> 10 #include <asm/csr.h> 11 #include <asm/insn-def.h> 12 #include <asm/kvm_mmu.h> 13 #include <asm/kvm_nacl.h> 14 #include "trace.h" 15 16 static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, 17 struct kvm_cpu_trap *trap) 18 { 19 struct kvm_gstage_mapping host_map; 20 struct kvm_memory_slot *memslot; 21 unsigned long hva; 22 gpa_t fault_addr; 23 bool writable; 24 gfn_t gfn; 25 int ret; 26 27 fault_addr = ((gpa_t)trap->htval << 2) | (trap->stval & 0x3); 28 gfn = fault_addr >> PAGE_SHIFT; 29 memslot = gfn_to_memslot(vcpu->kvm, gfn); 30 hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); 31 32 if (kvm_is_error_hva(hva) || 33 (trap->scause == EXC_STORE_GUEST_PAGE_FAULT && !writable)) { 34 switch (trap->scause) { 35 case EXC_LOAD_GUEST_PAGE_FAULT: 36 return kvm_riscv_vcpu_mmio_load(vcpu, run, 37 fault_addr, 38 trap->htinst); 39 case EXC_STORE_GUEST_PAGE_FAULT: 40 return kvm_riscv_vcpu_mmio_store(vcpu, run, 41 fault_addr, 42 trap->htinst); 43 case EXC_INST_GUEST_PAGE_FAULT: { 44 /* 45 * No memslot backs this GPA and an instruction fetch 46 * cannot be emulated as MMIO. On bare metal a fetch 47 * from an unbacked physical address raises an 48 * instruction access fault, so reflect that back to 49 * the guest. 50 */ 51 struct kvm_cpu_trap inst_trap = { 52 .sepc = trap->sepc, 53 .scause = EXC_INST_ACCESS, 54 .stval = trap->stval, 55 .htval = 0, 56 .htinst = 0, 57 }; 58 59 kvm_riscv_vcpu_trap_redirect(vcpu, &inst_trap); 60 return 1; 61 } 62 default: 63 return -EOPNOTSUPP; 64 }; 65 } 66 67 ret = kvm_riscv_mmu_map(vcpu, memslot, fault_addr, hva, 68 (trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false, 69 &host_map); 70 if (ret < 0) 71 return ret; 72 73 return 1; 74 } 75 76 /** 77 * kvm_riscv_vcpu_unpriv_read -- Read machine word from Guest memory 78 * 79 * @vcpu: The VCPU pointer 80 * @read_insn: Flag representing whether we are reading instruction 81 * @guest_addr: Guest address to read 82 * @trap: Output pointer to trap details 83 */ 84 unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu, 85 bool read_insn, 86 unsigned long guest_addr, 87 struct kvm_cpu_trap *trap) 88 { 89 register unsigned long taddr asm("a0") = (unsigned long)trap; 90 register unsigned long ttmp asm("a1"); 91 unsigned long flags, val, tmp, old_stvec, old_hstatus; 92 93 local_irq_save(flags); 94 95 old_hstatus = csr_swap(CSR_HSTATUS, vcpu->arch.guest_context.hstatus); 96 old_stvec = csr_swap(CSR_STVEC, (ulong)&__kvm_riscv_unpriv_trap); 97 98 if (read_insn) { 99 /* 100 * HLVX.HU instruction 101 * 0110010 00011 rs1 100 rd 1110011 102 */ 103 asm volatile ("\n" 104 ".option push\n" 105 ".option norvc\n" 106 "add %[ttmp], %[taddr], 0\n" 107 HLVX_HU(%[val], %[addr]) 108 "andi %[tmp], %[val], 3\n" 109 "addi %[tmp], %[tmp], -3\n" 110 "bne %[tmp], zero, 2f\n" 111 "addi %[addr], %[addr], 2\n" 112 HLVX_HU(%[tmp], %[addr]) 113 "sll %[tmp], %[tmp], 16\n" 114 "add %[val], %[val], %[tmp]\n" 115 "2:\n" 116 ".option pop" 117 : [val] "=&r" (val), [tmp] "=&r" (tmp), 118 [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp), 119 [addr] "+&r" (guest_addr) : : "memory"); 120 121 if (trap->scause == EXC_LOAD_PAGE_FAULT) 122 trap->scause = EXC_INST_PAGE_FAULT; 123 } else { 124 /* 125 * HLV.D instruction 126 * 0110110 00000 rs1 100 rd 1110011 127 * 128 * HLV.W instruction 129 * 0110100 00000 rs1 100 rd 1110011 130 */ 131 asm volatile ("\n" 132 ".option push\n" 133 ".option norvc\n" 134 "add %[ttmp], %[taddr], 0\n" 135 #ifdef CONFIG_64BIT 136 HLV_D(%[val], %[addr]) 137 #else 138 HLV_W(%[val], %[addr]) 139 #endif 140 ".option pop" 141 : [val] "=&r" (val), 142 [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp) 143 : [addr] "r" (guest_addr) : "memory"); 144 } 145 146 csr_write(CSR_STVEC, old_stvec); 147 csr_write(CSR_HSTATUS, old_hstatus); 148 149 local_irq_restore(flags); 150 151 return val; 152 } 153 154 /** 155 * kvm_riscv_vcpu_trap_redirect -- Redirect trap to Guest 156 * 157 * @vcpu: The VCPU pointer 158 * @trap: Trap details 159 */ 160 void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu, 161 struct kvm_cpu_trap *trap) 162 { 163 unsigned long vsstatus = ncsr_read(CSR_VSSTATUS); 164 165 /* Change Guest SSTATUS.SPP bit */ 166 vsstatus &= ~SR_SPP; 167 if (vcpu->arch.guest_context.sstatus & SR_SPP) 168 vsstatus |= SR_SPP; 169 170 /* Change Guest SSTATUS.SPIE bit */ 171 vsstatus &= ~SR_SPIE; 172 if (vsstatus & SR_SIE) 173 vsstatus |= SR_SPIE; 174 175 /* Clear Guest SSTATUS.SIE bit */ 176 vsstatus &= ~SR_SIE; 177 178 /* Change Guest SSTATUS.SPELP bit */ 179 if (vcpu->arch.cfg.henvcfg & ENVCFG_LPE) { 180 vsstatus &= ~SR_SPELP; 181 vsstatus |= vcpu->arch.guest_context.sstatus & SR_SPELP; 182 vcpu->arch.guest_context.sstatus &= ~SR_SPELP; 183 } 184 185 /* Update Guest SSTATUS */ 186 ncsr_write(CSR_VSSTATUS, vsstatus); 187 188 /* Update Guest SCAUSE, STVAL, and SEPC */ 189 ncsr_write(CSR_VSCAUSE, trap->scause); 190 ncsr_write(CSR_VSTVAL, trap->stval); 191 ncsr_write(CSR_VSEPC, trap->sepc); 192 193 /* Set Guest PC to Guest exception vector */ 194 vcpu->arch.guest_context.sepc = ncsr_read(CSR_VSTVEC); 195 196 /* Set Guest privilege mode to supervisor */ 197 vcpu->arch.guest_context.sstatus |= SR_SPP; 198 } 199 200 static inline int vcpu_redirect(struct kvm_vcpu *vcpu, struct kvm_cpu_trap *trap) 201 { 202 int ret = -EFAULT; 203 204 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) { 205 kvm_riscv_vcpu_trap_redirect(vcpu, trap); 206 ret = 1; 207 } 208 return ret; 209 } 210 211 /* 212 * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on 213 * proper exit to userspace. 214 */ 215 int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, 216 struct kvm_cpu_trap *trap) 217 { 218 int ret; 219 220 /* If we got host interrupt then do nothing */ 221 if (trap->scause & CAUSE_IRQ_FLAG) 222 return 1; 223 224 trace_kvm_vcpu_exit(vcpu->vcpu_id, trap->sepc, trap->scause, 225 trap->stval, trap->htval, trap->htinst); 226 227 /* Handle guest traps */ 228 ret = -EFAULT; 229 run->exit_reason = KVM_EXIT_UNKNOWN; 230 switch (trap->scause) { 231 case EXC_INST_ILLEGAL: 232 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ILLEGAL_INSN); 233 vcpu->stat.instr_illegal_exits++; 234 ret = vcpu_redirect(vcpu, trap); 235 break; 236 case EXC_LOAD_MISALIGNED: 237 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_MISALIGNED_LOAD); 238 vcpu->stat.load_misaligned_exits++; 239 ret = vcpu_redirect(vcpu, trap); 240 break; 241 case EXC_STORE_MISALIGNED: 242 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_MISALIGNED_STORE); 243 vcpu->stat.store_misaligned_exits++; 244 ret = vcpu_redirect(vcpu, trap); 245 break; 246 case EXC_LOAD_ACCESS: 247 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ACCESS_LOAD); 248 vcpu->stat.load_access_exits++; 249 ret = vcpu_redirect(vcpu, trap); 250 break; 251 case EXC_STORE_ACCESS: 252 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ACCESS_STORE); 253 vcpu->stat.store_access_exits++; 254 ret = vcpu_redirect(vcpu, trap); 255 break; 256 case EXC_INST_ACCESS: 257 ret = vcpu_redirect(vcpu, trap); 258 break; 259 case EXC_VIRTUAL_INST_FAULT: 260 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) 261 ret = kvm_riscv_vcpu_virtual_insn(vcpu, run, trap); 262 break; 263 case EXC_INST_GUEST_PAGE_FAULT: 264 case EXC_LOAD_GUEST_PAGE_FAULT: 265 case EXC_STORE_GUEST_PAGE_FAULT: 266 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) 267 ret = gstage_page_fault(vcpu, run, trap); 268 break; 269 case EXC_SUPERVISOR_SYSCALL: 270 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) 271 ret = kvm_riscv_vcpu_sbi_ecall(vcpu, run); 272 break; 273 case EXC_BREAKPOINT: 274 run->exit_reason = KVM_EXIT_DEBUG; 275 ret = 0; 276 break; 277 case EXC_SOFTWARE_CHECK: 278 if (vcpu->arch.cfg.henvcfg & (ENVCFG_LPE | ENVCFG_SSE)) 279 ret = vcpu_redirect(vcpu, trap); 280 break; 281 default: 282 break; 283 } 284 285 /* Print details in-case of error */ 286 if (ret < 0) { 287 kvm_err("VCPU exit error %d\n", ret); 288 kvm_err("SEPC=0x%lx SSTATUS=0x%lx HSTATUS=0x%lx\n", 289 vcpu->arch.guest_context.sepc, 290 vcpu->arch.guest_context.sstatus, 291 vcpu->arch.guest_context.hstatus); 292 kvm_err("SCAUSE=0x%lx STVAL=0x%lx HTVAL=0x%lx HTINST=0x%lx\n", 293 trap->scause, trap->stval, trap->htval, trap->htinst); 294 } 295 296 return ret; 297 } 298