1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/kvm_host.h> 10 #include <asm/csr.h> 11 #include <asm/insn-def.h> 12 #include <asm/kvm_mmu.h> 13 #include <asm/kvm_nacl.h> 14 15 static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, 16 struct kvm_cpu_trap *trap) 17 { 18 struct kvm_gstage_mapping host_map; 19 struct kvm_memory_slot *memslot; 20 unsigned long hva, fault_addr; 21 bool writable; 22 gfn_t gfn; 23 int ret; 24 25 fault_addr = (trap->htval << 2) | (trap->stval & 0x3); 26 gfn = fault_addr >> PAGE_SHIFT; 27 memslot = gfn_to_memslot(vcpu->kvm, gfn); 28 hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); 29 30 if (kvm_is_error_hva(hva) || 31 (trap->scause == EXC_STORE_GUEST_PAGE_FAULT && !writable)) { 32 switch (trap->scause) { 33 case EXC_LOAD_GUEST_PAGE_FAULT: 34 return kvm_riscv_vcpu_mmio_load(vcpu, run, 35 fault_addr, 36 trap->htinst); 37 case EXC_STORE_GUEST_PAGE_FAULT: 38 return kvm_riscv_vcpu_mmio_store(vcpu, run, 39 fault_addr, 40 trap->htinst); 41 case EXC_INST_GUEST_PAGE_FAULT: { 42 /* 43 * No memslot backs this GPA and an instruction fetch 44 * cannot be emulated as MMIO. On bare metal a fetch 45 * from an unbacked physical address raises an 46 * instruction access fault, so reflect that back to 47 * the guest. 48 */ 49 struct kvm_cpu_trap inst_trap = { 50 .sepc = trap->sepc, 51 .scause = EXC_INST_ACCESS, 52 .stval = trap->stval, 53 .htval = 0, 54 .htinst = 0, 55 }; 56 57 kvm_riscv_vcpu_trap_redirect(vcpu, &inst_trap); 58 return 1; 59 } 60 default: 61 return -EOPNOTSUPP; 62 }; 63 } 64 65 ret = kvm_riscv_mmu_map(vcpu, memslot, fault_addr, hva, 66 (trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false, 67 &host_map); 68 if (ret < 0) 69 return ret; 70 71 return 1; 72 } 73 74 /** 75 * kvm_riscv_vcpu_unpriv_read -- Read machine word from Guest memory 76 * 77 * @vcpu: The VCPU pointer 78 * @read_insn: Flag representing whether we are reading instruction 79 * @guest_addr: Guest address to read 80 * @trap: Output pointer to trap details 81 */ 82 unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu, 83 bool read_insn, 84 unsigned long guest_addr, 85 struct kvm_cpu_trap *trap) 86 { 87 register unsigned long taddr asm("a0") = (unsigned long)trap; 88 register unsigned long ttmp asm("a1"); 89 unsigned long flags, val, tmp, old_stvec, old_hstatus; 90 91 local_irq_save(flags); 92 93 old_hstatus = csr_swap(CSR_HSTATUS, vcpu->arch.guest_context.hstatus); 94 old_stvec = csr_swap(CSR_STVEC, (ulong)&__kvm_riscv_unpriv_trap); 95 96 if (read_insn) { 97 /* 98 * HLVX.HU instruction 99 * 0110010 00011 rs1 100 rd 1110011 100 */ 101 asm volatile ("\n" 102 ".option push\n" 103 ".option norvc\n" 104 "add %[ttmp], %[taddr], 0\n" 105 HLVX_HU(%[val], %[addr]) 106 "andi %[tmp], %[val], 3\n" 107 "addi %[tmp], %[tmp], -3\n" 108 "bne %[tmp], zero, 2f\n" 109 "addi %[addr], %[addr], 2\n" 110 HLVX_HU(%[tmp], %[addr]) 111 "sll %[tmp], %[tmp], 16\n" 112 "add %[val], %[val], %[tmp]\n" 113 "2:\n" 114 ".option pop" 115 : [val] "=&r" (val), [tmp] "=&r" (tmp), 116 [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp), 117 [addr] "+&r" (guest_addr) : : "memory"); 118 119 if (trap->scause == EXC_LOAD_PAGE_FAULT) 120 trap->scause = EXC_INST_PAGE_FAULT; 121 } else { 122 /* 123 * HLV.D instruction 124 * 0110110 00000 rs1 100 rd 1110011 125 * 126 * HLV.W instruction 127 * 0110100 00000 rs1 100 rd 1110011 128 */ 129 asm volatile ("\n" 130 ".option push\n" 131 ".option norvc\n" 132 "add %[ttmp], %[taddr], 0\n" 133 #ifdef CONFIG_64BIT 134 HLV_D(%[val], %[addr]) 135 #else 136 HLV_W(%[val], %[addr]) 137 #endif 138 ".option pop" 139 : [val] "=&r" (val), 140 [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp) 141 : [addr] "r" (guest_addr) : "memory"); 142 } 143 144 csr_write(CSR_STVEC, old_stvec); 145 csr_write(CSR_HSTATUS, old_hstatus); 146 147 local_irq_restore(flags); 148 149 return val; 150 } 151 152 /** 153 * kvm_riscv_vcpu_trap_redirect -- Redirect trap to Guest 154 * 155 * @vcpu: The VCPU pointer 156 * @trap: Trap details 157 */ 158 void kvm_riscv_vcpu_trap_redirect(struct kvm_vcpu *vcpu, 159 struct kvm_cpu_trap *trap) 160 { 161 unsigned long vsstatus = ncsr_read(CSR_VSSTATUS); 162 163 /* Change Guest SSTATUS.SPP bit */ 164 vsstatus &= ~SR_SPP; 165 if (vcpu->arch.guest_context.sstatus & SR_SPP) 166 vsstatus |= SR_SPP; 167 168 /* Change Guest SSTATUS.SPIE bit */ 169 vsstatus &= ~SR_SPIE; 170 if (vsstatus & SR_SIE) 171 vsstatus |= SR_SPIE; 172 173 /* Clear Guest SSTATUS.SIE bit */ 174 vsstatus &= ~SR_SIE; 175 176 /* Update Guest SSTATUS */ 177 ncsr_write(CSR_VSSTATUS, vsstatus); 178 179 /* Update Guest SCAUSE, STVAL, and SEPC */ 180 ncsr_write(CSR_VSCAUSE, trap->scause); 181 ncsr_write(CSR_VSTVAL, trap->stval); 182 ncsr_write(CSR_VSEPC, trap->sepc); 183 184 /* Set Guest PC to Guest exception vector */ 185 vcpu->arch.guest_context.sepc = ncsr_read(CSR_VSTVEC); 186 187 /* Set Guest privilege mode to supervisor */ 188 vcpu->arch.guest_context.sstatus |= SR_SPP; 189 } 190 191 static inline int vcpu_redirect(struct kvm_vcpu *vcpu, struct kvm_cpu_trap *trap) 192 { 193 int ret = -EFAULT; 194 195 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) { 196 kvm_riscv_vcpu_trap_redirect(vcpu, trap); 197 ret = 1; 198 } 199 return ret; 200 } 201 202 /* 203 * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on 204 * proper exit to userspace. 205 */ 206 int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, 207 struct kvm_cpu_trap *trap) 208 { 209 int ret; 210 211 /* If we got host interrupt then do nothing */ 212 if (trap->scause & CAUSE_IRQ_FLAG) 213 return 1; 214 215 /* Handle guest traps */ 216 ret = -EFAULT; 217 run->exit_reason = KVM_EXIT_UNKNOWN; 218 switch (trap->scause) { 219 case EXC_INST_ILLEGAL: 220 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ILLEGAL_INSN); 221 vcpu->stat.instr_illegal_exits++; 222 ret = vcpu_redirect(vcpu, trap); 223 break; 224 case EXC_LOAD_MISALIGNED: 225 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_MISALIGNED_LOAD); 226 vcpu->stat.load_misaligned_exits++; 227 ret = vcpu_redirect(vcpu, trap); 228 break; 229 case EXC_STORE_MISALIGNED: 230 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_MISALIGNED_STORE); 231 vcpu->stat.store_misaligned_exits++; 232 ret = vcpu_redirect(vcpu, trap); 233 break; 234 case EXC_LOAD_ACCESS: 235 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ACCESS_LOAD); 236 vcpu->stat.load_access_exits++; 237 ret = vcpu_redirect(vcpu, trap); 238 break; 239 case EXC_STORE_ACCESS: 240 kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_ACCESS_STORE); 241 vcpu->stat.store_access_exits++; 242 ret = vcpu_redirect(vcpu, trap); 243 break; 244 case EXC_INST_ACCESS: 245 ret = vcpu_redirect(vcpu, trap); 246 break; 247 case EXC_VIRTUAL_INST_FAULT: 248 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) 249 ret = kvm_riscv_vcpu_virtual_insn(vcpu, run, trap); 250 break; 251 case EXC_INST_GUEST_PAGE_FAULT: 252 case EXC_LOAD_GUEST_PAGE_FAULT: 253 case EXC_STORE_GUEST_PAGE_FAULT: 254 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) 255 ret = gstage_page_fault(vcpu, run, trap); 256 break; 257 case EXC_SUPERVISOR_SYSCALL: 258 if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) 259 ret = kvm_riscv_vcpu_sbi_ecall(vcpu, run); 260 break; 261 case EXC_BREAKPOINT: 262 run->exit_reason = KVM_EXIT_DEBUG; 263 ret = 0; 264 break; 265 default: 266 break; 267 } 268 269 /* Print details in-case of error */ 270 if (ret < 0) { 271 kvm_err("VCPU exit error %d\n", ret); 272 kvm_err("SEPC=0x%lx SSTATUS=0x%lx HSTATUS=0x%lx\n", 273 vcpu->arch.guest_context.sepc, 274 vcpu->arch.guest_context.sstatus, 275 vcpu->arch.guest_context.hstatus); 276 kvm_err("SCAUSE=0x%lx STVAL=0x%lx HTVAL=0x%lx HTINST=0x%lx\n", 277 trap->scause, trap->stval, trap->htval, trap->htinst); 278 } 279 280 return ret; 281 } 282