1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * RISC-V code 4 * 5 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 6 */ 7 8 #include <linux/compiler.h> 9 #include <assert.h> 10 11 #include "kvm_util.h" 12 #include "processor.h" 13 14 #define DEFAULT_RISCV_GUEST_STACK_VADDR_MIN 0xac0000 15 16 static vm_vaddr_t exception_handlers; 17 18 static uint64_t page_align(struct kvm_vm *vm, uint64_t v) 19 { 20 return (v + vm->page_size) & ~(vm->page_size - 1); 21 } 22 23 static uint64_t pte_addr(struct kvm_vm *vm, uint64_t entry) 24 { 25 return ((entry & PGTBL_PTE_ADDR_MASK) >> PGTBL_PTE_ADDR_SHIFT) << 26 PGTBL_PAGE_SIZE_SHIFT; 27 } 28 29 static uint64_t ptrs_per_pte(struct kvm_vm *vm) 30 { 31 return PGTBL_PAGE_SIZE / sizeof(uint64_t); 32 } 33 34 static uint64_t pte_index_mask[] = { 35 PGTBL_L0_INDEX_MASK, 36 PGTBL_L1_INDEX_MASK, 37 PGTBL_L2_INDEX_MASK, 38 PGTBL_L3_INDEX_MASK, 39 }; 40 41 static uint32_t pte_index_shift[] = { 42 PGTBL_L0_INDEX_SHIFT, 43 PGTBL_L1_INDEX_SHIFT, 44 PGTBL_L2_INDEX_SHIFT, 45 PGTBL_L3_INDEX_SHIFT, 46 }; 47 48 static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva, int level) 49 { 50 TEST_ASSERT(level > -1, 51 "Negative page table level (%d) not possible", level); 52 TEST_ASSERT(level < vm->pgtable_levels, 53 "Invalid page table level (%d)", level); 54 55 return (gva & pte_index_mask[level]) >> pte_index_shift[level]; 56 } 57 58 void virt_arch_pgd_alloc(struct kvm_vm *vm) 59 { 60 size_t nr_pages = page_align(vm, ptrs_per_pte(vm) * 8) / vm->page_size; 61 62 if (vm->pgd_created) 63 return; 64 65 vm->pgd = vm_phy_pages_alloc(vm, nr_pages, 66 KVM_GUEST_PAGE_TABLE_MIN_PADDR, 67 vm->memslots[MEM_REGION_PT]); 68 vm->pgd_created = true; 69 } 70 71 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) 72 { 73 uint64_t *ptep, next_ppn; 74 int level = vm->pgtable_levels - 1; 75 76 TEST_ASSERT((vaddr % vm->page_size) == 0, 77 "Virtual address not on page boundary,\n" 78 " vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size); 79 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, 80 (vaddr >> vm->page_shift)), 81 "Invalid virtual address, vaddr: 0x%lx", vaddr); 82 TEST_ASSERT((paddr % vm->page_size) == 0, 83 "Physical address not on page boundary,\n" 84 " paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size); 85 TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn, 86 "Physical address beyond maximum supported,\n" 87 " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x", 88 paddr, vm->max_gfn, vm->page_size); 89 90 ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, vaddr, level) * 8; 91 if (!*ptep) { 92 next_ppn = vm_alloc_page_table(vm) >> PGTBL_PAGE_SIZE_SHIFT; 93 *ptep = (next_ppn << PGTBL_PTE_ADDR_SHIFT) | 94 PGTBL_PTE_VALID_MASK; 95 } 96 level--; 97 98 while (level > -1) { 99 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + 100 pte_index(vm, vaddr, level) * 8; 101 if (!*ptep && level > 0) { 102 next_ppn = vm_alloc_page_table(vm) >> 103 PGTBL_PAGE_SIZE_SHIFT; 104 *ptep = (next_ppn << PGTBL_PTE_ADDR_SHIFT) | 105 PGTBL_PTE_VALID_MASK; 106 } 107 level--; 108 } 109 110 paddr = paddr >> PGTBL_PAGE_SIZE_SHIFT; 111 *ptep = (paddr << PGTBL_PTE_ADDR_SHIFT) | 112 PGTBL_PTE_PERM_MASK | PGTBL_PTE_VALID_MASK; 113 } 114 115 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) 116 { 117 uint64_t *ptep; 118 int level = vm->pgtable_levels - 1; 119 120 if (!vm->pgd_created) 121 goto unmapped_gva; 122 123 ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, gva, level) * 8; 124 if (!ptep) 125 goto unmapped_gva; 126 level--; 127 128 while (level > -1) { 129 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + 130 pte_index(vm, gva, level) * 8; 131 if (!ptep) 132 goto unmapped_gva; 133 level--; 134 } 135 136 return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1)); 137 138 unmapped_gva: 139 TEST_FAIL("No mapping for vm virtual address gva: 0x%lx level: %d", 140 gva, level); 141 exit(1); 142 } 143 144 static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, 145 uint64_t page, int level) 146 { 147 #ifdef DEBUG 148 static const char *const type[] = { "pte", "pmd", "pud", "p4d"}; 149 uint64_t pte, *ptep; 150 151 if (level < 0) 152 return; 153 154 for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) { 155 ptep = addr_gpa2hva(vm, pte); 156 if (!*ptep) 157 continue; 158 fprintf(stream, "%*s%s: %lx: %lx at %p\n", indent, "", 159 type[level], pte, *ptep, ptep); 160 pte_dump(stream, vm, indent + 1, 161 pte_addr(vm, *ptep), level - 1); 162 } 163 #endif 164 } 165 166 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) 167 { 168 int level = vm->pgtable_levels - 1; 169 uint64_t pgd, *ptep; 170 171 if (!vm->pgd_created) 172 return; 173 174 for (pgd = vm->pgd; pgd < vm->pgd + ptrs_per_pte(vm) * 8; pgd += 8) { 175 ptep = addr_gpa2hva(vm, pgd); 176 if (!*ptep) 177 continue; 178 fprintf(stream, "%*spgd: %lx: %lx at %p\n", indent, "", 179 pgd, *ptep, ptep); 180 pte_dump(stream, vm, indent + 1, 181 pte_addr(vm, *ptep), level - 1); 182 } 183 } 184 185 void riscv_vcpu_mmu_setup(struct kvm_vcpu *vcpu) 186 { 187 struct kvm_vm *vm = vcpu->vm; 188 unsigned long satp; 189 190 /* 191 * The RISC-V Sv48 MMU mode supports 56-bit physical address 192 * for 48-bit virtual address with 4KB last level page size. 193 */ 194 switch (vm->mode) { 195 case VM_MODE_P52V48_4K: 196 case VM_MODE_P48V48_4K: 197 case VM_MODE_P40V48_4K: 198 break; 199 default: 200 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode); 201 } 202 203 satp = (vm->pgd >> PGTBL_PAGE_SIZE_SHIFT) & SATP_PPN; 204 satp |= SATP_MODE_48; 205 206 vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(satp), satp); 207 } 208 209 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) 210 { 211 struct kvm_riscv_core core; 212 213 vcpu_get_reg(vcpu, RISCV_CORE_REG(mode), &core.mode); 214 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.pc), &core.regs.pc); 215 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.ra), &core.regs.ra); 216 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.sp), &core.regs.sp); 217 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.gp), &core.regs.gp); 218 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.tp), &core.regs.tp); 219 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t0), &core.regs.t0); 220 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t1), &core.regs.t1); 221 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t2), &core.regs.t2); 222 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s0), &core.regs.s0); 223 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s1), &core.regs.s1); 224 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.a0), &core.regs.a0); 225 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.a1), &core.regs.a1); 226 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.a2), &core.regs.a2); 227 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.a3), &core.regs.a3); 228 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.a4), &core.regs.a4); 229 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.a5), &core.regs.a5); 230 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.a6), &core.regs.a6); 231 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.a7), &core.regs.a7); 232 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s2), &core.regs.s2); 233 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s3), &core.regs.s3); 234 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s4), &core.regs.s4); 235 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s5), &core.regs.s5); 236 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s6), &core.regs.s6); 237 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s7), &core.regs.s7); 238 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s8), &core.regs.s8); 239 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s9), &core.regs.s9); 240 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s10), &core.regs.s10); 241 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.s11), &core.regs.s11); 242 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t3), &core.regs.t3); 243 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t4), &core.regs.t4); 244 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t5), &core.regs.t5); 245 vcpu_get_reg(vcpu, RISCV_CORE_REG(regs.t6), &core.regs.t6); 246 247 fprintf(stream, 248 " MODE: 0x%lx\n", core.mode); 249 fprintf(stream, 250 " PC: 0x%016lx RA: 0x%016lx SP: 0x%016lx GP: 0x%016lx\n", 251 core.regs.pc, core.regs.ra, core.regs.sp, core.regs.gp); 252 fprintf(stream, 253 " TP: 0x%016lx T0: 0x%016lx T1: 0x%016lx T2: 0x%016lx\n", 254 core.regs.tp, core.regs.t0, core.regs.t1, core.regs.t2); 255 fprintf(stream, 256 " S0: 0x%016lx S1: 0x%016lx A0: 0x%016lx A1: 0x%016lx\n", 257 core.regs.s0, core.regs.s1, core.regs.a0, core.regs.a1); 258 fprintf(stream, 259 " A2: 0x%016lx A3: 0x%016lx A4: 0x%016lx A5: 0x%016lx\n", 260 core.regs.a2, core.regs.a3, core.regs.a4, core.regs.a5); 261 fprintf(stream, 262 " A6: 0x%016lx A7: 0x%016lx S2: 0x%016lx S3: 0x%016lx\n", 263 core.regs.a6, core.regs.a7, core.regs.s2, core.regs.s3); 264 fprintf(stream, 265 " S4: 0x%016lx S5: 0x%016lx S6: 0x%016lx S7: 0x%016lx\n", 266 core.regs.s4, core.regs.s5, core.regs.s6, core.regs.s7); 267 fprintf(stream, 268 " S8: 0x%016lx S9: 0x%016lx S10: 0x%016lx S11: 0x%016lx\n", 269 core.regs.s8, core.regs.s9, core.regs.s10, core.regs.s11); 270 fprintf(stream, 271 " T3: 0x%016lx T4: 0x%016lx T5: 0x%016lx T6: 0x%016lx\n", 272 core.regs.t3, core.regs.t4, core.regs.t5, core.regs.t6); 273 } 274 275 static void __aligned(16) guest_unexp_trap(void) 276 { 277 sbi_ecall(KVM_RISCV_SELFTESTS_SBI_EXT, 278 KVM_RISCV_SELFTESTS_SBI_UNEXP, 279 0, 0, 0, 0, 0, 0); 280 } 281 282 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, 283 void *guest_code) 284 { 285 int r; 286 size_t stack_size; 287 unsigned long stack_vaddr; 288 unsigned long current_gp = 0; 289 struct kvm_mp_state mps; 290 struct kvm_vcpu *vcpu; 291 292 stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size : 293 vm->page_size; 294 stack_vaddr = __vm_vaddr_alloc(vm, stack_size, 295 DEFAULT_RISCV_GUEST_STACK_VADDR_MIN, 296 MEM_REGION_DATA); 297 298 vcpu = __vm_vcpu_add(vm, vcpu_id); 299 riscv_vcpu_mmu_setup(vcpu); 300 301 /* 302 * With SBI HSM support in KVM RISC-V, all secondary VCPUs are 303 * powered-off by default so we ensure that all secondary VCPUs 304 * are powered-on using KVM_SET_MP_STATE ioctl(). 305 */ 306 mps.mp_state = KVM_MP_STATE_RUNNABLE; 307 r = __vcpu_ioctl(vcpu, KVM_SET_MP_STATE, &mps); 308 TEST_ASSERT(!r, "IOCTL KVM_SET_MP_STATE failed (error %d)", r); 309 310 /* Setup global pointer of guest to be same as the host */ 311 asm volatile ( 312 "add %0, gp, zero" : "=r" (current_gp) : : "memory"); 313 vcpu_set_reg(vcpu, RISCV_CORE_REG(regs.gp), current_gp); 314 315 /* Setup stack pointer and program counter of guest */ 316 vcpu_set_reg(vcpu, RISCV_CORE_REG(regs.sp), stack_vaddr + stack_size); 317 vcpu_set_reg(vcpu, RISCV_CORE_REG(regs.pc), (unsigned long)guest_code); 318 319 /* Setup sscratch for guest_get_vcpuid() */ 320 vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(sscratch), vcpu_id); 321 322 /* Setup default exception vector of guest */ 323 vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(stvec), (unsigned long)guest_unexp_trap); 324 325 return vcpu; 326 } 327 328 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...) 329 { 330 va_list ap; 331 uint64_t id = RISCV_CORE_REG(regs.a0); 332 int i; 333 334 TEST_ASSERT(num >= 1 && num <= 8, "Unsupported number of args,\n" 335 " num: %u", num); 336 337 va_start(ap, num); 338 339 for (i = 0; i < num; i++) { 340 switch (i) { 341 case 0: 342 id = RISCV_CORE_REG(regs.a0); 343 break; 344 case 1: 345 id = RISCV_CORE_REG(regs.a1); 346 break; 347 case 2: 348 id = RISCV_CORE_REG(regs.a2); 349 break; 350 case 3: 351 id = RISCV_CORE_REG(regs.a3); 352 break; 353 case 4: 354 id = RISCV_CORE_REG(regs.a4); 355 break; 356 case 5: 357 id = RISCV_CORE_REG(regs.a5); 358 break; 359 case 6: 360 id = RISCV_CORE_REG(regs.a6); 361 break; 362 case 7: 363 id = RISCV_CORE_REG(regs.a7); 364 break; 365 } 366 vcpu_set_reg(vcpu, id, va_arg(ap, uint64_t)); 367 } 368 369 va_end(ap); 370 } 371 372 void kvm_exit_unexpected_exception(int vector, int ec) 373 { 374 ucall(UCALL_UNHANDLED, 2, vector, ec); 375 } 376 377 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) 378 { 379 struct ucall uc; 380 381 if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED) { 382 TEST_FAIL("Unexpected exception (vector:0x%lx, ec:0x%lx)", 383 uc.args[0], uc.args[1]); 384 } 385 } 386 387 struct handlers { 388 exception_handler_fn exception_handlers[NR_VECTORS][NR_EXCEPTIONS]; 389 }; 390 391 void route_exception(struct ex_regs *regs) 392 { 393 struct handlers *handlers = (struct handlers *)exception_handlers; 394 int vector = 0, ec; 395 396 ec = regs->cause & ~CAUSE_IRQ_FLAG; 397 if (ec >= NR_EXCEPTIONS) 398 goto unexpected_exception; 399 400 /* Use the same handler for all the interrupts */ 401 if (regs->cause & CAUSE_IRQ_FLAG) { 402 vector = 1; 403 ec = 0; 404 } 405 406 if (handlers && handlers->exception_handlers[vector][ec]) 407 return handlers->exception_handlers[vector][ec](regs); 408 409 unexpected_exception: 410 return kvm_exit_unexpected_exception(vector, ec); 411 } 412 413 void vcpu_init_vector_tables(struct kvm_vcpu *vcpu) 414 { 415 extern char exception_vectors; 416 417 vcpu_set_reg(vcpu, RISCV_GENERAL_CSR_REG(stvec), (unsigned long)&exception_vectors); 418 } 419 420 void vm_init_vector_tables(struct kvm_vm *vm) 421 { 422 vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers), 423 vm->page_size, MEM_REGION_DATA); 424 425 *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers; 426 } 427 428 void vm_install_exception_handler(struct kvm_vm *vm, int vector, exception_handler_fn handler) 429 { 430 struct handlers *handlers = addr_gva2hva(vm, vm->handlers); 431 432 assert(vector < NR_EXCEPTIONS); 433 handlers->exception_handlers[0][vector] = handler; 434 } 435 436 void vm_install_interrupt_handler(struct kvm_vm *vm, exception_handler_fn handler) 437 { 438 struct handlers *handlers = addr_gva2hva(vm, vm->handlers); 439 440 handlers->exception_handlers[1][0] = handler; 441 } 442 443 uint32_t guest_get_vcpuid(void) 444 { 445 return csr_read(CSR_SSCRATCH); 446 } 447 448 struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, 449 unsigned long arg1, unsigned long arg2, 450 unsigned long arg3, unsigned long arg4, 451 unsigned long arg5) 452 { 453 register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); 454 register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); 455 register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); 456 register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); 457 register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); 458 register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); 459 register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); 460 register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); 461 struct sbiret ret; 462 463 asm volatile ( 464 "ecall" 465 : "+r" (a0), "+r" (a1) 466 : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) 467 : "memory"); 468 ret.error = a0; 469 ret.value = a1; 470 471 return ret; 472 } 473 474 bool guest_sbi_probe_extension(int extid, long *out_val) 475 { 476 struct sbiret ret; 477 478 ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid, 479 0, 0, 0, 0, 0); 480 481 __GUEST_ASSERT(!ret.error || ret.error == SBI_ERR_NOT_SUPPORTED, 482 "ret.error=%ld, ret.value=%ld\n", ret.error, ret.value); 483 484 if (ret.error == SBI_ERR_NOT_SUPPORTED) 485 return false; 486 487 if (out_val) 488 *out_val = ret.value; 489 490 return true; 491 } 492