1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <assert.h> 4 #include <linux/compiler.h> 5 6 #include <asm/kvm.h> 7 #include "kvm_util.h" 8 #include "pmu.h" 9 #include "processor.h" 10 #include "ucall_common.h" 11 12 #define LOONGARCH_PAGE_TABLE_PHYS_MIN 0x200000 13 #define LOONGARCH_GUEST_STACK_VADDR_MIN 0x200000 14 15 static gpa_t invalid_pgtable[4]; 16 static gva_t exception_handlers; 17 18 static u64 virt_pte_index(struct kvm_vm *vm, gva_t gva, int level) 19 { 20 unsigned int shift; 21 u64 mask; 22 23 shift = level * (vm->page_shift - 3) + vm->page_shift; 24 mask = (1UL << (vm->page_shift - 3)) - 1; 25 return (gva >> shift) & mask; 26 } 27 28 static u64 pte_addr(struct kvm_vm *vm, u64 entry) 29 { 30 return entry & ~((0x1UL << vm->page_shift) - 1); 31 } 32 33 static u64 ptrs_per_pte(struct kvm_vm *vm) 34 { 35 return 1 << (vm->page_shift - 3); 36 } 37 38 static void virt_set_pgtable(struct kvm_vm *vm, gpa_t table, gpa_t child) 39 { 40 u64 *ptep; 41 int i, ptrs_per_pte; 42 43 ptep = addr_gpa2hva(vm, table); 44 ptrs_per_pte = 1 << (vm->page_shift - 3); 45 for (i = 0; i < ptrs_per_pte; i++) 46 WRITE_ONCE(*(ptep + i), child); 47 } 48 49 void virt_arch_pgd_alloc(struct kvm_vm *vm) 50 { 51 int i; 52 gpa_t child, table; 53 54 if (vm->mmu.pgd_created) 55 return; 56 57 child = table = 0; 58 for (i = 0; i < vm->mmu.pgtable_levels; i++) { 59 invalid_pgtable[i] = child; 60 table = vm_phy_page_alloc(vm, LOONGARCH_PAGE_TABLE_PHYS_MIN, 61 vm->memslots[MEM_REGION_PT]); 62 TEST_ASSERT(table, "Fail to allocate page tale at level %d\n", i); 63 virt_set_pgtable(vm, table, child); 64 child = table; 65 } 66 vm->mmu.pgd = table; 67 vm->mmu.pgd_created = true; 68 } 69 70 static int virt_pte_none(u64 *ptep, int level) 71 { 72 return *ptep == invalid_pgtable[level]; 73 } 74 75 static u64 *virt_populate_pte(struct kvm_vm *vm, gva_t gva, int alloc) 76 { 77 int level; 78 u64 *ptep; 79 gpa_t child; 80 81 if (!vm->mmu.pgd_created) 82 goto unmapped_gva; 83 84 child = vm->mmu.pgd; 85 level = vm->mmu.pgtable_levels - 1; 86 while (level > 0) { 87 ptep = addr_gpa2hva(vm, child) + virt_pte_index(vm, gva, level) * 8; 88 if (virt_pte_none(ptep, level)) { 89 if (alloc) { 90 child = vm_alloc_page_table(vm); 91 virt_set_pgtable(vm, child, invalid_pgtable[level - 1]); 92 WRITE_ONCE(*ptep, child); 93 } else 94 goto unmapped_gva; 95 96 } else 97 child = pte_addr(vm, *ptep); 98 level--; 99 } 100 101 ptep = addr_gpa2hva(vm, child) + virt_pte_index(vm, gva, level) * 8; 102 return ptep; 103 104 unmapped_gva: 105 TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva); 106 exit(EXIT_FAILURE); 107 } 108 109 gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva) 110 { 111 u64 *ptep; 112 113 ptep = virt_populate_pte(vm, gva, 0); 114 TEST_ASSERT(*ptep != 0, "Virtual address gva: 0x%lx not mapped\n", gva); 115 116 return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1)); 117 } 118 119 void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa) 120 { 121 u32 prot_bits; 122 u64 *ptep; 123 124 TEST_ASSERT((gva % vm->page_size) == 0, 125 "Virtual address not on page boundary,\n" 126 "gva: 0x%lx vm->page_size: 0x%x", gva, vm->page_size); 127 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)), 128 "Invalid virtual address, gva: 0x%lx", gva); 129 TEST_ASSERT((gpa % vm->page_size) == 0, 130 "Physical address not on page boundary,\n" 131 "gpa: 0x%lx vm->page_size: 0x%x", gpa, vm->page_size); 132 TEST_ASSERT((gpa >> vm->page_shift) <= vm->max_gfn, 133 "Physical address beyond maximum supported,\n" 134 "gpa: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x", 135 gpa, vm->max_gfn, vm->page_size); 136 137 ptep = virt_populate_pte(vm, gva, 1); 138 prot_bits = _PAGE_PRESENT | __READABLE | __WRITEABLE | _CACHE_CC | _PAGE_USER; 139 WRITE_ONCE(*ptep, gpa | prot_bits); 140 } 141 142 static void pte_dump(FILE *stream, struct kvm_vm *vm, u8 indent, u64 page, int level) 143 { 144 u64 pte, *ptep; 145 static const char * const type[] = { "pte", "pmd", "pud", "pgd"}; 146 147 if (level < 0) 148 return; 149 150 for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) { 151 ptep = addr_gpa2hva(vm, pte); 152 if (virt_pte_none(ptep, level)) 153 continue; 154 fprintf(stream, "%*s%s: %lx: %lx at %p\n", 155 indent, "", type[level], pte, *ptep, ptep); 156 pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level--); 157 } 158 } 159 160 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent) 161 { 162 int level; 163 164 if (!vm->mmu.pgd_created) 165 return; 166 167 level = vm->mmu.pgtable_levels - 1; 168 pte_dump(stream, vm, indent, vm->mmu.pgd, level); 169 } 170 171 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) 172 { 173 } 174 175 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) 176 { 177 struct ucall uc; 178 179 if (get_ucall(vcpu, &uc) != UCALL_UNHANDLED) 180 return; 181 182 TEST_FAIL("Unexpected exception (pc:0x%lx, estat:0x%lx, badv:0x%lx)", 183 uc.args[0], uc.args[1], uc.args[2]); 184 } 185 186 void route_exception(struct ex_regs *regs) 187 { 188 int vector; 189 unsigned long pc, estat, badv; 190 struct handlers *handlers; 191 192 handlers = (struct handlers *)exception_handlers; 193 vector = (regs->estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT; 194 if (handlers && handlers->exception_handlers[vector]) 195 return handlers->exception_handlers[vector](regs); 196 197 pc = regs->pc; 198 badv = regs->badv; 199 estat = regs->estat; 200 ucall(UCALL_UNHANDLED, 3, pc, estat, badv); 201 while (1) ; 202 } 203 204 void vm_init_descriptor_tables(struct kvm_vm *vm) 205 { 206 void *addr; 207 208 vm->handlers = __vm_alloc(vm, sizeof(struct handlers), 209 LOONGARCH_GUEST_STACK_VADDR_MIN, 210 MEM_REGION_DATA); 211 212 addr = addr_gva2hva(vm, vm->handlers); 213 memset(addr, 0, vm->page_size); 214 exception_handlers = vm->handlers; 215 sync_global_to_guest(vm, exception_handlers); 216 } 217 218 void vm_install_exception_handler(struct kvm_vm *vm, int vector, handler_fn handler) 219 { 220 struct handlers *handlers = addr_gva2hva(vm, vm->handlers); 221 222 assert(vector < VECTOR_NUM); 223 handlers->exception_handlers[vector] = handler; 224 } 225 226 u32 guest_get_vcpuid(void) 227 { 228 return csr_read(LOONGARCH_CSR_CPUID); 229 } 230 231 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...) 232 { 233 int i; 234 va_list ap; 235 struct kvm_regs regs; 236 237 TEST_ASSERT(num >= 1 && num <= 8, "Unsupported number of args,\n" 238 "num: %u\n", num); 239 240 vcpu_regs_get(vcpu, ®s); 241 242 va_start(ap, num); 243 for (i = 0; i < num; i++) 244 regs.gpr[i + 4] = va_arg(ap, u64); 245 va_end(ap); 246 247 vcpu_regs_set(vcpu, ®s); 248 } 249 250 static void loongarch_set_reg(struct kvm_vcpu *vcpu, u64 id, u64 val) 251 { 252 __vcpu_set_reg(vcpu, id, val); 253 } 254 255 static void loongarch_set_cpucfg(struct kvm_vcpu *vcpu, u64 id, u64 val) 256 { 257 u64 cfgid; 258 259 cfgid = KVM_REG_LOONGARCH_CPUCFG | KVM_REG_SIZE_U64 | 8 * id; 260 __vcpu_set_reg(vcpu, cfgid, val); 261 } 262 263 static void loongarch_get_csr(struct kvm_vcpu *vcpu, u64 id, void *addr) 264 { 265 u64 csrid; 266 267 csrid = KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U64 | 8 * id; 268 __vcpu_get_reg(vcpu, csrid, addr); 269 } 270 271 static void loongarch_set_csr(struct kvm_vcpu *vcpu, u64 id, u64 val) 272 { 273 u64 csrid; 274 275 csrid = KVM_REG_LOONGARCH_CSR | KVM_REG_SIZE_U64 | 8 * id; 276 __vcpu_set_reg(vcpu, csrid, val); 277 } 278 279 void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) 280 { 281 int ret, width; 282 unsigned int cfg; 283 unsigned long val; 284 struct kvm_vm *vm = vcpu->vm; 285 286 switch (vm->mode) { 287 case VM_MODE_P36V47_16K: 288 case VM_MODE_P47V47_16K: 289 break; 290 291 default: 292 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode); 293 } 294 295 /* Enable LSX/LASX feature by detecting host capability */ 296 cfg = CPUCFG2_FP | CPUCFG2_FPSP | CPUCFG2_FPDP | CPUCFG2_LLFTP; 297 ret = __kvm_has_device_attr(vm->fd, KVM_LOONGARCH_VM_FEAT_CTRL, KVM_LOONGARCH_VM_FEAT_LSX); 298 if (!ret) 299 cfg |= CPUCFG2_LSX; 300 ret = __kvm_has_device_attr(vm->fd, KVM_LOONGARCH_VM_FEAT_CTRL, KVM_LOONGARCH_VM_FEAT_LASX); 301 if (!ret) 302 cfg |= CPUCFG2_LASX; 303 loongarch_set_cpucfg(vcpu, LOONGARCH_CPUCFG2, cfg); 304 cfg = read_cpucfg(LOONGARCH_CPUCFG6); 305 loongarch_set_cpucfg(vcpu, LOONGARCH_CPUCFG6, cfg); 306 307 /* kernel mode and page enable mode */ 308 val = PLV_KERN | CSR_CRMD_PG; 309 loongarch_set_csr(vcpu, LOONGARCH_CSR_CRMD, val); 310 loongarch_set_csr(vcpu, LOONGARCH_CSR_PRMD, val); 311 loongarch_set_csr(vcpu, LOONGARCH_CSR_EUEN, 1); 312 loongarch_set_csr(vcpu, LOONGARCH_CSR_ECFG, 0); 313 loongarch_set_csr(vcpu, LOONGARCH_CSR_TCFG, 0); 314 loongarch_set_csr(vcpu, LOONGARCH_CSR_ASID, 1); 315 316 /* time count start from 0 */ 317 val = 0; 318 loongarch_set_reg(vcpu, KVM_REG_LOONGARCH_COUNTER, val); 319 320 width = vm->page_shift - 3; 321 322 switch (vm->mmu.pgtable_levels) { 323 case 4: 324 /* pud page shift and width */ 325 val = (vm->page_shift + width * 2) << 20 | (width << 25); 326 /* fall throuth */ 327 case 3: 328 /* pmd page shift and width */ 329 val |= (vm->page_shift + width) << 10 | (width << 15); 330 /* pte page shift and width */ 331 val |= vm->page_shift | width << 5; 332 break; 333 default: 334 TEST_FAIL("Got %u page table levels, expected 3 or 4", vm->mmu.pgtable_levels); 335 } 336 337 loongarch_set_csr(vcpu, LOONGARCH_CSR_PWCTL0, val); 338 339 /* PGD page shift and width */ 340 val = (vm->page_shift + width * (vm->mmu.pgtable_levels - 1)) | width << 6; 341 loongarch_set_csr(vcpu, LOONGARCH_CSR_PWCTL1, val); 342 loongarch_set_csr(vcpu, LOONGARCH_CSR_PGDL, vm->mmu.pgd); 343 344 /* 345 * Refill exception runs on real mode 346 * Entry address should be physical address 347 */ 348 val = addr_gva2gpa(vm, (unsigned long)handle_tlb_refill); 349 loongarch_set_csr(vcpu, LOONGARCH_CSR_TLBRENTRY, val); 350 351 /* 352 * General exception runs on page-enabled mode 353 * Entry address should be virtual address 354 */ 355 val = (unsigned long)handle_exception; 356 loongarch_set_csr(vcpu, LOONGARCH_CSR_EENTRY, val); 357 358 loongarch_get_csr(vcpu, LOONGARCH_CSR_TLBIDX, &val); 359 val &= ~CSR_TLBIDX_SIZEM; 360 val |= PS_DEFAULT_SIZE << CSR_TLBIDX_SIZE; 361 loongarch_set_csr(vcpu, LOONGARCH_CSR_TLBIDX, val); 362 363 loongarch_set_csr(vcpu, LOONGARCH_CSR_STLBPGSIZE, PS_DEFAULT_SIZE); 364 365 /* LOONGARCH_CSR_KS1 is used for exception stack */ 366 val = __vm_alloc(vm, vm->page_size, LOONGARCH_GUEST_STACK_VADDR_MIN, 367 MEM_REGION_DATA); 368 TEST_ASSERT(val != 0, "No memory for exception stack"); 369 val = val + vm->page_size; 370 loongarch_set_csr(vcpu, LOONGARCH_CSR_KS1, val); 371 372 loongarch_get_csr(vcpu, LOONGARCH_CSR_TLBREHI, &val); 373 val &= ~CSR_TLBREHI_PS; 374 val |= PS_DEFAULT_SIZE << CSR_TLBREHI_PS_SHIFT; 375 loongarch_set_csr(vcpu, LOONGARCH_CSR_TLBREHI, val); 376 377 loongarch_set_csr(vcpu, LOONGARCH_CSR_CPUID, vcpu->id); 378 loongarch_set_csr(vcpu, LOONGARCH_CSR_TMID, vcpu->id); 379 } 380 381 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id) 382 { 383 size_t stack_size; 384 u64 stack_gva; 385 struct kvm_regs regs; 386 struct kvm_vcpu *vcpu; 387 388 vcpu = __vm_vcpu_add(vm, vcpu_id); 389 stack_size = vm->page_size; 390 stack_gva = __vm_alloc(vm, stack_size, 391 LOONGARCH_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA); 392 TEST_ASSERT(stack_gva != 0, "No memory for vm stack"); 393 394 loongarch_vcpu_setup(vcpu); 395 /* Setup guest general purpose registers */ 396 vcpu_regs_get(vcpu, ®s); 397 regs.gpr[3] = stack_gva + stack_size; 398 vcpu_regs_set(vcpu, ®s); 399 400 return vcpu; 401 } 402 403 void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) 404 { 405 struct kvm_regs regs; 406 407 /* Setup guest PC register */ 408 vcpu_regs_get(vcpu, ®s); 409 regs.pc = (u64)guest_code; 410 vcpu_regs_set(vcpu, ®s); 411 } 412