17 static uint64_t virt_pte_index(struct kvm_vm *vm, vm_vaddr_t gva, int level)  in virt_pte_index()  argument
22 	shift = level * (vm->page_shift - 3) + vm->page_shift;  in virt_pte_index()
23 	mask = (1UL << (vm->page_shift - 3)) - 1;  in virt_pte_index()
27 static uint64_t pte_addr(struct kvm_vm *vm, uint64_t entry)  in pte_addr()  argument
29 	return entry &  ~((0x1UL << vm->page_shift) - 1);  in pte_addr()
32 static uint64_t ptrs_per_pte(struct kvm_vm *vm)  in ptrs_per_pte()  argument
34 	return 1 << (vm->page_shift - 3);  in ptrs_per_pte()
37 static void virt_set_pgtable(struct kvm_vm *vm, vm_paddr_t table, vm_paddr_t child)  in virt_set_pgtable()  argument
42 	ptep = addr_gpa2hva(vm, table);  in virt_set_pgtable()
43 	ptrs_per_pte = 1 << (vm->page_shift - 3);  in virt_set_pgtable()
48 void virt_arch_pgd_alloc(struct kvm_vm *vm)  in virt_arch_pgd_alloc()  argument
53 	if (vm->pgd_created)  in virt_arch_pgd_alloc()
57 	for (i = 0; i < vm->pgtable_levels; i++) {  in virt_arch_pgd_alloc()
59 		table = vm_phy_page_alloc(vm, LOONGARCH_PAGE_TABLE_PHYS_MIN,  in virt_arch_pgd_alloc()
60 				vm->memslots[MEM_REGION_PT]);  in virt_arch_pgd_alloc()
62 		virt_set_pgtable(vm, table, child);  in virt_arch_pgd_alloc()
65 	vm->pgd = table;  in virt_arch_pgd_alloc()
66 	vm->pgd_created = true;  in virt_arch_pgd_alloc()
74 static uint64_t *virt_populate_pte(struct kvm_vm *vm, vm_vaddr_t gva, int alloc)  in virt_populate_pte()  argument
80 	if (!vm->pgd_created)  in virt_populate_pte()
83 	child = vm->pgd;  in virt_populate_pte()
84 	level = vm->pgtable_levels - 1;  in virt_populate_pte()
86 		ptep = addr_gpa2hva(vm, child) + virt_pte_index(vm, gva, level) * 8;  in virt_populate_pte()
89 				child = vm_alloc_page_table(vm);  in virt_populate_pte()
90 				virt_set_pgtable(vm, child, invalid_pgtable[level - 1]);  in virt_populate_pte()
96 			child = pte_addr(vm, *ptep);  in virt_populate_pte()
100 	ptep = addr_gpa2hva(vm, child) + virt_pte_index(vm, gva, level) * 8;  in virt_populate_pte()
108 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)  in addr_arch_gva2gpa()  argument
112 	ptep = virt_populate_pte(vm, gva, 0);  in addr_arch_gva2gpa()
115 	return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));  in addr_arch_gva2gpa()
118 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)  in virt_arch_pg_map()  argument
123 	TEST_ASSERT((vaddr % vm->page_size) == 0,  in virt_arch_pg_map()
125 			"vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);  in virt_arch_pg_map()
126 	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,  in virt_arch_pg_map()
127 			(vaddr >> vm->page_shift)),  in virt_arch_pg_map()
129 	TEST_ASSERT((paddr % vm->page_size) == 0,  in virt_arch_pg_map()
131 			"paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);  in virt_arch_pg_map()
132 	TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,  in virt_arch_pg_map()
135 			paddr, vm->max_gfn, vm->page_size);  in virt_arch_pg_map()
137 	ptep = virt_populate_pte(vm, vaddr, 1);  in virt_arch_pg_map()
142 static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)  in pte_dump()  argument
150 	for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) {  in pte_dump()
151 		ptep = addr_gpa2hva(vm, pte);  in pte_dump()
156 		pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level--);  in pte_dump()
160 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)  in virt_arch_dump()  argument
164 	if (!vm->pgd_created)  in virt_arch_dump()
167 	level = vm->pgtable_levels - 1;  in virt_arch_dump()
168 	pte_dump(stream, vm, indent, vm->pgd, level);  in virt_arch_dump()
204 void vm_init_descriptor_tables(struct kvm_vm *vm)  in vm_init_descriptor_tables()  argument
208 	vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers),  in vm_init_descriptor_tables()
211 	addr = addr_gva2hva(vm, vm->handlers);  in vm_init_descriptor_tables()
212 	memset(addr, 0, vm->page_size);  in vm_init_descriptor_tables()
213 	exception_handlers = vm->handlers;  in vm_init_descriptor_tables()
214 	sync_global_to_guest(vm, exception_handlers);  in vm_init_descriptor_tables()
217 void vm_install_exception_handler(struct kvm_vm *vm, int vector, handler_fn handler)  in vm_install_exception_handler()  argument
219 	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);  in vm_install_exception_handler()
274 	struct kvm_vm *vm = vcpu->vm;  in loongarch_vcpu_setup()  local
276 	switch (vm->mode) {  in loongarch_vcpu_setup()
282 		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);  in loongarch_vcpu_setup()
298 	width = vm->page_shift - 3;  in loongarch_vcpu_setup()
300 	switch (vm->pgtable_levels) {  in loongarch_vcpu_setup()
303 		val = (vm->page_shift + width * 2) << 20 | (width << 25);  in loongarch_vcpu_setup()
307 		val |= (vm->page_shift + width) << 10 | (width << 15);  in loongarch_vcpu_setup()
309 		val |= vm->page_shift | width << 5;  in loongarch_vcpu_setup()
312 		TEST_FAIL("Got %u page table levels, expected 3 or 4", vm->pgtable_levels);  in loongarch_vcpu_setup()
318 	val = (vm->page_shift + width * (vm->pgtable_levels - 1)) | width << 6;  in loongarch_vcpu_setup()
320 	loongarch_set_csr(vcpu, LOONGARCH_CSR_PGDL, vm->pgd);  in loongarch_vcpu_setup()
326 	val = addr_gva2gpa(vm, (unsigned long)handle_tlb_refill);  in loongarch_vcpu_setup()
344 	val = __vm_vaddr_alloc(vm, vm->page_size,  in loongarch_vcpu_setup()
347 	val = val + vm->page_size;  in loongarch_vcpu_setup()
359 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)  in vm_arch_vcpu_add()  argument
366 	vcpu = __vm_vcpu_add(vm, vcpu_id);  in vm_arch_vcpu_add()
367 	stack_size = vm->page_size;  in vm_arch_vcpu_add()
368 	stack_vaddr = __vm_vaddr_alloc(vm, stack_size,  in vm_arch_vcpu_add()