164 static void virt_mmu_init(struct kvm_vm *vm, struct kvm_mmu *mmu,  in virt_mmu_init()  argument
168 	if (!mmu->pgd_created) {  in virt_mmu_init()
169 		mmu->pgd = vm_alloc_page_table(vm);  in virt_mmu_init()
170 		mmu->pgd_created = true;  in virt_mmu_init()
171 		mmu->arch.pte_masks = *pte_masks;  in virt_mmu_init()
174 	TEST_ASSERT(mmu->pgtable_levels == 4 || mmu->pgtable_levels == 5,  in virt_mmu_init()
175 		    "Selftests MMU only supports 4-level and 5-level paging, not %u-level paging",  in virt_mmu_init()
176 		    mmu->pgtable_levels);  in virt_mmu_init()
197 	virt_mmu_init(vm, &vm->mmu, &pte_masks);  in virt_arch_pgd_alloc()
203 	TEST_ASSERT(!vm->stage2_mmu.pgtable_levels, "TDP MMU already initialized");  in tdp_mmu_init()
209 static void *virt_get_pte(struct kvm_vm *vm, struct kvm_mmu *mmu,  in virt_get_pte()  argument
216 	TEST_ASSERT((*parent_pte == mmu->pgd) || is_present_pte(mmu, parent_pte),  in virt_get_pte()
224 				  struct kvm_mmu *mmu,  in virt_create_upper_pte()  argument
231 	u64 *pte = virt_get_pte(vm, mmu, parent_pte, gva, current_level);  in virt_create_upper_pte()
235 	if (!is_present_pte(mmu, pte)) {  in virt_create_upper_pte()
236 		*pte = PTE_PRESENT_MASK(mmu) | PTE_READABLE_MASK(mmu) |  in virt_create_upper_pte()
237 		       PTE_WRITABLE_MASK(mmu) | PTE_EXECUTABLE_MASK(mmu) |  in virt_create_upper_pte()
238 		       PTE_ALWAYS_SET_MASK(mmu);  in virt_create_upper_pte()
240 			*pte |= PTE_HUGE_MASK(mmu) | (gpa & PHYSICAL_PAGE_MASK);  in virt_create_upper_pte()
252 		TEST_ASSERT(!is_huge_pte(mmu, pte),  in virt_create_upper_pte()
259 void __virt_pg_map(struct kvm_vm *vm, struct kvm_mmu *mmu, gva_t gva,  in __virt_pg_map()  argument
263 	u64 *pte = &mmu->pgd;  in __virt_pg_map()
284 	TEST_ASSERT(!PTE_EXECUTABLE_MASK(mmu) || !PTE_NX_MASK(mmu),  in __virt_pg_map()
291 	for (current_level = mmu->pgtable_levels;  in __virt_pg_map()
294 		pte = virt_create_upper_pte(vm, mmu, pte, gva, gpa,  in __virt_pg_map()
296 		if (is_huge_pte(mmu, pte))  in __virt_pg_map()
301 	pte = virt_get_pte(vm, mmu, pte, gva, PG_LEVEL_4K);  in __virt_pg_map()
302 	TEST_ASSERT(!is_present_pte(mmu, pte),  in __virt_pg_map()
304 	*pte = PTE_PRESENT_MASK(mmu) | PTE_READABLE_MASK(mmu) |  in __virt_pg_map()
305 	       PTE_WRITABLE_MASK(mmu) | PTE_EXECUTABLE_MASK(mmu) |  in __virt_pg_map()
306 	       PTE_ALWAYS_SET_MASK(mmu) | (gpa & PHYSICAL_PAGE_MASK);  in __virt_pg_map()
313 		*pte |= PTE_C_BIT_MASK(mmu);  in __virt_pg_map()
315 		*pte |= PTE_S_BIT_MASK(mmu);  in __virt_pg_map()
320 	__virt_pg_map(vm, &vm->mmu, gva, gpa, PG_LEVEL_4K);  in virt_arch_pg_map()
335 		__virt_pg_map(vm, &vm->mmu, gva, gpa, level);  in virt_map_level()
344 static bool vm_is_target_pte(struct kvm_mmu *mmu, u64 *pte,  in vm_is_target_pte()  argument
347 	if (is_huge_pte(mmu, pte)) {  in vm_is_target_pte()
358 				      struct kvm_mmu *mmu,  in __vm_get_page_table_entry()  argument
362 	int va_width = 12 + (mmu->pgtable_levels) * 9;  in __vm_get_page_table_entry()
363 	u64 *pte = &mmu->pgd;  in __vm_get_page_table_entry()
369 	TEST_ASSERT(*level >= PG_LEVEL_NONE && *level <= mmu->pgtable_levels,  in __vm_get_page_table_entry()
382 	for (current_level = mmu->pgtable_levels;  in __vm_get_page_table_entry()
385 		pte = virt_get_pte(vm, mmu, pte, gva, current_level);  in __vm_get_page_table_entry()
386 		if (vm_is_target_pte(mmu, pte, level, current_level))  in __vm_get_page_table_entry()
390 	return virt_get_pte(vm, mmu, pte, gva, PG_LEVEL_4K);  in __vm_get_page_table_entry()
404 	return __vm_get_page_table_entry(vm, &vm->mmu, gva, &level);  in vm_get_pte()
409 	struct kvm_mmu *mmu = &vm->mmu;  in virt_arch_dump()  local
415 	if (!mmu->pgd_created)  in virt_arch_dump()
423 	pml4e_start = (u64 *)addr_gpa2hva(vm, mmu->pgd);  in virt_arch_dump()
426 		if (!is_present_pte(mmu, pml4e))  in virt_arch_dump()
433 			is_writable_pte(mmu, pml4e), is_nx_pte(mmu, pml4e));  in virt_arch_dump()
438 			if (!is_present_pte(mmu, pdpe))  in virt_arch_dump()
445 				PTE_GET_PFN(*pdpe), is_writable_pte(mmu, pdpe),  in virt_arch_dump()
446 				is_nx_pte(mmu, pdpe));  in virt_arch_dump()
451 				if (!is_present_pte(mmu, pde))  in virt_arch_dump()
457 					PTE_GET_PFN(*pde), is_writable_pte(mmu, pde),  in virt_arch_dump()
458 					is_nx_pte(mmu, pde));  in virt_arch_dump()
463 					if (!is_present_pte(mmu, pte))  in virt_arch_dump()
472 						is_writable_pte(mmu, pte),  in virt_arch_dump()
473 						is_nx_pte(mmu, pte),  in virt_arch_dump()
474 						is_dirty_pte(mmu, pte),  in virt_arch_dump()
619 	u64 *pte = __vm_get_page_table_entry(vm, &vm->mmu, gva, &level);  in addr_arch_gva2gpa()
621 	TEST_ASSERT(is_present_pte(&vm->mmu, pte),  in addr_arch_gva2gpa()
660 	if (vm->mmu.pgtable_levels == 5)  in vcpu_init_sregs()
671 	sregs.cr3 = vm->mmu.pgd;  in vcpu_init_sregs()