1 // SPDX-License-Identifier: GPL-2.0-only
3  * Kernel-based Virtual Machine driver for Linux
5  * This module enables machines with Intel VT-x extensions to run virtual
57 #include <asm/spec-ctrl.h>
64 int __read_mostly nx_huge_pages = -1;
100  * When setting this variable to true it enables Two-Dimensional-Paging
102  * 1. the guest-virtual to guest-physical
103  * 2. while doing 1. it walks guest-physical to host-physical
203 	return !!(regs->reg & flag);					\
225 	return !!(mmu->cpu_role. base_or_ext . reg##_##name);	\
228 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pse);
229 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smep);
230 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smap);
231 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pke);
232 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, la57);
234 BUILD_MMU_ROLE_ACCESSOR(ext,  efer, lma);
238         return mmu->cpu_role.base.level > 0;  in is_cr0_pg()
243         return !mmu->cpu_role.base.has_4_byte_gpte;  in is_cr4_pae()
251 		.efer = vcpu->arch.efer,  in vcpu_to_role_regs()
265 	if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && mmu->get_guest_pgd == get_guest_cr3)  in kvm_mmu_get_guest_pgd()
268 	return mmu->get_guest_pgd(vcpu);  in kvm_mmu_get_guest_pgd()
288 	kvm_flush_remote_tlbs_gfn(kvm, gfn, sp->role.level);  in kvm_flush_remote_tlbs_sptep()
317 	u64 kvm_gen, spte_gen, gen;  in check_mmio_spte()  local
319 	gen = kvm_vcpu_memslots(vcpu)->generation;  in check_mmio_spte()
320 	if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS))  in check_mmio_spte()
323 	kvm_gen = gen & MMIO_SPTE_GEN_MASK;  in check_mmio_spte()
376 	sp->clear_spte_count++;  in count_spte_clear()
386 	ssptep->spte_high = sspte.spte_high;  in __set_spte()
395 	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);  in __set_spte()
405 	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);  in __update_clear_spte_fast()
413 	ssptep->spte_high = sspte.spte_high;  in __update_clear_spte_fast()
425 	orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);  in __update_clear_spte_slow()
426 	orig.spte_high = ssptep->spte_high;  in __update_clear_spte_slow()
427 	ssptep->spte_high = sspte.spte_high;  in __update_clear_spte_slow()
439  * we need to protect against in-progress updates of the spte.
442  * for the high part of the spte.  The race is fine for a present->non-present
443  * change (because the high part of the spte is ignored for non-present spte),
444  * but for a present->present change we must reread the spte.
446  * All such changes are done in two steps (present->non-present and
447  * non-present->present), hence it is enough to count the number of
448  * present->non-present updates: if it changed while reading the spte,
458 	count = sp->clear_spte_count;  in __get_spte_lockless()
461 	spte.spte_low = orig->spte_low;  in __get_spte_lockless()
464 	spte.spte_high = orig->spte_high;  in __get_spte_lockless()
467 	if (unlikely(spte.spte_low != orig->spte_low ||  in __get_spte_lockless()
468 	      count != sp->clear_spte_count))  in __get_spte_lockless()
524 	int level = sptep_to_sp(sptep)->role.level;  in mmu_spte_clear_track_bits()
535 	kvm_update_page_stats(kvm, level, -1);  in mmu_spte_clear_track_bits()
556 	return tdp_mmu_enabled && vcpu->arch.mmu->root_role.direct;  in is_tdp_mmu_active()
565 		 * Prevent page table teardown by making any free-er wait during  in walk_shadow_page_lockless_begin()
572 		 * to vcpu->mode.  in walk_shadow_page_lockless_begin()
574 		smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);  in walk_shadow_page_lockless_begin()
584 		 * Make sure the write to vcpu->mode is not reordered in front of  in walk_shadow_page_lockless_end()
588 		smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);  in walk_shadow_page_lockless_end()
598 	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,  in mmu_topup_memory_caches()
602 	if (kvm_has_mirrored_tdp(vcpu->kvm)) {  in mmu_topup_memory_caches()
603 		r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_external_spt_cache,  in mmu_topup_memory_caches()
608 	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,  in mmu_topup_memory_caches()
613 		r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadowed_info_cache,  in mmu_topup_memory_caches()
618 	return kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,  in mmu_topup_memory_caches()
624 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache);  in mmu_free_memory_caches()
625 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadow_page_cache);  in mmu_free_memory_caches()
626 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadowed_info_cache);  in mmu_free_memory_caches()
627 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_external_spt_cache);  in mmu_free_memory_caches()
628 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);  in mmu_free_memory_caches()
640 	if (sp->role.passthrough)  in kvm_mmu_page_get_gfn()
641 		return sp->gfn;  in kvm_mmu_page_get_gfn()
643 	if (sp->shadowed_translation)  in kvm_mmu_page_get_gfn()
644 		return sp->shadowed_translation[index] >> PAGE_SHIFT;  in kvm_mmu_page_get_gfn()
646 	return sp->gfn + (index << ((sp->role.level - 1) * SPTE_LEVEL_BITS));  in kvm_mmu_page_get_gfn()
657 	if (sp->shadowed_translation)  in kvm_mmu_page_get_access()
658 		return sp->shadowed_translation[index] & ACC_ALL;  in kvm_mmu_page_get_access()
661 	 * For direct MMUs (e.g. TDP or non-paging guests) or passthrough SPs,  in kvm_mmu_page_get_access()
670 	 * In both cases, sp->role.access contains the correct access bits.  in kvm_mmu_page_get_access()
672 	return sp->role.access;  in kvm_mmu_page_get_access()
678 	if (sp->shadowed_translation) {  in kvm_mmu_page_set_translation()
679 		sp->shadowed_translation[index] = (gfn << PAGE_SHIFT) | access;  in kvm_mmu_page_set_translation()
685 	          sp->role.passthrough ? "passthrough" : "direct",  in kvm_mmu_page_set_translation()
686 	          sp->gfn, kvm_mmu_page_get_access(sp, index), access);  in kvm_mmu_page_set_translation()
690 	          sp->role.passthrough ? "passthrough" : "direct",  in kvm_mmu_page_set_translation()
691 	          sp->gfn, kvm_mmu_page_get_gfn(sp, index), gfn);  in kvm_mmu_page_set_translation()
711 	idx = gfn_to_index(gfn, slot->base_gfn, level);  in lpage_info_slot()
712 	return &slot->arch.lpage_info[level - 2][idx];  in lpage_info_slot()
732 		old = linfo->disallow_lpage;  in update_gfn_disallow_lpage_count()
733 		linfo->disallow_lpage += count;  in update_gfn_disallow_lpage_count()
734 		WARN_ON_ONCE((old ^ linfo->disallow_lpage) & KVM_LPAGE_MIXED_FLAG);  in update_gfn_disallow_lpage_count()
745 	update_gfn_disallow_lpage_count(slot, gfn, -1);  in kvm_mmu_gfn_allow_lpage()
754 	kvm->arch.indirect_shadow_pages++;  in account_shadowed()
756 	 * Ensure indirect_shadow_pages is elevated prior to re-reading guest  in account_shadowed()
757 	 * child PTEs in FNAME(gpte_changed), i.e. guarantee either in-flight  in account_shadowed()
758 	 * emulated writes are visible before re-reading guest PTEs, or that  in account_shadowed()
764 	gfn = sp->gfn;  in account_shadowed()
765 	slots = kvm_memslots_for_spte_role(kvm, sp->role);  in account_shadowed()
768 	/* the non-leaf shadow pages are keeping readonly. */  in account_shadowed()
769 	if (sp->role.level > PG_LEVEL_4K)  in account_shadowed()
788 	if (!list_empty(&sp->possible_nx_huge_page_link))  in track_possible_nx_huge_page()
791 	++kvm->stat.nx_lpage_splits;  in track_possible_nx_huge_page()
792 	list_add_tail(&sp->possible_nx_huge_page_link,  in track_possible_nx_huge_page()
793 		      &kvm->arch.possible_nx_huge_pages);  in track_possible_nx_huge_page()
799 	sp->nx_huge_page_disallowed = true;  in account_nx_huge_page()
811 	kvm->arch.indirect_shadow_pages--;  in unaccount_shadowed()
812 	gfn = sp->gfn;  in unaccount_shadowed()
813 	slots = kvm_memslots_for_spte_role(kvm, sp->role);  in unaccount_shadowed()
815 	if (sp->role.level > PG_LEVEL_4K)  in unaccount_shadowed()
823 	if (list_empty(&sp->possible_nx_huge_page_link))  in untrack_possible_nx_huge_page()
826 	--kvm->stat.nx_lpage_splits;  in untrack_possible_nx_huge_page()
827 	list_del_init(&sp->possible_nx_huge_page_link);  in untrack_possible_nx_huge_page()
832 	sp->nx_huge_page_disallowed = false;  in unaccount_nx_huge_page()
844 	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)  in gfn_to_memslot_dirty_bitmap()
855  * If the bit zero of rmap_head->val is clear, then it points to the only spte
856  * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
870 	if (!rmap_head->val) {  in pte_list_add()
871 		rmap_head->val = (unsigned long)spte;  in pte_list_add()
872 	} else if (!(rmap_head->val & KVM_RMAP_MANY)) {  in pte_list_add()
874 		desc->sptes[0] = (u64 *)rmap_head->val;  in pte_list_add()
875 		desc->sptes[1] = spte;  in pte_list_add()
876 		desc->spte_count = 2;  in pte_list_add()
877 		desc->tail_count = 0;  in pte_list_add()
878 		rmap_head->val = (unsigned long)desc | KVM_RMAP_MANY;  in pte_list_add()
881 		desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);  in pte_list_add()
882 		count = desc->tail_count + desc->spte_count;  in pte_list_add()
888 		if (desc->spte_count == PTE_LIST_EXT) {  in pte_list_add()
890 			desc->more = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);  in pte_list_add()
891 			desc->spte_count = 0;  in pte_list_add()
892 			desc->tail_count = count;  in pte_list_add()
893 			rmap_head->val = (unsigned long)desc | KVM_RMAP_MANY;  in pte_list_add()
895 		desc->sptes[desc->spte_count++] = spte;  in pte_list_add()
904 	struct pte_list_desc *head_desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);  in pte_list_desc_remove_entry()
905 	int j = head_desc->spte_count - 1;  in pte_list_desc_remove_entry()
915 	 * Replace the to-be-freed SPTE with the last valid entry from the head  in pte_list_desc_remove_entry()
919 	desc->sptes[i] = head_desc->sptes[j];  in pte_list_desc_remove_entry()
920 	head_desc->sptes[j] = NULL;  in pte_list_desc_remove_entry()
921 	head_desc->spte_count--;  in pte_list_desc_remove_entry()
922 	if (head_desc->spte_count)  in pte_list_desc_remove_entry()
930 	if (!head_desc->more)  in pte_list_desc_remove_entry()
931 		rmap_head->val = 0;  in pte_list_desc_remove_entry()
933 		rmap_head->val = (unsigned long)head_desc->more | KVM_RMAP_MANY;  in pte_list_desc_remove_entry()
943 	if (KVM_BUG_ON_DATA_CORRUPTION(!rmap_head->val, kvm))  in pte_list_remove()
946 	if (!(rmap_head->val & KVM_RMAP_MANY)) {  in pte_list_remove()
947 		if (KVM_BUG_ON_DATA_CORRUPTION((u64 *)rmap_head->val != spte, kvm))  in pte_list_remove()
950 		rmap_head->val = 0;  in pte_list_remove()
952 		desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);  in pte_list_remove()
954 			for (i = 0; i < desc->spte_count; ++i) {  in pte_list_remove()
955 				if (desc->sptes[i] == spte) {  in pte_list_remove()
961 			desc = desc->more;  in pte_list_remove()
982 	if (!rmap_head->val)  in kvm_zap_all_rmap_sptes()
985 	if (!(rmap_head->val & KVM_RMAP_MANY)) {  in kvm_zap_all_rmap_sptes()
986 		mmu_spte_clear_track_bits(kvm, (u64 *)rmap_head->val);  in kvm_zap_all_rmap_sptes()
990 	desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);  in kvm_zap_all_rmap_sptes()
993 		for (i = 0; i < desc->spte_count; i++)  in kvm_zap_all_rmap_sptes()
994 			mmu_spte_clear_track_bits(kvm, desc->sptes[i]);  in kvm_zap_all_rmap_sptes()
995 		next = desc->more;  in kvm_zap_all_rmap_sptes()
1000 	rmap_head->val = 0;  in kvm_zap_all_rmap_sptes()
1008 	if (!rmap_head->val)  in pte_list_count()
1010 	else if (!(rmap_head->val & KVM_RMAP_MANY))  in pte_list_count()
1013 	desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);  in pte_list_count()
1014 	return desc->tail_count + desc->spte_count;  in pte_list_count()
1022 	idx = gfn_to_index(gfn, slot->base_gfn, level);  in gfn_to_rmap()
1023 	return &slot->arch.rmap[level - PG_LEVEL_4K][idx];  in gfn_to_rmap()
1040 	 * information in sp->role.  in rmap_remove()
1042 	slots = kvm_memslots_for_spte_role(kvm, sp->role);  in rmap_remove()
1045 	rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);  in rmap_remove()
1072 	if (!rmap_head->val)  in rmap_get_first()
1075 	if (!(rmap_head->val & KVM_RMAP_MANY)) {  in rmap_get_first()
1076 		iter->desc = NULL;  in rmap_get_first()
1077 		sptep = (u64 *)rmap_head->val;  in rmap_get_first()
1081 	iter->desc = (struct pte_list_desc *)(rmap_head->val & ~KVM_RMAP_MANY);  in rmap_get_first()
1082 	iter->pos = 0;  in rmap_get_first()
1083 	sptep = iter->desc->sptes[iter->pos];  in rmap_get_first()
1098 	if (iter->desc) {  in rmap_get_next()
1099 		if (iter->pos < PTE_LIST_EXT - 1) {  in rmap_get_next()
1100 			++iter->pos;  in rmap_get_next()
1101 			sptep = iter->desc->sptes[iter->pos];  in rmap_get_next()
1106 		iter->desc = iter->desc->more;  in rmap_get_next()
1108 		if (iter->desc) {  in rmap_get_next()
1109 			iter->pos = 0;  in rmap_get_next()
1110 			/* desc->sptes[0] cannot be NULL */  in rmap_get_next()
1111 			sptep = iter->desc->sptes[iter->pos];  in rmap_get_next()
1139 	WARN_ON_ONCE(sp->role.level == PG_LEVEL_4K);  in drop_large_spte()
1148  * Write-protect on the specified @sptep, @pt_protect indicates whether
1149  * spte write-protection is caused by protecting shadow page table.
1153  * - for dirty logging, the spte can be set to writable at anytime if
1155  * - for spte protection, the spte can be writable only after unsync-ing
1199  *	- D bit on ad-enabled SPTEs, and
1200  *	- W bit on ad-disabled SPTEs.
1228 				slot->base_gfn + gfn_offset, mask, true);  in kvm_mmu_write_protect_pt_masked()
1234 		rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),  in kvm_mmu_write_protect_pt_masked()
1239 		mask &= mask - 1;  in kvm_mmu_write_protect_pt_masked()
1251 				slot->base_gfn + gfn_offset, mask, false);  in kvm_mmu_clear_dirty_pt_masked()
1257 		rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),  in kvm_mmu_clear_dirty_pt_masked()
1262 		mask &= mask - 1;  in kvm_mmu_clear_dirty_pt_masked()
1271 	 * If the slot was assumed to be "initially all dirty", write-protect  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1282 		gfn_t start = slot->base_gfn + gfn_offset + __ffs(mask);  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1283 		gfn_t end = slot->base_gfn + gfn_offset + __fls(mask);  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1299 	 * mask.  If PML is enabled and the GFN doesn't need to be write-  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1345 	return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn, PG_LEVEL_4K);  in kvm_vcpu_write_protect_gfn()
1374 	iterator->level = level;  in rmap_walk_init_level()
1375 	iterator->gfn = iterator->start_gfn;  in rmap_walk_init_level()
1376 	iterator->rmap = gfn_to_rmap(iterator->gfn, level, iterator->slot);  in rmap_walk_init_level()
1377 	iterator->end_rmap = gfn_to_rmap(iterator->end_gfn, level, iterator->slot);  in rmap_walk_init_level()
1385 	iterator->slot = slot;  in slot_rmap_walk_init()
1386 	iterator->start_level = start_level;  in slot_rmap_walk_init()
1387 	iterator->end_level = end_level;  in slot_rmap_walk_init()
1388 	iterator->start_gfn = start_gfn;  in slot_rmap_walk_init()
1389 	iterator->end_gfn = end_gfn;  in slot_rmap_walk_init()
1391 	rmap_walk_init_level(iterator, iterator->start_level);  in slot_rmap_walk_init()
1396 	return !!iterator->rmap;  in slot_rmap_walk_okay()
1401 	while (++iterator->rmap <= iterator->end_rmap) {  in slot_rmap_walk_next()
1402 		iterator->gfn += KVM_PAGES_PER_HPAGE(iterator->level);  in slot_rmap_walk_next()
1404 		if (iterator->rmap->val)  in slot_rmap_walk_next()
1408 	if (++iterator->level > iterator->end_level) {  in slot_rmap_walk_next()
1409 		iterator->rmap = NULL;  in slot_rmap_walk_next()
1413 	rmap_walk_init_level(iterator, iterator->level);  in slot_rmap_walk_next()
1438 	lockdep_assert_held_write(&kvm->mmu_lock);  in __walk_slot_rmaps()
1448 		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {  in __walk_slot_rmaps()
1451 							    iterator.gfn - start_gfn + 1);  in __walk_slot_rmaps()
1454 			cond_resched_rwlock_write(&kvm->mmu_lock);  in __walk_slot_rmaps()
1468 				 slot->base_gfn, slot->base_gfn + slot->npages - 1,  in walk_slot_rmaps()
1487 				 start, end - 1, can_yield, true, flush);  in __kvm_rmap_zap_gfn_range()
1502 	lockdep_assert_once(kvm->mmu_invalidate_in_progress ||  in kvm_unmap_gfn_range()
1503 			    lockdep_is_held(&kvm->slots_lock));  in kvm_unmap_gfn_range()
1506 		flush = __kvm_rmap_zap_gfn_range(kvm, range->slot,  in kvm_unmap_gfn_range()
1507 						 range->start, range->end,  in kvm_unmap_gfn_range()
1508 						 range->may_block, flush);  in kvm_unmap_gfn_range()
1514 	    range->slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT)  in kvm_unmap_gfn_range()
1533 	kvm_update_page_stats(kvm, sp->role.level, 1);  in __rmap_add()
1535 	rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);  in __rmap_add()
1538 	if (rmap_count > kvm->stat.max_mmu_rmap_size)  in __rmap_add()
1539 		kvm->stat.max_mmu_rmap_size = rmap_count;  in __rmap_add()
1542 		kvm_flush_remote_tlbs_gfn(kvm, gfn, sp->role.level);  in __rmap_add()
1549 	struct kvm_mmu_memory_cache *cache = &vcpu->arch.mmu_pte_list_desc_cache;  in rmap_add()
1551 	__rmap_add(vcpu->kvm, cache, slot, spte, gfn, access);  in rmap_add()
1562 	for_each_slot_rmap_range(range->slot, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,  in kvm_rmap_age_gfn_range()
1563 				 range->start, range->end - 1, &iterator) {  in kvm_rmap_age_gfn_range()
1574 				clear_bit((ffs(shadow_accessed_mask) - 1),  in kvm_rmap_age_gfn_range()
1623 		if (KVM_MMU_WARN_ON(is_shadow_present_pte(sp->spt[i])))  in kvm_mmu_check_sptes_at_free()
1624 			pr_err_ratelimited("SPTE %llx (@ %p) for gfn %llx shadow-present at free",  in kvm_mmu_check_sptes_at_free()
1625 					   sp->spt[i], &sp->spt[i],  in kvm_mmu_check_sptes_at_free()
1633 	kvm->arch.n_used_mmu_pages++;  in kvm_account_mmu_page()
1634 	kvm_account_pgtable_pages((void *)sp->spt, +1);  in kvm_account_mmu_page()
1639 	kvm->arch.n_used_mmu_pages--;  in kvm_unaccount_mmu_page()
1640 	kvm_account_pgtable_pages((void *)sp->spt, -1);  in kvm_unaccount_mmu_page()
1647 	hlist_del(&sp->hash_link);  in kvm_mmu_free_shadow_page()
1648 	list_del(&sp->link);  in kvm_mmu_free_shadow_page()
1649 	free_page((unsigned long)sp->spt);  in kvm_mmu_free_shadow_page()
1650 	free_page((unsigned long)sp->shadowed_translation);  in kvm_mmu_free_shadow_page()
1665 	pte_list_add(cache, parent_pte, &sp->parent_ptes);  in mmu_page_add_parent_pte()
1671 	pte_list_remove(kvm, parent_pte, &sp->parent_ptes);  in mmu_page_remove_parent_pte()
1687 	for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {  in kvm_mmu_mark_parents_unsync()
1697 	if (__test_and_set_bit(spte_index(spte), sp->unsync_child_bitmap))  in mark_unsync()
1699 	if (sp->unsync_children++)  in mark_unsync()
1719 	if (sp->unsync)  in mmu_pages_add()
1720 		for (i=0; i < pvec->nr; i++)  in mmu_pages_add()
1721 			if (pvec->page[i].sp == sp)  in mmu_pages_add()
1724 	pvec->page[pvec->nr].sp = sp;  in mmu_pages_add()
1725 	pvec->page[pvec->nr].idx = idx;  in mmu_pages_add()
1726 	pvec->nr++;  in mmu_pages_add()
1727 	return (pvec->nr == KVM_PAGE_ARRAY_NR);  in mmu_pages_add()
1732 	--sp->unsync_children;  in clear_unsync_child_bit()
1733 	WARN_ON_ONCE((int)sp->unsync_children < 0);  in clear_unsync_child_bit()
1734 	__clear_bit(idx, sp->unsync_child_bitmap);  in clear_unsync_child_bit()
1742 	for_each_set_bit(i, sp->unsync_child_bitmap, 512) {  in __mmu_unsync_walk()
1744 		u64 ent = sp->spt[i];  in __mmu_unsync_walk()
1753 		if (child->unsync_children) {  in __mmu_unsync_walk()
1755 				return -ENOSPC;  in __mmu_unsync_walk()
1765 		} else if (child->unsync) {  in __mmu_unsync_walk()
1768 				return -ENOSPC;  in __mmu_unsync_walk()
1776 #define INVALID_INDEX (-1)
1781 	pvec->nr = 0;  in mmu_unsync_walk()
1782 	if (!sp->unsync_children)  in mmu_unsync_walk()
1791 	WARN_ON_ONCE(!sp->unsync);  in kvm_unlink_unsync_page()
1793 	sp->unsync = 0;  in kvm_unlink_unsync_page()
1794 	--kvm->stat.mmu_unsync;  in kvm_unlink_unsync_page()
1804 	if (sp->role.direct)  in sp_has_gptes()
1807 	if (sp->role.passthrough)  in sp_has_gptes()
1820 	  &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)])	\
1821 		if ((_sp)->gfn != (_gfn) || !sp_has_gptes(_sp)) {} else
1825 	union kvm_mmu_page_role root_role = vcpu->arch.mmu->root_role;  in kvm_sync_page_check()
1831 	 *  - level: not part of the overall MMU role and will never match as the MMU's  in kvm_sync_page_check()
1833 	 *  - access: updated based on the new guest PTE  in kvm_sync_page_check()
1834 	 *  - quadrant: not part of the overall MMU role (similar to level)  in kvm_sync_page_check()
1846 	 * differs then the memslot lookup (SMM vs. non-SMM) will be bogus, the  in kvm_sync_page_check()
1849 	if (WARN_ON_ONCE(sp->role.direct || !vcpu->arch.mmu->sync_spte ||  in kvm_sync_page_check()
1850 			 (sp->role.word ^ root_role.word) & ~sync_role_ign.word))  in kvm_sync_page_check()
1858 	/* sp->spt[i] has initial value of shadow page table allocation */  in kvm_sync_spte()
1859 	if (sp->spt[i] == SHADOW_NONPRESENT_VALUE)  in kvm_sync_spte()
1862 	return vcpu->arch.mmu->sync_spte(vcpu, sp, i);  in kvm_sync_spte()
1871 		return -1;  in __kvm_sync_page()
1876 		if (ret < -1)  in __kvm_sync_page()
1877 			return -1;  in __kvm_sync_page()
1899 		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);  in kvm_sync_page()
1919 	if (sp->role.invalid)  in is_obsolete_sp()
1924 	       unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);  in is_obsolete_sp()
1943 	for (n = i+1; n < pvec->nr; n++) {  in mmu_pages_next()
1944 		struct kvm_mmu_page *sp = pvec->page[n].sp;  in mmu_pages_next()
1945 		unsigned idx = pvec->page[n].idx;  in mmu_pages_next()
1946 		int level = sp->role.level;  in mmu_pages_next()
1948 		parents->idx[level-1] = idx;  in mmu_pages_next()
1952 		parents->parent[level-2] = sp;  in mmu_pages_next()
1964 	if (pvec->nr == 0)  in mmu_pages_first()
1967 	WARN_ON_ONCE(pvec->page[0].idx != INVALID_INDEX);  in mmu_pages_first()
1969 	sp = pvec->page[0].sp;  in mmu_pages_first()
1970 	level = sp->role.level;  in mmu_pages_first()
1973 	parents->parent[level-2] = sp;  in mmu_pages_first()
1978 	parents->parent[level-1] = NULL;  in mmu_pages_first()
1988 		unsigned int idx = parents->idx[level];  in mmu_pages_clear_parents()
1989 		sp = parents->parent[level];  in mmu_pages_clear_parents()
1996 	} while (!sp->unsync_children);  in mmu_pages_clear_parents()
2013 			protected |= kvm_vcpu_write_protect_gfn(vcpu, sp->gfn);  in mmu_sync_children()
2016 			kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, true);  in mmu_sync_children()
2021 			kvm_unlink_unsync_page(vcpu->kvm, sp);  in mmu_sync_children()
2025 		if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) {  in mmu_sync_children()
2026 			kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);  in mmu_sync_children()
2029 				return -EINTR;  in mmu_sync_children()
2032 			cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);  in mmu_sync_children()
2037 	kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);  in mmu_sync_children()
2043 	atomic_set(&sp->write_flooding_count,  0);  in __clear_sp_write_flooding_count()
2069 		if (sp->gfn != gfn) {  in kvm_mmu_find_shadow_page()
2074 		if (sp->role.word != role.word) {  in kvm_mmu_find_shadow_page()
2076 			 * If the guest is creating an upper-level page, zap  in kvm_mmu_find_shadow_page()
2082 			 * upper-level page will be write-protected.  in kvm_mmu_find_shadow_page()
2084 			if (role.level > PG_LEVEL_4K && sp->unsync)  in kvm_mmu_find_shadow_page()
2090 		/* unsync and write-flooding only apply to indirect SPs. */  in kvm_mmu_find_shadow_page()
2091 		if (sp->role.direct)  in kvm_mmu_find_shadow_page()
2094 		if (sp->unsync) {  in kvm_mmu_find_shadow_page()
2101 			 * it doesn't write-protect the page or mark it synchronized!  in kvm_mmu_find_shadow_page()
2125 	++kvm->stat.mmu_cache_miss;  in kvm_mmu_find_shadow_page()
2130 	if (collisions > kvm->stat.max_mmu_page_hash_collisions)  in kvm_mmu_find_shadow_page()
2131 		kvm->stat.max_mmu_page_hash_collisions = collisions;  in kvm_mmu_find_shadow_page()
2150 	sp = kvm_mmu_memory_cache_alloc(caches->page_header_cache);  in kvm_mmu_alloc_shadow_page()
2151 	sp->spt = kvm_mmu_memory_cache_alloc(caches->shadow_page_cache);  in kvm_mmu_alloc_shadow_page()
2153 		sp->shadowed_translation = kvm_mmu_memory_cache_alloc(caches->shadowed_info_cache);  in kvm_mmu_alloc_shadow_page()
2155 	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);  in kvm_mmu_alloc_shadow_page()
2157 	INIT_LIST_HEAD(&sp->possible_nx_huge_page_link);  in kvm_mmu_alloc_shadow_page()
2164 	sp->mmu_valid_gen = kvm->arch.mmu_valid_gen;  in kvm_mmu_alloc_shadow_page()
2165 	list_add(&sp->link, &kvm->arch.active_mmu_pages);  in kvm_mmu_alloc_shadow_page()
2168 	sp->gfn = gfn;  in kvm_mmu_alloc_shadow_page()
2169 	sp->role = role;  in kvm_mmu_alloc_shadow_page()
2170 	hlist_add_head(&sp->hash_link, sp_list);  in kvm_mmu_alloc_shadow_page()
2188 	sp_list = &kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)];  in __kvm_mmu_get_shadow_page()
2205 		.page_header_cache = &vcpu->arch.mmu_page_header_cache,  in kvm_mmu_get_shadow_page()
2206 		.shadow_page_cache = &vcpu->arch.mmu_shadow_page_cache,  in kvm_mmu_get_shadow_page()
2207 		.shadowed_info_cache = &vcpu->arch.mmu_shadowed_info_cache,  in kvm_mmu_get_shadow_page()
2210 	return __kvm_mmu_get_shadow_page(vcpu->kvm, vcpu, &caches, gfn, role);  in kvm_mmu_get_shadow_page()
2219 	role = parent_sp->role;  in kvm_mmu_child_role()
2220 	role.level--;  in kvm_mmu_child_role()
2226 	 * If the guest has 4-byte PTEs then that means it's using 32-bit,  in kvm_mmu_child_role()
2227 	 * 2-level, non-PAE paging. KVM shadows such guests with PAE paging  in kvm_mmu_child_role()
2228 	 * (i.e. 8-byte PTEs). The difference in PTE size means that KVM must  in kvm_mmu_child_role()
2241 	 * Concretely, a 4-byte PDE consumes bits 31:22, while an 8-byte PDE  in kvm_mmu_child_role()
2243 	 * PDPTEs; those 4 PAE page directories are pre-allocated and their  in kvm_mmu_child_role()
2244 	 * quadrant is assigned in mmu_alloc_root().   A 4-byte PTE consumes  in kvm_mmu_child_role()
2245 	 * bits 21:12, while an 8-byte PTE consumes bits 20:12.  To consume  in kvm_mmu_child_role()
2247 	 * quadrant, i.e. sets quadrant to '0' or '1'.  The parent 8-byte PDE  in kvm_mmu_child_role()
2266 		return ERR_PTR(-EEXIST);  in kvm_mmu_get_child_sp()
2276 	iterator->addr = addr;  in shadow_walk_init_using_root()
2277 	iterator->shadow_addr = root;  in shadow_walk_init_using_root()
2278 	iterator->level = vcpu->arch.mmu->root_role.level;  in shadow_walk_init_using_root()
2280 	if (iterator->level >= PT64_ROOT_4LEVEL &&  in shadow_walk_init_using_root()
2281 	    vcpu->arch.mmu->cpu_role.base.level < PT64_ROOT_4LEVEL &&  in shadow_walk_init_using_root()
2282 	    !vcpu->arch.mmu->root_role.direct)  in shadow_walk_init_using_root()
2283 		iterator->level = PT32E_ROOT_LEVEL;  in shadow_walk_init_using_root()
2285 	if (iterator->level == PT32E_ROOT_LEVEL) {  in shadow_walk_init_using_root()
2287 		 * prev_root is currently only used for 64-bit hosts. So only  in shadow_walk_init_using_root()
2290 		BUG_ON(root != vcpu->arch.mmu->root.hpa);  in shadow_walk_init_using_root()
2292 		iterator->shadow_addr  in shadow_walk_init_using_root()
2293 			= vcpu->arch.mmu->pae_root[(addr >> 30) & 3];  in shadow_walk_init_using_root()
2294 		iterator->shadow_addr &= SPTE_BASE_ADDR_MASK;  in shadow_walk_init_using_root()
2295 		--iterator->level;  in shadow_walk_init_using_root()
2296 		if (!iterator->shadow_addr)  in shadow_walk_init_using_root()
2297 			iterator->level = 0;  in shadow_walk_init_using_root()
2304 	shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root.hpa,  in shadow_walk_init()
2310 	if (iterator->level < PG_LEVEL_4K)  in shadow_walk_okay()
2313 	iterator->index = SPTE_INDEX(iterator->addr, iterator->level);  in shadow_walk_okay()
2314 	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;  in shadow_walk_okay()
2321 	if (!is_shadow_present_pte(spte) || is_last_spte(spte, iterator->level)) {  in __shadow_walk_next()
2322 		iterator->level = 0;  in __shadow_walk_next()
2326 	iterator->shadow_addr = spte & SPTE_BASE_ADDR_MASK;  in __shadow_walk_next()
2327 	--iterator->level;  in __shadow_walk_next()
2332 	__shadow_walk_next(iterator, *iterator->sptep);  in shadow_walk_next()
2351 	spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));  in __link_shadow_page()
2358 	 * The non-direct sub-pagetable must be updated before linking.  For  in __link_shadow_page()
2360 	 * kvm_mmu_find_shadow_page() without write-protecting the gfn,  in __link_shadow_page()
2361 	 * so sp->unsync can be true or false.  For higher level non-direct  in __link_shadow_page()
2363 	 * FNAME(fetch)(), so sp->unsync_children can only be false.  in __link_shadow_page()
2366 	if (WARN_ON_ONCE(sp->unsync_children) || sp->unsync)  in __link_shadow_page()
2373 	__link_shadow_page(vcpu->kvm, &vcpu->arch.mmu_pte_list_desc_cache, sptep, sp, true);  in link_shadow_page()
2385 		 * sp's access: allow writable in the read-only sp,  in validate_direct_spte()
2390 		if (child->role.access == direct_access)  in validate_direct_spte()
2393 		drop_parent_pte(vcpu->kvm, child, sptep);  in validate_direct_spte()
2394 		kvm_flush_remote_tlbs_sptep(vcpu->kvm, sptep);  in validate_direct_spte()
2398 /* Returns the number of zapped non-leaf child shadow pages. */
2407 		if (is_last_spte(pte, sp->role.level)) {  in mmu_page_zap_pte()
2419 			    child->role.guest_mode && !child->parent_ptes.val)  in mmu_page_zap_pte()
2437 		zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list);  in kvm_mmu_page_unlink_children()
2447 	while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))  in kvm_mmu_unlink_parents()
2459 	if (parent->role.level == PG_LEVEL_4K)  in mmu_zap_unsync_children()
2482 	lockdep_assert_held_write(&kvm->mmu_lock);  in __kvm_mmu_prepare_zap_page()
2484 	++kvm->stat.mmu_shadow_zapped;  in __kvm_mmu_prepare_zap_page()
2492 	if (!sp->role.invalid && sp_has_gptes(sp))  in __kvm_mmu_prepare_zap_page()
2495 	if (sp->unsync)  in __kvm_mmu_prepare_zap_page()
2497 	if (!sp->root_count) {  in __kvm_mmu_prepare_zap_page()
2504 		 * !sp->root_count.  in __kvm_mmu_prepare_zap_page()
2506 		if (sp->role.invalid)  in __kvm_mmu_prepare_zap_page()
2507 			list_add(&sp->link, invalid_list);  in __kvm_mmu_prepare_zap_page()
2509 			list_move(&sp->link, invalid_list);  in __kvm_mmu_prepare_zap_page()
2516 		list_del(&sp->link);  in __kvm_mmu_prepare_zap_page()
2526 	if (sp->nx_huge_page_disallowed)  in __kvm_mmu_prepare_zap_page()
2529 	sp->role.invalid = 1;  in __kvm_mmu_prepare_zap_page()
2559 	 * the page tables and see changes to vcpu->mode here. The barrier  in kvm_mmu_commit_zap_page()
2569 		WARN_ON_ONCE(!sp->role.invalid || sp->root_count);  in kvm_mmu_commit_zap_page()
2583 	if (list_empty(&kvm->arch.active_mmu_pages))  in kvm_mmu_zap_oldest_mmu_pages()
2587 	list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {  in kvm_mmu_zap_oldest_mmu_pages()
2592 		if (sp->root_count)  in kvm_mmu_zap_oldest_mmu_pages()
2607 	kvm->stat.mmu_recycled += total_zapped;  in kvm_mmu_zap_oldest_mmu_pages()
2613 	if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)  in kvm_mmu_available_pages()
2614 		return kvm->arch.n_max_mmu_pages -  in kvm_mmu_available_pages()
2615 			kvm->arch.n_used_mmu_pages;  in kvm_mmu_available_pages()
2622 	unsigned long avail = kvm_mmu_available_pages(vcpu->kvm);  in make_mmu_pages_available()
2627 	kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail);  in make_mmu_pages_available()
2632 	 * four pages, e.g. for PAE roots or for 5-level paging.  Temporarily  in make_mmu_pages_available()
2638 	if (!kvm_mmu_available_pages(vcpu->kvm))  in make_mmu_pages_available()
2639 		return -ENOSPC;  in make_mmu_pages_available()
2649 	write_lock(&kvm->mmu_lock);  in kvm_mmu_change_mmu_pages()
2651 	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {  in kvm_mmu_change_mmu_pages()
2652 		kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -  in kvm_mmu_change_mmu_pages()
2655 		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;  in kvm_mmu_change_mmu_pages()
2658 	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;  in kvm_mmu_change_mmu_pages()
2660 	write_unlock(&kvm->mmu_lock);  in kvm_mmu_change_mmu_pages()
2666 	struct kvm *kvm = vcpu->kvm;  in __kvm_mmu_unprotect_gfn_and_retry()
2673 	 * Bail early if there aren't any write-protected shadow pages to avoid  in __kvm_mmu_unprotect_gfn_and_retry()
2674 	 * unnecessarily taking mmu_lock lock, e.g. if the gfn is write-tracked  in __kvm_mmu_unprotect_gfn_and_retry()
2680 	if (!READ_ONCE(kvm->arch.indirect_shadow_pages))  in __kvm_mmu_unprotect_gfn_and_retry()
2683 	if (!vcpu->arch.mmu->root_role.direct) {  in __kvm_mmu_unprotect_gfn_and_retry()
2689 	write_lock(&kvm->mmu_lock);  in __kvm_mmu_unprotect_gfn_and_retry()
2699 	write_unlock(&kvm->mmu_lock);  in __kvm_mmu_unprotect_gfn_and_retry()
2703 		vcpu->arch.last_retry_eip = kvm_rip_read(vcpu);  in __kvm_mmu_unprotect_gfn_and_retry()
2704 		vcpu->arch.last_retry_addr = cr2_or_gpa;  in __kvm_mmu_unprotect_gfn_and_retry()
2712 	++kvm->stat.mmu_unsync;  in kvm_unsync_page()
2713 	sp->unsync = 1;  in kvm_unsync_page()
2721  * were marked unsync (or if there is no shadow page), -EPERM if the SPTE must
2722  * be write-protected.
2731 	 * Force write-protection if the page is being tracked.  Note, the page  in mmu_try_to_unsync_pages()
2732 	 * track machinery is used to write-protect upper-level shadow pages,  in mmu_try_to_unsync_pages()
2736 		return -EPERM;  in mmu_try_to_unsync_pages()
2739 	 * The page is not write-tracked, mark existing shadow pages unsync  in mmu_try_to_unsync_pages()
2746 			return -EPERM;  in mmu_try_to_unsync_pages()
2748 		if (sp->unsync)  in mmu_try_to_unsync_pages()
2752 			return -EEXIST;  in mmu_try_to_unsync_pages()
2763 			spin_lock(&kvm->arch.mmu_unsync_pages_lock);  in mmu_try_to_unsync_pages()
2769 			 * possible as clearing sp->unsync _must_ hold mmu_lock  in mmu_try_to_unsync_pages()
2770 			 * for write, i.e. unsync cannot transition from 1->0  in mmu_try_to_unsync_pages()
2773 			if (READ_ONCE(sp->unsync))  in mmu_try_to_unsync_pages()
2777 		WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K);  in mmu_try_to_unsync_pages()
2781 		spin_unlock(&kvm->arch.mmu_unsync_pages_lock);  in mmu_try_to_unsync_pages()
2788 	 * before the page had been marked as unsync-ed, something like the  in mmu_try_to_unsync_pages()
2792 	 * ---------------------------------------------------------------------  in mmu_try_to_unsync_pages()
2805 	 *                      2.3 Walking of unsync pages sees sp->unsync is  in mmu_try_to_unsync_pages()
2814 	 *     (sp->unsync = true)  in mmu_try_to_unsync_pages()
2830 	int level = sp->role.level;  in mmu_set_spte()
2838 	bool host_writable = !fault || fault->map_writable;  in mmu_set_spte()
2839 	bool prefetch = !fault || fault->prefetch;  in mmu_set_spte()
2840 	bool write_fault = fault && fault->write;  in mmu_set_spte()
2843 		vcpu->stat.pf_mmio_spte_created++;  in mmu_set_spte()
2861 			drop_parent_pte(vcpu->kvm, child, sptep);  in mmu_set_spte()
2864 			drop_spte(vcpu->kvm, sptep);  in mmu_set_spte()
2884 		kvm_flush_remote_tlbs_gfn(vcpu->kvm, gfn, level);  in mmu_set_spte()
2939 	unsigned int access = sp->role.access;  in direct_pte_prefetch_many()
2941 	return kvm_mmu_prefetch_sptes(vcpu, gfn, start, end - start, access);  in direct_pte_prefetch_many()
2950 	WARN_ON_ONCE(!sp->role.direct);  in __direct_pte_prefetch()
2952 	i = spte_index(sptep) & ~(PTE_PREFETCH_NUM - 1);  in __direct_pte_prefetch()
2953 	spte = sp->spt + i;  in __direct_pte_prefetch()
2984 	if (sp->role.level > PG_LEVEL_4K)  in direct_pte_prefetch()
2991 	if (unlikely(vcpu->kvm->mmu_invalidate_in_progress))  in direct_pte_prefetch()
3005  * - Check mmu_invalidate_retry_gfn() after grabbing the mapping level, before
3009  * - Hold mmu_lock AND ensure there is no in-progress MMU notifier invalidation
3013  * - Do not use the result to install new mappings, e.g. use the host mapping
3034 	 * Note, using the already-retrieved memslot and __gfn_to_hva_memslot()  in host_pfn_mapping_level()
3037 	 * read-only memslots due to gfn_to_hva() assuming writes.  Earlier  in host_pfn_mapping_level()
3039 	 * read-only memslot.  in host_pfn_mapping_level()
3051 	 * Read each entry once.  As above, a non-leaf entry can be promoted to  in host_pfn_mapping_level()
3052 	 * a huge page _during_ this walk.  Re-reading the entry could send the  in host_pfn_mapping_level()
3057 	pgd = READ_ONCE(*pgd_offset(kvm->mm, hva));  in host_pfn_mapping_level()
3094 	for ( ; max_level > PG_LEVEL_4K; max_level--) {  in __kvm_mmu_max_mapping_level()
3096 		if (!linfo->disallow_lpage)  in __kvm_mmu_max_mapping_level()
3121 	struct kvm_memory_slot *slot = fault->slot;  in kvm_mmu_hugepage_adjust()
3124 	fault->huge_page_disallowed = fault->exec && fault->nx_huge_page_workaround_enabled;  in kvm_mmu_hugepage_adjust()
3126 	if (unlikely(fault->max_level == PG_LEVEL_4K))  in kvm_mmu_hugepage_adjust()
3129 	if (is_error_noslot_pfn(fault->pfn))  in kvm_mmu_hugepage_adjust()
3139 	fault->req_level = __kvm_mmu_max_mapping_level(vcpu->kvm, slot,  in kvm_mmu_hugepage_adjust()
3140 						       fault->gfn, fault->max_level,  in kvm_mmu_hugepage_adjust()
3141 						       fault->is_private);  in kvm_mmu_hugepage_adjust()
3142 	if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)  in kvm_mmu_hugepage_adjust()
3149 	fault->goal_level = fault->req_level;  in kvm_mmu_hugepage_adjust()
3150 	mask = KVM_PAGES_PER_HPAGE(fault->goal_level) - 1;  in kvm_mmu_hugepage_adjust()
3151 	VM_BUG_ON((fault->gfn & mask) != (fault->pfn & mask));  in kvm_mmu_hugepage_adjust()
3152 	fault->pfn &= ~mask;  in kvm_mmu_hugepage_adjust()
3158 	    cur_level == fault->goal_level &&  in disallowed_hugepage_adjust()
3161 	    spte_to_child_sp(spte)->nx_huge_page_disallowed) {  in disallowed_hugepage_adjust()
3169 		u64 page_mask = KVM_PAGES_PER_HPAGE(cur_level) -  in disallowed_hugepage_adjust()
3170 				KVM_PAGES_PER_HPAGE(cur_level - 1);  in disallowed_hugepage_adjust()
3171 		fault->pfn |= fault->gfn & page_mask;  in disallowed_hugepage_adjust()
3172 		fault->goal_level--;  in disallowed_hugepage_adjust()
3181 	gfn_t base_gfn = fault->gfn;  in direct_map()
3186 	for_each_shadow_entry(vcpu, fault->addr, it) {  in direct_map()
3191 		if (fault->nx_huge_page_workaround_enabled)  in direct_map()
3194 		base_gfn = gfn_round_for_level(fault->gfn, it.level);  in direct_map()
3195 		if (it.level == fault->goal_level)  in direct_map()
3199 		if (sp == ERR_PTR(-EEXIST))  in direct_map()
3203 		if (fault->huge_page_disallowed)  in direct_map()
3204 			account_nx_huge_page(vcpu->kvm, sp,  in direct_map()
3205 					     fault->req_level >= it.level);  in direct_map()
3208 	if (WARN_ON_ONCE(it.level != fault->goal_level))  in direct_map()
3209 		return -EFAULT;  in direct_map()
3211 	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL,  in direct_map()
3212 			   base_gfn, fault->pfn, fault);  in direct_map()
3229 	if (is_sigpending_pfn(fault->pfn)) {  in kvm_handle_error_pfn()
3231 		return -EINTR;  in kvm_handle_error_pfn()
3239 	if (fault->pfn == KVM_PFN_ERR_RO_FAULT)  in kvm_handle_error_pfn()
3242 	if (fault->pfn == KVM_PFN_ERR_HWPOISON) {  in kvm_handle_error_pfn()
3243 		kvm_send_hwpoison_signal(fault->slot, fault->gfn);  in kvm_handle_error_pfn()
3247 	return -EFAULT;  in kvm_handle_error_pfn()
3254 	gva_t gva = fault->is_tdp ? 0 : fault->addr;  in kvm_handle_noslot_fault()
3256 	if (fault->is_private) {  in kvm_handle_noslot_fault()
3258 		return -EFAULT;  in kvm_handle_noslot_fault()
3261 	vcpu_cache_mmio_info(vcpu, gva, fault->gfn,  in kvm_handle_noslot_fault()
3264 	fault->slot = NULL;  in kvm_handle_noslot_fault()
3265 	fault->pfn = KVM_PFN_NOSLOT;  in kvm_handle_noslot_fault()
3266 	fault->map_writable = false;  in kvm_handle_noslot_fault()
3283 	if (unlikely(fault->gfn > kvm_mmu_max_gfn()))  in kvm_handle_noslot_fault()
3297 	if (fault->rsvd)  in page_fault_can_be_fast()
3301 	 * For hardware-protected VMs, certain conditions like attempting to  in page_fault_can_be_fast()
3305 	 * result of a write-protected access, and treat it as a spurious case  in page_fault_can_be_fast()
3316 	if (kvm->arch.has_private_mem &&  in page_fault_can_be_fast()
3317 	    fault->is_private != kvm_mem_is_private(kvm, fault->gfn))  in page_fault_can_be_fast()
3331 	 *    the fault was caused by a write-protection violation.  If the  in page_fault_can_be_fast()
3332 	 *    SPTE is MMU-writable (determined later), the fault can be fixed  in page_fault_can_be_fast()
3335 	if (!fault->present)  in page_fault_can_be_fast()
3342 	return fault->write;  in page_fault_can_be_fast()
3361 	 * so non-PML cases won't be impacted.  in fast_pf_fix_direct_spte()
3369 		mark_page_dirty_in_slot(vcpu->kvm, fault->slot, fault->gfn);  in fast_pf_fix_direct_spte()
3376  * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
3380  *  - Must be called between walk_shadow_page_lockless_{begin,end}.
3381  *  - The returned sptep must not be used after walk_shadow_page_lockless_end.
3408 	if (!page_fault_can_be_fast(vcpu->kvm, fault))  in fast_page_fault()
3417 			sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, fault->gfn, &spte);  in fast_page_fault()
3419 			sptep = fast_pf_get_last_sptep(vcpu, fault->addr, &spte);  in fast_page_fault()
3433 		if (!is_last_spte(spte, sp->role.level))  in fast_page_fault()
3456 		 * uses A/D bits for non-nested MMUs.  Thus, if A/D bits are  in fast_page_fault()
3457 		 * enabled, the SPTE can't be an access-tracked SPTE.  in fast_page_fault()
3464 		 * To keep things simple, only SPTEs that are MMU-writable can  in fast_page_fault()
3466 		 * that were write-protected for dirty-logging or access  in fast_page_fault()
3471 		 * shadow-present, i.e. except for access tracking restoration  in fast_page_fault()
3474 		if (fault->write && is_mmu_writable_spte(spte)) {  in fast_page_fault()
3478 			 * Do not fix write-permission on the large spte when  in fast_page_fault()
3480 			 * first page into the dirty-bitmap in  in fast_page_fault()
3487 			if (sp->role.level > PG_LEVEL_4K &&  in fast_page_fault()
3488 			    kvm_slot_dirty_track_enabled(fault->slot))  in fast_page_fault()
3518 		vcpu->stat.pf_fast++;  in fast_page_fault()
3536 		lockdep_assert_held_read(&kvm->mmu_lock);  in mmu_free_root_page()
3539 		lockdep_assert_held_write(&kvm->mmu_lock);  in mmu_free_root_page()
3540 		if (!--sp->root_count && sp->role.invalid)  in mmu_free_root_page()
3551 	bool is_tdp_mmu = tdp_mmu_enabled && mmu->root_role.direct;  in kvm_mmu_free_roots()
3562 		&& VALID_PAGE(mmu->root.hpa);  in kvm_mmu_free_roots()
3567 			    VALID_PAGE(mmu->prev_roots[i].hpa))  in kvm_mmu_free_roots()
3575 		read_lock(&kvm->mmu_lock);  in kvm_mmu_free_roots()
3577 		write_lock(&kvm->mmu_lock);  in kvm_mmu_free_roots()
3581 			mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa,  in kvm_mmu_free_roots()
3585 		if (kvm_mmu_is_dummy_root(mmu->root.hpa)) {  in kvm_mmu_free_roots()
3587 		} else if (root_to_sp(mmu->root.hpa)) {  in kvm_mmu_free_roots()
3588 			mmu_free_root_page(kvm, &mmu->root.hpa, &invalid_list);  in kvm_mmu_free_roots()
3589 		} else if (mmu->pae_root) {  in kvm_mmu_free_roots()
3591 				if (!IS_VALID_PAE_ROOT(mmu->pae_root[i]))  in kvm_mmu_free_roots()
3594 				mmu_free_root_page(kvm, &mmu->pae_root[i],  in kvm_mmu_free_roots()
3596 				mmu->pae_root[i] = INVALID_PAE_ROOT;  in kvm_mmu_free_roots()
3599 		mmu->root.hpa = INVALID_PAGE;  in kvm_mmu_free_roots()
3600 		mmu->root.pgd = 0;  in kvm_mmu_free_roots()
3604 		read_unlock(&kvm->mmu_lock);  in kvm_mmu_free_roots()
3608 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_free_roots()
3624 	WARN_ON_ONCE(mmu->root_role.guest_mode);  in kvm_mmu_free_guest_mode_roots()
3627 		root_hpa = mmu->prev_roots[i].hpa;  in kvm_mmu_free_guest_mode_roots()
3632 		if (!sp || sp->role.guest_mode)  in kvm_mmu_free_guest_mode_roots()
3643 	union kvm_mmu_page_role role = vcpu->arch.mmu->root_role;  in mmu_alloc_root()
3653 	++sp->root_count;  in mmu_alloc_root()
3655 	return __pa(sp->spt);  in mmu_alloc_root()
3660 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in mmu_alloc_direct_roots()
3661 	u8 shadow_root_level = mmu->root_role.level;  in mmu_alloc_direct_roots()
3667 		if (kvm_has_mirrored_tdp(vcpu->kvm) &&  in mmu_alloc_direct_roots()
3668 		    !VALID_PAGE(mmu->mirror_root_hpa))  in mmu_alloc_direct_roots()
3674 	write_lock(&vcpu->kvm->mmu_lock);  in mmu_alloc_direct_roots()
3681 		mmu->root.hpa = root;  in mmu_alloc_direct_roots()
3683 		if (WARN_ON_ONCE(!mmu->pae_root)) {  in mmu_alloc_direct_roots()
3684 			r = -EIO;  in mmu_alloc_direct_roots()
3689 			WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));  in mmu_alloc_direct_roots()
3691 			root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT), 0,  in mmu_alloc_direct_roots()
3693 			mmu->pae_root[i] = root | PT_PRESENT_MASK |  in mmu_alloc_direct_roots()
3696 		mmu->root.hpa = __pa(mmu->pae_root);  in mmu_alloc_direct_roots()
3699 		r = -EIO;  in mmu_alloc_direct_roots()
3704 	mmu->root.pgd = 0;  in mmu_alloc_direct_roots()
3706 	write_unlock(&vcpu->kvm->mmu_lock);  in mmu_alloc_direct_roots()
3723 	mutex_lock(&kvm->slots_arch_lock);  in mmu_first_shadow_root_alloc()
3741 			 * Both of these functions are no-ops if the target is  in mmu_first_shadow_root_alloc()
3750 			r = memslot_rmap_alloc(slot, slot->npages);  in mmu_first_shadow_root_alloc()
3764 	smp_store_release(&kvm->arch.shadow_root_allocated, true);  in mmu_first_shadow_root_alloc()
3767 	mutex_unlock(&kvm->slots_arch_lock);  in mmu_first_shadow_root_alloc()
3773 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in mmu_alloc_shadow_roots()
3783 		mmu->root.hpa = kvm_mmu_get_dummy_root();  in mmu_alloc_shadow_roots()
3791 	if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {  in mmu_alloc_shadow_roots()
3793 			pdptrs[i] = mmu->get_pdptr(vcpu, i);  in mmu_alloc_shadow_roots()
3802 	r = mmu_first_shadow_root_alloc(vcpu->kvm);  in mmu_alloc_shadow_roots()
3806 	write_lock(&vcpu->kvm->mmu_lock);  in mmu_alloc_shadow_roots()
3813 	 * write-protect the guests page table root.  in mmu_alloc_shadow_roots()
3815 	if (mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {  in mmu_alloc_shadow_roots()
3817 				      mmu->root_role.level);  in mmu_alloc_shadow_roots()
3818 		mmu->root.hpa = root;  in mmu_alloc_shadow_roots()
3822 	if (WARN_ON_ONCE(!mmu->pae_root)) {  in mmu_alloc_shadow_roots()
3823 		r = -EIO;  in mmu_alloc_shadow_roots()
3828 	 * We shadow a 32 bit page table. This may be a legacy 2-level  in mmu_alloc_shadow_roots()
3829 	 * or a PAE 3-level page table. In either case we need to be aware that  in mmu_alloc_shadow_roots()
3833 	if (mmu->root_role.level >= PT64_ROOT_4LEVEL) {  in mmu_alloc_shadow_roots()
3836 		if (WARN_ON_ONCE(!mmu->pml4_root)) {  in mmu_alloc_shadow_roots()
3837 			r = -EIO;  in mmu_alloc_shadow_roots()
3840 		mmu->pml4_root[0] = __pa(mmu->pae_root) | pm_mask;  in mmu_alloc_shadow_roots()
3842 		if (mmu->root_role.level == PT64_ROOT_5LEVEL) {  in mmu_alloc_shadow_roots()
3843 			if (WARN_ON_ONCE(!mmu->pml5_root)) {  in mmu_alloc_shadow_roots()
3844 				r = -EIO;  in mmu_alloc_shadow_roots()
3847 			mmu->pml5_root[0] = __pa(mmu->pml4_root) | pm_mask;  in mmu_alloc_shadow_roots()
3852 		WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));  in mmu_alloc_shadow_roots()
3854 		if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {  in mmu_alloc_shadow_roots()
3856 				mmu->pae_root[i] = INVALID_PAE_ROOT;  in mmu_alloc_shadow_roots()
3863 		 * If shadowing 32-bit non-PAE page tables, each PAE page  in mmu_alloc_shadow_roots()
3864 		 * directory maps one quarter of the guest's non-PAE page  in mmu_alloc_shadow_roots()
3868 		quadrant = (mmu->cpu_role.base.level == PT32_ROOT_LEVEL) ? i : 0;  in mmu_alloc_shadow_roots()
3871 		mmu->pae_root[i] = root | pm_mask;  in mmu_alloc_shadow_roots()
3874 	if (mmu->root_role.level == PT64_ROOT_5LEVEL)  in mmu_alloc_shadow_roots()
3875 		mmu->root.hpa = __pa(mmu->pml5_root);  in mmu_alloc_shadow_roots()
3876 	else if (mmu->root_role.level == PT64_ROOT_4LEVEL)  in mmu_alloc_shadow_roots()
3877 		mmu->root.hpa = __pa(mmu->pml4_root);  in mmu_alloc_shadow_roots()
3879 		mmu->root.hpa = __pa(mmu->pae_root);  in mmu_alloc_shadow_roots()
3882 	mmu->root.pgd = root_pgd;  in mmu_alloc_shadow_roots()
3884 	write_unlock(&vcpu->kvm->mmu_lock);  in mmu_alloc_shadow_roots()
3891 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in mmu_alloc_special_roots()
3892 	bool need_pml5 = mmu->root_role.level > PT64_ROOT_4LEVEL;  in mmu_alloc_special_roots()
3898 	 * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP  in mmu_alloc_special_roots()
3901 	 * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.  in mmu_alloc_special_roots()
3903 	if (mmu->root_role.direct ||  in mmu_alloc_special_roots()
3904 	    mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL ||  in mmu_alloc_special_roots()
3905 	    mmu->root_role.level < PT64_ROOT_4LEVEL)  in mmu_alloc_special_roots()
3910 	 * of levels for the shadow page tables, e.g. all MMUs are 4-level or  in mmu_alloc_special_roots()
3911 	 * all MMus are 5-level.  Thus, this can safely require that pml5_root  in mmu_alloc_special_roots()
3915 	if (mmu->pae_root && mmu->pml4_root && (!need_pml5 || mmu->pml5_root))  in mmu_alloc_special_roots()
3922 	if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->pml4_root ||  in mmu_alloc_special_roots()
3923 			 (need_pml5 && mmu->pml5_root)))  in mmu_alloc_special_roots()
3924 		return -EIO;  in mmu_alloc_special_roots()
3927 	 * Unlike 32-bit NPT, the PDP table doesn't need to be in low mem, and  in mmu_alloc_special_roots()
3932 		return -ENOMEM;  in mmu_alloc_special_roots()
3946 	mmu->pae_root = pae_root;  in mmu_alloc_special_roots()
3947 	mmu->pml4_root = pml4_root;  in mmu_alloc_special_roots()
3948 	mmu->pml5_root = pml5_root;  in mmu_alloc_special_roots()
3957 	return -ENOMEM;  in mmu_alloc_special_roots()
3970 	 * walk before the reads of sp->unsync/sp->unsync_children here.  in is_unsync_root()
3972 	 * Even if another CPU was marking the SP as unsync-ed simultaneously,  in is_unsync_root()
3990 	if (sp->unsync || sp->unsync_children)  in is_unsync_root()
4001 	if (vcpu->arch.mmu->root_role.direct)  in kvm_mmu_sync_roots()
4004 	if (!VALID_PAGE(vcpu->arch.mmu->root.hpa))  in kvm_mmu_sync_roots()
4009 	if (vcpu->arch.mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {  in kvm_mmu_sync_roots()
4010 		hpa_t root = vcpu->arch.mmu->root.hpa;  in kvm_mmu_sync_roots()
4017 		write_lock(&vcpu->kvm->mmu_lock);  in kvm_mmu_sync_roots()
4019 		write_unlock(&vcpu->kvm->mmu_lock);  in kvm_mmu_sync_roots()
4023 	write_lock(&vcpu->kvm->mmu_lock);  in kvm_mmu_sync_roots()
4026 		hpa_t root = vcpu->arch.mmu->pae_root[i];  in kvm_mmu_sync_roots()
4034 	write_unlock(&vcpu->kvm->mmu_lock);  in kvm_mmu_sync_roots()
4043 		if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa))  in kvm_mmu_sync_prev_roots()
4047 	kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);  in kvm_mmu_sync_prev_roots()
4055 		exception->error_code = 0;  in nonpaging_gva_to_gpa()
4076  * That SPTE may be non-present.
4083 	int leaf = -1;  in get_walk()
4115 /* return true if reserved bit(s) are detected on a valid, non-MMIO SPTE. */
4135 	 * to detect reserved bits on non-MMIO SPTEs. i.e. buggy SPTEs.  in get_mmio_spte()
4140 	rsvd_check = &vcpu->arch.mmu->shadow_zero_check;  in get_mmio_spte()
4142 	for (level = root; level >= leaf; level--)  in get_mmio_spte()
4146 		pr_err("%s: reserved bits set on MMU-present spte, addr 0x%llx, hierarchy:\n",  in get_mmio_spte()
4148 		for (level = root; level >= leaf; level--)  in get_mmio_spte()
4149 			pr_err("------ spte = 0x%llx level = %d, rsvd bits = 0x%llx",  in get_mmio_spte()
4167 		return -EINVAL;  in handle_mmio_page_fault()
4169 	if (is_mmio_spte(vcpu->kvm, spte)) {  in handle_mmio_page_fault()
4194 	if (unlikely(fault->rsvd))  in page_fault_handle_page_track()
4197 	if (!fault->present || !fault->write)  in page_fault_handle_page_track()
4204 	if (kvm_gfn_is_write_tracked(vcpu->kvm, fault->slot, fault->gfn))  in page_fault_handle_page_track()
4224 	u32 id = vcpu->arch.apf.id;  in alloc_apf_token()
4227 		vcpu->arch.apf.id = 1;  in alloc_apf_token()
4229 	return (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;  in alloc_apf_token()
4238 	arch.gfn = fault->gfn;  in kvm_arch_setup_async_pf()
4239 	arch.error_code = fault->error_code;  in kvm_arch_setup_async_pf()
4240 	arch.direct_map = vcpu->arch.mmu->root_role.direct;  in kvm_arch_setup_async_pf()
4241 	arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu);  in kvm_arch_setup_async_pf()
4243 	return kvm_setup_async_pf(vcpu, fault->addr,  in kvm_arch_setup_async_pf()
4244 				  kvm_vcpu_gfn_to_hva(vcpu, fault->gfn), &arch);  in kvm_arch_setup_async_pf()
4251 	if (WARN_ON_ONCE(work->arch.error_code & PFERR_PRIVATE_ACCESS))  in kvm_arch_async_page_ready()
4254 	if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) ||  in kvm_arch_async_page_ready()
4255 	      work->wakeup_all)  in kvm_arch_async_page_ready()
4262 	if (!vcpu->arch.mmu->root_role.direct &&  in kvm_arch_async_page_ready()
4263 	      work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu))  in kvm_arch_async_page_ready()
4266 	r = kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, work->arch.error_code,  in kvm_arch_async_page_ready()
4271 	 * ignore stats for all other return times.  Page-ready "faults" aren't  in kvm_arch_async_page_ready()
4275 		vcpu->stat.pf_fixed++;  in kvm_arch_async_page_ready()
4317 	kvm_release_faultin_page(vcpu->kvm, fault->refcounted_page,  in kvm_mmu_finish_page_fault()
4318 				 r == RET_PF_RETRY, fault->map_writable);  in kvm_mmu_finish_page_fault()
4326 	if (!kvm_slot_can_be_private(fault->slot)) {  in kvm_mmu_faultin_pfn_private()
4328 		return -EFAULT;  in kvm_mmu_faultin_pfn_private()
4331 	r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn,  in kvm_mmu_faultin_pfn_private()
4332 			     &fault->refcounted_page, &max_order);  in kvm_mmu_faultin_pfn_private()
4338 	fault->map_writable = !(fault->slot->flags & KVM_MEM_READONLY);  in kvm_mmu_faultin_pfn_private()
4339 	fault->max_level = kvm_max_private_mapping_level(vcpu->kvm, fault->pfn,  in kvm_mmu_faultin_pfn_private()
4340 							 fault->max_level, max_order);  in kvm_mmu_faultin_pfn_private()
4348 	unsigned int foll = fault->write ? FOLL_WRITE : 0;  in __kvm_mmu_faultin_pfn()
4350 	if (fault->is_private)  in __kvm_mmu_faultin_pfn()
4354 	fault->pfn = __kvm_faultin_pfn(fault->slot, fault->gfn, foll,  in __kvm_mmu_faultin_pfn()
4355 				       &fault->map_writable, &fault->refcounted_page);  in __kvm_mmu_faultin_pfn()
4358 	 * If resolving the page failed because I/O is needed to fault-in the  in __kvm_mmu_faultin_pfn()
4363 	if (fault->pfn != KVM_PFN_ERR_NEEDS_IO)  in __kvm_mmu_faultin_pfn()
4366 	if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {  in __kvm_mmu_faultin_pfn()
4367 		trace_kvm_try_async_get_page(fault->addr, fault->gfn);  in __kvm_mmu_faultin_pfn()
4368 		if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {  in __kvm_mmu_faultin_pfn()
4369 			trace_kvm_async_pf_repeated_fault(fault->addr, fault->gfn);  in __kvm_mmu_faultin_pfn()
4378 	 * Allow gup to bail on pending non-fatal signals when it's also allowed  in __kvm_mmu_faultin_pfn()
4384 	fault->pfn = __kvm_faultin_pfn(fault->slot, fault->gfn, foll,  in __kvm_mmu_faultin_pfn()
4385 				       &fault->map_writable, &fault->refcounted_page);  in __kvm_mmu_faultin_pfn()
4393 	struct kvm_memory_slot *slot = fault->slot;  in kvm_mmu_faultin_pfn()
4394 	struct kvm *kvm = vcpu->kvm;  in kvm_mmu_faultin_pfn()
4397 	if (KVM_BUG_ON(kvm_is_gfn_alias(kvm, fault->gfn), kvm))  in kvm_mmu_faultin_pfn()
4398 		return -EFAULT;  in kvm_mmu_faultin_pfn()
4403 	 * invalidation relate to fault->fn and resume the guest without  in kvm_mmu_faultin_pfn()
4406 	fault->mmu_seq = vcpu->kvm->mmu_invalidate_seq;  in kvm_mmu_faultin_pfn()
4413 	if (fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) {  in kvm_mmu_faultin_pfn()
4415 		return -EFAULT;  in kvm_mmu_faultin_pfn()
4426 	if (slot->flags & KVM_MEMSLOT_INVALID)  in kvm_mmu_faultin_pfn()
4429 	if (slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT) {  in kvm_mmu_faultin_pfn()
4448 		 * when the AVIC is re-enabled.  in kvm_mmu_faultin_pfn()
4450 		if (!kvm_apicv_activated(vcpu->kvm))  in kvm_mmu_faultin_pfn()
4458 	 * For mmu_lock, if there is an in-progress invalidation and the kernel  in kvm_mmu_faultin_pfn()
4460 	 * in response to mmu_lock being contended, which is *very* counter-  in kvm_mmu_faultin_pfn()
4470 	 * Do the pre-check even for non-preemtible kernels, i.e. even if KVM  in kvm_mmu_faultin_pfn()
4475 	if (mmu_invalidate_retry_gfn_unsafe(kvm, fault->mmu_seq, fault->gfn))  in kvm_mmu_faultin_pfn()
4482 	if (unlikely(is_error_pfn(fault->pfn)))  in kvm_mmu_faultin_pfn()
4485 	if (WARN_ON_ONCE(!fault->slot || is_noslot_pfn(fault->pfn)))  in kvm_mmu_faultin_pfn()
4495 	if (mmu_invalidate_retry_gfn_unsafe(kvm, fault->mmu_seq, fault->gfn)) {  in kvm_mmu_faultin_pfn()
4510 	struct kvm_mmu_page *sp = root_to_sp(vcpu->arch.mmu->root.hpa);  in is_page_fault_stale()
4513 	if (sp && is_obsolete_sp(vcpu->kvm, sp))  in is_page_fault_stale()
4532 	return fault->slot &&  in is_page_fault_stale()
4533 	       mmu_invalidate_retry_gfn(vcpu->kvm, fault->mmu_seq, fault->gfn);  in is_page_fault_stale()
4541 	if (WARN_ON_ONCE(kvm_mmu_is_dummy_root(vcpu->arch.mmu->root.hpa)))  in direct_page_fault()
4560 	write_lock(&vcpu->kvm->mmu_lock);  in direct_page_fault()
4573 	write_unlock(&vcpu->kvm->mmu_lock);  in direct_page_fault()
4581 	fault->max_level = PG_LEVEL_2M;  in nonpaging_page_fault()
4589 	u32 flags = vcpu->arch.apf.host_apf_flags;  in kvm_handle_page_fault()
4592 	/* A 64-bit CR2 should be impossible on 32-bit KVM. */  in kvm_handle_page_fault()
4594 		return -EFAULT;  in kvm_handle_page_fault()
4597 	 * Legacy #PF exception only have a 32-bit error code.  Simply drop the  in kvm_handle_page_fault()
4599 	 * set), and to ensure there are no collisions with KVM-defined bits.  in kvm_handle_page_fault()
4605 	 * Restrict KVM-defined flags to bits 63:32 so that it's impossible for  in kvm_handle_page_fault()
4610 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_handle_page_fault()
4617 		vcpu->arch.apf.host_apf_flags = 0;  in kvm_handle_page_fault()
4651 	read_lock(&vcpu->kvm->mmu_lock);  in kvm_tdp_mmu_page_fault()
4660 	read_unlock(&vcpu->kvm->mmu_lock);  in kvm_tdp_mmu_page_fault()
4668 	 * When EPT is enabled (shadow_memtype_mask is non-zero), and the VM  in kvm_mmu_may_ignore_guest_pat()
4669 	 * has non-coherent DMA (DMA doesn't snoop CPU caches), KVM's ABI is to  in kvm_mmu_may_ignore_guest_pat()
4672 	 * result, KVM _may_ ignore guest PAT, whereas without non-coherent DMA,  in kvm_mmu_may_ignore_guest_pat()
4697 	if (vcpu->arch.mmu->page_fault != kvm_tdp_page_fault)  in kvm_tdp_map_page()
4698 		return -EOPNOTSUPP;  in kvm_tdp_map_page()
4702 			return -EINTR;  in kvm_tdp_map_page()
4717 		return -ENOENT;  in kvm_tdp_map_page()
4724 		return -EIO;  in kvm_tdp_map_page()
4736 	if (!vcpu->kvm->arch.pre_fault_allowed)  in kvm_arch_vcpu_pre_fault_memory()
4737 		return -EOPNOTSUPP;  in kvm_arch_vcpu_pre_fault_memory()
4747 	if (kvm_arch_has_private_mem(vcpu->kvm) &&  in kvm_arch_vcpu_pre_fault_memory()
4748 	    kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa)))  in kvm_arch_vcpu_pre_fault_memory()
4753 	 * two-dimensional paging.  in kvm_arch_vcpu_pre_fault_memory()
4755 	r = kvm_tdp_map_page(vcpu, range->gpa, error_code, &level);  in kvm_arch_vcpu_pre_fault_memory()
4760 	 * If the mapping that covers range->gpa can use a huge page, it  in kvm_arch_vcpu_pre_fault_memory()
4761 	 * may start below it or end after range->gpa + range->size.  in kvm_arch_vcpu_pre_fault_memory()
4763 	end = (range->gpa & KVM_HPAGE_MASK(level)) + KVM_HPAGE_SIZE(level);  in kvm_arch_vcpu_pre_fault_memory()
4764 	return min(range->size, end - range->gpa);  in kvm_arch_vcpu_pre_fault_memory()
4769 	context->page_fault = nonpaging_page_fault;  in nonpaging_init_context()
4770 	context->gva_to_gpa = nonpaging_gva_to_gpa;  in nonpaging_init_context()
4771 	context->sync_spte = NULL;  in nonpaging_init_context()
4779 	if (!VALID_PAGE(root->hpa))  in is_root_usable()
4782 	if (!role.direct && pgd != root->pgd)  in is_root_usable()
4785 	sp = root_to_sp(root->hpa);  in is_root_usable()
4789 	return role.word == sp->role.word;  in is_root_usable()
4795  * If a matching root is found, it is assigned to kvm_mmu->root and
4797  * If no match is found, kvm_mmu->root is left invalid, the LRU root is
4806 	if (is_root_usable(&mmu->root, new_pgd, new_role))  in cached_root_find_and_keep_current()
4818 		swap(mmu->root, mmu->prev_roots[i]);  in cached_root_find_and_keep_current()
4819 		if (is_root_usable(&mmu->root, new_pgd, new_role))  in cached_root_find_and_keep_current()
4829  * On entry, mmu->root is invalid.
4830  * If a matching root is found, it is assigned to kvm_mmu->root, the LRU entry
4832  * If no match is found, kvm_mmu->root is left invalid and false is returned.
4841 		if (is_root_usable(&mmu->prev_roots[i], new_pgd, new_role))  in cached_root_find_without_current()
4847 	swap(mmu->root, mmu->prev_roots[i]);  in cached_root_find_without_current()
4849 	for (; i < KVM_MMU_NUM_PREV_ROOTS - 1; i++)  in cached_root_find_without_current()
4850 		mmu->prev_roots[i] = mmu->prev_roots[i + 1];  in cached_root_find_without_current()
4851 	mmu->prev_roots[i].hpa = INVALID_PAGE;  in cached_root_find_without_current()
4859 	 * Limit reuse to 64-bit hosts+VMs without "special" roots in order to  in fast_pgd_switch()
4862 	if (VALID_PAGE(mmu->root.hpa) && !root_to_sp(mmu->root.hpa))  in fast_pgd_switch()
4865 	if (VALID_PAGE(mmu->root.hpa))  in fast_pgd_switch()
4873 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_mmu_new_pgd()
4874 	union kvm_mmu_page_role new_role = mmu->root_role;  in kvm_mmu_new_pgd()
4878 	 * will establish a valid root prior to the next VM-Enter.  in kvm_mmu_new_pgd()
4880 	if (!fast_pgd_switch(vcpu->kvm, mmu, new_pgd, new_role))  in kvm_mmu_new_pgd()
4898 	 * switching to a new CR3, that GVA->GPA mapping may no longer be  in kvm_mmu_new_pgd()
4909 		struct kvm_mmu_page *sp = root_to_sp(vcpu->arch.mmu->root.hpa);  in kvm_mmu_new_pgd()
4920 	if (unlikely(is_mmio_spte(vcpu->kvm, *sptep))) {  in sync_mmio_spte()
4954 	rsvd_check->bad_mt_xwr = 0;  in __reset_rsvds_bits_mask()
4969 	 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for  in __reset_rsvds_bits_mask()
4978 		rsvd_check->rsvd_bits_mask[0][1] = 0;  in __reset_rsvds_bits_mask()
4979 		rsvd_check->rsvd_bits_mask[0][0] = 0;  in __reset_rsvds_bits_mask()
4980 		rsvd_check->rsvd_bits_mask[1][0] =  in __reset_rsvds_bits_mask()
4981 			rsvd_check->rsvd_bits_mask[0][0];  in __reset_rsvds_bits_mask()
4984 			rsvd_check->rsvd_bits_mask[1][1] = 0;  in __reset_rsvds_bits_mask()
4990 			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);  in __reset_rsvds_bits_mask()
4993 			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);  in __reset_rsvds_bits_mask()
4996 		rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |  in __reset_rsvds_bits_mask()
5000 		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;	/* PDE */  in __reset_rsvds_bits_mask()
5001 		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;	/* PTE */  in __reset_rsvds_bits_mask()
5002 		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5004 		rsvd_check->rsvd_bits_mask[1][0] =  in __reset_rsvds_bits_mask()
5005 			rsvd_check->rsvd_bits_mask[0][0];  in __reset_rsvds_bits_mask()
5008 		rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5011 		rsvd_check->rsvd_bits_mask[1][4] =  in __reset_rsvds_bits_mask()
5012 			rsvd_check->rsvd_bits_mask[0][4];  in __reset_rsvds_bits_mask()
5015 		rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5018 		rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5020 		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;  in __reset_rsvds_bits_mask()
5021 		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;  in __reset_rsvds_bits_mask()
5022 		rsvd_check->rsvd_bits_mask[1][3] =  in __reset_rsvds_bits_mask()
5023 			rsvd_check->rsvd_bits_mask[0][3];  in __reset_rsvds_bits_mask()
5024 		rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5027 		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5029 		rsvd_check->rsvd_bits_mask[1][0] =  in __reset_rsvds_bits_mask()
5030 			rsvd_check->rsvd_bits_mask[0][0];  in __reset_rsvds_bits_mask()
5038 	__reset_rsvds_bits_mask(&context->guest_rsvd_check,  in reset_guest_rsvds_bits_mask()
5039 				vcpu->arch.reserved_gpa_bits,  in reset_guest_rsvds_bits_mask()
5040 				context->cpu_role.base.level, is_efer_nx(context),  in reset_guest_rsvds_bits_mask()
5059 	rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);  in __reset_rsvds_bits_mask_ept()
5060 	rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);  in __reset_rsvds_bits_mask_ept()
5061 	rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;  in __reset_rsvds_bits_mask_ept()
5062 	rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;  in __reset_rsvds_bits_mask_ept()
5063 	rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;  in __reset_rsvds_bits_mask_ept()
5066 	rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];  in __reset_rsvds_bits_mask_ept()
5067 	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];  in __reset_rsvds_bits_mask_ept()
5068 	rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;  in __reset_rsvds_bits_mask_ept()
5069 	rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;  in __reset_rsvds_bits_mask_ept()
5070 	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];  in __reset_rsvds_bits_mask_ept()
5081 	rsvd_check->bad_mt_xwr = bad_mt_xwr;  in __reset_rsvds_bits_mask_ept()
5087 	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,  in reset_rsvds_bits_mask_ept()
5088 				    vcpu->arch.reserved_gpa_bits, execonly,  in reset_rsvds_bits_mask_ept()
5107 	/* KVM doesn't use 2-level page tables for the shadow MMU. */  in reset_shadow_zero_bits_mask()
5112 	WARN_ON_ONCE(context->root_role.level < PT32E_ROOT_LEVEL);  in reset_shadow_zero_bits_mask()
5114 	shadow_zero_check = &context->shadow_zero_check;  in reset_shadow_zero_bits_mask()
5116 				context->root_role.level,  in reset_shadow_zero_bits_mask()
5117 				context->root_role.efer_nx,  in reset_shadow_zero_bits_mask()
5124 	for (i = context->root_role.level; --i >= 0;) {  in reset_shadow_zero_bits_mask()
5131 		shadow_zero_check->rsvd_bits_mask[0][i] |= shadow_me_mask;  in reset_shadow_zero_bits_mask()
5132 		shadow_zero_check->rsvd_bits_mask[1][i] |= shadow_me_mask;  in reset_shadow_zero_bits_mask()
5133 		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_value;  in reset_shadow_zero_bits_mask()
5134 		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_value;  in reset_shadow_zero_bits_mask()
5147  * possible, however, kvm currently does not do execution-protection.
5154 	shadow_zero_check = &context->shadow_zero_check;  in reset_tdp_shadow_zero_bits_mask()
5158 					context->root_role.level, true,  in reset_tdp_shadow_zero_bits_mask()
5169 	for (i = context->root_role.level; --i >= 0;) {  in reset_tdp_shadow_zero_bits_mask()
5170 		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;  in reset_tdp_shadow_zero_bits_mask()
5171 		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;  in reset_tdp_shadow_zero_bits_mask()
5182 	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,  in reset_ept_shadow_zero_bits_mask()
5210 	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {  in update_permission_bitmask()
5218 		/* Faults from writes to non-writable pages */  in update_permission_bitmask()
5222 		/* Faults from fetches of non-executable pages*/  in update_permission_bitmask()
5246 			 * SMAP:kernel-mode data accesses from user-mode  in update_permission_bitmask()
5250 			 *   - X86_CR4_SMAP is set in CR4  in update_permission_bitmask()
5251 			 *   - A user page is accessed  in update_permission_bitmask()
5252 			 *   - The access is not a fetch  in update_permission_bitmask()
5253 			 *   - The access is supervisor mode  in update_permission_bitmask()
5254 			 *   - If implicit supervisor access or X86_EFLAGS_AC is clear  in update_permission_bitmask()
5265 		mmu->permissions[byte] = ff | uf | wf | smepf | smapf;  in update_permission_bitmask()
5271 * user-mode addresses based on the value in the PKRU register.  Protection
5280 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
5281 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
5282 * - PK is always zero if U=0 in the page tables
5283 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
5298 	mmu->pkru_mask = 0;  in update_pkru_bitmask()
5305 	for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {  in update_pkru_bitmask()
5333 		mmu->pkru_mask |= (pkey_bits & 3) << pfec;  in update_pkru_bitmask()
5350 	context->page_fault = paging64_page_fault;  in paging64_init_context()
5351 	context->gva_to_gpa = paging64_gva_to_gpa;  in paging64_init_context()
5352 	context->sync_spte = paging64_sync_spte;  in paging64_init_context()
5357 	context->page_fault = paging32_page_fault;  in paging32_init_context()
5358 	context->gva_to_gpa = paging32_gva_to_gpa;  in paging32_init_context()
5359 	context->sync_spte = paging32_sync_spte;  in paging32_init_context()
5370 	role.ext.valid = 1;  in kvm_calc_cpu_role()
5391 	role.ext.cr4_smep = ____is_cr4_smep(regs);  in kvm_calc_cpu_role()
5392 	role.ext.cr4_smap = ____is_cr4_smap(regs);  in kvm_calc_cpu_role()
5393 	role.ext.cr4_pse = ____is_cr4_pse(regs);  in kvm_calc_cpu_role()
5396 	role.ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs);  in kvm_calc_cpu_role()
5397 	role.ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs);  in kvm_calc_cpu_role()
5398 	role.ext.efer_lma = ____is_efer_lma(regs);  in kvm_calc_cpu_role()
5413 	mmu->cpu_role.base.cr0_wp = cr0_wp;  in __kvm_mmu_refresh_passthrough_bits()
5423 	/* Use 5-level TDP if and only if it's useful/necessary. */  in kvm_mmu_get_tdp_level()
5457 	struct kvm_mmu *context = &vcpu->arch.root_mmu;  in init_kvm_tdp_mmu()
5460 	if (cpu_role.as_u64 == context->cpu_role.as_u64 &&  in init_kvm_tdp_mmu()
5461 	    root_role.word == context->root_role.word)  in init_kvm_tdp_mmu()
5464 	context->cpu_role.as_u64 = cpu_role.as_u64;  in init_kvm_tdp_mmu()
5465 	context->root_role.word = root_role.word;  in init_kvm_tdp_mmu()
5466 	context->page_fault = kvm_tdp_page_fault;  in init_kvm_tdp_mmu()
5467 	context->sync_spte = NULL;  in init_kvm_tdp_mmu()
5468 	context->get_guest_pgd = get_guest_cr3;  in init_kvm_tdp_mmu()
5469 	context->get_pdptr = kvm_pdptr_read;  in init_kvm_tdp_mmu()
5470 	context->inject_page_fault = kvm_inject_page_fault;  in init_kvm_tdp_mmu()
5473 		context->gva_to_gpa = nonpaging_gva_to_gpa;  in init_kvm_tdp_mmu()
5475 		context->gva_to_gpa = paging64_gva_to_gpa;  in init_kvm_tdp_mmu()
5477 		context->gva_to_gpa = paging32_gva_to_gpa;  in init_kvm_tdp_mmu()
5487 	if (cpu_role.as_u64 == context->cpu_role.as_u64 &&  in shadow_mmu_init_context()
5488 	    root_role.word == context->root_role.word)  in shadow_mmu_init_context()
5491 	context->cpu_role.as_u64 = cpu_role.as_u64;  in shadow_mmu_init_context()
5492 	context->root_role.word = root_role.word;  in shadow_mmu_init_context()
5508 	struct kvm_mmu *context = &vcpu->arch.root_mmu;  in kvm_init_shadow_mmu()
5513 	/* KVM uses PAE paging whenever the guest isn't using 64-bit paging. */  in kvm_init_shadow_mmu()
5519 	 * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and  in kvm_init_shadow_mmu()
5521 	 * The iTLB multi-hit workaround can be toggled at any time, so assume  in kvm_init_shadow_mmu()
5522 	 * NX can be used by any non-nested shadow MMU to avoid having to reset  in kvm_init_shadow_mmu()
5533 	struct kvm_mmu *context = &vcpu->arch.guest_mmu;  in kvm_init_shadow_npt_mmu()
5574 	role.ext.word = 0;  in kvm_calc_shadow_ept_root_page_role()
5575 	role.ext.execonly = execonly;  in kvm_calc_shadow_ept_root_page_role()
5576 	role.ext.valid = 1;  in kvm_calc_shadow_ept_root_page_role()
5585 	struct kvm_mmu *context = &vcpu->arch.guest_mmu;  in kvm_init_shadow_ept_mmu()
5591 	if (new_mode.as_u64 != context->cpu_role.as_u64) {  in kvm_init_shadow_ept_mmu()
5593 		context->cpu_role.as_u64 = new_mode.as_u64;  in kvm_init_shadow_ept_mmu()
5594 		context->root_role.word = new_mode.base.word;  in kvm_init_shadow_ept_mmu()
5596 		context->page_fault = ept_page_fault;  in kvm_init_shadow_ept_mmu()
5597 		context->gva_to_gpa = ept_gva_to_gpa;  in kvm_init_shadow_ept_mmu()
5598 		context->sync_spte = ept_sync_spte;  in kvm_init_shadow_ept_mmu()
5601 		context->pkru_mask = 0;  in kvm_init_shadow_ept_mmu()
5613 	struct kvm_mmu *context = &vcpu->arch.root_mmu;  in init_kvm_softmmu()
5617 	context->get_guest_pgd     = get_guest_cr3;  in init_kvm_softmmu()
5618 	context->get_pdptr         = kvm_pdptr_read;  in init_kvm_softmmu()
5619 	context->inject_page_fault = kvm_inject_page_fault;  in init_kvm_softmmu()
5625 	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;  in init_kvm_nested_mmu()
5627 	if (new_mode.as_u64 == g_context->cpu_role.as_u64)  in init_kvm_nested_mmu()
5630 	g_context->cpu_role.as_u64   = new_mode.as_u64;  in init_kvm_nested_mmu()
5631 	g_context->get_guest_pgd     = get_guest_cr3;  in init_kvm_nested_mmu()
5632 	g_context->get_pdptr         = kvm_pdptr_read;  in init_kvm_nested_mmu()
5633 	g_context->inject_page_fault = kvm_inject_page_fault;  in init_kvm_nested_mmu()
5639 	g_context->sync_spte         = NULL;  in init_kvm_nested_mmu()
5642 	 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using  in init_kvm_nested_mmu()
5650 		g_context->gva_to_gpa = nonpaging_gva_to_gpa;  in init_kvm_nested_mmu()
5652 		g_context->gva_to_gpa = paging64_gva_to_gpa;  in init_kvm_nested_mmu()
5654 		g_context->gva_to_gpa = paging64_gva_to_gpa;  in init_kvm_nested_mmu()
5656 		g_context->gva_to_gpa = paging32_gva_to_gpa;  in init_kvm_nested_mmu()
5689 	vcpu->arch.root_mmu.root_role.invalid = 1;  in kvm_mmu_after_set_cpuid()
5690 	vcpu->arch.guest_mmu.root_role.invalid = 1;  in kvm_mmu_after_set_cpuid()
5691 	vcpu->arch.nested_mmu.root_role.invalid = 1;  in kvm_mmu_after_set_cpuid()
5692 	vcpu->arch.root_mmu.cpu_role.ext.valid = 0;  in kvm_mmu_after_set_cpuid()
5693 	vcpu->arch.guest_mmu.cpu_role.ext.valid = 0;  in kvm_mmu_after_set_cpuid()
5694 	vcpu->arch.nested_mmu.cpu_role.ext.valid = 0;  in kvm_mmu_after_set_cpuid()
5701 	KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm);  in kvm_mmu_after_set_cpuid()
5715 	r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->root_role.direct);  in kvm_mmu_load()
5721 	if (vcpu->arch.mmu->root_role.direct)  in kvm_mmu_load()
5746 	struct kvm *kvm = vcpu->kvm;  in kvm_mmu_unload()
5748 	kvm_mmu_free_roots(kvm, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);  in kvm_mmu_unload()
5749 	WARN_ON_ONCE(VALID_PAGE(vcpu->arch.root_mmu.root.hpa));  in kvm_mmu_unload()
5750 	kvm_mmu_free_roots(kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);  in kvm_mmu_unload()
5751 	WARN_ON_ONCE(VALID_PAGE(vcpu->arch.guest_mmu.root.hpa));  in kvm_mmu_unload()
5773 	 *      is unlikely to zap an in-use PGD.  in is_obsolete_root()
5787 	if (is_obsolete_root(kvm, mmu->root.hpa))  in __kvm_mmu_free_obsolete_roots()
5791 		if (is_obsolete_root(kvm, mmu->prev_roots[i].hpa))  in __kvm_mmu_free_obsolete_roots()
5801 	__kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);  in kvm_mmu_free_obsolete_roots()
5802 	__kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);  in kvm_mmu_free_obsolete_roots()
5817 		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */  in mmu_pte_write_fetch_gpte()
5838 	 * Skip write-flooding detected for the sp whose level is 1, because  in detect_write_flooding()
5839 	 * it can become unsync, then the guest page is not write-protected.  in detect_write_flooding()
5841 	if (sp->role.level == PG_LEVEL_4K)  in detect_write_flooding()
5844 	atomic_inc(&sp->write_flooding_count);  in detect_write_flooding()
5845 	return atomic_read(&sp->write_flooding_count) >= 3;  in detect_write_flooding()
5858 	pte_size = sp->role.has_4_byte_gpte ? 4 : 8;  in detect_write_misaligned()
5864 	if (!(offset & (pte_size - 1)) && bytes == 1)  in detect_write_misaligned()
5867 	misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);  in detect_write_misaligned()
5880 	level = sp->role.level;  in get_written_sptes()
5882 	if (sp->role.has_4_byte_gpte) {  in get_written_sptes()
5883 		page_offset <<= 1;	/* 32->64 */  in get_written_sptes()
5885 		 * A 32-bit pde maps 4MB while the shadow pdes map  in get_written_sptes()
5896 		if (quadrant != sp->role.quadrant)  in get_written_sptes()
5900 	spte = &sp->spt[page_offset / sizeof(*spte)];  in get_written_sptes()
5919 	 * a non-zero indirect_shadow_pages.  Pairs with the smp_mb() in  in kvm_mmu_track_write()
5923 	if (!vcpu->kvm->arch.indirect_shadow_pages)  in kvm_mmu_track_write()
5926 	write_lock(&vcpu->kvm->mmu_lock);  in kvm_mmu_track_write()
5930 	++vcpu->kvm->stat.mmu_pte_write;  in kvm_mmu_track_write()
5932 	for_each_gfn_valid_sp_with_gptes(vcpu->kvm, sp, gfn) {  in kvm_mmu_track_write()
5935 			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);  in kvm_mmu_track_write()
5936 			++vcpu->kvm->stat.mmu_flooded;  in kvm_mmu_track_write()
5944 		while (npte--) {  in kvm_mmu_track_write()
5946 			mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);  in kvm_mmu_track_write()
5947 			if (gentry && sp->role.level != PG_LEVEL_4K)  in kvm_mmu_track_write()
5948 				++vcpu->kvm->stat.mmu_pde_zapped;  in kvm_mmu_track_write()
5954 	kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);  in kvm_mmu_track_write()
5955 	write_unlock(&vcpu->kvm->mmu_lock);  in kvm_mmu_track_write()
5968 	bool direct = vcpu->arch.mmu->root_role.direct;  in kvm_mmu_write_protect_fault()
5971 	 * Do not try to unprotect and retry if the vCPU re-faulted on the same  in kvm_mmu_write_protect_fault()
5974 	 * a non-page-table modifying instruction on the PDE that points to the  in kvm_mmu_write_protect_fault()
5978 	if (vcpu->arch.last_retry_eip == kvm_rip_read(vcpu) &&  in kvm_mmu_write_protect_fault()
5979 	    vcpu->arch.last_retry_addr == cr2_or_gpa)  in kvm_mmu_write_protect_fault()
5988 	vcpu->arch.last_retry_eip = 0;  in kvm_mmu_write_protect_fault()
5989 	vcpu->arch.last_retry_addr = 0;  in kvm_mmu_write_protect_fault()
5998 	 * will keep faulting on the non-existent MMIO address.  in kvm_mmu_write_protect_fault()
6005 	 * to a read-only violation while the CPU was walking non-nested NPT  in kvm_mmu_write_protect_fault()
6009 	 * (L0) write-protects the nested NPTs, i.e. npt12 entries, KVM is also  in kvm_mmu_write_protect_fault()
6010 	 * unknowingly write-protecting L1's guest page tables, which KVM isn't  in kvm_mmu_write_protect_fault()
6018 	 * isn't shadowed by KVM, there is no need to write-protect L1's gPTEs  in kvm_mmu_write_protect_fault()
6024 	 * the instruction.  If no shadow pages were zapped, then the write-  in kvm_mmu_write_protect_fault()
6044 	 * The gfn is write-protected, but if KVM detects its emulating an  in kvm_mmu_write_protect_fault()
6047 	 * re-execute the instruction that caused the page fault.  Do not allow  in kvm_mmu_write_protect_fault()
6062 	bool direct = vcpu->arch.mmu->root_role.direct;  in kvm_mmu_page_fault()
6064 	if (WARN_ON_ONCE(!VALID_PAGE(vcpu->arch.mmu->root.hpa)))  in kvm_mmu_page_fault()
6068 	 * Except for reserved faults (emulated MMIO is shared-only), set the  in kvm_mmu_page_fault()
6069 	 * PFERR_PRIVATE_ACCESS flag for software-protected VMs based on the gfn's  in kvm_mmu_page_fault()
6073 	 * for software-protected VMs.  in kvm_mmu_page_fault()
6077 	    vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM &&  in kvm_mmu_page_fault()
6078 	    kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)))  in kvm_mmu_page_fault()
6084 			return -EFAULT;  in kvm_mmu_page_fault()
6092 		vcpu->stat.pf_taken++;  in kvm_mmu_page_fault()
6096 		if (KVM_BUG_ON(r == RET_PF_INVALID, vcpu->kvm))  in kvm_mmu_page_fault()
6097 			return -EIO;  in kvm_mmu_page_fault()
6108 		vcpu->stat.pf_fixed++;  in kvm_mmu_page_fault()
6110 		vcpu->stat.pf_emulate++;  in kvm_mmu_page_fault()
6112 		vcpu->stat.pf_spurious++;  in kvm_mmu_page_fault()
6141 	for (level = root_level; level >= leaf; level--)  in kvm_mmu_print_sptes()
6159 	if (WARN_ON_ONCE(mmu != vcpu->arch.mmu))  in __kvm_mmu_invalidate_addr()
6165 	write_lock(&vcpu->kvm->mmu_lock);  in __kvm_mmu_invalidate_addr()
6169 		if (sp->unsync) {  in __kvm_mmu_invalidate_addr()
6173 				mmu_page_zap_pte(vcpu->kvm, sp, iterator.sptep, NULL);  in __kvm_mmu_invalidate_addr()
6175 				kvm_flush_remote_tlbs_sptep(vcpu->kvm, iterator.sptep);  in __kvm_mmu_invalidate_addr()
6178 		if (!sp->unsync_children)  in __kvm_mmu_invalidate_addr()
6181 	write_unlock(&vcpu->kvm->mmu_lock);  in __kvm_mmu_invalidate_addr()
6191 	/* It's actually a GPA for vcpu->arch.guest_mmu.  */  in kvm_mmu_invalidate_addr()
6192 	if (mmu != &vcpu->arch.guest_mmu) {  in kvm_mmu_invalidate_addr()
6193 		/* INVLPG on a non-canonical address is a NOP according to the SDM.  */  in kvm_mmu_invalidate_addr()
6200 	if (!mmu->sync_spte)  in kvm_mmu_invalidate_addr()
6204 		__kvm_mmu_invalidate_addr(vcpu, mmu, addr, mmu->root.hpa);  in kvm_mmu_invalidate_addr()
6208 			__kvm_mmu_invalidate_addr(vcpu, mmu, addr, mmu->prev_roots[i].hpa);  in kvm_mmu_invalidate_addr()
6225 	kvm_mmu_invalidate_addr(vcpu, vcpu->arch.walk_mmu, gva, KVM_MMU_ROOTS_ALL);  in kvm_mmu_invlpg()
6226 	++vcpu->stat.invlpg;  in kvm_mmu_invlpg()
6233 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_mmu_invpcid_gva()
6241 		if (VALID_PAGE(mmu->prev_roots[i].hpa) &&  in kvm_mmu_invpcid_gva()
6242 		    pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd))  in kvm_mmu_invpcid_gva()
6248 	++vcpu->stat.invlpg;  in kvm_mmu_invpcid_gva()
6285 	if (!tdp_enabled && mmu->pae_root)  in free_mmu_pages()
6286 		set_memory_encrypted((unsigned long)mmu->pae_root, 1);  in free_mmu_pages()
6287 	free_page((unsigned long)mmu->pae_root);  in free_mmu_pages()
6288 	free_page((unsigned long)mmu->pml4_root);  in free_mmu_pages()
6289 	free_page((unsigned long)mmu->pml5_root);  in free_mmu_pages()
6297 	mmu->root.hpa = INVALID_PAGE;  in __kvm_mmu_create()
6298 	mmu->root.pgd = 0;  in __kvm_mmu_create()
6299 	mmu->mirror_root_hpa = INVALID_PAGE;  in __kvm_mmu_create()
6301 		mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;  in __kvm_mmu_create()
6303 	/* vcpu->arch.guest_mmu isn't used when !tdp_enabled. */  in __kvm_mmu_create()
6304 	if (!tdp_enabled && mmu == &vcpu->arch.guest_mmu)  in __kvm_mmu_create()
6309 	 * while the PDP table is a per-vCPU construct that's allocated at MMU  in __kvm_mmu_create()
6310 	 * creation.  When emulating 32-bit mode, cr3 is only 32 bits even on  in __kvm_mmu_create()
6314 	 * table.  The main exception, handled here, is SVM's 32-bit NPT.  The  in __kvm_mmu_create()
6315 	 * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit  in __kvm_mmu_create()
6316 	 * KVM; that horror is handled on-demand by mmu_alloc_special_roots().  in __kvm_mmu_create()
6323 		return -ENOMEM;  in __kvm_mmu_create()
6325 	mmu->pae_root = page_address(page);  in __kvm_mmu_create()
6331 	 * only necessary when using shadow paging, as 64-bit NPT can get at  in __kvm_mmu_create()
6332 	 * the C-bit even when shadowing 32-bit NPT, and SME isn't supported  in __kvm_mmu_create()
6333 	 * by 32-bit kernels (when KVM itself uses 32-bit NPT).  in __kvm_mmu_create()
6336 		set_memory_decrypted((unsigned long)mmu->pae_root, 1);  in __kvm_mmu_create()
6341 		mmu->pae_root[i] = INVALID_PAE_ROOT;  in __kvm_mmu_create()
6350 	vcpu->arch.mmu_pte_list_desc_cache.kmem_cache = pte_list_desc_cache;  in kvm_mmu_create()
6351 	vcpu->arch.mmu_pte_list_desc_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_create()
6353 	vcpu->arch.mmu_page_header_cache.kmem_cache = mmu_page_header_cache;  in kvm_mmu_create()
6354 	vcpu->arch.mmu_page_header_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_create()
6356 	vcpu->arch.mmu_shadow_page_cache.init_value =  in kvm_mmu_create()
6358 	if (!vcpu->arch.mmu_shadow_page_cache.init_value)  in kvm_mmu_create()
6359 		vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_create()
6361 	vcpu->arch.mmu = &vcpu->arch.root_mmu;  in kvm_mmu_create()
6362 	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;  in kvm_mmu_create()
6364 	ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);  in kvm_mmu_create()
6368 	ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu);  in kvm_mmu_create()
6374 	free_mmu_pages(&vcpu->arch.guest_mmu);  in kvm_mmu_create()
6386 	lockdep_assert_held(&kvm->slots_lock);  in kvm_zap_obsolete_pages()
6390 	      &kvm->arch.active_mmu_pages, link) {  in kvm_zap_obsolete_pages()
6403 		if (WARN_ON_ONCE(sp->role.invalid))  in kvm_zap_obsolete_pages()
6413 		    cond_resched_rwlock_write(&kvm->mmu_lock)) {  in kvm_zap_obsolete_pages()
6439  * Fast invalidate all shadow pages and use lock-break technique
6444  * not use any resource of the being-deleted slot or all slots
6449 	lockdep_assert_held(&kvm->slots_lock);  in kvm_mmu_zap_all_fast()
6451 	write_lock(&kvm->mmu_lock);  in kvm_mmu_zap_all_fast()
6461 	kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1;  in kvm_mmu_zap_all_fast()
6464 	 * In order to ensure all vCPUs drop their soon-to-be invalid roots,  in kvm_mmu_zap_all_fast()
6489 	write_unlock(&kvm->mmu_lock);  in kvm_mmu_zap_all_fast()
6497 	 * lead to use-after-free.  in kvm_mmu_zap_all_fast()
6505 	kvm->arch.shadow_mmio_value = shadow_mmio_value;  in kvm_mmu_init_vm()
6506 	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);  in kvm_mmu_init_vm()
6507 	INIT_LIST_HEAD(&kvm->arch.possible_nx_huge_pages);  in kvm_mmu_init_vm()
6508 	spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);  in kvm_mmu_init_vm()
6513 	kvm->arch.split_page_header_cache.kmem_cache = mmu_page_header_cache;  in kvm_mmu_init_vm()
6514 	kvm->arch.split_page_header_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_init_vm()
6516 	kvm->arch.split_shadow_page_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_init_vm()
6518 	kvm->arch.split_desc_cache.kmem_cache = pte_list_desc_cache;  in kvm_mmu_init_vm()
6519 	kvm->arch.split_desc_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_init_vm()
6524 	kvm_mmu_free_memory_cache(&kvm->arch.split_desc_cache);  in mmu_free_vm_memory_caches()
6525 	kvm_mmu_free_memory_cache(&kvm->arch.split_page_header_cache);  in mmu_free_vm_memory_caches()
6526 	kvm_mmu_free_memory_cache(&kvm->arch.split_shadow_page_cache);  in mmu_free_vm_memory_caches()
6554 			start = max(gfn_start, memslot->base_gfn);  in kvm_rmap_zap_gfn_range()
6555 			end = min(gfn_end, memslot->base_gfn + memslot->npages);  in kvm_rmap_zap_gfn_range()
6578 	write_lock(&kvm->mmu_lock);  in kvm_zap_gfn_range()
6590 		kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);  in kvm_zap_gfn_range()
6594 	write_unlock(&kvm->mmu_lock);  in kvm_zap_gfn_range()
6609 		write_lock(&kvm->mmu_lock);  in kvm_mmu_slot_remove_write_access()
6612 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_remove_write_access()
6616 		read_lock(&kvm->mmu_lock);  in kvm_mmu_slot_remove_write_access()
6618 		read_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_remove_write_access()
6629 	if (need_resched() || rwlock_needbreak(&kvm->mmu_lock))  in need_topup_split_caches_or_resched()
6637 	return need_topup(&kvm->arch.split_desc_cache, SPLIT_DESC_CACHE_MIN_NR_OBJECTS) ||  in need_topup_split_caches_or_resched()
6638 	       need_topup(&kvm->arch.split_page_header_cache, 1) ||  in need_topup_split_caches_or_resched()
6639 	       need_topup(&kvm->arch.split_shadow_page_cache, 1);  in need_topup_split_caches_or_resched()
6651 	 * but aliasing rarely occurs post-boot or for many gfns.  If there is  in topup_split_caches()
6652 	 * only one rmap entry, rmap->val points directly at that one entry and  in topup_split_caches()
6661 	lockdep_assert_held(&kvm->slots_lock);  in topup_split_caches()
6663 	r = __kvm_mmu_topup_memory_cache(&kvm->arch.split_desc_cache, capacity,  in topup_split_caches()
6668 	r = kvm_mmu_topup_memory_cache(&kvm->arch.split_page_header_cache, 1);  in topup_split_caches()
6672 	return kvm_mmu_topup_memory_cache(&kvm->arch.split_shadow_page_cache, 1);  in topup_split_caches()
6695 	caches.page_header_cache = &kvm->arch.split_page_header_cache;  in shadow_mmu_get_sp_for_split()
6696 	caches.shadow_page_cache = &kvm->arch.split_shadow_page_cache;  in shadow_mmu_get_sp_for_split()
6707 	struct kvm_mmu_memory_cache *cache = &kvm->arch.split_desc_cache;  in shadow_mmu_split_huge_page()
6718 		sptep = &sp->spt[index];  in shadow_mmu_split_huge_page()
6725 		 * gfn-to-pfn translation since the SP is direct, so no need to  in shadow_mmu_split_huge_page()
6736 			flush |= !is_last_spte(*sptep, sp->role.level);  in shadow_mmu_split_huge_page()
6740 		spte = make_small_spte(kvm, huge_spte, sp->role, index);  in shadow_mmu_split_huge_page()
6742 		__rmap_add(kvm, cache, slot, sptep, gfn, sp->role.access);  in shadow_mmu_split_huge_page()
6759 	level = huge_sp->role.level;  in shadow_mmu_try_split_huge_page()
6763 		r = -ENOSPC;  in shadow_mmu_try_split_huge_page()
6768 		write_unlock(&kvm->mmu_lock);  in shadow_mmu_try_split_huge_page()
6771 		 * If the topup succeeds, return -EAGAIN to indicate that the  in shadow_mmu_try_split_huge_page()
6775 		r = topup_split_caches(kvm) ?: -EAGAIN;  in shadow_mmu_try_split_huge_page()
6776 		write_lock(&kvm->mmu_lock);  in shadow_mmu_try_split_huge_page()
6801 		if (WARN_ON_ONCE(!sp->role.guest_mode))  in shadow_mmu_try_split_huge_pages()
6804 		/* The rmaps should never contain non-leaf SPTEs. */  in shadow_mmu_try_split_huge_pages()
6809 		if (WARN_ON_ONCE(sp->unsync))  in shadow_mmu_try_split_huge_pages()
6813 		if (sp->role.invalid)  in shadow_mmu_try_split_huge_pages()
6823 		if (!r || r == -EAGAIN)  in shadow_mmu_try_split_huge_pages()
6826 		/* The split failed and shouldn't be retried (e.g. -ENOMEM). */  in shadow_mmu_try_split_huge_pages()
6846 	for (level = KVM_MAX_HUGEPAGE_LEVEL; level > target_level; level--)  in kvm_shadow_mmu_try_split_huge_pages()
6848 				  level, level, start, end - 1, true, true, false);  in kvm_shadow_mmu_try_split_huge_pages()
6851 /* Must be called with the mmu_lock held in write-mode. */
6875 	u64 start = memslot->base_gfn;  in kvm_mmu_slot_try_split_huge_pages()
6876 	u64 end = start + memslot->npages;  in kvm_mmu_slot_try_split_huge_pages()
6882 		write_lock(&kvm->mmu_lock);  in kvm_mmu_slot_try_split_huge_pages()
6884 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_try_split_huge_pages()
6887 	read_lock(&kvm->mmu_lock);  in kvm_mmu_slot_try_split_huge_pages()
6889 	read_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_try_split_huge_pages()
6893 	 * write-protecting and/or clearing dirty on the newly split SPTEs to  in kvm_mmu_slot_try_split_huge_pages()
6922 		if (sp->role.direct &&  in kvm_mmu_zap_collapsible_spte()
6923 		    sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn)) {  in kvm_mmu_zap_collapsible_spte()
6943 	 * Note, use KVM_MAX_HUGEPAGE_LEVEL - 1 since there's no need to zap  in kvm_rmap_zap_collapsible_sptes()
6947 			    PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL - 1, true))  in kvm_rmap_zap_collapsible_sptes()
6955 		write_lock(&kvm->mmu_lock);  in kvm_mmu_recover_huge_pages()
6957 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_recover_huge_pages()
6961 		read_lock(&kvm->mmu_lock);  in kvm_mmu_recover_huge_pages()
6963 		read_unlock(&kvm->mmu_lock);  in kvm_mmu_recover_huge_pages()
6971 		write_lock(&kvm->mmu_lock);  in kvm_mmu_slot_leaf_clear_dirty()
6977 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_leaf_clear_dirty()
6981 		read_lock(&kvm->mmu_lock);  in kvm_mmu_slot_leaf_clear_dirty()
6983 		read_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_leaf_clear_dirty()
7002 	write_lock(&kvm->mmu_lock);  in kvm_mmu_zap_all()
7004 	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {  in kvm_mmu_zap_all()
7005 		if (WARN_ON_ONCE(sp->role.invalid))  in kvm_mmu_zap_all()
7009 		if (cond_resched_rwlock_write(&kvm->mmu_lock))  in kvm_mmu_zap_all()
7018 	write_unlock(&kvm->mmu_lock);  in kvm_mmu_zap_all()
7033 	if (list_empty(&kvm->arch.active_mmu_pages))  in kvm_mmu_zap_memslot_pages_and_flush()
7040 	 * will result in use-after-free, e.g. in unaccount_shadowed().  in kvm_mmu_zap_memslot_pages_and_flush()
7042 	for (i = 0; i < slot->npages; i++) {  in kvm_mmu_zap_memslot_pages_and_flush()
7044 		gfn_t gfn = slot->base_gfn + i;  in kvm_mmu_zap_memslot_pages_and_flush()
7049 		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {  in kvm_mmu_zap_memslot_pages_and_flush()
7052 			cond_resched_rwlock_write(&kvm->mmu_lock);  in kvm_mmu_zap_memslot_pages_and_flush()
7065 		.start = slot->base_gfn,  in kvm_mmu_zap_memslot()
7066 		.end = slot->base_gfn + slot->npages,  in kvm_mmu_zap_memslot()
7071 	write_lock(&kvm->mmu_lock);  in kvm_mmu_zap_memslot()
7074 	write_unlock(&kvm->mmu_lock);  in kvm_mmu_zap_memslot()
7079 	return kvm->arch.vm_type == KVM_X86_DEFAULT_VM &&  in kvm_memslot_flush_zap_all()
7092 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)  in kvm_mmu_invalidate_mmio_sptes()  argument
7094 	WARN_ON_ONCE(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);  in kvm_mmu_invalidate_mmio_sptes()
7096 	gen &= MMIO_SPTE_GEN_MASK;  in kvm_mmu_invalidate_mmio_sptes()
7105 	gen &= ~((u64)kvm_arch_nr_memslot_as_ids(kvm) - 1);  in kvm_mmu_invalidate_mmio_sptes()
7111 	if (unlikely(gen == 0)) {  in kvm_mmu_invalidate_mmio_sptes()
7126 	 * The NX recovery thread is spawned on-demand at the first KVM_RUN and  in kvm_wake_nx_recovery_thread()
7130 	struct vhost_task *nx_thread = READ_ONCE(kvm->arch.nx_huge_page_recovery_thread);  in kvm_wake_nx_recovery_thread()
7161 		return -EPERM;  in set_nx_huge_pages()
7176 			return -EBUSY;  in set_nx_huge_pages()
7181 		return -EINVAL;  in set_nx_huge_pages()
7192 			mutex_lock(&kvm->slots_lock);  in set_nx_huge_pages()
7194 			mutex_unlock(&kvm->slots_lock);  in set_nx_huge_pages()
7206  * its default value of -1 is technically undefined behavior for a boolean.
7212 	if (nx_huge_pages == -1)  in kvm_mmu_x86_module_init()
7232 	int ret = -ENOMEM;  in kvm_mmu_vendor_module_init()
7267 		read_lock(&vcpu->kvm->mmu_lock);  in kvm_mmu_destroy()
7268 		mmu_free_root_page(vcpu->kvm, &vcpu->arch.mmu->mirror_root_hpa,  in kvm_mmu_destroy()
7270 		read_unlock(&vcpu->kvm->mmu_lock);  in kvm_mmu_destroy()
7272 	free_mmu_pages(&vcpu->arch.root_mmu);  in kvm_mmu_destroy()
7273 	free_mmu_pages(&vcpu->arch.guest_mmu);  in kvm_mmu_destroy()
7314 		return -EPERM;  in set_nx_huge_pages_recovery_param()
7341 	unsigned long nx_lpage_splits = kvm->stat.nx_lpage_splits;  in kvm_recover_nx_huge_pages()
7350 	rcu_idx = srcu_read_lock(&kvm->srcu);  in kvm_recover_nx_huge_pages()
7351 	write_lock(&kvm->mmu_lock);  in kvm_recover_nx_huge_pages()
7362 	for ( ; to_zap; --to_zap) {  in kvm_recover_nx_huge_pages()
7363 		if (list_empty(&kvm->arch.possible_nx_huge_pages))  in kvm_recover_nx_huge_pages()
7373 		sp = list_first_entry(&kvm->arch.possible_nx_huge_pages,  in kvm_recover_nx_huge_pages()
7376 		WARN_ON_ONCE(!sp->nx_huge_page_disallowed);  in kvm_recover_nx_huge_pages()
7377 		WARN_ON_ONCE(!sp->role.direct);  in kvm_recover_nx_huge_pages()
7393 		 * of kvm->nr_memslots_dirty_logging is not a problem: if it is  in kvm_recover_nx_huge_pages()
7400 		if (atomic_read(&kvm->nr_memslots_dirty_logging)) {  in kvm_recover_nx_huge_pages()
7403 			slots = kvm_memslots_for_spte_role(kvm, sp->role);  in kvm_recover_nx_huge_pages()
7404 			slot = __gfn_to_memslot(slots, sp->gfn);  in kvm_recover_nx_huge_pages()
7414 		WARN_ON_ONCE(sp->nx_huge_page_disallowed);  in kvm_recover_nx_huge_pages()
7416 		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {  in kvm_recover_nx_huge_pages()
7420 			cond_resched_rwlock_write(&kvm->mmu_lock);  in kvm_recover_nx_huge_pages()
7430 	write_unlock(&kvm->mmu_lock);  in kvm_recover_nx_huge_pages()
7431 	srcu_read_unlock(&kvm->srcu, rcu_idx);  in kvm_recover_nx_huge_pages()
7449 	remaining_time = kvm->arch.nx_huge_page_last + msecs_to_jiffies(period)  in kvm_nx_huge_page_recovery_worker()
7450 		- get_jiffies_64();  in kvm_nx_huge_page_recovery_worker()
7459 	kvm->arch.nx_huge_page_last = get_jiffies_64();  in kvm_nx_huge_page_recovery_worker()
7469 	kvm->arch.nx_huge_page_last = get_jiffies_64();  in kvm_mmu_start_lpage_recovery()
7472 				      kvm, "kvm-nx-lpage-recovery");  in kvm_mmu_start_lpage_recovery()
7480 	WRITE_ONCE(kvm->arch.nx_huge_page_recovery_thread, nx_thread);  in kvm_mmu_start_lpage_recovery()
7489 	return call_once(&kvm->arch.nx_once, kvm_mmu_start_lpage_recovery);  in kvm_mmu_post_init_vm()
7494 	if (kvm->arch.nx_huge_page_recovery_thread)  in kvm_mmu_pre_destroy_vm()
7495 		vhost_task_stop(kvm->arch.nx_huge_page_recovery_thread);  in kvm_mmu_pre_destroy_vm()
7517 	if (range->arg.attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE)  in kvm_arch_pre_set_memory_attributes()
7518 		range->attr_filter = KVM_FILTER_SHARED;  in kvm_arch_pre_set_memory_attributes()
7520 		range->attr_filter = KVM_FILTER_PRIVATE;  in kvm_arch_pre_set_memory_attributes()
7528 	return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG;  in hugepage_test_mixed()
7534 	lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG;  in hugepage_clear_mixed()
7540 	lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG;  in hugepage_set_mixed()
7552 	for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) {  in hugepage_has_attrs()
7553 		if (hugepage_test_mixed(slot, gfn, level - 1) ||  in hugepage_has_attrs()
7563 	unsigned long attrs = range->arg.attributes;  in kvm_arch_post_set_memory_attributes()
7564 	struct kvm_memory_slot *slot = range->slot;  in kvm_arch_post_set_memory_attributes()
7567 	lockdep_assert_held_write(&kvm->mmu_lock);  in kvm_arch_post_set_memory_attributes()
7568 	lockdep_assert_held(&kvm->slots_lock);  in kvm_arch_post_set_memory_attributes()
7585 		gfn_t gfn = gfn_round_for_level(range->start, level);  in kvm_arch_post_set_memory_attributes()
7588 		if (gfn != range->start || gfn + nr_pages > range->end) {  in kvm_arch_post_set_memory_attributes()
7594 			if (gfn >= slot->base_gfn &&  in kvm_arch_post_set_memory_attributes()
7595 			    gfn + nr_pages <= slot->base_gfn + slot->npages) {  in kvm_arch_post_set_memory_attributes()
7608 		for ( ; gfn + nr_pages <= range->end; gfn += nr_pages)  in kvm_arch_post_set_memory_attributes()
7616 		if (gfn < range->end &&  in kvm_arch_post_set_memory_attributes()
7617 		    (gfn + nr_pages) <= (slot->base_gfn + slot->npages)) {  in kvm_arch_post_set_memory_attributes()
7641 		gfn_t end = gfn_round_for_level(slot->base_gfn + slot->npages, level);  in kvm_mmu_init_memslot_memory_attributes()
7642 		gfn_t start = gfn_round_for_level(slot->base_gfn, level);  in kvm_mmu_init_memslot_memory_attributes()
7646 		if (start < slot->base_gfn)  in kvm_mmu_init_memslot_memory_attributes()