1 // SPDX-License-Identifier: GPL-2.0-only
3  * Kernel-based Virtual Machine driver for Linux
5  * This module enables machines with Intel VT-x extensions to run virtual
57 #include <asm/spec-ctrl.h>
64 int __read_mostly nx_huge_pages = -1;
100  * When setting this variable to true it enables Two-Dimensional-Paging
102  * 1. the guest-virtual to guest-physical
103  * 2. while doing 1. it walks guest-physical to host-physical
204 	return !!(regs->reg & flag);					\
226 	return !!(mmu->cpu_role. base_or_ext . reg##_##name);	\
239         return mmu->cpu_role.base.level > 0;  in is_cr0_pg()
244         return !mmu->cpu_role.base.has_4_byte_gpte;  in is_cr4_pae()
252 		.efer = vcpu->arch.efer,  in vcpu_to_role_regs()
266 	if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && mmu->get_guest_pgd == get_guest_cr3)  in kvm_mmu_get_guest_pgd()
269 	return mmu->get_guest_pgd(vcpu);  in kvm_mmu_get_guest_pgd()
289 	kvm_flush_remote_tlbs_gfn(kvm, gfn, sp->role.level);  in kvm_flush_remote_tlbs_sptep()
320 	gen = kvm_vcpu_memslots(vcpu)->generation;  in check_mmio_spte()
377 	sp->clear_spte_count++;  in count_spte_clear()
387 	ssptep->spte_high = sspte.spte_high;  in __set_spte()
396 	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);  in __set_spte()
406 	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);  in __update_clear_spte_fast()
414 	ssptep->spte_high = sspte.spte_high;  in __update_clear_spte_fast()
426 	orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);  in __update_clear_spte_slow()
427 	orig.spte_high = ssptep->spte_high;  in __update_clear_spte_slow()
428 	ssptep->spte_high = sspte.spte_high;  in __update_clear_spte_slow()
440  * we need to protect against in-progress updates of the spte.
443  * for the high part of the spte.  The race is fine for a present->non-present
444  * change (because the high part of the spte is ignored for non-present spte),
445  * but for a present->present change we must reread the spte.
447  * All such changes are done in two steps (present->non-present and
448  * non-present->present), hence it is enough to count the number of
449  * present->non-present updates: if it changed while reading the spte,
459 	count = sp->clear_spte_count;  in __get_spte_lockless()
462 	spte.spte_low = orig->spte_low;  in __get_spte_lockless()
465 	spte.spte_high = orig->spte_high;  in __get_spte_lockless()
468 	if (unlikely(spte.spte_low != orig->spte_low ||  in __get_spte_lockless()
469 	      count != sp->clear_spte_count))  in __get_spte_lockless()
525 	int level = sptep_to_sp(sptep)->role.level;  in mmu_spte_clear_track_bits()
536 	kvm_update_page_stats(kvm, level, -1);  in mmu_spte_clear_track_bits()
557 	return tdp_mmu_enabled && vcpu->arch.mmu->root_role.direct;  in is_tdp_mmu_active()
566 		 * Prevent page table teardown by making any free-er wait during  in walk_shadow_page_lockless_begin()
573 		 * to vcpu->mode.  in walk_shadow_page_lockless_begin()
575 		smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);  in walk_shadow_page_lockless_begin()
585 		 * Make sure the write to vcpu->mode is not reordered in front of  in walk_shadow_page_lockless_end()
589 		smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);  in walk_shadow_page_lockless_end()
599 	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,  in mmu_topup_memory_caches()
603 	if (kvm_has_mirrored_tdp(vcpu->kvm)) {  in mmu_topup_memory_caches()
604 		r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_external_spt_cache,  in mmu_topup_memory_caches()
609 	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,  in mmu_topup_memory_caches()
614 		r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadowed_info_cache,  in mmu_topup_memory_caches()
619 	return kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,  in mmu_topup_memory_caches()
625 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache);  in mmu_free_memory_caches()
626 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadow_page_cache);  in mmu_free_memory_caches()
627 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadowed_info_cache);  in mmu_free_memory_caches()
628 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_external_spt_cache);  in mmu_free_memory_caches()
629 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);  in mmu_free_memory_caches()
641 	if (sp->role.passthrough)  in kvm_mmu_page_get_gfn()
642 		return sp->gfn;  in kvm_mmu_page_get_gfn()
644 	if (sp->shadowed_translation)  in kvm_mmu_page_get_gfn()
645 		return sp->shadowed_translation[index] >> PAGE_SHIFT;  in kvm_mmu_page_get_gfn()
647 	return sp->gfn + (index << ((sp->role.level - 1) * SPTE_LEVEL_BITS));  in kvm_mmu_page_get_gfn()
658 	if (sp->shadowed_translation)  in kvm_mmu_page_get_access()
659 		return sp->shadowed_translation[index] & ACC_ALL;  in kvm_mmu_page_get_access()
662 	 * For direct MMUs (e.g. TDP or non-paging guests) or passthrough SPs,  in kvm_mmu_page_get_access()
671 	 * In both cases, sp->role.access contains the correct access bits.  in kvm_mmu_page_get_access()
673 	return sp->role.access;  in kvm_mmu_page_get_access()
679 	if (sp->shadowed_translation) {  in kvm_mmu_page_set_translation()
680 		sp->shadowed_translation[index] = (gfn << PAGE_SHIFT) | access;  in kvm_mmu_page_set_translation()
686 	          sp->role.passthrough ? "passthrough" : "direct",  in kvm_mmu_page_set_translation()
687 	          sp->gfn, kvm_mmu_page_get_access(sp, index), access);  in kvm_mmu_page_set_translation()
691 	          sp->role.passthrough ? "passthrough" : "direct",  in kvm_mmu_page_set_translation()
692 	          sp->gfn, kvm_mmu_page_get_gfn(sp, index), gfn);  in kvm_mmu_page_set_translation()
712 	idx = gfn_to_index(gfn, slot->base_gfn, level);  in lpage_info_slot()
713 	return &slot->arch.lpage_info[level - 2][idx];  in lpage_info_slot()
733 		old = linfo->disallow_lpage;  in update_gfn_disallow_lpage_count()
734 		linfo->disallow_lpage += count;  in update_gfn_disallow_lpage_count()
735 		WARN_ON_ONCE((old ^ linfo->disallow_lpage) & KVM_LPAGE_MIXED_FLAG);  in update_gfn_disallow_lpage_count()
746 	update_gfn_disallow_lpage_count(slot, gfn, -1);  in kvm_mmu_gfn_allow_lpage()
755 	kvm->arch.indirect_shadow_pages++;  in account_shadowed()
757 	 * Ensure indirect_shadow_pages is elevated prior to re-reading guest  in account_shadowed()
758 	 * child PTEs in FNAME(gpte_changed), i.e. guarantee either in-flight  in account_shadowed()
759 	 * emulated writes are visible before re-reading guest PTEs, or that  in account_shadowed()
765 	gfn = sp->gfn;  in account_shadowed()
766 	slots = kvm_memslots_for_spte_role(kvm, sp->role);  in account_shadowed()
769 	/* the non-leaf shadow pages are keeping readonly. */  in account_shadowed()
770 	if (sp->role.level > PG_LEVEL_4K)  in account_shadowed()
790 	if (!list_empty(&sp->possible_nx_huge_page_link))  in track_possible_nx_huge_page()
793 	++kvm->stat.nx_lpage_splits;  in track_possible_nx_huge_page()
794 	++kvm->arch.possible_nx_huge_pages[mmu_type].nr_pages;  in track_possible_nx_huge_page()
795 	list_add_tail(&sp->possible_nx_huge_page_link,  in track_possible_nx_huge_page()
796 		      &kvm->arch.possible_nx_huge_pages[mmu_type].pages);  in track_possible_nx_huge_page()
802 	sp->nx_huge_page_disallowed = true;  in account_nx_huge_page()
814 	kvm->arch.indirect_shadow_pages--;  in unaccount_shadowed()
815 	gfn = sp->gfn;  in unaccount_shadowed()
816 	slots = kvm_memslots_for_spte_role(kvm, sp->role);  in unaccount_shadowed()
818 	if (sp->role.level > PG_LEVEL_4K)  in unaccount_shadowed()
827 	if (list_empty(&sp->possible_nx_huge_page_link))  in untrack_possible_nx_huge_page()
830 	--kvm->stat.nx_lpage_splits;  in untrack_possible_nx_huge_page()
831 	--kvm->arch.possible_nx_huge_pages[mmu_type].nr_pages;  in untrack_possible_nx_huge_page()
832 	list_del_init(&sp->possible_nx_huge_page_link);  in untrack_possible_nx_huge_page()
837 	sp->nx_huge_page_disallowed = false;  in unaccount_nx_huge_page()
849 	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)  in gfn_to_memslot_dirty_bitmap()
860  * If the bit zero of rmap_head->val is clear, then it points to the only spte
861  * in this rmap chain. Otherwise, (rmap_head->val & ~3) points to a struct
891 	 * Elide the lock if the rmap is empty, as lockless walkers (read-only  in __kvm_rmap_lock()
892 	 * mode) don't need to (and can't) walk an empty rmap, nor can they add  in __kvm_rmap_lock()
896 	old_val = atomic_long_read(&rmap_head->val);  in __kvm_rmap_lock()
908 			old_val = atomic_long_read(&rmap_head->val);  in __kvm_rmap_lock()
929 	} while (!atomic_long_try_cmpxchg_acquire(&rmap_head->val, &old_val, new_val));  in __kvm_rmap_lock()
933 	 * impossible for the return value to be 0 (see above), i.e. the read-  in __kvm_rmap_lock()
942 	lockdep_assert_held_write(&kvm->mmu_lock);  in kvm_rmap_lock()
957 	atomic_long_set_release(&rmap_head->val, val);  in __kvm_rmap_unlock()
964 	lockdep_assert_held_write(&kvm->mmu_lock);  in kvm_rmap_unlock()
971 	return atomic_long_read(&rmap_head->val) & ~KVM_RMAP_LOCKED;  in kvm_rmap_get()
975  * If mmu_lock isn't held, rmaps can only be locked in read-only mode.  The
1020 		desc->sptes[0] = (u64 *)old_val;  in pte_list_add()
1021 		desc->sptes[1] = spte;  in pte_list_add()
1022 		desc->spte_count = 2;  in pte_list_add()
1023 		desc->tail_count = 0;  in pte_list_add()
1028 		count = desc->tail_count + desc->spte_count;  in pte_list_add()
1034 		if (desc->spte_count == PTE_LIST_EXT) {  in pte_list_add()
1036 			desc->more = (struct pte_list_desc *)(old_val & ~KVM_RMAP_MANY);  in pte_list_add()
1037 			desc->spte_count = 0;  in pte_list_add()
1038 			desc->tail_count = count;  in pte_list_add()
1043 		desc->sptes[desc->spte_count++] = spte;  in pte_list_add()
1055 	int j = head_desc->spte_count - 1;  in pte_list_desc_remove_entry()
1065 	 * Replace the to-be-freed SPTE with the last valid entry from the head  in pte_list_desc_remove_entry()
1069 	desc->sptes[i] = head_desc->sptes[j];  in pte_list_desc_remove_entry()
1070 	head_desc->sptes[j] = NULL;  in pte_list_desc_remove_entry()
1071 	head_desc->spte_count--;  in pte_list_desc_remove_entry()
1072 	if (head_desc->spte_count)  in pte_list_desc_remove_entry()
1080 	if (!head_desc->more)  in pte_list_desc_remove_entry()
1083 		*rmap_val = (unsigned long)head_desc->more | KVM_RMAP_MANY;  in pte_list_desc_remove_entry()
1106 			for (i = 0; i < desc->spte_count; ++i) {  in pte_list_remove()
1107 				if (desc->sptes[i] == spte) {  in pte_list_remove()
1113 			desc = desc->more;  in pte_list_remove()
1150 		for (i = 0; i < desc->spte_count; i++)  in kvm_zap_all_rmap_sptes()
1151 			mmu_spte_clear_track_bits(kvm, desc->sptes[i]);  in kvm_zap_all_rmap_sptes()
1152 		next = desc->more;  in kvm_zap_all_rmap_sptes()
1172 	return desc->tail_count + desc->spte_count;  in pte_list_count()
1180 	idx = gfn_to_index(gfn, slot->base_gfn, level);  in gfn_to_rmap()
1181 	return &slot->arch.rmap[level - PG_LEVEL_4K][idx];  in gfn_to_rmap()
1198 	 * information in sp->role.  in rmap_remove()
1200 	slots = kvm_memslots_for_spte_role(kvm, sp->role);  in rmap_remove()
1203 	rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);  in rmap_remove()
1235 		iter->desc = NULL;  in rmap_get_first()
1239 	iter->desc = (struct pte_list_desc *)(rmap_val & ~KVM_RMAP_MANY);  in rmap_get_first()
1240 	iter->pos = 0;  in rmap_get_first()
1241 	return iter->desc->sptes[iter->pos];  in rmap_get_first()
1251 	if (iter->desc) {  in rmap_get_next()
1252 		if (iter->pos < PTE_LIST_EXT - 1) {  in rmap_get_next()
1253 			++iter->pos;  in rmap_get_next()
1254 			if (iter->desc->sptes[iter->pos])  in rmap_get_next()
1255 				return iter->desc->sptes[iter->pos];  in rmap_get_next()
1258 		iter->desc = iter->desc->more;  in rmap_get_next()
1260 		if (iter->desc) {  in rmap_get_next()
1261 			iter->pos = 0;  in rmap_get_next()
1262 			/* desc->sptes[0] cannot be NULL */  in rmap_get_next()
1263 			return iter->desc->sptes[iter->pos];  in rmap_get_next()
1295 	WARN_ON_ONCE(sp->role.level == PG_LEVEL_4K);  in drop_large_spte()
1304  * Write-protect on the specified @sptep, @pt_protect indicates whether
1305  * spte write-protection is caused by protecting shadow page table.
1309  * - for dirty logging, the spte can be set to writable at anytime if
1311  * - for spte protection, the spte can be writable only after unsync-ing
1355  *	- D bit on ad-enabled SPTEs, and
1356  *	- W bit on ad-disabled SPTEs.
1385 				slot->base_gfn + gfn_offset, mask, true);  in kvm_mmu_write_protect_pt_masked()
1391 		rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),  in kvm_mmu_write_protect_pt_masked()
1396 		mask &= mask - 1;  in kvm_mmu_write_protect_pt_masked()
1408 				slot->base_gfn + gfn_offset, mask, false);  in kvm_mmu_clear_dirty_pt_masked()
1414 		rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),  in kvm_mmu_clear_dirty_pt_masked()
1419 		mask &= mask - 1;  in kvm_mmu_clear_dirty_pt_masked()
1428 	 * If the slot was assumed to be "initially all dirty", write-protect  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1439 		gfn_t start = slot->base_gfn + gfn_offset + __ffs(mask);  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1440 		gfn_t end = slot->base_gfn + gfn_offset + __fls(mask);  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1456 	 * mask.  If PML is enabled and the GFN doesn't need to be write-  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1464 	if (kvm->arch.cpu_dirty_log_size)  in kvm_arch_mmu_enable_log_dirty_pt_masked()
1472 	return kvm->arch.cpu_dirty_log_size;  in kvm_cpu_dirty_log_size()
1502 	return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn, PG_LEVEL_4K);  in kvm_vcpu_write_protect_gfn()
1531 	iterator->level = level;  in rmap_walk_init_level()
1532 	iterator->gfn = iterator->start_gfn;  in rmap_walk_init_level()
1533 	iterator->rmap = gfn_to_rmap(iterator->gfn, level, iterator->slot);  in rmap_walk_init_level()
1534 	iterator->end_rmap = gfn_to_rmap(iterator->end_gfn, level, iterator->slot);  in rmap_walk_init_level()
1542 	iterator->slot = slot;  in slot_rmap_walk_init()
1543 	iterator->start_level = start_level;  in slot_rmap_walk_init()
1544 	iterator->end_level = end_level;  in slot_rmap_walk_init()
1545 	iterator->start_gfn = start_gfn;  in slot_rmap_walk_init()
1546 	iterator->end_gfn = end_gfn;  in slot_rmap_walk_init()
1548 	rmap_walk_init_level(iterator, iterator->start_level);  in slot_rmap_walk_init()
1553 	return !!iterator->rmap;  in slot_rmap_walk_okay()
1558 	while (++iterator->rmap <= iterator->end_rmap) {  in slot_rmap_walk_next()
1559 		iterator->gfn += KVM_PAGES_PER_HPAGE(iterator->level);  in slot_rmap_walk_next()
1561 		if (atomic_long_read(&iterator->rmap->val))  in slot_rmap_walk_next()
1565 	if (++iterator->level > iterator->end_level) {  in slot_rmap_walk_next()
1566 		iterator->rmap = NULL;  in slot_rmap_walk_next()
1570 	rmap_walk_init_level(iterator, iterator->level);  in slot_rmap_walk_next()
1595 	lockdep_assert_held_write(&kvm->mmu_lock);  in __walk_slot_rmaps()
1605 		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {  in __walk_slot_rmaps()
1608 							    iterator.gfn - start_gfn + 1);  in __walk_slot_rmaps()
1611 			cond_resched_rwlock_write(&kvm->mmu_lock);  in __walk_slot_rmaps()
1625 				 slot->base_gfn, slot->base_gfn + slot->npages - 1,  in walk_slot_rmaps()
1644 				 start, end - 1, can_yield, true, flush);  in __kvm_rmap_zap_gfn_range()
1659 	lockdep_assert_once(kvm->mmu_invalidate_in_progress ||  in kvm_unmap_gfn_range()
1660 			    lockdep_is_held(&kvm->slots_lock));  in kvm_unmap_gfn_range()
1663 		flush = __kvm_rmap_zap_gfn_range(kvm, range->slot,  in kvm_unmap_gfn_range()
1664 						 range->start, range->end,  in kvm_unmap_gfn_range()
1665 						 range->may_block, flush);  in kvm_unmap_gfn_range()
1671 	    range->slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT)  in kvm_unmap_gfn_range()
1690 	kvm_update_page_stats(kvm, sp->role.level, 1);  in __rmap_add()
1692 	rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);  in __rmap_add()
1695 	if (rmap_count > kvm->stat.max_mmu_rmap_size)  in __rmap_add()
1696 		kvm->stat.max_mmu_rmap_size = rmap_count;  in __rmap_add()
1699 		kvm_flush_remote_tlbs_gfn(kvm, gfn, sp->role.level);  in __rmap_add()
1706 	struct kvm_mmu_memory_cache *cache = &vcpu->arch.mmu_pte_list_desc_cache;  in rmap_add()
1708 	__rmap_add(vcpu->kvm, cache, slot, spte, gfn, access);  in rmap_add()
1725 		for (gfn = range->start; gfn < range->end;  in kvm_rmap_age_gfn_range()
1727 			rmap_head = gfn_to_rmap(gfn, level, range->slot);  in kvm_rmap_age_gfn_range()
1740 					clear_bit((ffs(shadow_accessed_mask) - 1),  in kvm_rmap_age_gfn_range()
1761 	return !tdp_mmu_enabled || READ_ONCE(kvm->arch.indirect_shadow_pages);  in kvm_may_have_shadow_mmu_sptes()
1799 		if (KVM_MMU_WARN_ON(is_shadow_present_pte(sp->spt[i])))  in kvm_mmu_check_sptes_at_free()
1800 			pr_err_ratelimited("SPTE %llx (@ %p) for gfn %llx shadow-present at free",  in kvm_mmu_check_sptes_at_free()
1801 					   sp->spt[i], &sp->spt[i],  in kvm_mmu_check_sptes_at_free()
1809 	kvm->arch.n_used_mmu_pages++;  in kvm_account_mmu_page()
1810 	kvm_account_pgtable_pages((void *)sp->spt, +1);  in kvm_account_mmu_page()
1815 	kvm->arch.n_used_mmu_pages--;  in kvm_unaccount_mmu_page()
1816 	kvm_account_pgtable_pages((void *)sp->spt, -1);  in kvm_unaccount_mmu_page()
1823 	hlist_del(&sp->hash_link);  in kvm_mmu_free_shadow_page()
1824 	list_del(&sp->link);  in kvm_mmu_free_shadow_page()
1825 	free_page((unsigned long)sp->spt);  in kvm_mmu_free_shadow_page()
1826 	free_page((unsigned long)sp->shadowed_translation);  in kvm_mmu_free_shadow_page()
1842 	pte_list_add(kvm, cache, parent_pte, &sp->parent_ptes);  in mmu_page_add_parent_pte()
1848 	pte_list_remove(kvm, parent_pte, &sp->parent_ptes);  in mmu_page_remove_parent_pte()
1864 	for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {  in kvm_mmu_mark_parents_unsync()
1874 	if (__test_and_set_bit(spte_index(spte), sp->unsync_child_bitmap))  in mark_unsync()
1876 	if (sp->unsync_children++)  in mark_unsync()
1896 	if (sp->unsync)  in mmu_pages_add()
1897 		for (i=0; i < pvec->nr; i++)  in mmu_pages_add()
1898 			if (pvec->page[i].sp == sp)  in mmu_pages_add()
1901 	pvec->page[pvec->nr].sp = sp;  in mmu_pages_add()
1902 	pvec->page[pvec->nr].idx = idx;  in mmu_pages_add()
1903 	pvec->nr++;  in mmu_pages_add()
1904 	return (pvec->nr == KVM_PAGE_ARRAY_NR);  in mmu_pages_add()
1909 	--sp->unsync_children;  in clear_unsync_child_bit()
1910 	WARN_ON_ONCE((int)sp->unsync_children < 0);  in clear_unsync_child_bit()
1911 	__clear_bit(idx, sp->unsync_child_bitmap);  in clear_unsync_child_bit()
1919 	for_each_set_bit(i, sp->unsync_child_bitmap, 512) {  in __mmu_unsync_walk()
1921 		u64 ent = sp->spt[i];  in __mmu_unsync_walk()
1930 		if (child->unsync_children) {  in __mmu_unsync_walk()
1932 				return -ENOSPC;  in __mmu_unsync_walk()
1942 		} else if (child->unsync) {  in __mmu_unsync_walk()
1945 				return -ENOSPC;  in __mmu_unsync_walk()
1953 #define INVALID_INDEX (-1)
1958 	pvec->nr = 0;  in mmu_unsync_walk()
1959 	if (!sp->unsync_children)  in mmu_unsync_walk()
1968 	WARN_ON_ONCE(!sp->unsync);  in kvm_unlink_unsync_page()
1970 	sp->unsync = 0;  in kvm_unlink_unsync_page()
1971 	--kvm->stat.mmu_unsync;  in kvm_unlink_unsync_page()
1981 	if (sp->role.direct)  in sp_has_gptes()
1984 	if (sp->role.passthrough)  in sp_has_gptes()
2002 	struct hlist_head *page_hash = smp_load_acquire(&kvm->arch.mmu_page_hash);  in kvm_get_mmu_page_hash()
2004 	lockdep_assert_held(&kvm->mmu_lock);  in kvm_get_mmu_page_hash()
2019 		if ((_sp)->gfn != (_gfn) || !sp_has_gptes(_sp)) {} else
2023 	union kvm_mmu_page_role root_role = vcpu->arch.mmu->root_role;  in kvm_sync_page_check()
2029 	 *  - level: not part of the overall MMU role and will never match as the MMU's  in kvm_sync_page_check()
2031 	 *  - access: updated based on the new guest PTE  in kvm_sync_page_check()
2032 	 *  - quadrant: not part of the overall MMU role (similar to level)  in kvm_sync_page_check()
2044 	 * differs then the memslot lookup (SMM vs. non-SMM) will be bogus, the  in kvm_sync_page_check()
2047 	if (WARN_ON_ONCE(sp->role.direct || !vcpu->arch.mmu->sync_spte ||  in kvm_sync_page_check()
2048 			 (sp->role.word ^ root_role.word) & ~sync_role_ign.word))  in kvm_sync_page_check()
2056 	/* sp->spt[i] has initial value of shadow page table allocation */  in kvm_sync_spte()
2057 	if (sp->spt[i] == SHADOW_NONPRESENT_VALUE)  in kvm_sync_spte()
2060 	return vcpu->arch.mmu->sync_spte(vcpu, sp, i);  in kvm_sync_spte()
2069 		return -1;  in __kvm_sync_page()
2074 		if (ret < -1)  in __kvm_sync_page()
2075 			return -1;  in __kvm_sync_page()
2097 		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);  in kvm_sync_page()
2117 	if (sp->role.invalid)  in is_obsolete_sp()
2122 	       unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);  in is_obsolete_sp()
2141 	for (n = i+1; n < pvec->nr; n++) {  in mmu_pages_next()
2142 		struct kvm_mmu_page *sp = pvec->page[n].sp;  in mmu_pages_next()
2143 		unsigned idx = pvec->page[n].idx;  in mmu_pages_next()
2144 		int level = sp->role.level;  in mmu_pages_next()
2146 		parents->idx[level-1] = idx;  in mmu_pages_next()
2150 		parents->parent[level-2] = sp;  in mmu_pages_next()
2162 	if (pvec->nr == 0)  in mmu_pages_first()
2165 	WARN_ON_ONCE(pvec->page[0].idx != INVALID_INDEX);  in mmu_pages_first()
2167 	sp = pvec->page[0].sp;  in mmu_pages_first()
2168 	level = sp->role.level;  in mmu_pages_first()
2171 	parents->parent[level-2] = sp;  in mmu_pages_first()
2176 	parents->parent[level-1] = NULL;  in mmu_pages_first()
2186 		unsigned int idx = parents->idx[level];  in mmu_pages_clear_parents()
2187 		sp = parents->parent[level];  in mmu_pages_clear_parents()
2194 	} while (!sp->unsync_children);  in mmu_pages_clear_parents()
2211 			protected |= kvm_vcpu_write_protect_gfn(vcpu, sp->gfn);  in mmu_sync_children()
2214 			kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, true);  in mmu_sync_children()
2219 			kvm_unlink_unsync_page(vcpu->kvm, sp);  in mmu_sync_children()
2223 		if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) {  in mmu_sync_children()
2224 			kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);  in mmu_sync_children()
2227 				return -EINTR;  in mmu_sync_children()
2230 			cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);  in mmu_sync_children()
2235 	kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);  in mmu_sync_children()
2241 	atomic_set(&sp->write_flooding_count,  0);  in __clear_sp_write_flooding_count()
2267 		if (sp->gfn != gfn) {  in kvm_mmu_find_shadow_page()
2272 		if (sp->role.word != role.word) {  in kvm_mmu_find_shadow_page()
2274 			 * If the guest is creating an upper-level page, zap  in kvm_mmu_find_shadow_page()
2280 			 * upper-level page will be write-protected.  in kvm_mmu_find_shadow_page()
2282 			if (role.level > PG_LEVEL_4K && sp->unsync)  in kvm_mmu_find_shadow_page()
2288 		/* unsync and write-flooding only apply to indirect SPs. */  in kvm_mmu_find_shadow_page()
2289 		if (sp->role.direct)  in kvm_mmu_find_shadow_page()
2292 		if (sp->unsync) {  in kvm_mmu_find_shadow_page()
2299 			 * it doesn't write-protect the page or mark it synchronized!  in kvm_mmu_find_shadow_page()
2323 	++kvm->stat.mmu_cache_miss;  in kvm_mmu_find_shadow_page()
2328 	if (collisions > kvm->stat.max_mmu_page_hash_collisions)  in kvm_mmu_find_shadow_page()
2329 		kvm->stat.max_mmu_page_hash_collisions = collisions;  in kvm_mmu_find_shadow_page()
2348 	sp = kvm_mmu_memory_cache_alloc(caches->page_header_cache);  in kvm_mmu_alloc_shadow_page()
2349 	sp->spt = kvm_mmu_memory_cache_alloc(caches->shadow_page_cache);  in kvm_mmu_alloc_shadow_page()
2351 		sp->shadowed_translation = kvm_mmu_memory_cache_alloc(caches->shadowed_info_cache);  in kvm_mmu_alloc_shadow_page()
2353 	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);  in kvm_mmu_alloc_shadow_page()
2355 	INIT_LIST_HEAD(&sp->possible_nx_huge_page_link);  in kvm_mmu_alloc_shadow_page()
2362 	sp->mmu_valid_gen = kvm->arch.mmu_valid_gen;  in kvm_mmu_alloc_shadow_page()
2363 	list_add(&sp->link, &kvm->arch.active_mmu_pages);  in kvm_mmu_alloc_shadow_page()
2366 	sp->gfn = gfn;  in kvm_mmu_alloc_shadow_page()
2367 	sp->role = role;  in kvm_mmu_alloc_shadow_page()
2368 	hlist_add_head(&sp->hash_link, sp_list);  in kvm_mmu_alloc_shadow_page()
2391 	BUG_ON(!kvm->arch.mmu_page_hash);  in __kvm_mmu_get_shadow_page()
2392 	sp_list = &kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)];  in __kvm_mmu_get_shadow_page()
2409 		.page_header_cache = &vcpu->arch.mmu_page_header_cache,  in kvm_mmu_get_shadow_page()
2410 		.shadow_page_cache = &vcpu->arch.mmu_shadow_page_cache,  in kvm_mmu_get_shadow_page()
2411 		.shadowed_info_cache = &vcpu->arch.mmu_shadowed_info_cache,  in kvm_mmu_get_shadow_page()
2414 	return __kvm_mmu_get_shadow_page(vcpu->kvm, vcpu, &caches, gfn, role);  in kvm_mmu_get_shadow_page()
2423 	role = parent_sp->role;  in kvm_mmu_child_role()
2424 	role.level--;  in kvm_mmu_child_role()
2430 	 * If the guest has 4-byte PTEs then that means it's using 32-bit,  in kvm_mmu_child_role()
2431 	 * 2-level, non-PAE paging. KVM shadows such guests with PAE paging  in kvm_mmu_child_role()
2432 	 * (i.e. 8-byte PTEs). The difference in PTE size means that KVM must  in kvm_mmu_child_role()
2445 	 * Concretely, a 4-byte PDE consumes bits 31:22, while an 8-byte PDE  in kvm_mmu_child_role()
2447 	 * PDPTEs; those 4 PAE page directories are pre-allocated and their  in kvm_mmu_child_role()
2448 	 * quadrant is assigned in mmu_alloc_root().   A 4-byte PTE consumes  in kvm_mmu_child_role()
2449 	 * bits 21:12, while an 8-byte PTE consumes bits 20:12.  To consume  in kvm_mmu_child_role()
2451 	 * quadrant, i.e. sets quadrant to '0' or '1'.  The parent 8-byte PDE  in kvm_mmu_child_role()
2470 		return ERR_PTR(-EEXIST);  in kvm_mmu_get_child_sp()
2480 	iterator->addr = addr;  in shadow_walk_init_using_root()
2481 	iterator->shadow_addr = root;  in shadow_walk_init_using_root()
2482 	iterator->level = vcpu->arch.mmu->root_role.level;  in shadow_walk_init_using_root()
2484 	if (iterator->level >= PT64_ROOT_4LEVEL &&  in shadow_walk_init_using_root()
2485 	    vcpu->arch.mmu->cpu_role.base.level < PT64_ROOT_4LEVEL &&  in shadow_walk_init_using_root()
2486 	    !vcpu->arch.mmu->root_role.direct)  in shadow_walk_init_using_root()
2487 		iterator->level = PT32E_ROOT_LEVEL;  in shadow_walk_init_using_root()
2489 	if (iterator->level == PT32E_ROOT_LEVEL) {  in shadow_walk_init_using_root()
2491 		 * prev_root is currently only used for 64-bit hosts. So only  in shadow_walk_init_using_root()
2494 		BUG_ON(root != vcpu->arch.mmu->root.hpa);  in shadow_walk_init_using_root()
2496 		iterator->shadow_addr  in shadow_walk_init_using_root()
2497 			= vcpu->arch.mmu->pae_root[(addr >> 30) & 3];  in shadow_walk_init_using_root()
2498 		iterator->shadow_addr &= SPTE_BASE_ADDR_MASK;  in shadow_walk_init_using_root()
2499 		--iterator->level;  in shadow_walk_init_using_root()
2500 		if (!iterator->shadow_addr)  in shadow_walk_init_using_root()
2501 			iterator->level = 0;  in shadow_walk_init_using_root()
2508 	shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root.hpa,  in shadow_walk_init()
2514 	if (iterator->level < PG_LEVEL_4K)  in shadow_walk_okay()
2517 	iterator->index = SPTE_INDEX(iterator->addr, iterator->level);  in shadow_walk_okay()
2518 	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;  in shadow_walk_okay()
2525 	if (!is_shadow_present_pte(spte) || is_last_spte(spte, iterator->level)) {  in __shadow_walk_next()
2526 		iterator->level = 0;  in __shadow_walk_next()
2530 	iterator->shadow_addr = spte & SPTE_BASE_ADDR_MASK;  in __shadow_walk_next()
2531 	--iterator->level;  in __shadow_walk_next()
2536 	__shadow_walk_next(iterator, *iterator->sptep);  in shadow_walk_next()
2555 	spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));  in __link_shadow_page()
2562 	 * The non-direct sub-pagetable must be updated before linking.  For  in __link_shadow_page()
2564 	 * kvm_mmu_find_shadow_page() without write-protecting the gfn,  in __link_shadow_page()
2565 	 * so sp->unsync can be true or false.  For higher level non-direct  in __link_shadow_page()
2567 	 * FNAME(fetch)(), so sp->unsync_children can only be false.  in __link_shadow_page()
2570 	if (WARN_ON_ONCE(sp->unsync_children) || sp->unsync)  in __link_shadow_page()
2577 	__link_shadow_page(vcpu->kvm, &vcpu->arch.mmu_pte_list_desc_cache, sptep, sp, true);  in link_shadow_page()
2589 		 * sp's access: allow writable in the read-only sp,  in validate_direct_spte()
2594 		if (child->role.access == direct_access)  in validate_direct_spte()
2597 		drop_parent_pte(vcpu->kvm, child, sptep);  in validate_direct_spte()
2598 		kvm_flush_remote_tlbs_sptep(vcpu->kvm, sptep);  in validate_direct_spte()
2602 /* Returns the number of zapped non-leaf child shadow pages. */
2611 		if (is_last_spte(pte, sp->role.level)) {  in mmu_page_zap_pte()
2623 			    child->role.guest_mode &&  in mmu_page_zap_pte()
2624 			    !atomic_long_read(&child->parent_ptes.val))  in mmu_page_zap_pte()
2642 		zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list);  in kvm_mmu_page_unlink_children()
2652 	while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))  in kvm_mmu_unlink_parents()
2664 	if (parent->role.level == PG_LEVEL_4K)  in mmu_zap_unsync_children()
2687 	lockdep_assert_held_write(&kvm->mmu_lock);  in __kvm_mmu_prepare_zap_page()
2689 	++kvm->stat.mmu_shadow_zapped;  in __kvm_mmu_prepare_zap_page()
2697 	if (!sp->role.invalid && sp_has_gptes(sp))  in __kvm_mmu_prepare_zap_page()
2700 	if (sp->unsync)  in __kvm_mmu_prepare_zap_page()
2702 	if (!sp->root_count) {  in __kvm_mmu_prepare_zap_page()
2709 		 * !sp->root_count.  in __kvm_mmu_prepare_zap_page()
2711 		if (sp->role.invalid)  in __kvm_mmu_prepare_zap_page()
2712 			list_add(&sp->link, invalid_list);  in __kvm_mmu_prepare_zap_page()
2714 			list_move(&sp->link, invalid_list);  in __kvm_mmu_prepare_zap_page()
2721 		list_del(&sp->link);  in __kvm_mmu_prepare_zap_page()
2731 	if (sp->nx_huge_page_disallowed)  in __kvm_mmu_prepare_zap_page()
2734 	sp->role.invalid = 1;  in __kvm_mmu_prepare_zap_page()
2764 	 * the page tables and see changes to vcpu->mode here. The barrier  in kvm_mmu_commit_zap_page()
2769 	 * guest mode and/or lockless shadow page table walks.  in kvm_mmu_commit_zap_page()
2774 		WARN_ON_ONCE(!sp->role.invalid || sp->root_count);  in kvm_mmu_commit_zap_page()
2788 	if (list_empty(&kvm->arch.active_mmu_pages))  in kvm_mmu_zap_oldest_mmu_pages()
2792 	list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {  in kvm_mmu_zap_oldest_mmu_pages()
2797 		if (sp->root_count)  in kvm_mmu_zap_oldest_mmu_pages()
2812 	kvm->stat.mmu_recycled += total_zapped;  in kvm_mmu_zap_oldest_mmu_pages()
2818 	if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)  in kvm_mmu_available_pages()
2819 		return kvm->arch.n_max_mmu_pages -  in kvm_mmu_available_pages()
2820 			kvm->arch.n_used_mmu_pages;  in kvm_mmu_available_pages()
2827 	unsigned long avail = kvm_mmu_available_pages(vcpu->kvm);  in make_mmu_pages_available()
2832 	kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail);  in make_mmu_pages_available()
2837 	 * four pages, e.g. for PAE roots or for 5-level paging.  Temporarily  in make_mmu_pages_available()
2843 	if (!kvm_mmu_available_pages(vcpu->kvm))  in make_mmu_pages_available()
2844 		return -ENOSPC;  in make_mmu_pages_available()
2854 	write_lock(&kvm->mmu_lock);  in kvm_mmu_change_mmu_pages()
2856 	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {  in kvm_mmu_change_mmu_pages()
2857 		kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -  in kvm_mmu_change_mmu_pages()
2860 		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;  in kvm_mmu_change_mmu_pages()
2863 	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;  in kvm_mmu_change_mmu_pages()
2865 	write_unlock(&kvm->mmu_lock);  in kvm_mmu_change_mmu_pages()
2871 	struct kvm *kvm = vcpu->kvm;  in __kvm_mmu_unprotect_gfn_and_retry()
2878 	 * Bail early if there aren't any write-protected shadow pages to avoid  in __kvm_mmu_unprotect_gfn_and_retry()
2879 	 * unnecessarily taking mmu_lock lock, e.g. if the gfn is write-tracked  in __kvm_mmu_unprotect_gfn_and_retry()
2885 	if (!READ_ONCE(kvm->arch.indirect_shadow_pages))  in __kvm_mmu_unprotect_gfn_and_retry()
2888 	if (!vcpu->arch.mmu->root_role.direct) {  in __kvm_mmu_unprotect_gfn_and_retry()
2894 	write_lock(&kvm->mmu_lock);  in __kvm_mmu_unprotect_gfn_and_retry()
2904 	write_unlock(&kvm->mmu_lock);  in __kvm_mmu_unprotect_gfn_and_retry()
2908 		vcpu->arch.last_retry_eip = kvm_rip_read(vcpu);  in __kvm_mmu_unprotect_gfn_and_retry()
2909 		vcpu->arch.last_retry_addr = cr2_or_gpa;  in __kvm_mmu_unprotect_gfn_and_retry()
2917 	++kvm->stat.mmu_unsync;  in kvm_unsync_page()
2918 	sp->unsync = 1;  in kvm_unsync_page()
2926  * were marked unsync (or if there is no shadow page), -EPERM if the SPTE must
2927  * be write-protected.
2936 	 * Force write-protection if the page is being tracked.  Note, the page  in mmu_try_to_unsync_pages()
2937 	 * track machinery is used to write-protect upper-level shadow pages,  in mmu_try_to_unsync_pages()
2941 		return -EPERM;  in mmu_try_to_unsync_pages()
2944 	 * The page is not write-tracked, mark existing shadow pages unsync  in mmu_try_to_unsync_pages()
2951 			return -EPERM;  in mmu_try_to_unsync_pages()
2953 		if (sp->unsync)  in mmu_try_to_unsync_pages()
2957 			return -EEXIST;  in mmu_try_to_unsync_pages()
2968 			spin_lock(&kvm->arch.mmu_unsync_pages_lock);  in mmu_try_to_unsync_pages()
2974 			 * possible as clearing sp->unsync _must_ hold mmu_lock  in mmu_try_to_unsync_pages()
2975 			 * for write, i.e. unsync cannot transition from 1->0  in mmu_try_to_unsync_pages()
2978 			if (READ_ONCE(sp->unsync))  in mmu_try_to_unsync_pages()
2982 		WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K);  in mmu_try_to_unsync_pages()
2986 		spin_unlock(&kvm->arch.mmu_unsync_pages_lock);  in mmu_try_to_unsync_pages()
2989 	 * We need to ensure that the marking of unsync pages is visible  in mmu_try_to_unsync_pages()
2993 	 * before the page had been marked as unsync-ed, something like the  in mmu_try_to_unsync_pages()
2997 	 * ---------------------------------------------------------------------  in mmu_try_to_unsync_pages()
3010 	 *                      2.3 Walking of unsync pages sees sp->unsync is  in mmu_try_to_unsync_pages()
3019 	 *     (sp->unsync = true)  in mmu_try_to_unsync_pages()
3035 	int level = sp->role.level;  in mmu_set_spte()
3043 	bool host_writable = !fault || fault->map_writable;  in mmu_set_spte()
3044 	bool prefetch = !fault || fault->prefetch;  in mmu_set_spte()
3045 	bool write_fault = fault && fault->write;  in mmu_set_spte()
3048 		vcpu->stat.pf_mmio_spte_created++;  in mmu_set_spte()
3067 			drop_parent_pte(vcpu->kvm, child, sptep);  in mmu_set_spte()
3070 			drop_spte(vcpu->kvm, sptep);  in mmu_set_spte()
3090 		kvm_flush_remote_tlbs_gfn(vcpu->kvm, gfn, level);  in mmu_set_spte()
3145 	unsigned int access = sp->role.access;  in direct_pte_prefetch_many()
3147 	return kvm_mmu_prefetch_sptes(vcpu, gfn, start, end - start, access);  in direct_pte_prefetch_many()
3156 	WARN_ON_ONCE(!sp->role.direct);  in __direct_pte_prefetch()
3158 	i = spte_index(sptep) & ~(PTE_PREFETCH_NUM - 1);  in __direct_pte_prefetch()
3159 	spte = sp->spt + i;  in __direct_pte_prefetch()
3190 	if (sp->role.level > PG_LEVEL_4K)  in direct_pte_prefetch()
3197 	if (unlikely(vcpu->kvm->mmu_invalidate_in_progress))  in direct_pte_prefetch()
3211  * - Check mmu_invalidate_retry_gfn() after grabbing the mapping level, before
3215  * - Hold mmu_lock AND ensure there is no in-progress MMU notifier invalidation
3219  * - Do not use the result to install new mappings, e.g. use the host mapping
3240 	 * Note, using the already-retrieved memslot and __gfn_to_hva_memslot()  in host_pfn_mapping_level()
3243 	 * read-only memslots due to gfn_to_hva() assuming writes.  Earlier  in host_pfn_mapping_level()
3245 	 * read-only memslot.  in host_pfn_mapping_level()
3257 	 * Read each entry once.  As above, a non-leaf entry can be promoted to  in host_pfn_mapping_level()
3258 	 * a huge page _during_ this walk.  Re-reading the entry could send the  in host_pfn_mapping_level()
3263 	pgd = READ_ONCE(*pgd_offset(kvm->mm, hva));  in host_pfn_mapping_level()
3318 		pfn = fault->pfn;  in kvm_gmem_max_mapping_level()
3319 		max_level = fault->max_level;  in kvm_gmem_max_mapping_level()
3331 	 * CoCo may influence the max mapping level, e.g. due to RMP or S-EPT  in kvm_gmem_max_mapping_level()
3349 	lockdep_assert_held(&kvm->mmu_lock);  in kvm_mmu_max_mapping_level()
3352 		max_level = fault->max_level;  in kvm_mmu_max_mapping_level()
3353 		is_private = fault->is_private;  in kvm_mmu_max_mapping_level()
3360 	for ( ; max_level > PG_LEVEL_4K; max_level--) {  in kvm_mmu_max_mapping_level()
3362 		if (!linfo->disallow_lpage)  in kvm_mmu_max_mapping_level()
3379 	struct kvm_memory_slot *slot = fault->slot;  in kvm_mmu_hugepage_adjust()
3382 	fault->huge_page_disallowed = fault->exec && fault->nx_huge_page_workaround_enabled;  in kvm_mmu_hugepage_adjust()
3384 	if (unlikely(fault->max_level == PG_LEVEL_4K))  in kvm_mmu_hugepage_adjust()
3387 	if (is_error_noslot_pfn(fault->pfn))  in kvm_mmu_hugepage_adjust()
3397 	fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, fault,  in kvm_mmu_hugepage_adjust()
3398 						     fault->slot, fault->gfn);  in kvm_mmu_hugepage_adjust()
3399 	if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)  in kvm_mmu_hugepage_adjust()
3406 	fault->goal_level = fault->req_level;  in kvm_mmu_hugepage_adjust()
3407 	mask = KVM_PAGES_PER_HPAGE(fault->goal_level) - 1;  in kvm_mmu_hugepage_adjust()
3408 	VM_BUG_ON((fault->gfn & mask) != (fault->pfn & mask));  in kvm_mmu_hugepage_adjust()
3409 	fault->pfn &= ~mask;  in kvm_mmu_hugepage_adjust()
3415 	    cur_level == fault->goal_level &&  in disallowed_hugepage_adjust()
3418 	    spte_to_child_sp(spte)->nx_huge_page_disallowed) {  in disallowed_hugepage_adjust()
3426 		u64 page_mask = KVM_PAGES_PER_HPAGE(cur_level) -  in disallowed_hugepage_adjust()
3427 				KVM_PAGES_PER_HPAGE(cur_level - 1);  in disallowed_hugepage_adjust()
3428 		fault->pfn |= fault->gfn & page_mask;  in disallowed_hugepage_adjust()
3429 		fault->goal_level--;  in disallowed_hugepage_adjust()
3438 	gfn_t base_gfn = fault->gfn;  in direct_map()
3443 	for_each_shadow_entry(vcpu, fault->addr, it) {  in direct_map()
3448 		if (fault->nx_huge_page_workaround_enabled)  in direct_map()
3451 		base_gfn = gfn_round_for_level(fault->gfn, it.level);  in direct_map()
3452 		if (it.level == fault->goal_level)  in direct_map()
3456 		if (sp == ERR_PTR(-EEXIST))  in direct_map()
3460 		if (fault->huge_page_disallowed)  in direct_map()
3461 			account_nx_huge_page(vcpu->kvm, sp,  in direct_map()
3462 					     fault->req_level >= it.level);  in direct_map()
3465 	if (WARN_ON_ONCE(it.level != fault->goal_level))  in direct_map()
3466 		return -EFAULT;  in direct_map()
3468 	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL,  in direct_map()
3469 			   base_gfn, fault->pfn, fault);  in direct_map()
3486 	if (is_sigpending_pfn(fault->pfn)) {  in kvm_handle_error_pfn()
3488 		return -EINTR;  in kvm_handle_error_pfn()
3496 	if (fault->pfn == KVM_PFN_ERR_RO_FAULT)  in kvm_handle_error_pfn()
3499 	if (fault->pfn == KVM_PFN_ERR_HWPOISON) {  in kvm_handle_error_pfn()
3500 		kvm_send_hwpoison_signal(fault->slot, fault->gfn);  in kvm_handle_error_pfn()
3504 	return -EFAULT;  in kvm_handle_error_pfn()
3511 	gva_t gva = fault->is_tdp ? 0 : fault->addr;  in kvm_handle_noslot_fault()
3513 	if (fault->is_private) {  in kvm_handle_noslot_fault()
3515 		return -EFAULT;  in kvm_handle_noslot_fault()
3518 	vcpu_cache_mmio_info(vcpu, gva, fault->gfn,  in kvm_handle_noslot_fault()
3521 	fault->slot = NULL;  in kvm_handle_noslot_fault()
3522 	fault->pfn = KVM_PFN_NOSLOT;  in kvm_handle_noslot_fault()
3523 	fault->map_writable = false;  in kvm_handle_noslot_fault()
3540 	if (unlikely(fault->gfn > kvm_mmu_max_gfn()))  in kvm_handle_noslot_fault()
3554 	if (fault->rsvd)  in page_fault_can_be_fast()
3558 	 * For hardware-protected VMs, certain conditions like attempting to  in page_fault_can_be_fast()
3562 	 * result of a write-protected access, and treat it as a spurious case  in page_fault_can_be_fast()
3573 	if (kvm->arch.has_private_mem &&  in page_fault_can_be_fast()
3574 	    fault->is_private != kvm_mem_is_private(kvm, fault->gfn))  in page_fault_can_be_fast()
3588 	 *    the fault was caused by a write-protection violation.  If the  in page_fault_can_be_fast()
3589 	 *    SPTE is MMU-writable (determined later), the fault can be fixed  in page_fault_can_be_fast()
3592 	if (!fault->present)  in page_fault_can_be_fast()
3599 	return fault->write;  in page_fault_can_be_fast()
3618 	 * so non-PML cases won't be impacted.  in fast_pf_fix_direct_spte()
3626 		mark_page_dirty_in_slot(vcpu->kvm, fault->slot, fault->gfn);  in fast_pf_fix_direct_spte()
3633  * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
3637  *  - Must be called between walk_shadow_page_lockless_{begin,end}.
3638  *  - The returned sptep must not be used after walk_shadow_page_lockless_end.
3665 	if (!page_fault_can_be_fast(vcpu->kvm, fault))  in fast_page_fault()
3674 			sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, fault->gfn, &spte);  in fast_page_fault()
3676 			sptep = fast_pf_get_last_sptep(vcpu, fault->addr, &spte);  in fast_page_fault()
3690 		if (!is_last_spte(spte, sp->role.level))  in fast_page_fault()
3713 		 * uses A/D bits for non-nested MMUs.  Thus, if A/D bits are  in fast_page_fault()
3714 		 * enabled, the SPTE can't be an access-tracked SPTE.  in fast_page_fault()
3721 		 * To keep things simple, only SPTEs that are MMU-writable can  in fast_page_fault()
3723 		 * that were write-protected for dirty-logging or access  in fast_page_fault()
3728 		 * shadow-present, i.e. except for access tracking restoration  in fast_page_fault()
3731 		if (fault->write && is_mmu_writable_spte(spte)) {  in fast_page_fault()
3735 			 * Do not fix write-permission on the large spte when  in fast_page_fault()
3737 			 * first page into the dirty-bitmap in  in fast_page_fault()
3744 			if (sp->role.level > PG_LEVEL_4K &&  in fast_page_fault()
3745 			    kvm_slot_dirty_track_enabled(fault->slot))  in fast_page_fault()
3775 		vcpu->stat.pf_fast++;  in fast_page_fault()
3793 		lockdep_assert_held_read(&kvm->mmu_lock);  in mmu_free_root_page()
3796 		lockdep_assert_held_write(&kvm->mmu_lock);  in mmu_free_root_page()
3797 		if (!--sp->root_count && sp->role.invalid)  in mmu_free_root_page()
3808 	bool is_tdp_mmu = tdp_mmu_enabled && mmu->root_role.direct;  in kvm_mmu_free_roots()
3819 		&& VALID_PAGE(mmu->root.hpa);  in kvm_mmu_free_roots()
3824 			    VALID_PAGE(mmu->prev_roots[i].hpa))  in kvm_mmu_free_roots()
3832 		read_lock(&kvm->mmu_lock);  in kvm_mmu_free_roots()
3834 		write_lock(&kvm->mmu_lock);  in kvm_mmu_free_roots()
3838 			mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa,  in kvm_mmu_free_roots()
3842 		if (kvm_mmu_is_dummy_root(mmu->root.hpa)) {  in kvm_mmu_free_roots()
3844 		} else if (root_to_sp(mmu->root.hpa)) {  in kvm_mmu_free_roots()
3845 			mmu_free_root_page(kvm, &mmu->root.hpa, &invalid_list);  in kvm_mmu_free_roots()
3846 		} else if (mmu->pae_root) {  in kvm_mmu_free_roots()
3848 				if (!IS_VALID_PAE_ROOT(mmu->pae_root[i]))  in kvm_mmu_free_roots()
3851 				mmu_free_root_page(kvm, &mmu->pae_root[i],  in kvm_mmu_free_roots()
3853 				mmu->pae_root[i] = INVALID_PAE_ROOT;  in kvm_mmu_free_roots()
3856 		mmu->root.hpa = INVALID_PAGE;  in kvm_mmu_free_roots()
3857 		mmu->root.pgd = 0;  in kvm_mmu_free_roots()
3861 		read_unlock(&kvm->mmu_lock);  in kvm_mmu_free_roots()
3865 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_free_roots()
3881 	WARN_ON_ONCE(mmu->root_role.guest_mode);  in kvm_mmu_free_guest_mode_roots()
3884 		root_hpa = mmu->prev_roots[i].hpa;  in kvm_mmu_free_guest_mode_roots()
3889 		if (!sp || sp->role.guest_mode)  in kvm_mmu_free_guest_mode_roots()
3900 	union kvm_mmu_page_role role = vcpu->arch.mmu->root_role;  in mmu_alloc_root()
3910 	++sp->root_count;  in mmu_alloc_root()
3912 	return __pa(sp->spt);  in mmu_alloc_root()
3917 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in mmu_alloc_direct_roots()
3918 	u8 shadow_root_level = mmu->root_role.level;  in mmu_alloc_direct_roots()
3924 		if (kvm_has_mirrored_tdp(vcpu->kvm) &&  in mmu_alloc_direct_roots()
3925 		    !VALID_PAGE(mmu->mirror_root_hpa))  in mmu_alloc_direct_roots()
3931 	write_lock(&vcpu->kvm->mmu_lock);  in mmu_alloc_direct_roots()
3938 		mmu->root.hpa = root;  in mmu_alloc_direct_roots()
3940 		if (WARN_ON_ONCE(!mmu->pae_root)) {  in mmu_alloc_direct_roots()
3941 			r = -EIO;  in mmu_alloc_direct_roots()
3946 			WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));  in mmu_alloc_direct_roots()
3948 			root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT), 0,  in mmu_alloc_direct_roots()
3950 			mmu->pae_root[i] = root | PT_PRESENT_MASK |  in mmu_alloc_direct_roots()
3953 		mmu->root.hpa = __pa(mmu->pae_root);  in mmu_alloc_direct_roots()
3956 		r = -EIO;  in mmu_alloc_direct_roots()
3961 	mmu->root.pgd = 0;  in mmu_alloc_direct_roots()
3963 	write_unlock(&vcpu->kvm->mmu_lock);  in mmu_alloc_direct_roots()
3971 	if (kvm->arch.mmu_page_hash)  in kvm_mmu_alloc_page_hash()
3976 		return -ENOMEM;  in kvm_mmu_alloc_page_hash()
3985 	smp_store_release(&kvm->arch.mmu_page_hash, h);  in kvm_mmu_alloc_page_hash()
4002 	mutex_lock(&kvm->slots_arch_lock);  in mmu_first_shadow_root_alloc()
4024 			 * Both of these functions are no-ops if the target is  in mmu_first_shadow_root_alloc()
4033 			r = memslot_rmap_alloc(slot, slot->npages);  in mmu_first_shadow_root_alloc()
4047 	smp_store_release(&kvm->arch.shadow_root_allocated, true);  in mmu_first_shadow_root_alloc()
4050 	mutex_unlock(&kvm->slots_arch_lock);  in mmu_first_shadow_root_alloc()
4056 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in mmu_alloc_shadow_roots()
4066 		mmu->root.hpa = kvm_mmu_get_dummy_root();  in mmu_alloc_shadow_roots()
4074 	if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {  in mmu_alloc_shadow_roots()
4076 			pdptrs[i] = mmu->get_pdptr(vcpu, i);  in mmu_alloc_shadow_roots()
4085 	r = mmu_first_shadow_root_alloc(vcpu->kvm);  in mmu_alloc_shadow_roots()
4089 	write_lock(&vcpu->kvm->mmu_lock);  in mmu_alloc_shadow_roots()
4095 	 * Do we shadow a long mode page table? If so we need to  in mmu_alloc_shadow_roots()
4096 	 * write-protect the guests page table root.  in mmu_alloc_shadow_roots()
4098 	if (mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {  in mmu_alloc_shadow_roots()
4100 				      mmu->root_role.level);  in mmu_alloc_shadow_roots()
4101 		mmu->root.hpa = root;  in mmu_alloc_shadow_roots()
4105 	if (WARN_ON_ONCE(!mmu->pae_root)) {  in mmu_alloc_shadow_roots()
4106 		r = -EIO;  in mmu_alloc_shadow_roots()
4111 	 * We shadow a 32 bit page table. This may be a legacy 2-level  in mmu_alloc_shadow_roots()
4112 	 * or a PAE 3-level page table. In either case we need to be aware that  in mmu_alloc_shadow_roots()
4113 	 * the shadow page table may be a PAE or a long mode page table.  in mmu_alloc_shadow_roots()
4116 	if (mmu->root_role.level >= PT64_ROOT_4LEVEL) {  in mmu_alloc_shadow_roots()
4119 		if (WARN_ON_ONCE(!mmu->pml4_root)) {  in mmu_alloc_shadow_roots()
4120 			r = -EIO;  in mmu_alloc_shadow_roots()
4123 		mmu->pml4_root[0] = __pa(mmu->pae_root) | pm_mask;  in mmu_alloc_shadow_roots()
4125 		if (mmu->root_role.level == PT64_ROOT_5LEVEL) {  in mmu_alloc_shadow_roots()
4126 			if (WARN_ON_ONCE(!mmu->pml5_root)) {  in mmu_alloc_shadow_roots()
4127 				r = -EIO;  in mmu_alloc_shadow_roots()
4130 			mmu->pml5_root[0] = __pa(mmu->pml4_root) | pm_mask;  in mmu_alloc_shadow_roots()
4135 		WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));  in mmu_alloc_shadow_roots()
4137 		if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {  in mmu_alloc_shadow_roots()
4139 				mmu->pae_root[i] = INVALID_PAE_ROOT;  in mmu_alloc_shadow_roots()
4146 		 * If shadowing 32-bit non-PAE page tables, each PAE page  in mmu_alloc_shadow_roots()
4147 		 * directory maps one quarter of the guest's non-PAE page  in mmu_alloc_shadow_roots()
4151 		quadrant = (mmu->cpu_role.base.level == PT32_ROOT_LEVEL) ? i : 0;  in mmu_alloc_shadow_roots()
4154 		mmu->pae_root[i] = root | pm_mask;  in mmu_alloc_shadow_roots()
4157 	if (mmu->root_role.level == PT64_ROOT_5LEVEL)  in mmu_alloc_shadow_roots()
4158 		mmu->root.hpa = __pa(mmu->pml5_root);  in mmu_alloc_shadow_roots()
4159 	else if (mmu->root_role.level == PT64_ROOT_4LEVEL)  in mmu_alloc_shadow_roots()
4160 		mmu->root.hpa = __pa(mmu->pml4_root);  in mmu_alloc_shadow_roots()
4162 		mmu->root.hpa = __pa(mmu->pae_root);  in mmu_alloc_shadow_roots()
4165 	mmu->root.pgd = root_pgd;  in mmu_alloc_shadow_roots()
4167 	write_unlock(&vcpu->kvm->mmu_lock);  in mmu_alloc_shadow_roots()
4174 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in mmu_alloc_special_roots()
4175 	bool need_pml5 = mmu->root_role.level > PT64_ROOT_4LEVEL;  in mmu_alloc_special_roots()
4181 	 * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP  in mmu_alloc_special_roots()
4184 	 * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.  in mmu_alloc_special_roots()
4186 	if (mmu->root_role.direct ||  in mmu_alloc_special_roots()
4187 	    mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL ||  in mmu_alloc_special_roots()
4188 	    mmu->root_role.level < PT64_ROOT_4LEVEL)  in mmu_alloc_special_roots()
4192 	 * NPT, the only paging mode that uses this horror, uses a fixed number  in mmu_alloc_special_roots()
4193 	 * of levels for the shadow page tables, e.g. all MMUs are 4-level or  in mmu_alloc_special_roots()
4194 	 * all MMus are 5-level.  Thus, this can safely require that pml5_root  in mmu_alloc_special_roots()
4198 	if (mmu->pae_root && mmu->pml4_root && (!need_pml5 || mmu->pml5_root))  in mmu_alloc_special_roots()
4205 	if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->pml4_root ||  in mmu_alloc_special_roots()
4206 			 (need_pml5 && mmu->pml5_root)))  in mmu_alloc_special_roots()
4207 		return -EIO;  in mmu_alloc_special_roots()
4210 	 * Unlike 32-bit NPT, the PDP table doesn't need to be in low mem, and  in mmu_alloc_special_roots()
4215 		return -ENOMEM;  in mmu_alloc_special_roots()
4229 	mmu->pae_root = pae_root;  in mmu_alloc_special_roots()
4230 	mmu->pml4_root = pml4_root;  in mmu_alloc_special_roots()
4231 	mmu->pml5_root = pml5_root;  in mmu_alloc_special_roots()
4240 	return -ENOMEM;  in mmu_alloc_special_roots()
4253 	 * walk before the reads of sp->unsync/sp->unsync_children here.  in is_unsync_root()
4255 	 * Even if another CPU was marking the SP as unsync-ed simultaneously,  in is_unsync_root()
4256 	 * any guest page table changes are not guaranteed to be visible anyway  in is_unsync_root()
4273 	if (sp->unsync || sp->unsync_children)  in is_unsync_root()
4284 	if (vcpu->arch.mmu->root_role.direct)  in kvm_mmu_sync_roots()
4287 	if (!VALID_PAGE(vcpu->arch.mmu->root.hpa))  in kvm_mmu_sync_roots()
4292 	if (vcpu->arch.mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {  in kvm_mmu_sync_roots()
4293 		hpa_t root = vcpu->arch.mmu->root.hpa;  in kvm_mmu_sync_roots()
4300 		write_lock(&vcpu->kvm->mmu_lock);  in kvm_mmu_sync_roots()
4302 		write_unlock(&vcpu->kvm->mmu_lock);  in kvm_mmu_sync_roots()
4306 	write_lock(&vcpu->kvm->mmu_lock);  in kvm_mmu_sync_roots()
4309 		hpa_t root = vcpu->arch.mmu->pae_root[i];  in kvm_mmu_sync_roots()
4317 	write_unlock(&vcpu->kvm->mmu_lock);  in kvm_mmu_sync_roots()
4326 		if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa))  in kvm_mmu_sync_prev_roots()
4330 	kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);  in kvm_mmu_sync_prev_roots()
4338 		exception->error_code = 0;  in nonpaging_gva_to_gpa()
4359  * That SPTE may be non-present.
4366 	int leaf = -1;  in get_walk()
4398 /* return true if reserved bit(s) are detected on a valid, non-MMIO SPTE. */
4418 	 * to detect reserved bits on non-MMIO SPTEs. i.e. buggy SPTEs.  in get_mmio_spte()
4423 	rsvd_check = &vcpu->arch.mmu->shadow_zero_check;  in get_mmio_spte()
4425 	for (level = root; level >= leaf; level--)  in get_mmio_spte()
4429 		pr_err("%s: reserved bits set on MMU-present spte, addr 0x%llx, hierarchy:\n",  in get_mmio_spte()
4431 		for (level = root; level >= leaf; level--)  in get_mmio_spte()
4432 			pr_err("------ spte = 0x%llx level = %d, rsvd bits = 0x%llx",  in get_mmio_spte()
4450 		return -EINVAL;  in handle_mmio_page_fault()
4452 	if (is_mmio_spte(vcpu->kvm, spte)) {  in handle_mmio_page_fault()
4477 	if (unlikely(fault->rsvd))  in page_fault_handle_page_track()
4480 	if (!fault->present || !fault->write)  in page_fault_handle_page_track()
4487 	if (kvm_gfn_is_write_tracked(vcpu->kvm, fault->slot, fault->gfn))  in page_fault_handle_page_track()
4507 	u32 id = vcpu->arch.apf.id;  in alloc_apf_token()
4510 		vcpu->arch.apf.id = 1;  in alloc_apf_token()
4512 	return (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;  in alloc_apf_token()
4521 	arch.gfn = fault->gfn;  in kvm_arch_setup_async_pf()
4522 	arch.error_code = fault->error_code;  in kvm_arch_setup_async_pf()
4523 	arch.direct_map = vcpu->arch.mmu->root_role.direct;  in kvm_arch_setup_async_pf()
4524 	arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu);  in kvm_arch_setup_async_pf()
4526 	return kvm_setup_async_pf(vcpu, fault->addr,  in kvm_arch_setup_async_pf()
4527 				  kvm_vcpu_gfn_to_hva(vcpu, fault->gfn), &arch);  in kvm_arch_setup_async_pf()
4534 	if (WARN_ON_ONCE(work->arch.error_code & PFERR_PRIVATE_ACCESS))  in kvm_arch_async_page_ready()
4537 	if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) ||  in kvm_arch_async_page_ready()
4538 	      work->wakeup_all)  in kvm_arch_async_page_ready()
4545 	if (!vcpu->arch.mmu->root_role.direct &&  in kvm_arch_async_page_ready()
4546 	      work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu))  in kvm_arch_async_page_ready()
4549 	r = kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, work->arch.error_code,  in kvm_arch_async_page_ready()
4554 	 * ignore stats for all other return times.  Page-ready "faults" aren't  in kvm_arch_async_page_ready()
4558 		vcpu->stat.pf_fixed++;  in kvm_arch_async_page_ready()
4564 	kvm_release_faultin_page(vcpu->kvm, fault->refcounted_page,  in kvm_mmu_finish_page_fault()
4565 				 r == RET_PF_RETRY, fault->map_writable);  in kvm_mmu_finish_page_fault()
4573 	if (!kvm_slot_has_gmem(fault->slot)) {  in kvm_mmu_faultin_pfn_gmem()
4575 		return -EFAULT;  in kvm_mmu_faultin_pfn_gmem()
4578 	r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn,  in kvm_mmu_faultin_pfn_gmem()
4579 			     &fault->refcounted_page, &max_order);  in kvm_mmu_faultin_pfn_gmem()
4585 	fault->map_writable = !(fault->slot->flags & KVM_MEM_READONLY);  in kvm_mmu_faultin_pfn_gmem()
4586 	fault->max_level = kvm_max_level_for_order(max_order);  in kvm_mmu_faultin_pfn_gmem()
4594 	unsigned int foll = fault->write ? FOLL_WRITE : 0;  in __kvm_mmu_faultin_pfn()
4596 	if (fault->is_private || kvm_memslot_is_gmem_only(fault->slot))  in __kvm_mmu_faultin_pfn()
4600 	fault->pfn = __kvm_faultin_pfn(fault->slot, fault->gfn, foll,  in __kvm_mmu_faultin_pfn()
4601 				       &fault->map_writable, &fault->refcounted_page);  in __kvm_mmu_faultin_pfn()
4604 	 * If resolving the page failed because I/O is needed to fault-in the  in __kvm_mmu_faultin_pfn()
4609 	if (fault->pfn != KVM_PFN_ERR_NEEDS_IO)  in __kvm_mmu_faultin_pfn()
4612 	if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {  in __kvm_mmu_faultin_pfn()
4613 		trace_kvm_try_async_get_page(fault->addr, fault->gfn);  in __kvm_mmu_faultin_pfn()
4614 		if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {  in __kvm_mmu_faultin_pfn()
4615 			trace_kvm_async_pf_repeated_fault(fault->addr, fault->gfn);  in __kvm_mmu_faultin_pfn()
4624 	 * Allow gup to bail on pending non-fatal signals when it's also allowed  in __kvm_mmu_faultin_pfn()
4630 	fault->pfn = __kvm_faultin_pfn(fault->slot, fault->gfn, foll,  in __kvm_mmu_faultin_pfn()
4631 				       &fault->map_writable, &fault->refcounted_page);  in __kvm_mmu_faultin_pfn()
4639 	struct kvm_memory_slot *slot = fault->slot;  in kvm_mmu_faultin_pfn()
4640 	struct kvm *kvm = vcpu->kvm;  in kvm_mmu_faultin_pfn()
4643 	if (KVM_BUG_ON(kvm_is_gfn_alias(kvm, fault->gfn), kvm))  in kvm_mmu_faultin_pfn()
4644 		return -EFAULT;  in kvm_mmu_faultin_pfn()
4649 	 * invalidation relate to fault->fn and resume the guest without  in kvm_mmu_faultin_pfn()
4652 	fault->mmu_seq = vcpu->kvm->mmu_invalidate_seq;  in kvm_mmu_faultin_pfn()
4659 	if (fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) {  in kvm_mmu_faultin_pfn()
4661 		return -EFAULT;  in kvm_mmu_faultin_pfn()
4674 	if (slot->flags & KVM_MEMSLOT_INVALID) {  in kvm_mmu_faultin_pfn()
4675 		if (fault->prefetch)  in kvm_mmu_faultin_pfn()
4676 			return -EAGAIN;  in kvm_mmu_faultin_pfn()
4681 	if (slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT) {  in kvm_mmu_faultin_pfn()
4700 		 * when the AVIC is re-enabled.  in kvm_mmu_faultin_pfn()
4702 		if (!kvm_apicv_activated(vcpu->kvm))  in kvm_mmu_faultin_pfn()
4710 	 * For mmu_lock, if there is an in-progress invalidation and the kernel  in kvm_mmu_faultin_pfn()
4712 	 * in response to mmu_lock being contended, which is *very* counter-  in kvm_mmu_faultin_pfn()
4722 	 * Do the pre-check even for non-preemtible kernels, i.e. even if KVM  in kvm_mmu_faultin_pfn()
4727 	if (mmu_invalidate_retry_gfn_unsafe(kvm, fault->mmu_seq, fault->gfn))  in kvm_mmu_faultin_pfn()
4734 	if (unlikely(is_error_pfn(fault->pfn)))  in kvm_mmu_faultin_pfn()
4737 	if (WARN_ON_ONCE(!fault->slot || is_noslot_pfn(fault->pfn)))  in kvm_mmu_faultin_pfn()
4747 	if (mmu_invalidate_retry_gfn_unsafe(kvm, fault->mmu_seq, fault->gfn)) {  in kvm_mmu_faultin_pfn()
4762 	struct kvm_mmu_page *sp = root_to_sp(vcpu->arch.mmu->root.hpa);  in is_page_fault_stale()
4765 	if (sp && is_obsolete_sp(vcpu->kvm, sp))  in is_page_fault_stale()
4784 	return fault->slot &&  in is_page_fault_stale()
4785 	       mmu_invalidate_retry_gfn(vcpu->kvm, fault->mmu_seq, fault->gfn);  in is_page_fault_stale()
4793 	if (WARN_ON_ONCE(kvm_mmu_is_dummy_root(vcpu->arch.mmu->root.hpa)))  in direct_page_fault()
4812 	write_lock(&vcpu->kvm->mmu_lock);  in direct_page_fault()
4825 	write_unlock(&vcpu->kvm->mmu_lock);  in direct_page_fault()
4833 	fault->max_level = PG_LEVEL_2M;  in nonpaging_page_fault()
4841 	u32 flags = vcpu->arch.apf.host_apf_flags;  in kvm_handle_page_fault()
4844 	/* A 64-bit CR2 should be impossible on 32-bit KVM. */  in kvm_handle_page_fault()
4846 		return -EFAULT;  in kvm_handle_page_fault()
4849 	 * Legacy #PF exception only have a 32-bit error code.  Simply drop the  in kvm_handle_page_fault()
4851 	 * set), and to ensure there are no collisions with KVM-defined bits.  in kvm_handle_page_fault()
4857 	 * Restrict KVM-defined flags to bits 63:32 so that it's impossible for  in kvm_handle_page_fault()
4862 	vcpu->arch.l1tf_flush_l1d = true;  in kvm_handle_page_fault()
4869 		vcpu->arch.apf.host_apf_flags = 0;  in kvm_handle_page_fault()
4903 	read_lock(&vcpu->kvm->mmu_lock);  in kvm_tdp_mmu_page_fault()
4912 	read_unlock(&vcpu->kvm->mmu_lock);  in kvm_tdp_mmu_page_fault()
4935 	if (vcpu->arch.mmu->page_fault != kvm_tdp_page_fault)  in kvm_tdp_map_page()
4936 		return -EOPNOTSUPP;  in kvm_tdp_map_page()
4940 			return -EINTR;  in kvm_tdp_map_page()
4943 			return -EIO;  in kvm_tdp_map_page()
4959 		return -ENOENT;  in kvm_tdp_map_page()
4966 		return -EIO;  in kvm_tdp_map_page()
4980 	if (!vcpu->kvm->arch.pre_fault_allowed)  in kvm_arch_vcpu_pre_fault_memory()
4981 		return -EOPNOTSUPP;  in kvm_arch_vcpu_pre_fault_memory()
4983 	if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa)))  in kvm_arch_vcpu_pre_fault_memory()
4984 		return -EINVAL;  in kvm_arch_vcpu_pre_fault_memory()
4995 	if (kvm_arch_has_private_mem(vcpu->kvm) &&  in kvm_arch_vcpu_pre_fault_memory()
4996 	    kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa)))  in kvm_arch_vcpu_pre_fault_memory()
4999 		direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));  in kvm_arch_vcpu_pre_fault_memory()
5003 	 * two-dimensional paging.  in kvm_arch_vcpu_pre_fault_memory()
5005 	r = kvm_tdp_map_page(vcpu, range->gpa | direct_bits, error_code, &level);  in kvm_arch_vcpu_pre_fault_memory()
5010 	 * If the mapping that covers range->gpa can use a huge page, it  in kvm_arch_vcpu_pre_fault_memory()
5011 	 * may start below it or end after range->gpa + range->size.  in kvm_arch_vcpu_pre_fault_memory()
5013 	end = (range->gpa & KVM_HPAGE_MASK(level)) + KVM_HPAGE_SIZE(level);  in kvm_arch_vcpu_pre_fault_memory()
5014 	return min(range->size, end - range->gpa);  in kvm_arch_vcpu_pre_fault_memory()
5019 	context->page_fault = nonpaging_page_fault;  in nonpaging_init_context()
5020 	context->gva_to_gpa = nonpaging_gva_to_gpa;  in nonpaging_init_context()
5021 	context->sync_spte = NULL;  in nonpaging_init_context()
5029 	if (!VALID_PAGE(root->hpa))  in is_root_usable()
5032 	if (!role.direct && pgd != root->pgd)  in is_root_usable()
5035 	sp = root_to_sp(root->hpa);  in is_root_usable()
5039 	return role.word == sp->role.word;  in is_root_usable()
5045  * If a matching root is found, it is assigned to kvm_mmu->root and
5047  * If no match is found, kvm_mmu->root is left invalid, the LRU root is
5056 	if (is_root_usable(&mmu->root, new_pgd, new_role))  in cached_root_find_and_keep_current()
5068 		swap(mmu->root, mmu->prev_roots[i]);  in cached_root_find_and_keep_current()
5069 		if (is_root_usable(&mmu->root, new_pgd, new_role))  in cached_root_find_and_keep_current()
5079  * On entry, mmu->root is invalid.
5080  * If a matching root is found, it is assigned to kvm_mmu->root, the LRU entry
5082  * If no match is found, kvm_mmu->root is left invalid and false is returned.
5091 		if (is_root_usable(&mmu->prev_roots[i], new_pgd, new_role))  in cached_root_find_without_current()
5097 	swap(mmu->root, mmu->prev_roots[i]);  in cached_root_find_without_current()
5099 	for (; i < KVM_MMU_NUM_PREV_ROOTS - 1; i++)  in cached_root_find_without_current()
5100 		mmu->prev_roots[i] = mmu->prev_roots[i + 1];  in cached_root_find_without_current()
5101 	mmu->prev_roots[i].hpa = INVALID_PAGE;  in cached_root_find_without_current()
5109 	 * Limit reuse to 64-bit hosts+VMs without "special" roots in order to  in fast_pgd_switch()
5112 	if (VALID_PAGE(mmu->root.hpa) && !root_to_sp(mmu->root.hpa))  in fast_pgd_switch()
5115 	if (VALID_PAGE(mmu->root.hpa))  in fast_pgd_switch()
5123 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_mmu_new_pgd()
5124 	union kvm_mmu_page_role new_role = mmu->root_role;  in kvm_mmu_new_pgd()
5128 	 * will establish a valid root prior to the next VM-Enter.  in kvm_mmu_new_pgd()
5130 	if (!fast_pgd_switch(vcpu->kvm, mmu, new_pgd, new_role))  in kvm_mmu_new_pgd()
5148 	 * switching to a new CR3, that GVA->GPA mapping may no longer be  in kvm_mmu_new_pgd()
5159 		struct kvm_mmu_page *sp = root_to_sp(vcpu->arch.mmu->root.hpa);  in kvm_mmu_new_pgd()
5170 	if (unlikely(is_mmio_spte(vcpu->kvm, *sptep))) {  in sync_mmio_spte()
5204 	rsvd_check->bad_mt_xwr = 0;  in __reset_rsvds_bits_mask()
5219 	 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for  in __reset_rsvds_bits_mask()
5228 		rsvd_check->rsvd_bits_mask[0][1] = 0;  in __reset_rsvds_bits_mask()
5229 		rsvd_check->rsvd_bits_mask[0][0] = 0;  in __reset_rsvds_bits_mask()
5230 		rsvd_check->rsvd_bits_mask[1][0] =  in __reset_rsvds_bits_mask()
5231 			rsvd_check->rsvd_bits_mask[0][0];  in __reset_rsvds_bits_mask()
5234 			rsvd_check->rsvd_bits_mask[1][1] = 0;  in __reset_rsvds_bits_mask()
5240 			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);  in __reset_rsvds_bits_mask()
5243 			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);  in __reset_rsvds_bits_mask()
5246 		rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |  in __reset_rsvds_bits_mask()
5250 		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;	/* PDE */  in __reset_rsvds_bits_mask()
5251 		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;	/* PTE */  in __reset_rsvds_bits_mask()
5252 		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5254 		rsvd_check->rsvd_bits_mask[1][0] =  in __reset_rsvds_bits_mask()
5255 			rsvd_check->rsvd_bits_mask[0][0];  in __reset_rsvds_bits_mask()
5258 		rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5261 		rsvd_check->rsvd_bits_mask[1][4] =  in __reset_rsvds_bits_mask()
5262 			rsvd_check->rsvd_bits_mask[0][4];  in __reset_rsvds_bits_mask()
5265 		rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5268 		rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5270 		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;  in __reset_rsvds_bits_mask()
5271 		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;  in __reset_rsvds_bits_mask()
5272 		rsvd_check->rsvd_bits_mask[1][3] =  in __reset_rsvds_bits_mask()
5273 			rsvd_check->rsvd_bits_mask[0][3];  in __reset_rsvds_bits_mask()
5274 		rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5277 		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |  in __reset_rsvds_bits_mask()
5279 		rsvd_check->rsvd_bits_mask[1][0] =  in __reset_rsvds_bits_mask()
5280 			rsvd_check->rsvd_bits_mask[0][0];  in __reset_rsvds_bits_mask()
5288 	__reset_rsvds_bits_mask(&context->guest_rsvd_check,  in reset_guest_rsvds_bits_mask()
5289 				vcpu->arch.reserved_gpa_bits,  in reset_guest_rsvds_bits_mask()
5290 				context->cpu_role.base.level, is_efer_nx(context),  in reset_guest_rsvds_bits_mask()
5309 	rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);  in __reset_rsvds_bits_mask_ept()
5310 	rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);  in __reset_rsvds_bits_mask_ept()
5311 	rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;  in __reset_rsvds_bits_mask_ept()
5312 	rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;  in __reset_rsvds_bits_mask_ept()
5313 	rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;  in __reset_rsvds_bits_mask_ept()
5316 	rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];  in __reset_rsvds_bits_mask_ept()
5317 	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];  in __reset_rsvds_bits_mask_ept()
5318 	rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;  in __reset_rsvds_bits_mask_ept()
5319 	rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;  in __reset_rsvds_bits_mask_ept()
5320 	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];  in __reset_rsvds_bits_mask_ept()
5331 	rsvd_check->bad_mt_xwr = bad_mt_xwr;  in __reset_rsvds_bits_mask_ept()
5337 	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,  in reset_rsvds_bits_mask_ept()
5338 				    vcpu->arch.reserved_gpa_bits, execonly,  in reset_rsvds_bits_mask_ept()
5357 	/* KVM doesn't use 2-level page tables for the shadow MMU. */  in reset_shadow_zero_bits_mask()
5362 	WARN_ON_ONCE(context->root_role.level < PT32E_ROOT_LEVEL);  in reset_shadow_zero_bits_mask()
5364 	shadow_zero_check = &context->shadow_zero_check;  in reset_shadow_zero_bits_mask()
5366 				context->root_role.level,  in reset_shadow_zero_bits_mask()
5367 				context->root_role.efer_nx,  in reset_shadow_zero_bits_mask()
5374 	for (i = context->root_role.level; --i >= 0;) {  in reset_shadow_zero_bits_mask()
5381 		shadow_zero_check->rsvd_bits_mask[0][i] |= shadow_me_mask;  in reset_shadow_zero_bits_mask()
5382 		shadow_zero_check->rsvd_bits_mask[1][i] |= shadow_me_mask;  in reset_shadow_zero_bits_mask()
5383 		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_value;  in reset_shadow_zero_bits_mask()
5384 		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_value;  in reset_shadow_zero_bits_mask()
5397  * possible, however, kvm currently does not do execution-protection.
5404 	shadow_zero_check = &context->shadow_zero_check;  in reset_tdp_shadow_zero_bits_mask()
5408 					context->root_role.level, true,  in reset_tdp_shadow_zero_bits_mask()
5419 	for (i = context->root_role.level; --i >= 0;) {  in reset_tdp_shadow_zero_bits_mask()
5420 		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;  in reset_tdp_shadow_zero_bits_mask()
5421 		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;  in reset_tdp_shadow_zero_bits_mask()
5432 	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,  in reset_ept_shadow_zero_bits_mask()
5460 	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {  in update_permission_bitmask()
5468 		/* Faults from writes to non-writable pages */  in update_permission_bitmask()
5470 		/* Faults from user mode accesses to supervisor pages */  in update_permission_bitmask()
5472 		/* Faults from fetches of non-executable pages*/  in update_permission_bitmask()
5474 		/* Faults from kernel mode fetches of user pages */  in update_permission_bitmask()
5476 		/* Faults from kernel mode accesses of user pages */  in update_permission_bitmask()
5480 			/* Faults from kernel mode accesses to user pages */  in update_permission_bitmask()
5487 			/* Allow supervisor writes if !cr0.wp */  in update_permission_bitmask()
5491 			/* Disallow supervisor fetches of user code if cr4.smep */  in update_permission_bitmask()
5496 			 * SMAP:kernel-mode data accesses from user-mode  in update_permission_bitmask()
5500 			 *   - X86_CR4_SMAP is set in CR4  in update_permission_bitmask()
5501 			 *   - A user page is accessed  in update_permission_bitmask()
5502 			 *   - The access is not a fetch  in update_permission_bitmask()
5503 			 *   - The access is supervisor mode  in update_permission_bitmask()
5504 			 *   - If implicit supervisor access or X86_EFLAGS_AC is clear  in update_permission_bitmask()
5515 		mmu->permissions[byte] = ff | uf | wf | smepf | smapf;  in update_permission_bitmask()
5521 * user-mode addresses based on the value in the PKRU register.  Protection
5530 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
5531 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
5532 * - PK is always zero if U=0 in the page tables
5533 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
5548 	mmu->pkru_mask = 0;  in update_pkru_bitmask()
5555 	for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {  in update_pkru_bitmask()
5583 		mmu->pkru_mask |= (pkey_bits & 3) << pfec;  in update_pkru_bitmask()
5600 	context->page_fault = paging64_page_fault;  in paging64_init_context()
5601 	context->gva_to_gpa = paging64_gva_to_gpa;  in paging64_init_context()
5602 	context->sync_spte = paging64_sync_spte;  in paging64_init_context()
5607 	context->page_fault = paging32_page_fault;  in paging32_init_context()
5608 	context->gva_to_gpa = paging32_gva_to_gpa;  in paging32_init_context()
5609 	context->sync_spte = paging32_sync_spte;  in paging32_init_context()
5645 	/* PKEY and LA57 are active iff long mode is active. */  in kvm_calc_cpu_role()
5663 	mmu->cpu_role.base.cr0_wp = cr0_wp;  in __kvm_mmu_refresh_passthrough_bits()
5671 	if (vcpu->kvm->arch.vm_type == KVM_X86_TDX_VM)  in kvm_mmu_get_tdp_level()
5680 	/* Use 5-level TDP if and only if it's useful/necessary. */  in kvm_mmu_get_tdp_level()
5714 	struct kvm_mmu *context = &vcpu->arch.root_mmu;  in init_kvm_tdp_mmu()
5717 	if (cpu_role.as_u64 == context->cpu_role.as_u64 &&  in init_kvm_tdp_mmu()
5718 	    root_role.word == context->root_role.word)  in init_kvm_tdp_mmu()
5721 	context->cpu_role.as_u64 = cpu_role.as_u64;  in init_kvm_tdp_mmu()
5722 	context->root_role.word = root_role.word;  in init_kvm_tdp_mmu()
5723 	context->page_fault = kvm_tdp_page_fault;  in init_kvm_tdp_mmu()
5724 	context->sync_spte = NULL;  in init_kvm_tdp_mmu()
5725 	context->get_guest_pgd = get_guest_cr3;  in init_kvm_tdp_mmu()
5726 	context->get_pdptr = kvm_pdptr_read;  in init_kvm_tdp_mmu()
5727 	context->inject_page_fault = kvm_inject_page_fault;  in init_kvm_tdp_mmu()
5730 		context->gva_to_gpa = nonpaging_gva_to_gpa;  in init_kvm_tdp_mmu()
5732 		context->gva_to_gpa = paging64_gva_to_gpa;  in init_kvm_tdp_mmu()
5734 		context->gva_to_gpa = paging32_gva_to_gpa;  in init_kvm_tdp_mmu()
5744 	if (cpu_role.as_u64 == context->cpu_role.as_u64 &&  in shadow_mmu_init_context()
5745 	    root_role.word == context->root_role.word)  in shadow_mmu_init_context()
5748 	context->cpu_role.as_u64 = cpu_role.as_u64;  in shadow_mmu_init_context()
5749 	context->root_role.word = root_role.word;  in shadow_mmu_init_context()
5765 	struct kvm_mmu *context = &vcpu->arch.root_mmu;  in kvm_init_shadow_mmu()
5770 	/* KVM uses PAE paging whenever the guest isn't using 64-bit paging. */  in kvm_init_shadow_mmu()
5776 	 * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and  in kvm_init_shadow_mmu()
5778 	 * The iTLB multi-hit workaround can be toggled at any time, so assume  in kvm_init_shadow_mmu()
5779 	 * NX can be used by any non-nested shadow MMU to avoid having to reset  in kvm_init_shadow_mmu()
5790 	struct kvm_mmu *context = &vcpu->arch.guest_mmu;  in kvm_init_shadow_npt_mmu()
5842 	struct kvm_mmu *context = &vcpu->arch.guest_mmu;  in kvm_init_shadow_ept_mmu()
5848 	if (new_mode.as_u64 != context->cpu_role.as_u64) {  in kvm_init_shadow_ept_mmu()
5850 		context->cpu_role.as_u64 = new_mode.as_u64;  in kvm_init_shadow_ept_mmu()
5851 		context->root_role.word = new_mode.base.word;  in kvm_init_shadow_ept_mmu()
5853 		context->page_fault = ept_page_fault;  in kvm_init_shadow_ept_mmu()
5854 		context->gva_to_gpa = ept_gva_to_gpa;  in kvm_init_shadow_ept_mmu()
5855 		context->sync_spte = ept_sync_spte;  in kvm_init_shadow_ept_mmu()
5858 		context->pkru_mask = 0;  in kvm_init_shadow_ept_mmu()
5870 	struct kvm_mmu *context = &vcpu->arch.root_mmu;  in init_kvm_softmmu()
5874 	context->get_guest_pgd     = get_guest_cr3;  in init_kvm_softmmu()
5875 	context->get_pdptr         = kvm_pdptr_read;  in init_kvm_softmmu()
5876 	context->inject_page_fault = kvm_inject_page_fault;  in init_kvm_softmmu()
5882 	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;  in init_kvm_nested_mmu()
5884 	if (new_mode.as_u64 == g_context->cpu_role.as_u64)  in init_kvm_nested_mmu()
5887 	g_context->cpu_role.as_u64   = new_mode.as_u64;  in init_kvm_nested_mmu()
5888 	g_context->get_guest_pgd     = get_guest_cr3;  in init_kvm_nested_mmu()
5889 	g_context->get_pdptr         = kvm_pdptr_read;  in init_kvm_nested_mmu()
5890 	g_context->inject_page_fault = kvm_inject_page_fault;  in init_kvm_nested_mmu()
5896 	g_context->sync_spte         = NULL;  in init_kvm_nested_mmu()
5899 	 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using  in init_kvm_nested_mmu()
5907 		g_context->gva_to_gpa = nonpaging_gva_to_gpa;  in init_kvm_nested_mmu()
5909 		g_context->gva_to_gpa = paging64_gva_to_gpa;  in init_kvm_nested_mmu()
5911 		g_context->gva_to_gpa = paging64_gva_to_gpa;  in init_kvm_nested_mmu()
5913 		g_context->gva_to_gpa = paging32_gva_to_gpa;  in init_kvm_nested_mmu()
5946 	vcpu->arch.root_mmu.root_role.invalid = 1;  in kvm_mmu_after_set_cpuid()
5947 	vcpu->arch.guest_mmu.root_role.invalid = 1;  in kvm_mmu_after_set_cpuid()
5948 	vcpu->arch.nested_mmu.root_role.invalid = 1;  in kvm_mmu_after_set_cpuid()
5949 	vcpu->arch.root_mmu.cpu_role.ext.valid = 0;  in kvm_mmu_after_set_cpuid()
5950 	vcpu->arch.guest_mmu.cpu_role.ext.valid = 0;  in kvm_mmu_after_set_cpuid()
5951 	vcpu->arch.nested_mmu.cpu_role.ext.valid = 0;  in kvm_mmu_after_set_cpuid()
5958 	KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm);  in kvm_mmu_after_set_cpuid()
5972 	r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->root_role.direct);  in kvm_mmu_load()
5978 	if (vcpu->arch.mmu->root_role.direct)  in kvm_mmu_load()
6004 	struct kvm *kvm = vcpu->kvm;  in kvm_mmu_unload()
6006 	kvm_mmu_free_roots(kvm, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);  in kvm_mmu_unload()
6007 	WARN_ON_ONCE(VALID_PAGE(vcpu->arch.root_mmu.root.hpa));  in kvm_mmu_unload()
6008 	kvm_mmu_free_roots(kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);  in kvm_mmu_unload()
6009 	WARN_ON_ONCE(VALID_PAGE(vcpu->arch.guest_mmu.root.hpa));  in kvm_mmu_unload()
6031 	 *      is unlikely to zap an in-use PGD.  in is_obsolete_root()
6045 	if (is_obsolete_root(kvm, mmu->root.hpa))  in __kvm_mmu_free_obsolete_roots()
6049 		if (is_obsolete_root(kvm, mmu->prev_roots[i].hpa))  in __kvm_mmu_free_obsolete_roots()
6059 	__kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);  in kvm_mmu_free_obsolete_roots()
6060 	__kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);  in kvm_mmu_free_obsolete_roots()
6072 	 * as the current vcpu paging mode since we update the sptes only  in mmu_pte_write_fetch_gpte()
6073 	 * when they have the same mode.  in mmu_pte_write_fetch_gpte()
6076 		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */  in mmu_pte_write_fetch_gpte()
6097 	 * Skip write-flooding detected for the sp whose level is 1, because  in detect_write_flooding()
6098 	 * it can become unsync, then the guest page is not write-protected.  in detect_write_flooding()
6100 	if (sp->role.level == PG_LEVEL_4K)  in detect_write_flooding()
6103 	atomic_inc(&sp->write_flooding_count);  in detect_write_flooding()
6104 	return atomic_read(&sp->write_flooding_count) >= 3;  in detect_write_flooding()
6117 	pte_size = sp->role.has_4_byte_gpte ? 4 : 8;  in detect_write_misaligned()
6123 	if (!(offset & (pte_size - 1)) && bytes == 1)  in detect_write_misaligned()
6126 	misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);  in detect_write_misaligned()
6139 	level = sp->role.level;  in get_written_sptes()
6141 	if (sp->role.has_4_byte_gpte) {  in get_written_sptes()
6142 		page_offset <<= 1;	/* 32->64 */  in get_written_sptes()
6144 		 * A 32-bit pde maps 4MB while the shadow pdes map  in get_written_sptes()
6155 		if (quadrant != sp->role.quadrant)  in get_written_sptes()
6159 	spte = &sp->spt[page_offset / sizeof(*spte)];  in get_written_sptes()
6174 	 * When emulating guest writes, ensure the written value is visible to  in kvm_mmu_track_write()
6178 	 * a non-zero indirect_shadow_pages.  Pairs with the smp_mb() in  in kvm_mmu_track_write()
6182 	if (!vcpu->kvm->arch.indirect_shadow_pages)  in kvm_mmu_track_write()
6185 	write_lock(&vcpu->kvm->mmu_lock);  in kvm_mmu_track_write()
6189 	++vcpu->kvm->stat.mmu_pte_write;  in kvm_mmu_track_write()
6191 	for_each_gfn_valid_sp_with_gptes(vcpu->kvm, sp, gfn) {  in kvm_mmu_track_write()
6194 			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);  in kvm_mmu_track_write()
6195 			++vcpu->kvm->stat.mmu_flooded;  in kvm_mmu_track_write()
6203 		while (npte--) {  in kvm_mmu_track_write()
6205 			mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);  in kvm_mmu_track_write()
6206 			if (gentry && sp->role.level != PG_LEVEL_4K)  in kvm_mmu_track_write()
6207 				++vcpu->kvm->stat.mmu_pde_zapped;  in kvm_mmu_track_write()
6213 	kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);  in kvm_mmu_track_write()
6214 	write_unlock(&vcpu->kvm->mmu_lock);  in kvm_mmu_track_write()
6227 	bool direct = vcpu->arch.mmu->root_role.direct;  in kvm_mmu_write_protect_fault()
6230 	 * Do not try to unprotect and retry if the vCPU re-faulted on the same  in kvm_mmu_write_protect_fault()
6233 	 * a non-page-table modifying instruction on the PDE that points to the  in kvm_mmu_write_protect_fault()
6237 	if (vcpu->arch.last_retry_eip == kvm_rip_read(vcpu) &&  in kvm_mmu_write_protect_fault()
6238 	    vcpu->arch.last_retry_addr == cr2_or_gpa)  in kvm_mmu_write_protect_fault()
6247 	vcpu->arch.last_retry_eip = 0;  in kvm_mmu_write_protect_fault()
6248 	vcpu->arch.last_retry_addr = 0;  in kvm_mmu_write_protect_fault()
6257 	 * will keep faulting on the non-existent MMIO address.  in kvm_mmu_write_protect_fault()
6264 	 * to a read-only violation while the CPU was walking non-nested NPT  in kvm_mmu_write_protect_fault()
6268 	 * (L0) write-protects the nested NPTs, i.e. npt12 entries, KVM is also  in kvm_mmu_write_protect_fault()
6269 	 * unknowingly write-protecting L1's guest page tables, which KVM isn't  in kvm_mmu_write_protect_fault()
6277 	 * isn't shadowed by KVM, there is no need to write-protect L1's gPTEs  in kvm_mmu_write_protect_fault()
6283 	 * the instruction.  If no shadow pages were zapped, then the write-  in kvm_mmu_write_protect_fault()
6303 	 * The gfn is write-protected, but if KVM detects its emulating an  in kvm_mmu_write_protect_fault()
6306 	 * re-execute the instruction that caused the page fault.  Do not allow  in kvm_mmu_write_protect_fault()
6321 	bool direct = vcpu->arch.mmu->root_role.direct;  in kvm_mmu_page_fault()
6323 	if (WARN_ON_ONCE(!VALID_PAGE(vcpu->arch.mmu->root.hpa)))  in kvm_mmu_page_fault()
6327 	 * Except for reserved faults (emulated MMIO is shared-only), set the  in kvm_mmu_page_fault()
6328 	 * PFERR_PRIVATE_ACCESS flag for software-protected VMs based on the gfn's  in kvm_mmu_page_fault()
6332 	 * for software-protected VMs.  in kvm_mmu_page_fault()
6336 	    vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM &&  in kvm_mmu_page_fault()
6337 	    kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)))  in kvm_mmu_page_fault()
6343 			return -EFAULT;  in kvm_mmu_page_fault()
6351 		vcpu->stat.pf_taken++;  in kvm_mmu_page_fault()
6355 		if (KVM_BUG_ON(r == RET_PF_INVALID, vcpu->kvm))  in kvm_mmu_page_fault()
6356 			return -EIO;  in kvm_mmu_page_fault()
6367 		vcpu->stat.pf_fixed++;  in kvm_mmu_page_fault()
6369 		vcpu->stat.pf_emulate++;  in kvm_mmu_page_fault()
6371 		vcpu->stat.pf_spurious++;  in kvm_mmu_page_fault()
6400 	for (level = root_level; level >= leaf; level--)  in kvm_mmu_print_sptes()
6418 	if (WARN_ON_ONCE(mmu != vcpu->arch.mmu))  in __kvm_mmu_invalidate_addr()
6424 	write_lock(&vcpu->kvm->mmu_lock);  in __kvm_mmu_invalidate_addr()
6428 		if (sp->unsync) {  in __kvm_mmu_invalidate_addr()
6432 				mmu_page_zap_pte(vcpu->kvm, sp, iterator.sptep, NULL);  in __kvm_mmu_invalidate_addr()
6434 				kvm_flush_remote_tlbs_sptep(vcpu->kvm, iterator.sptep);  in __kvm_mmu_invalidate_addr()
6437 		if (!sp->unsync_children)  in __kvm_mmu_invalidate_addr()
6440 	write_unlock(&vcpu->kvm->mmu_lock);  in __kvm_mmu_invalidate_addr()
6450 	/* It's actually a GPA for vcpu->arch.guest_mmu.  */  in kvm_mmu_invalidate_addr()
6451 	if (mmu != &vcpu->arch.guest_mmu) {  in kvm_mmu_invalidate_addr()
6452 		/* INVLPG on a non-canonical address is a NOP according to the SDM.  */  in kvm_mmu_invalidate_addr()
6459 	if (!mmu->sync_spte)  in kvm_mmu_invalidate_addr()
6463 		__kvm_mmu_invalidate_addr(vcpu, mmu, addr, mmu->root.hpa);  in kvm_mmu_invalidate_addr()
6467 			__kvm_mmu_invalidate_addr(vcpu, mmu, addr, mmu->prev_roots[i].hpa);  in kvm_mmu_invalidate_addr()
6484 	kvm_mmu_invalidate_addr(vcpu, vcpu->arch.walk_mmu, gva, KVM_MMU_ROOTS_ALL);  in kvm_mmu_invlpg()
6485 	++vcpu->stat.invlpg;  in kvm_mmu_invlpg()
6492 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_mmu_invpcid_gva()
6500 		if (VALID_PAGE(mmu->prev_roots[i].hpa) &&  in kvm_mmu_invpcid_gva()
6501 		    pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd))  in kvm_mmu_invpcid_gva()
6507 	++vcpu->stat.invlpg;  in kvm_mmu_invpcid_gva()
6544 	if (!tdp_enabled && mmu->pae_root)  in free_mmu_pages()
6545 		set_memory_encrypted((unsigned long)mmu->pae_root, 1);  in free_mmu_pages()
6546 	free_page((unsigned long)mmu->pae_root);  in free_mmu_pages()
6547 	free_page((unsigned long)mmu->pml4_root);  in free_mmu_pages()
6548 	free_page((unsigned long)mmu->pml5_root);  in free_mmu_pages()
6556 	mmu->root.hpa = INVALID_PAGE;  in __kvm_mmu_create()
6557 	mmu->root.pgd = 0;  in __kvm_mmu_create()
6558 	mmu->mirror_root_hpa = INVALID_PAGE;  in __kvm_mmu_create()
6560 		mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;  in __kvm_mmu_create()
6562 	/* vcpu->arch.guest_mmu isn't used when !tdp_enabled. */  in __kvm_mmu_create()
6563 	if (!tdp_enabled && mmu == &vcpu->arch.guest_mmu)  in __kvm_mmu_create()
6568 	 * while the PDP table is a per-vCPU construct that's allocated at MMU  in __kvm_mmu_create()
6569 	 * creation.  When emulating 32-bit mode, cr3 is only 32 bits even on  in __kvm_mmu_create()
6573 	 * table.  The main exception, handled here, is SVM's 32-bit NPT.  The  in __kvm_mmu_create()
6574 	 * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit  in __kvm_mmu_create()
6575 	 * KVM; that horror is handled on-demand by mmu_alloc_special_roots().  in __kvm_mmu_create()
6582 		return -ENOMEM;  in __kvm_mmu_create()
6584 	mmu->pae_root = page_address(page);  in __kvm_mmu_create()
6590 	 * only necessary when using shadow paging, as 64-bit NPT can get at  in __kvm_mmu_create()
6591 	 * the C-bit even when shadowing 32-bit NPT, and SME isn't supported  in __kvm_mmu_create()
6592 	 * by 32-bit kernels (when KVM itself uses 32-bit NPT).  in __kvm_mmu_create()
6595 		set_memory_decrypted((unsigned long)mmu->pae_root, 1);  in __kvm_mmu_create()
6600 		mmu->pae_root[i] = INVALID_PAE_ROOT;  in __kvm_mmu_create()
6609 	vcpu->arch.mmu_pte_list_desc_cache.kmem_cache = pte_list_desc_cache;  in kvm_mmu_create()
6610 	vcpu->arch.mmu_pte_list_desc_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_create()
6612 	vcpu->arch.mmu_page_header_cache.kmem_cache = mmu_page_header_cache;  in kvm_mmu_create()
6613 	vcpu->arch.mmu_page_header_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_create()
6615 	vcpu->arch.mmu_shadow_page_cache.init_value =  in kvm_mmu_create()
6617 	if (!vcpu->arch.mmu_shadow_page_cache.init_value)  in kvm_mmu_create()
6618 		vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_create()
6620 	vcpu->arch.mmu = &vcpu->arch.root_mmu;  in kvm_mmu_create()
6621 	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;  in kvm_mmu_create()
6623 	ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);  in kvm_mmu_create()
6627 	ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu);  in kvm_mmu_create()
6633 	free_mmu_pages(&vcpu->arch.guest_mmu);  in kvm_mmu_create()
6645 	lockdep_assert_held(&kvm->slots_lock);  in kvm_zap_obsolete_pages()
6649 	      &kvm->arch.active_mmu_pages, link) {  in kvm_zap_obsolete_pages()
6662 		if (WARN_ON_ONCE(sp->role.invalid))  in kvm_zap_obsolete_pages()
6672 		    cond_resched_rwlock_write(&kvm->mmu_lock)) {  in kvm_zap_obsolete_pages()
6698  * Fast invalidate all shadow pages and use lock-break technique
6703  * not use any resource of the being-deleted slot or all slots
6708 	lockdep_assert_held(&kvm->slots_lock);  in kvm_mmu_zap_all_fast()
6710 	write_lock(&kvm->mmu_lock);  in kvm_mmu_zap_all_fast()
6720 	kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1;  in kvm_mmu_zap_all_fast()
6723 	 * In order to ensure all vCPUs drop their soon-to-be invalid roots,  in kvm_mmu_zap_all_fast()
6748 	write_unlock(&kvm->mmu_lock);  in kvm_mmu_zap_all_fast()
6756 	 * lead to use-after-free.  in kvm_mmu_zap_all_fast()
6766 	kvm->arch.shadow_mmio_value = shadow_mmio_value;  in kvm_mmu_init_vm()
6767 	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);  in kvm_mmu_init_vm()
6769 		INIT_LIST_HEAD(&kvm->arch.possible_nx_huge_pages[i].pages);  in kvm_mmu_init_vm()
6770 	spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);  in kvm_mmu_init_vm()
6780 	kvm->arch.split_page_header_cache.kmem_cache = mmu_page_header_cache;  in kvm_mmu_init_vm()
6781 	kvm->arch.split_page_header_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_init_vm()
6783 	kvm->arch.split_shadow_page_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_init_vm()
6785 	kvm->arch.split_desc_cache.kmem_cache = pte_list_desc_cache;  in kvm_mmu_init_vm()
6786 	kvm->arch.split_desc_cache.gfp_zero = __GFP_ZERO;  in kvm_mmu_init_vm()
6792 	kvm_mmu_free_memory_cache(&kvm->arch.split_desc_cache);  in mmu_free_vm_memory_caches()
6793 	kvm_mmu_free_memory_cache(&kvm->arch.split_page_header_cache);  in mmu_free_vm_memory_caches()
6794 	kvm_mmu_free_memory_cache(&kvm->arch.split_shadow_page_cache);  in mmu_free_vm_memory_caches()
6799 	kvfree(kvm->arch.mmu_page_hash);  in kvm_mmu_uninit_vm()
6824 			start = max(gfn_start, memslot->base_gfn);  in kvm_rmap_zap_gfn_range()
6825 			end = min(gfn_end, memslot->base_gfn + memslot->npages);  in kvm_rmap_zap_gfn_range()
6848 	write_lock(&kvm->mmu_lock);  in kvm_zap_gfn_range()
6860 		kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);  in kvm_zap_gfn_range()
6864 	write_unlock(&kvm->mmu_lock);  in kvm_zap_gfn_range()
6879 		write_lock(&kvm->mmu_lock);  in kvm_mmu_slot_remove_write_access()
6882 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_remove_write_access()
6886 		read_lock(&kvm->mmu_lock);  in kvm_mmu_slot_remove_write_access()
6888 		read_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_remove_write_access()
6899 	if (need_resched() || rwlock_needbreak(&kvm->mmu_lock))  in need_topup_split_caches_or_resched()
6907 	return need_topup(&kvm->arch.split_desc_cache, SPLIT_DESC_CACHE_MIN_NR_OBJECTS) ||  in need_topup_split_caches_or_resched()
6908 	       need_topup(&kvm->arch.split_page_header_cache, 1) ||  in need_topup_split_caches_or_resched()
6909 	       need_topup(&kvm->arch.split_shadow_page_cache, 1);  in need_topup_split_caches_or_resched()
6921 	 * but aliasing rarely occurs post-boot or for many gfns.  If there is  in topup_split_caches()
6922 	 * only one rmap entry, rmap->val points directly at that one entry and  in topup_split_caches()
6931 	lockdep_assert_held(&kvm->slots_lock);  in topup_split_caches()
6933 	r = __kvm_mmu_topup_memory_cache(&kvm->arch.split_desc_cache, capacity,  in topup_split_caches()
6938 	r = kvm_mmu_topup_memory_cache(&kvm->arch.split_page_header_cache, 1);  in topup_split_caches()
6942 	return kvm_mmu_topup_memory_cache(&kvm->arch.split_shadow_page_cache, 1);  in topup_split_caches()
6965 	caches.page_header_cache = &kvm->arch.split_page_header_cache;  in shadow_mmu_get_sp_for_split()
6966 	caches.shadow_page_cache = &kvm->arch.split_shadow_page_cache;  in shadow_mmu_get_sp_for_split()
6977 	struct kvm_mmu_memory_cache *cache = &kvm->arch.split_desc_cache;  in shadow_mmu_split_huge_page()
6988 		sptep = &sp->spt[index];  in shadow_mmu_split_huge_page()
6995 		 * gfn-to-pfn translation since the SP is direct, so no need to  in shadow_mmu_split_huge_page()
7006 			flush |= !is_last_spte(*sptep, sp->role.level);  in shadow_mmu_split_huge_page()
7010 		spte = make_small_spte(kvm, huge_spte, sp->role, index);  in shadow_mmu_split_huge_page()
7012 		__rmap_add(kvm, cache, slot, sptep, gfn, sp->role.access);  in shadow_mmu_split_huge_page()
7029 	level = huge_sp->role.level;  in shadow_mmu_try_split_huge_page()
7033 		r = -ENOSPC;  in shadow_mmu_try_split_huge_page()
7038 		write_unlock(&kvm->mmu_lock);  in shadow_mmu_try_split_huge_page()
7041 		 * If the topup succeeds, return -EAGAIN to indicate that the  in shadow_mmu_try_split_huge_page()
7045 		r = topup_split_caches(kvm) ?: -EAGAIN;  in shadow_mmu_try_split_huge_page()
7046 		write_lock(&kvm->mmu_lock);  in shadow_mmu_try_split_huge_page()
7071 		if (WARN_ON_ONCE(!sp->role.guest_mode))  in shadow_mmu_try_split_huge_pages()
7074 		/* The rmaps should never contain non-leaf SPTEs. */  in shadow_mmu_try_split_huge_pages()
7079 		if (WARN_ON_ONCE(sp->unsync))  in shadow_mmu_try_split_huge_pages()
7083 		if (sp->role.invalid)  in shadow_mmu_try_split_huge_pages()
7093 		if (!r || r == -EAGAIN)  in shadow_mmu_try_split_huge_pages()
7096 		/* The split failed and shouldn't be retried (e.g. -ENOMEM). */  in shadow_mmu_try_split_huge_pages()
7116 	for (level = KVM_MAX_HUGEPAGE_LEVEL; level > target_level; level--)  in kvm_shadow_mmu_try_split_huge_pages()
7118 				  level, level, start, end - 1, true, true, false);  in kvm_shadow_mmu_try_split_huge_pages()
7121 /* Must be called with the mmu_lock held in write-mode. */
7145 	u64 start = memslot->base_gfn;  in kvm_mmu_slot_try_split_huge_pages()
7146 	u64 end = start + memslot->npages;  in kvm_mmu_slot_try_split_huge_pages()
7152 		write_lock(&kvm->mmu_lock);  in kvm_mmu_slot_try_split_huge_pages()
7154 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_try_split_huge_pages()
7157 	read_lock(&kvm->mmu_lock);  in kvm_mmu_slot_try_split_huge_pages()
7159 	read_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_try_split_huge_pages()
7163 	 * write-protecting and/or clearing dirty on the newly split SPTEs to  in kvm_mmu_slot_try_split_huge_pages()
7192 		if (sp->role.direct &&  in kvm_mmu_zap_collapsible_spte()
7193 		    sp->role.level < kvm_mmu_max_mapping_level(kvm, NULL, slot, sp->gfn)) {  in kvm_mmu_zap_collapsible_spte()
7213 	 * Note, use KVM_MAX_HUGEPAGE_LEVEL - 1 since there's no need to zap  in kvm_rmap_zap_collapsible_sptes()
7217 			    PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL - 1, true))  in kvm_rmap_zap_collapsible_sptes()
7225 		write_lock(&kvm->mmu_lock);  in kvm_mmu_recover_huge_pages()
7227 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_recover_huge_pages()
7231 		read_lock(&kvm->mmu_lock);  in kvm_mmu_recover_huge_pages()
7233 		read_unlock(&kvm->mmu_lock);  in kvm_mmu_recover_huge_pages()
7241 		write_lock(&kvm->mmu_lock);  in kvm_mmu_slot_leaf_clear_dirty()
7247 		write_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_leaf_clear_dirty()
7251 		read_lock(&kvm->mmu_lock);  in kvm_mmu_slot_leaf_clear_dirty()
7253 		read_unlock(&kvm->mmu_lock);  in kvm_mmu_slot_leaf_clear_dirty()
7272 	write_lock(&kvm->mmu_lock);  in kvm_mmu_zap_all()
7274 	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {  in kvm_mmu_zap_all()
7275 		if (WARN_ON_ONCE(sp->role.invalid))  in kvm_mmu_zap_all()
7279 		if (cond_resched_rwlock_write(&kvm->mmu_lock))  in kvm_mmu_zap_all()
7288 	write_unlock(&kvm->mmu_lock);  in kvm_mmu_zap_all()
7303 	if (list_empty(&kvm->arch.active_mmu_pages))  in kvm_mmu_zap_memslot_pages_and_flush()
7310 	 * will result in use-after-free, e.g. in unaccount_shadowed().  in kvm_mmu_zap_memslot_pages_and_flush()
7312 	for (i = 0; i < slot->npages; i++) {  in kvm_mmu_zap_memslot_pages_and_flush()
7314 		gfn_t gfn = slot->base_gfn + i;  in kvm_mmu_zap_memslot_pages_and_flush()
7319 		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {  in kvm_mmu_zap_memslot_pages_and_flush()
7322 			cond_resched_rwlock_write(&kvm->mmu_lock);  in kvm_mmu_zap_memslot_pages_and_flush()
7335 		.start = slot->base_gfn,  in kvm_mmu_zap_memslot()
7336 		.end = slot->base_gfn + slot->npages,  in kvm_mmu_zap_memslot()
7342 	write_lock(&kvm->mmu_lock);  in kvm_mmu_zap_memslot()
7345 	write_unlock(&kvm->mmu_lock);  in kvm_mmu_zap_memslot()
7350 	return kvm->arch.vm_type == KVM_X86_DEFAULT_VM &&  in kvm_memslot_flush_zap_all()
7376 	gen &= ~((u64)kvm_arch_nr_memslot_as_ids(kvm) - 1);  in kvm_mmu_invalidate_mmio_sptes()
7397 	 * The NX recovery thread is spawned on-demand at the first KVM_RUN and  in kvm_wake_nx_recovery_thread()
7398 	 * may not be valid even though the VM is globally visible.  Do nothing,  in kvm_wake_nx_recovery_thread()
7401 	struct vhost_task *nx_thread = READ_ONCE(kvm->arch.nx_huge_page_recovery_thread);  in kvm_wake_nx_recovery_thread()
7432 		return -EPERM;  in set_nx_huge_pages()
7434 	/* In "auto" mode deploy workaround only if CPU has the bug. */  in set_nx_huge_pages()
7447 			return -EBUSY;  in set_nx_huge_pages()
7452 		return -EINVAL;  in set_nx_huge_pages()
7463 			mutex_lock(&kvm->slots_lock);  in set_nx_huge_pages()
7465 			mutex_unlock(&kvm->slots_lock);  in set_nx_huge_pages()
7477  * its default value of -1 is technically undefined behavior for a boolean.
7483 	if (nx_huge_pages == -1)  in kvm_mmu_x86_module_init()
7503 	int ret = -ENOMEM;  in kvm_mmu_vendor_module_init()
7538 		read_lock(&vcpu->kvm->mmu_lock);  in kvm_mmu_destroy()
7539 		mmu_free_root_page(vcpu->kvm, &vcpu->arch.mmu->mirror_root_hpa,  in kvm_mmu_destroy()
7541 		read_unlock(&vcpu->kvm->mmu_lock);  in kvm_mmu_destroy()
7543 	free_mmu_pages(&vcpu->arch.root_mmu);  in kvm_mmu_destroy()
7544 	free_mmu_pages(&vcpu->arch.guest_mmu);  in kvm_mmu_destroy()
7585 		return -EPERM;  in set_nx_huge_pages_recovery_param()
7613 	unsigned long pages = READ_ONCE(kvm->arch.possible_nx_huge_pages[mmu_type].nr_pages);  in nx_huge_pages_to_zap()
7629 	 * kvm->nr_memslots_dirty_logging is not a problem: if it is becoming  in kvm_mmu_sp_dirty_logging_enabled()
7634 	if (!atomic_read(&kvm->nr_memslots_dirty_logging))  in kvm_mmu_sp_dirty_logging_enabled()
7637 	slot = __gfn_to_memslot(kvm_memslots_for_spte_role(kvm, sp->role), sp->gfn);  in kvm_mmu_sp_dirty_logging_enabled()
7649 	spinlock_t *tdp_mmu_pages_lock = &kvm->arch.tdp_mmu_pages_lock;  in kvm_recover_nx_huge_pages()
7661 	nx_huge_pages = &kvm->arch.possible_nx_huge_pages[mmu_type].pages;  in kvm_recover_nx_huge_pages()
7663 	rcu_idx = srcu_read_lock(&kvm->srcu);  in kvm_recover_nx_huge_pages()
7665 		read_lock(&kvm->mmu_lock);  in kvm_recover_nx_huge_pages()
7667 		write_lock(&kvm->mmu_lock);  in kvm_recover_nx_huge_pages()
7676 	for ( ; to_zap; --to_zap) {  in kvm_recover_nx_huge_pages()
7696 		WARN_ON_ONCE(!sp->nx_huge_page_disallowed);  in kvm_recover_nx_huge_pages()
7697 		WARN_ON_ONCE(!sp->role.direct);  in kvm_recover_nx_huge_pages()
7719 		WARN_ON_ONCE(sp->nx_huge_page_disallowed);  in kvm_recover_nx_huge_pages()
7721 		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {  in kvm_recover_nx_huge_pages()
7726 				cond_resched_rwlock_read(&kvm->mmu_lock);  in kvm_recover_nx_huge_pages()
7728 				cond_resched_rwlock_write(&kvm->mmu_lock);  in kvm_recover_nx_huge_pages()
7739 		read_unlock(&kvm->mmu_lock);  in kvm_recover_nx_huge_pages()
7741 		write_unlock(&kvm->mmu_lock);  in kvm_recover_nx_huge_pages()
7742 	srcu_read_unlock(&kvm->srcu, rcu_idx);  in kvm_recover_nx_huge_pages()
7761 	remaining_time = kvm->arch.nx_huge_page_last + msecs_to_jiffies(period)  in kvm_nx_huge_page_recovery_worker()
7762 		- get_jiffies_64();  in kvm_nx_huge_page_recovery_worker()
7772 	kvm->arch.nx_huge_page_last = get_jiffies_64();  in kvm_nx_huge_page_recovery_worker()
7782 	kvm->arch.nx_huge_page_last = get_jiffies_64();  in kvm_mmu_start_lpage_recovery()
7785 				      kvm, "kvm-nx-lpage-recovery");  in kvm_mmu_start_lpage_recovery()
7792 	/* Make the task visible only once it is fully started. */  in kvm_mmu_start_lpage_recovery()
7793 	WRITE_ONCE(kvm->arch.nx_huge_page_recovery_thread, nx_thread);  in kvm_mmu_start_lpage_recovery()
7802 	return call_once(&kvm->arch.nx_once, kvm_mmu_start_lpage_recovery);  in kvm_mmu_post_init_vm()
7807 	if (kvm->arch.nx_huge_page_recovery_thread)  in kvm_mmu_pre_destroy_vm()
7808 		vhost_task_stop(kvm->arch.nx_huge_page_recovery_thread);  in kvm_mmu_pre_destroy_vm()
7815 	return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG;  in hugepage_test_mixed()
7821 	lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG;  in hugepage_clear_mixed()
7827 	lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG;  in hugepage_set_mixed()
7833 	struct kvm_memory_slot *slot = range->slot;  in kvm_arch_pre_set_memory_attributes()
7850 	if (WARN_ON_ONCE(range->end <= range->start))  in kvm_arch_pre_set_memory_attributes()
7863 		gfn_t start = gfn_round_for_level(range->start, level);  in kvm_arch_pre_set_memory_attributes()
7864 		gfn_t end = gfn_round_for_level(range->end - 1, level);  in kvm_arch_pre_set_memory_attributes()
7867 		if ((start != range->start || start + nr_pages > range->end) &&  in kvm_arch_pre_set_memory_attributes()
7868 		    start >= slot->base_gfn &&  in kvm_arch_pre_set_memory_attributes()
7869 		    start + nr_pages <= slot->base_gfn + slot->npages &&  in kvm_arch_pre_set_memory_attributes()
7876 		if ((end + nr_pages) > range->end &&  in kvm_arch_pre_set_memory_attributes()
7877 		    (end + nr_pages) <= (slot->base_gfn + slot->npages) &&  in kvm_arch_pre_set_memory_attributes()
7883 	if (range->arg.attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE)  in kvm_arch_pre_set_memory_attributes()
7884 		range->attr_filter = KVM_FILTER_SHARED;  in kvm_arch_pre_set_memory_attributes()
7886 		range->attr_filter = KVM_FILTER_PRIVATE;  in kvm_arch_pre_set_memory_attributes()
7902 	for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) {  in hugepage_has_attrs()
7903 		if (hugepage_test_mixed(slot, gfn, level - 1) ||  in hugepage_has_attrs()
7913 	unsigned long attrs = range->arg.attributes;  in kvm_arch_post_set_memory_attributes()
7914 	struct kvm_memory_slot *slot = range->slot;  in kvm_arch_post_set_memory_attributes()
7917 	lockdep_assert_held_write(&kvm->mmu_lock);  in kvm_arch_post_set_memory_attributes()
7918 	lockdep_assert_held(&kvm->slots_lock);  in kvm_arch_post_set_memory_attributes()
7935 		gfn_t gfn = gfn_round_for_level(range->start, level);  in kvm_arch_post_set_memory_attributes()
7938 		if (gfn != range->start || gfn + nr_pages > range->end) {  in kvm_arch_post_set_memory_attributes()
7944 			if (gfn >= slot->base_gfn &&  in kvm_arch_post_set_memory_attributes()
7945 			    gfn + nr_pages <= slot->base_gfn + slot->npages) {  in kvm_arch_post_set_memory_attributes()
7958 		for ( ; gfn + nr_pages <= range->end; gfn += nr_pages)  in kvm_arch_post_set_memory_attributes()
7966 		if (gfn < range->end &&  in kvm_arch_post_set_memory_attributes()
7967 		    (gfn + nr_pages) <= (slot->base_gfn + slot->npages)) {  in kvm_arch_post_set_memory_attributes()
7991 		gfn_t end = gfn_round_for_level(slot->base_gfn + slot->npages, level);  in kvm_mmu_init_memslot_memory_attributes()
7992 		gfn_t start = gfn_round_for_level(slot->base_gfn, level);  in kvm_mmu_init_memslot_memory_attributes()
7996 		if (start < slot->base_gfn)  in kvm_mmu_init_memslot_memory_attributes()