1 // SPDX-License-Identifier: GPL-2.0
17 	INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);  in kvm_mmu_init_tdp_mmu()
18 	spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);  in kvm_mmu_init_tdp_mmu()
26 		lockdep_assert_held_read(&kvm->mmu_lock);  in kvm_lockdep_assert_mmu_lock_held()
28 		lockdep_assert_held_write(&kvm->mmu_lock);  in kvm_lockdep_assert_mmu_lock_held()
44 	KVM_MMU_WARN_ON(atomic64_read(&kvm->arch.tdp_mmu_pages));  in kvm_mmu_uninit_tdp_mmu()
46 	WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));  in kvm_mmu_uninit_tdp_mmu()
58 	free_page((unsigned long)sp->external_spt);  in tdp_mmu_free_sp()
59 	free_page((unsigned long)sp->spt);  in tdp_mmu_free_sp()
81 	if (!refcount_dec_and_test(&root->tdp_mmu_root_count))  in kvm_tdp_mmu_put_root()
89 	KVM_BUG_ON(!is_tdp_mmu_page(root) || !root->role.invalid, kvm);  in kvm_tdp_mmu_put_root()
91 	spin_lock(&kvm->arch.tdp_mmu_pages_lock);  in kvm_tdp_mmu_put_root()
92 	list_del_rcu(&root->link);  in kvm_tdp_mmu_put_root()
93 	spin_unlock(&kvm->arch.tdp_mmu_pages_lock);  in kvm_tdp_mmu_put_root()
94 	call_rcu(&root->rcu_head, tdp_mmu_free_sp_rcu_callback);  in kvm_tdp_mmu_put_root()
103 	if (root->role.invalid && !(types & KVM_INVALID_ROOTS))  in tdp_mmu_root_match()
115  * must hold a reference to @prev_root if it's non-NULL).
128 	 * While the roots themselves are RCU-protected, fields such as  in tdp_mmu_next_root()
131 	lockdep_assert_held(&kvm->mmu_lock);  in tdp_mmu_next_root()
136 		next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots,  in tdp_mmu_next_root()
137 						  &prev_root->link,  in tdp_mmu_next_root()
140 		next_root = list_first_or_null_rcu(&kvm->arch.tdp_mmu_roots,  in tdp_mmu_next_root()
148 		next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots,  in tdp_mmu_next_root()
149 				&next_root->link, typeof(*next_root), link);  in tdp_mmu_next_root()
171 	     ({ lockdep_assert_held(&(_kvm)->mmu_lock); }), _root;		\
181 	     ({ lockdep_assert_held(&(_kvm)->mmu_lock); }), _root;	\
187  * inherently yield-friendly and should use the yield-safe variant above.
192 	list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link)		\
199  * Iterate over all TDP MMU roots in an RCU read-side critical section.
206 	list_for_each_entry_rcu(_root, &_kvm->arch.tdp_mmu_roots, link)		\
218 	sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);  in tdp_mmu_alloc_sp()
219 	sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache);  in tdp_mmu_alloc_sp()
227 	INIT_LIST_HEAD(&sp->possible_nx_huge_page_link);  in tdp_mmu_init_sp()
229 	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);  in tdp_mmu_init_sp()
231 	sp->role = role;  in tdp_mmu_init_sp()
232 	sp->gfn = gfn;  in tdp_mmu_init_sp()
233 	sp->ptep = sptep;  in tdp_mmu_init_sp()
234 	sp->tdp_mmu_page = true;  in tdp_mmu_init_sp()
245 	parent_sp = sptep_to_sp(rcu_dereference(iter->sptep));  in tdp_mmu_init_child_sp()
247 	role = parent_sp->role;  in tdp_mmu_init_child_sp()
248 	role.level--;  in tdp_mmu_init_child_sp()
250 	tdp_mmu_init_sp(child_sp, iter->sptep, iter->gfn, role);  in tdp_mmu_init_child_sp()
255 	struct kvm_mmu *mmu = vcpu->arch.mmu;  in kvm_tdp_mmu_alloc_root()
256 	union kvm_mmu_page_role role = mmu->root_role;  in kvm_tdp_mmu_alloc_root()
258 	struct kvm *kvm = vcpu->kvm;  in kvm_tdp_mmu_alloc_root()
270 	read_lock(&kvm->mmu_lock);  in kvm_tdp_mmu_alloc_root()
273 		if (root->role.word == role.word)  in kvm_tdp_mmu_alloc_root()
277 	spin_lock(&kvm->arch.tdp_mmu_pages_lock);  in kvm_tdp_mmu_alloc_root()
287 	list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {  in kvm_tdp_mmu_alloc_root()
288 		if (root->role.word == role.word &&  in kvm_tdp_mmu_alloc_root()
303 	refcount_set(&root->tdp_mmu_root_count, 2);  in kvm_tdp_mmu_alloc_root()
304 	list_add_rcu(&root->link, &kvm->arch.tdp_mmu_roots);  in kvm_tdp_mmu_alloc_root()
307 	spin_unlock(&kvm->arch.tdp_mmu_pages_lock);  in kvm_tdp_mmu_alloc_root()
309 	read_unlock(&kvm->mmu_lock);  in kvm_tdp_mmu_alloc_root()
316 		mmu->mirror_root_hpa = __pa(root->spt);  in kvm_tdp_mmu_alloc_root()
318 		mmu->root.hpa = __pa(root->spt);  in kvm_tdp_mmu_alloc_root()
319 		mmu->root.pgd = 0;  in kvm_tdp_mmu_alloc_root()
329 	kvm_account_pgtable_pages((void *)sp->spt, +1);  in tdp_account_mmu_page()
331 	atomic64_inc(&kvm->arch.tdp_mmu_pages);  in tdp_account_mmu_page()
337 	kvm_account_pgtable_pages((void *)sp->spt, -1);  in tdp_unaccount_mmu_page()
339 	atomic64_dec(&kvm->arch.tdp_mmu_pages);  in tdp_unaccount_mmu_page()
344  * tdp_mmu_unlink_sp() - Remove a shadow page from the list of used pages
353 	if (!sp->nx_huge_page_disallowed)  in tdp_mmu_unlink_sp()
356 	spin_lock(&kvm->arch.tdp_mmu_pages_lock);  in tdp_mmu_unlink_sp()
357 	sp->nx_huge_page_disallowed = false;  in tdp_mmu_unlink_sp()
359 	spin_unlock(&kvm->arch.tdp_mmu_pages_lock);  in tdp_mmu_unlink_sp()
371 	 * But remove_external_spte() will be called on non-leaf PTEs via  in remove_external_spte()
379 	lockdep_assert_held_write(&kvm->mmu_lock);  in remove_external_spte()
386  * handle_removed_pt() - handle a page table removed from the TDP structure
405 	int level = sp->role.level;  in handle_removed_pt()
406 	gfn_t base_gfn = sp->gfn;  in handle_removed_pt()
435 			 * If the SPTE is not MMU-present, there is no backing  in handle_removed_pt()
451 			 * Even though the parent SPTE is !PRESENT, the TLB  in handle_removed_pt()
453 			 * document that A/D assists can use upper-level PxE  in handle_removed_pt()
454 			 * entries that are cached in the TLB, i.e. the CPU can  in handle_removed_pt()
467 			 * scenario where a non-atomic update could result in a  in handle_removed_pt()
488 	    WARN_ON(kvm_x86_call(free_external_spt)(kvm, base_gfn, sp->role.level,  in handle_removed_pt()
489 						    sp->external_spt))) {  in handle_removed_pt()
496 		sp->external_spt = NULL;  in handle_removed_pt()
499 	call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback);  in handle_removed_pt()
507 		WARN_ON_ONCE(sp->role.level + 1 != level);  in get_external_spt()
508 		WARN_ON_ONCE(sp->gfn != gfn);  in get_external_spt()
509 		return sp->external_spt;  in get_external_spt()
527 	lockdep_assert_held(&kvm->mmu_lock);  in set_external_spte_present()
534 		return -EBUSY;  in set_external_spte_present()
556  * handle_changed_spte - handle bookkeeping associated with an SPTE change
583 	WARN_ON_ONCE(gfn & (KVM_PAGES_PER_HPAGE(level) - 1));  in handle_changed_spte()
598 		       "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",  in handle_changed_spte()
617 	 * The only times a SPTE should be changed from a non-present to  in handle_changed_spte()
618 	 * non-present state is when an MMIO entry is installed/modified/  in handle_changed_spte()
636 			       "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",  in handle_changed_spte()
642 		kvm_update_page_stats(kvm, level, is_leaf ? 1 : -1);  in handle_changed_spte()
662 	 * and pre-checking before inserting a new SPTE is advantageous as it  in __tdp_mmu_set_spte_atomic()
665 	WARN_ON_ONCE(iter->yielded || is_frozen_spte(iter->old_spte));  in __tdp_mmu_set_spte_atomic()
667 	if (is_mirror_sptep(iter->sptep) && !is_frozen_spte(new_spte)) {  in __tdp_mmu_set_spte_atomic()
675 			return -EBUSY;  in __tdp_mmu_set_spte_atomic()
677 		ret = set_external_spte_present(kvm, iter->sptep, iter->gfn,  in __tdp_mmu_set_spte_atomic()
678 						iter->old_spte, new_spte, iter->level);  in __tdp_mmu_set_spte_atomic()
682 		u64 *sptep = rcu_dereference(iter->sptep);  in __tdp_mmu_set_spte_atomic()
688 		 * updates iter->old_spte with the current value, so the caller  in __tdp_mmu_set_spte_atomic()
692 		if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte))  in __tdp_mmu_set_spte_atomic()
693 			return -EBUSY;  in __tdp_mmu_set_spte_atomic()
700  * tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically
704  * If setting the SPTE fails because it has changed, iter->old_spte will be
711  * * 0      - If the SPTE was set.
712  * * -EBUSY - If the SPTE cannot be set. In this case this function will have
713  *            no side-effects other than setting iter->old_spte to the last
722 	lockdep_assert_held_read(&kvm->mmu_lock);  in tdp_mmu_set_spte_atomic()
728 	handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte,  in tdp_mmu_set_spte_atomic()
729 			    new_spte, iter->level, true);  in tdp_mmu_set_spte_atomic()
735  * tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping
750 	lockdep_assert_held_write(&kvm->mmu_lock);  in tdp_mmu_set_spte()
766 	 * Users that do non-atomic setting of PTEs don't operate on mirror  in tdp_mmu_set_spte()
780 	WARN_ON_ONCE(iter->yielded);  in tdp_mmu_iter_set_spte()
781 	iter->old_spte = tdp_mmu_set_spte(kvm, iter->as_id, iter->sptep,  in tdp_mmu_iter_set_spte()
782 					  iter->old_spte, new_spte,  in tdp_mmu_iter_set_spte()
783 					  iter->gfn, iter->level);  in tdp_mmu_iter_set_spte()
799 	if (!need_resched() && !rwlock_needbreak(&kvm->mmu_lock))  in tdp_mmu_iter_need_resched()
803 	return iter->next_last_level_gfn != iter->yielded_gfn;  in tdp_mmu_iter_need_resched()
811  * TLB flush before yielding.
813  * If this function yields, iter->yielded is set and the caller must skip to
824 	KVM_MMU_WARN_ON(iter->yielded);  in tdp_mmu_iter_cond_resched()
835 		cond_resched_rwlock_read(&kvm->mmu_lock);  in tdp_mmu_iter_cond_resched()
837 		cond_resched_rwlock_write(&kvm->mmu_lock);  in tdp_mmu_iter_cond_resched()
841 	WARN_ON_ONCE(iter->gfn > iter->next_last_level_gfn);  in tdp_mmu_iter_cond_resched()
843 	iter->yielded = true;  in tdp_mmu_iter_cond_resched()
892 	 * and lead to use-after-free as zapping a SPTE triggers "writeback" of  in tdp_mmu_zap_root()
895 	WARN_ON_ONCE(!refcount_read(&root->tdp_mmu_root_count));  in tdp_mmu_zap_root()
904 	 * preempt models) or mmu_lock contention (full or real-time models).  in tdp_mmu_zap_root()
923 	__tdp_mmu_zap_root(kvm, root, shared, root->role.level);  in tdp_mmu_zap_root()
932 		.old_spte = sp->ptep ? kvm_tdp_mmu_read_spte(sp->ptep) : 0,  in kvm_tdp_mmu_zap_possible_nx_huge_page()
933 		.sptep = sp->ptep,  in kvm_tdp_mmu_zap_possible_nx_huge_page()
934 		.level = sp->role.level + 1,  in kvm_tdp_mmu_zap_possible_nx_huge_page()
935 		.gfn = sp->gfn,  in kvm_tdp_mmu_zap_possible_nx_huge_page()
939 	lockdep_assert_held_read(&kvm->mmu_lock);  in kvm_tdp_mmu_zap_possible_nx_huge_page()
948 	if (WARN_ON_ONCE(!sp->ptep))  in kvm_tdp_mmu_zap_possible_nx_huge_page()
954 	 * points at the SP's page table, as checking shadow-present isn't  in kvm_tdp_mmu_zap_possible_nx_huge_page()
957 	 * shadow-present, i.e. guards against zapping a frozen SPTE.  in kvm_tdp_mmu_zap_possible_nx_huge_page()
959 	if ((tdp_ptep_t)sp->spt != spte_to_child_pt(iter.old_spte, iter.level))  in kvm_tdp_mmu_zap_possible_nx_huge_page()
964 	 * for the SPTE to still be used for the to-be-zapped SP. Non-leaf  in kvm_tdp_mmu_zap_possible_nx_huge_page()
965 	 * SPTEs don't have Dirty bits, KVM always sets the Accessed bit when  in kvm_tdp_mmu_zap_possible_nx_huge_page()
966 	 * creating non-leaf SPTEs, and all other bits are immutable for non-  in kvm_tdp_mmu_zap_possible_nx_huge_page()
967 	 * leaf SPTEs, i.e. the only legal operations for non-leaf SPTEs are  in kvm_tdp_mmu_zap_possible_nx_huge_page()
971 		WARN_ON_ONCE((tdp_ptep_t)sp->spt == spte_to_child_pt(iter.old_spte, iter.level));  in kvm_tdp_mmu_zap_possible_nx_huge_page()
992 	lockdep_assert_held_write(&kvm->mmu_lock);  in tdp_mmu_zap_leafs()
1010 		 * Zappings SPTEs in invalid roots doesn't require a TLB flush,  in tdp_mmu_zap_leafs()
1013 		if (!root->role.invalid)  in tdp_mmu_zap_leafs()
1028  * Returns true if a TLB flush is needed before releasing the MMU lock, i.e. if
1035 	lockdep_assert_held_write(&kvm->mmu_lock);  in kvm_tdp_mmu_zap_leafs()
1036 	for_each_valid_tdp_mmu_root_yield_safe(kvm, root, -1)  in kvm_tdp_mmu_zap_leafs()
1052 	 * "fast zap".  Walking zapped top-level SPTEs isn't all that expensive  in kvm_tdp_mmu_zap_all()
1055 	 * A TLB flush is unnecessary, KVM zaps everything if and only the VM  in kvm_tdp_mmu_zap_all()
1059 	lockdep_assert_held_write(&kvm->mmu_lock);  in kvm_tdp_mmu_zap_all()
1060 	__for_each_tdp_mmu_root_yield_safe(kvm, root, -1,  in kvm_tdp_mmu_zap_all()
1074 		read_lock(&kvm->mmu_lock);  in kvm_tdp_mmu_zap_invalidated_roots()
1076 		write_lock(&kvm->mmu_lock);  in kvm_tdp_mmu_zap_invalidated_roots()
1079 		if (!root->tdp_mmu_scheduled_root_to_zap)  in kvm_tdp_mmu_zap_invalidated_roots()
1082 		root->tdp_mmu_scheduled_root_to_zap = false;  in kvm_tdp_mmu_zap_invalidated_roots()
1083 		KVM_BUG_ON(!root->role.invalid, kvm);  in kvm_tdp_mmu_zap_invalidated_roots()
1086 		 * A TLB flush is not necessary as KVM performs a local TLB  in kvm_tdp_mmu_zap_invalidated_roots()
1089 		 * TLB flush on reuse also invalidates paging-structure-cache  in kvm_tdp_mmu_zap_invalidated_roots()
1090 		 * entries, i.e. TLB entries for intermediate paging structures,  in kvm_tdp_mmu_zap_invalidated_roots()
1105 		read_unlock(&kvm->mmu_lock);  in kvm_tdp_mmu_zap_invalidated_roots()
1107 		write_unlock(&kvm->mmu_lock);  in kvm_tdp_mmu_zap_invalidated_roots()
1144 	    refcount_read(&kvm->users_count) && kvm->created_vcpus)  in kvm_tdp_mmu_invalidate_roots()
1145 		lockdep_assert_held_write(&kvm->mmu_lock);  in kvm_tdp_mmu_invalidate_roots()
1152 	list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {  in kvm_tdp_mmu_invalidate_roots()
1162 		if (!root->role.invalid) {  in kvm_tdp_mmu_invalidate_roots()
1163 			root->tdp_mmu_scheduled_root_to_zap = true;  in kvm_tdp_mmu_invalidate_roots()
1164 			root->role.invalid = true;  in kvm_tdp_mmu_invalidate_roots()
1170  * Installs a last-level SPTE to handle a TDP page fault.
1177 	struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep));  in tdp_mmu_map_handle_target_level()
1182 	if (WARN_ON_ONCE(sp->role.level != fault->goal_level))  in tdp_mmu_map_handle_target_level()
1185 	if (is_shadow_present_pte(iter->old_spte) &&  in tdp_mmu_map_handle_target_level()
1186 	    (fault->prefetch || is_access_allowed(fault, iter->old_spte)) &&  in tdp_mmu_map_handle_target_level()
1187 	    is_last_spte(iter->old_spte, iter->level)) {  in tdp_mmu_map_handle_target_level()
1188 		WARN_ON_ONCE(fault->pfn != spte_to_pfn(iter->old_spte));  in tdp_mmu_map_handle_target_level()
1192 	if (unlikely(!fault->slot))  in tdp_mmu_map_handle_target_level()
1193 		new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL);  in tdp_mmu_map_handle_target_level()
1195 		wrprot = make_spte(vcpu, sp, fault->slot, ACC_ALL, iter->gfn,  in tdp_mmu_map_handle_target_level()
1196 				   fault->pfn, iter->old_spte, fault->prefetch,  in tdp_mmu_map_handle_target_level()
1197 				   false, fault->map_writable, &new_spte);  in tdp_mmu_map_handle_target_level()
1199 	if (new_spte == iter->old_spte)  in tdp_mmu_map_handle_target_level()
1201 	else if (tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte))  in tdp_mmu_map_handle_target_level()
1203 	else if (is_shadow_present_pte(iter->old_spte) &&  in tdp_mmu_map_handle_target_level()
1204 		 (!is_last_spte(iter->old_spte, iter->level) ||  in tdp_mmu_map_handle_target_level()
1205 		  WARN_ON_ONCE(leaf_spte_change_needs_tlb_flush(iter->old_spte, new_spte))))  in tdp_mmu_map_handle_target_level()
1206 		kvm_flush_remote_tlbs_gfn(vcpu->kvm, iter->gfn, iter->level);  in tdp_mmu_map_handle_target_level()
1213 	if (wrprot && fault->write)  in tdp_mmu_map_handle_target_level()
1217 	if (unlikely(is_mmio_spte(vcpu->kvm, new_spte))) {  in tdp_mmu_map_handle_target_level()
1218 		vcpu->stat.pf_mmio_spte_created++;  in tdp_mmu_map_handle_target_level()
1219 		trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn,  in tdp_mmu_map_handle_target_level()
1223 		trace_kvm_mmu_set_spte(iter->level, iter->gfn,  in tdp_mmu_map_handle_target_level()
1224 				       rcu_dereference(iter->sptep));  in tdp_mmu_map_handle_target_level()
1231  * tdp_mmu_link_sp - Replace the given spte with an spte pointing to the
1239  * Returns: 0 if the new page table was installed. Non-0 if the page table
1240  *          could not be installed (e.g. the atomic compare-exchange failed).
1245 	u64 spte = make_nonleaf_spte(sp->spt, !kvm_ad_enabled);  in tdp_mmu_link_sp()
1271 	struct kvm *kvm = vcpu->kvm;  in kvm_tdp_mmu_map()
1282 	for_each_tdp_pte(iter, kvm, root, fault->gfn, fault->gfn + 1) {  in kvm_tdp_mmu_map()
1285 		if (fault->nx_huge_page_workaround_enabled)  in kvm_tdp_mmu_map()
1295 		if (iter.level == fault->goal_level)  in kvm_tdp_mmu_map()
1304 		 * The SPTE is either non-present or points to a huge page that  in kvm_tdp_mmu_map()
1312 		sp->nx_huge_page_disallowed = fault->huge_page_disallowed;  in kvm_tdp_mmu_map()
1316 			KVM_BUG_ON(is_mirror_sptep(iter.sptep), vcpu->kvm);  in kvm_tdp_mmu_map()
1331 		if (fault->huge_page_disallowed &&  in kvm_tdp_mmu_map()
1332 		    fault->req_level >= iter.level) {  in kvm_tdp_mmu_map()
1333 			spin_lock(&kvm->arch.tdp_mmu_pages_lock);  in kvm_tdp_mmu_map()
1334 			if (sp->nx_huge_page_disallowed)  in kvm_tdp_mmu_map()
1336 			spin_unlock(&kvm->arch.tdp_mmu_pages_lock);  in kvm_tdp_mmu_map()
1344 	WARN_ON_ONCE(iter.level == fault->goal_level);  in kvm_tdp_mmu_map()
1362 	types = kvm_gfn_range_filter_to_root_types(kvm, range->attr_filter) | KVM_INVALID_ROOTS;  in kvm_tdp_mmu_unmap_gfn_range()
1364 	__for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, types)  in kvm_tdp_mmu_unmap_gfn_range()
1365 		flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end,  in kvm_tdp_mmu_unmap_gfn_range()
1366 					  range->may_block, flush);  in kvm_tdp_mmu_unmap_gfn_range()
1372  * Mark the SPTEs range of GFNs [start, end) unaccessed and return non-zero
1383 	if (spte_ad_enabled(iter->old_spte)) {  in kvm_tdp_mmu_age_spte()
1384 		iter->old_spte = tdp_mmu_clear_spte_bits_atomic(iter->sptep,  in kvm_tdp_mmu_age_spte()
1386 		new_spte = iter->old_spte & ~shadow_accessed_mask;  in kvm_tdp_mmu_age_spte()
1388 		new_spte = mark_spte_for_access_track(iter->old_spte);  in kvm_tdp_mmu_age_spte()
1397 	trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level,  in kvm_tdp_mmu_age_spte()
1398 				       iter->old_spte, new_spte);  in kvm_tdp_mmu_age_spte()
1410 	types = kvm_gfn_range_filter_to_root_types(kvm, range->attr_filter);  in __kvm_tdp_mmu_age_gfn_range()
1414 	 * into this helper allow blocking; it'd be dead, wasteful code.  Note,  in __kvm_tdp_mmu_age_gfn_range()
1421 	for_each_tdp_mmu_root_rcu(kvm, root, range->slot->as_id, types) {  in __kvm_tdp_mmu_age_gfn_range()
1422 		tdp_root_for_each_leaf_pte(iter, kvm, root, range->start, range->end) {  in __kvm_tdp_mmu_age_gfn_range()
1496 	lockdep_assert_held_read(&kvm->mmu_lock);  in kvm_tdp_mmu_wrprot_slot()
1498 	for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id)  in kvm_tdp_mmu_wrprot_slot()
1499 		spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn,  in kvm_tdp_mmu_wrprot_slot()
1500 			     slot->base_gfn + slot->npages, min_level);  in kvm_tdp_mmu_wrprot_slot()
1513 	sp->spt = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);  in tdp_mmu_alloc_sp_for_split()
1514 	if (!sp->spt) {  in tdp_mmu_alloc_sp_for_split()
1526 	const u64 huge_spte = iter->old_spte;  in tdp_mmu_split_huge_page()
1527 	const int level = iter->level;  in tdp_mmu_split_huge_page()
1531 	 * No need for atomics when writing to sp->spt since the page table has  in tdp_mmu_split_huge_page()
1535 		sp->spt[i] = make_small_spte(kvm, huge_spte, sp->role, i);  in tdp_mmu_split_huge_page()
1539 	 * page table. Since we are making this change without a TLB flush vCPUs  in tdp_mmu_split_huge_page()
1541 	 * depending on what's currently in their TLB. This is fine from a  in tdp_mmu_split_huge_page()
1554 	kvm_update_page_stats(kvm, level - 1, SPTE_ENT_PER_PAGE);  in tdp_mmu_split_huge_page()
1557 	trace_kvm_mmu_split_huge_page(iter->gfn, huge_spte, level, ret);  in tdp_mmu_split_huge_page()
1576 	 * Since the TDP iterator uses a pre-order traversal, we are guaranteed  in tdp_mmu_split_huge_pages_root()
1594 				read_unlock(&kvm->mmu_lock);  in tdp_mmu_split_huge_pages_root()
1596 				write_unlock(&kvm->mmu_lock);  in tdp_mmu_split_huge_pages_root()
1601 				read_lock(&kvm->mmu_lock);  in tdp_mmu_split_huge_pages_root()
1603 				write_lock(&kvm->mmu_lock);  in tdp_mmu_split_huge_pages_root()
1608 							      iter.level, -ENOMEM);  in tdp_mmu_split_huge_pages_root()
1609 				return -ENOMEM;  in tdp_mmu_split_huge_pages_root()
1652 	for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id) {  in kvm_tdp_mmu_try_split_huge_pages()
1703  * Clear the dirty status (D-bit or W-bit) of all the SPTEs mapping GFNs in the
1711 	lockdep_assert_held_read(&kvm->mmu_lock);  in kvm_tdp_mmu_clear_dirty_slot()
1712 	for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id)  in kvm_tdp_mmu_clear_dirty_slot()
1713 		clear_dirty_gfn_range(kvm, root, slot->base_gfn,  in kvm_tdp_mmu_clear_dirty_slot()
1714 				      slot->base_gfn + slot->npages);  in kvm_tdp_mmu_clear_dirty_slot()
1724 	lockdep_assert_held_write(&kvm->mmu_lock);  in clear_dirty_pt_masked()
1737 		    !(mask & (1UL << (iter.gfn - gfn))))  in clear_dirty_pt_masked()
1740 		mask &= ~(1UL << (iter.gfn - gfn));  in clear_dirty_pt_masked()
1758  * Clear the dirty status (D-bit or W-bit) of all the 4k SPTEs mapping GFNs for
1769 	for_each_valid_tdp_mmu_root(kvm, root, slot->as_id)  in kvm_tdp_mmu_clear_dirty_pt_masked()
1777 	struct kvm_mmu_page *root = spte_to_child_sp(parent->old_spte);  in tdp_mmu_make_huge_spte()
1778 	gfn_t start = parent->gfn;  in tdp_mmu_make_huge_spte()
1779 	gfn_t end = start + KVM_PAGES_PER_HPAGE(parent->level);  in tdp_mmu_make_huge_spte()
1786 		 * returning -EAGAIN here and then failing the forward progress  in tdp_mmu_make_huge_spte()
1790 			return -EAGAIN;  in tdp_mmu_make_huge_spte()
1792 		*huge_spte = make_huge_spte(kvm, iter.old_spte, parent->level);  in tdp_mmu_make_huge_spte()
1796 	return -ENOENT;  in tdp_mmu_make_huge_spte()
1803 	gfn_t start = slot->base_gfn;  in recover_huge_pages_range()
1804 	gfn_t end = start + slot->npages;  in recover_huge_pages_range()
1839 		 * to query that info from slot->arch.lpage_info will cause an  in recover_huge_pages_range()
1840 		 * out-of-bounds access.  in recover_huge_pages_range()
1850 		if (r == -EAGAIN)  in recover_huge_pages_range()
1868  * Recover huge page mappings within the slot by replacing non-leaf SPTEs with
1876 	lockdep_assert_held_read(&kvm->mmu_lock);  in kvm_tdp_mmu_recover_huge_pages()
1877 	for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id)  in kvm_tdp_mmu_recover_huge_pages()
1883  * MMU-writable bit to ensure future writes continue to be intercepted.
1884  * Returns true if an SPTE was set and a TLB flush is needed.
1919  * MMU-writable bit to ensure future writes continue to be intercepted.
1920  * Returns true if an SPTE was set and a TLB flush is needed.
1929 	lockdep_assert_held_write(&kvm->mmu_lock);  in kvm_tdp_mmu_write_protect_gfn()
1930 	for_each_valid_tdp_mmu_root(kvm, root, slot->as_id)  in kvm_tdp_mmu_write_protect_gfn()
1938  * That SPTE may be non-present.
1947 	int leaf = -1;  in __kvm_tdp_mmu_get_walk()
1949 	for_each_tdp_pte(iter, vcpu->kvm, root, gfn, gfn + 1) {  in __kvm_tdp_mmu_get_walk()
1960 	struct kvm_mmu_page *root = root_to_sp(vcpu->arch.mmu->root.hpa);  in kvm_tdp_mmu_get_walk()
1961 	*root_level = vcpu->arch.mmu->root_role.level;  in kvm_tdp_mmu_get_walk()
1968 	struct kvm *kvm = vcpu->kvm;  in kvm_tdp_mmu_gpa_is_mapped()
1970 	hpa_t root = is_direct ? vcpu->arch.mmu->root.hpa :  in kvm_tdp_mmu_gpa_is_mapped()
1971 				 vcpu->arch.mmu->mirror_root_hpa;  in kvm_tdp_mmu_gpa_is_mapped()
1975 	lockdep_assert_held(&kvm->mmu_lock);  in kvm_tdp_mmu_gpa_is_mapped()
1989  * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
1993  *  - Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}.
1994  *  - The returned sptep must not be used after kvm_tdp_mmu_walk_lockless_end.
2006 	for_each_tdp_pte(iter, vcpu->kvm, root, gfn, gfn + 1) {  in kvm_tdp_mmu_fast_pf_get_last_sptep()
2014 	 * legacy MMU and thus does not retain the TDP MMU-specific __rcu  in kvm_tdp_mmu_fast_pf_get_last_sptep()