1 // SPDX-License-Identifier: GPL-2.0 2 3 #ifndef __KVM_X86_MMU_TDP_ITER_H 4 #define __KVM_X86_MMU_TDP_ITER_H 5 6 #include <linux/kvm_host.h> 7 8 #include "mmu.h" 9 #include "spte.h" 10 11 /* 12 * TDP MMU SPTEs are RCU protected to allow paging structures (non-leaf SPTEs) 13 * to be zapped while holding mmu_lock for read, and to allow TLB flushes to be 14 * batched without having to collect the list of zapped SPs. Flows that can 15 * remove SPs must service pending TLB flushes prior to dropping RCU protection. 16 */ 17 static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep) 18 { 19 return READ_ONCE(*rcu_dereference(sptep)); 20 } 21 22 static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte) 23 { 24 KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte)); 25 return xchg(rcu_dereference(sptep), new_spte); 26 } 27 28 static inline u64 tdp_mmu_clear_spte_bits_atomic(tdp_ptep_t sptep, u64 mask) 29 { 30 atomic64_t *sptep_atomic = (atomic64_t *)rcu_dereference(sptep); 31 32 return (u64)atomic64_fetch_and(~mask, sptep_atomic); 33 } 34 35 static inline void __kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 new_spte) 36 { 37 KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte)); 38 WRITE_ONCE(*rcu_dereference(sptep), new_spte); 39 } 40 41 /* 42 * SPTEs must be modified atomically if they are shadow-present, leaf SPTEs, 43 * and have volatile bits (bits that can be set outside of mmu_lock) that 44 * must not be clobbered. 45 */ 46 static inline bool kvm_tdp_mmu_spte_need_atomic_update(u64 old_spte, int level) 47 { 48 return is_shadow_present_pte(old_spte) && 49 is_last_spte(old_spte, level) && 50 spte_needs_atomic_update(old_spte); 51 } 52 53 static inline u64 kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 old_spte, 54 u64 new_spte, int level) 55 { 56 if (kvm_tdp_mmu_spte_need_atomic_update(old_spte, level)) 57 return kvm_tdp_mmu_write_spte_atomic(sptep, new_spte); 58 59 __kvm_tdp_mmu_write_spte(sptep, new_spte); 60 return old_spte; 61 } 62 63 static inline u64 tdp_mmu_clear_spte_bits(tdp_ptep_t sptep, u64 old_spte, 64 u64 mask, int level) 65 { 66 if (kvm_tdp_mmu_spte_need_atomic_update(old_spte, level)) 67 return tdp_mmu_clear_spte_bits_atomic(sptep, mask); 68 69 __kvm_tdp_mmu_write_spte(sptep, old_spte & ~mask); 70 return old_spte; 71 } 72 73 /* 74 * A TDP iterator performs a pre-order walk over a TDP paging structure. 75 */ 76 struct tdp_iter { 77 /* 78 * The iterator will traverse the paging structure towards the mapping 79 * for this GFN. 80 */ 81 gfn_t next_last_level_gfn; 82 /* 83 * The next_last_level_gfn at the time when the thread last 84 * yielded. Only yielding when the next_last_level_gfn != 85 * yielded_gfn helps ensure forward progress. 86 */ 87 gfn_t yielded_gfn; 88 /* Pointers to the page tables traversed to reach the current SPTE */ 89 tdp_ptep_t pt_path[PT64_ROOT_MAX_LEVEL]; 90 /* A pointer to the current SPTE */ 91 tdp_ptep_t sptep; 92 /* The lowest GFN (mask bits excluded) mapped by the current SPTE */ 93 gfn_t gfn; 94 /* Mask applied to convert the GFN to the mapping GPA */ 95 gfn_t gfn_bits; 96 /* The level of the root page given to the iterator */ 97 int root_level; 98 /* The lowest level the iterator should traverse to */ 99 int min_level; 100 /* The iterator's current level within the paging structure */ 101 int level; 102 /* The address space ID, i.e. SMM vs. regular. */ 103 int as_id; 104 /* A snapshot of the value at sptep */ 105 u64 old_spte; 106 /* 107 * Whether the iterator has a valid state. This will be false if the 108 * iterator walks off the end of the paging structure. 109 */ 110 bool valid; 111 /* 112 * True if KVM dropped mmu_lock and yielded in the middle of a walk, in 113 * which case tdp_iter_next() needs to restart the walk at the root 114 * level instead of advancing to the next entry. 115 */ 116 bool yielded; 117 }; 118 119 /* 120 * Iterates over every SPTE mapping the GFN range [start, end) in a 121 * preorder traversal. 122 */ 123 #define for_each_tdp_pte_min_level(iter, kvm, root, min_level, start, end) \ 124 for (tdp_iter_start(&iter, root, min_level, start, kvm_gfn_root_bits(kvm, root)); \ 125 iter.valid && iter.gfn < end; \ 126 tdp_iter_next(&iter)) 127 128 #define for_each_tdp_pte_min_level_all(iter, root, min_level) \ 129 for (tdp_iter_start(&iter, root, min_level, 0, 0); \ 130 iter.valid && iter.gfn < tdp_mmu_max_gfn_exclusive(); \ 131 tdp_iter_next(&iter)) 132 133 #define for_each_tdp_pte(iter, kvm, root, start, end) \ 134 for_each_tdp_pte_min_level(iter, kvm, root, PG_LEVEL_4K, start, end) 135 136 tdp_ptep_t spte_to_child_pt(u64 pte, int level); 137 138 void tdp_iter_start(struct tdp_iter *iter, struct kvm_mmu_page *root, 139 int min_level, gfn_t next_last_level_gfn, gfn_t gfn_bits); 140 void tdp_iter_next(struct tdp_iter *iter); 141 void tdp_iter_restart(struct tdp_iter *iter); 142 143 #endif /* __KVM_X86_MMU_TDP_ITER_H */ 144