1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #ifndef KVM_X86_MMU_SPTE_H 4 #define KVM_X86_MMU_SPTE_H 5 6 #include "mmu_internal.h" 7 8 #define PT_FIRST_AVAIL_BITS_SHIFT 10 9 #define PT64_SECOND_AVAIL_BITS_SHIFT 54 10 11 /* 12 * The mask used to denote special SPTEs, which can be either MMIO SPTEs or 13 * Access Tracking SPTEs. 14 */ 15 #define SPTE_SPECIAL_MASK (3ULL << 52) 16 #define SPTE_AD_ENABLED_MASK (0ULL << 52) 17 #define SPTE_AD_DISABLED_MASK (1ULL << 52) 18 #define SPTE_AD_WRPROT_ONLY_MASK (2ULL << 52) 19 #define SPTE_MMIO_MASK (3ULL << 52) 20 21 #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK 22 #define PT64_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1)) 23 #else 24 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1)) 25 #endif 26 #define PT64_LVL_ADDR_MASK(level) \ 27 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \ 28 * PT64_LEVEL_BITS))) - 1)) 29 #define PT64_LVL_OFFSET_MASK(level) \ 30 (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \ 31 * PT64_LEVEL_BITS))) - 1)) 32 33 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \ 34 | shadow_x_mask | shadow_nx_mask | shadow_me_mask) 35 36 #define ACC_EXEC_MASK 1 37 #define ACC_WRITE_MASK PT_WRITABLE_MASK 38 #define ACC_USER_MASK PT_USER_MASK 39 #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK) 40 41 /* The mask for the R/X bits in EPT PTEs */ 42 #define PT64_EPT_READABLE_MASK 0x1ull 43 #define PT64_EPT_EXECUTABLE_MASK 0x4ull 44 45 #define PT64_LEVEL_BITS 9 46 47 #define PT64_LEVEL_SHIFT(level) \ 48 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS) 49 50 #define PT64_INDEX(address, level)\ 51 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1)) 52 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) 53 54 55 #define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT) 56 #define SPTE_MMU_WRITEABLE (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1)) 57 58 /* 59 * Due to limited space in PTEs, the MMIO generation is a 18 bit subset of 60 * the memslots generation and is derived as follows: 61 * 62 * Bits 0-8 of the MMIO generation are propagated to spte bits 3-11 63 * Bits 9-17 of the MMIO generation are propagated to spte bits 54-62 64 * 65 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in 66 * the MMIO generation number, as doing so would require stealing a bit from 67 * the "real" generation number and thus effectively halve the maximum number 68 * of MMIO generations that can be handled before encountering a wrap (which 69 * requires a full MMU zap). The flag is instead explicitly queried when 70 * checking for MMIO spte cache hits. 71 */ 72 73 #define MMIO_SPTE_GEN_LOW_START 3 74 #define MMIO_SPTE_GEN_LOW_END 11 75 76 #define MMIO_SPTE_GEN_HIGH_START PT64_SECOND_AVAIL_BITS_SHIFT 77 #define MMIO_SPTE_GEN_HIGH_END 62 78 79 #define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \ 80 MMIO_SPTE_GEN_LOW_START) 81 #define MMIO_SPTE_GEN_HIGH_MASK GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \ 82 MMIO_SPTE_GEN_HIGH_START) 83 84 #define MMIO_SPTE_GEN_LOW_BITS (MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1) 85 #define MMIO_SPTE_GEN_HIGH_BITS (MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1) 86 87 /* remember to adjust the comment above as well if you change these */ 88 static_assert(MMIO_SPTE_GEN_LOW_BITS == 9 && MMIO_SPTE_GEN_HIGH_BITS == 9); 89 90 #define MMIO_SPTE_GEN_LOW_SHIFT (MMIO_SPTE_GEN_LOW_START - 0) 91 #define MMIO_SPTE_GEN_HIGH_SHIFT (MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS) 92 93 #define MMIO_SPTE_GEN_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0) 94 95 extern u64 __read_mostly shadow_nx_mask; 96 extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */ 97 extern u64 __read_mostly shadow_user_mask; 98 extern u64 __read_mostly shadow_accessed_mask; 99 extern u64 __read_mostly shadow_dirty_mask; 100 extern u64 __read_mostly shadow_mmio_value; 101 extern u64 __read_mostly shadow_mmio_access_mask; 102 extern u64 __read_mostly shadow_present_mask; 103 extern u64 __read_mostly shadow_me_mask; 104 105 /* 106 * SPTEs used by MMUs without A/D bits are marked with SPTE_AD_DISABLED_MASK; 107 * shadow_acc_track_mask is the set of bits to be cleared in non-accessed 108 * pages. 109 */ 110 extern u64 __read_mostly shadow_acc_track_mask; 111 112 /* 113 * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order 114 * to guard against L1TF attacks. 115 */ 116 extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask; 117 118 /* 119 * The number of high-order 1 bits to use in the mask above. 120 */ 121 #define SHADOW_NONPRESENT_OR_RSVD_MASK_LEN 5 122 123 /* 124 * The mask/shift to use for saving the original R/X bits when marking the PTE 125 * as not-present for access tracking purposes. We do not save the W bit as the 126 * PTEs being access tracked also need to be dirty tracked, so the W bit will be 127 * restored only when a write is attempted to the page. 128 */ 129 #define SHADOW_ACC_TRACK_SAVED_BITS_MASK (PT64_EPT_READABLE_MASK | \ 130 PT64_EPT_EXECUTABLE_MASK) 131 #define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT PT64_SECOND_AVAIL_BITS_SHIFT 132 133 /* 134 * If a thread running without exclusive control of the MMU lock must perform a 135 * multi-part operation on an SPTE, it can set the SPTE to REMOVED_SPTE as a 136 * non-present intermediate value. Other threads which encounter this value 137 * should not modify the SPTE. 138 * 139 * This constant works because it is considered non-present on both AMD and 140 * Intel CPUs and does not create a L1TF vulnerability because the pfn section 141 * is zeroed out. 142 * 143 * Only used by the TDP MMU. 144 */ 145 #define REMOVED_SPTE (1ull << 59) 146 147 static inline bool is_removed_spte(u64 spte) 148 { 149 return spte == REMOVED_SPTE; 150 } 151 152 /* 153 * In some cases, we need to preserve the GFN of a non-present or reserved 154 * SPTE when we usurp the upper five bits of the physical address space to 155 * defend against L1TF, e.g. for MMIO SPTEs. To preserve the GFN, we'll 156 * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask 157 * left into the reserved bits, i.e. the GFN in the SPTE will be split into 158 * high and low parts. This mask covers the lower bits of the GFN. 159 */ 160 extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask; 161 162 /* 163 * The number of non-reserved physical address bits irrespective of features 164 * that repurpose legal bits, e.g. MKTME. 165 */ 166 extern u8 __read_mostly shadow_phys_bits; 167 168 static inline bool is_mmio_spte(u64 spte) 169 { 170 return (spte & SPTE_SPECIAL_MASK) == SPTE_MMIO_MASK; 171 } 172 173 static inline bool sp_ad_disabled(struct kvm_mmu_page *sp) 174 { 175 return sp->role.ad_disabled; 176 } 177 178 static inline bool spte_ad_enabled(u64 spte) 179 { 180 MMU_WARN_ON(is_mmio_spte(spte)); 181 return (spte & SPTE_SPECIAL_MASK) != SPTE_AD_DISABLED_MASK; 182 } 183 184 static inline bool spte_ad_need_write_protect(u64 spte) 185 { 186 MMU_WARN_ON(is_mmio_spte(spte)); 187 return (spte & SPTE_SPECIAL_MASK) != SPTE_AD_ENABLED_MASK; 188 } 189 190 static inline u64 spte_shadow_accessed_mask(u64 spte) 191 { 192 MMU_WARN_ON(is_mmio_spte(spte)); 193 return spte_ad_enabled(spte) ? shadow_accessed_mask : 0; 194 } 195 196 static inline u64 spte_shadow_dirty_mask(u64 spte) 197 { 198 MMU_WARN_ON(is_mmio_spte(spte)); 199 return spte_ad_enabled(spte) ? shadow_dirty_mask : 0; 200 } 201 202 static inline bool is_access_track_spte(u64 spte) 203 { 204 return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0; 205 } 206 207 static inline bool is_shadow_present_pte(u64 pte) 208 { 209 return (pte != 0) && !is_mmio_spte(pte) && !is_removed_spte(pte); 210 } 211 212 static inline bool is_large_pte(u64 pte) 213 { 214 return pte & PT_PAGE_SIZE_MASK; 215 } 216 217 static inline bool is_last_spte(u64 pte, int level) 218 { 219 return (level == PG_LEVEL_4K) || is_large_pte(pte); 220 } 221 222 static inline bool is_executable_pte(u64 spte) 223 { 224 return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask; 225 } 226 227 static inline kvm_pfn_t spte_to_pfn(u64 pte) 228 { 229 return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; 230 } 231 232 static inline bool is_accessed_spte(u64 spte) 233 { 234 u64 accessed_mask = spte_shadow_accessed_mask(spte); 235 236 return accessed_mask ? spte & accessed_mask 237 : !is_access_track_spte(spte); 238 } 239 240 static inline bool is_dirty_spte(u64 spte) 241 { 242 u64 dirty_mask = spte_shadow_dirty_mask(spte); 243 244 return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK; 245 } 246 247 static inline bool spte_can_locklessly_be_made_writable(u64 spte) 248 { 249 return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) == 250 (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE); 251 } 252 253 static inline u64 get_mmio_spte_generation(u64 spte) 254 { 255 u64 gen; 256 257 gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT; 258 gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT; 259 return gen; 260 } 261 262 /* Bits which may be returned by set_spte() */ 263 #define SET_SPTE_WRITE_PROTECTED_PT BIT(0) 264 #define SET_SPTE_NEED_REMOTE_TLB_FLUSH BIT(1) 265 #define SET_SPTE_SPURIOUS BIT(2) 266 267 int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, 268 gfn_t gfn, kvm_pfn_t pfn, u64 old_spte, bool speculative, 269 bool can_unsync, bool host_writable, bool ad_disabled, 270 u64 *new_spte); 271 u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled); 272 u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access); 273 u64 mark_spte_for_access_track(u64 spte); 274 u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn); 275 276 void kvm_mmu_reset_all_pte_masks(void); 277 278 #endif 279