1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __KVM_X86_MMU_H
3 #define __KVM_X86_MMU_H
4
5 #include <linux/kvm_host.h>
6 #include "regs.h"
7 #include "cpuid.h"
8
9 extern bool __read_mostly tdp_enabled;
10 #ifdef CONFIG_X86_64
11 extern bool __read_mostly tdp_mmu_enabled;
12 #else
13 #define tdp_mmu_enabled false
14 #endif
15 extern bool __read_mostly enable_mmio_caching;
16 extern bool __read_mostly eager_page_split;
17
18 #define KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO 50
19 #define KVM_MIN_ALLOC_MMU_PAGES 64UL
20 #define KVM_MMU_HASH_SHIFT 12
21 #define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT)
22 #define KVM_MIN_FREE_MMU_PAGES 5
23 #define KVM_REFILL_PAGES 25
24
25 #define PT_WRITABLE_SHIFT 1
26 #define PT_USER_SHIFT 2
27
28 #define PT_PRESENT_MASK (1ULL << 0)
29 #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
30 #define PT_USER_MASK (1ULL << PT_USER_SHIFT)
31 #define PT_PWT_MASK (1ULL << 3)
32 #define PT_PCD_MASK (1ULL << 4)
33 #define PT_ACCESSED_SHIFT 5
34 #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
35 #define PT_DIRTY_SHIFT 6
36 #define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
37 #define PT_PAGE_SIZE_SHIFT 7
38 #define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT)
39 #define PT_PAT_MASK (1ULL << 7)
40 #define PT_GLOBAL_MASK (1ULL << 8)
41 #define PT64_NX_SHIFT 63
42 #define PT64_NX_MASK (1ULL << PT64_NX_SHIFT)
43
44 #define PT_PAT_SHIFT 7
45 #define PT_DIR_PAT_SHIFT 12
46 #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
47
48 #define PT64_ROOT_5LEVEL 5
49 #define PT64_ROOT_4LEVEL 4
50 #define PT32_ROOT_LEVEL 2
51 #define PT32E_ROOT_LEVEL 3
52
53 #define ACC_READ_MASK PT_PRESENT_MASK
54 #define ACC_WRITE_MASK PT_WRITABLE_MASK
55 #define ACC_USER_MASK PT_USER_MASK /* non EPT */
56 #define ACC_USER_EXEC_MASK ACC_USER_MASK /* EPT only */
57 #define ACC_EXEC_MASK 8
58 #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK | ACC_READ_MASK)
59
60 #define KVM_MMU_CR4_ROLE_BITS (X86_CR4_PSE | X86_CR4_PAE | X86_CR4_LA57 | \
61 X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE)
62
63 #define KVM_MMU_CR0_ROLE_BITS (X86_CR0_PG | X86_CR0_WP)
64 #define KVM_MMU_EFER_ROLE_BITS (EFER_LME | EFER_NX)
65
rsvd_bits(int s,int e)66 static __always_inline u64 rsvd_bits(int s, int e)
67 {
68 BUILD_BUG_ON(__builtin_constant_p(e) && __builtin_constant_p(s) && e < s);
69
70 if (__builtin_constant_p(e))
71 BUILD_BUG_ON(e > 63);
72 else
73 e &= 63;
74
75 if (e < s)
76 return 0;
77
78 return ((2ULL << (e - s)) - 1) << s;
79 }
80
kvm_mmu_max_gfn(void)81 static inline gfn_t kvm_mmu_max_gfn(void)
82 {
83 /*
84 * Note that this uses the host MAXPHYADDR, not the guest's.
85 * EPT/NPT cannot support GPAs that would exceed host.MAXPHYADDR;
86 * assuming KVM is running on bare metal, guest accesses beyond
87 * host.MAXPHYADDR will hit a #PF(RSVD) and never cause a vmexit
88 * (either EPT Violation/Misconfig or #NPF), and so KVM will never
89 * install a SPTE for such addresses. If KVM is running as a VM
90 * itself, on the other hand, it might see a MAXPHYADDR that is less
91 * than hardware's real MAXPHYADDR. Using the host MAXPHYADDR
92 * disallows such SPTEs entirely and simplifies the TDP MMU.
93 */
94 int max_gpa_bits = likely(tdp_enabled) ? kvm_host.maxphyaddr : 52;
95
96 return (1ULL << (max_gpa_bits - PAGE_SHIFT)) - 1;
97 }
98
mmu_has_mbec(struct kvm_mmu * mmu)99 static inline bool mmu_has_mbec(struct kvm_mmu *mmu)
100 {
101 return mmu->root_role.cr4_smep;
102 }
103
104 u8 kvm_mmu_get_max_tdp_level(void);
105
106 void __init kvm_mmu_x86_module_init(void);
107 int kvm_mmu_vendor_module_init(void);
108 void kvm_mmu_vendor_module_exit(void);
109
110 void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
111 int kvm_mmu_create(struct kvm_vcpu *vcpu);
112 int kvm_mmu_init_vm(struct kvm *kvm);
113 void kvm_mmu_uninit_vm(struct kvm *kvm);
114
115 void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
116 struct kvm_memory_slot *slot);
117
118 void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu);
119 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
120 void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
121 const struct kvm_memory_slot *memslot,
122 int start_level);
123 void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
124 const struct kvm_memory_slot *memslot,
125 int target_level);
126 void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
127 const struct kvm_memory_slot *memslot,
128 u64 start, u64 end,
129 int target_level);
130 void kvm_mmu_recover_huge_pages(struct kvm *kvm,
131 const struct kvm_memory_slot *memslot);
132 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
133 const struct kvm_memory_slot *memslot);
134 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen);
135 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages);
136 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);
137
138 void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask);
139 void kvm_mmu_set_mmio_spte_value(struct kvm *kvm, u64 mmio_value);
140 void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask);
141 void kvm_mmu_set_ept_masks(bool has_ad_bits);
142
143 void kvm_init_mmu(struct kvm_vcpu *vcpu);
144 void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
145 u64 efer, gpa_t nested_cr3, u64 misc_ctl);
146 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
147 int huge_page_level, bool accessed_dirty,
148 bool mbec, gpa_t new_eptp);
149
150 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
151 void *insn, int insn_len);
152 void kvm_mmu_print_sptes(struct kvm_vcpu *vcpu, gpa_t gpa, const char *msg);
153 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
154 void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
155 u64 addr, unsigned long roots);
156 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
157 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
158
159 void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
160 int tdp_max_root_level, int tdp_huge_page_level);
161
162 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu);
163 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
164 u64 fault_address, char *insn, int insn_len);
165 void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
166 struct kvm_pagewalk *pw);
167
168 int kvm_mmu_load(struct kvm_vcpu *vcpu);
169 void kvm_mmu_unload(struct kvm_vcpu *vcpu);
170 void kvm_mmu_free_obsolete_roots(struct kvm_vcpu *vcpu);
171 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
172 void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu);
173 void kvm_mmu_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
174 int bytes);
175
176 bool __kvm_mmu_unprotect_gfn_and_retry(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
177 bool always_retry);
178
kvm_mmu_unprotect_gfn_and_retry(struct kvm_vcpu * vcpu,gpa_t cr2_or_gpa)179 static inline bool kvm_mmu_unprotect_gfn_and_retry(struct kvm_vcpu *vcpu,
180 gpa_t cr2_or_gpa)
181 {
182 return __kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa, false);
183 }
184
185 void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
186 ulong roots_to_free);
187 void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu);
188 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
189 struct x86_exception *exception);
190 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
191 struct x86_exception *exception);
192 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
193 struct x86_exception *exception);
194
kvm_mmu_reload(struct kvm_vcpu * vcpu)195 static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu)
196 {
197 if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
198 kvm_mmu_free_obsolete_roots(vcpu);
199
200 /*
201 * Checking root.hpa is sufficient even when KVM has mirror root.
202 * We can have either:
203 * (1) mirror_root_hpa = INVALID_PAGE, root.hpa = INVALID_PAGE
204 * (2) mirror_root_hpa = root, root.hpa = INVALID_PAGE
205 * (3) mirror_root_hpa = root1, root.hpa = root2
206 * We don't ever have:
207 * mirror_root_hpa = INVALID_PAGE, root.hpa = root
208 */
209 if (likely(vcpu->arch.mmu->root.hpa != INVALID_PAGE))
210 return 0;
211
212 return kvm_mmu_load(vcpu);
213 }
214
kvm_get_pcid(struct kvm_vcpu * vcpu,gpa_t cr3)215 static inline unsigned long kvm_get_pcid(struct kvm_vcpu *vcpu, gpa_t cr3)
216 {
217 BUILD_BUG_ON((X86_CR3_PCID_MASK & PAGE_MASK) != 0);
218
219 return kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)
220 ? cr3 & X86_CR3_PCID_MASK
221 : 0;
222 }
223
kvm_get_active_pcid(struct kvm_vcpu * vcpu)224 static inline unsigned long kvm_get_active_pcid(struct kvm_vcpu *vcpu)
225 {
226 return kvm_get_pcid(vcpu, kvm_read_cr3(vcpu));
227 }
228
kvm_get_active_cr3_lam_bits(struct kvm_vcpu * vcpu)229 static inline unsigned long kvm_get_active_cr3_lam_bits(struct kvm_vcpu *vcpu)
230 {
231 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LAM))
232 return 0;
233
234 return kvm_read_cr3(vcpu) & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57);
235 }
236
kvm_mmu_load_pgd(struct kvm_vcpu * vcpu)237 static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)
238 {
239 u64 root_hpa = vcpu->arch.mmu->root.hpa;
240
241 if (!VALID_PAGE(root_hpa))
242 return;
243
244 kvm_x86_call(load_mmu_pgd)(vcpu, root_hpa,
245 vcpu->arch.mmu->root_role.level);
246 }
247
kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu * vcpu,struct kvm_pagewalk * w)248 static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
249 struct kvm_pagewalk *w)
250 {
251 /*
252 * When EPT is enabled, KVM may passthrough CR0.WP to the guest, i.e.
253 * @w's snapshot of CR0.WP and thus all related paging metadata may
254 * be stale. Refresh CR0.WP and the metadata on-demand when checking
255 * for permission faults. Exempt nested MMUs, i.e. MMUs for shadowing
256 * nEPT and nNPT, as CR0.WP is ignored in both cases. Note, KVM will
257 * still refresh gva_walk, so as to honor L2's CR0.WP when translating
258 * L2 GVAs to GPAs.
259 */
260 if (!tdp_enabled || w == &vcpu->arch.ngpa_walk)
261 return;
262
263 __kvm_mmu_refresh_passthrough_bits(vcpu, w);
264 }
265
266 /*
267 * Check if a given access (described through the I/D, W/R and U/S bits of a
268 * page fault error code pfec) causes a permission fault with the given PTE
269 * access rights (in ACC_* format).
270 *
271 * Return zero if the access does not fault; return the page fault error code
272 * if the access faults.
273 */
permission_fault(struct kvm_vcpu * vcpu,struct kvm_pagewalk * w,unsigned pte_access,unsigned pte_pkey,u64 access)274 static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
275 unsigned pte_access, unsigned pte_pkey,
276 u64 access)
277 {
278 /*
279 * Of the error codes that do not contribute to the index in
280 * fmt->permissions, PK and SS are not supported by shadow paging
281 * and also not provided to EPT violations, but SS *is* included
282 * in the NPT exit code. RSVD faults are handled elsewhere.
283 */
284 unsigned int pfec = access & ~PFERR_SS_MASK;
285 if (WARN_ON_ONCE(pfec & ~(PFERR_PRESENT_MASK | PFERR_WRITE_MASK |
286 PFERR_USER_MASK | PFERR_FETCH_MASK)))
287 pfec &= (PFERR_PRESENT_MASK | PFERR_WRITE_MASK |
288 PFERR_USER_MASK | PFERR_FETCH_MASK);
289
290 unsigned long rflags = kvm_x86_call(get_rflags)(vcpu);
291
292 /*
293 * For explicit supervisor accesses, SMAP is disabled if EFLAGS.AC = 1.
294 * For implicit supervisor accesses, SMAP cannot be overridden.
295 *
296 * SMAP works on supervisor accesses only, and not_smap can
297 * be set or not set when user access with neither has any bearing
298 * on the result.
299 *
300 * We put the SMAP checking bit in place of the PFERR_RSVD_MASK bit;
301 * this bit will always be zero in pfec, but it will be one in index
302 * if SMAP checks are being disabled.
303 */
304 u64 implicit_access = access & PFERR_IMPLICIT_ACCESS;
305 bool not_smap = ((rflags & X86_EFLAGS_AC) | implicit_access) == X86_EFLAGS_AC;
306 int index = (pfec | (not_smap ? PFERR_RSVD_MASK : 0)) >> 1;
307 struct kvm_page_format *fmt = &w->fmt;
308 u32 errcode = PFERR_PRESENT_MASK;
309 bool fault;
310
311 kvm_mmu_refresh_passthrough_bits(vcpu, w);
312
313 fault = (fmt->permissions[index] >> pte_access) & 1;
314 if (unlikely(fmt->pkru_mask)) {
315 u32 pkru_bits, offset;
316
317 /*
318 * PKRU defines 32 bits, there are 16 domains and 2
319 * attribute bits per domain in pkru. pte_pkey is the
320 * index of the protection domain, so pte_pkey * 2 is
321 * is the index of the first bit for the domain.
322 */
323 pkru_bits = (vcpu->arch.pkru >> (pte_pkey * 2)) & 3;
324
325 /* clear present bit, replace PFEC.RSVD with ACC_USER_MASK. */
326 offset = (pfec & ~1) | ((pte_access & PT_USER_MASK) ? PFERR_RSVD_MASK : 0);
327
328 pkru_bits &= fmt->pkru_mask >> offset;
329 errcode |= -pkru_bits & PFERR_PK_MASK;
330 fault |= (pkru_bits != 0);
331 }
332
333 return -(u32)fault & errcode;
334 }
335
336 int kvm_mmu_post_init_vm(struct kvm *kvm);
337 void kvm_mmu_pre_destroy_vm(struct kvm *kvm);
338
kvm_shadow_root_allocated(struct kvm * kvm)339 static inline bool kvm_shadow_root_allocated(struct kvm *kvm)
340 {
341 /*
342 * Read shadow_root_allocated before related pointers. Hence, threads
343 * reading shadow_root_allocated in any lock context are guaranteed to
344 * see the pointers. Pairs with smp_store_release in
345 * mmu_first_shadow_root_alloc.
346 */
347 return smp_load_acquire(&kvm->arch.shadow_root_allocated);
348 }
349
350 int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn);
351
kvm_memslots_have_rmaps(struct kvm * kvm)352 static inline bool kvm_memslots_have_rmaps(struct kvm *kvm)
353 {
354 return !tdp_mmu_enabled || kvm_shadow_root_allocated(kvm);
355 }
356
gfn_to_index(gfn_t gfn,gfn_t base_gfn,int level)357 static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
358 {
359 /* KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K) must be 0. */
360 return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
361 (base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
362 }
363
364 static inline unsigned long
__kvm_mmu_slot_lpages(struct kvm_memory_slot * slot,unsigned long npages,int level)365 __kvm_mmu_slot_lpages(struct kvm_memory_slot *slot, unsigned long npages,
366 int level)
367 {
368 return gfn_to_index(slot->base_gfn + npages - 1,
369 slot->base_gfn, level) + 1;
370 }
371
372 static inline unsigned long
kvm_mmu_slot_lpages(struct kvm_memory_slot * slot,int level)373 kvm_mmu_slot_lpages(struct kvm_memory_slot *slot, int level)
374 {
375 return __kvm_mmu_slot_lpages(slot, slot->npages, level);
376 }
377
kvm_update_page_stats(struct kvm * kvm,int level,int count)378 static inline void kvm_update_page_stats(struct kvm *kvm, int level, int count)
379 {
380 atomic64_add(count, &kvm->stat.pages[level - 1]);
381 }
382
mmu_is_nested(struct kvm_vcpu * vcpu)383 static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
384 {
385 return vcpu->arch.mmu == &vcpu->arch.guest_mmu;
386 }
387
kvm_translate_gpa(struct kvm_vcpu * vcpu,struct kvm_pagewalk * w,gpa_t gpa,u64 access,struct x86_exception * exception,u64 pte_access)388 static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu,
389 struct kvm_pagewalk *w,
390 gpa_t gpa, u64 access,
391 struct x86_exception *exception,
392 u64 pte_access)
393 {
394 if (!mmu_is_nested(vcpu) || w == &vcpu->arch.ngpa_walk)
395 return gpa;
396 return kvm_nested_call(translate_nested_gpa)(vcpu, gpa, access,
397 exception, pte_access);
398 }
399
kvm_has_mirrored_tdp(const struct kvm * kvm)400 static inline bool kvm_has_mirrored_tdp(const struct kvm *kvm)
401 {
402 return kvm->arch.vm_type == KVM_X86_TDX_VM;
403 }
404
kvm_gfn_direct_bits(const struct kvm * kvm)405 static inline gfn_t kvm_gfn_direct_bits(const struct kvm *kvm)
406 {
407 return kvm->arch.gfn_direct_bits;
408 }
409
kvm_is_addr_direct(struct kvm * kvm,gpa_t gpa)410 static inline bool kvm_is_addr_direct(struct kvm *kvm, gpa_t gpa)
411 {
412 gpa_t gpa_direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(kvm));
413
414 return !gpa_direct_bits || (gpa & gpa_direct_bits);
415 }
416
kvm_is_gfn_alias(struct kvm * kvm,gfn_t gfn)417 static inline bool kvm_is_gfn_alias(struct kvm *kvm, gfn_t gfn)
418 {
419 return gfn & kvm_gfn_direct_bits(kvm);
420 }
421 #endif
422