xref: /linux/arch/x86/kvm/mmu/paging_tmpl.h (revision 0750b8fcf313845b21c71344b4bea8ad7d3cee84)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Kernel-based Virtual Machine driver for Linux
4  *
5  * This module enables machines with Intel VT-x extensions to run virtual
6  * machines without emulation or binary translation.
7  *
8  * MMU support
9  *
10  * Copyright (C) 2006 Qumranet, Inc.
11  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
12  *
13  * Authors:
14  *   Yaniv Kamay  <yaniv@qumranet.com>
15  *   Avi Kivity   <avi@qumranet.com>
16  */
17 
18 /*
19  * We need the mmu code to access both 32-bit and 64-bit guest ptes,
20  * so the code in this file is compiled twice, once per pte size.
21  */
22 
23 #if PTTYPE == 64
24 	#define pt_element_t u64
25 	#define guest_walker guest_walker64
26 	#define FNAME(name) paging##64_##name
27 	#define PT_BASE_ADDR_MASK GUEST_PT64_BASE_ADDR_MASK
28 	#define PT_LVL_ADDR_MASK(lvl) PT64_LVL_ADDR_MASK(lvl)
29 	#define PT_LVL_OFFSET_MASK(lvl) PT64_LVL_OFFSET_MASK(lvl)
30 	#define PT_INDEX(addr, level) PT64_INDEX(addr, level)
31 	#define PT_LEVEL_BITS PT64_LEVEL_BITS
32 	#define PT_GUEST_DIRTY_SHIFT PT_DIRTY_SHIFT
33 	#define PT_GUEST_ACCESSED_SHIFT PT_ACCESSED_SHIFT
34 	#define PT_HAVE_ACCESSED_DIRTY(mmu) true
35 	#ifdef CONFIG_X86_64
36 	#define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL
37 	#define CMPXCHG cmpxchg
38 	#else
39 	#define CMPXCHG cmpxchg64
40 	#define PT_MAX_FULL_LEVELS 2
41 	#endif
42 #elif PTTYPE == 32
43 	#define pt_element_t u32
44 	#define guest_walker guest_walker32
45 	#define FNAME(name) paging##32_##name
46 	#define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
47 	#define PT_LVL_ADDR_MASK(lvl) PT32_LVL_ADDR_MASK(lvl)
48 	#define PT_LVL_OFFSET_MASK(lvl) PT32_LVL_OFFSET_MASK(lvl)
49 	#define PT_INDEX(addr, level) PT32_INDEX(addr, level)
50 	#define PT_LEVEL_BITS PT32_LEVEL_BITS
51 	#define PT_MAX_FULL_LEVELS 2
52 	#define PT_GUEST_DIRTY_SHIFT PT_DIRTY_SHIFT
53 	#define PT_GUEST_ACCESSED_SHIFT PT_ACCESSED_SHIFT
54 	#define PT_HAVE_ACCESSED_DIRTY(mmu) true
55 	#define CMPXCHG cmpxchg
56 #elif PTTYPE == PTTYPE_EPT
57 	#define pt_element_t u64
58 	#define guest_walker guest_walkerEPT
59 	#define FNAME(name) ept_##name
60 	#define PT_BASE_ADDR_MASK GUEST_PT64_BASE_ADDR_MASK
61 	#define PT_LVL_ADDR_MASK(lvl) PT64_LVL_ADDR_MASK(lvl)
62 	#define PT_LVL_OFFSET_MASK(lvl) PT64_LVL_OFFSET_MASK(lvl)
63 	#define PT_INDEX(addr, level) PT64_INDEX(addr, level)
64 	#define PT_LEVEL_BITS PT64_LEVEL_BITS
65 	#define PT_GUEST_DIRTY_SHIFT 9
66 	#define PT_GUEST_ACCESSED_SHIFT 8
67 	#define PT_HAVE_ACCESSED_DIRTY(mmu) ((mmu)->ept_ad)
68 	#define CMPXCHG cmpxchg64
69 	#define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL
70 #else
71 	#error Invalid PTTYPE value
72 #endif
73 
74 #define PT_GUEST_DIRTY_MASK    (1 << PT_GUEST_DIRTY_SHIFT)
75 #define PT_GUEST_ACCESSED_MASK (1 << PT_GUEST_ACCESSED_SHIFT)
76 
77 #define gpte_to_gfn_lvl FNAME(gpte_to_gfn_lvl)
78 #define gpte_to_gfn(pte) gpte_to_gfn_lvl((pte), PG_LEVEL_4K)
79 
80 /*
81  * The guest_walker structure emulates the behavior of the hardware page
82  * table walker.
83  */
84 struct guest_walker {
85 	int level;
86 	unsigned max_level;
87 	gfn_t table_gfn[PT_MAX_FULL_LEVELS];
88 	pt_element_t ptes[PT_MAX_FULL_LEVELS];
89 	pt_element_t prefetch_ptes[PTE_PREFETCH_NUM];
90 	gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
91 	pt_element_t __user *ptep_user[PT_MAX_FULL_LEVELS];
92 	bool pte_writable[PT_MAX_FULL_LEVELS];
93 	unsigned int pt_access[PT_MAX_FULL_LEVELS];
94 	unsigned int pte_access;
95 	gfn_t gfn;
96 	struct x86_exception fault;
97 };
98 
99 static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
100 {
101 	return (gpte & PT_LVL_ADDR_MASK(lvl)) >> PAGE_SHIFT;
102 }
103 
104 static inline void FNAME(protect_clean_gpte)(struct kvm_mmu *mmu, unsigned *access,
105 					     unsigned gpte)
106 {
107 	unsigned mask;
108 
109 	/* dirty bit is not supported, so no need to track it */
110 	if (!PT_HAVE_ACCESSED_DIRTY(mmu))
111 		return;
112 
113 	BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
114 
115 	mask = (unsigned)~ACC_WRITE_MASK;
116 	/* Allow write access to dirty gptes */
117 	mask |= (gpte >> (PT_GUEST_DIRTY_SHIFT - PT_WRITABLE_SHIFT)) &
118 		PT_WRITABLE_MASK;
119 	*access &= mask;
120 }
121 
122 static inline int FNAME(is_present_gpte)(unsigned long pte)
123 {
124 #if PTTYPE != PTTYPE_EPT
125 	return pte & PT_PRESENT_MASK;
126 #else
127 	return pte & 7;
128 #endif
129 }
130 
131 static bool FNAME(is_bad_mt_xwr)(struct rsvd_bits_validate *rsvd_check, u64 gpte)
132 {
133 #if PTTYPE != PTTYPE_EPT
134 	return false;
135 #else
136 	return __is_bad_mt_xwr(rsvd_check, gpte);
137 #endif
138 }
139 
140 static bool FNAME(is_rsvd_bits_set)(struct kvm_mmu *mmu, u64 gpte, int level)
141 {
142 	return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level) ||
143 	       FNAME(is_bad_mt_xwr)(&mmu->guest_rsvd_check, gpte);
144 }
145 
146 static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
147 			       pt_element_t __user *ptep_user, unsigned index,
148 			       pt_element_t orig_pte, pt_element_t new_pte)
149 {
150 	int npages;
151 	pt_element_t ret;
152 	pt_element_t *table;
153 	struct page *page;
154 
155 	npages = get_user_pages_fast((unsigned long)ptep_user, 1, FOLL_WRITE, &page);
156 	if (likely(npages == 1)) {
157 		table = kmap_atomic(page);
158 		ret = CMPXCHG(&table[index], orig_pte, new_pte);
159 		kunmap_atomic(table);
160 
161 		kvm_release_page_dirty(page);
162 	} else {
163 		struct vm_area_struct *vma;
164 		unsigned long vaddr = (unsigned long)ptep_user & PAGE_MASK;
165 		unsigned long pfn;
166 		unsigned long paddr;
167 
168 		mmap_read_lock(current->mm);
169 		vma = find_vma_intersection(current->mm, vaddr, vaddr + PAGE_SIZE);
170 		if (!vma || !(vma->vm_flags & VM_PFNMAP)) {
171 			mmap_read_unlock(current->mm);
172 			return -EFAULT;
173 		}
174 		pfn = ((vaddr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
175 		paddr = pfn << PAGE_SHIFT;
176 		table = memremap(paddr, PAGE_SIZE, MEMREMAP_WB);
177 		if (!table) {
178 			mmap_read_unlock(current->mm);
179 			return -EFAULT;
180 		}
181 		ret = CMPXCHG(&table[index], orig_pte, new_pte);
182 		memunmap(table);
183 		mmap_read_unlock(current->mm);
184 	}
185 
186 	return (ret != orig_pte);
187 }
188 
189 static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
190 				  struct kvm_mmu_page *sp, u64 *spte,
191 				  u64 gpte)
192 {
193 	if (!FNAME(is_present_gpte)(gpte))
194 		goto no_present;
195 
196 	/* if accessed bit is not supported prefetch non accessed gpte */
197 	if (PT_HAVE_ACCESSED_DIRTY(vcpu->arch.mmu) &&
198 	    !(gpte & PT_GUEST_ACCESSED_MASK))
199 		goto no_present;
200 
201 	if (FNAME(is_rsvd_bits_set)(vcpu->arch.mmu, gpte, PG_LEVEL_4K))
202 		goto no_present;
203 
204 	return false;
205 
206 no_present:
207 	drop_spte(vcpu->kvm, spte);
208 	return true;
209 }
210 
211 /*
212  * For PTTYPE_EPT, a page table can be executable but not readable
213  * on supported processors. Therefore, set_spte does not automatically
214  * set bit 0 if execute only is supported. Here, we repurpose ACC_USER_MASK
215  * to signify readability since it isn't used in the EPT case
216  */
217 static inline unsigned FNAME(gpte_access)(u64 gpte)
218 {
219 	unsigned access;
220 #if PTTYPE == PTTYPE_EPT
221 	access = ((gpte & VMX_EPT_WRITABLE_MASK) ? ACC_WRITE_MASK : 0) |
222 		((gpte & VMX_EPT_EXECUTABLE_MASK) ? ACC_EXEC_MASK : 0) |
223 		((gpte & VMX_EPT_READABLE_MASK) ? ACC_USER_MASK : 0);
224 #else
225 	BUILD_BUG_ON(ACC_EXEC_MASK != PT_PRESENT_MASK);
226 	BUILD_BUG_ON(ACC_EXEC_MASK != 1);
227 	access = gpte & (PT_WRITABLE_MASK | PT_USER_MASK | PT_PRESENT_MASK);
228 	/* Combine NX with P (which is set here) to get ACC_EXEC_MASK.  */
229 	access ^= (gpte >> PT64_NX_SHIFT);
230 #endif
231 
232 	return access;
233 }
234 
235 static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
236 					     struct kvm_mmu *mmu,
237 					     struct guest_walker *walker,
238 					     gpa_t addr, int write_fault)
239 {
240 	unsigned level, index;
241 	pt_element_t pte, orig_pte;
242 	pt_element_t __user *ptep_user;
243 	gfn_t table_gfn;
244 	int ret;
245 
246 	/* dirty/accessed bits are not supported, so no need to update them */
247 	if (!PT_HAVE_ACCESSED_DIRTY(mmu))
248 		return 0;
249 
250 	for (level = walker->max_level; level >= walker->level; --level) {
251 		pte = orig_pte = walker->ptes[level - 1];
252 		table_gfn = walker->table_gfn[level - 1];
253 		ptep_user = walker->ptep_user[level - 1];
254 		index = offset_in_page(ptep_user) / sizeof(pt_element_t);
255 		if (!(pte & PT_GUEST_ACCESSED_MASK)) {
256 			trace_kvm_mmu_set_accessed_bit(table_gfn, index, sizeof(pte));
257 			pte |= PT_GUEST_ACCESSED_MASK;
258 		}
259 		if (level == walker->level && write_fault &&
260 				!(pte & PT_GUEST_DIRTY_MASK)) {
261 			trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
262 #if PTTYPE == PTTYPE_EPT
263 			if (kvm_x86_ops.nested_ops->write_log_dirty(vcpu, addr))
264 				return -EINVAL;
265 #endif
266 			pte |= PT_GUEST_DIRTY_MASK;
267 		}
268 		if (pte == orig_pte)
269 			continue;
270 
271 		/*
272 		 * If the slot is read-only, simply do not process the accessed
273 		 * and dirty bits.  This is the correct thing to do if the slot
274 		 * is ROM, and page tables in read-as-ROM/write-as-MMIO slots
275 		 * are only supported if the accessed and dirty bits are already
276 		 * set in the ROM (so that MMIO writes are never needed).
277 		 *
278 		 * Note that NPT does not allow this at all and faults, since
279 		 * it always wants nested page table entries for the guest
280 		 * page tables to be writable.  And EPT works but will simply
281 		 * overwrite the read-only memory to set the accessed and dirty
282 		 * bits.
283 		 */
284 		if (unlikely(!walker->pte_writable[level - 1]))
285 			continue;
286 
287 		ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, orig_pte, pte);
288 		if (ret)
289 			return ret;
290 
291 		kvm_vcpu_mark_page_dirty(vcpu, table_gfn);
292 		walker->ptes[level - 1] = pte;
293 	}
294 	return 0;
295 }
296 
297 static inline unsigned FNAME(gpte_pkeys)(struct kvm_vcpu *vcpu, u64 gpte)
298 {
299 	unsigned pkeys = 0;
300 #if PTTYPE == 64
301 	pte_t pte = {.pte = gpte};
302 
303 	pkeys = pte_flags_pkey(pte_flags(pte));
304 #endif
305 	return pkeys;
306 }
307 
308 static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu,
309 				       unsigned int level, unsigned int gpte)
310 {
311 	/*
312 	 * For EPT and PAE paging (both variants), bit 7 is either reserved at
313 	 * all level or indicates a huge page (ignoring CR3/EPTP).  In either
314 	 * case, bit 7 being set terminates the walk.
315 	 */
316 #if PTTYPE == 32
317 	/*
318 	 * 32-bit paging requires special handling because bit 7 is ignored if
319 	 * CR4.PSE=0, not reserved.  Clear bit 7 in the gpte if the level is
320 	 * greater than the last level for which bit 7 is the PAGE_SIZE bit.
321 	 *
322 	 * The RHS has bit 7 set iff level < (2 + PSE).  If it is clear, bit 7
323 	 * is not reserved and does not indicate a large page at this level,
324 	 * so clear PT_PAGE_SIZE_MASK in gpte if that is the case.
325 	 */
326 	gpte &= level - (PT32_ROOT_LEVEL + mmu->mmu_role.ext.cr4_pse);
327 #endif
328 	/*
329 	 * PG_LEVEL_4K always terminates.  The RHS has bit 7 set
330 	 * iff level <= PG_LEVEL_4K, which for our purpose means
331 	 * level == PG_LEVEL_4K; set PT_PAGE_SIZE_MASK in gpte then.
332 	 */
333 	gpte |= level - PG_LEVEL_4K - 1;
334 
335 	return gpte & PT_PAGE_SIZE_MASK;
336 }
337 /*
338  * Fetch a guest pte for a guest virtual address, or for an L2's GPA.
339  */
340 static int FNAME(walk_addr_generic)(struct guest_walker *walker,
341 				    struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
342 				    gpa_t addr, u32 access)
343 {
344 	int ret;
345 	pt_element_t pte;
346 	pt_element_t __user *ptep_user;
347 	gfn_t table_gfn;
348 	u64 pt_access, pte_access;
349 	unsigned index, accessed_dirty, pte_pkey;
350 	unsigned nested_access;
351 	gpa_t pte_gpa;
352 	bool have_ad;
353 	int offset;
354 	u64 walk_nx_mask = 0;
355 	const int write_fault = access & PFERR_WRITE_MASK;
356 	const int user_fault  = access & PFERR_USER_MASK;
357 	const int fetch_fault = access & PFERR_FETCH_MASK;
358 	u16 errcode = 0;
359 	gpa_t real_gpa;
360 	gfn_t gfn;
361 
362 	trace_kvm_mmu_pagetable_walk(addr, access);
363 retry_walk:
364 	walker->level = mmu->root_level;
365 	pte           = mmu->get_guest_pgd(vcpu);
366 	have_ad       = PT_HAVE_ACCESSED_DIRTY(mmu);
367 
368 #if PTTYPE == 64
369 	walk_nx_mask = 1ULL << PT64_NX_SHIFT;
370 	if (walker->level == PT32E_ROOT_LEVEL) {
371 		pte = mmu->get_pdptr(vcpu, (addr >> 30) & 3);
372 		trace_kvm_mmu_paging_element(pte, walker->level);
373 		if (!FNAME(is_present_gpte)(pte))
374 			goto error;
375 		--walker->level;
376 	}
377 #endif
378 	walker->max_level = walker->level;
379 	ASSERT(!(is_long_mode(vcpu) && !is_pae(vcpu)));
380 
381 	/*
382 	 * FIXME: on Intel processors, loads of the PDPTE registers for PAE paging
383 	 * by the MOV to CR instruction are treated as reads and do not cause the
384 	 * processor to set the dirty flag in any EPT paging-structure entry.
385 	 */
386 	nested_access = (have_ad ? PFERR_WRITE_MASK : 0) | PFERR_USER_MASK;
387 
388 	pte_access = ~0;
389 	++walker->level;
390 
391 	do {
392 		unsigned long host_addr;
393 
394 		pt_access = pte_access;
395 		--walker->level;
396 
397 		index = PT_INDEX(addr, walker->level);
398 		table_gfn = gpte_to_gfn(pte);
399 		offset    = index * sizeof(pt_element_t);
400 		pte_gpa   = gfn_to_gpa(table_gfn) + offset;
401 
402 		BUG_ON(walker->level < 1);
403 		walker->table_gfn[walker->level - 1] = table_gfn;
404 		walker->pte_gpa[walker->level - 1] = pte_gpa;
405 
406 		real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(table_gfn),
407 					     nested_access, &walker->fault);
408 
409 		/*
410 		 * FIXME: This can happen if emulation (for of an INS/OUTS
411 		 * instruction) triggers a nested page fault.  The exit
412 		 * qualification / exit info field will incorrectly have
413 		 * "guest page access" as the nested page fault's cause,
414 		 * instead of "guest page structure access".  To fix this,
415 		 * the x86_exception struct should be augmented with enough
416 		 * information to fix the exit_qualification or exit_info_1
417 		 * fields.
418 		 */
419 		if (unlikely(real_gpa == UNMAPPED_GVA))
420 			return 0;
421 
422 		host_addr = kvm_vcpu_gfn_to_hva_prot(vcpu, gpa_to_gfn(real_gpa),
423 					    &walker->pte_writable[walker->level - 1]);
424 		if (unlikely(kvm_is_error_hva(host_addr)))
425 			goto error;
426 
427 		ptep_user = (pt_element_t __user *)((void *)host_addr + offset);
428 		if (unlikely(__get_user(pte, ptep_user)))
429 			goto error;
430 		walker->ptep_user[walker->level - 1] = ptep_user;
431 
432 		trace_kvm_mmu_paging_element(pte, walker->level);
433 
434 		/*
435 		 * Inverting the NX it lets us AND it like other
436 		 * permission bits.
437 		 */
438 		pte_access = pt_access & (pte ^ walk_nx_mask);
439 
440 		if (unlikely(!FNAME(is_present_gpte)(pte)))
441 			goto error;
442 
443 		if (unlikely(FNAME(is_rsvd_bits_set)(mmu, pte, walker->level))) {
444 			errcode = PFERR_RSVD_MASK | PFERR_PRESENT_MASK;
445 			goto error;
446 		}
447 
448 		walker->ptes[walker->level - 1] = pte;
449 
450 		/* Convert to ACC_*_MASK flags for struct guest_walker.  */
451 		walker->pt_access[walker->level - 1] = FNAME(gpte_access)(pt_access ^ walk_nx_mask);
452 	} while (!FNAME(is_last_gpte)(mmu, walker->level, pte));
453 
454 	pte_pkey = FNAME(gpte_pkeys)(vcpu, pte);
455 	accessed_dirty = have_ad ? pte_access & PT_GUEST_ACCESSED_MASK : 0;
456 
457 	/* Convert to ACC_*_MASK flags for struct guest_walker.  */
458 	walker->pte_access = FNAME(gpte_access)(pte_access ^ walk_nx_mask);
459 	errcode = permission_fault(vcpu, mmu, walker->pte_access, pte_pkey, access);
460 	if (unlikely(errcode))
461 		goto error;
462 
463 	gfn = gpte_to_gfn_lvl(pte, walker->level);
464 	gfn += (addr & PT_LVL_OFFSET_MASK(walker->level)) >> PAGE_SHIFT;
465 
466 	if (PTTYPE == 32 && walker->level > PG_LEVEL_4K && is_cpuid_PSE36())
467 		gfn += pse36_gfn_delta(pte);
468 
469 	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(gfn), access, &walker->fault);
470 	if (real_gpa == UNMAPPED_GVA)
471 		return 0;
472 
473 	walker->gfn = real_gpa >> PAGE_SHIFT;
474 
475 	if (!write_fault)
476 		FNAME(protect_clean_gpte)(mmu, &walker->pte_access, pte);
477 	else
478 		/*
479 		 * On a write fault, fold the dirty bit into accessed_dirty.
480 		 * For modes without A/D bits support accessed_dirty will be
481 		 * always clear.
482 		 */
483 		accessed_dirty &= pte >>
484 			(PT_GUEST_DIRTY_SHIFT - PT_GUEST_ACCESSED_SHIFT);
485 
486 	if (unlikely(!accessed_dirty)) {
487 		ret = FNAME(update_accessed_dirty_bits)(vcpu, mmu, walker,
488 							addr, write_fault);
489 		if (unlikely(ret < 0))
490 			goto error;
491 		else if (ret)
492 			goto retry_walk;
493 	}
494 
495 	pgprintk("%s: pte %llx pte_access %x pt_access %x\n",
496 		 __func__, (u64)pte, walker->pte_access,
497 		 walker->pt_access[walker->level - 1]);
498 	return 1;
499 
500 error:
501 	errcode |= write_fault | user_fault;
502 	if (fetch_fault && (is_efer_nx(mmu) || is_cr4_smep(mmu)))
503 		errcode |= PFERR_FETCH_MASK;
504 
505 	walker->fault.vector = PF_VECTOR;
506 	walker->fault.error_code_valid = true;
507 	walker->fault.error_code = errcode;
508 
509 #if PTTYPE == PTTYPE_EPT
510 	/*
511 	 * Use PFERR_RSVD_MASK in error_code to to tell if EPT
512 	 * misconfiguration requires to be injected. The detection is
513 	 * done by is_rsvd_bits_set() above.
514 	 *
515 	 * We set up the value of exit_qualification to inject:
516 	 * [2:0] - Derive from the access bits. The exit_qualification might be
517 	 *         out of date if it is serving an EPT misconfiguration.
518 	 * [5:3] - Calculated by the page walk of the guest EPT page tables
519 	 * [7:8] - Derived from [7:8] of real exit_qualification
520 	 *
521 	 * The other bits are set to 0.
522 	 */
523 	if (!(errcode & PFERR_RSVD_MASK)) {
524 		vcpu->arch.exit_qualification &= 0x180;
525 		if (write_fault)
526 			vcpu->arch.exit_qualification |= EPT_VIOLATION_ACC_WRITE;
527 		if (user_fault)
528 			vcpu->arch.exit_qualification |= EPT_VIOLATION_ACC_READ;
529 		if (fetch_fault)
530 			vcpu->arch.exit_qualification |= EPT_VIOLATION_ACC_INSTR;
531 		vcpu->arch.exit_qualification |= (pte_access & 0x7) << 3;
532 	}
533 #endif
534 	walker->fault.address = addr;
535 	walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;
536 	walker->fault.async_page_fault = false;
537 
538 	trace_kvm_mmu_walker_error(walker->fault.error_code);
539 	return 0;
540 }
541 
542 static int FNAME(walk_addr)(struct guest_walker *walker,
543 			    struct kvm_vcpu *vcpu, gpa_t addr, u32 access)
544 {
545 	return FNAME(walk_addr_generic)(walker, vcpu, vcpu->arch.mmu, addr,
546 					access);
547 }
548 
549 static bool
550 FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
551 		     u64 *spte, pt_element_t gpte, bool no_dirty_log)
552 {
553 	struct kvm_memory_slot *slot;
554 	unsigned pte_access;
555 	gfn_t gfn;
556 	kvm_pfn_t pfn;
557 
558 	if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
559 		return false;
560 
561 	pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
562 
563 	gfn = gpte_to_gfn(gpte);
564 	pte_access = sp->role.access & FNAME(gpte_access)(gpte);
565 	FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
566 
567 	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn,
568 			no_dirty_log && (pte_access & ACC_WRITE_MASK));
569 	if (!slot)
570 		return false;
571 
572 	pfn = gfn_to_pfn_memslot_atomic(slot, gfn);
573 	if (is_error_pfn(pfn))
574 		return false;
575 
576 	mmu_set_spte(vcpu, slot, spte, pte_access, gfn, pfn, NULL);
577 	kvm_release_pfn_clean(pfn);
578 	return true;
579 }
580 
581 static bool FNAME(gpte_changed)(struct kvm_vcpu *vcpu,
582 				struct guest_walker *gw, int level)
583 {
584 	pt_element_t curr_pte;
585 	gpa_t base_gpa, pte_gpa = gw->pte_gpa[level - 1];
586 	u64 mask;
587 	int r, index;
588 
589 	if (level == PG_LEVEL_4K) {
590 		mask = PTE_PREFETCH_NUM * sizeof(pt_element_t) - 1;
591 		base_gpa = pte_gpa & ~mask;
592 		index = (pte_gpa - base_gpa) / sizeof(pt_element_t);
593 
594 		r = kvm_vcpu_read_guest_atomic(vcpu, base_gpa,
595 				gw->prefetch_ptes, sizeof(gw->prefetch_ptes));
596 		curr_pte = gw->prefetch_ptes[index];
597 	} else
598 		r = kvm_vcpu_read_guest_atomic(vcpu, pte_gpa,
599 				  &curr_pte, sizeof(curr_pte));
600 
601 	return r || curr_pte != gw->ptes[level - 1];
602 }
603 
604 static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
605 				u64 *sptep)
606 {
607 	struct kvm_mmu_page *sp;
608 	pt_element_t *gptep = gw->prefetch_ptes;
609 	u64 *spte;
610 	int i;
611 
612 	sp = sptep_to_sp(sptep);
613 
614 	if (sp->role.level > PG_LEVEL_4K)
615 		return;
616 
617 	/*
618 	 * If addresses are being invalidated, skip prefetching to avoid
619 	 * accidentally prefetching those addresses.
620 	 */
621 	if (unlikely(vcpu->kvm->mmu_notifier_count))
622 		return;
623 
624 	if (sp->role.direct)
625 		return __direct_pte_prefetch(vcpu, sp, sptep);
626 
627 	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
628 	spte = sp->spt + i;
629 
630 	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
631 		if (spte == sptep)
632 			continue;
633 
634 		if (is_shadow_present_pte(*spte))
635 			continue;
636 
637 		if (!FNAME(prefetch_gpte)(vcpu, sp, spte, gptep[i], true))
638 			break;
639 	}
640 }
641 
642 /*
643  * Fetch a shadow pte for a specific level in the paging hierarchy.
644  * If the guest tries to write a write-protected page, we need to
645  * emulate this operation, return 1 to indicate this case.
646  */
647 static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
648 			 struct guest_walker *gw)
649 {
650 	struct kvm_mmu_page *sp = NULL;
651 	struct kvm_shadow_walk_iterator it;
652 	unsigned int direct_access, access;
653 	int top_level, ret;
654 	gfn_t base_gfn = fault->gfn;
655 
656 	WARN_ON_ONCE(gw->gfn != base_gfn);
657 	direct_access = gw->pte_access;
658 
659 	top_level = vcpu->arch.mmu->root_level;
660 	if (top_level == PT32E_ROOT_LEVEL)
661 		top_level = PT32_ROOT_LEVEL;
662 	/*
663 	 * Verify that the top-level gpte is still there.  Since the page
664 	 * is a root page, it is either write protected (and cannot be
665 	 * changed from now on) or it is invalid (in which case, we don't
666 	 * really care if it changes underneath us after this point).
667 	 */
668 	if (FNAME(gpte_changed)(vcpu, gw, top_level))
669 		goto out_gpte_changed;
670 
671 	if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa)))
672 		goto out_gpte_changed;
673 
674 	for (shadow_walk_init(&it, vcpu, fault->addr);
675 	     shadow_walk_okay(&it) && it.level > gw->level;
676 	     shadow_walk_next(&it)) {
677 		gfn_t table_gfn;
678 
679 		clear_sp_write_flooding_count(it.sptep);
680 		drop_large_spte(vcpu, it.sptep);
681 
682 		sp = NULL;
683 		if (!is_shadow_present_pte(*it.sptep)) {
684 			table_gfn = gw->table_gfn[it.level - 2];
685 			access = gw->pt_access[it.level - 2];
686 			sp = kvm_mmu_get_page(vcpu, table_gfn, fault->addr,
687 					      it.level-1, false, access);
688 			/*
689 			 * We must synchronize the pagetable before linking it
690 			 * because the guest doesn't need to flush tlb when
691 			 * the gpte is changed from non-present to present.
692 			 * Otherwise, the guest may use the wrong mapping.
693 			 *
694 			 * For PG_LEVEL_4K, kvm_mmu_get_page() has already
695 			 * synchronized it transiently via kvm_sync_page().
696 			 *
697 			 * For higher level pagetable, we synchronize it via
698 			 * the slower mmu_sync_children().  If it needs to
699 			 * break, some progress has been made; return
700 			 * RET_PF_RETRY and retry on the next #PF.
701 			 * KVM_REQ_MMU_SYNC is not necessary but it
702 			 * expedites the process.
703 			 */
704 			if (sp->unsync_children &&
705 			    mmu_sync_children(vcpu, sp, false))
706 				return RET_PF_RETRY;
707 		}
708 
709 		/*
710 		 * Verify that the gpte in the page we've just write
711 		 * protected is still there.
712 		 */
713 		if (FNAME(gpte_changed)(vcpu, gw, it.level - 1))
714 			goto out_gpte_changed;
715 
716 		if (sp)
717 			link_shadow_page(vcpu, it.sptep, sp);
718 	}
719 
720 	kvm_mmu_hugepage_adjust(vcpu, fault);
721 
722 	trace_kvm_mmu_spte_requested(fault);
723 
724 	for (; shadow_walk_okay(&it); shadow_walk_next(&it)) {
725 		clear_sp_write_flooding_count(it.sptep);
726 
727 		/*
728 		 * We cannot overwrite existing page tables with an NX
729 		 * large page, as the leaf could be executable.
730 		 */
731 		if (fault->nx_huge_page_workaround_enabled)
732 			disallowed_hugepage_adjust(fault, *it.sptep, it.level);
733 
734 		base_gfn = fault->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1);
735 		if (it.level == fault->goal_level)
736 			break;
737 
738 		validate_direct_spte(vcpu, it.sptep, direct_access);
739 
740 		drop_large_spte(vcpu, it.sptep);
741 
742 		if (!is_shadow_present_pte(*it.sptep)) {
743 			sp = kvm_mmu_get_page(vcpu, base_gfn, fault->addr,
744 					      it.level - 1, true, direct_access);
745 			link_shadow_page(vcpu, it.sptep, sp);
746 			if (fault->huge_page_disallowed &&
747 			    fault->req_level >= it.level)
748 				account_huge_nx_page(vcpu->kvm, sp);
749 		}
750 	}
751 
752 	if (WARN_ON_ONCE(it.level != fault->goal_level))
753 		return -EFAULT;
754 
755 	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, gw->pte_access,
756 			   base_gfn, fault->pfn, fault);
757 	if (ret == RET_PF_SPURIOUS)
758 		return ret;
759 
760 	FNAME(pte_prefetch)(vcpu, gw, it.sptep);
761 	++vcpu->stat.pf_fixed;
762 	return ret;
763 
764 out_gpte_changed:
765 	return RET_PF_RETRY;
766 }
767 
768  /*
769  * To see whether the mapped gfn can write its page table in the current
770  * mapping.
771  *
772  * It is the helper function of FNAME(page_fault). When guest uses large page
773  * size to map the writable gfn which is used as current page table, we should
774  * force kvm to use small page size to map it because new shadow page will be
775  * created when kvm establishes shadow page table that stop kvm using large
776  * page size. Do it early can avoid unnecessary #PF and emulation.
777  *
778  * @write_fault_to_shadow_pgtable will return true if the fault gfn is
779  * currently used as its page table.
780  *
781  * Note: the PDPT page table is not checked for PAE-32 bit guest. It is ok
782  * since the PDPT is always shadowed, that means, we can not use large page
783  * size to map the gfn which is used as PDPT.
784  */
785 static bool
786 FNAME(is_self_change_mapping)(struct kvm_vcpu *vcpu,
787 			      struct guest_walker *walker, bool user_fault,
788 			      bool *write_fault_to_shadow_pgtable)
789 {
790 	int level;
791 	gfn_t mask = ~(KVM_PAGES_PER_HPAGE(walker->level) - 1);
792 	bool self_changed = false;
793 
794 	if (!(walker->pte_access & ACC_WRITE_MASK ||
795 	    (!is_cr0_wp(vcpu->arch.mmu) && !user_fault)))
796 		return false;
797 
798 	for (level = walker->level; level <= walker->max_level; level++) {
799 		gfn_t gfn = walker->gfn ^ walker->table_gfn[level - 1];
800 
801 		self_changed |= !(gfn & mask);
802 		*write_fault_to_shadow_pgtable |= !gfn;
803 	}
804 
805 	return self_changed;
806 }
807 
808 /*
809  * Page fault handler.  There are several causes for a page fault:
810  *   - there is no shadow pte for the guest pte
811  *   - write access through a shadow pte marked read only so that we can set
812  *     the dirty bit
813  *   - write access to a shadow pte marked read only so we can update the page
814  *     dirty bitmap, when userspace requests it
815  *   - mmio access; in this case we will never install a present shadow pte
816  *   - normal guest page fault due to the guest pte marked not present, not
817  *     writable, or not executable
818  *
819  *  Returns: 1 if we need to emulate the instruction, 0 otherwise, or
820  *           a negative value on error.
821  */
822 static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
823 {
824 	struct guest_walker walker;
825 	int r;
826 	unsigned long mmu_seq;
827 	bool is_self_change_mapping;
828 
829 	pgprintk("%s: addr %lx err %x\n", __func__, fault->addr, fault->error_code);
830 	WARN_ON_ONCE(fault->is_tdp);
831 
832 	/*
833 	 * Look up the guest pte for the faulting address.
834 	 * If PFEC.RSVD is set, this is a shadow page fault.
835 	 * The bit needs to be cleared before walking guest page tables.
836 	 */
837 	r = FNAME(walk_addr)(&walker, vcpu, fault->addr,
838 			     fault->error_code & ~PFERR_RSVD_MASK);
839 
840 	/*
841 	 * The page is not mapped by the guest.  Let the guest handle it.
842 	 */
843 	if (!r) {
844 		pgprintk("%s: guest page fault\n", __func__);
845 		if (!fault->prefetch)
846 			kvm_inject_emulated_page_fault(vcpu, &walker.fault);
847 
848 		return RET_PF_RETRY;
849 	}
850 
851 	fault->gfn = walker.gfn;
852 	fault->slot = kvm_vcpu_gfn_to_memslot(vcpu, fault->gfn);
853 
854 	if (page_fault_handle_page_track(vcpu, fault)) {
855 		shadow_page_table_clear_flood(vcpu, fault->addr);
856 		return RET_PF_EMULATE;
857 	}
858 
859 	r = mmu_topup_memory_caches(vcpu, true);
860 	if (r)
861 		return r;
862 
863 	vcpu->arch.write_fault_to_shadow_pgtable = false;
864 
865 	is_self_change_mapping = FNAME(is_self_change_mapping)(vcpu,
866 	      &walker, fault->user, &vcpu->arch.write_fault_to_shadow_pgtable);
867 
868 	if (is_self_change_mapping)
869 		fault->max_level = PG_LEVEL_4K;
870 	else
871 		fault->max_level = walker.level;
872 
873 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
874 	smp_rmb();
875 
876 	if (kvm_faultin_pfn(vcpu, fault, &r))
877 		return r;
878 
879 	if (handle_abnormal_pfn(vcpu, fault, walker.pte_access, &r))
880 		return r;
881 
882 	/*
883 	 * Do not change pte_access if the pfn is a mmio page, otherwise
884 	 * we will cache the incorrect access into mmio spte.
885 	 */
886 	if (fault->write && !(walker.pte_access & ACC_WRITE_MASK) &&
887 	    !is_cr0_wp(vcpu->arch.mmu) && !fault->user && fault->slot) {
888 		walker.pte_access |= ACC_WRITE_MASK;
889 		walker.pte_access &= ~ACC_USER_MASK;
890 
891 		/*
892 		 * If we converted a user page to a kernel page,
893 		 * so that the kernel can write to it when cr0.wp=0,
894 		 * then we should prevent the kernel from executing it
895 		 * if SMEP is enabled.
896 		 */
897 		if (is_cr4_smep(vcpu->arch.mmu))
898 			walker.pte_access &= ~ACC_EXEC_MASK;
899 	}
900 
901 	r = RET_PF_RETRY;
902 	write_lock(&vcpu->kvm->mmu_lock);
903 
904 	if (is_page_fault_stale(vcpu, fault, mmu_seq))
905 		goto out_unlock;
906 
907 	kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT);
908 	r = make_mmu_pages_available(vcpu);
909 	if (r)
910 		goto out_unlock;
911 	r = FNAME(fetch)(vcpu, fault, &walker);
912 	kvm_mmu_audit(vcpu, AUDIT_POST_PAGE_FAULT);
913 
914 out_unlock:
915 	write_unlock(&vcpu->kvm->mmu_lock);
916 	kvm_release_pfn_clean(fault->pfn);
917 	return r;
918 }
919 
920 static gpa_t FNAME(get_level1_sp_gpa)(struct kvm_mmu_page *sp)
921 {
922 	int offset = 0;
923 
924 	WARN_ON(sp->role.level != PG_LEVEL_4K);
925 
926 	if (PTTYPE == 32)
927 		offset = sp->role.quadrant << PT64_LEVEL_BITS;
928 
929 	return gfn_to_gpa(sp->gfn) + offset * sizeof(pt_element_t);
930 }
931 
932 static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa)
933 {
934 	struct kvm_shadow_walk_iterator iterator;
935 	struct kvm_mmu_page *sp;
936 	u64 old_spte;
937 	int level;
938 	u64 *sptep;
939 
940 	vcpu_clear_mmio_info(vcpu, gva);
941 
942 	/*
943 	 * No need to check return value here, rmap_can_add() can
944 	 * help us to skip pte prefetch later.
945 	 */
946 	mmu_topup_memory_caches(vcpu, true);
947 
948 	if (!VALID_PAGE(root_hpa)) {
949 		WARN_ON(1);
950 		return;
951 	}
952 
953 	write_lock(&vcpu->kvm->mmu_lock);
954 	for_each_shadow_entry_using_root(vcpu, root_hpa, gva, iterator) {
955 		level = iterator.level;
956 		sptep = iterator.sptep;
957 
958 		sp = sptep_to_sp(sptep);
959 		old_spte = *sptep;
960 		if (is_last_spte(old_spte, level)) {
961 			pt_element_t gpte;
962 			gpa_t pte_gpa;
963 
964 			if (!sp->unsync)
965 				break;
966 
967 			pte_gpa = FNAME(get_level1_sp_gpa)(sp);
968 			pte_gpa += (sptep - sp->spt) * sizeof(pt_element_t);
969 
970 			mmu_page_zap_pte(vcpu->kvm, sp, sptep, NULL);
971 			if (is_shadow_present_pte(old_spte))
972 				kvm_flush_remote_tlbs_with_address(vcpu->kvm,
973 					sp->gfn, KVM_PAGES_PER_HPAGE(sp->role.level));
974 
975 			if (!rmap_can_add(vcpu))
976 				break;
977 
978 			if (kvm_vcpu_read_guest_atomic(vcpu, pte_gpa, &gpte,
979 						       sizeof(pt_element_t)))
980 				break;
981 
982 			FNAME(prefetch_gpte)(vcpu, sp, sptep, gpte, false);
983 		}
984 
985 		if (!sp->unsync_children)
986 			break;
987 	}
988 	write_unlock(&vcpu->kvm->mmu_lock);
989 }
990 
991 /* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */
992 static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
993 			       gpa_t addr, u32 access,
994 			       struct x86_exception *exception)
995 {
996 	struct guest_walker walker;
997 	gpa_t gpa = UNMAPPED_GVA;
998 	int r;
999 
1000 #ifndef CONFIG_X86_64
1001 	/* A 64-bit GVA should be impossible on 32-bit KVM. */
1002 	WARN_ON_ONCE((addr >> 32) && mmu == vcpu->arch.walk_mmu);
1003 #endif
1004 
1005 	r = FNAME(walk_addr_generic)(&walker, vcpu, mmu, addr, access);
1006 
1007 	if (r) {
1008 		gpa = gfn_to_gpa(walker.gfn);
1009 		gpa |= addr & ~PAGE_MASK;
1010 	} else if (exception)
1011 		*exception = walker.fault;
1012 
1013 	return gpa;
1014 }
1015 
1016 /*
1017  * Using the cached information from sp->gfns is safe because:
1018  * - The spte has a reference to the struct page, so the pfn for a given gfn
1019  *   can't change unless all sptes pointing to it are nuked first.
1020  *
1021  * Returns
1022  * < 0: the sp should be zapped
1023  *   0: the sp is synced and no tlb flushing is required
1024  * > 0: the sp is synced and tlb flushing is required
1025  */
1026 static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1027 {
1028 	union kvm_mmu_page_role mmu_role = vcpu->arch.mmu->mmu_role.base;
1029 	int i;
1030 	bool host_writable;
1031 	gpa_t first_pte_gpa;
1032 	bool flush = false;
1033 
1034 	/*
1035 	 * Ignore various flags when verifying that it's safe to sync a shadow
1036 	 * page using the current MMU context.
1037 	 *
1038 	 *  - level: not part of the overall MMU role and will never match as the MMU's
1039 	 *           level tracks the root level
1040 	 *  - access: updated based on the new guest PTE
1041 	 *  - quadrant: not part of the overall MMU role (similar to level)
1042 	 */
1043 	const union kvm_mmu_page_role sync_role_ign = {
1044 		.level = 0xf,
1045 		.access = 0x7,
1046 		.quadrant = 0x3,
1047 	};
1048 
1049 	/*
1050 	 * Direct pages can never be unsync, and KVM should never attempt to
1051 	 * sync a shadow page for a different MMU context, e.g. if the role
1052 	 * differs then the memslot lookup (SMM vs. non-SMM) will be bogus, the
1053 	 * reserved bits checks will be wrong, etc...
1054 	 */
1055 	if (WARN_ON_ONCE(sp->role.direct ||
1056 			 (sp->role.word ^ mmu_role.word) & ~sync_role_ign.word))
1057 		return -1;
1058 
1059 	first_pte_gpa = FNAME(get_level1_sp_gpa)(sp);
1060 
1061 	for (i = 0; i < PT64_ENT_PER_PAGE; i++) {
1062 		u64 *sptep, spte;
1063 		struct kvm_memory_slot *slot;
1064 		unsigned pte_access;
1065 		pt_element_t gpte;
1066 		gpa_t pte_gpa;
1067 		gfn_t gfn;
1068 
1069 		if (!sp->spt[i])
1070 			continue;
1071 
1072 		pte_gpa = first_pte_gpa + i * sizeof(pt_element_t);
1073 
1074 		if (kvm_vcpu_read_guest_atomic(vcpu, pte_gpa, &gpte,
1075 					       sizeof(pt_element_t)))
1076 			return -1;
1077 
1078 		if (FNAME(prefetch_invalid_gpte)(vcpu, sp, &sp->spt[i], gpte)) {
1079 			flush = true;
1080 			continue;
1081 		}
1082 
1083 		gfn = gpte_to_gfn(gpte);
1084 		pte_access = sp->role.access;
1085 		pte_access &= FNAME(gpte_access)(gpte);
1086 		FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
1087 
1088 		if (sync_mmio_spte(vcpu, &sp->spt[i], gfn, pte_access))
1089 			continue;
1090 
1091 		if (gfn != sp->gfns[i]) {
1092 			drop_spte(vcpu->kvm, &sp->spt[i]);
1093 			flush = true;
1094 			continue;
1095 		}
1096 
1097 		sptep = &sp->spt[i];
1098 		spte = *sptep;
1099 		host_writable = spte & shadow_host_writable_mask;
1100 		slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1101 		make_spte(vcpu, sp, slot, pte_access, gfn,
1102 			  spte_to_pfn(spte), spte, true, false,
1103 			  host_writable, &spte);
1104 
1105 		flush |= mmu_spte_update(sptep, spte);
1106 	}
1107 
1108 	return flush;
1109 }
1110 
1111 #undef pt_element_t
1112 #undef guest_walker
1113 #undef FNAME
1114 #undef PT_BASE_ADDR_MASK
1115 #undef PT_INDEX
1116 #undef PT_LVL_ADDR_MASK
1117 #undef PT_LVL_OFFSET_MASK
1118 #undef PT_LEVEL_BITS
1119 #undef PT_MAX_FULL_LEVELS
1120 #undef gpte_to_gfn
1121 #undef gpte_to_gfn_lvl
1122 #undef CMPXCHG
1123 #undef PT_GUEST_ACCESSED_MASK
1124 #undef PT_GUEST_DIRTY_MASK
1125 #undef PT_GUEST_DIRTY_SHIFT
1126 #undef PT_GUEST_ACCESSED_SHIFT
1127 #undef PT_HAVE_ACCESSED_DIRTY
1128