xref: /illumos-gate/usr/src/uts/intel/io/vmm/vmm_sol_ept.c (revision b9b43e848e8a93ea5f612b51b05c8bb8bf612ee2)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
12 
13 /*
14  * Copyright 2019 Joyent, Inc.
15  * Copyright 2023 Oxide Computer Company
16  */
17 
18 #include <sys/types.h>
19 #include <sys/param.h>
20 #include <sys/atomic.h>
21 #include <sys/kmem.h>
22 #include <sys/machsystm.h>
23 #include <sys/mman.h>
24 #include <sys/x86_archext.h>
25 #include <vm/hat_pte.h>
26 
27 #include <sys/vmm_gpt.h>
28 #include <sys/vmm_vm.h>
29 
30 #define	EPT_R		(1 << 0)
31 #define	EPT_W		(1 << 1)
32 #define	EPT_X		(1 << 2)
33 #define	EPT_RWX		(EPT_R | EPT_W | EPT_X)
34 #define	EPT_LGPG	(1 << 7)
35 #define	EPT_ACCESSED	(1 << 8)
36 #define	EPT_DIRTY	(1 << 9)
37 
38 #define	EPT_PA_MASK	(0x000ffffffffff000ull)
39 
40 #define	EPT_MAX_LEVELS	4
41 CTASSERT(EPT_MAX_LEVELS <= MAX_GPT_LEVEL);
42 
43 #define	EPTP_FLAG_ACCESSED_DIRTY	(1 << 6)
44 
45 CTASSERT(EPT_R == PROT_READ);
46 CTASSERT(EPT_W == PROT_WRITE);
47 CTASSERT(EPT_X == PROT_EXEC);
48 
49 static uint_t
ept_pte_prot(uint64_t pte)50 ept_pte_prot(uint64_t pte)
51 {
52 	return (pte & EPT_RWX);
53 }
54 
55 static inline uint64_t
ept_attr_to_pat(uint8_t attr)56 ept_attr_to_pat(uint8_t attr)
57 {
58 	uint64_t bits = attr & 0x7;
59 	return (bits << 3);
60 }
61 
62 static uint64_t
ept_map_table(uint64_t pfn)63 ept_map_table(uint64_t pfn)
64 {
65 	const uint64_t paddr = pfn_to_pa(pfn) & EPT_PA_MASK;
66 	return (paddr | EPT_RWX);
67 }
68 
69 static uint64_t
ept_map_page(uint64_t pfn,uint_t prot,uint8_t attr)70 ept_map_page(uint64_t pfn, uint_t prot, uint8_t attr)
71 {
72 	const uint64_t paddr = pfn_to_pa(pfn) & EPT_PA_MASK;
73 	const uint64_t pat = ept_attr_to_pat(attr);
74 	const uint64_t rprot = prot & EPT_RWX;
75 	return (paddr | pat | rprot);
76 }
77 
78 static uint64_t
ept_pte_pfn(uint64_t pte)79 ept_pte_pfn(uint64_t pte)
80 {
81 	return (mmu_btop(pte & PT_PADDR));
82 }
83 
84 static bool
ept_pte_is_present(uint64_t pte)85 ept_pte_is_present(uint64_t pte)
86 {
87 	return ((pte & EPT_RWX) != 0);
88 }
89 
90 static uint_t
ept_reset_bits(volatile uint64_t * entry,uint64_t mask,uint64_t bits)91 ept_reset_bits(volatile uint64_t *entry, uint64_t mask, uint64_t bits)
92 {
93 	uint64_t pte, newpte, oldpte = 0;
94 
95 	/*
96 	 * We use volatile and atomic ops here because we may be
97 	 * racing against hardware modifying these bits.
98 	 */
99 	VERIFY3P(entry, !=, NULL);
100 	oldpte = *entry;
101 	do {
102 		pte = oldpte;
103 		newpte = (pte & ~mask) | bits;
104 		oldpte = atomic_cas_64(entry, pte, newpte);
105 	} while (oldpte != pte);
106 
107 	return (oldpte & mask);
108 }
109 
110 static uint_t
ept_reset_dirty(uint64_t * entry,bool on)111 ept_reset_dirty(uint64_t *entry, bool on)
112 {
113 	return (ept_reset_bits(entry, EPT_DIRTY,
114 	    on ? (EPT_DIRTY | EPT_ACCESSED) : 0));
115 }
116 
117 static uint_t
ept_reset_accessed(uint64_t * entry,bool on)118 ept_reset_accessed(uint64_t *entry, bool on)
119 {
120 	return (ept_reset_bits(entry, EPT_DIRTY | EPT_ACCESSED,
121 	    on ? EPT_ACCESSED : 0));
122 }
123 
124 static bool
ept_query(uint64_t * entry,vmm_gpt_query_t query)125 ept_query(uint64_t *entry, vmm_gpt_query_t query)
126 {
127 	ASSERT(entry != NULL);
128 
129 	const uint64_t pte = *entry;
130 	switch (query) {
131 	case VGQ_ACCESSED:
132 		return ((pte & EPT_ACCESSED) != 0);
133 	case VGQ_DIRTY:
134 		return ((pte & EPT_DIRTY) != 0);
135 	default:
136 		panic("unrecognized query: %d", query);
137 	}
138 }
139 
140 static uint64_t
ept_get_pmtp(pfn_t root_pfn,bool track_dirty)141 ept_get_pmtp(pfn_t root_pfn, bool track_dirty)
142 {
143 	const uint64_t ad_flag = track_dirty ? EPTP_FLAG_ACCESSED_DIRTY : 0;
144 	return ((root_pfn << PAGESHIFT | ad_flag |
145 	    (EPT_MAX_LEVELS - 1) << 3 | MTRR_TYPE_WB));
146 }
147 
148 static bool
ept_hw_ad_supported(void)149 ept_hw_ad_supported(void)
150 {
151 	uint64_t ept_caps = rdmsr(MSR_IA32_VMX_EPT_VPID_CAP);
152 	return ((ept_caps & IA32_VMX_EPT_VPID_HW_AD) != 0);
153 }
154 
155 vmm_pte_ops_t ept_pte_ops = {
156 	.vpeo_map_table		= ept_map_table,
157 	.vpeo_map_page		= ept_map_page,
158 	.vpeo_pte_pfn		= ept_pte_pfn,
159 	.vpeo_pte_is_present	= ept_pte_is_present,
160 	.vpeo_pte_prot		= ept_pte_prot,
161 	.vpeo_reset_dirty	= ept_reset_dirty,
162 	.vpeo_reset_accessed	= ept_reset_accessed,
163 	.vpeo_query		= ept_query,
164 	.vpeo_get_pmtp		= ept_get_pmtp,
165 	.vpeo_hw_ad_supported	= ept_hw_ad_supported,
166 };
167