1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11 /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
12
13 /*
14 * Copyright 2019 Joyent, Inc.
15 * Copyright 2023 Oxide Computer Company
16 */
17
18 #include <sys/types.h>
19 #include <sys/param.h>
20 #include <sys/atomic.h>
21 #include <sys/kmem.h>
22 #include <sys/machsystm.h>
23 #include <sys/mman.h>
24 #include <sys/x86_archext.h>
25 #include <vm/hat_pte.h>
26
27 #include <sys/vmm_gpt.h>
28 #include <sys/vmm_vm.h>
29
30 #define EPT_R (1 << 0)
31 #define EPT_W (1 << 1)
32 #define EPT_X (1 << 2)
33 #define EPT_RWX (EPT_R | EPT_W | EPT_X)
34 #define EPT_LGPG (1 << 7)
35 #define EPT_ACCESSED (1 << 8)
36 #define EPT_DIRTY (1 << 9)
37
38 #define EPT_PA_MASK (0x000ffffffffff000ull)
39
40 #define EPT_MAX_LEVELS 4
41 CTASSERT(EPT_MAX_LEVELS <= MAX_GPT_LEVEL);
42
43 #define EPTP_FLAG_ACCESSED_DIRTY (1 << 6)
44
45 CTASSERT(EPT_R == PROT_READ);
46 CTASSERT(EPT_W == PROT_WRITE);
47 CTASSERT(EPT_X == PROT_EXEC);
48
49 static uint_t
ept_pte_prot(uint64_t pte)50 ept_pte_prot(uint64_t pte)
51 {
52 return (pte & EPT_RWX);
53 }
54
55 static inline uint64_t
ept_attr_to_pat(uint8_t attr)56 ept_attr_to_pat(uint8_t attr)
57 {
58 uint64_t bits = attr & 0x7;
59 return (bits << 3);
60 }
61
62 static uint64_t
ept_map_table(uint64_t pfn)63 ept_map_table(uint64_t pfn)
64 {
65 const uint64_t paddr = pfn_to_pa(pfn) & EPT_PA_MASK;
66 return (paddr | EPT_RWX);
67 }
68
69 static uint64_t
ept_map_page(uint64_t pfn,uint_t prot,uint8_t attr)70 ept_map_page(uint64_t pfn, uint_t prot, uint8_t attr)
71 {
72 const uint64_t paddr = pfn_to_pa(pfn) & EPT_PA_MASK;
73 const uint64_t pat = ept_attr_to_pat(attr);
74 const uint64_t rprot = prot & EPT_RWX;
75 return (paddr | pat | rprot);
76 }
77
78 static uint64_t
ept_pte_pfn(uint64_t pte)79 ept_pte_pfn(uint64_t pte)
80 {
81 return (mmu_btop(pte & PT_PADDR));
82 }
83
84 static bool
ept_pte_is_present(uint64_t pte)85 ept_pte_is_present(uint64_t pte)
86 {
87 return ((pte & EPT_RWX) != 0);
88 }
89
90 static uint_t
ept_reset_bits(volatile uint64_t * entry,uint64_t mask,uint64_t bits)91 ept_reset_bits(volatile uint64_t *entry, uint64_t mask, uint64_t bits)
92 {
93 uint64_t pte, newpte, oldpte = 0;
94
95 /*
96 * We use volatile and atomic ops here because we may be
97 * racing against hardware modifying these bits.
98 */
99 VERIFY3P(entry, !=, NULL);
100 oldpte = *entry;
101 do {
102 pte = oldpte;
103 newpte = (pte & ~mask) | bits;
104 oldpte = atomic_cas_64(entry, pte, newpte);
105 } while (oldpte != pte);
106
107 return (oldpte & mask);
108 }
109
110 static uint_t
ept_reset_dirty(uint64_t * entry,bool on)111 ept_reset_dirty(uint64_t *entry, bool on)
112 {
113 return (ept_reset_bits(entry, EPT_DIRTY,
114 on ? (EPT_DIRTY | EPT_ACCESSED) : 0));
115 }
116
117 static uint_t
ept_reset_accessed(uint64_t * entry,bool on)118 ept_reset_accessed(uint64_t *entry, bool on)
119 {
120 return (ept_reset_bits(entry, EPT_DIRTY | EPT_ACCESSED,
121 on ? EPT_ACCESSED : 0));
122 }
123
124 static bool
ept_query(uint64_t * entry,vmm_gpt_query_t query)125 ept_query(uint64_t *entry, vmm_gpt_query_t query)
126 {
127 ASSERT(entry != NULL);
128
129 const uint64_t pte = *entry;
130 switch (query) {
131 case VGQ_ACCESSED:
132 return ((pte & EPT_ACCESSED) != 0);
133 case VGQ_DIRTY:
134 return ((pte & EPT_DIRTY) != 0);
135 default:
136 panic("unrecognized query: %d", query);
137 }
138 }
139
140 static uint64_t
ept_get_pmtp(pfn_t root_pfn,bool track_dirty)141 ept_get_pmtp(pfn_t root_pfn, bool track_dirty)
142 {
143 const uint64_t ad_flag = track_dirty ? EPTP_FLAG_ACCESSED_DIRTY : 0;
144 return ((root_pfn << PAGESHIFT | ad_flag |
145 (EPT_MAX_LEVELS - 1) << 3 | MTRR_TYPE_WB));
146 }
147
148 static bool
ept_hw_ad_supported(void)149 ept_hw_ad_supported(void)
150 {
151 uint64_t ept_caps = rdmsr(MSR_IA32_VMX_EPT_VPID_CAP);
152 return ((ept_caps & IA32_VMX_EPT_VPID_HW_AD) != 0);
153 }
154
155 vmm_pte_ops_t ept_pte_ops = {
156 .vpeo_map_table = ept_map_table,
157 .vpeo_map_page = ept_map_page,
158 .vpeo_pte_pfn = ept_pte_pfn,
159 .vpeo_pte_is_present = ept_pte_is_present,
160 .vpeo_pte_prot = ept_pte_prot,
161 .vpeo_reset_dirty = ept_reset_dirty,
162 .vpeo_reset_accessed = ept_reset_accessed,
163 .vpeo_query = ept_query,
164 .vpeo_get_pmtp = ept_get_pmtp,
165 .vpeo_hw_ad_supported = ept_hw_ad_supported,
166 };
167