1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ 12 13 /* 14 * Copyright 2019 Joyent, Inc. 15 * Copyright 2023 Oxide Computer Company 16 */ 17 18 #include <sys/types.h> 19 #include <sys/param.h> 20 #include <sys/atomic.h> 21 #include <sys/kmem.h> 22 #include <sys/machsystm.h> 23 #include <sys/mman.h> 24 #include <sys/x86_archext.h> 25 #include <vm/hat_pte.h> 26 27 #include <sys/vmm_gpt.h> 28 #include <sys/vmm_vm.h> 29 30 #define EPT_R (1 << 0) 31 #define EPT_W (1 << 1) 32 #define EPT_X (1 << 2) 33 #define EPT_RWX (EPT_R | EPT_W | EPT_X) 34 #define EPT_LGPG (1 << 7) 35 #define EPT_ACCESSED (1 << 8) 36 #define EPT_DIRTY (1 << 9) 37 38 #define EPT_PA_MASK (0x000ffffffffff000ull) 39 40 #define EPT_MAX_LEVELS 4 41 CTASSERT(EPT_MAX_LEVELS <= MAX_GPT_LEVEL); 42 43 #define EPTP_FLAG_ACCESSED_DIRTY (1 << 6) 44 45 CTASSERT(EPT_R == PROT_READ); 46 CTASSERT(EPT_W == PROT_WRITE); 47 CTASSERT(EPT_X == PROT_EXEC); 48 49 static uint_t 50 ept_pte_prot(uint64_t pte) 51 { 52 return (pte & EPT_RWX); 53 } 54 55 static inline uint64_t 56 ept_attr_to_pat(uint8_t attr) 57 { 58 uint64_t bits = attr & 0x7; 59 return (bits << 3); 60 } 61 62 static uint64_t 63 ept_map_table(uint64_t pfn) 64 { 65 const uint64_t paddr = pfn_to_pa(pfn) & EPT_PA_MASK; 66 return (paddr | EPT_RWX); 67 } 68 69 static uint64_t 70 ept_map_page(uint64_t pfn, uint_t prot, uint8_t attr) 71 { 72 const uint64_t paddr = pfn_to_pa(pfn) & EPT_PA_MASK; 73 const uint64_t pat = ept_attr_to_pat(attr); 74 const uint64_t rprot = prot & EPT_RWX; 75 return (paddr | pat | rprot); 76 } 77 78 static uint64_t 79 ept_pte_pfn(uint64_t pte) 80 { 81 return (mmu_btop(pte & PT_PADDR)); 82 } 83 84 static bool 85 ept_pte_is_present(uint64_t pte) 86 { 87 return ((pte & EPT_RWX) != 0); 88 } 89 90 static uint_t 91 ept_reset_bits(volatile uint64_t *entry, uint64_t mask, uint64_t bits) 92 { 93 uint64_t pte, newpte, oldpte = 0; 94 95 /* 96 * We use volatile and atomic ops here because we may be 97 * racing against hardware modifying these bits. 98 */ 99 VERIFY3P(entry, !=, NULL); 100 oldpte = *entry; 101 do { 102 pte = oldpte; 103 newpte = (pte & ~mask) | bits; 104 oldpte = atomic_cas_64(entry, pte, newpte); 105 } while (oldpte != pte); 106 107 return (oldpte & mask); 108 } 109 110 static uint_t 111 ept_reset_dirty(uint64_t *entry, bool on) 112 { 113 return (ept_reset_bits(entry, EPT_DIRTY, 114 on ? (EPT_DIRTY | EPT_ACCESSED) : 0)); 115 } 116 117 static uint_t 118 ept_reset_accessed(uint64_t *entry, bool on) 119 { 120 return (ept_reset_bits(entry, EPT_DIRTY | EPT_ACCESSED, 121 on ? EPT_ACCESSED : 0)); 122 } 123 124 static bool 125 ept_query(uint64_t *entry, vmm_gpt_query_t query) 126 { 127 ASSERT(entry != NULL); 128 129 const uint64_t pte = *entry; 130 switch (query) { 131 case VGQ_ACCESSED: 132 return ((pte & EPT_ACCESSED) != 0); 133 case VGQ_DIRTY: 134 return ((pte & EPT_DIRTY) != 0); 135 default: 136 panic("unrecognized query: %d", query); 137 } 138 } 139 140 static uint64_t 141 ept_get_pmtp(pfn_t root_pfn, bool track_dirty) 142 { 143 const uint64_t ad_flag = track_dirty ? EPTP_FLAG_ACCESSED_DIRTY : 0; 144 return ((root_pfn << PAGESHIFT | ad_flag | 145 (EPT_MAX_LEVELS - 1) << 3 | MTRR_TYPE_WB)); 146 } 147 148 static bool 149 ept_hw_ad_supported(void) 150 { 151 uint64_t ept_caps = rdmsr(MSR_IA32_VMX_EPT_VPID_CAP); 152 return ((ept_caps & IA32_VMX_EPT_VPID_HW_AD) != 0); 153 } 154 155 vmm_pte_ops_t ept_pte_ops = { 156 .vpeo_map_table = ept_map_table, 157 .vpeo_map_page = ept_map_page, 158 .vpeo_pte_pfn = ept_pte_pfn, 159 .vpeo_pte_is_present = ept_pte_is_present, 160 .vpeo_pte_prot = ept_pte_prot, 161 .vpeo_reset_dirty = ept_reset_dirty, 162 .vpeo_reset_accessed = ept_reset_accessed, 163 .vpeo_query = ept_query, 164 .vpeo_get_pmtp = ept_get_pmtp, 165 .vpeo_hw_ad_supported = ept_hw_ad_supported, 166 }; 167