1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/kernel.h> 36 #include <sys/types.h> 37 #include <sys/systm.h> 38 #include <sys/smp.h> 39 #include <sys/sysctl.h> 40 41 #include <vm/vm.h> 42 #include <vm/pmap.h> 43 #include <vm/vm_extern.h> 44 45 #include <machine/vmm.h> 46 47 #include "vmx_cpufunc.h" 48 #include "ept.h" 49 50 #define EPT_SUPPORTS_EXEC_ONLY(cap) ((cap) & (1UL << 0)) 51 #define EPT_PWL4(cap) ((cap) & (1UL << 6)) 52 #define EPT_MEMORY_TYPE_WB(cap) ((cap) & (1UL << 14)) 53 #define EPT_PDE_SUPERPAGE(cap) ((cap) & (1UL << 16)) /* 2MB pages */ 54 #define EPT_PDPTE_SUPERPAGE(cap) ((cap) & (1UL << 17)) /* 1GB pages */ 55 #define INVEPT_SUPPORTED(cap) ((cap) & (1UL << 20)) 56 #define AD_BITS_SUPPORTED(cap) ((cap) & (1UL << 21)) 57 #define INVVPID_SUPPORTED(cap) ((cap) & (1UL << 32)) 58 59 #define INVVPID_ALL_TYPES_MASK 0xF0000000000UL 60 #define INVVPID_ALL_TYPES_SUPPORTED(cap) \ 61 (((cap) & INVVPID_ALL_TYPES_MASK) == INVVPID_ALL_TYPES_MASK) 62 63 #define INVEPT_ALL_TYPES_MASK 0x6000000UL 64 #define INVEPT_ALL_TYPES_SUPPORTED(cap) \ 65 (((cap) & INVEPT_ALL_TYPES_MASK) == INVEPT_ALL_TYPES_MASK) 66 67 #define EPT_PWLEVELS 4 /* page walk levels */ 68 #define EPT_ENABLE_AD_BITS (1 << 6) 69 70 SYSCTL_DECL(_hw_vmm); 71 SYSCTL_NODE(_hw_vmm, OID_AUTO, ept, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 72 NULL); 73 74 static int ept_enable_ad_bits; 75 76 static int ept_pmap_flags; 77 SYSCTL_INT(_hw_vmm_ept, OID_AUTO, pmap_flags, CTLFLAG_RD, 78 &ept_pmap_flags, 0, NULL); 79 80 int 81 ept_init(int ipinum) 82 { 83 int use_hw_ad_bits, use_superpages, use_exec_only; 84 uint64_t cap; 85 86 cap = rdmsr(MSR_VMX_EPT_VPID_CAP); 87 88 /* 89 * Verify that: 90 * - page walk length is 4 steps 91 * - extended page tables can be laid out in write-back memory 92 * - invvpid instruction with all possible types is supported 93 * - invept instruction with all possible types is supported 94 */ 95 if (!EPT_PWL4(cap) || 96 !EPT_MEMORY_TYPE_WB(cap) || 97 !INVVPID_SUPPORTED(cap) || 98 !INVVPID_ALL_TYPES_SUPPORTED(cap) || 99 !INVEPT_SUPPORTED(cap) || 100 !INVEPT_ALL_TYPES_SUPPORTED(cap)) 101 return (EINVAL); 102 103 ept_pmap_flags = ipinum & PMAP_NESTED_IPIMASK; 104 105 use_superpages = 1; 106 TUNABLE_INT_FETCH("hw.vmm.ept.use_superpages", &use_superpages); 107 if (use_superpages && EPT_PDE_SUPERPAGE(cap)) 108 ept_pmap_flags |= PMAP_PDE_SUPERPAGE; /* 2MB superpage */ 109 110 use_hw_ad_bits = 1; 111 TUNABLE_INT_FETCH("hw.vmm.ept.use_hw_ad_bits", &use_hw_ad_bits); 112 if (use_hw_ad_bits && AD_BITS_SUPPORTED(cap)) 113 ept_enable_ad_bits = 1; 114 else 115 ept_pmap_flags |= PMAP_EMULATE_AD_BITS; 116 117 use_exec_only = 1; 118 TUNABLE_INT_FETCH("hw.vmm.ept.use_exec_only", &use_exec_only); 119 if (use_exec_only && EPT_SUPPORTS_EXEC_ONLY(cap)) 120 ept_pmap_flags |= PMAP_SUPPORTS_EXEC_ONLY; 121 122 return (0); 123 } 124 125 #if 0 126 static void 127 ept_dump(uint64_t *ptp, int nlevels) 128 { 129 int i, t, tabs; 130 uint64_t *ptpnext, ptpval; 131 132 if (--nlevels < 0) 133 return; 134 135 tabs = 3 - nlevels; 136 for (t = 0; t < tabs; t++) 137 printf("\t"); 138 printf("PTP = %p\n", ptp); 139 140 for (i = 0; i < 512; i++) { 141 ptpval = ptp[i]; 142 143 if (ptpval == 0) 144 continue; 145 146 for (t = 0; t < tabs; t++) 147 printf("\t"); 148 printf("%3d 0x%016lx\n", i, ptpval); 149 150 if (nlevels != 0 && (ptpval & EPT_PG_SUPERPAGE) == 0) { 151 ptpnext = (uint64_t *) 152 PHYS_TO_DMAP(ptpval & EPT_ADDR_MASK); 153 ept_dump(ptpnext, nlevels); 154 } 155 } 156 } 157 #endif 158 159 static void 160 invept_single_context(void *arg) 161 { 162 struct invept_desc desc = *(struct invept_desc *)arg; 163 164 invept(INVEPT_TYPE_SINGLE_CONTEXT, desc); 165 } 166 167 void 168 ept_invalidate_mappings(u_long eptp) 169 { 170 struct invept_desc invept_desc = { 0 }; 171 172 invept_desc.eptp = eptp; 173 174 smp_rendezvous(NULL, invept_single_context, NULL, &invept_desc); 175 } 176 177 static int 178 ept_pinit(pmap_t pmap) 179 { 180 181 return (pmap_pinit_type(pmap, PT_EPT, ept_pmap_flags)); 182 } 183 184 struct vmspace * 185 ept_vmspace_alloc(vm_offset_t min, vm_offset_t max) 186 { 187 188 return (vmspace_alloc(min, max, ept_pinit)); 189 } 190 191 void 192 ept_vmspace_free(struct vmspace *vmspace) 193 { 194 195 vmspace_free(vmspace); 196 } 197 198 uint64_t 199 eptp(uint64_t pml4) 200 { 201 uint64_t eptp_val; 202 203 eptp_val = pml4 | (EPT_PWLEVELS - 1) << 3 | PAT_WRITE_BACK; 204 if (ept_enable_ad_bits) 205 eptp_val |= EPT_ENABLE_AD_BITS; 206 207 return (eptp_val); 208 } 209