1 /*- 2 * Copyright (c) 2011 NetApp, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/types.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/sysctl.h> 37 38 #include <dev/pci/pcivar.h> 39 #include <dev/pci/pcireg.h> 40 41 #include <machine/cpu.h> 42 #include <machine/md_var.h> 43 44 #include "vmm_util.h" 45 #include "vmm_mem.h" 46 #include "iommu.h" 47 48 SYSCTL_DECL(_hw_vmm); 49 SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW, 0, "bhyve iommu parameters"); 50 51 static int iommu_avail; 52 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail, 53 0, "bhyve iommu initialized?"); 54 55 static int iommu_enable = 1; 56 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0, 57 "Enable use of I/O MMU (required for PCI passthrough)."); 58 59 static struct iommu_ops *ops; 60 static void *host_domain; 61 62 static __inline int 63 IOMMU_INIT(void) 64 { 65 if (ops != NULL) 66 return ((*ops->init)()); 67 else 68 return (ENXIO); 69 } 70 71 static __inline void 72 IOMMU_CLEANUP(void) 73 { 74 if (ops != NULL && iommu_avail) 75 (*ops->cleanup)(); 76 } 77 78 static __inline void * 79 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr) 80 { 81 82 if (ops != NULL && iommu_avail) 83 return ((*ops->create_domain)(maxaddr)); 84 else 85 return (NULL); 86 } 87 88 static __inline void 89 IOMMU_DESTROY_DOMAIN(void *dom) 90 { 91 92 if (ops != NULL && iommu_avail) 93 (*ops->destroy_domain)(dom); 94 } 95 96 static __inline uint64_t 97 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) 98 { 99 100 if (ops != NULL && iommu_avail) 101 return ((*ops->create_mapping)(domain, gpa, hpa, len)); 102 else 103 return (len); /* XXX */ 104 } 105 106 static __inline uint64_t 107 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len) 108 { 109 110 if (ops != NULL && iommu_avail) 111 return ((*ops->remove_mapping)(domain, gpa, len)); 112 else 113 return (len); /* XXX */ 114 } 115 116 static __inline void 117 IOMMU_ADD_DEVICE(void *domain, uint16_t rid) 118 { 119 120 if (ops != NULL && iommu_avail) 121 (*ops->add_device)(domain, rid); 122 } 123 124 static __inline void 125 IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid) 126 { 127 128 if (ops != NULL && iommu_avail) 129 (*ops->remove_device)(domain, rid); 130 } 131 132 static __inline void 133 IOMMU_INVALIDATE_TLB(void *domain) 134 { 135 136 if (ops != NULL && iommu_avail) 137 (*ops->invalidate_tlb)(domain); 138 } 139 140 static __inline void 141 IOMMU_ENABLE(void) 142 { 143 144 if (ops != NULL && iommu_avail) 145 (*ops->enable)(); 146 } 147 148 static __inline void 149 IOMMU_DISABLE(void) 150 { 151 152 if (ops != NULL && iommu_avail) 153 (*ops->disable)(); 154 } 155 156 static void 157 iommu_init(void) 158 { 159 int error, bus, slot, func; 160 vm_paddr_t maxaddr; 161 device_t dev; 162 163 if (!iommu_enable) 164 return; 165 166 if (vmm_is_intel()) 167 ops = &iommu_ops_intel; 168 else if (vmm_is_amd()) 169 ops = &iommu_ops_amd; 170 else 171 ops = NULL; 172 173 error = IOMMU_INIT(); 174 if (error) 175 return; 176 177 iommu_avail = 1; 178 179 /* 180 * Create a domain for the devices owned by the host 181 */ 182 maxaddr = vmm_mem_maxaddr(); 183 host_domain = IOMMU_CREATE_DOMAIN(maxaddr); 184 if (host_domain == NULL) { 185 printf("iommu_init: unable to create a host domain"); 186 IOMMU_CLEANUP(); 187 ops = NULL; 188 iommu_avail = 0; 189 return; 190 } 191 192 /* 193 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to 194 * the host 195 */ 196 iommu_create_mapping(host_domain, 0, 0, maxaddr); 197 198 for (bus = 0; bus <= PCI_BUSMAX; bus++) { 199 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 200 for (func = 0; func <= PCI_FUNCMAX; func++) { 201 dev = pci_find_dbsf(0, bus, slot, func); 202 if (dev == NULL) 203 continue; 204 205 /* Everything belongs to the host domain. */ 206 iommu_add_device(host_domain, 207 pci_get_rid(dev)); 208 } 209 } 210 } 211 IOMMU_ENABLE(); 212 213 } 214 215 void 216 iommu_cleanup(void) 217 { 218 IOMMU_DISABLE(); 219 IOMMU_DESTROY_DOMAIN(host_domain); 220 IOMMU_CLEANUP(); 221 } 222 223 void * 224 iommu_create_domain(vm_paddr_t maxaddr) 225 { 226 static volatile int iommu_initted; 227 228 if (iommu_initted < 2) { 229 if (atomic_cmpset_int(&iommu_initted, 0, 1)) { 230 iommu_init(); 231 atomic_store_rel_int(&iommu_initted, 2); 232 } else 233 while (iommu_initted == 1) 234 cpu_spinwait(); 235 } 236 return (IOMMU_CREATE_DOMAIN(maxaddr)); 237 } 238 239 void 240 iommu_destroy_domain(void *dom) 241 { 242 243 IOMMU_DESTROY_DOMAIN(dom); 244 } 245 246 void 247 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 248 { 249 uint64_t mapped, remaining; 250 251 remaining = len; 252 253 while (remaining > 0) { 254 mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining); 255 gpa += mapped; 256 hpa += mapped; 257 remaining -= mapped; 258 } 259 } 260 261 void 262 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len) 263 { 264 uint64_t unmapped, remaining; 265 266 remaining = len; 267 268 while (remaining > 0) { 269 unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining); 270 gpa += unmapped; 271 remaining -= unmapped; 272 } 273 } 274 275 void * 276 iommu_host_domain(void) 277 { 278 279 return (host_domain); 280 } 281 282 void 283 iommu_add_device(void *dom, uint16_t rid) 284 { 285 286 IOMMU_ADD_DEVICE(dom, rid); 287 } 288 289 void 290 iommu_remove_device(void *dom, uint16_t rid) 291 { 292 293 IOMMU_REMOVE_DEVICE(dom, rid); 294 } 295 296 void 297 iommu_invalidate_tlb(void *domain) 298 { 299 300 IOMMU_INVALIDATE_TLB(domain); 301 } 302