1 /*- 2 * Copyright (c) 2011 NetApp, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/types.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/sysctl.h> 37 38 #include <dev/pci/pcivar.h> 39 #include <dev/pci/pcireg.h> 40 41 #include <machine/cpu.h> 42 #include <machine/md_var.h> 43 44 #include "vmm_util.h" 45 #include "vmm_mem.h" 46 #include "iommu.h" 47 48 SYSCTL_DECL(_hw_vmm); 49 SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW, 0, "bhyve iommu parameters"); 50 51 static int iommu_avail; 52 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail, 53 0, "bhyve iommu initialized?"); 54 55 static int iommu_enable = 1; 56 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0, 57 "Enable use of I/O MMU (required for PCI passthrough)."); 58 59 static struct iommu_ops *ops; 60 static void *host_domain; 61 static eventhandler_tag add_tag, delete_tag; 62 63 static __inline int 64 IOMMU_INIT(void) 65 { 66 if (ops != NULL) 67 return ((*ops->init)()); 68 else 69 return (ENXIO); 70 } 71 72 static __inline void 73 IOMMU_CLEANUP(void) 74 { 75 if (ops != NULL && iommu_avail) 76 (*ops->cleanup)(); 77 } 78 79 static __inline void * 80 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr) 81 { 82 83 if (ops != NULL && iommu_avail) 84 return ((*ops->create_domain)(maxaddr)); 85 else 86 return (NULL); 87 } 88 89 static __inline void 90 IOMMU_DESTROY_DOMAIN(void *dom) 91 { 92 93 if (ops != NULL && iommu_avail) 94 (*ops->destroy_domain)(dom); 95 } 96 97 static __inline uint64_t 98 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) 99 { 100 101 if (ops != NULL && iommu_avail) 102 return ((*ops->create_mapping)(domain, gpa, hpa, len)); 103 else 104 return (len); /* XXX */ 105 } 106 107 static __inline uint64_t 108 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len) 109 { 110 111 if (ops != NULL && iommu_avail) 112 return ((*ops->remove_mapping)(domain, gpa, len)); 113 else 114 return (len); /* XXX */ 115 } 116 117 static __inline void 118 IOMMU_ADD_DEVICE(void *domain, uint16_t rid) 119 { 120 121 if (ops != NULL && iommu_avail) 122 (*ops->add_device)(domain, rid); 123 } 124 125 static __inline void 126 IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid) 127 { 128 129 if (ops != NULL && iommu_avail) 130 (*ops->remove_device)(domain, rid); 131 } 132 133 static __inline void 134 IOMMU_INVALIDATE_TLB(void *domain) 135 { 136 137 if (ops != NULL && iommu_avail) 138 (*ops->invalidate_tlb)(domain); 139 } 140 141 static __inline void 142 IOMMU_ENABLE(void) 143 { 144 145 if (ops != NULL && iommu_avail) 146 (*ops->enable)(); 147 } 148 149 static __inline void 150 IOMMU_DISABLE(void) 151 { 152 153 if (ops != NULL && iommu_avail) 154 (*ops->disable)(); 155 } 156 157 static void 158 iommu_pci_add(void *arg, device_t dev) 159 { 160 161 /* Add new devices to the host domain. */ 162 iommu_add_device(host_domain, pci_get_rid(dev)); 163 } 164 165 static void 166 iommu_pci_delete(void *arg, device_t dev) 167 { 168 169 iommu_remove_device(host_domain, pci_get_rid(dev)); 170 } 171 172 static void 173 iommu_init(void) 174 { 175 int error, bus, slot, func; 176 vm_paddr_t maxaddr; 177 devclass_t dc; 178 device_t dev; 179 180 if (!iommu_enable) 181 return; 182 183 if (vmm_is_intel()) 184 ops = &iommu_ops_intel; 185 else if (vmm_is_amd()) 186 ops = &iommu_ops_amd; 187 else 188 ops = NULL; 189 190 error = IOMMU_INIT(); 191 if (error) 192 return; 193 194 iommu_avail = 1; 195 196 /* 197 * Create a domain for the devices owned by the host 198 */ 199 maxaddr = vmm_mem_maxaddr(); 200 host_domain = IOMMU_CREATE_DOMAIN(maxaddr); 201 if (host_domain == NULL) { 202 printf("iommu_init: unable to create a host domain"); 203 IOMMU_CLEANUP(); 204 ops = NULL; 205 iommu_avail = 0; 206 return; 207 } 208 209 /* 210 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to 211 * the host 212 */ 213 iommu_create_mapping(host_domain, 0, 0, maxaddr); 214 215 add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0); 216 delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete, 217 NULL, 0); 218 dc = devclass_find("ppt"); 219 for (bus = 0; bus <= PCI_BUSMAX; bus++) { 220 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 221 for (func = 0; func <= PCI_FUNCMAX; func++) { 222 dev = pci_find_dbsf(0, bus, slot, func); 223 if (dev == NULL) 224 continue; 225 226 /* Skip passthrough devices. */ 227 if (dc != NULL && 228 device_get_devclass(dev) == dc) 229 continue; 230 231 /* 232 * Everything else belongs to the host 233 * domain. 234 */ 235 iommu_add_device(host_domain, 236 pci_get_rid(dev)); 237 } 238 } 239 } 240 IOMMU_ENABLE(); 241 242 } 243 244 void 245 iommu_cleanup(void) 246 { 247 248 if (add_tag != NULL) { 249 EVENTHANDLER_DEREGISTER(pci_add_device, add_tag); 250 add_tag = NULL; 251 } 252 if (delete_tag != NULL) { 253 EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag); 254 delete_tag = NULL; 255 } 256 IOMMU_DISABLE(); 257 IOMMU_DESTROY_DOMAIN(host_domain); 258 IOMMU_CLEANUP(); 259 } 260 261 void * 262 iommu_create_domain(vm_paddr_t maxaddr) 263 { 264 static volatile int iommu_initted; 265 266 if (iommu_initted < 2) { 267 if (atomic_cmpset_int(&iommu_initted, 0, 1)) { 268 iommu_init(); 269 atomic_store_rel_int(&iommu_initted, 2); 270 } else 271 while (iommu_initted == 1) 272 cpu_spinwait(); 273 } 274 return (IOMMU_CREATE_DOMAIN(maxaddr)); 275 } 276 277 void 278 iommu_destroy_domain(void *dom) 279 { 280 281 IOMMU_DESTROY_DOMAIN(dom); 282 } 283 284 void 285 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 286 { 287 uint64_t mapped, remaining; 288 289 remaining = len; 290 291 while (remaining > 0) { 292 mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining); 293 gpa += mapped; 294 hpa += mapped; 295 remaining -= mapped; 296 } 297 } 298 299 void 300 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len) 301 { 302 uint64_t unmapped, remaining; 303 304 remaining = len; 305 306 while (remaining > 0) { 307 unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining); 308 gpa += unmapped; 309 remaining -= unmapped; 310 } 311 } 312 313 void * 314 iommu_host_domain(void) 315 { 316 317 return (host_domain); 318 } 319 320 void 321 iommu_add_device(void *dom, uint16_t rid) 322 { 323 324 IOMMU_ADD_DEVICE(dom, rid); 325 } 326 327 void 328 iommu_remove_device(void *dom, uint16_t rid) 329 { 330 331 IOMMU_REMOVE_DEVICE(dom, rid); 332 } 333 334 void 335 iommu_invalidate_tlb(void *domain) 336 { 337 338 IOMMU_INVALIDATE_TLB(domain); 339 } 340