1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/bus.h> 34 #include <sys/eventhandler.h> 35 #include <sys/sysctl.h> 36 #include <sys/systm.h> 37 38 #include <dev/pci/pcivar.h> 39 #include <dev/pci/pcireg.h> 40 41 #include <machine/cpu.h> 42 #include <machine/md_var.h> 43 44 #include "vmm_util.h" 45 #include "vmm_mem.h" 46 #include "iommu.h" 47 48 SYSCTL_DECL(_hw_vmm); 49 SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 50 "bhyve iommu parameters"); 51 52 static int iommu_avail; 53 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail, 54 0, "bhyve iommu initialized?"); 55 56 static int iommu_enable = 1; 57 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0, 58 "Enable use of I/O MMU (required for PCI passthrough)."); 59 60 static const struct iommu_ops *ops; 61 static void *host_domain; 62 static eventhandler_tag add_tag, delete_tag; 63 64 static __inline int 65 IOMMU_INIT(void) 66 { 67 if (ops != NULL) 68 return ((*ops->init)()); 69 else 70 return (ENXIO); 71 } 72 73 static __inline void 74 IOMMU_CLEANUP(void) 75 { 76 if (ops != NULL && iommu_avail) 77 (*ops->cleanup)(); 78 } 79 80 static __inline void * 81 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr) 82 { 83 84 if (ops != NULL && iommu_avail) 85 return ((*ops->create_domain)(maxaddr)); 86 else 87 return (NULL); 88 } 89 90 static __inline void 91 IOMMU_DESTROY_DOMAIN(void *dom) 92 { 93 94 if (ops != NULL && iommu_avail) 95 (*ops->destroy_domain)(dom); 96 } 97 98 static __inline uint64_t 99 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) 100 { 101 102 if (ops != NULL && iommu_avail) 103 return ((*ops->create_mapping)(domain, gpa, hpa, len)); 104 else 105 return (len); /* XXX */ 106 } 107 108 static __inline uint64_t 109 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len) 110 { 111 112 if (ops != NULL && iommu_avail) 113 return ((*ops->remove_mapping)(domain, gpa, len)); 114 else 115 return (len); /* XXX */ 116 } 117 118 static __inline void 119 IOMMU_ADD_DEVICE(void *domain, uint16_t rid) 120 { 121 122 if (ops != NULL && iommu_avail) 123 (*ops->add_device)(domain, rid); 124 } 125 126 static __inline void 127 IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid) 128 { 129 130 if (ops != NULL && iommu_avail) 131 (*ops->remove_device)(domain, rid); 132 } 133 134 static __inline void 135 IOMMU_INVALIDATE_TLB(void *domain) 136 { 137 138 if (ops != NULL && iommu_avail) 139 (*ops->invalidate_tlb)(domain); 140 } 141 142 static __inline void 143 IOMMU_ENABLE(void) 144 { 145 146 if (ops != NULL && iommu_avail) 147 (*ops->enable)(); 148 } 149 150 static __inline void 151 IOMMU_DISABLE(void) 152 { 153 154 if (ops != NULL && iommu_avail) 155 (*ops->disable)(); 156 } 157 158 static void 159 iommu_pci_add(void *arg, device_t dev) 160 { 161 162 /* Add new devices to the host domain. */ 163 iommu_add_device(host_domain, pci_get_rid(dev)); 164 } 165 166 static void 167 iommu_pci_delete(void *arg, device_t dev) 168 { 169 170 iommu_remove_device(host_domain, pci_get_rid(dev)); 171 } 172 173 static void 174 iommu_init(void) 175 { 176 int error, bus, slot, func; 177 vm_paddr_t maxaddr; 178 devclass_t dc; 179 device_t dev; 180 181 if (!iommu_enable) 182 return; 183 184 if (vmm_is_intel()) 185 ops = &iommu_ops_intel; 186 else if (vmm_is_svm()) 187 ops = &iommu_ops_amd; 188 else 189 ops = NULL; 190 191 error = IOMMU_INIT(); 192 if (error) 193 return; 194 195 iommu_avail = 1; 196 197 /* 198 * Create a domain for the devices owned by the host 199 */ 200 maxaddr = vmm_mem_maxaddr(); 201 host_domain = IOMMU_CREATE_DOMAIN(maxaddr); 202 if (host_domain == NULL) { 203 printf("iommu_init: unable to create a host domain"); 204 IOMMU_CLEANUP(); 205 ops = NULL; 206 iommu_avail = 0; 207 return; 208 } 209 210 /* 211 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to 212 * the host 213 */ 214 iommu_create_mapping(host_domain, 0, 0, maxaddr); 215 216 add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0); 217 delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete, 218 NULL, 0); 219 dc = devclass_find("ppt"); 220 for (bus = 0; bus <= PCI_BUSMAX; bus++) { 221 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 222 for (func = 0; func <= PCI_FUNCMAX; func++) { 223 dev = pci_find_dbsf(0, bus, slot, func); 224 if (dev == NULL) 225 continue; 226 227 /* Skip passthrough devices. */ 228 if (dc != NULL && 229 device_get_devclass(dev) == dc) 230 continue; 231 232 /* 233 * Everything else belongs to the host 234 * domain. 235 */ 236 iommu_add_device(host_domain, 237 pci_get_rid(dev)); 238 } 239 } 240 } 241 IOMMU_ENABLE(); 242 243 } 244 245 void 246 iommu_cleanup(void) 247 { 248 249 if (add_tag != NULL) { 250 EVENTHANDLER_DEREGISTER(pci_add_device, add_tag); 251 add_tag = NULL; 252 } 253 if (delete_tag != NULL) { 254 EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag); 255 delete_tag = NULL; 256 } 257 IOMMU_DISABLE(); 258 IOMMU_DESTROY_DOMAIN(host_domain); 259 host_domain = NULL; 260 IOMMU_CLEANUP(); 261 } 262 263 void * 264 iommu_create_domain(vm_paddr_t maxaddr) 265 { 266 static volatile int iommu_initted; 267 268 if (iommu_initted < 2) { 269 if (atomic_cmpset_int(&iommu_initted, 0, 1)) { 270 iommu_init(); 271 atomic_store_rel_int(&iommu_initted, 2); 272 } else 273 while (iommu_initted == 1) 274 cpu_spinwait(); 275 } 276 return (IOMMU_CREATE_DOMAIN(maxaddr)); 277 } 278 279 void 280 iommu_destroy_domain(void *dom) 281 { 282 283 IOMMU_DESTROY_DOMAIN(dom); 284 } 285 286 void 287 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 288 { 289 uint64_t mapped, remaining; 290 291 remaining = len; 292 293 while (remaining > 0) { 294 mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining); 295 gpa += mapped; 296 hpa += mapped; 297 remaining -= mapped; 298 } 299 } 300 301 void 302 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len) 303 { 304 uint64_t unmapped, remaining; 305 306 remaining = len; 307 308 while (remaining > 0) { 309 unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining); 310 gpa += unmapped; 311 remaining -= unmapped; 312 } 313 } 314 315 void * 316 iommu_host_domain(void) 317 { 318 319 return (host_domain); 320 } 321 322 void 323 iommu_add_device(void *dom, uint16_t rid) 324 { 325 326 IOMMU_ADD_DEVICE(dom, rid); 327 } 328 329 void 330 iommu_remove_device(void *dom, uint16_t rid) 331 { 332 333 IOMMU_REMOVE_DEVICE(dom, rid); 334 } 335 336 void 337 iommu_invalidate_tlb(void *domain) 338 { 339 340 IOMMU_INVALIDATE_TLB(domain); 341 } 342