1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/bus.h> 31 #include <sys/eventhandler.h> 32 #include <sys/sysctl.h> 33 #include <sys/systm.h> 34 35 #include <dev/pci/pcivar.h> 36 #include <dev/pci/pcireg.h> 37 38 #include <machine/cpu.h> 39 #include <machine/md_var.h> 40 41 #include "vmm_util.h" 42 #include "vmm_mem.h" 43 #include "iommu.h" 44 45 SYSCTL_DECL(_hw_vmm); 46 SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 47 "bhyve iommu parameters"); 48 49 static int iommu_avail; 50 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail, 51 0, "bhyve iommu initialized?"); 52 53 static int iommu_enable = 1; 54 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0, 55 "Enable use of I/O MMU (required for PCI passthrough)."); 56 57 static const struct iommu_ops *ops; 58 static void *host_domain; 59 static eventhandler_tag add_tag, delete_tag; 60 61 static __inline int 62 IOMMU_INIT(void) 63 { 64 if (ops != NULL) 65 return ((*ops->init)()); 66 else 67 return (ENXIO); 68 } 69 70 static __inline void 71 IOMMU_CLEANUP(void) 72 { 73 if (ops != NULL && iommu_avail) 74 (*ops->cleanup)(); 75 } 76 77 static __inline void * 78 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr) 79 { 80 81 if (ops != NULL && iommu_avail) 82 return ((*ops->create_domain)(maxaddr)); 83 else 84 return (NULL); 85 } 86 87 static __inline void 88 IOMMU_DESTROY_DOMAIN(void *dom) 89 { 90 91 if (ops != NULL && iommu_avail) 92 (*ops->destroy_domain)(dom); 93 } 94 95 static __inline uint64_t 96 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) 97 { 98 99 if (ops != NULL && iommu_avail) 100 return ((*ops->create_mapping)(domain, gpa, hpa, len)); 101 else 102 return (len); /* XXX */ 103 } 104 105 static __inline uint64_t 106 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len) 107 { 108 109 if (ops != NULL && iommu_avail) 110 return ((*ops->remove_mapping)(domain, gpa, len)); 111 else 112 return (len); /* XXX */ 113 } 114 115 static __inline void 116 IOMMU_ADD_DEVICE(void *domain, uint16_t rid) 117 { 118 119 if (ops != NULL && iommu_avail) 120 (*ops->add_device)(domain, rid); 121 } 122 123 static __inline void 124 IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid) 125 { 126 127 if (ops != NULL && iommu_avail) 128 (*ops->remove_device)(domain, rid); 129 } 130 131 static __inline void 132 IOMMU_INVALIDATE_TLB(void *domain) 133 { 134 135 if (ops != NULL && iommu_avail) 136 (*ops->invalidate_tlb)(domain); 137 } 138 139 static __inline void 140 IOMMU_ENABLE(void) 141 { 142 143 if (ops != NULL && iommu_avail) 144 (*ops->enable)(); 145 } 146 147 static __inline void 148 IOMMU_DISABLE(void) 149 { 150 151 if (ops != NULL && iommu_avail) 152 (*ops->disable)(); 153 } 154 155 static void 156 iommu_pci_add(void *arg, device_t dev) 157 { 158 159 /* Add new devices to the host domain. */ 160 iommu_add_device(host_domain, pci_get_rid(dev)); 161 } 162 163 static void 164 iommu_pci_delete(void *arg, device_t dev) 165 { 166 167 iommu_remove_device(host_domain, pci_get_rid(dev)); 168 } 169 170 static void 171 iommu_init(void) 172 { 173 int error, bus, slot, func; 174 vm_paddr_t maxaddr; 175 devclass_t dc; 176 device_t dev; 177 178 if (!iommu_enable) 179 return; 180 181 if (vmm_is_intel()) 182 ops = &iommu_ops_intel; 183 else if (vmm_is_svm()) 184 ops = &iommu_ops_amd; 185 else 186 ops = NULL; 187 188 error = IOMMU_INIT(); 189 if (error) 190 return; 191 192 iommu_avail = 1; 193 194 /* 195 * Create a domain for the devices owned by the host 196 */ 197 maxaddr = vmm_mem_maxaddr(); 198 host_domain = IOMMU_CREATE_DOMAIN(maxaddr); 199 if (host_domain == NULL) { 200 printf("iommu_init: unable to create a host domain"); 201 IOMMU_CLEANUP(); 202 ops = NULL; 203 iommu_avail = 0; 204 return; 205 } 206 207 /* 208 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to 209 * the host 210 */ 211 iommu_create_mapping(host_domain, 0, 0, maxaddr); 212 213 add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0); 214 delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete, 215 NULL, 0); 216 dc = devclass_find("ppt"); 217 for (bus = 0; bus <= PCI_BUSMAX; bus++) { 218 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 219 for (func = 0; func <= PCI_FUNCMAX; func++) { 220 dev = pci_find_dbsf(0, bus, slot, func); 221 if (dev == NULL) 222 continue; 223 224 /* Skip passthrough devices. */ 225 if (dc != NULL && 226 device_get_devclass(dev) == dc) 227 continue; 228 229 /* 230 * Everything else belongs to the host 231 * domain. 232 */ 233 iommu_add_device(host_domain, 234 pci_get_rid(dev)); 235 } 236 } 237 } 238 IOMMU_ENABLE(); 239 240 } 241 242 void 243 iommu_cleanup(void) 244 { 245 246 if (add_tag != NULL) { 247 EVENTHANDLER_DEREGISTER(pci_add_device, add_tag); 248 add_tag = NULL; 249 } 250 if (delete_tag != NULL) { 251 EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag); 252 delete_tag = NULL; 253 } 254 IOMMU_DISABLE(); 255 IOMMU_DESTROY_DOMAIN(host_domain); 256 host_domain = NULL; 257 IOMMU_CLEANUP(); 258 } 259 260 void * 261 iommu_create_domain(vm_paddr_t maxaddr) 262 { 263 static volatile int iommu_initted; 264 265 if (iommu_initted < 2) { 266 if (atomic_cmpset_int(&iommu_initted, 0, 1)) { 267 iommu_init(); 268 atomic_store_rel_int(&iommu_initted, 2); 269 } else 270 while (iommu_initted == 1) 271 cpu_spinwait(); 272 } 273 return (IOMMU_CREATE_DOMAIN(maxaddr)); 274 } 275 276 void 277 iommu_destroy_domain(void *dom) 278 { 279 280 IOMMU_DESTROY_DOMAIN(dom); 281 } 282 283 void 284 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 285 { 286 uint64_t mapped, remaining; 287 288 remaining = len; 289 290 while (remaining > 0) { 291 mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining); 292 gpa += mapped; 293 hpa += mapped; 294 remaining -= mapped; 295 } 296 } 297 298 void 299 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len) 300 { 301 uint64_t unmapped, remaining; 302 303 remaining = len; 304 305 while (remaining > 0) { 306 unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining); 307 gpa += unmapped; 308 remaining -= unmapped; 309 } 310 } 311 312 void * 313 iommu_host_domain(void) 314 { 315 316 return (host_domain); 317 } 318 319 void 320 iommu_add_device(void *dom, uint16_t rid) 321 { 322 323 IOMMU_ADD_DEVICE(dom, rid); 324 } 325 326 void 327 iommu_remove_device(void *dom, uint16_t rid) 328 { 329 330 IOMMU_REMOVE_DEVICE(dom, rid); 331 } 332 333 void 334 iommu_invalidate_tlb(void *domain) 335 { 336 337 IOMMU_INVALIDATE_TLB(domain); 338 } 339