1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/bus.h> 31 #include <sys/eventhandler.h> 32 #include <sys/sysctl.h> 33 #include <sys/systm.h> 34 35 #include <dev/pci/pcivar.h> 36 #include <dev/pci/pcireg.h> 37 38 #include <machine/cpu.h> 39 #include <machine/md_var.h> 40 41 #include "vmm_util.h" 42 #include "vmm_mem.h" 43 #include "iommu.h" 44 45 SYSCTL_DECL(_hw_vmm); 46 SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 47 "bhyve iommu parameters"); 48 49 static int iommu_avail; 50 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail, 51 0, "bhyve iommu initialized?"); 52 53 static int iommu_enable = 1; 54 SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0, 55 "Enable use of I/O MMU (required for PCI passthrough)."); 56 57 static const struct iommu_ops *ops; 58 static void *host_domain; 59 static eventhandler_tag add_tag, delete_tag; 60 61 static void iommu_cleanup_int(bool iommu_disable); 62 63 static __inline int 64 IOMMU_INIT(void) 65 { 66 if (ops != NULL) 67 return ((*ops->init)()); 68 else 69 return (ENXIO); 70 } 71 72 static __inline void 73 IOMMU_CLEANUP(void) 74 { 75 if (ops != NULL && iommu_avail) 76 (*ops->cleanup)(); 77 } 78 79 static __inline void * 80 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr) 81 { 82 83 if (ops != NULL && iommu_avail) 84 return ((*ops->create_domain)(maxaddr)); 85 else 86 return (NULL); 87 } 88 89 static __inline void 90 IOMMU_DESTROY_DOMAIN(void *dom) 91 { 92 93 if (ops != NULL && iommu_avail) 94 (*ops->destroy_domain)(dom); 95 } 96 97 static __inline int 98 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, 99 uint64_t len, uint64_t *res_len) 100 { 101 102 if (ops != NULL && iommu_avail) 103 return ((*ops->create_mapping)(domain, gpa, hpa, len, res_len)); 104 return (EOPNOTSUPP); 105 } 106 107 static __inline uint64_t 108 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len, 109 uint64_t *res_len) 110 { 111 112 if (ops != NULL && iommu_avail) 113 return ((*ops->remove_mapping)(domain, gpa, len, res_len)); 114 return (EOPNOTSUPP); 115 } 116 117 static __inline int 118 IOMMU_ADD_DEVICE(void *domain, device_t dev, uint16_t rid) 119 { 120 121 if (ops != NULL && iommu_avail) 122 return ((*ops->add_device)(domain, dev, rid)); 123 return (EOPNOTSUPP); 124 } 125 126 static __inline int 127 IOMMU_REMOVE_DEVICE(void *domain, device_t dev, uint16_t rid) 128 { 129 130 if (ops != NULL && iommu_avail) 131 return ((*ops->remove_device)(domain, dev, rid)); 132 return (0); /* To allow ppt_attach() to succeed. */ 133 } 134 135 static __inline int 136 IOMMU_INVALIDATE_TLB(void *domain) 137 { 138 139 if (ops != NULL && iommu_avail) 140 return ((*ops->invalidate_tlb)(domain)); 141 return (0); 142 } 143 144 static __inline void 145 IOMMU_ENABLE(void) 146 { 147 148 if (ops != NULL && iommu_avail) 149 (*ops->enable)(); 150 } 151 152 static __inline void 153 IOMMU_DISABLE(void) 154 { 155 156 if (ops != NULL && iommu_avail) 157 (*ops->disable)(); 158 } 159 160 static void 161 iommu_pci_add(void *arg, device_t dev) 162 { 163 164 /* Add new devices to the host domain. */ 165 iommu_add_device(host_domain, dev, pci_get_rid(dev)); 166 } 167 168 static void 169 iommu_pci_delete(void *arg, device_t dev) 170 { 171 172 iommu_remove_device(host_domain, dev, pci_get_rid(dev)); 173 } 174 175 static void 176 iommu_init(void) 177 { 178 int error, bus, slot, func; 179 vm_paddr_t maxaddr; 180 devclass_t dc; 181 device_t dev; 182 183 if (!iommu_enable) 184 return; 185 186 if (vmm_is_intel()) 187 ops = &iommu_ops_intel; 188 else if (vmm_is_svm()) 189 ops = &iommu_ops_amd; 190 else 191 ops = NULL; 192 193 error = IOMMU_INIT(); 194 if (error) 195 return; 196 197 iommu_avail = 1; 198 199 /* 200 * Create a domain for the devices owned by the host 201 */ 202 maxaddr = vmm_mem_maxaddr(); 203 host_domain = IOMMU_CREATE_DOMAIN(maxaddr); 204 if (host_domain == NULL) { 205 printf("iommu_init: unable to create a host domain"); 206 IOMMU_CLEANUP(); 207 ops = NULL; 208 iommu_avail = 0; 209 return; 210 } 211 212 /* 213 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to 214 * the host 215 */ 216 iommu_create_mapping(host_domain, 0, 0, maxaddr); 217 218 add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0); 219 delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete, 220 NULL, 0); 221 dc = devclass_find("ppt"); 222 for (bus = 0; bus <= PCI_BUSMAX; bus++) { 223 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 224 for (func = 0; func <= PCI_FUNCMAX; func++) { 225 dev = pci_find_dbsf(0, bus, slot, func); 226 if (dev == NULL) 227 continue; 228 229 /* Skip passthrough devices. */ 230 if (dc != NULL && 231 device_get_devclass(dev) == dc) 232 continue; 233 234 /* 235 * Everything else belongs to the host 236 * domain. 237 */ 238 error = iommu_add_device(host_domain, dev, 239 pci_get_rid(dev)); 240 if (error != 0 && error != ENXIO) { 241 printf( 242 "iommu_add_device(%s rid %#x) failed, error %d\n", 243 device_get_name(dev), 244 pci_get_rid(dev), error); 245 iommu_cleanup_int(false); 246 return; 247 } 248 } 249 } 250 } 251 IOMMU_ENABLE(); 252 } 253 254 static void 255 iommu_cleanup_int(bool iommu_disable) 256 { 257 258 if (add_tag != NULL) { 259 EVENTHANDLER_DEREGISTER(pci_add_device, add_tag); 260 add_tag = NULL; 261 } 262 if (delete_tag != NULL) { 263 EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag); 264 delete_tag = NULL; 265 } 266 if (iommu_disable) 267 IOMMU_DISABLE(); 268 IOMMU_DESTROY_DOMAIN(host_domain); 269 host_domain = NULL; 270 IOMMU_CLEANUP(); 271 } 272 273 void 274 iommu_cleanup(void) 275 { 276 iommu_cleanup_int(true); 277 } 278 279 void * 280 iommu_create_domain(vm_paddr_t maxaddr) 281 { 282 static volatile int iommu_initted; 283 284 if (iommu_initted < 2) { 285 if (atomic_cmpset_int(&iommu_initted, 0, 1)) { 286 iommu_init(); 287 atomic_store_rel_int(&iommu_initted, 2); 288 } else 289 while (iommu_initted == 1) 290 cpu_spinwait(); 291 } 292 return (IOMMU_CREATE_DOMAIN(maxaddr)); 293 } 294 295 void 296 iommu_destroy_domain(void *dom) 297 { 298 299 IOMMU_DESTROY_DOMAIN(dom); 300 } 301 302 int 303 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 304 { 305 uint64_t mapped, remaining; 306 int error; 307 308 for (remaining = len; remaining > 0; gpa += mapped, hpa += mapped, 309 remaining -= mapped) { 310 error = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining, 311 &mapped); 312 if (error != 0) { 313 /* XXXKIB rollback */ 314 return (error); 315 } 316 } 317 return (0); 318 } 319 320 int 321 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len) 322 { 323 uint64_t unmapped, remaining; 324 int error; 325 326 for (remaining = len; remaining > 0; gpa += unmapped, 327 remaining -= unmapped) { 328 error = IOMMU_REMOVE_MAPPING(dom, gpa, remaining, &unmapped); 329 if (error != 0) { 330 /* XXXKIB ? */ 331 return (error); 332 } 333 } 334 return (0); 335 } 336 337 void * 338 iommu_host_domain(void) 339 { 340 341 return (host_domain); 342 } 343 344 int 345 iommu_add_device(void *dom, device_t dev, uint16_t rid) 346 { 347 348 return (IOMMU_ADD_DEVICE(dom, dev, rid)); 349 } 350 351 int 352 iommu_remove_device(void *dom, device_t dev, uint16_t rid) 353 { 354 355 return (IOMMU_REMOVE_DEVICE(dom, dev, rid)); 356 } 357 358 int 359 iommu_invalidate_tlb(void *domain) 360 { 361 362 return (IOMMU_INVALIDATE_TLB(domain)); 363 } 364