1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for the VirtIO PCI interface. */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/module.h> 39 #include <sys/malloc.h> 40 #include <sys/endian.h> 41 42 #include <machine/bus.h> 43 #include <machine/resource.h> 44 #include <sys/bus.h> 45 #include <sys/rman.h> 46 47 #include <dev/pci/pcivar.h> 48 #include <dev/pci/pcireg.h> 49 50 #include <dev/virtio/virtio.h> 51 #include <dev/virtio/virtqueue.h> 52 #include <dev/virtio/pci/virtio_pci.h> 53 54 #include "virtio_bus_if.h" 55 #include "virtio_if.h" 56 57 struct vtpci_interrupt { 58 struct resource *vti_irq; 59 int vti_rid; 60 void *vti_handler; 61 }; 62 63 struct vtpci_virtqueue { 64 struct virtqueue *vtv_vq; 65 int vtv_no_intr; 66 }; 67 68 struct vtpci_softc { 69 device_t vtpci_dev; 70 struct resource *vtpci_res; 71 struct resource *vtpci_msix_res; 72 uint64_t vtpci_features; 73 uint32_t vtpci_flags; 74 #define VTPCI_FLAG_NO_MSI 0x0001 75 #define VTPCI_FLAG_NO_MSIX 0x0002 76 #define VTPCI_FLAG_LEGACY 0x1000 77 #define VTPCI_FLAG_MSI 0x2000 78 #define VTPCI_FLAG_MSIX 0x4000 79 #define VTPCI_FLAG_SHARED_MSIX 0x8000 80 #define VTPCI_FLAG_ITYPE_MASK 0xF000 81 82 /* This "bus" will only ever have one child. */ 83 device_t vtpci_child_dev; 84 struct virtio_feature_desc *vtpci_child_feat_desc; 85 86 int vtpci_nvqs; 87 struct vtpci_virtqueue *vtpci_vqs; 88 89 /* 90 * Ideally, each virtqueue that the driver provides a callback for will 91 * receive its own MSIX vector. If there are not sufficient vectors 92 * available, then attempt to have all the VQs share one vector. For 93 * MSIX, the configuration changed notifications must be on their own 94 * vector. 95 * 96 * If MSIX is not available, we will attempt to have the whole device 97 * share one MSI vector, and then, finally, one legacy interrupt. 98 */ 99 struct vtpci_interrupt vtpci_device_interrupt; 100 struct vtpci_interrupt *vtpci_msix_vq_interrupts; 101 int vtpci_nmsix_resources; 102 }; 103 104 static int vtpci_probe(device_t); 105 static int vtpci_attach(device_t); 106 static int vtpci_detach(device_t); 107 static int vtpci_suspend(device_t); 108 static int vtpci_resume(device_t); 109 static int vtpci_shutdown(device_t); 110 static void vtpci_driver_added(device_t, driver_t *); 111 static void vtpci_child_detached(device_t, device_t); 112 static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); 113 static int vtpci_write_ivar(device_t, device_t, int, uintptr_t); 114 115 static uint64_t vtpci_negotiate_features(device_t, uint64_t); 116 static int vtpci_with_feature(device_t, uint64_t); 117 static int vtpci_alloc_virtqueues(device_t, int, int, 118 struct vq_alloc_info *); 119 static int vtpci_setup_intr(device_t, enum intr_type); 120 static void vtpci_stop(device_t); 121 static int vtpci_reinit(device_t, uint64_t); 122 static void vtpci_reinit_complete(device_t); 123 static void vtpci_notify_virtqueue(device_t, uint16_t); 124 static uint8_t vtpci_get_status(device_t); 125 static void vtpci_set_status(device_t, uint8_t); 126 static void vtpci_read_dev_config(device_t, bus_size_t, void *, int); 127 static void vtpci_write_dev_config(device_t, bus_size_t, void *, int); 128 129 static void vtpci_describe_features(struct vtpci_softc *, const char *, 130 uint64_t); 131 static void vtpci_probe_and_attach_child(struct vtpci_softc *); 132 133 static int vtpci_alloc_msix(struct vtpci_softc *, int); 134 static int vtpci_alloc_msi(struct vtpci_softc *); 135 static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *); 136 static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *); 137 static int vtpci_alloc_intr_msi(struct vtpci_softc *); 138 static int vtpci_alloc_intr_legacy(struct vtpci_softc *); 139 static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int, 140 struct vtpci_interrupt *); 141 static int vtpci_alloc_intr_resources(struct vtpci_softc *); 142 143 static int vtpci_setup_legacy_interrupt(struct vtpci_softc *, 144 enum intr_type); 145 static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *, 146 enum intr_type); 147 static int vtpci_setup_msix_interrupts(struct vtpci_softc *, 148 enum intr_type); 149 static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type); 150 151 static int vtpci_register_msix_vector(struct vtpci_softc *, int, 152 struct vtpci_interrupt *); 153 static int vtpci_set_host_msix_vectors(struct vtpci_softc *); 154 static int vtpci_reinit_virtqueue(struct vtpci_softc *, int); 155 156 static void vtpci_free_interrupt(struct vtpci_softc *, 157 struct vtpci_interrupt *); 158 static void vtpci_free_interrupts(struct vtpci_softc *); 159 static void vtpci_free_virtqueues(struct vtpci_softc *); 160 static void vtpci_release_child_resources(struct vtpci_softc *); 161 static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *); 162 static void vtpci_reset(struct vtpci_softc *); 163 164 static void vtpci_select_virtqueue(struct vtpci_softc *, int); 165 166 static void vtpci_legacy_intr(void *); 167 static int vtpci_vq_shared_intr_filter(void *); 168 static void vtpci_vq_shared_intr(void *); 169 static int vtpci_vq_intr_filter(void *); 170 static void vtpci_vq_intr(void *); 171 static void vtpci_config_intr(void *); 172 173 #define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt 174 175 #define VIRTIO_PCI_CONFIG(_sc) \ 176 VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0) 177 178 /* 179 * I/O port read/write wrappers. 180 */ 181 #define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) 182 #define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) 183 184 /* 185 * Virtio-pci specifies that PCI Configuration area is guest endian. However, 186 * since PCI devices are inherently little-endian, on BE systems the bus layer 187 * transparently converts it to BE. For virtio-legacy, this conversion is 188 * undesired, so an extra byte swap is required to fix it. 189 */ 190 #define vtpci_read_config_2(sc, o) le16toh(bus_read_2((sc)->vtpci_res, (o))) 191 #define vtpci_read_config_4(sc, o) le32toh(bus_read_4((sc)->vtpci_res, (o))) 192 #define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (htole16(v))) 193 #define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (htole32(v))) 194 195 /* PCI Header LE. On BE systems the bus layer takes care of byte swapping */ 196 #define vtpci_read_header_2(sc, o) (bus_read_2((sc)->vtpci_res, (o))) 197 #define vtpci_read_header_4(sc, o) (bus_read_4((sc)->vtpci_res, (o))) 198 #define vtpci_write_header_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) 199 #define vtpci_write_header_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) 200 201 /* Tunables. */ 202 static int vtpci_disable_msix = 0; 203 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); 204 205 static device_method_t vtpci_methods[] = { 206 /* Device interface. */ 207 DEVMETHOD(device_probe, vtpci_probe), 208 DEVMETHOD(device_attach, vtpci_attach), 209 DEVMETHOD(device_detach, vtpci_detach), 210 DEVMETHOD(device_suspend, vtpci_suspend), 211 DEVMETHOD(device_resume, vtpci_resume), 212 DEVMETHOD(device_shutdown, vtpci_shutdown), 213 214 /* Bus interface. */ 215 DEVMETHOD(bus_driver_added, vtpci_driver_added), 216 DEVMETHOD(bus_child_detached, vtpci_child_detached), 217 DEVMETHOD(bus_child_pnpinfo_str, virtio_child_pnpinfo_str), 218 DEVMETHOD(bus_read_ivar, vtpci_read_ivar), 219 DEVMETHOD(bus_write_ivar, vtpci_write_ivar), 220 221 /* VirtIO bus interface. */ 222 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), 223 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), 224 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), 225 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), 226 DEVMETHOD(virtio_bus_stop, vtpci_stop), 227 DEVMETHOD(virtio_bus_reinit, vtpci_reinit), 228 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), 229 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), 230 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), 231 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), 232 233 DEVMETHOD_END 234 }; 235 236 static driver_t vtpci_driver = { 237 "virtio_pci", 238 vtpci_methods, 239 sizeof(struct vtpci_softc) 240 }; 241 242 devclass_t vtpci_devclass; 243 244 DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); 245 MODULE_VERSION(virtio_pci, 1); 246 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); 247 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); 248 249 static int 250 vtpci_probe(device_t dev) 251 { 252 char desc[36]; 253 const char *name; 254 255 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 256 return (ENXIO); 257 258 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 259 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) 260 return (ENXIO); 261 262 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) 263 return (ENXIO); 264 265 name = virtio_device_name(pci_get_subdevice(dev)); 266 if (name == NULL) 267 name = "Unknown"; 268 269 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); 270 device_set_desc_copy(dev, desc); 271 272 return (BUS_PROBE_DEFAULT); 273 } 274 275 static int 276 vtpci_attach(device_t dev) 277 { 278 struct vtpci_softc *sc; 279 device_t child; 280 int rid; 281 282 sc = device_get_softc(dev); 283 sc->vtpci_dev = dev; 284 285 pci_enable_busmaster(dev); 286 287 rid = PCIR_BAR(0); 288 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 289 RF_ACTIVE); 290 if (sc->vtpci_res == NULL) { 291 device_printf(dev, "cannot map I/O space\n"); 292 return (ENXIO); 293 } 294 295 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) 296 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI; 297 298 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 299 rid = PCIR_BAR(1); 300 sc->vtpci_msix_res = bus_alloc_resource_any(dev, 301 SYS_RES_MEMORY, &rid, RF_ACTIVE); 302 } 303 304 if (sc->vtpci_msix_res == NULL) 305 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX; 306 307 vtpci_reset(sc); 308 309 /* Tell the host we've noticed this device. */ 310 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 311 312 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 313 device_printf(dev, "cannot create child device\n"); 314 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 315 vtpci_detach(dev); 316 return (ENOMEM); 317 } 318 319 sc->vtpci_child_dev = child; 320 vtpci_probe_and_attach_child(sc); 321 322 return (0); 323 } 324 325 static int 326 vtpci_detach(device_t dev) 327 { 328 struct vtpci_softc *sc; 329 device_t child; 330 int error; 331 332 sc = device_get_softc(dev); 333 334 if ((child = sc->vtpci_child_dev) != NULL) { 335 error = device_delete_child(dev, child); 336 if (error) 337 return (error); 338 sc->vtpci_child_dev = NULL; 339 } 340 341 vtpci_reset(sc); 342 343 if (sc->vtpci_msix_res != NULL) { 344 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), 345 sc->vtpci_msix_res); 346 sc->vtpci_msix_res = NULL; 347 } 348 349 if (sc->vtpci_res != NULL) { 350 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), 351 sc->vtpci_res); 352 sc->vtpci_res = NULL; 353 } 354 355 return (0); 356 } 357 358 static int 359 vtpci_suspend(device_t dev) 360 { 361 362 return (bus_generic_suspend(dev)); 363 } 364 365 static int 366 vtpci_resume(device_t dev) 367 { 368 369 return (bus_generic_resume(dev)); 370 } 371 372 static int 373 vtpci_shutdown(device_t dev) 374 { 375 376 (void) bus_generic_shutdown(dev); 377 /* Forcibly stop the host device. */ 378 vtpci_stop(dev); 379 380 return (0); 381 } 382 383 static void 384 vtpci_driver_added(device_t dev, driver_t *driver) 385 { 386 struct vtpci_softc *sc; 387 388 sc = device_get_softc(dev); 389 390 vtpci_probe_and_attach_child(sc); 391 } 392 393 static void 394 vtpci_child_detached(device_t dev, device_t child) 395 { 396 struct vtpci_softc *sc; 397 398 sc = device_get_softc(dev); 399 400 vtpci_reset(sc); 401 vtpci_release_child_resources(sc); 402 } 403 404 static int 405 vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 406 { 407 struct vtpci_softc *sc; 408 409 sc = device_get_softc(dev); 410 411 if (sc->vtpci_child_dev != child) 412 return (ENOENT); 413 414 switch (index) { 415 case VIRTIO_IVAR_DEVTYPE: 416 case VIRTIO_IVAR_SUBDEVICE: 417 *result = pci_get_subdevice(dev); 418 break; 419 case VIRTIO_IVAR_VENDOR: 420 *result = pci_get_vendor(dev); 421 break; 422 case VIRTIO_IVAR_DEVICE: 423 *result = pci_get_device(dev); 424 break; 425 case VIRTIO_IVAR_SUBVENDOR: 426 *result = pci_get_subvendor(dev); 427 break; 428 default: 429 return (ENOENT); 430 } 431 432 return (0); 433 } 434 435 static int 436 vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 437 { 438 struct vtpci_softc *sc; 439 440 sc = device_get_softc(dev); 441 442 if (sc->vtpci_child_dev != child) 443 return (ENOENT); 444 445 switch (index) { 446 case VIRTIO_IVAR_FEATURE_DESC: 447 sc->vtpci_child_feat_desc = (void *) value; 448 break; 449 default: 450 return (ENOENT); 451 } 452 453 return (0); 454 } 455 456 static uint64_t 457 vtpci_negotiate_features(device_t dev, uint64_t child_features) 458 { 459 struct vtpci_softc *sc; 460 uint64_t host_features, features; 461 462 sc = device_get_softc(dev); 463 464 host_features = vtpci_read_header_4(sc, VIRTIO_PCI_HOST_FEATURES); 465 vtpci_describe_features(sc, "host", host_features); 466 467 /* 468 * Limit negotiated features to what the driver, virtqueue, and 469 * host all support. 470 */ 471 features = host_features & child_features; 472 features = virtqueue_filter_features(features); 473 sc->vtpci_features = features; 474 475 vtpci_describe_features(sc, "negotiated", features); 476 vtpci_write_header_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); 477 478 return (features); 479 } 480 481 static int 482 vtpci_with_feature(device_t dev, uint64_t feature) 483 { 484 struct vtpci_softc *sc; 485 486 sc = device_get_softc(dev); 487 488 return ((sc->vtpci_features & feature) != 0); 489 } 490 491 static int 492 vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, 493 struct vq_alloc_info *vq_info) 494 { 495 struct vtpci_softc *sc; 496 struct virtqueue *vq; 497 struct vtpci_virtqueue *vqx; 498 struct vq_alloc_info *info; 499 int idx, error; 500 uint16_t size; 501 502 sc = device_get_softc(dev); 503 504 if (sc->vtpci_nvqs != 0) 505 return (EALREADY); 506 if (nvqs <= 0) 507 return (EINVAL); 508 509 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), 510 M_DEVBUF, M_NOWAIT | M_ZERO); 511 if (sc->vtpci_vqs == NULL) 512 return (ENOMEM); 513 514 for (idx = 0; idx < nvqs; idx++) { 515 vqx = &sc->vtpci_vqs[idx]; 516 info = &vq_info[idx]; 517 518 vtpci_select_virtqueue(sc, idx); 519 size = vtpci_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM); 520 521 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN, 522 ~(vm_paddr_t)0, info, &vq); 523 if (error) { 524 device_printf(dev, 525 "cannot allocate virtqueue %d: %d\n", idx, error); 526 break; 527 } 528 529 vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, 530 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 531 532 vqx->vtv_vq = *info->vqai_vq = vq; 533 vqx->vtv_no_intr = info->vqai_intr == NULL; 534 535 sc->vtpci_nvqs++; 536 } 537 538 if (error) 539 vtpci_free_virtqueues(sc); 540 541 return (error); 542 } 543 544 static int 545 vtpci_setup_intr(device_t dev, enum intr_type type) 546 { 547 struct vtpci_softc *sc; 548 int attempt, error; 549 550 sc = device_get_softc(dev); 551 552 for (attempt = 0; attempt < 5; attempt++) { 553 /* 554 * Start with the most desirable interrupt configuration and 555 * fallback towards less desirable ones. 556 */ 557 switch (attempt) { 558 case 0: 559 error = vtpci_alloc_intr_msix_pervq(sc); 560 break; 561 case 1: 562 error = vtpci_alloc_intr_msix_shared(sc); 563 break; 564 case 2: 565 error = vtpci_alloc_intr_msi(sc); 566 break; 567 case 3: 568 error = vtpci_alloc_intr_legacy(sc); 569 break; 570 default: 571 device_printf(dev, 572 "exhausted all interrupt allocation attempts\n"); 573 return (ENXIO); 574 } 575 576 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0) 577 break; 578 579 vtpci_cleanup_setup_intr_attempt(sc); 580 } 581 582 if (bootverbose) { 583 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 584 device_printf(dev, "using legacy interrupt\n"); 585 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 586 device_printf(dev, "using MSI interrupt\n"); 587 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) 588 device_printf(dev, "using shared MSIX interrupts\n"); 589 else 590 device_printf(dev, "using per VQ MSIX interrupts\n"); 591 } 592 593 return (0); 594 } 595 596 static void 597 vtpci_stop(device_t dev) 598 { 599 600 vtpci_reset(device_get_softc(dev)); 601 } 602 603 static int 604 vtpci_reinit(device_t dev, uint64_t features) 605 { 606 struct vtpci_softc *sc; 607 int idx, error; 608 609 sc = device_get_softc(dev); 610 611 /* 612 * Redrive the device initialization. This is a bit of an abuse of 613 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 614 * play nice. 615 * 616 * We do not allow the host device to change from what was originally 617 * negotiated beyond what the guest driver changed. MSIX state should 618 * not change, number of virtqueues and their size remain the same, etc. 619 * This will need to be rethought when we want to support migration. 620 */ 621 622 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 623 vtpci_stop(dev); 624 625 /* 626 * Quickly drive the status through ACK and DRIVER. The device 627 * does not become usable again until vtpci_reinit_complete(). 628 */ 629 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 630 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 631 632 vtpci_negotiate_features(dev, features); 633 634 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 635 error = vtpci_reinit_virtqueue(sc, idx); 636 if (error) 637 return (error); 638 } 639 640 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 641 error = vtpci_set_host_msix_vectors(sc); 642 if (error) 643 return (error); 644 } 645 646 return (0); 647 } 648 649 static void 650 vtpci_reinit_complete(device_t dev) 651 { 652 653 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 654 } 655 656 static void 657 vtpci_notify_virtqueue(device_t dev, uint16_t queue) 658 { 659 struct vtpci_softc *sc; 660 661 sc = device_get_softc(dev); 662 663 vtpci_write_header_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); 664 } 665 666 static uint8_t 667 vtpci_get_status(device_t dev) 668 { 669 struct vtpci_softc *sc; 670 671 sc = device_get_softc(dev); 672 673 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); 674 } 675 676 static void 677 vtpci_set_status(device_t dev, uint8_t status) 678 { 679 struct vtpci_softc *sc; 680 681 sc = device_get_softc(dev); 682 683 if (status != VIRTIO_CONFIG_STATUS_RESET) 684 status |= vtpci_get_status(dev); 685 686 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); 687 } 688 689 static void 690 vtpci_read_dev_config(device_t dev, bus_size_t offset, 691 void *dst, int length) 692 { 693 struct vtpci_softc *sc; 694 bus_size_t off; 695 uint8_t *d; 696 int size; 697 698 sc = device_get_softc(dev); 699 off = VIRTIO_PCI_CONFIG(sc) + offset; 700 701 for (d = dst; length > 0; d += size, off += size, length -= size) { 702 if (length >= 4) { 703 size = 4; 704 *(uint32_t *)d = vtpci_read_config_4(sc, off); 705 } else if (length >= 2) { 706 size = 2; 707 *(uint16_t *)d = vtpci_read_config_2(sc, off); 708 } else { 709 size = 1; 710 *d = vtpci_read_config_1(sc, off); 711 } 712 } 713 } 714 715 static void 716 vtpci_write_dev_config(device_t dev, bus_size_t offset, 717 void *src, int length) 718 { 719 struct vtpci_softc *sc; 720 bus_size_t off; 721 uint8_t *s; 722 int size; 723 724 sc = device_get_softc(dev); 725 off = VIRTIO_PCI_CONFIG(sc) + offset; 726 727 for (s = src; length > 0; s += size, off += size, length -= size) { 728 if (length >= 4) { 729 size = 4; 730 vtpci_write_config_4(sc, off, *(uint32_t *)s); 731 } else if (length >= 2) { 732 size = 2; 733 vtpci_write_config_2(sc, off, *(uint16_t *)s); 734 } else { 735 size = 1; 736 vtpci_write_config_1(sc, off, *s); 737 } 738 } 739 } 740 741 static void 742 vtpci_describe_features(struct vtpci_softc *sc, const char *msg, 743 uint64_t features) 744 { 745 device_t dev, child; 746 747 dev = sc->vtpci_dev; 748 child = sc->vtpci_child_dev; 749 750 if (device_is_attached(child) || bootverbose == 0) 751 return; 752 753 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); 754 } 755 756 static void 757 vtpci_probe_and_attach_child(struct vtpci_softc *sc) 758 { 759 device_t dev, child; 760 761 dev = sc->vtpci_dev; 762 child = sc->vtpci_child_dev; 763 764 if (child == NULL) 765 return; 766 767 if (device_get_state(child) != DS_NOTPRESENT) 768 return; 769 770 if (device_probe(child) != 0) 771 return; 772 773 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 774 if (device_attach(child) != 0) { 775 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 776 vtpci_reset(sc); 777 vtpci_release_child_resources(sc); 778 /* Reset status for future attempt. */ 779 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 780 } else { 781 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 782 VIRTIO_ATTACH_COMPLETED(child); 783 } 784 } 785 786 static int 787 vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) 788 { 789 device_t dev; 790 int nmsix, cnt, required; 791 792 dev = sc->vtpci_dev; 793 794 /* Allocate an additional vector for the config changes. */ 795 required = nvectors + 1; 796 797 nmsix = pci_msix_count(dev); 798 if (nmsix < required) 799 return (1); 800 801 cnt = required; 802 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 803 sc->vtpci_nmsix_resources = required; 804 return (0); 805 } 806 807 pci_release_msi(dev); 808 809 return (1); 810 } 811 812 static int 813 vtpci_alloc_msi(struct vtpci_softc *sc) 814 { 815 device_t dev; 816 int nmsi, cnt, required; 817 818 dev = sc->vtpci_dev; 819 required = 1; 820 821 nmsi = pci_msi_count(dev); 822 if (nmsi < required) 823 return (1); 824 825 cnt = required; 826 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) 827 return (0); 828 829 pci_release_msi(dev); 830 831 return (1); 832 } 833 834 static int 835 vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc) 836 { 837 int i, nvectors, error; 838 839 if (vtpci_disable_msix != 0 || 840 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 841 return (ENOTSUP); 842 843 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) { 844 if (sc->vtpci_vqs[i].vtv_no_intr == 0) 845 nvectors++; 846 } 847 848 error = vtpci_alloc_msix(sc, nvectors); 849 if (error) 850 return (error); 851 852 sc->vtpci_flags |= VTPCI_FLAG_MSIX; 853 854 return (0); 855 } 856 857 static int 858 vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc) 859 { 860 int error; 861 862 if (vtpci_disable_msix != 0 || 863 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 864 return (ENOTSUP); 865 866 error = vtpci_alloc_msix(sc, 1); 867 if (error) 868 return (error); 869 870 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; 871 872 return (0); 873 } 874 875 static int 876 vtpci_alloc_intr_msi(struct vtpci_softc *sc) 877 { 878 int error; 879 880 /* Only BHyVe supports MSI. */ 881 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI) 882 return (ENOTSUP); 883 884 error = vtpci_alloc_msi(sc); 885 if (error) 886 return (error); 887 888 sc->vtpci_flags |= VTPCI_FLAG_MSI; 889 890 return (0); 891 } 892 893 static int 894 vtpci_alloc_intr_legacy(struct vtpci_softc *sc) 895 { 896 897 sc->vtpci_flags |= VTPCI_FLAG_LEGACY; 898 899 return (0); 900 } 901 902 static int 903 vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, 904 struct vtpci_interrupt *intr) 905 { 906 struct resource *irq; 907 908 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags); 909 if (irq == NULL) 910 return (ENXIO); 911 912 intr->vti_irq = irq; 913 intr->vti_rid = rid; 914 915 return (0); 916 } 917 918 static int 919 vtpci_alloc_intr_resources(struct vtpci_softc *sc) 920 { 921 struct vtpci_interrupt *intr; 922 int i, rid, flags, nvq_intrs, error; 923 924 rid = 0; 925 flags = RF_ACTIVE; 926 927 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 928 flags |= RF_SHAREABLE; 929 else 930 rid = 1; 931 932 /* 933 * For legacy and MSI interrupts, this single resource handles all 934 * interrupts. For MSIX, this resource is used for the configuration 935 * changed interrupt. 936 */ 937 intr = &sc->vtpci_device_interrupt; 938 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 939 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI)) 940 return (error); 941 942 /* Subtract one for the configuration changed interrupt. */ 943 nvq_intrs = sc->vtpci_nmsix_resources - 1; 944 945 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs * 946 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); 947 if (sc->vtpci_msix_vq_interrupts == NULL) 948 return (ENOMEM); 949 950 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { 951 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 952 if (error) 953 return (error); 954 } 955 956 return (0); 957 } 958 959 static int 960 vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type) 961 { 962 struct vtpci_interrupt *intr; 963 int error; 964 965 intr = &sc->vtpci_device_interrupt; 966 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL, 967 vtpci_legacy_intr, sc, &intr->vti_handler); 968 969 return (error); 970 } 971 972 static int 973 vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 974 { 975 struct vtpci_virtqueue *vqx; 976 struct vtpci_interrupt *intr; 977 int i, error; 978 979 intr = sc->vtpci_msix_vq_interrupts; 980 981 for (i = 0; i < sc->vtpci_nvqs; i++) { 982 vqx = &sc->vtpci_vqs[i]; 983 984 if (vqx->vtv_no_intr) 985 continue; 986 987 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, 988 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, 989 &intr->vti_handler); 990 if (error) 991 return (error); 992 993 intr++; 994 } 995 996 return (0); 997 } 998 999 static int 1000 vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 1001 { 1002 device_t dev; 1003 struct vtpci_interrupt *intr; 1004 int error; 1005 1006 dev = sc->vtpci_dev; 1007 intr = &sc->vtpci_device_interrupt; 1008 1009 error = bus_setup_intr(dev, intr->vti_irq, type, NULL, 1010 vtpci_config_intr, sc, &intr->vti_handler); 1011 if (error) 1012 return (error); 1013 1014 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { 1015 intr = sc->vtpci_msix_vq_interrupts; 1016 error = bus_setup_intr(dev, intr->vti_irq, type, 1017 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc, 1018 &intr->vti_handler); 1019 } else 1020 error = vtpci_setup_pervq_msix_interrupts(sc, type); 1021 1022 return (error ? error : vtpci_set_host_msix_vectors(sc)); 1023 } 1024 1025 static int 1026 vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type) 1027 { 1028 int error; 1029 1030 type |= INTR_MPSAFE; 1031 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, 1032 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags)); 1033 1034 error = vtpci_alloc_intr_resources(sc); 1035 if (error) 1036 return (error); 1037 1038 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 1039 error = vtpci_setup_legacy_interrupt(sc, type); 1040 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 1041 error = vtpci_setup_msi_interrupt(sc, type); 1042 else 1043 error = vtpci_setup_msix_interrupts(sc, type); 1044 1045 return (error); 1046 } 1047 1048 static int 1049 vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, 1050 struct vtpci_interrupt *intr) 1051 { 1052 device_t dev; 1053 uint16_t vector; 1054 1055 dev = sc->vtpci_dev; 1056 1057 if (intr != NULL) { 1058 /* Map from guest rid to host vector. */ 1059 vector = intr->vti_rid - 1; 1060 } else 1061 vector = VIRTIO_MSI_NO_VECTOR; 1062 1063 vtpci_write_header_2(sc, offset, vector); 1064 1065 /* Read vector to determine if the host had sufficient resources. */ 1066 if (vtpci_read_header_2(sc, offset) != vector) { 1067 device_printf(dev, 1068 "insufficient host resources for MSIX interrupts\n"); 1069 return (ENODEV); 1070 } 1071 1072 return (0); 1073 } 1074 1075 static int 1076 vtpci_set_host_msix_vectors(struct vtpci_softc *sc) 1077 { 1078 struct vtpci_interrupt *intr, *tintr; 1079 int idx, offset, error; 1080 1081 intr = &sc->vtpci_device_interrupt; 1082 offset = VIRTIO_MSI_CONFIG_VECTOR; 1083 1084 error = vtpci_register_msix_vector(sc, offset, intr); 1085 if (error) 1086 return (error); 1087 1088 intr = sc->vtpci_msix_vq_interrupts; 1089 offset = VIRTIO_MSI_QUEUE_VECTOR; 1090 1091 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1092 vtpci_select_virtqueue(sc, idx); 1093 1094 if (sc->vtpci_vqs[idx].vtv_no_intr) 1095 tintr = NULL; 1096 else 1097 tintr = intr; 1098 1099 error = vtpci_register_msix_vector(sc, offset, tintr); 1100 if (error) 1101 break; 1102 1103 /* 1104 * For shared MSIX, all the virtqueues share the first 1105 * interrupt. 1106 */ 1107 if (!sc->vtpci_vqs[idx].vtv_no_intr && 1108 (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) 1109 intr++; 1110 } 1111 1112 return (error); 1113 } 1114 1115 static int 1116 vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx) 1117 { 1118 struct vtpci_virtqueue *vqx; 1119 struct virtqueue *vq; 1120 int error; 1121 uint16_t size; 1122 1123 vqx = &sc->vtpci_vqs[idx]; 1124 vq = vqx->vtv_vq; 1125 1126 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 1127 1128 vtpci_select_virtqueue(sc, idx); 1129 size = vtpci_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM); 1130 1131 error = virtqueue_reinit(vq, size); 1132 if (error) 1133 return (error); 1134 1135 vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, 1136 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 1137 1138 return (0); 1139 } 1140 1141 static void 1142 vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr) 1143 { 1144 device_t dev; 1145 1146 dev = sc->vtpci_dev; 1147 1148 if (intr->vti_handler != NULL) { 1149 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); 1150 intr->vti_handler = NULL; 1151 } 1152 1153 if (intr->vti_irq != NULL) { 1154 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, 1155 intr->vti_irq); 1156 intr->vti_irq = NULL; 1157 intr->vti_rid = -1; 1158 } 1159 } 1160 1161 static void 1162 vtpci_free_interrupts(struct vtpci_softc *sc) 1163 { 1164 struct vtpci_interrupt *intr; 1165 int i, nvq_intrs; 1166 1167 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt); 1168 1169 if (sc->vtpci_nmsix_resources != 0) { 1170 nvq_intrs = sc->vtpci_nmsix_resources - 1; 1171 sc->vtpci_nmsix_resources = 0; 1172 1173 intr = sc->vtpci_msix_vq_interrupts; 1174 if (intr != NULL) { 1175 for (i = 0; i < nvq_intrs; i++, intr++) 1176 vtpci_free_interrupt(sc, intr); 1177 1178 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF); 1179 sc->vtpci_msix_vq_interrupts = NULL; 1180 } 1181 } 1182 1183 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) 1184 pci_release_msi(sc->vtpci_dev); 1185 1186 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; 1187 } 1188 1189 static void 1190 vtpci_free_virtqueues(struct vtpci_softc *sc) 1191 { 1192 struct vtpci_virtqueue *vqx; 1193 int idx; 1194 1195 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1196 vqx = &sc->vtpci_vqs[idx]; 1197 1198 vtpci_select_virtqueue(sc, idx); 1199 vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); 1200 1201 virtqueue_free(vqx->vtv_vq); 1202 vqx->vtv_vq = NULL; 1203 } 1204 1205 free(sc->vtpci_vqs, M_DEVBUF); 1206 sc->vtpci_vqs = NULL; 1207 sc->vtpci_nvqs = 0; 1208 } 1209 1210 static void 1211 vtpci_release_child_resources(struct vtpci_softc *sc) 1212 { 1213 1214 vtpci_free_interrupts(sc); 1215 vtpci_free_virtqueues(sc); 1216 } 1217 1218 static void 1219 vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc) 1220 { 1221 int idx; 1222 1223 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 1224 vtpci_write_header_2(sc, VIRTIO_MSI_CONFIG_VECTOR, 1225 VIRTIO_MSI_NO_VECTOR); 1226 1227 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1228 vtpci_select_virtqueue(sc, idx); 1229 vtpci_write_header_2(sc, VIRTIO_MSI_QUEUE_VECTOR, 1230 VIRTIO_MSI_NO_VECTOR); 1231 } 1232 } 1233 1234 vtpci_free_interrupts(sc); 1235 } 1236 1237 static void 1238 vtpci_reset(struct vtpci_softc *sc) 1239 { 1240 1241 /* 1242 * Setting the status to RESET sets the host device to 1243 * the original, uninitialized state. 1244 */ 1245 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); 1246 } 1247 1248 static void 1249 vtpci_select_virtqueue(struct vtpci_softc *sc, int idx) 1250 { 1251 1252 vtpci_write_header_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); 1253 } 1254 1255 static void 1256 vtpci_legacy_intr(void *xsc) 1257 { 1258 struct vtpci_softc *sc; 1259 struct vtpci_virtqueue *vqx; 1260 int i; 1261 uint8_t isr; 1262 1263 sc = xsc; 1264 vqx = &sc->vtpci_vqs[0]; 1265 1266 /* Reading the ISR also clears it. */ 1267 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); 1268 1269 if (isr & VIRTIO_PCI_ISR_CONFIG) 1270 vtpci_config_intr(sc); 1271 1272 if (isr & VIRTIO_PCI_ISR_INTR) { 1273 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1274 if (vqx->vtv_no_intr == 0) 1275 virtqueue_intr(vqx->vtv_vq); 1276 } 1277 } 1278 } 1279 1280 static int 1281 vtpci_vq_shared_intr_filter(void *xsc) 1282 { 1283 struct vtpci_softc *sc; 1284 struct vtpci_virtqueue *vqx; 1285 int i, rc; 1286 1287 rc = 0; 1288 sc = xsc; 1289 vqx = &sc->vtpci_vqs[0]; 1290 1291 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1292 if (vqx->vtv_no_intr == 0) 1293 rc |= virtqueue_intr_filter(vqx->vtv_vq); 1294 } 1295 1296 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1297 } 1298 1299 static void 1300 vtpci_vq_shared_intr(void *xsc) 1301 { 1302 struct vtpci_softc *sc; 1303 struct vtpci_virtqueue *vqx; 1304 int i; 1305 1306 sc = xsc; 1307 vqx = &sc->vtpci_vqs[0]; 1308 1309 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1310 if (vqx->vtv_no_intr == 0) 1311 virtqueue_intr(vqx->vtv_vq); 1312 } 1313 } 1314 1315 static int 1316 vtpci_vq_intr_filter(void *xvq) 1317 { 1318 struct virtqueue *vq; 1319 int rc; 1320 1321 vq = xvq; 1322 rc = virtqueue_intr_filter(vq); 1323 1324 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1325 } 1326 1327 static void 1328 vtpci_vq_intr(void *xvq) 1329 { 1330 struct virtqueue *vq; 1331 1332 vq = xvq; 1333 virtqueue_intr(vq); 1334 } 1335 1336 static void 1337 vtpci_config_intr(void *xsc) 1338 { 1339 struct vtpci_softc *sc; 1340 device_t child; 1341 1342 sc = xsc; 1343 child = sc->vtpci_child_dev; 1344 1345 if (child != NULL) 1346 VIRTIO_CONFIG_CHANGE(child); 1347 } 1348