1 /*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 /* Driver for the VirtIO PCI interface. */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/malloc.h> 38 39 #include <machine/bus.h> 40 #include <machine/resource.h> 41 #include <sys/bus.h> 42 #include <sys/rman.h> 43 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pcireg.h> 46 47 #include <dev/virtio/virtio.h> 48 #include <dev/virtio/virtqueue.h> 49 #include <dev/virtio/pci/virtio_pci.h> 50 51 #include "virtio_bus_if.h" 52 #include "virtio_if.h" 53 54 struct vtpci_interrupt { 55 struct resource *vti_irq; 56 int vti_rid; 57 void *vti_handler; 58 }; 59 60 struct vtpci_virtqueue { 61 struct virtqueue *vtv_vq; 62 int vtv_no_intr; 63 }; 64 65 struct vtpci_softc { 66 device_t vtpci_dev; 67 struct resource *vtpci_res; 68 struct resource *vtpci_msix_res; 69 uint64_t vtpci_features; 70 uint32_t vtpci_flags; 71 #define VTPCI_FLAG_NO_MSI 0x0001 72 #define VTPCI_FLAG_NO_MSIX 0x0002 73 #define VTPCI_FLAG_LEGACY 0x1000 74 #define VTPCI_FLAG_MSI 0x2000 75 #define VTPCI_FLAG_MSIX 0x4000 76 #define VTPCI_FLAG_SHARED_MSIX 0x8000 77 #define VTPCI_FLAG_ITYPE_MASK 0xF000 78 79 /* This "bus" will only ever have one child. */ 80 device_t vtpci_child_dev; 81 struct virtio_feature_desc *vtpci_child_feat_desc; 82 83 int vtpci_nvqs; 84 struct vtpci_virtqueue *vtpci_vqs; 85 86 /* 87 * Ideally, each virtqueue that the driver provides a callback for will 88 * receive its own MSIX vector. If there are not sufficient vectors 89 * available, then attempt to have all the VQs share one vector. For 90 * MSIX, the configuration changed notifications must be on their own 91 * vector. 92 * 93 * If MSIX is not available, we will attempt to have the whole device 94 * share one MSI vector, and then, finally, one legacy interrupt. 95 */ 96 struct vtpci_interrupt vtpci_device_interrupt; 97 struct vtpci_interrupt *vtpci_msix_vq_interrupts; 98 int vtpci_nmsix_resources; 99 }; 100 101 static int vtpci_probe(device_t); 102 static int vtpci_attach(device_t); 103 static int vtpci_detach(device_t); 104 static int vtpci_suspend(device_t); 105 static int vtpci_resume(device_t); 106 static int vtpci_shutdown(device_t); 107 static void vtpci_driver_added(device_t, driver_t *); 108 static void vtpci_child_detached(device_t, device_t); 109 static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); 110 static int vtpci_write_ivar(device_t, device_t, int, uintptr_t); 111 112 static uint64_t vtpci_negotiate_features(device_t, uint64_t); 113 static int vtpci_with_feature(device_t, uint64_t); 114 static int vtpci_alloc_virtqueues(device_t, int, int, 115 struct vq_alloc_info *); 116 static int vtpci_setup_intr(device_t, enum intr_type); 117 static void vtpci_stop(device_t); 118 static int vtpci_reinit(device_t, uint64_t); 119 static void vtpci_reinit_complete(device_t); 120 static void vtpci_notify_virtqueue(device_t, uint16_t); 121 static uint8_t vtpci_get_status(device_t); 122 static void vtpci_set_status(device_t, uint8_t); 123 static void vtpci_read_dev_config(device_t, bus_size_t, void *, int); 124 static void vtpci_write_dev_config(device_t, bus_size_t, void *, int); 125 126 static void vtpci_describe_features(struct vtpci_softc *, const char *, 127 uint64_t); 128 static void vtpci_probe_and_attach_child(struct vtpci_softc *); 129 130 static int vtpci_alloc_msix(struct vtpci_softc *, int); 131 static int vtpci_alloc_msi(struct vtpci_softc *); 132 static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *); 133 static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *); 134 static int vtpci_alloc_intr_msi(struct vtpci_softc *); 135 static int vtpci_alloc_intr_legacy(struct vtpci_softc *); 136 static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int, 137 struct vtpci_interrupt *); 138 static int vtpci_alloc_intr_resources(struct vtpci_softc *); 139 140 static int vtpci_setup_legacy_interrupt(struct vtpci_softc *, 141 enum intr_type); 142 static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *, 143 enum intr_type); 144 static int vtpci_setup_msix_interrupts(struct vtpci_softc *, 145 enum intr_type); 146 static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type); 147 148 static int vtpci_register_msix_vector(struct vtpci_softc *, int, 149 struct vtpci_interrupt *); 150 static int vtpci_set_host_msix_vectors(struct vtpci_softc *); 151 static int vtpci_reinit_virtqueue(struct vtpci_softc *, int); 152 153 static void vtpci_free_interrupt(struct vtpci_softc *, 154 struct vtpci_interrupt *); 155 static void vtpci_free_interrupts(struct vtpci_softc *); 156 static void vtpci_free_virtqueues(struct vtpci_softc *); 157 static void vtpci_release_child_resources(struct vtpci_softc *); 158 static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *); 159 static void vtpci_reset(struct vtpci_softc *); 160 161 static void vtpci_select_virtqueue(struct vtpci_softc *, int); 162 163 static void vtpci_legacy_intr(void *); 164 static int vtpci_vq_shared_intr_filter(void *); 165 static void vtpci_vq_shared_intr(void *); 166 static int vtpci_vq_intr_filter(void *); 167 static void vtpci_vq_intr(void *); 168 static void vtpci_config_intr(void *); 169 170 #define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt 171 172 #define VIRTIO_PCI_CONFIG(_sc) \ 173 VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0) 174 175 /* 176 * I/O port read/write wrappers. 177 */ 178 #define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) 179 #define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o)) 180 #define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o)) 181 #define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) 182 #define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) 183 #define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) 184 185 /* Tunables. */ 186 static int vtpci_disable_msix = 0; 187 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); 188 189 static device_method_t vtpci_methods[] = { 190 /* Device interface. */ 191 DEVMETHOD(device_probe, vtpci_probe), 192 DEVMETHOD(device_attach, vtpci_attach), 193 DEVMETHOD(device_detach, vtpci_detach), 194 DEVMETHOD(device_suspend, vtpci_suspend), 195 DEVMETHOD(device_resume, vtpci_resume), 196 DEVMETHOD(device_shutdown, vtpci_shutdown), 197 198 /* Bus interface. */ 199 DEVMETHOD(bus_driver_added, vtpci_driver_added), 200 DEVMETHOD(bus_child_detached, vtpci_child_detached), 201 DEVMETHOD(bus_read_ivar, vtpci_read_ivar), 202 DEVMETHOD(bus_write_ivar, vtpci_write_ivar), 203 204 /* VirtIO bus interface. */ 205 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), 206 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), 207 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), 208 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), 209 DEVMETHOD(virtio_bus_stop, vtpci_stop), 210 DEVMETHOD(virtio_bus_reinit, vtpci_reinit), 211 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), 212 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), 213 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), 214 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), 215 216 DEVMETHOD_END 217 }; 218 219 static driver_t vtpci_driver = { 220 "virtio_pci", 221 vtpci_methods, 222 sizeof(struct vtpci_softc) 223 }; 224 225 devclass_t vtpci_devclass; 226 227 DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); 228 MODULE_VERSION(virtio_pci, 1); 229 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); 230 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); 231 232 static int 233 vtpci_probe(device_t dev) 234 { 235 char desc[36]; 236 const char *name; 237 238 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 239 return (ENXIO); 240 241 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 242 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) 243 return (ENXIO); 244 245 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) 246 return (ENXIO); 247 248 name = virtio_device_name(pci_get_subdevice(dev)); 249 if (name == NULL) 250 name = "Unknown"; 251 252 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); 253 device_set_desc_copy(dev, desc); 254 255 return (BUS_PROBE_DEFAULT); 256 } 257 258 static int 259 vtpci_attach(device_t dev) 260 { 261 struct vtpci_softc *sc; 262 device_t child; 263 int rid; 264 265 sc = device_get_softc(dev); 266 sc->vtpci_dev = dev; 267 268 pci_enable_busmaster(dev); 269 270 rid = PCIR_BAR(0); 271 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 272 RF_ACTIVE); 273 if (sc->vtpci_res == NULL) { 274 device_printf(dev, "cannot map I/O space\n"); 275 return (ENXIO); 276 } 277 278 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) 279 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI; 280 281 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 282 rid = PCIR_BAR(1); 283 sc->vtpci_msix_res = bus_alloc_resource_any(dev, 284 SYS_RES_MEMORY, &rid, RF_ACTIVE); 285 } 286 287 if (sc->vtpci_msix_res == NULL) 288 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX; 289 290 vtpci_reset(sc); 291 292 /* Tell the host we've noticed this device. */ 293 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 294 295 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 296 device_printf(dev, "cannot create child device\n"); 297 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 298 vtpci_detach(dev); 299 return (ENOMEM); 300 } 301 302 sc->vtpci_child_dev = child; 303 vtpci_probe_and_attach_child(sc); 304 305 return (0); 306 } 307 308 static int 309 vtpci_detach(device_t dev) 310 { 311 struct vtpci_softc *sc; 312 device_t child; 313 int error; 314 315 sc = device_get_softc(dev); 316 317 if ((child = sc->vtpci_child_dev) != NULL) { 318 error = device_delete_child(dev, child); 319 if (error) 320 return (error); 321 sc->vtpci_child_dev = NULL; 322 } 323 324 vtpci_reset(sc); 325 326 if (sc->vtpci_msix_res != NULL) { 327 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), 328 sc->vtpci_msix_res); 329 sc->vtpci_msix_res = NULL; 330 } 331 332 if (sc->vtpci_res != NULL) { 333 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), 334 sc->vtpci_res); 335 sc->vtpci_res = NULL; 336 } 337 338 return (0); 339 } 340 341 static int 342 vtpci_suspend(device_t dev) 343 { 344 345 return (bus_generic_suspend(dev)); 346 } 347 348 static int 349 vtpci_resume(device_t dev) 350 { 351 352 return (bus_generic_resume(dev)); 353 } 354 355 static int 356 vtpci_shutdown(device_t dev) 357 { 358 359 (void) bus_generic_shutdown(dev); 360 /* Forcibly stop the host device. */ 361 vtpci_stop(dev); 362 363 return (0); 364 } 365 366 static void 367 vtpci_driver_added(device_t dev, driver_t *driver) 368 { 369 struct vtpci_softc *sc; 370 371 sc = device_get_softc(dev); 372 373 vtpci_probe_and_attach_child(sc); 374 } 375 376 static void 377 vtpci_child_detached(device_t dev, device_t child) 378 { 379 struct vtpci_softc *sc; 380 381 sc = device_get_softc(dev); 382 383 vtpci_reset(sc); 384 vtpci_release_child_resources(sc); 385 } 386 387 static int 388 vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 389 { 390 struct vtpci_softc *sc; 391 392 sc = device_get_softc(dev); 393 394 if (sc->vtpci_child_dev != child) 395 return (ENOENT); 396 397 switch (index) { 398 case VIRTIO_IVAR_DEVTYPE: 399 case VIRTIO_IVAR_SUBDEVICE: 400 *result = pci_get_subdevice(dev); 401 break; 402 case VIRTIO_IVAR_VENDOR: 403 *result = pci_get_vendor(dev); 404 break; 405 case VIRTIO_IVAR_DEVICE: 406 *result = pci_get_device(dev); 407 break; 408 case VIRTIO_IVAR_SUBVENDOR: 409 *result = pci_get_subdevice(dev); 410 break; 411 default: 412 return (ENOENT); 413 } 414 415 return (0); 416 } 417 418 static int 419 vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 420 { 421 struct vtpci_softc *sc; 422 423 sc = device_get_softc(dev); 424 425 if (sc->vtpci_child_dev != child) 426 return (ENOENT); 427 428 switch (index) { 429 case VIRTIO_IVAR_FEATURE_DESC: 430 sc->vtpci_child_feat_desc = (void *) value; 431 break; 432 default: 433 return (ENOENT); 434 } 435 436 return (0); 437 } 438 439 static uint64_t 440 vtpci_negotiate_features(device_t dev, uint64_t child_features) 441 { 442 struct vtpci_softc *sc; 443 uint64_t host_features, features; 444 445 sc = device_get_softc(dev); 446 447 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES); 448 vtpci_describe_features(sc, "host", host_features); 449 450 /* 451 * Limit negotiated features to what the driver, virtqueue, and 452 * host all support. 453 */ 454 features = host_features & child_features; 455 features = virtqueue_filter_features(features); 456 sc->vtpci_features = features; 457 458 vtpci_describe_features(sc, "negotiated", features); 459 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); 460 461 return (features); 462 } 463 464 static int 465 vtpci_with_feature(device_t dev, uint64_t feature) 466 { 467 struct vtpci_softc *sc; 468 469 sc = device_get_softc(dev); 470 471 return ((sc->vtpci_features & feature) != 0); 472 } 473 474 static int 475 vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, 476 struct vq_alloc_info *vq_info) 477 { 478 struct vtpci_softc *sc; 479 struct virtqueue *vq; 480 struct vtpci_virtqueue *vqx; 481 struct vq_alloc_info *info; 482 int idx, error; 483 uint16_t size; 484 485 sc = device_get_softc(dev); 486 487 if (sc->vtpci_nvqs != 0) 488 return (EALREADY); 489 if (nvqs <= 0) 490 return (EINVAL); 491 492 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), 493 M_DEVBUF, M_NOWAIT | M_ZERO); 494 if (sc->vtpci_vqs == NULL) 495 return (ENOMEM); 496 497 for (idx = 0; idx < nvqs; idx++) { 498 vqx = &sc->vtpci_vqs[idx]; 499 info = &vq_info[idx]; 500 501 vtpci_select_virtqueue(sc, idx); 502 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 503 504 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN, 505 0xFFFFFFFFUL, info, &vq); 506 if (error) { 507 device_printf(dev, 508 "cannot allocate virtqueue %d: %d\n", idx, error); 509 break; 510 } 511 512 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 513 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 514 515 vqx->vtv_vq = *info->vqai_vq = vq; 516 vqx->vtv_no_intr = info->vqai_intr == NULL; 517 518 sc->vtpci_nvqs++; 519 } 520 521 if (error) 522 vtpci_free_virtqueues(sc); 523 524 return (error); 525 } 526 527 static int 528 vtpci_setup_intr(device_t dev, enum intr_type type) 529 { 530 struct vtpci_softc *sc; 531 int attempt, error; 532 533 sc = device_get_softc(dev); 534 535 for (attempt = 0; attempt < 5; attempt++) { 536 /* 537 * Start with the most desirable interrupt configuration and 538 * fallback towards less desirable ones. 539 */ 540 switch (attempt) { 541 case 0: 542 error = vtpci_alloc_intr_msix_pervq(sc); 543 break; 544 case 1: 545 error = vtpci_alloc_intr_msix_shared(sc); 546 break; 547 case 2: 548 error = vtpci_alloc_intr_msi(sc); 549 break; 550 case 3: 551 error = vtpci_alloc_intr_legacy(sc); 552 break; 553 default: 554 device_printf(dev, 555 "exhausted all interrupt allocation attempts\n"); 556 return (ENXIO); 557 } 558 559 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0) 560 break; 561 562 vtpci_cleanup_setup_intr_attempt(sc); 563 } 564 565 if (bootverbose) { 566 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 567 device_printf(dev, "using legacy interrupt\n"); 568 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 569 device_printf(dev, "using MSI interrupt\n"); 570 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) 571 device_printf(dev, "using shared MSIX interrupts\n"); 572 else 573 device_printf(dev, "using per VQ MSIX interrupts\n"); 574 } 575 576 return (0); 577 } 578 579 static void 580 vtpci_stop(device_t dev) 581 { 582 583 vtpci_reset(device_get_softc(dev)); 584 } 585 586 static int 587 vtpci_reinit(device_t dev, uint64_t features) 588 { 589 struct vtpci_softc *sc; 590 int idx, error; 591 592 sc = device_get_softc(dev); 593 594 /* 595 * Redrive the device initialization. This is a bit of an abuse of 596 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 597 * play nice. 598 * 599 * We do not allow the host device to change from what was originally 600 * negotiated beyond what the guest driver changed. MSIX state should 601 * not change, number of virtqueues and their size remain the same, etc. 602 * This will need to be rethought when we want to support migration. 603 */ 604 605 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 606 vtpci_stop(dev); 607 608 /* 609 * Quickly drive the status through ACK and DRIVER. The device 610 * does not become usable again until vtpci_reinit_complete(). 611 */ 612 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 613 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 614 615 vtpci_negotiate_features(dev, features); 616 617 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 618 error = vtpci_reinit_virtqueue(sc, idx); 619 if (error) 620 return (error); 621 } 622 623 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 624 error = vtpci_set_host_msix_vectors(sc); 625 if (error) 626 return (error); 627 } 628 629 return (0); 630 } 631 632 static void 633 vtpci_reinit_complete(device_t dev) 634 { 635 636 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 637 } 638 639 static void 640 vtpci_notify_virtqueue(device_t dev, uint16_t queue) 641 { 642 struct vtpci_softc *sc; 643 644 sc = device_get_softc(dev); 645 646 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); 647 } 648 649 static uint8_t 650 vtpci_get_status(device_t dev) 651 { 652 struct vtpci_softc *sc; 653 654 sc = device_get_softc(dev); 655 656 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); 657 } 658 659 static void 660 vtpci_set_status(device_t dev, uint8_t status) 661 { 662 struct vtpci_softc *sc; 663 664 sc = device_get_softc(dev); 665 666 if (status != VIRTIO_CONFIG_STATUS_RESET) 667 status |= vtpci_get_status(dev); 668 669 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); 670 } 671 672 static void 673 vtpci_read_dev_config(device_t dev, bus_size_t offset, 674 void *dst, int length) 675 { 676 struct vtpci_softc *sc; 677 bus_size_t off; 678 uint8_t *d; 679 int size; 680 681 sc = device_get_softc(dev); 682 off = VIRTIO_PCI_CONFIG(sc) + offset; 683 684 for (d = dst; length > 0; d += size, off += size, length -= size) { 685 if (length >= 4) { 686 size = 4; 687 *(uint32_t *)d = vtpci_read_config_4(sc, off); 688 } else if (length >= 2) { 689 size = 2; 690 *(uint16_t *)d = vtpci_read_config_2(sc, off); 691 } else { 692 size = 1; 693 *d = vtpci_read_config_1(sc, off); 694 } 695 } 696 } 697 698 static void 699 vtpci_write_dev_config(device_t dev, bus_size_t offset, 700 void *src, int length) 701 { 702 struct vtpci_softc *sc; 703 bus_size_t off; 704 uint8_t *s; 705 int size; 706 707 sc = device_get_softc(dev); 708 off = VIRTIO_PCI_CONFIG(sc) + offset; 709 710 for (s = src; length > 0; s += size, off += size, length -= size) { 711 if (length >= 4) { 712 size = 4; 713 vtpci_write_config_4(sc, off, *(uint32_t *)s); 714 } else if (length >= 2) { 715 size = 2; 716 vtpci_write_config_2(sc, off, *(uint16_t *)s); 717 } else { 718 size = 1; 719 vtpci_write_config_1(sc, off, *s); 720 } 721 } 722 } 723 724 static void 725 vtpci_describe_features(struct vtpci_softc *sc, const char *msg, 726 uint64_t features) 727 { 728 device_t dev, child; 729 730 dev = sc->vtpci_dev; 731 child = sc->vtpci_child_dev; 732 733 if (device_is_attached(child) || bootverbose == 0) 734 return; 735 736 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); 737 } 738 739 static void 740 vtpci_probe_and_attach_child(struct vtpci_softc *sc) 741 { 742 device_t dev, child; 743 744 dev = sc->vtpci_dev; 745 child = sc->vtpci_child_dev; 746 747 if (child == NULL) 748 return; 749 750 if (device_get_state(child) != DS_NOTPRESENT) 751 return; 752 753 if (device_probe(child) != 0) 754 return; 755 756 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 757 if (device_attach(child) != 0) { 758 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 759 vtpci_reset(sc); 760 vtpci_release_child_resources(sc); 761 /* Reset status for future attempt. */ 762 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 763 } else { 764 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 765 VIRTIO_ATTACH_COMPLETED(child); 766 } 767 } 768 769 static int 770 vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) 771 { 772 device_t dev; 773 int nmsix, cnt, required; 774 775 dev = sc->vtpci_dev; 776 777 /* Allocate an additional vector for the config changes. */ 778 required = nvectors + 1; 779 780 nmsix = pci_msix_count(dev); 781 if (nmsix < required) 782 return (1); 783 784 cnt = required; 785 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 786 sc->vtpci_nmsix_resources = required; 787 return (0); 788 } 789 790 pci_release_msi(dev); 791 792 return (1); 793 } 794 795 static int 796 vtpci_alloc_msi(struct vtpci_softc *sc) 797 { 798 device_t dev; 799 int nmsi, cnt, required; 800 801 dev = sc->vtpci_dev; 802 required = 1; 803 804 nmsi = pci_msi_count(dev); 805 if (nmsi < required) 806 return (1); 807 808 cnt = required; 809 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) 810 return (0); 811 812 pci_release_msi(dev); 813 814 return (1); 815 } 816 817 static int 818 vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc) 819 { 820 int i, nvectors, error; 821 822 if (vtpci_disable_msix != 0 || 823 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 824 return (ENOTSUP); 825 826 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) { 827 if (sc->vtpci_vqs[i].vtv_no_intr == 0) 828 nvectors++; 829 } 830 831 error = vtpci_alloc_msix(sc, nvectors); 832 if (error) 833 return (error); 834 835 sc->vtpci_flags |= VTPCI_FLAG_MSIX; 836 837 return (0); 838 } 839 840 static int 841 vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc) 842 { 843 int error; 844 845 if (vtpci_disable_msix != 0 || 846 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 847 return (ENOTSUP); 848 849 error = vtpci_alloc_msix(sc, 1); 850 if (error) 851 return (error); 852 853 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; 854 855 return (0); 856 } 857 858 static int 859 vtpci_alloc_intr_msi(struct vtpci_softc *sc) 860 { 861 int error; 862 863 /* Only BHyVe supports MSI. */ 864 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI) 865 return (ENOTSUP); 866 867 error = vtpci_alloc_msi(sc); 868 if (error) 869 return (error); 870 871 sc->vtpci_flags |= VTPCI_FLAG_MSI; 872 873 return (0); 874 } 875 876 static int 877 vtpci_alloc_intr_legacy(struct vtpci_softc *sc) 878 { 879 880 sc->vtpci_flags |= VTPCI_FLAG_LEGACY; 881 882 return (0); 883 } 884 885 static int 886 vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, 887 struct vtpci_interrupt *intr) 888 { 889 struct resource *irq; 890 891 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags); 892 if (irq == NULL) 893 return (ENXIO); 894 895 intr->vti_irq = irq; 896 intr->vti_rid = rid; 897 898 return (0); 899 } 900 901 static int 902 vtpci_alloc_intr_resources(struct vtpci_softc *sc) 903 { 904 struct vtpci_interrupt *intr; 905 int i, rid, flags, nvq_intrs, error; 906 907 rid = 0; 908 flags = RF_ACTIVE; 909 910 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 911 flags |= RF_SHAREABLE; 912 else 913 rid = 1; 914 915 /* 916 * For legacy and MSI interrupts, this single resource handles all 917 * interrupts. For MSIX, this resource is used for the configuration 918 * changed interrupt. 919 */ 920 intr = &sc->vtpci_device_interrupt; 921 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 922 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI)) 923 return (error); 924 925 /* Subtract one for the configuration changed interrupt. */ 926 nvq_intrs = sc->vtpci_nmsix_resources - 1; 927 928 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs * 929 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); 930 if (sc->vtpci_msix_vq_interrupts == NULL) 931 return (ENOMEM); 932 933 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { 934 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 935 if (error) 936 return (error); 937 } 938 939 return (0); 940 } 941 942 static int 943 vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type) 944 { 945 struct vtpci_interrupt *intr; 946 int error; 947 948 intr = &sc->vtpci_device_interrupt; 949 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL, 950 vtpci_legacy_intr, sc, &intr->vti_handler); 951 952 return (error); 953 } 954 955 static int 956 vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 957 { 958 struct vtpci_virtqueue *vqx; 959 struct vtpci_interrupt *intr; 960 int i, error; 961 962 intr = sc->vtpci_msix_vq_interrupts; 963 964 for (i = 0; i < sc->vtpci_nvqs; i++) { 965 vqx = &sc->vtpci_vqs[i]; 966 967 if (vqx->vtv_no_intr) 968 continue; 969 970 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, 971 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, 972 &intr->vti_handler); 973 if (error) 974 return (error); 975 976 intr++; 977 } 978 979 return (0); 980 } 981 982 static int 983 vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 984 { 985 device_t dev; 986 struct vtpci_interrupt *intr; 987 int error; 988 989 dev = sc->vtpci_dev; 990 intr = &sc->vtpci_device_interrupt; 991 992 error = bus_setup_intr(dev, intr->vti_irq, type, NULL, 993 vtpci_config_intr, sc, &intr->vti_handler); 994 if (error) 995 return (error); 996 997 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { 998 intr = sc->vtpci_msix_vq_interrupts; 999 error = bus_setup_intr(dev, intr->vti_irq, type, 1000 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc, 1001 &intr->vti_handler); 1002 } else 1003 error = vtpci_setup_pervq_msix_interrupts(sc, type); 1004 1005 return (error ? error : vtpci_set_host_msix_vectors(sc)); 1006 } 1007 1008 static int 1009 vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type) 1010 { 1011 int error; 1012 1013 type |= INTR_MPSAFE; 1014 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, 1015 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags)); 1016 1017 error = vtpci_alloc_intr_resources(sc); 1018 if (error) 1019 return (error); 1020 1021 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 1022 error = vtpci_setup_legacy_interrupt(sc, type); 1023 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 1024 error = vtpci_setup_msi_interrupt(sc, type); 1025 else 1026 error = vtpci_setup_msix_interrupts(sc, type); 1027 1028 return (error); 1029 } 1030 1031 static int 1032 vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, 1033 struct vtpci_interrupt *intr) 1034 { 1035 device_t dev; 1036 uint16_t vector; 1037 1038 dev = sc->vtpci_dev; 1039 1040 if (intr != NULL) { 1041 /* Map from guest rid to host vector. */ 1042 vector = intr->vti_rid - 1; 1043 } else 1044 vector = VIRTIO_MSI_NO_VECTOR; 1045 1046 vtpci_write_config_2(sc, offset, vector); 1047 1048 /* Read vector to determine if the host had sufficient resources. */ 1049 if (vtpci_read_config_2(sc, offset) != vector) { 1050 device_printf(dev, 1051 "insufficient host resources for MSIX interrupts\n"); 1052 return (ENODEV); 1053 } 1054 1055 return (0); 1056 } 1057 1058 static int 1059 vtpci_set_host_msix_vectors(struct vtpci_softc *sc) 1060 { 1061 struct vtpci_interrupt *intr, *tintr; 1062 int idx, offset, error; 1063 1064 intr = &sc->vtpci_device_interrupt; 1065 offset = VIRTIO_MSI_CONFIG_VECTOR; 1066 1067 error = vtpci_register_msix_vector(sc, offset, intr); 1068 if (error) 1069 return (error); 1070 1071 intr = sc->vtpci_msix_vq_interrupts; 1072 offset = VIRTIO_MSI_QUEUE_VECTOR; 1073 1074 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1075 vtpci_select_virtqueue(sc, idx); 1076 1077 if (sc->vtpci_vqs[idx].vtv_no_intr) 1078 tintr = NULL; 1079 else 1080 tintr = intr; 1081 1082 error = vtpci_register_msix_vector(sc, offset, tintr); 1083 if (error) 1084 break; 1085 1086 /* 1087 * For shared MSIX, all the virtqueues share the first 1088 * interrupt. 1089 */ 1090 if (!sc->vtpci_vqs[idx].vtv_no_intr && 1091 (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) 1092 intr++; 1093 } 1094 1095 return (error); 1096 } 1097 1098 static int 1099 vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx) 1100 { 1101 struct vtpci_virtqueue *vqx; 1102 struct virtqueue *vq; 1103 int error; 1104 uint16_t size; 1105 1106 vqx = &sc->vtpci_vqs[idx]; 1107 vq = vqx->vtv_vq; 1108 1109 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 1110 1111 vtpci_select_virtqueue(sc, idx); 1112 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 1113 1114 error = virtqueue_reinit(vq, size); 1115 if (error) 1116 return (error); 1117 1118 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 1119 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 1120 1121 return (0); 1122 } 1123 1124 static void 1125 vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr) 1126 { 1127 device_t dev; 1128 1129 dev = sc->vtpci_dev; 1130 1131 if (intr->vti_handler != NULL) { 1132 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); 1133 intr->vti_handler = NULL; 1134 } 1135 1136 if (intr->vti_irq != NULL) { 1137 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, 1138 intr->vti_irq); 1139 intr->vti_irq = NULL; 1140 intr->vti_rid = -1; 1141 } 1142 } 1143 1144 static void 1145 vtpci_free_interrupts(struct vtpci_softc *sc) 1146 { 1147 struct vtpci_interrupt *intr; 1148 int i, nvq_intrs; 1149 1150 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt); 1151 1152 if (sc->vtpci_nmsix_resources != 0) { 1153 nvq_intrs = sc->vtpci_nmsix_resources - 1; 1154 sc->vtpci_nmsix_resources = 0; 1155 1156 intr = sc->vtpci_msix_vq_interrupts; 1157 if (intr != NULL) { 1158 for (i = 0; i < nvq_intrs; i++, intr++) 1159 vtpci_free_interrupt(sc, intr); 1160 1161 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF); 1162 sc->vtpci_msix_vq_interrupts = NULL; 1163 } 1164 } 1165 1166 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) 1167 pci_release_msi(sc->vtpci_dev); 1168 1169 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; 1170 } 1171 1172 static void 1173 vtpci_free_virtqueues(struct vtpci_softc *sc) 1174 { 1175 struct vtpci_virtqueue *vqx; 1176 int idx; 1177 1178 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1179 vqx = &sc->vtpci_vqs[idx]; 1180 1181 vtpci_select_virtqueue(sc, idx); 1182 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); 1183 1184 virtqueue_free(vqx->vtv_vq); 1185 vqx->vtv_vq = NULL; 1186 } 1187 1188 free(sc->vtpci_vqs, M_DEVBUF); 1189 sc->vtpci_vqs = NULL; 1190 sc->vtpci_nvqs = 0; 1191 } 1192 1193 static void 1194 vtpci_release_child_resources(struct vtpci_softc *sc) 1195 { 1196 1197 vtpci_free_interrupts(sc); 1198 vtpci_free_virtqueues(sc); 1199 } 1200 1201 static void 1202 vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc) 1203 { 1204 int idx; 1205 1206 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 1207 vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR, 1208 VIRTIO_MSI_NO_VECTOR); 1209 1210 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1211 vtpci_select_virtqueue(sc, idx); 1212 vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR, 1213 VIRTIO_MSI_NO_VECTOR); 1214 } 1215 } 1216 1217 vtpci_free_interrupts(sc); 1218 } 1219 1220 static void 1221 vtpci_reset(struct vtpci_softc *sc) 1222 { 1223 1224 /* 1225 * Setting the status to RESET sets the host device to 1226 * the original, uninitialized state. 1227 */ 1228 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); 1229 } 1230 1231 static void 1232 vtpci_select_virtqueue(struct vtpci_softc *sc, int idx) 1233 { 1234 1235 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); 1236 } 1237 1238 static void 1239 vtpci_legacy_intr(void *xsc) 1240 { 1241 struct vtpci_softc *sc; 1242 struct vtpci_virtqueue *vqx; 1243 int i; 1244 uint8_t isr; 1245 1246 sc = xsc; 1247 vqx = &sc->vtpci_vqs[0]; 1248 1249 /* Reading the ISR also clears it. */ 1250 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); 1251 1252 if (isr & VIRTIO_PCI_ISR_CONFIG) 1253 vtpci_config_intr(sc); 1254 1255 if (isr & VIRTIO_PCI_ISR_INTR) { 1256 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1257 if (vqx->vtv_no_intr == 0) 1258 virtqueue_intr(vqx->vtv_vq); 1259 } 1260 } 1261 } 1262 1263 static int 1264 vtpci_vq_shared_intr_filter(void *xsc) 1265 { 1266 struct vtpci_softc *sc; 1267 struct vtpci_virtqueue *vqx; 1268 int i, rc; 1269 1270 rc = 0; 1271 sc = xsc; 1272 vqx = &sc->vtpci_vqs[0]; 1273 1274 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1275 if (vqx->vtv_no_intr == 0) 1276 rc |= virtqueue_intr_filter(vqx->vtv_vq); 1277 } 1278 1279 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1280 } 1281 1282 static void 1283 vtpci_vq_shared_intr(void *xsc) 1284 { 1285 struct vtpci_softc *sc; 1286 struct vtpci_virtqueue *vqx; 1287 int i; 1288 1289 sc = xsc; 1290 vqx = &sc->vtpci_vqs[0]; 1291 1292 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1293 if (vqx->vtv_no_intr == 0) 1294 virtqueue_intr(vqx->vtv_vq); 1295 } 1296 } 1297 1298 static int 1299 vtpci_vq_intr_filter(void *xvq) 1300 { 1301 struct virtqueue *vq; 1302 int rc; 1303 1304 vq = xvq; 1305 rc = virtqueue_intr_filter(vq); 1306 1307 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1308 } 1309 1310 static void 1311 vtpci_vq_intr(void *xvq) 1312 { 1313 struct virtqueue *vq; 1314 1315 vq = xvq; 1316 virtqueue_intr(vq); 1317 } 1318 1319 static void 1320 vtpci_config_intr(void *xsc) 1321 { 1322 struct vtpci_softc *sc; 1323 device_t child; 1324 1325 sc = xsc; 1326 child = sc->vtpci_child_dev; 1327 1328 if (child != NULL) 1329 VIRTIO_CONFIG_CHANGE(child); 1330 } 1331