1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for the VirtIO PCI interface. */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/module.h> 39 #include <sys/malloc.h> 40 41 #include <machine/bus.h> 42 #include <machine/resource.h> 43 #include <sys/bus.h> 44 #include <sys/rman.h> 45 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pcireg.h> 48 49 #include <dev/virtio/virtio.h> 50 #include <dev/virtio/virtqueue.h> 51 #include <dev/virtio/pci/virtio_pci.h> 52 53 #include "virtio_bus_if.h" 54 #include "virtio_if.h" 55 56 struct vtpci_interrupt { 57 struct resource *vti_irq; 58 int vti_rid; 59 void *vti_handler; 60 }; 61 62 struct vtpci_virtqueue { 63 struct virtqueue *vtv_vq; 64 int vtv_no_intr; 65 }; 66 67 struct vtpci_softc { 68 device_t vtpci_dev; 69 struct resource *vtpci_res; 70 struct resource *vtpci_msix_res; 71 uint64_t vtpci_features; 72 uint32_t vtpci_flags; 73 #define VTPCI_FLAG_NO_MSI 0x0001 74 #define VTPCI_FLAG_NO_MSIX 0x0002 75 #define VTPCI_FLAG_LEGACY 0x1000 76 #define VTPCI_FLAG_MSI 0x2000 77 #define VTPCI_FLAG_MSIX 0x4000 78 #define VTPCI_FLAG_SHARED_MSIX 0x8000 79 #define VTPCI_FLAG_ITYPE_MASK 0xF000 80 81 /* This "bus" will only ever have one child. */ 82 device_t vtpci_child_dev; 83 struct virtio_feature_desc *vtpci_child_feat_desc; 84 85 int vtpci_nvqs; 86 struct vtpci_virtqueue *vtpci_vqs; 87 88 /* 89 * Ideally, each virtqueue that the driver provides a callback for will 90 * receive its own MSIX vector. If there are not sufficient vectors 91 * available, then attempt to have all the VQs share one vector. For 92 * MSIX, the configuration changed notifications must be on their own 93 * vector. 94 * 95 * If MSIX is not available, we will attempt to have the whole device 96 * share one MSI vector, and then, finally, one legacy interrupt. 97 */ 98 struct vtpci_interrupt vtpci_device_interrupt; 99 struct vtpci_interrupt *vtpci_msix_vq_interrupts; 100 int vtpci_nmsix_resources; 101 }; 102 103 static int vtpci_probe(device_t); 104 static int vtpci_attach(device_t); 105 static int vtpci_detach(device_t); 106 static int vtpci_suspend(device_t); 107 static int vtpci_resume(device_t); 108 static int vtpci_shutdown(device_t); 109 static void vtpci_driver_added(device_t, driver_t *); 110 static void vtpci_child_detached(device_t, device_t); 111 static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); 112 static int vtpci_write_ivar(device_t, device_t, int, uintptr_t); 113 114 static uint64_t vtpci_negotiate_features(device_t, uint64_t); 115 static int vtpci_with_feature(device_t, uint64_t); 116 static int vtpci_alloc_virtqueues(device_t, int, int, 117 struct vq_alloc_info *); 118 static int vtpci_setup_intr(device_t, enum intr_type); 119 static void vtpci_stop(device_t); 120 static int vtpci_reinit(device_t, uint64_t); 121 static void vtpci_reinit_complete(device_t); 122 static void vtpci_notify_virtqueue(device_t, uint16_t); 123 static uint8_t vtpci_get_status(device_t); 124 static void vtpci_set_status(device_t, uint8_t); 125 static void vtpci_read_dev_config(device_t, bus_size_t, void *, int); 126 static void vtpci_write_dev_config(device_t, bus_size_t, void *, int); 127 128 static void vtpci_describe_features(struct vtpci_softc *, const char *, 129 uint64_t); 130 static void vtpci_probe_and_attach_child(struct vtpci_softc *); 131 132 static int vtpci_alloc_msix(struct vtpci_softc *, int); 133 static int vtpci_alloc_msi(struct vtpci_softc *); 134 static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *); 135 static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *); 136 static int vtpci_alloc_intr_msi(struct vtpci_softc *); 137 static int vtpci_alloc_intr_legacy(struct vtpci_softc *); 138 static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int, 139 struct vtpci_interrupt *); 140 static int vtpci_alloc_intr_resources(struct vtpci_softc *); 141 142 static int vtpci_setup_legacy_interrupt(struct vtpci_softc *, 143 enum intr_type); 144 static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *, 145 enum intr_type); 146 static int vtpci_setup_msix_interrupts(struct vtpci_softc *, 147 enum intr_type); 148 static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type); 149 150 static int vtpci_register_msix_vector(struct vtpci_softc *, int, 151 struct vtpci_interrupt *); 152 static int vtpci_set_host_msix_vectors(struct vtpci_softc *); 153 static int vtpci_reinit_virtqueue(struct vtpci_softc *, int); 154 155 static void vtpci_free_interrupt(struct vtpci_softc *, 156 struct vtpci_interrupt *); 157 static void vtpci_free_interrupts(struct vtpci_softc *); 158 static void vtpci_free_virtqueues(struct vtpci_softc *); 159 static void vtpci_release_child_resources(struct vtpci_softc *); 160 static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *); 161 static void vtpci_reset(struct vtpci_softc *); 162 163 static void vtpci_select_virtqueue(struct vtpci_softc *, int); 164 165 static void vtpci_legacy_intr(void *); 166 static int vtpci_vq_shared_intr_filter(void *); 167 static void vtpci_vq_shared_intr(void *); 168 static int vtpci_vq_intr_filter(void *); 169 static void vtpci_vq_intr(void *); 170 static void vtpci_config_intr(void *); 171 172 #define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt 173 174 #define VIRTIO_PCI_CONFIG(_sc) \ 175 VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0) 176 177 /* 178 * I/O port read/write wrappers. 179 */ 180 #define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) 181 #define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o)) 182 #define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o)) 183 #define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) 184 #define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) 185 #define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) 186 187 /* Tunables. */ 188 static int vtpci_disable_msix = 0; 189 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); 190 191 static device_method_t vtpci_methods[] = { 192 /* Device interface. */ 193 DEVMETHOD(device_probe, vtpci_probe), 194 DEVMETHOD(device_attach, vtpci_attach), 195 DEVMETHOD(device_detach, vtpci_detach), 196 DEVMETHOD(device_suspend, vtpci_suspend), 197 DEVMETHOD(device_resume, vtpci_resume), 198 DEVMETHOD(device_shutdown, vtpci_shutdown), 199 200 /* Bus interface. */ 201 DEVMETHOD(bus_driver_added, vtpci_driver_added), 202 DEVMETHOD(bus_child_detached, vtpci_child_detached), 203 DEVMETHOD(bus_read_ivar, vtpci_read_ivar), 204 DEVMETHOD(bus_write_ivar, vtpci_write_ivar), 205 206 /* VirtIO bus interface. */ 207 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), 208 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), 209 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), 210 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), 211 DEVMETHOD(virtio_bus_stop, vtpci_stop), 212 DEVMETHOD(virtio_bus_reinit, vtpci_reinit), 213 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), 214 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), 215 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), 216 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), 217 218 DEVMETHOD_END 219 }; 220 221 static driver_t vtpci_driver = { 222 "virtio_pci", 223 vtpci_methods, 224 sizeof(struct vtpci_softc) 225 }; 226 227 devclass_t vtpci_devclass; 228 229 DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); 230 MODULE_VERSION(virtio_pci, 1); 231 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); 232 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); 233 234 static int 235 vtpci_probe(device_t dev) 236 { 237 char desc[36]; 238 const char *name; 239 240 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 241 return (ENXIO); 242 243 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 244 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) 245 return (ENXIO); 246 247 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) 248 return (ENXIO); 249 250 name = virtio_device_name(pci_get_subdevice(dev)); 251 if (name == NULL) 252 name = "Unknown"; 253 254 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); 255 device_set_desc_copy(dev, desc); 256 257 return (BUS_PROBE_DEFAULT); 258 } 259 260 static int 261 vtpci_attach(device_t dev) 262 { 263 struct vtpci_softc *sc; 264 device_t child; 265 int rid; 266 267 sc = device_get_softc(dev); 268 sc->vtpci_dev = dev; 269 270 pci_enable_busmaster(dev); 271 272 rid = PCIR_BAR(0); 273 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 274 RF_ACTIVE); 275 if (sc->vtpci_res == NULL) { 276 device_printf(dev, "cannot map I/O space\n"); 277 return (ENXIO); 278 } 279 280 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) 281 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI; 282 283 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 284 rid = PCIR_BAR(1); 285 sc->vtpci_msix_res = bus_alloc_resource_any(dev, 286 SYS_RES_MEMORY, &rid, RF_ACTIVE); 287 } 288 289 if (sc->vtpci_msix_res == NULL) 290 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX; 291 292 vtpci_reset(sc); 293 294 /* Tell the host we've noticed this device. */ 295 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 296 297 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 298 device_printf(dev, "cannot create child device\n"); 299 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 300 vtpci_detach(dev); 301 return (ENOMEM); 302 } 303 304 sc->vtpci_child_dev = child; 305 vtpci_probe_and_attach_child(sc); 306 307 return (0); 308 } 309 310 static int 311 vtpci_detach(device_t dev) 312 { 313 struct vtpci_softc *sc; 314 device_t child; 315 int error; 316 317 sc = device_get_softc(dev); 318 319 if ((child = sc->vtpci_child_dev) != NULL) { 320 error = device_delete_child(dev, child); 321 if (error) 322 return (error); 323 sc->vtpci_child_dev = NULL; 324 } 325 326 vtpci_reset(sc); 327 328 if (sc->vtpci_msix_res != NULL) { 329 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), 330 sc->vtpci_msix_res); 331 sc->vtpci_msix_res = NULL; 332 } 333 334 if (sc->vtpci_res != NULL) { 335 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), 336 sc->vtpci_res); 337 sc->vtpci_res = NULL; 338 } 339 340 return (0); 341 } 342 343 static int 344 vtpci_suspend(device_t dev) 345 { 346 347 return (bus_generic_suspend(dev)); 348 } 349 350 static int 351 vtpci_resume(device_t dev) 352 { 353 354 return (bus_generic_resume(dev)); 355 } 356 357 static int 358 vtpci_shutdown(device_t dev) 359 { 360 361 (void) bus_generic_shutdown(dev); 362 /* Forcibly stop the host device. */ 363 vtpci_stop(dev); 364 365 return (0); 366 } 367 368 static void 369 vtpci_driver_added(device_t dev, driver_t *driver) 370 { 371 struct vtpci_softc *sc; 372 373 sc = device_get_softc(dev); 374 375 vtpci_probe_and_attach_child(sc); 376 } 377 378 static void 379 vtpci_child_detached(device_t dev, device_t child) 380 { 381 struct vtpci_softc *sc; 382 383 sc = device_get_softc(dev); 384 385 vtpci_reset(sc); 386 vtpci_release_child_resources(sc); 387 } 388 389 static int 390 vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 391 { 392 struct vtpci_softc *sc; 393 394 sc = device_get_softc(dev); 395 396 if (sc->vtpci_child_dev != child) 397 return (ENOENT); 398 399 switch (index) { 400 case VIRTIO_IVAR_DEVTYPE: 401 case VIRTIO_IVAR_SUBDEVICE: 402 *result = pci_get_subdevice(dev); 403 break; 404 case VIRTIO_IVAR_VENDOR: 405 *result = pci_get_vendor(dev); 406 break; 407 case VIRTIO_IVAR_DEVICE: 408 *result = pci_get_device(dev); 409 break; 410 case VIRTIO_IVAR_SUBVENDOR: 411 *result = pci_get_subdevice(dev); 412 break; 413 default: 414 return (ENOENT); 415 } 416 417 return (0); 418 } 419 420 static int 421 vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 422 { 423 struct vtpci_softc *sc; 424 425 sc = device_get_softc(dev); 426 427 if (sc->vtpci_child_dev != child) 428 return (ENOENT); 429 430 switch (index) { 431 case VIRTIO_IVAR_FEATURE_DESC: 432 sc->vtpci_child_feat_desc = (void *) value; 433 break; 434 default: 435 return (ENOENT); 436 } 437 438 return (0); 439 } 440 441 static uint64_t 442 vtpci_negotiate_features(device_t dev, uint64_t child_features) 443 { 444 struct vtpci_softc *sc; 445 uint64_t host_features, features; 446 447 sc = device_get_softc(dev); 448 449 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES); 450 vtpci_describe_features(sc, "host", host_features); 451 452 /* 453 * Limit negotiated features to what the driver, virtqueue, and 454 * host all support. 455 */ 456 features = host_features & child_features; 457 features = virtqueue_filter_features(features); 458 sc->vtpci_features = features; 459 460 vtpci_describe_features(sc, "negotiated", features); 461 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); 462 463 return (features); 464 } 465 466 static int 467 vtpci_with_feature(device_t dev, uint64_t feature) 468 { 469 struct vtpci_softc *sc; 470 471 sc = device_get_softc(dev); 472 473 return ((sc->vtpci_features & feature) != 0); 474 } 475 476 static int 477 vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, 478 struct vq_alloc_info *vq_info) 479 { 480 struct vtpci_softc *sc; 481 struct virtqueue *vq; 482 struct vtpci_virtqueue *vqx; 483 struct vq_alloc_info *info; 484 int idx, error; 485 uint16_t size; 486 487 sc = device_get_softc(dev); 488 489 if (sc->vtpci_nvqs != 0) 490 return (EALREADY); 491 if (nvqs <= 0) 492 return (EINVAL); 493 494 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), 495 M_DEVBUF, M_NOWAIT | M_ZERO); 496 if (sc->vtpci_vqs == NULL) 497 return (ENOMEM); 498 499 for (idx = 0; idx < nvqs; idx++) { 500 vqx = &sc->vtpci_vqs[idx]; 501 info = &vq_info[idx]; 502 503 vtpci_select_virtqueue(sc, idx); 504 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 505 506 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN, 507 0xFFFFFFFFUL, info, &vq); 508 if (error) { 509 device_printf(dev, 510 "cannot allocate virtqueue %d: %d\n", idx, error); 511 break; 512 } 513 514 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 515 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 516 517 vqx->vtv_vq = *info->vqai_vq = vq; 518 vqx->vtv_no_intr = info->vqai_intr == NULL; 519 520 sc->vtpci_nvqs++; 521 } 522 523 if (error) 524 vtpci_free_virtqueues(sc); 525 526 return (error); 527 } 528 529 static int 530 vtpci_setup_intr(device_t dev, enum intr_type type) 531 { 532 struct vtpci_softc *sc; 533 int attempt, error; 534 535 sc = device_get_softc(dev); 536 537 for (attempt = 0; attempt < 5; attempt++) { 538 /* 539 * Start with the most desirable interrupt configuration and 540 * fallback towards less desirable ones. 541 */ 542 switch (attempt) { 543 case 0: 544 error = vtpci_alloc_intr_msix_pervq(sc); 545 break; 546 case 1: 547 error = vtpci_alloc_intr_msix_shared(sc); 548 break; 549 case 2: 550 error = vtpci_alloc_intr_msi(sc); 551 break; 552 case 3: 553 error = vtpci_alloc_intr_legacy(sc); 554 break; 555 default: 556 device_printf(dev, 557 "exhausted all interrupt allocation attempts\n"); 558 return (ENXIO); 559 } 560 561 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0) 562 break; 563 564 vtpci_cleanup_setup_intr_attempt(sc); 565 } 566 567 if (bootverbose) { 568 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 569 device_printf(dev, "using legacy interrupt\n"); 570 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 571 device_printf(dev, "using MSI interrupt\n"); 572 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) 573 device_printf(dev, "using shared MSIX interrupts\n"); 574 else 575 device_printf(dev, "using per VQ MSIX interrupts\n"); 576 } 577 578 return (0); 579 } 580 581 static void 582 vtpci_stop(device_t dev) 583 { 584 585 vtpci_reset(device_get_softc(dev)); 586 } 587 588 static int 589 vtpci_reinit(device_t dev, uint64_t features) 590 { 591 struct vtpci_softc *sc; 592 int idx, error; 593 594 sc = device_get_softc(dev); 595 596 /* 597 * Redrive the device initialization. This is a bit of an abuse of 598 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 599 * play nice. 600 * 601 * We do not allow the host device to change from what was originally 602 * negotiated beyond what the guest driver changed. MSIX state should 603 * not change, number of virtqueues and their size remain the same, etc. 604 * This will need to be rethought when we want to support migration. 605 */ 606 607 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 608 vtpci_stop(dev); 609 610 /* 611 * Quickly drive the status through ACK and DRIVER. The device 612 * does not become usable again until vtpci_reinit_complete(). 613 */ 614 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 615 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 616 617 vtpci_negotiate_features(dev, features); 618 619 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 620 error = vtpci_reinit_virtqueue(sc, idx); 621 if (error) 622 return (error); 623 } 624 625 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 626 error = vtpci_set_host_msix_vectors(sc); 627 if (error) 628 return (error); 629 } 630 631 return (0); 632 } 633 634 static void 635 vtpci_reinit_complete(device_t dev) 636 { 637 638 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 639 } 640 641 static void 642 vtpci_notify_virtqueue(device_t dev, uint16_t queue) 643 { 644 struct vtpci_softc *sc; 645 646 sc = device_get_softc(dev); 647 648 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); 649 } 650 651 static uint8_t 652 vtpci_get_status(device_t dev) 653 { 654 struct vtpci_softc *sc; 655 656 sc = device_get_softc(dev); 657 658 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); 659 } 660 661 static void 662 vtpci_set_status(device_t dev, uint8_t status) 663 { 664 struct vtpci_softc *sc; 665 666 sc = device_get_softc(dev); 667 668 if (status != VIRTIO_CONFIG_STATUS_RESET) 669 status |= vtpci_get_status(dev); 670 671 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); 672 } 673 674 static void 675 vtpci_read_dev_config(device_t dev, bus_size_t offset, 676 void *dst, int length) 677 { 678 struct vtpci_softc *sc; 679 bus_size_t off; 680 uint8_t *d; 681 int size; 682 683 sc = device_get_softc(dev); 684 off = VIRTIO_PCI_CONFIG(sc) + offset; 685 686 for (d = dst; length > 0; d += size, off += size, length -= size) { 687 if (length >= 4) { 688 size = 4; 689 *(uint32_t *)d = vtpci_read_config_4(sc, off); 690 } else if (length >= 2) { 691 size = 2; 692 *(uint16_t *)d = vtpci_read_config_2(sc, off); 693 } else { 694 size = 1; 695 *d = vtpci_read_config_1(sc, off); 696 } 697 } 698 } 699 700 static void 701 vtpci_write_dev_config(device_t dev, bus_size_t offset, 702 void *src, int length) 703 { 704 struct vtpci_softc *sc; 705 bus_size_t off; 706 uint8_t *s; 707 int size; 708 709 sc = device_get_softc(dev); 710 off = VIRTIO_PCI_CONFIG(sc) + offset; 711 712 for (s = src; length > 0; s += size, off += size, length -= size) { 713 if (length >= 4) { 714 size = 4; 715 vtpci_write_config_4(sc, off, *(uint32_t *)s); 716 } else if (length >= 2) { 717 size = 2; 718 vtpci_write_config_2(sc, off, *(uint16_t *)s); 719 } else { 720 size = 1; 721 vtpci_write_config_1(sc, off, *s); 722 } 723 } 724 } 725 726 static void 727 vtpci_describe_features(struct vtpci_softc *sc, const char *msg, 728 uint64_t features) 729 { 730 device_t dev, child; 731 732 dev = sc->vtpci_dev; 733 child = sc->vtpci_child_dev; 734 735 if (device_is_attached(child) || bootverbose == 0) 736 return; 737 738 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); 739 } 740 741 static void 742 vtpci_probe_and_attach_child(struct vtpci_softc *sc) 743 { 744 device_t dev, child; 745 746 dev = sc->vtpci_dev; 747 child = sc->vtpci_child_dev; 748 749 if (child == NULL) 750 return; 751 752 if (device_get_state(child) != DS_NOTPRESENT) 753 return; 754 755 if (device_probe(child) != 0) 756 return; 757 758 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 759 if (device_attach(child) != 0) { 760 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 761 vtpci_reset(sc); 762 vtpci_release_child_resources(sc); 763 /* Reset status for future attempt. */ 764 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 765 } else { 766 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 767 VIRTIO_ATTACH_COMPLETED(child); 768 } 769 } 770 771 static int 772 vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) 773 { 774 device_t dev; 775 int nmsix, cnt, required; 776 777 dev = sc->vtpci_dev; 778 779 /* Allocate an additional vector for the config changes. */ 780 required = nvectors + 1; 781 782 nmsix = pci_msix_count(dev); 783 if (nmsix < required) 784 return (1); 785 786 cnt = required; 787 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 788 sc->vtpci_nmsix_resources = required; 789 return (0); 790 } 791 792 pci_release_msi(dev); 793 794 return (1); 795 } 796 797 static int 798 vtpci_alloc_msi(struct vtpci_softc *sc) 799 { 800 device_t dev; 801 int nmsi, cnt, required; 802 803 dev = sc->vtpci_dev; 804 required = 1; 805 806 nmsi = pci_msi_count(dev); 807 if (nmsi < required) 808 return (1); 809 810 cnt = required; 811 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) 812 return (0); 813 814 pci_release_msi(dev); 815 816 return (1); 817 } 818 819 static int 820 vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc) 821 { 822 int i, nvectors, error; 823 824 if (vtpci_disable_msix != 0 || 825 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 826 return (ENOTSUP); 827 828 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) { 829 if (sc->vtpci_vqs[i].vtv_no_intr == 0) 830 nvectors++; 831 } 832 833 error = vtpci_alloc_msix(sc, nvectors); 834 if (error) 835 return (error); 836 837 sc->vtpci_flags |= VTPCI_FLAG_MSIX; 838 839 return (0); 840 } 841 842 static int 843 vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc) 844 { 845 int error; 846 847 if (vtpci_disable_msix != 0 || 848 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 849 return (ENOTSUP); 850 851 error = vtpci_alloc_msix(sc, 1); 852 if (error) 853 return (error); 854 855 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; 856 857 return (0); 858 } 859 860 static int 861 vtpci_alloc_intr_msi(struct vtpci_softc *sc) 862 { 863 int error; 864 865 /* Only BHyVe supports MSI. */ 866 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI) 867 return (ENOTSUP); 868 869 error = vtpci_alloc_msi(sc); 870 if (error) 871 return (error); 872 873 sc->vtpci_flags |= VTPCI_FLAG_MSI; 874 875 return (0); 876 } 877 878 static int 879 vtpci_alloc_intr_legacy(struct vtpci_softc *sc) 880 { 881 882 sc->vtpci_flags |= VTPCI_FLAG_LEGACY; 883 884 return (0); 885 } 886 887 static int 888 vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, 889 struct vtpci_interrupt *intr) 890 { 891 struct resource *irq; 892 893 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags); 894 if (irq == NULL) 895 return (ENXIO); 896 897 intr->vti_irq = irq; 898 intr->vti_rid = rid; 899 900 return (0); 901 } 902 903 static int 904 vtpci_alloc_intr_resources(struct vtpci_softc *sc) 905 { 906 struct vtpci_interrupt *intr; 907 int i, rid, flags, nvq_intrs, error; 908 909 rid = 0; 910 flags = RF_ACTIVE; 911 912 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 913 flags |= RF_SHAREABLE; 914 else 915 rid = 1; 916 917 /* 918 * For legacy and MSI interrupts, this single resource handles all 919 * interrupts. For MSIX, this resource is used for the configuration 920 * changed interrupt. 921 */ 922 intr = &sc->vtpci_device_interrupt; 923 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 924 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI)) 925 return (error); 926 927 /* Subtract one for the configuration changed interrupt. */ 928 nvq_intrs = sc->vtpci_nmsix_resources - 1; 929 930 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs * 931 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); 932 if (sc->vtpci_msix_vq_interrupts == NULL) 933 return (ENOMEM); 934 935 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { 936 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 937 if (error) 938 return (error); 939 } 940 941 return (0); 942 } 943 944 static int 945 vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type) 946 { 947 struct vtpci_interrupt *intr; 948 int error; 949 950 intr = &sc->vtpci_device_interrupt; 951 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL, 952 vtpci_legacy_intr, sc, &intr->vti_handler); 953 954 return (error); 955 } 956 957 static int 958 vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 959 { 960 struct vtpci_virtqueue *vqx; 961 struct vtpci_interrupt *intr; 962 int i, error; 963 964 intr = sc->vtpci_msix_vq_interrupts; 965 966 for (i = 0; i < sc->vtpci_nvqs; i++) { 967 vqx = &sc->vtpci_vqs[i]; 968 969 if (vqx->vtv_no_intr) 970 continue; 971 972 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, 973 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, 974 &intr->vti_handler); 975 if (error) 976 return (error); 977 978 intr++; 979 } 980 981 return (0); 982 } 983 984 static int 985 vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 986 { 987 device_t dev; 988 struct vtpci_interrupt *intr; 989 int error; 990 991 dev = sc->vtpci_dev; 992 intr = &sc->vtpci_device_interrupt; 993 994 error = bus_setup_intr(dev, intr->vti_irq, type, NULL, 995 vtpci_config_intr, sc, &intr->vti_handler); 996 if (error) 997 return (error); 998 999 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { 1000 intr = sc->vtpci_msix_vq_interrupts; 1001 error = bus_setup_intr(dev, intr->vti_irq, type, 1002 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc, 1003 &intr->vti_handler); 1004 } else 1005 error = vtpci_setup_pervq_msix_interrupts(sc, type); 1006 1007 return (error ? error : vtpci_set_host_msix_vectors(sc)); 1008 } 1009 1010 static int 1011 vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type) 1012 { 1013 int error; 1014 1015 type |= INTR_MPSAFE; 1016 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, 1017 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags)); 1018 1019 error = vtpci_alloc_intr_resources(sc); 1020 if (error) 1021 return (error); 1022 1023 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 1024 error = vtpci_setup_legacy_interrupt(sc, type); 1025 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 1026 error = vtpci_setup_msi_interrupt(sc, type); 1027 else 1028 error = vtpci_setup_msix_interrupts(sc, type); 1029 1030 return (error); 1031 } 1032 1033 static int 1034 vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, 1035 struct vtpci_interrupt *intr) 1036 { 1037 device_t dev; 1038 uint16_t vector; 1039 1040 dev = sc->vtpci_dev; 1041 1042 if (intr != NULL) { 1043 /* Map from guest rid to host vector. */ 1044 vector = intr->vti_rid - 1; 1045 } else 1046 vector = VIRTIO_MSI_NO_VECTOR; 1047 1048 vtpci_write_config_2(sc, offset, vector); 1049 1050 /* Read vector to determine if the host had sufficient resources. */ 1051 if (vtpci_read_config_2(sc, offset) != vector) { 1052 device_printf(dev, 1053 "insufficient host resources for MSIX interrupts\n"); 1054 return (ENODEV); 1055 } 1056 1057 return (0); 1058 } 1059 1060 static int 1061 vtpci_set_host_msix_vectors(struct vtpci_softc *sc) 1062 { 1063 struct vtpci_interrupt *intr, *tintr; 1064 int idx, offset, error; 1065 1066 intr = &sc->vtpci_device_interrupt; 1067 offset = VIRTIO_MSI_CONFIG_VECTOR; 1068 1069 error = vtpci_register_msix_vector(sc, offset, intr); 1070 if (error) 1071 return (error); 1072 1073 intr = sc->vtpci_msix_vq_interrupts; 1074 offset = VIRTIO_MSI_QUEUE_VECTOR; 1075 1076 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1077 vtpci_select_virtqueue(sc, idx); 1078 1079 if (sc->vtpci_vqs[idx].vtv_no_intr) 1080 tintr = NULL; 1081 else 1082 tintr = intr; 1083 1084 error = vtpci_register_msix_vector(sc, offset, tintr); 1085 if (error) 1086 break; 1087 1088 /* 1089 * For shared MSIX, all the virtqueues share the first 1090 * interrupt. 1091 */ 1092 if (!sc->vtpci_vqs[idx].vtv_no_intr && 1093 (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) 1094 intr++; 1095 } 1096 1097 return (error); 1098 } 1099 1100 static int 1101 vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx) 1102 { 1103 struct vtpci_virtqueue *vqx; 1104 struct virtqueue *vq; 1105 int error; 1106 uint16_t size; 1107 1108 vqx = &sc->vtpci_vqs[idx]; 1109 vq = vqx->vtv_vq; 1110 1111 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 1112 1113 vtpci_select_virtqueue(sc, idx); 1114 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 1115 1116 error = virtqueue_reinit(vq, size); 1117 if (error) 1118 return (error); 1119 1120 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 1121 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 1122 1123 return (0); 1124 } 1125 1126 static void 1127 vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr) 1128 { 1129 device_t dev; 1130 1131 dev = sc->vtpci_dev; 1132 1133 if (intr->vti_handler != NULL) { 1134 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); 1135 intr->vti_handler = NULL; 1136 } 1137 1138 if (intr->vti_irq != NULL) { 1139 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, 1140 intr->vti_irq); 1141 intr->vti_irq = NULL; 1142 intr->vti_rid = -1; 1143 } 1144 } 1145 1146 static void 1147 vtpci_free_interrupts(struct vtpci_softc *sc) 1148 { 1149 struct vtpci_interrupt *intr; 1150 int i, nvq_intrs; 1151 1152 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt); 1153 1154 if (sc->vtpci_nmsix_resources != 0) { 1155 nvq_intrs = sc->vtpci_nmsix_resources - 1; 1156 sc->vtpci_nmsix_resources = 0; 1157 1158 intr = sc->vtpci_msix_vq_interrupts; 1159 if (intr != NULL) { 1160 for (i = 0; i < nvq_intrs; i++, intr++) 1161 vtpci_free_interrupt(sc, intr); 1162 1163 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF); 1164 sc->vtpci_msix_vq_interrupts = NULL; 1165 } 1166 } 1167 1168 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) 1169 pci_release_msi(sc->vtpci_dev); 1170 1171 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; 1172 } 1173 1174 static void 1175 vtpci_free_virtqueues(struct vtpci_softc *sc) 1176 { 1177 struct vtpci_virtqueue *vqx; 1178 int idx; 1179 1180 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1181 vqx = &sc->vtpci_vqs[idx]; 1182 1183 vtpci_select_virtqueue(sc, idx); 1184 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); 1185 1186 virtqueue_free(vqx->vtv_vq); 1187 vqx->vtv_vq = NULL; 1188 } 1189 1190 free(sc->vtpci_vqs, M_DEVBUF); 1191 sc->vtpci_vqs = NULL; 1192 sc->vtpci_nvqs = 0; 1193 } 1194 1195 static void 1196 vtpci_release_child_resources(struct vtpci_softc *sc) 1197 { 1198 1199 vtpci_free_interrupts(sc); 1200 vtpci_free_virtqueues(sc); 1201 } 1202 1203 static void 1204 vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc) 1205 { 1206 int idx; 1207 1208 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 1209 vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR, 1210 VIRTIO_MSI_NO_VECTOR); 1211 1212 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1213 vtpci_select_virtqueue(sc, idx); 1214 vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR, 1215 VIRTIO_MSI_NO_VECTOR); 1216 } 1217 } 1218 1219 vtpci_free_interrupts(sc); 1220 } 1221 1222 static void 1223 vtpci_reset(struct vtpci_softc *sc) 1224 { 1225 1226 /* 1227 * Setting the status to RESET sets the host device to 1228 * the original, uninitialized state. 1229 */ 1230 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); 1231 } 1232 1233 static void 1234 vtpci_select_virtqueue(struct vtpci_softc *sc, int idx) 1235 { 1236 1237 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); 1238 } 1239 1240 static void 1241 vtpci_legacy_intr(void *xsc) 1242 { 1243 struct vtpci_softc *sc; 1244 struct vtpci_virtqueue *vqx; 1245 int i; 1246 uint8_t isr; 1247 1248 sc = xsc; 1249 vqx = &sc->vtpci_vqs[0]; 1250 1251 /* Reading the ISR also clears it. */ 1252 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); 1253 1254 if (isr & VIRTIO_PCI_ISR_CONFIG) 1255 vtpci_config_intr(sc); 1256 1257 if (isr & VIRTIO_PCI_ISR_INTR) { 1258 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1259 if (vqx->vtv_no_intr == 0) 1260 virtqueue_intr(vqx->vtv_vq); 1261 } 1262 } 1263 } 1264 1265 static int 1266 vtpci_vq_shared_intr_filter(void *xsc) 1267 { 1268 struct vtpci_softc *sc; 1269 struct vtpci_virtqueue *vqx; 1270 int i, rc; 1271 1272 rc = 0; 1273 sc = xsc; 1274 vqx = &sc->vtpci_vqs[0]; 1275 1276 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1277 if (vqx->vtv_no_intr == 0) 1278 rc |= virtqueue_intr_filter(vqx->vtv_vq); 1279 } 1280 1281 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1282 } 1283 1284 static void 1285 vtpci_vq_shared_intr(void *xsc) 1286 { 1287 struct vtpci_softc *sc; 1288 struct vtpci_virtqueue *vqx; 1289 int i; 1290 1291 sc = xsc; 1292 vqx = &sc->vtpci_vqs[0]; 1293 1294 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1295 if (vqx->vtv_no_intr == 0) 1296 virtqueue_intr(vqx->vtv_vq); 1297 } 1298 } 1299 1300 static int 1301 vtpci_vq_intr_filter(void *xvq) 1302 { 1303 struct virtqueue *vq; 1304 int rc; 1305 1306 vq = xvq; 1307 rc = virtqueue_intr_filter(vq); 1308 1309 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1310 } 1311 1312 static void 1313 vtpci_vq_intr(void *xvq) 1314 { 1315 struct virtqueue *vq; 1316 1317 vq = xvq; 1318 virtqueue_intr(vq); 1319 } 1320 1321 static void 1322 vtpci_config_intr(void *xsc) 1323 { 1324 struct vtpci_softc *sc; 1325 device_t child; 1326 1327 sc = xsc; 1328 child = sc->vtpci_child_dev; 1329 1330 if (child != NULL) 1331 VIRTIO_CONFIG_CHANGE(child); 1332 } 1333