1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/systm.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/sbuf.h> 34 #include <sys/sysctl.h> 35 #include <sys/module.h> 36 #include <sys/malloc.h> 37 38 #include <machine/bus.h> 39 #include <machine/resource.h> 40 #include <sys/bus.h> 41 #include <sys/rman.h> 42 43 #include <dev/pci/pcivar.h> 44 #include <dev/pci/pcireg.h> 45 46 #include <dev/virtio/virtio.h> 47 #include <dev/virtio/virtqueue.h> 48 #include <dev/virtio/pci/virtio_pci.h> 49 #include <dev/virtio/pci/virtio_pci_var.h> 50 51 #include "virtio_pci_if.h" 52 #include "virtio_if.h" 53 54 static void vtpci_describe_features(struct vtpci_common *, const char *, 55 uint64_t); 56 static int vtpci_alloc_msix(struct vtpci_common *, int); 57 static int vtpci_alloc_msi(struct vtpci_common *); 58 static int vtpci_alloc_intr_msix_pervq(struct vtpci_common *); 59 static int vtpci_alloc_intr_msix_shared(struct vtpci_common *); 60 static int vtpci_alloc_intr_msi(struct vtpci_common *); 61 static int vtpci_alloc_intr_intx(struct vtpci_common *); 62 static int vtpci_alloc_interrupt(struct vtpci_common *, int, int, 63 struct vtpci_interrupt *); 64 static void vtpci_free_interrupt(struct vtpci_common *, 65 struct vtpci_interrupt *); 66 67 static void vtpci_free_interrupts(struct vtpci_common *); 68 static void vtpci_free_virtqueues(struct vtpci_common *); 69 static void vtpci_cleanup_setup_intr_attempt(struct vtpci_common *); 70 static int vtpci_alloc_intr_resources(struct vtpci_common *); 71 static int vtpci_setup_intx_interrupt(struct vtpci_common *, 72 enum intr_type); 73 static int vtpci_setup_pervq_msix_interrupts(struct vtpci_common *, 74 enum intr_type); 75 static int vtpci_set_host_msix_vectors(struct vtpci_common *); 76 static int vtpci_setup_msix_interrupts(struct vtpci_common *, 77 enum intr_type); 78 static int vtpci_setup_intrs(struct vtpci_common *, enum intr_type); 79 static int vtpci_reinit_virtqueue(struct vtpci_common *, int); 80 static void vtpci_intx_intr(void *); 81 static int vtpci_vq_shared_intr_filter(void *); 82 static void vtpci_vq_shared_intr(void *); 83 static int vtpci_vq_intr_filter(void *); 84 static void vtpci_vq_intr(void *); 85 static void vtpci_config_intr(void *); 86 87 static void vtpci_setup_sysctl(struct vtpci_common *); 88 89 #define vtpci_setup_msi_interrupt vtpci_setup_intx_interrupt 90 91 /* 92 * This module contains two drivers: 93 * - virtio_pci_legacy for pre-V1 support 94 * - virtio_pci_modern for V1 support 95 */ 96 MODULE_VERSION(virtio_pci, 1); 97 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); 98 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); 99 100 int vtpci_disable_msix = 0; 101 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); 102 103 static uint8_t 104 vtpci_read_isr(struct vtpci_common *cn) 105 { 106 return (VIRTIO_PCI_READ_ISR(cn->vtpci_dev)); 107 } 108 109 static uint16_t 110 vtpci_get_vq_size(struct vtpci_common *cn, int idx) 111 { 112 return (VIRTIO_PCI_GET_VQ_SIZE(cn->vtpci_dev, idx)); 113 } 114 115 static bus_size_t 116 vtpci_get_vq_notify_off(struct vtpci_common *cn, int idx) 117 { 118 return (VIRTIO_PCI_GET_VQ_NOTIFY_OFF(cn->vtpci_dev, idx)); 119 } 120 121 static void 122 vtpci_set_vq(struct vtpci_common *cn, struct virtqueue *vq) 123 { 124 VIRTIO_PCI_SET_VQ(cn->vtpci_dev, vq); 125 } 126 127 static void 128 vtpci_disable_vq(struct vtpci_common *cn, int idx) 129 { 130 VIRTIO_PCI_DISABLE_VQ(cn->vtpci_dev, idx); 131 } 132 133 static int 134 vtpci_register_cfg_msix(struct vtpci_common *cn, struct vtpci_interrupt *intr) 135 { 136 return (VIRTIO_PCI_REGISTER_CFG_MSIX(cn->vtpci_dev, intr)); 137 } 138 139 static int 140 vtpci_register_vq_msix(struct vtpci_common *cn, int idx, 141 struct vtpci_interrupt *intr) 142 { 143 return (VIRTIO_PCI_REGISTER_VQ_MSIX(cn->vtpci_dev, idx, intr)); 144 } 145 146 void 147 vtpci_init(struct vtpci_common *cn, device_t dev, bool modern) 148 { 149 150 cn->vtpci_dev = dev; 151 152 pci_enable_busmaster(dev); 153 154 if (modern) 155 cn->vtpci_flags |= VTPCI_FLAG_MODERN; 156 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) 157 cn->vtpci_flags |= VTPCI_FLAG_NO_MSI; 158 if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) 159 cn->vtpci_flags |= VTPCI_FLAG_NO_MSIX; 160 161 vtpci_setup_sysctl(cn); 162 } 163 164 int 165 vtpci_add_child(struct vtpci_common *cn) 166 { 167 device_t dev, child; 168 169 dev = cn->vtpci_dev; 170 171 child = device_add_child(dev, NULL, DEVICE_UNIT_ANY); 172 if (child == NULL) { 173 device_printf(dev, "cannot create child device\n"); 174 return (ENOMEM); 175 } 176 177 cn->vtpci_child_dev = child; 178 179 return (0); 180 } 181 182 int 183 vtpci_delete_child(struct vtpci_common *cn) 184 { 185 device_t dev; 186 int error; 187 188 dev = cn->vtpci_dev; 189 190 error = bus_generic_detach(dev); 191 if (error) 192 return (error); 193 194 return (0); 195 } 196 197 void 198 vtpci_child_detached(struct vtpci_common *cn) 199 { 200 201 vtpci_release_child_resources(cn); 202 203 cn->vtpci_child_feat_desc = NULL; 204 cn->vtpci_host_features = 0; 205 cn->vtpci_features = 0; 206 } 207 208 int 209 vtpci_reinit(struct vtpci_common *cn) 210 { 211 int idx, error; 212 213 for (idx = 0; idx < cn->vtpci_nvqs; idx++) { 214 error = vtpci_reinit_virtqueue(cn, idx); 215 if (error) 216 return (error); 217 } 218 219 if (vtpci_is_msix_enabled(cn)) { 220 error = vtpci_set_host_msix_vectors(cn); 221 if (error) 222 return (error); 223 } 224 225 return (0); 226 } 227 228 static void 229 vtpci_describe_features(struct vtpci_common *cn, const char *msg, 230 uint64_t features) 231 { 232 device_t dev, child; 233 234 dev = cn->vtpci_dev; 235 child = cn->vtpci_child_dev; 236 237 if (device_is_attached(child) || bootverbose == 0) 238 return; 239 240 virtio_describe(dev, msg, features, cn->vtpci_child_feat_desc); 241 } 242 243 uint64_t 244 vtpci_negotiate_features(struct vtpci_common *cn, 245 uint64_t child_features, uint64_t host_features) 246 { 247 uint64_t features; 248 249 cn->vtpci_host_features = host_features; 250 vtpci_describe_features(cn, "host", host_features); 251 252 /* 253 * Limit negotiated features to what the driver, virtqueue, and 254 * host all support. 255 */ 256 features = host_features & child_features; 257 features = virtio_filter_transport_features(features); 258 259 cn->vtpci_features = features; 260 vtpci_describe_features(cn, "negotiated", features); 261 262 return (features); 263 } 264 265 bool 266 vtpci_with_feature(struct vtpci_common *cn, uint64_t feature) 267 { 268 return ((cn->vtpci_features & feature) != 0); 269 } 270 271 int 272 vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result) 273 { 274 device_t dev; 275 int error; 276 277 dev = cn->vtpci_dev; 278 error = 0; 279 280 switch (index) { 281 case VIRTIO_IVAR_SUBDEVICE: 282 *result = pci_get_subdevice(dev); 283 break; 284 case VIRTIO_IVAR_VENDOR: 285 *result = pci_get_vendor(dev); 286 break; 287 case VIRTIO_IVAR_DEVICE: 288 *result = pci_get_device(dev); 289 break; 290 case VIRTIO_IVAR_SUBVENDOR: 291 *result = pci_get_subvendor(dev); 292 break; 293 case VIRTIO_IVAR_MODERN: 294 *result = vtpci_is_modern(cn); 295 break; 296 default: 297 error = ENOENT; 298 } 299 300 return (error); 301 } 302 303 int 304 vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value) 305 { 306 int error; 307 308 error = 0; 309 310 switch (index) { 311 case VIRTIO_IVAR_FEATURE_DESC: 312 cn->vtpci_child_feat_desc = (void *) value; 313 break; 314 default: 315 error = ENOENT; 316 } 317 318 return (error); 319 } 320 321 int 322 vtpci_alloc_virtqueues(struct vtpci_common *cn, int nvqs, 323 struct vq_alloc_info *vq_info) 324 { 325 device_t dev; 326 int idx, align, error; 327 328 dev = cn->vtpci_dev; 329 330 /* 331 * This is VIRTIO_PCI_VRING_ALIGN from legacy VirtIO. In modern VirtIO, 332 * the tables do not have to be allocated contiguously, but we do so 333 * anyways. 334 */ 335 align = 4096; 336 337 if (cn->vtpci_nvqs != 0) 338 return (EALREADY); 339 if (nvqs <= 0) 340 return (EINVAL); 341 342 cn->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), 343 M_DEVBUF, M_NOWAIT | M_ZERO); 344 if (cn->vtpci_vqs == NULL) 345 return (ENOMEM); 346 347 for (idx = 0; idx < nvqs; idx++) { 348 struct vtpci_virtqueue *vqx; 349 struct vq_alloc_info *info; 350 struct virtqueue *vq; 351 bus_size_t notify_offset; 352 uint16_t size; 353 354 vqx = &cn->vtpci_vqs[idx]; 355 info = &vq_info[idx]; 356 357 size = vtpci_get_vq_size(cn, idx); 358 notify_offset = vtpci_get_vq_notify_off(cn, idx); 359 360 error = virtqueue_alloc(dev, idx, size, notify_offset, align, 361 ~(vm_paddr_t)0, info, &vq); 362 if (error) { 363 device_printf(dev, 364 "cannot allocate virtqueue %d: %d\n", idx, error); 365 break; 366 } 367 368 vtpci_set_vq(cn, vq); 369 370 vqx->vtv_vq = *info->vqai_vq = vq; 371 vqx->vtv_no_intr = info->vqai_intr == NULL; 372 373 cn->vtpci_nvqs++; 374 } 375 376 if (error) 377 vtpci_free_virtqueues(cn); 378 379 return (error); 380 } 381 382 static int 383 vtpci_alloc_msix(struct vtpci_common *cn, int nvectors) 384 { 385 device_t dev; 386 int nmsix, cnt, required; 387 388 dev = cn->vtpci_dev; 389 390 /* Allocate an additional vector for the config changes. */ 391 required = nvectors + 1; 392 393 nmsix = pci_msix_count(dev); 394 if (nmsix < required) 395 return (1); 396 397 cnt = required; 398 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 399 cn->vtpci_nmsix_resources = required; 400 return (0); 401 } 402 403 pci_release_msi(dev); 404 405 return (1); 406 } 407 408 static int 409 vtpci_alloc_msi(struct vtpci_common *cn) 410 { 411 device_t dev; 412 int nmsi, cnt, required; 413 414 dev = cn->vtpci_dev; 415 required = 1; 416 417 nmsi = pci_msi_count(dev); 418 if (nmsi < required) 419 return (1); 420 421 cnt = required; 422 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) 423 return (0); 424 425 pci_release_msi(dev); 426 427 return (1); 428 } 429 430 static int 431 vtpci_alloc_intr_msix_pervq(struct vtpci_common *cn) 432 { 433 int i, nvectors, error; 434 435 if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) 436 return (ENOTSUP); 437 438 for (nvectors = 0, i = 0; i < cn->vtpci_nvqs; i++) { 439 if (cn->vtpci_vqs[i].vtv_no_intr == 0) 440 nvectors++; 441 } 442 443 error = vtpci_alloc_msix(cn, nvectors); 444 if (error) 445 return (error); 446 447 cn->vtpci_flags |= VTPCI_FLAG_MSIX; 448 449 return (0); 450 } 451 452 static int 453 vtpci_alloc_intr_msix_shared(struct vtpci_common *cn) 454 { 455 int error; 456 457 if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) 458 return (ENOTSUP); 459 460 error = vtpci_alloc_msix(cn, 1); 461 if (error) 462 return (error); 463 464 cn->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; 465 466 return (0); 467 } 468 469 static int 470 vtpci_alloc_intr_msi(struct vtpci_common *cn) 471 { 472 int error; 473 474 /* Only BHyVe supports MSI. */ 475 if (cn->vtpci_flags & VTPCI_FLAG_NO_MSI) 476 return (ENOTSUP); 477 478 error = vtpci_alloc_msi(cn); 479 if (error) 480 return (error); 481 482 cn->vtpci_flags |= VTPCI_FLAG_MSI; 483 484 return (0); 485 } 486 487 static int 488 vtpci_alloc_intr_intx(struct vtpci_common *cn) 489 { 490 491 cn->vtpci_flags |= VTPCI_FLAG_INTX; 492 493 return (0); 494 } 495 496 static int 497 vtpci_alloc_interrupt(struct vtpci_common *cn, int rid, int flags, 498 struct vtpci_interrupt *intr) 499 { 500 struct resource *irq; 501 502 irq = bus_alloc_resource_any(cn->vtpci_dev, SYS_RES_IRQ, &rid, flags); 503 if (irq == NULL) 504 return (ENXIO); 505 506 intr->vti_irq = irq; 507 intr->vti_rid = rid; 508 509 return (0); 510 } 511 512 static void 513 vtpci_free_interrupt(struct vtpci_common *cn, struct vtpci_interrupt *intr) 514 { 515 device_t dev; 516 517 dev = cn->vtpci_dev; 518 519 if (intr->vti_handler != NULL) { 520 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); 521 intr->vti_handler = NULL; 522 } 523 524 if (intr->vti_irq != NULL) { 525 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, 526 intr->vti_irq); 527 intr->vti_irq = NULL; 528 intr->vti_rid = -1; 529 } 530 } 531 532 static void 533 vtpci_free_interrupts(struct vtpci_common *cn) 534 { 535 struct vtpci_interrupt *intr; 536 int i, nvq_intrs; 537 538 vtpci_free_interrupt(cn, &cn->vtpci_device_interrupt); 539 540 if (cn->vtpci_nmsix_resources != 0) { 541 nvq_intrs = cn->vtpci_nmsix_resources - 1; 542 cn->vtpci_nmsix_resources = 0; 543 544 if ((intr = cn->vtpci_msix_vq_interrupts) != NULL) { 545 for (i = 0; i < nvq_intrs; i++, intr++) 546 vtpci_free_interrupt(cn, intr); 547 548 free(cn->vtpci_msix_vq_interrupts, M_DEVBUF); 549 cn->vtpci_msix_vq_interrupts = NULL; 550 } 551 } 552 553 if (cn->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) 554 pci_release_msi(cn->vtpci_dev); 555 556 cn->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; 557 } 558 559 static void 560 vtpci_free_virtqueues(struct vtpci_common *cn) 561 { 562 struct vtpci_virtqueue *vqx; 563 int idx; 564 565 for (idx = 0; idx < cn->vtpci_nvqs; idx++) { 566 vtpci_disable_vq(cn, idx); 567 568 vqx = &cn->vtpci_vqs[idx]; 569 virtqueue_free(vqx->vtv_vq); 570 vqx->vtv_vq = NULL; 571 } 572 573 free(cn->vtpci_vqs, M_DEVBUF); 574 cn->vtpci_vqs = NULL; 575 cn->vtpci_nvqs = 0; 576 } 577 578 void 579 vtpci_release_child_resources(struct vtpci_common *cn) 580 { 581 582 vtpci_free_interrupts(cn); 583 vtpci_free_virtqueues(cn); 584 } 585 586 static void 587 vtpci_cleanup_setup_intr_attempt(struct vtpci_common *cn) 588 { 589 int idx; 590 591 if (cn->vtpci_flags & VTPCI_FLAG_MSIX) { 592 vtpci_register_cfg_msix(cn, NULL); 593 594 for (idx = 0; idx < cn->vtpci_nvqs; idx++) 595 vtpci_register_vq_msix(cn, idx, NULL); 596 } 597 598 vtpci_free_interrupts(cn); 599 } 600 601 static int 602 vtpci_alloc_intr_resources(struct vtpci_common *cn) 603 { 604 struct vtpci_interrupt *intr; 605 int i, rid, flags, nvq_intrs, error; 606 607 flags = RF_ACTIVE; 608 609 if (cn->vtpci_flags & VTPCI_FLAG_INTX) { 610 rid = 0; 611 flags |= RF_SHAREABLE; 612 } else 613 rid = 1; 614 615 /* 616 * When using INTX or MSI interrupts, this resource handles all 617 * interrupts. When using MSIX, this resource handles just the 618 * configuration changed interrupt. 619 */ 620 intr = &cn->vtpci_device_interrupt; 621 622 error = vtpci_alloc_interrupt(cn, rid, flags, intr); 623 if (error || cn->vtpci_flags & (VTPCI_FLAG_INTX | VTPCI_FLAG_MSI)) 624 return (error); 625 626 /* 627 * Now allocate the interrupts for the virtqueues. This may be one 628 * for all the virtqueues, or one for each virtqueue. Subtract one 629 * below for because of the configuration changed interrupt. 630 */ 631 nvq_intrs = cn->vtpci_nmsix_resources - 1; 632 633 cn->vtpci_msix_vq_interrupts = malloc(nvq_intrs * 634 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); 635 if (cn->vtpci_msix_vq_interrupts == NULL) 636 return (ENOMEM); 637 638 intr = cn->vtpci_msix_vq_interrupts; 639 640 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { 641 error = vtpci_alloc_interrupt(cn, rid, flags, intr); 642 if (error) 643 return (error); 644 } 645 646 return (0); 647 } 648 649 static int 650 vtpci_setup_intx_interrupt(struct vtpci_common *cn, enum intr_type type) 651 { 652 struct vtpci_interrupt *intr; 653 int error; 654 655 intr = &cn->vtpci_device_interrupt; 656 657 error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, 658 vtpci_intx_intr, cn, &intr->vti_handler); 659 660 return (error); 661 } 662 663 static int 664 vtpci_setup_pervq_msix_interrupts(struct vtpci_common *cn, enum intr_type type) 665 { 666 struct vtpci_virtqueue *vqx; 667 struct vtpci_interrupt *intr; 668 int i, error; 669 670 intr = cn->vtpci_msix_vq_interrupts; 671 672 for (i = 0; i < cn->vtpci_nvqs; i++) { 673 vqx = &cn->vtpci_vqs[i]; 674 675 if (vqx->vtv_no_intr) 676 continue; 677 678 error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, 679 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, 680 &intr->vti_handler); 681 if (error) 682 return (error); 683 684 intr++; 685 } 686 687 return (0); 688 } 689 690 static int 691 vtpci_set_host_msix_vectors(struct vtpci_common *cn) 692 { 693 struct vtpci_interrupt *intr, *tintr; 694 int idx, error; 695 696 intr = &cn->vtpci_device_interrupt; 697 error = vtpci_register_cfg_msix(cn, intr); 698 if (error) 699 return (error); 700 701 intr = cn->vtpci_msix_vq_interrupts; 702 for (idx = 0; idx < cn->vtpci_nvqs; idx++) { 703 if (cn->vtpci_vqs[idx].vtv_no_intr) 704 tintr = NULL; 705 else 706 tintr = intr; 707 708 error = vtpci_register_vq_msix(cn, idx, tintr); 709 if (error) 710 break; 711 712 /* 713 * For shared MSIX, all the virtqueues share the first 714 * interrupt. 715 */ 716 if (!cn->vtpci_vqs[idx].vtv_no_intr && 717 (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) 718 intr++; 719 } 720 721 return (error); 722 } 723 724 static int 725 vtpci_setup_msix_interrupts(struct vtpci_common *cn, enum intr_type type) 726 { 727 struct vtpci_interrupt *intr; 728 int error; 729 730 intr = &cn->vtpci_device_interrupt; 731 732 error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, 733 vtpci_config_intr, cn, &intr->vti_handler); 734 if (error) 735 return (error); 736 737 if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { 738 intr = &cn->vtpci_msix_vq_interrupts[0]; 739 740 error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, 741 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, cn, 742 &intr->vti_handler); 743 } else 744 error = vtpci_setup_pervq_msix_interrupts(cn, type); 745 746 return (error ? error : vtpci_set_host_msix_vectors(cn)); 747 } 748 749 static int 750 vtpci_setup_intrs(struct vtpci_common *cn, enum intr_type type) 751 { 752 int error; 753 754 type |= INTR_MPSAFE; 755 KASSERT(cn->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, 756 ("%s: no interrupt type selected %#x", __func__, cn->vtpci_flags)); 757 758 error = vtpci_alloc_intr_resources(cn); 759 if (error) 760 return (error); 761 762 if (cn->vtpci_flags & VTPCI_FLAG_INTX) 763 error = vtpci_setup_intx_interrupt(cn, type); 764 else if (cn->vtpci_flags & VTPCI_FLAG_MSI) 765 error = vtpci_setup_msi_interrupt(cn, type); 766 else 767 error = vtpci_setup_msix_interrupts(cn, type); 768 769 return (error); 770 } 771 772 int 773 vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type) 774 { 775 device_t dev; 776 int attempt, error; 777 778 dev = cn->vtpci_dev; 779 780 for (attempt = 0; attempt < 5; attempt++) { 781 /* 782 * Start with the most desirable interrupt configuration and 783 * fallback towards less desirable ones. 784 */ 785 switch (attempt) { 786 case 0: 787 error = vtpci_alloc_intr_msix_pervq(cn); 788 break; 789 case 1: 790 error = vtpci_alloc_intr_msix_shared(cn); 791 break; 792 case 2: 793 error = vtpci_alloc_intr_msi(cn); 794 break; 795 case 3: 796 error = vtpci_alloc_intr_intx(cn); 797 break; 798 default: 799 device_printf(dev, 800 "exhausted all interrupt allocation attempts\n"); 801 return (ENXIO); 802 } 803 804 if (error == 0 && vtpci_setup_intrs(cn, type) == 0) 805 break; 806 807 vtpci_cleanup_setup_intr_attempt(cn); 808 } 809 810 if (bootverbose) { 811 if (cn->vtpci_flags & VTPCI_FLAG_INTX) 812 device_printf(dev, "using legacy interrupt\n"); 813 else if (cn->vtpci_flags & VTPCI_FLAG_MSI) 814 device_printf(dev, "using MSI interrupt\n"); 815 else if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) 816 device_printf(dev, "using shared MSIX interrupts\n"); 817 else 818 device_printf(dev, "using per VQ MSIX interrupts\n"); 819 } 820 821 return (0); 822 } 823 824 static int 825 vtpci_reinit_virtqueue(struct vtpci_common *cn, int idx) 826 { 827 struct vtpci_virtqueue *vqx; 828 struct virtqueue *vq; 829 int error; 830 831 vqx = &cn->vtpci_vqs[idx]; 832 vq = vqx->vtv_vq; 833 834 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 835 836 error = virtqueue_reinit(vq, vtpci_get_vq_size(cn, idx)); 837 if (error == 0) 838 vtpci_set_vq(cn, vq); 839 840 return (error); 841 } 842 843 static void 844 vtpci_intx_intr(void *xcn) 845 { 846 struct vtpci_common *cn; 847 struct vtpci_virtqueue *vqx; 848 int i; 849 uint8_t isr; 850 851 cn = xcn; 852 isr = vtpci_read_isr(cn); 853 854 if (isr & VIRTIO_PCI_ISR_CONFIG) 855 vtpci_config_intr(cn); 856 857 if (isr & VIRTIO_PCI_ISR_INTR) { 858 vqx = &cn->vtpci_vqs[0]; 859 for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { 860 if (vqx->vtv_no_intr == 0) 861 virtqueue_intr(vqx->vtv_vq); 862 } 863 } 864 } 865 866 static int 867 vtpci_vq_shared_intr_filter(void *xcn) 868 { 869 struct vtpci_common *cn; 870 struct vtpci_virtqueue *vqx; 871 int i, rc; 872 873 cn = xcn; 874 vqx = &cn->vtpci_vqs[0]; 875 rc = 0; 876 877 for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { 878 if (vqx->vtv_no_intr == 0) 879 rc |= virtqueue_intr_filter(vqx->vtv_vq); 880 } 881 882 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 883 } 884 885 static void 886 vtpci_vq_shared_intr(void *xcn) 887 { 888 struct vtpci_common *cn; 889 struct vtpci_virtqueue *vqx; 890 int i; 891 892 cn = xcn; 893 vqx = &cn->vtpci_vqs[0]; 894 895 for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { 896 if (vqx->vtv_no_intr == 0) 897 virtqueue_intr(vqx->vtv_vq); 898 } 899 } 900 901 static int 902 vtpci_vq_intr_filter(void *xvq) 903 { 904 struct virtqueue *vq; 905 int rc; 906 907 vq = xvq; 908 rc = virtqueue_intr_filter(vq); 909 910 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 911 } 912 913 static void 914 vtpci_vq_intr(void *xvq) 915 { 916 struct virtqueue *vq; 917 918 vq = xvq; 919 virtqueue_intr(vq); 920 } 921 922 static void 923 vtpci_config_intr(void *xcn) 924 { 925 struct vtpci_common *cn; 926 device_t child; 927 928 cn = xcn; 929 child = cn->vtpci_child_dev; 930 931 if (child != NULL) 932 VIRTIO_CONFIG_CHANGE(child); 933 } 934 935 static int 936 vtpci_feature_sysctl(struct sysctl_req *req, struct vtpci_common *cn, 937 uint64_t features) 938 { 939 struct sbuf *sb; 940 int error; 941 942 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 943 if (sb == NULL) 944 return (ENOMEM); 945 946 error = virtio_describe_sbuf(sb, features, cn->vtpci_child_feat_desc); 947 sbuf_delete(sb); 948 949 return (error); 950 } 951 952 static int 953 vtpci_host_features_sysctl(SYSCTL_HANDLER_ARGS) 954 { 955 struct vtpci_common *cn; 956 957 cn = arg1; 958 959 return (vtpci_feature_sysctl(req, cn, cn->vtpci_host_features)); 960 } 961 962 static int 963 vtpci_negotiated_features_sysctl(SYSCTL_HANDLER_ARGS) 964 { 965 struct vtpci_common *cn; 966 967 cn = arg1; 968 969 return (vtpci_feature_sysctl(req, cn, cn->vtpci_features)); 970 } 971 972 static void 973 vtpci_setup_sysctl(struct vtpci_common *cn) 974 { 975 device_t dev; 976 struct sysctl_ctx_list *ctx; 977 struct sysctl_oid *tree; 978 struct sysctl_oid_list *child; 979 980 dev = cn->vtpci_dev; 981 ctx = device_get_sysctl_ctx(dev); 982 tree = device_get_sysctl_tree(dev); 983 child = SYSCTL_CHILDREN(tree); 984 985 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nvqs", 986 CTLFLAG_RD, &cn->vtpci_nvqs, 0, "Number of virtqueues"); 987 988 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "host_features", 989 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0, 990 vtpci_host_features_sysctl, "A", "Features supported by the host"); 991 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "negotiated_features", 992 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0, 993 vtpci_negotiated_features_sysctl, "A", "Features negotiated"); 994 } 995