1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Portions of this software were developed by Andrew Turner 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * VirtIO MMIO interface. 37 * This driver is heavily based on VirtIO PCI interface driver. 38 */ 39 40 /* 41 * FDT example: 42 * virtio_block@1000 { 43 * compatible = "virtio,mmio"; 44 * reg = <0x1000 0x100>; 45 * interrupts = <63>; 46 * interrupt-parent = <&GIC>; 47 * }; 48 */ 49 50 #include <sys/cdefs.h> 51 __FBSDID("$FreeBSD$"); 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/bus.h> 56 #include <sys/kernel.h> 57 #include <sys/module.h> 58 #include <sys/malloc.h> 59 #include <sys/rman.h> 60 61 #include <machine/bus.h> 62 #include <machine/resource.h> 63 64 #include <dev/fdt/fdt_common.h> 65 #include <dev/ofw/openfirm.h> 66 #include <dev/ofw/ofw_bus.h> 67 #include <dev/ofw/ofw_bus_subr.h> 68 69 #include <dev/virtio/virtio.h> 70 #include <dev/virtio/virtqueue.h> 71 #include <dev/virtio/mmio/virtio_mmio.h> 72 73 #include "virtio_mmio_if.h" 74 #include "virtio_bus_if.h" 75 #include "virtio_if.h" 76 77 #define PAGE_SHIFT 12 78 79 struct vtmmio_virtqueue { 80 struct virtqueue *vtv_vq; 81 int vtv_no_intr; 82 }; 83 84 struct vtmmio_softc { 85 device_t dev; 86 device_t platform; 87 struct resource *res[2]; 88 89 uint64_t vtmmio_features; 90 uint32_t vtmmio_flags; 91 92 /* This "bus" will only ever have one child. */ 93 device_t vtmmio_child_dev; 94 struct virtio_feature_desc *vtmmio_child_feat_desc; 95 96 int vtmmio_nvqs; 97 struct vtmmio_virtqueue *vtmmio_vqs; 98 void *ih; 99 }; 100 101 static int vtmmio_probe(device_t); 102 static int vtmmio_attach(device_t); 103 static int vtmmio_detach(device_t); 104 static int vtmmio_suspend(device_t); 105 static int vtmmio_resume(device_t); 106 static int vtmmio_shutdown(device_t); 107 static void vtmmio_driver_added(device_t, driver_t *); 108 static void vtmmio_child_detached(device_t, device_t); 109 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *); 110 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t); 111 static uint64_t vtmmio_negotiate_features(device_t, uint64_t); 112 static int vtmmio_with_feature(device_t, uint64_t); 113 static int vtmmio_alloc_virtqueues(device_t, int, int, 114 struct vq_alloc_info *); 115 static int vtmmio_setup_intr(device_t, enum intr_type); 116 static void vtmmio_stop(device_t); 117 static void vtmmio_poll(device_t); 118 static int vtmmio_reinit(device_t, uint64_t); 119 static void vtmmio_reinit_complete(device_t); 120 static void vtmmio_notify_virtqueue(device_t, uint16_t); 121 static uint8_t vtmmio_get_status(device_t); 122 static void vtmmio_set_status(device_t, uint8_t); 123 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); 124 static void vtmmio_write_dev_config(device_t, bus_size_t, void *, int); 125 static void vtmmio_describe_features(struct vtmmio_softc *, const char *, 126 uint64_t); 127 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *); 128 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int); 129 static void vtmmio_free_interrupts(struct vtmmio_softc *); 130 static void vtmmio_free_virtqueues(struct vtmmio_softc *); 131 static void vtmmio_release_child_resources(struct vtmmio_softc *); 132 static void vtmmio_reset(struct vtmmio_softc *); 133 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int); 134 static void vtmmio_vq_intr(void *); 135 136 /* 137 * I/O port read/write wrappers. 138 */ 139 #define vtmmio_write_config_1(sc, o, v) \ 140 do { \ 141 if (sc->platform != NULL) \ 142 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 143 bus_write_1((sc)->res[0], (o), (v)); \ 144 if (sc->platform != NULL) \ 145 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 146 } while (0) 147 #define vtmmio_write_config_2(sc, o, v) \ 148 do { \ 149 if (sc->platform != NULL) \ 150 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 151 bus_write_2((sc)->res[0], (o), (v)); \ 152 if (sc->platform != NULL) \ 153 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 154 } while (0) 155 #define vtmmio_write_config_4(sc, o, v) \ 156 do { \ 157 if (sc->platform != NULL) \ 158 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 159 bus_write_4((sc)->res[0], (o), (v)); \ 160 if (sc->platform != NULL) \ 161 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 162 } while (0) 163 164 #define vtmmio_read_config_1(sc, o) \ 165 bus_read_1((sc)->res[0], (o)) 166 #define vtmmio_read_config_2(sc, o) \ 167 bus_read_2((sc)->res[0], (o)) 168 #define vtmmio_read_config_4(sc, o) \ 169 bus_read_4((sc)->res[0], (o)) 170 171 static device_method_t vtmmio_methods[] = { 172 /* Device interface. */ 173 DEVMETHOD(device_probe, vtmmio_probe), 174 DEVMETHOD(device_attach, vtmmio_attach), 175 DEVMETHOD(device_detach, vtmmio_detach), 176 DEVMETHOD(device_suspend, vtmmio_suspend), 177 DEVMETHOD(device_resume, vtmmio_resume), 178 DEVMETHOD(device_shutdown, vtmmio_shutdown), 179 180 /* Bus interface. */ 181 DEVMETHOD(bus_driver_added, vtmmio_driver_added), 182 DEVMETHOD(bus_child_detached, vtmmio_child_detached), 183 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar), 184 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar), 185 186 /* VirtIO bus interface. */ 187 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features), 188 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature), 189 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), 190 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), 191 DEVMETHOD(virtio_bus_stop, vtmmio_stop), 192 DEVMETHOD(virtio_bus_poll, vtmmio_poll), 193 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), 194 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), 195 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), 196 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config), 197 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config), 198 199 DEVMETHOD_END 200 }; 201 202 static driver_t vtmmio_driver = { 203 "virtio_mmio", 204 vtmmio_methods, 205 sizeof(struct vtmmio_softc) 206 }; 207 208 devclass_t vtmmio_devclass; 209 210 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0); 211 DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0); 212 MODULE_VERSION(virtio_mmio, 1); 213 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1); 214 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1); 215 216 static int 217 vtmmio_setup_intr(device_t dev, enum intr_type type) 218 { 219 struct vtmmio_softc *sc; 220 int rid; 221 int err; 222 223 sc = device_get_softc(dev); 224 225 if (sc->platform != NULL) { 226 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, 227 vtmmio_vq_intr, sc); 228 if (err == 0) { 229 /* Okay we have backend-specific interrupts */ 230 return (0); 231 } 232 } 233 234 rid = 0; 235 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 236 RF_ACTIVE); 237 if (!sc->res[1]) { 238 device_printf(dev, "Can't allocate interrupt\n"); 239 return (ENXIO); 240 } 241 242 if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, 243 NULL, vtmmio_vq_intr, sc, &sc->ih)) { 244 device_printf(dev, "Can't setup the interrupt\n"); 245 return (ENXIO); 246 } 247 248 return (0); 249 } 250 251 static int 252 vtmmio_probe(device_t dev) 253 { 254 255 if (!ofw_bus_status_okay(dev)) 256 return (ENXIO); 257 258 if (!ofw_bus_is_compatible(dev, "virtio,mmio")) 259 return (ENXIO); 260 261 device_set_desc(dev, "VirtIO MMIO adapter"); 262 return (BUS_PROBE_DEFAULT); 263 } 264 265 static int 266 vtmmio_setup_platform(struct vtmmio_softc *sc) 267 { 268 phandle_t platform_node; 269 struct fdt_ic *ic; 270 phandle_t xref; 271 phandle_t node; 272 273 sc->platform = NULL; 274 275 if ((node = ofw_bus_get_node(sc->dev)) == -1) 276 return (ENXIO); 277 278 if (OF_searchencprop(node, "platform", &xref, 279 sizeof(xref)) == -1) { 280 return (ENXIO); 281 } 282 283 platform_node = OF_node_from_xref(xref); 284 285 SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) { 286 if (ic->iph == platform_node) { 287 sc->platform = ic->dev; 288 break; 289 } 290 } 291 292 if (sc->platform == NULL) { 293 /* No platform-specific device. Ignore it. */ 294 } 295 296 return (0); 297 } 298 299 static int 300 vtmmio_attach(device_t dev) 301 { 302 struct vtmmio_softc *sc; 303 device_t child; 304 int rid; 305 306 sc = device_get_softc(dev); 307 sc->dev = dev; 308 309 vtmmio_setup_platform(sc); 310 311 rid = 0; 312 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 313 RF_ACTIVE); 314 if (!sc->res[0]) { 315 device_printf(dev, "Cannot allocate memory window.\n"); 316 return (ENXIO); 317 } 318 319 vtmmio_reset(sc); 320 321 /* Tell the host we've noticed this device. */ 322 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 323 324 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 325 device_printf(dev, "Cannot create child device.\n"); 326 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 327 vtmmio_detach(dev); 328 return (ENOMEM); 329 } 330 331 sc->vtmmio_child_dev = child; 332 vtmmio_probe_and_attach_child(sc); 333 334 return (0); 335 } 336 337 static int 338 vtmmio_detach(device_t dev) 339 { 340 struct vtmmio_softc *sc; 341 device_t child; 342 int error; 343 344 sc = device_get_softc(dev); 345 346 if ((child = sc->vtmmio_child_dev) != NULL) { 347 error = device_delete_child(dev, child); 348 if (error) 349 return (error); 350 sc->vtmmio_child_dev = NULL; 351 } 352 353 vtmmio_reset(sc); 354 355 if (sc->res[0] != NULL) { 356 bus_release_resource(dev, SYS_RES_MEMORY, 0, 357 sc->res[0]); 358 sc->res[0] = NULL; 359 } 360 361 return (0); 362 } 363 364 static int 365 vtmmio_suspend(device_t dev) 366 { 367 368 return (bus_generic_suspend(dev)); 369 } 370 371 static int 372 vtmmio_resume(device_t dev) 373 { 374 375 return (bus_generic_resume(dev)); 376 } 377 378 static int 379 vtmmio_shutdown(device_t dev) 380 { 381 382 (void) bus_generic_shutdown(dev); 383 384 /* Forcibly stop the host device. */ 385 vtmmio_stop(dev); 386 387 return (0); 388 } 389 390 static void 391 vtmmio_driver_added(device_t dev, driver_t *driver) 392 { 393 struct vtmmio_softc *sc; 394 395 sc = device_get_softc(dev); 396 397 vtmmio_probe_and_attach_child(sc); 398 } 399 400 static void 401 vtmmio_child_detached(device_t dev, device_t child) 402 { 403 struct vtmmio_softc *sc; 404 405 sc = device_get_softc(dev); 406 407 vtmmio_reset(sc); 408 vtmmio_release_child_resources(sc); 409 } 410 411 static int 412 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 413 { 414 struct vtmmio_softc *sc; 415 416 sc = device_get_softc(dev); 417 418 if (sc->vtmmio_child_dev != child) 419 return (ENOENT); 420 421 switch (index) { 422 case VIRTIO_IVAR_DEVTYPE: 423 case VIRTIO_IVAR_SUBDEVICE: 424 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID); 425 break; 426 case VIRTIO_IVAR_VENDOR: 427 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID); 428 break; 429 default: 430 return (ENOENT); 431 } 432 433 return (0); 434 } 435 436 static int 437 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 438 { 439 struct vtmmio_softc *sc; 440 441 sc = device_get_softc(dev); 442 443 if (sc->vtmmio_child_dev != child) 444 return (ENOENT); 445 446 switch (index) { 447 case VIRTIO_IVAR_FEATURE_DESC: 448 sc->vtmmio_child_feat_desc = (void *) value; 449 break; 450 default: 451 return (ENOENT); 452 } 453 454 return (0); 455 } 456 457 static uint64_t 458 vtmmio_negotiate_features(device_t dev, uint64_t child_features) 459 { 460 struct vtmmio_softc *sc; 461 uint64_t host_features, features; 462 463 sc = device_get_softc(dev); 464 465 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 466 vtmmio_describe_features(sc, "host", host_features); 467 468 /* 469 * Limit negotiated features to what the driver, virtqueue, and 470 * host all support. 471 */ 472 features = host_features & child_features; 473 features = virtqueue_filter_features(features); 474 sc->vtmmio_features = features; 475 476 vtmmio_describe_features(sc, "negotiated", features); 477 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features); 478 479 return (features); 480 } 481 482 static int 483 vtmmio_with_feature(device_t dev, uint64_t feature) 484 { 485 struct vtmmio_softc *sc; 486 487 sc = device_get_softc(dev); 488 489 return ((sc->vtmmio_features & feature) != 0); 490 } 491 492 static int 493 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, 494 struct vq_alloc_info *vq_info) 495 { 496 struct vtmmio_virtqueue *vqx; 497 struct vq_alloc_info *info; 498 struct vtmmio_softc *sc; 499 struct virtqueue *vq; 500 uint32_t size; 501 int idx, error; 502 503 sc = device_get_softc(dev); 504 505 if (sc->vtmmio_nvqs != 0) 506 return (EALREADY); 507 if (nvqs <= 0) 508 return (EINVAL); 509 510 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue), 511 M_DEVBUF, M_NOWAIT | M_ZERO); 512 if (sc->vtmmio_vqs == NULL) 513 return (ENOMEM); 514 515 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 1 << PAGE_SHIFT); 516 517 for (idx = 0; idx < nvqs; idx++) { 518 vqx = &sc->vtmmio_vqs[idx]; 519 info = &vq_info[idx]; 520 521 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 522 523 vtmmio_select_virtqueue(sc, idx); 524 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 525 526 error = virtqueue_alloc(dev, idx, size, 527 VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq); 528 if (error) { 529 device_printf(dev, 530 "cannot allocate virtqueue %d: %d\n", 531 idx, error); 532 break; 533 } 534 535 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 536 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 537 VIRTIO_MMIO_VRING_ALIGN); 538 #if 0 539 device_printf(dev, "virtqueue paddr 0x%08lx\n", 540 (uint64_t)virtqueue_paddr(vq)); 541 #endif 542 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 543 virtqueue_paddr(vq) >> PAGE_SHIFT); 544 545 vqx->vtv_vq = *info->vqai_vq = vq; 546 vqx->vtv_no_intr = info->vqai_intr == NULL; 547 548 sc->vtmmio_nvqs++; 549 } 550 551 if (error) 552 vtmmio_free_virtqueues(sc); 553 554 return (error); 555 } 556 557 static void 558 vtmmio_stop(device_t dev) 559 { 560 561 vtmmio_reset(device_get_softc(dev)); 562 } 563 564 static void 565 vtmmio_poll(device_t dev) 566 { 567 struct vtmmio_softc *sc; 568 569 sc = device_get_softc(dev); 570 571 if (sc->platform != NULL) 572 VIRTIO_MMIO_POLL(sc->platform); 573 } 574 575 static int 576 vtmmio_reinit(device_t dev, uint64_t features) 577 { 578 struct vtmmio_softc *sc; 579 int idx, error; 580 581 sc = device_get_softc(dev); 582 583 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 584 vtmmio_stop(dev); 585 586 /* 587 * Quickly drive the status through ACK and DRIVER. The device 588 * does not become usable again until vtmmio_reinit_complete(). 589 */ 590 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 591 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 592 593 vtmmio_negotiate_features(dev, features); 594 595 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 596 error = vtmmio_reinit_virtqueue(sc, idx); 597 if (error) 598 return (error); 599 } 600 601 return (0); 602 } 603 604 static void 605 vtmmio_reinit_complete(device_t dev) 606 { 607 608 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 609 } 610 611 static void 612 vtmmio_notify_virtqueue(device_t dev, uint16_t queue) 613 { 614 struct vtmmio_softc *sc; 615 616 sc = device_get_softc(dev); 617 618 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue); 619 } 620 621 static uint8_t 622 vtmmio_get_status(device_t dev) 623 { 624 struct vtmmio_softc *sc; 625 626 sc = device_get_softc(dev); 627 628 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS)); 629 } 630 631 static void 632 vtmmio_set_status(device_t dev, uint8_t status) 633 { 634 struct vtmmio_softc *sc; 635 636 sc = device_get_softc(dev); 637 638 if (status != VIRTIO_CONFIG_STATUS_RESET) 639 status |= vtmmio_get_status(dev); 640 641 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status); 642 } 643 644 static void 645 vtmmio_read_dev_config(device_t dev, bus_size_t offset, 646 void *dst, int length) 647 { 648 struct vtmmio_softc *sc; 649 bus_size_t off; 650 uint8_t *d; 651 int size; 652 653 sc = device_get_softc(dev); 654 off = VIRTIO_MMIO_CONFIG + offset; 655 656 for (d = dst; length > 0; d += size, off += size, length -= size) { 657 #ifdef ALLOW_WORD_ALIGNED_ACCESS 658 if (length >= 4) { 659 size = 4; 660 *(uint32_t *)d = vtmmio_read_config_4(sc, off); 661 } else if (length >= 2) { 662 size = 2; 663 *(uint16_t *)d = vtmmio_read_config_2(sc, off); 664 } else 665 #endif 666 { 667 size = 1; 668 *d = vtmmio_read_config_1(sc, off); 669 } 670 } 671 } 672 673 static void 674 vtmmio_write_dev_config(device_t dev, bus_size_t offset, 675 void *src, int length) 676 { 677 struct vtmmio_softc *sc; 678 bus_size_t off; 679 uint8_t *s; 680 int size; 681 682 sc = device_get_softc(dev); 683 off = VIRTIO_MMIO_CONFIG + offset; 684 685 for (s = src; length > 0; s += size, off += size, length -= size) { 686 #ifdef ALLOW_WORD_ALIGNED_ACCESS 687 if (length >= 4) { 688 size = 4; 689 vtmmio_write_config_4(sc, off, *(uint32_t *)s); 690 } else if (length >= 2) { 691 size = 2; 692 vtmmio_write_config_2(sc, off, *(uint16_t *)s); 693 } else 694 #endif 695 { 696 size = 1; 697 vtmmio_write_config_1(sc, off, *s); 698 } 699 } 700 } 701 702 static void 703 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg, 704 uint64_t features) 705 { 706 device_t dev, child; 707 708 dev = sc->dev; 709 child = sc->vtmmio_child_dev; 710 711 if (device_is_attached(child) || bootverbose == 0) 712 return; 713 714 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc); 715 } 716 717 static void 718 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc) 719 { 720 device_t dev, child; 721 722 dev = sc->dev; 723 child = sc->vtmmio_child_dev; 724 725 if (child == NULL) 726 return; 727 728 if (device_get_state(child) != DS_NOTPRESENT) { 729 return; 730 } 731 732 if (device_probe(child) != 0) { 733 return; 734 } 735 736 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 737 if (device_attach(child) != 0) { 738 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 739 vtmmio_reset(sc); 740 vtmmio_release_child_resources(sc); 741 /* Reset status for future attempt. */ 742 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 743 } else { 744 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 745 VIRTIO_ATTACH_COMPLETED(child); 746 } 747 } 748 749 static int 750 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx) 751 { 752 struct vtmmio_virtqueue *vqx; 753 struct virtqueue *vq; 754 int error; 755 uint16_t size; 756 757 vqx = &sc->vtmmio_vqs[idx]; 758 vq = vqx->vtv_vq; 759 760 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 761 762 vtmmio_select_virtqueue(sc, idx); 763 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 764 765 error = virtqueue_reinit(vq, size); 766 if (error) 767 return (error); 768 769 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 770 virtqueue_paddr(vq) >> PAGE_SHIFT); 771 772 return (0); 773 } 774 775 static void 776 vtmmio_free_interrupts(struct vtmmio_softc *sc) 777 { 778 779 if (sc->ih != NULL) 780 bus_teardown_intr(sc->dev, sc->res[1], sc->ih); 781 782 if (sc->res[1] != NULL) 783 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]); 784 } 785 786 static void 787 vtmmio_free_virtqueues(struct vtmmio_softc *sc) 788 { 789 struct vtmmio_virtqueue *vqx; 790 int idx; 791 792 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 793 vqx = &sc->vtmmio_vqs[idx]; 794 795 vtmmio_select_virtqueue(sc, idx); 796 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0); 797 798 virtqueue_free(vqx->vtv_vq); 799 vqx->vtv_vq = NULL; 800 } 801 802 free(sc->vtmmio_vqs, M_DEVBUF); 803 sc->vtmmio_vqs = NULL; 804 sc->vtmmio_nvqs = 0; 805 } 806 807 static void 808 vtmmio_release_child_resources(struct vtmmio_softc *sc) 809 { 810 811 vtmmio_free_interrupts(sc); 812 vtmmio_free_virtqueues(sc); 813 } 814 815 static void 816 vtmmio_reset(struct vtmmio_softc *sc) 817 { 818 819 /* 820 * Setting the status to RESET sets the host device to 821 * the original, uninitialized state. 822 */ 823 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET); 824 } 825 826 static void 827 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx) 828 { 829 830 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 831 } 832 833 static void 834 vtmmio_vq_intr(void *arg) 835 { 836 struct vtmmio_virtqueue *vqx; 837 struct vtmmio_softc *sc; 838 struct virtqueue *vq; 839 uint32_t status; 840 int idx; 841 842 sc = arg; 843 844 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS); 845 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status); 846 847 /* The config changed */ 848 if (status & VIRTIO_MMIO_INT_CONFIG) 849 if (sc->vtmmio_child_dev != NULL) 850 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev); 851 852 /* Notify all virtqueues. */ 853 if (status & VIRTIO_MMIO_INT_VRING) { 854 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 855 vqx = &sc->vtmmio_vqs[idx]; 856 if (vqx->vtv_no_intr == 0) { 857 vq = vqx->vtv_vq; 858 virtqueue_intr(vq); 859 } 860 } 861 } 862 } 863