1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Portions of this software were developed by Andrew Turner 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * VirtIO MMIO interface. 37 * This driver is heavily based on VirtIO PCI interface driver. 38 */ 39 40 /* 41 * FDT example: 42 * virtio_block@1000 { 43 * compatible = "virtio,mmio"; 44 * reg = <0x1000 0x100>; 45 * interrupts = <63>; 46 * interrupt-parent = <&GIC>; 47 * }; 48 */ 49 50 #include <sys/cdefs.h> 51 __FBSDID("$FreeBSD$"); 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/bus.h> 56 #include <sys/kernel.h> 57 #include <sys/module.h> 58 #include <sys/malloc.h> 59 #include <sys/rman.h> 60 61 #include <machine/bus.h> 62 #include <machine/resource.h> 63 64 #include <dev/fdt/fdt_common.h> 65 #include <dev/ofw/openfirm.h> 66 #include <dev/ofw/ofw_bus.h> 67 #include <dev/ofw/ofw_bus_subr.h> 68 69 #include <dev/virtio/virtio.h> 70 #include <dev/virtio/virtqueue.h> 71 #include <dev/virtio/mmio/virtio_mmio.h> 72 73 #include "virtio_mmio_if.h" 74 #include "virtio_bus_if.h" 75 #include "virtio_if.h" 76 77 #define PAGE_SHIFT 12 78 79 struct vtmmio_virtqueue { 80 struct virtqueue *vtv_vq; 81 int vtv_no_intr; 82 }; 83 84 struct vtmmio_softc { 85 device_t dev; 86 device_t platform; 87 struct resource *res[2]; 88 89 uint64_t vtmmio_features; 90 uint32_t vtmmio_flags; 91 92 /* This "bus" will only ever have one child. */ 93 device_t vtmmio_child_dev; 94 struct virtio_feature_desc *vtmmio_child_feat_desc; 95 96 int vtmmio_nvqs; 97 struct vtmmio_virtqueue *vtmmio_vqs; 98 void *ih; 99 }; 100 101 static int vtmmio_probe(device_t); 102 static int vtmmio_attach(device_t); 103 static int vtmmio_detach(device_t); 104 static int vtmmio_suspend(device_t); 105 static int vtmmio_resume(device_t); 106 static int vtmmio_shutdown(device_t); 107 static void vtmmio_driver_added(device_t, driver_t *); 108 static void vtmmio_child_detached(device_t, device_t); 109 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *); 110 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t); 111 static uint64_t vtmmio_negotiate_features(device_t, uint64_t); 112 static int vtmmio_with_feature(device_t, uint64_t); 113 static int vtmmio_alloc_virtqueues(device_t, int, int, 114 struct vq_alloc_info *); 115 static int vtmmio_setup_intr(device_t, enum intr_type); 116 static void vtmmio_stop(device_t); 117 static void vtmmio_poll(device_t); 118 static int vtmmio_reinit(device_t, uint64_t); 119 static void vtmmio_reinit_complete(device_t); 120 static void vtmmio_notify_virtqueue(device_t, uint16_t); 121 static uint8_t vtmmio_get_status(device_t); 122 static void vtmmio_set_status(device_t, uint8_t); 123 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); 124 static void vtmmio_write_dev_config(device_t, bus_size_t, void *, int); 125 static void vtmmio_describe_features(struct vtmmio_softc *, const char *, 126 uint64_t); 127 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *); 128 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int); 129 static void vtmmio_free_interrupts(struct vtmmio_softc *); 130 static void vtmmio_free_virtqueues(struct vtmmio_softc *); 131 static void vtmmio_release_child_resources(struct vtmmio_softc *); 132 static void vtmmio_reset(struct vtmmio_softc *); 133 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int); 134 static void vtmmio_vq_intr(void *); 135 136 /* 137 * I/O port read/write wrappers. 138 */ 139 #define vtmmio_write_config_1(sc, o, v) \ 140 do { \ 141 if (sc->platform != NULL) \ 142 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 143 bus_write_1((sc)->res[0], (o), (v)); \ 144 if (sc->platform != NULL) \ 145 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 146 } while (0) 147 #define vtmmio_write_config_2(sc, o, v) \ 148 do { \ 149 if (sc->platform != NULL) \ 150 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 151 bus_write_2((sc)->res[0], (o), (v)); \ 152 if (sc->platform != NULL) \ 153 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 154 } while (0) 155 #define vtmmio_write_config_4(sc, o, v) \ 156 do { \ 157 if (sc->platform != NULL) \ 158 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 159 bus_write_4((sc)->res[0], (o), (v)); \ 160 if (sc->platform != NULL) \ 161 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 162 } while (0) 163 164 #define vtmmio_read_config_1(sc, o) \ 165 bus_read_1((sc)->res[0], (o)) 166 #define vtmmio_read_config_2(sc, o) \ 167 bus_read_2((sc)->res[0], (o)) 168 #define vtmmio_read_config_4(sc, o) \ 169 bus_read_4((sc)->res[0], (o)) 170 171 static device_method_t vtmmio_methods[] = { 172 /* Device interface. */ 173 DEVMETHOD(device_probe, vtmmio_probe), 174 DEVMETHOD(device_attach, vtmmio_attach), 175 DEVMETHOD(device_detach, vtmmio_detach), 176 DEVMETHOD(device_suspend, vtmmio_suspend), 177 DEVMETHOD(device_resume, vtmmio_resume), 178 DEVMETHOD(device_shutdown, vtmmio_shutdown), 179 180 /* Bus interface. */ 181 DEVMETHOD(bus_driver_added, vtmmio_driver_added), 182 DEVMETHOD(bus_child_detached, vtmmio_child_detached), 183 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar), 184 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar), 185 186 /* VirtIO bus interface. */ 187 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features), 188 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature), 189 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), 190 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), 191 DEVMETHOD(virtio_bus_stop, vtmmio_stop), 192 DEVMETHOD(virtio_bus_poll, vtmmio_poll), 193 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), 194 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), 195 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), 196 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config), 197 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config), 198 199 DEVMETHOD_END 200 }; 201 202 static driver_t vtmmio_driver = { 203 "virtio_mmio", 204 vtmmio_methods, 205 sizeof(struct vtmmio_softc) 206 }; 207 208 devclass_t vtmmio_devclass; 209 210 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0); 211 DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0); 212 MODULE_VERSION(virtio_mmio, 1); 213 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1); 214 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1); 215 216 static int 217 vtmmio_setup_intr(device_t dev, enum intr_type type) 218 { 219 struct vtmmio_softc *sc; 220 int rid; 221 int err; 222 223 sc = device_get_softc(dev); 224 225 if (sc->platform != NULL) { 226 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, 227 vtmmio_vq_intr, sc); 228 if (err == 0) { 229 /* Okay we have backend-specific interrupts */ 230 return (0); 231 } 232 } 233 234 rid = 0; 235 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 236 RF_ACTIVE); 237 if (!sc->res[1]) { 238 device_printf(dev, "Can't allocate interrupt\n"); 239 return (ENXIO); 240 } 241 242 if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, 243 NULL, vtmmio_vq_intr, sc, &sc->ih)) { 244 device_printf(dev, "Can't setup the interrupt\n"); 245 return (ENXIO); 246 } 247 248 return (0); 249 } 250 251 static int 252 vtmmio_probe(device_t dev) 253 { 254 255 if (!ofw_bus_status_okay(dev)) 256 return (ENXIO); 257 258 if (!ofw_bus_is_compatible(dev, "virtio,mmio")) 259 return (ENXIO); 260 261 device_set_desc(dev, "VirtIO MMIO adapter"); 262 return (BUS_PROBE_DEFAULT); 263 } 264 265 static int 266 vtmmio_setup_platform(struct vtmmio_softc *sc) 267 { 268 phandle_t platform_node; 269 struct fdt_ic *ic; 270 phandle_t xref; 271 phandle_t node; 272 273 sc->platform = NULL; 274 275 if ((node = ofw_bus_get_node(sc->dev)) == -1) 276 return (ENXIO); 277 278 if (OF_searchencprop(node, "platform", &xref, 279 sizeof(xref)) == -1) { 280 return (ENXIO); 281 } 282 283 platform_node = OF_node_from_xref(xref); 284 285 SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) { 286 if (ic->iph == platform_node) { 287 sc->platform = ic->dev; 288 break; 289 } 290 } 291 292 if (sc->platform == NULL) { 293 /* No platform-specific device. Ignore it. */ 294 } 295 296 return (0); 297 } 298 299 static int 300 vtmmio_attach(device_t dev) 301 { 302 struct vtmmio_softc *sc; 303 device_t child; 304 int rid; 305 306 sc = device_get_softc(dev); 307 sc->dev = dev; 308 309 vtmmio_setup_platform(sc); 310 311 rid = 0; 312 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 313 RF_ACTIVE); 314 if (!sc->res[0]) { 315 device_printf(dev, "Cannot allocate memory window.\n"); 316 return (ENXIO); 317 } 318 319 vtmmio_reset(sc); 320 321 /* Tell the host we've noticed this device. */ 322 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 323 324 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 325 device_printf(dev, "Cannot create child device.\n"); 326 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 327 vtmmio_detach(dev); 328 return (ENOMEM); 329 } 330 331 sc->vtmmio_child_dev = child; 332 vtmmio_probe_and_attach_child(sc); 333 334 return (0); 335 } 336 337 static int 338 vtmmio_detach(device_t dev) 339 { 340 struct vtmmio_softc *sc; 341 device_t child; 342 int error; 343 344 sc = device_get_softc(dev); 345 346 if ((child = sc->vtmmio_child_dev) != NULL) { 347 error = device_delete_child(dev, child); 348 if (error) 349 return (error); 350 sc->vtmmio_child_dev = NULL; 351 } 352 353 vtmmio_reset(sc); 354 355 if (sc->res[0] != NULL) { 356 bus_release_resource(dev, SYS_RES_MEMORY, 0, 357 sc->res[0]); 358 sc->res[0] = NULL; 359 } 360 361 return (0); 362 } 363 364 static int 365 vtmmio_suspend(device_t dev) 366 { 367 368 return (bus_generic_suspend(dev)); 369 } 370 371 static int 372 vtmmio_resume(device_t dev) 373 { 374 375 return (bus_generic_resume(dev)); 376 } 377 378 static int 379 vtmmio_shutdown(device_t dev) 380 { 381 382 (void) bus_generic_shutdown(dev); 383 384 /* Forcibly stop the host device. */ 385 vtmmio_stop(dev); 386 387 return (0); 388 } 389 390 static void 391 vtmmio_driver_added(device_t dev, driver_t *driver) 392 { 393 struct vtmmio_softc *sc; 394 395 sc = device_get_softc(dev); 396 397 vtmmio_probe_and_attach_child(sc); 398 } 399 400 static void 401 vtmmio_child_detached(device_t dev, device_t child) 402 { 403 struct vtmmio_softc *sc; 404 405 sc = device_get_softc(dev); 406 407 vtmmio_reset(sc); 408 vtmmio_release_child_resources(sc); 409 } 410 411 static int 412 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 413 { 414 struct vtmmio_softc *sc; 415 416 sc = device_get_softc(dev); 417 418 if (sc->vtmmio_child_dev != child) 419 return (ENOENT); 420 421 switch (index) { 422 case VIRTIO_IVAR_DEVTYPE: 423 case VIRTIO_IVAR_SUBDEVICE: 424 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID); 425 break; 426 case VIRTIO_IVAR_VENDOR: 427 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID); 428 break; 429 default: 430 return (ENOENT); 431 } 432 433 return (0); 434 } 435 436 static int 437 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 438 { 439 struct vtmmio_softc *sc; 440 441 sc = device_get_softc(dev); 442 443 if (sc->vtmmio_child_dev != child) 444 return (ENOENT); 445 446 switch (index) { 447 case VIRTIO_IVAR_FEATURE_DESC: 448 sc->vtmmio_child_feat_desc = (void *) value; 449 break; 450 default: 451 return (ENOENT); 452 } 453 454 return (0); 455 } 456 457 static uint64_t 458 vtmmio_negotiate_features(device_t dev, uint64_t child_features) 459 { 460 struct vtmmio_softc *sc; 461 uint64_t host_features, features; 462 463 sc = device_get_softc(dev); 464 465 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 466 vtmmio_describe_features(sc, "host", host_features); 467 468 /* 469 * Limit negotiated features to what the driver, virtqueue, and 470 * host all support. 471 */ 472 features = host_features & child_features; 473 features = virtqueue_filter_features(features); 474 sc->vtmmio_features = features; 475 476 vtmmio_describe_features(sc, "negotiated", features); 477 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features); 478 479 return (features); 480 } 481 482 static int 483 vtmmio_with_feature(device_t dev, uint64_t feature) 484 { 485 struct vtmmio_softc *sc; 486 487 sc = device_get_softc(dev); 488 489 return ((sc->vtmmio_features & feature) != 0); 490 } 491 492 static int 493 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, 494 struct vq_alloc_info *vq_info) 495 { 496 struct vtmmio_virtqueue *vqx; 497 struct vq_alloc_info *info; 498 struct vtmmio_softc *sc; 499 struct virtqueue *vq; 500 uint32_t size; 501 int idx, error; 502 503 sc = device_get_softc(dev); 504 505 if (sc->vtmmio_nvqs != 0) 506 return (EALREADY); 507 if (nvqs <= 0) 508 return (EINVAL); 509 510 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue), 511 M_DEVBUF, M_NOWAIT | M_ZERO); 512 if (sc->vtmmio_vqs == NULL) 513 return (ENOMEM); 514 515 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 516 (1 << PAGE_SHIFT)); 517 518 for (idx = 0; idx < nvqs; idx++) { 519 vqx = &sc->vtmmio_vqs[idx]; 520 info = &vq_info[idx]; 521 522 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 523 524 vtmmio_select_virtqueue(sc, idx); 525 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 526 527 error = virtqueue_alloc(dev, idx, size, 528 VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq); 529 if (error) { 530 device_printf(dev, 531 "cannot allocate virtqueue %d: %d\n", 532 idx, error); 533 break; 534 } 535 536 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 537 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 538 VIRTIO_MMIO_VRING_ALIGN); 539 #if 0 540 device_printf(dev, "virtqueue paddr 0x%08lx\n", 541 (uint64_t)virtqueue_paddr(vq)); 542 #endif 543 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 544 virtqueue_paddr(vq) >> PAGE_SHIFT); 545 546 vqx->vtv_vq = *info->vqai_vq = vq; 547 vqx->vtv_no_intr = info->vqai_intr == NULL; 548 549 sc->vtmmio_nvqs++; 550 } 551 552 if (error) 553 vtmmio_free_virtqueues(sc); 554 555 return (error); 556 } 557 558 static void 559 vtmmio_stop(device_t dev) 560 { 561 562 vtmmio_reset(device_get_softc(dev)); 563 } 564 565 static void 566 vtmmio_poll(device_t dev) 567 { 568 struct vtmmio_softc *sc; 569 570 sc = device_get_softc(dev); 571 572 if (sc->platform != NULL) 573 VIRTIO_MMIO_POLL(sc->platform); 574 } 575 576 static int 577 vtmmio_reinit(device_t dev, uint64_t features) 578 { 579 struct vtmmio_softc *sc; 580 int idx, error; 581 582 sc = device_get_softc(dev); 583 584 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 585 vtmmio_stop(dev); 586 587 /* 588 * Quickly drive the status through ACK and DRIVER. The device 589 * does not become usable again until vtmmio_reinit_complete(). 590 */ 591 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 592 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 593 594 vtmmio_negotiate_features(dev, features); 595 596 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 597 (1 << PAGE_SHIFT)); 598 599 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 600 error = vtmmio_reinit_virtqueue(sc, idx); 601 if (error) 602 return (error); 603 } 604 605 return (0); 606 } 607 608 static void 609 vtmmio_reinit_complete(device_t dev) 610 { 611 612 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 613 } 614 615 static void 616 vtmmio_notify_virtqueue(device_t dev, uint16_t queue) 617 { 618 struct vtmmio_softc *sc; 619 620 sc = device_get_softc(dev); 621 622 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue); 623 } 624 625 static uint8_t 626 vtmmio_get_status(device_t dev) 627 { 628 struct vtmmio_softc *sc; 629 630 sc = device_get_softc(dev); 631 632 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS)); 633 } 634 635 static void 636 vtmmio_set_status(device_t dev, uint8_t status) 637 { 638 struct vtmmio_softc *sc; 639 640 sc = device_get_softc(dev); 641 642 if (status != VIRTIO_CONFIG_STATUS_RESET) 643 status |= vtmmio_get_status(dev); 644 645 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status); 646 } 647 648 static void 649 vtmmio_read_dev_config(device_t dev, bus_size_t offset, 650 void *dst, int length) 651 { 652 struct vtmmio_softc *sc; 653 bus_size_t off; 654 uint8_t *d; 655 int size; 656 657 sc = device_get_softc(dev); 658 off = VIRTIO_MMIO_CONFIG + offset; 659 660 for (d = dst; length > 0; d += size, off += size, length -= size) { 661 #ifdef ALLOW_WORD_ALIGNED_ACCESS 662 if (length >= 4) { 663 size = 4; 664 *(uint32_t *)d = vtmmio_read_config_4(sc, off); 665 } else if (length >= 2) { 666 size = 2; 667 *(uint16_t *)d = vtmmio_read_config_2(sc, off); 668 } else 669 #endif 670 { 671 size = 1; 672 *d = vtmmio_read_config_1(sc, off); 673 } 674 } 675 } 676 677 static void 678 vtmmio_write_dev_config(device_t dev, bus_size_t offset, 679 void *src, int length) 680 { 681 struct vtmmio_softc *sc; 682 bus_size_t off; 683 uint8_t *s; 684 int size; 685 686 sc = device_get_softc(dev); 687 off = VIRTIO_MMIO_CONFIG + offset; 688 689 for (s = src; length > 0; s += size, off += size, length -= size) { 690 #ifdef ALLOW_WORD_ALIGNED_ACCESS 691 if (length >= 4) { 692 size = 4; 693 vtmmio_write_config_4(sc, off, *(uint32_t *)s); 694 } else if (length >= 2) { 695 size = 2; 696 vtmmio_write_config_2(sc, off, *(uint16_t *)s); 697 } else 698 #endif 699 { 700 size = 1; 701 vtmmio_write_config_1(sc, off, *s); 702 } 703 } 704 } 705 706 static void 707 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg, 708 uint64_t features) 709 { 710 device_t dev, child; 711 712 dev = sc->dev; 713 child = sc->vtmmio_child_dev; 714 715 if (device_is_attached(child) || bootverbose == 0) 716 return; 717 718 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc); 719 } 720 721 static void 722 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc) 723 { 724 device_t dev, child; 725 726 dev = sc->dev; 727 child = sc->vtmmio_child_dev; 728 729 if (child == NULL) 730 return; 731 732 if (device_get_state(child) != DS_NOTPRESENT) { 733 return; 734 } 735 736 if (device_probe(child) != 0) { 737 return; 738 } 739 740 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 741 if (device_attach(child) != 0) { 742 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 743 vtmmio_reset(sc); 744 vtmmio_release_child_resources(sc); 745 /* Reset status for future attempt. */ 746 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 747 } else { 748 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 749 VIRTIO_ATTACH_COMPLETED(child); 750 } 751 } 752 753 static int 754 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx) 755 { 756 struct vtmmio_virtqueue *vqx; 757 struct virtqueue *vq; 758 int error; 759 uint16_t size; 760 761 vqx = &sc->vtmmio_vqs[idx]; 762 vq = vqx->vtv_vq; 763 764 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 765 766 vtmmio_select_virtqueue(sc, idx); 767 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 768 769 error = virtqueue_reinit(vq, size); 770 if (error) 771 return (error); 772 773 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 774 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 775 VIRTIO_MMIO_VRING_ALIGN); 776 #if 0 777 device_printf(sc->dev, "virtqueue paddr 0x%08lx\n", 778 (uint64_t)virtqueue_paddr(vq)); 779 #endif 780 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 781 virtqueue_paddr(vq) >> PAGE_SHIFT); 782 783 return (0); 784 } 785 786 static void 787 vtmmio_free_interrupts(struct vtmmio_softc *sc) 788 { 789 790 if (sc->ih != NULL) 791 bus_teardown_intr(sc->dev, sc->res[1], sc->ih); 792 793 if (sc->res[1] != NULL) 794 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]); 795 } 796 797 static void 798 vtmmio_free_virtqueues(struct vtmmio_softc *sc) 799 { 800 struct vtmmio_virtqueue *vqx; 801 int idx; 802 803 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 804 vqx = &sc->vtmmio_vqs[idx]; 805 806 vtmmio_select_virtqueue(sc, idx); 807 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0); 808 809 virtqueue_free(vqx->vtv_vq); 810 vqx->vtv_vq = NULL; 811 } 812 813 free(sc->vtmmio_vqs, M_DEVBUF); 814 sc->vtmmio_vqs = NULL; 815 sc->vtmmio_nvqs = 0; 816 } 817 818 static void 819 vtmmio_release_child_resources(struct vtmmio_softc *sc) 820 { 821 822 vtmmio_free_interrupts(sc); 823 vtmmio_free_virtqueues(sc); 824 } 825 826 static void 827 vtmmio_reset(struct vtmmio_softc *sc) 828 { 829 830 /* 831 * Setting the status to RESET sets the host device to 832 * the original, uninitialized state. 833 */ 834 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET); 835 } 836 837 static void 838 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx) 839 { 840 841 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 842 } 843 844 static void 845 vtmmio_vq_intr(void *arg) 846 { 847 struct vtmmio_virtqueue *vqx; 848 struct vtmmio_softc *sc; 849 struct virtqueue *vq; 850 uint32_t status; 851 int idx; 852 853 sc = arg; 854 855 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS); 856 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status); 857 858 /* The config changed */ 859 if (status & VIRTIO_MMIO_INT_CONFIG) 860 if (sc->vtmmio_child_dev != NULL) 861 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev); 862 863 /* Notify all virtqueues. */ 864 if (status & VIRTIO_MMIO_INT_VRING) { 865 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 866 vqx = &sc->vtmmio_vqs[idx]; 867 if (vqx->vtv_no_intr == 0) { 868 vq = vqx->vtv_vq; 869 virtqueue_intr(vq); 870 } 871 } 872 } 873 } 874