1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Portions of this software were developed by Andrew Turner 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * VirtIO MMIO interface. 37 * This driver is heavily based on VirtIO PCI interface driver. 38 */ 39 40 #include <sys/cdefs.h> 41 __FBSDID("$FreeBSD$"); 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/kernel.h> 47 #include <sys/module.h> 48 #include <sys/malloc.h> 49 #include <sys/rman.h> 50 #include <sys/endian.h> 51 52 #include <machine/bus.h> 53 #include <machine/resource.h> 54 55 #include <dev/virtio/virtio.h> 56 #include <dev/virtio/virtqueue.h> 57 #include <dev/virtio/mmio/virtio_mmio.h> 58 59 #include "virtio_mmio_if.h" 60 #include "virtio_bus_if.h" 61 #include "virtio_if.h" 62 63 #define PAGE_SHIFT 12 64 65 struct vtmmio_virtqueue { 66 struct virtqueue *vtv_vq; 67 int vtv_no_intr; 68 }; 69 70 static int vtmmio_detach(device_t); 71 static int vtmmio_suspend(device_t); 72 static int vtmmio_resume(device_t); 73 static int vtmmio_shutdown(device_t); 74 static void vtmmio_driver_added(device_t, driver_t *); 75 static void vtmmio_child_detached(device_t, device_t); 76 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *); 77 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t); 78 static uint64_t vtmmio_negotiate_features(device_t, uint64_t); 79 static int vtmmio_with_feature(device_t, uint64_t); 80 static void vtmmio_set_virtqueue(struct vtmmio_softc *sc, 81 struct virtqueue *vq, uint32_t size); 82 static int vtmmio_alloc_virtqueues(device_t, int, int, 83 struct vq_alloc_info *); 84 static int vtmmio_setup_intr(device_t, enum intr_type); 85 static void vtmmio_stop(device_t); 86 static void vtmmio_poll(device_t); 87 static int vtmmio_reinit(device_t, uint64_t); 88 static void vtmmio_reinit_complete(device_t); 89 static void vtmmio_notify_virtqueue(device_t, uint16_t); 90 static uint8_t vtmmio_get_status(device_t); 91 static void vtmmio_set_status(device_t, uint8_t); 92 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); 93 static void vtmmio_write_dev_config(device_t, bus_size_t, void *, int); 94 static void vtmmio_describe_features(struct vtmmio_softc *, const char *, 95 uint64_t); 96 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *); 97 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int); 98 static void vtmmio_free_interrupts(struct vtmmio_softc *); 99 static void vtmmio_free_virtqueues(struct vtmmio_softc *); 100 static void vtmmio_release_child_resources(struct vtmmio_softc *); 101 static void vtmmio_reset(struct vtmmio_softc *); 102 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int); 103 static void vtmmio_vq_intr(void *); 104 105 /* 106 * I/O port read/write wrappers. 107 */ 108 #define vtmmio_write_config_1(sc, o, v) \ 109 do { \ 110 if (sc->platform != NULL) \ 111 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 112 bus_write_1((sc)->res[0], (o), (v)); \ 113 if (sc->platform != NULL) \ 114 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 115 } while (0) 116 #define vtmmio_write_config_2(sc, o, v) \ 117 do { \ 118 if (sc->platform != NULL) \ 119 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 120 bus_write_2((sc)->res[0], (o), (v)); \ 121 if (sc->platform != NULL) \ 122 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 123 } while (0) 124 #define vtmmio_write_config_4(sc, o, v) \ 125 do { \ 126 if (sc->platform != NULL) \ 127 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 128 bus_write_4((sc)->res[0], (o), (v)); \ 129 if (sc->platform != NULL) \ 130 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 131 } while (0) 132 133 #define vtmmio_read_config_1(sc, o) \ 134 bus_read_1((sc)->res[0], (o)) 135 #define vtmmio_read_config_2(sc, o) \ 136 bus_read_2((sc)->res[0], (o)) 137 #define vtmmio_read_config_4(sc, o) \ 138 bus_read_4((sc)->res[0], (o)) 139 140 static device_method_t vtmmio_methods[] = { 141 /* Device interface. */ 142 DEVMETHOD(device_attach, vtmmio_attach), 143 DEVMETHOD(device_detach, vtmmio_detach), 144 DEVMETHOD(device_suspend, vtmmio_suspend), 145 DEVMETHOD(device_resume, vtmmio_resume), 146 DEVMETHOD(device_shutdown, vtmmio_shutdown), 147 148 /* Bus interface. */ 149 DEVMETHOD(bus_driver_added, vtmmio_driver_added), 150 DEVMETHOD(bus_child_detached, vtmmio_child_detached), 151 DEVMETHOD(bus_child_pnpinfo_str, virtio_child_pnpinfo_str), 152 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar), 153 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar), 154 155 /* VirtIO bus interface. */ 156 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features), 157 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature), 158 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), 159 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), 160 DEVMETHOD(virtio_bus_stop, vtmmio_stop), 161 DEVMETHOD(virtio_bus_poll, vtmmio_poll), 162 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), 163 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), 164 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), 165 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config), 166 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config), 167 168 DEVMETHOD_END 169 }; 170 171 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods, 172 sizeof(struct vtmmio_softc)); 173 174 MODULE_VERSION(virtio_mmio, 1); 175 176 static int 177 vtmmio_setup_intr(device_t dev, enum intr_type type) 178 { 179 struct vtmmio_softc *sc; 180 int rid; 181 int err; 182 183 sc = device_get_softc(dev); 184 185 if (sc->platform != NULL) { 186 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, 187 vtmmio_vq_intr, sc); 188 if (err == 0) { 189 /* Okay we have backend-specific interrupts */ 190 return (0); 191 } 192 } 193 194 rid = 0; 195 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 196 RF_ACTIVE); 197 if (!sc->res[1]) { 198 device_printf(dev, "Can't allocate interrupt\n"); 199 return (ENXIO); 200 } 201 202 if (bus_setup_intr(dev, sc->res[1], type | INTR_MPSAFE, 203 NULL, vtmmio_vq_intr, sc, &sc->ih)) { 204 device_printf(dev, "Can't setup the interrupt\n"); 205 return (ENXIO); 206 } 207 208 return (0); 209 } 210 211 int 212 vtmmio_attach(device_t dev) 213 { 214 struct vtmmio_softc *sc; 215 device_t child; 216 int rid; 217 218 sc = device_get_softc(dev); 219 sc->dev = dev; 220 221 rid = 0; 222 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 223 RF_ACTIVE); 224 if (!sc->res[0]) { 225 device_printf(dev, "Cannot allocate memory window.\n"); 226 return (ENXIO); 227 } 228 229 sc->vtmmio_version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION); 230 if (sc->vtmmio_version < 1 || sc->vtmmio_version > 2) { 231 device_printf(dev, "Unsupported version: %x\n", 232 sc->vtmmio_version); 233 bus_release_resource(dev, SYS_RES_MEMORY, 0, 234 sc->res[0]); 235 sc->res[0] = NULL; 236 return (ENXIO); 237 } 238 239 vtmmio_reset(sc); 240 241 /* Tell the host we've noticed this device. */ 242 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 243 244 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 245 device_printf(dev, "Cannot create child device.\n"); 246 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 247 vtmmio_detach(dev); 248 return (ENOMEM); 249 } 250 251 sc->vtmmio_child_dev = child; 252 vtmmio_probe_and_attach_child(sc); 253 254 return (0); 255 } 256 257 static int 258 vtmmio_detach(device_t dev) 259 { 260 struct vtmmio_softc *sc; 261 device_t child; 262 int error; 263 264 sc = device_get_softc(dev); 265 266 if ((child = sc->vtmmio_child_dev) != NULL) { 267 error = device_delete_child(dev, child); 268 if (error) 269 return (error); 270 sc->vtmmio_child_dev = NULL; 271 } 272 273 vtmmio_reset(sc); 274 275 if (sc->res[0] != NULL) { 276 bus_release_resource(dev, SYS_RES_MEMORY, 0, 277 sc->res[0]); 278 sc->res[0] = NULL; 279 } 280 281 return (0); 282 } 283 284 static int 285 vtmmio_suspend(device_t dev) 286 { 287 288 return (bus_generic_suspend(dev)); 289 } 290 291 static int 292 vtmmio_resume(device_t dev) 293 { 294 295 return (bus_generic_resume(dev)); 296 } 297 298 static int 299 vtmmio_shutdown(device_t dev) 300 { 301 302 (void) bus_generic_shutdown(dev); 303 304 /* Forcibly stop the host device. */ 305 vtmmio_stop(dev); 306 307 return (0); 308 } 309 310 static void 311 vtmmio_driver_added(device_t dev, driver_t *driver) 312 { 313 struct vtmmio_softc *sc; 314 315 sc = device_get_softc(dev); 316 317 vtmmio_probe_and_attach_child(sc); 318 } 319 320 static void 321 vtmmio_child_detached(device_t dev, device_t child) 322 { 323 struct vtmmio_softc *sc; 324 325 sc = device_get_softc(dev); 326 327 vtmmio_reset(sc); 328 vtmmio_release_child_resources(sc); 329 } 330 331 static int 332 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 333 { 334 struct vtmmio_softc *sc; 335 336 sc = device_get_softc(dev); 337 338 if (sc->vtmmio_child_dev != child) 339 return (ENOENT); 340 341 switch (index) { 342 case VIRTIO_IVAR_DEVTYPE: 343 case VIRTIO_IVAR_SUBDEVICE: 344 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID); 345 break; 346 case VIRTIO_IVAR_VENDOR: 347 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID); 348 break; 349 case VIRTIO_IVAR_SUBVENDOR: 350 case VIRTIO_IVAR_DEVICE: 351 /* 352 * Dummy value for fields not present in this bus. Used by 353 * bus-agnostic virtio_child_pnpinfo_str. 354 */ 355 *result = 0; 356 break; 357 default: 358 return (ENOENT); 359 } 360 361 return (0); 362 } 363 364 static int 365 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 366 { 367 struct vtmmio_softc *sc; 368 369 sc = device_get_softc(dev); 370 371 if (sc->vtmmio_child_dev != child) 372 return (ENOENT); 373 374 switch (index) { 375 case VIRTIO_IVAR_FEATURE_DESC: 376 sc->vtmmio_child_feat_desc = (void *) value; 377 break; 378 default: 379 return (ENOENT); 380 } 381 382 return (0); 383 } 384 385 static uint64_t 386 vtmmio_negotiate_features(device_t dev, uint64_t child_features) 387 { 388 struct vtmmio_softc *sc; 389 uint64_t host_features, features; 390 391 sc = device_get_softc(dev); 392 393 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 394 vtmmio_describe_features(sc, "host", host_features); 395 396 /* 397 * Limit negotiated features to what the driver, virtqueue, and 398 * host all support. 399 */ 400 features = host_features & child_features; 401 features = virtqueue_filter_features(features); 402 sc->vtmmio_features = features; 403 404 vtmmio_describe_features(sc, "negotiated", features); 405 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features); 406 407 return (features); 408 } 409 410 static int 411 vtmmio_with_feature(device_t dev, uint64_t feature) 412 { 413 struct vtmmio_softc *sc; 414 415 sc = device_get_softc(dev); 416 417 return ((sc->vtmmio_features & feature) != 0); 418 } 419 420 static void 421 vtmmio_set_virtqueue(struct vtmmio_softc *sc, struct virtqueue *vq, 422 uint32_t size) 423 { 424 vm_paddr_t paddr; 425 426 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 427 #if 0 428 device_printf(dev, "virtqueue paddr 0x%08lx\n", 429 (uint64_t)paddr); 430 #endif 431 if (sc->vtmmio_version == 1) { 432 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 433 VIRTIO_MMIO_VRING_ALIGN); 434 paddr = virtqueue_paddr(vq); 435 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 436 paddr >> PAGE_SHIFT); 437 } else { 438 paddr = virtqueue_desc_paddr(vq); 439 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_LOW, 440 paddr); 441 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_HIGH, 442 ((uint64_t)paddr) >> 32); 443 444 paddr = virtqueue_avail_paddr(vq); 445 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_LOW, 446 paddr); 447 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_HIGH, 448 ((uint64_t)paddr) >> 32); 449 450 paddr = virtqueue_used_paddr(vq); 451 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_LOW, 452 paddr); 453 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_HIGH, 454 ((uint64_t)paddr) >> 32); 455 456 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 1); 457 } 458 } 459 460 static int 461 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, 462 struct vq_alloc_info *vq_info) 463 { 464 struct vtmmio_virtqueue *vqx; 465 struct vq_alloc_info *info; 466 struct vtmmio_softc *sc; 467 struct virtqueue *vq; 468 uint32_t size; 469 int idx, error; 470 471 sc = device_get_softc(dev); 472 473 if (sc->vtmmio_nvqs != 0) 474 return (EALREADY); 475 if (nvqs <= 0) 476 return (EINVAL); 477 478 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue), 479 M_DEVBUF, M_NOWAIT | M_ZERO); 480 if (sc->vtmmio_vqs == NULL) 481 return (ENOMEM); 482 483 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 484 (1 << PAGE_SHIFT)); 485 486 for (idx = 0; idx < nvqs; idx++) { 487 vqx = &sc->vtmmio_vqs[idx]; 488 info = &vq_info[idx]; 489 490 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 491 492 vtmmio_select_virtqueue(sc, idx); 493 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 494 495 error = virtqueue_alloc(dev, idx, size, 496 VIRTIO_MMIO_VRING_ALIGN, ~(vm_paddr_t)0, info, &vq); 497 if (error) { 498 device_printf(dev, 499 "cannot allocate virtqueue %d: %d\n", 500 idx, error); 501 break; 502 } 503 504 vtmmio_set_virtqueue(sc, vq, size); 505 506 vqx->vtv_vq = *info->vqai_vq = vq; 507 vqx->vtv_no_intr = info->vqai_intr == NULL; 508 509 sc->vtmmio_nvqs++; 510 } 511 512 if (error) 513 vtmmio_free_virtqueues(sc); 514 515 return (error); 516 } 517 518 static void 519 vtmmio_stop(device_t dev) 520 { 521 522 vtmmio_reset(device_get_softc(dev)); 523 } 524 525 static void 526 vtmmio_poll(device_t dev) 527 { 528 struct vtmmio_softc *sc; 529 530 sc = device_get_softc(dev); 531 532 if (sc->platform != NULL) 533 VIRTIO_MMIO_POLL(sc->platform); 534 } 535 536 static int 537 vtmmio_reinit(device_t dev, uint64_t features) 538 { 539 struct vtmmio_softc *sc; 540 int idx, error; 541 542 sc = device_get_softc(dev); 543 544 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 545 vtmmio_stop(dev); 546 547 /* 548 * Quickly drive the status through ACK and DRIVER. The device 549 * does not become usable again until vtmmio_reinit_complete(). 550 */ 551 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 552 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 553 554 vtmmio_negotiate_features(dev, features); 555 556 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 557 (1 << PAGE_SHIFT)); 558 559 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 560 error = vtmmio_reinit_virtqueue(sc, idx); 561 if (error) 562 return (error); 563 } 564 565 return (0); 566 } 567 568 static void 569 vtmmio_reinit_complete(device_t dev) 570 { 571 572 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 573 } 574 575 static void 576 vtmmio_notify_virtqueue(device_t dev, uint16_t queue) 577 { 578 struct vtmmio_softc *sc; 579 580 sc = device_get_softc(dev); 581 582 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue); 583 } 584 585 static uint8_t 586 vtmmio_get_status(device_t dev) 587 { 588 struct vtmmio_softc *sc; 589 590 sc = device_get_softc(dev); 591 592 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS)); 593 } 594 595 static void 596 vtmmio_set_status(device_t dev, uint8_t status) 597 { 598 struct vtmmio_softc *sc; 599 600 sc = device_get_softc(dev); 601 602 if (status != VIRTIO_CONFIG_STATUS_RESET) 603 status |= vtmmio_get_status(dev); 604 605 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status); 606 } 607 608 static void 609 vtmmio_read_dev_config(device_t dev, bus_size_t offset, 610 void *dst, int length) 611 { 612 struct vtmmio_softc *sc; 613 bus_size_t off; 614 uint8_t *d; 615 int size; 616 uint64_t low32, high32; 617 618 sc = device_get_softc(dev); 619 off = VIRTIO_MMIO_CONFIG + offset; 620 621 /* 622 * The non-legacy MMIO specification adds the following restriction: 623 * 624 * 4.2.2.2: For the device-specific configuration space, the driver 625 * MUST use 8 bit wide accesses for 8 bit wide fields, 16 bit wide 626 * and aligned accesses for 16 bit wide fields and 32 bit wide and 627 * aligned accesses for 32 and 64 bit wide fields. 628 * 629 * The endianness also varies between non-legacy and legacy: 630 * 631 * 2.4: Note: The device configuration space uses the little-endian 632 * format for multi-byte fields. 633 * 634 * 2.4.3: Note that for legacy interfaces, device configuration space 635 * is generally the guest’s native endian, rather than PCI’s 636 * little-endian. The correct endian-ness is documented for each 637 * device. 638 */ 639 if (sc->vtmmio_version > 1) { 640 switch (length) { 641 case 1: 642 *(uint8_t *)dst = vtmmio_read_config_1(sc, off); 643 break; 644 case 2: 645 *(uint16_t *)dst = 646 le16toh(vtmmio_read_config_2(sc, off)); 647 break; 648 case 4: 649 *(uint32_t *)dst = 650 le32toh(vtmmio_read_config_4(sc, off)); 651 break; 652 case 8: 653 low32 = le32toh(vtmmio_read_config_4(sc, off)); 654 high32 = le32toh(vtmmio_read_config_4(sc, off + 4)); 655 *(uint64_t *)dst = (high32 << 32) | low32; 656 break; 657 default: 658 panic("%s: invalid length %d\n", __func__, length); 659 } 660 661 return; 662 } 663 664 for (d = dst; length > 0; d += size, off += size, length -= size) { 665 #ifdef ALLOW_WORD_ALIGNED_ACCESS 666 if (length >= 4) { 667 size = 4; 668 *(uint32_t *)d = vtmmio_read_config_4(sc, off); 669 } else if (length >= 2) { 670 size = 2; 671 *(uint16_t *)d = vtmmio_read_config_2(sc, off); 672 } else 673 #endif 674 { 675 size = 1; 676 *d = vtmmio_read_config_1(sc, off); 677 } 678 } 679 } 680 681 static void 682 vtmmio_write_dev_config(device_t dev, bus_size_t offset, 683 void *src, int length) 684 { 685 struct vtmmio_softc *sc; 686 bus_size_t off; 687 uint8_t *s; 688 int size; 689 690 sc = device_get_softc(dev); 691 off = VIRTIO_MMIO_CONFIG + offset; 692 693 /* 694 * The non-legacy MMIO specification adds size and alignment 695 * restrctions. It also changes the endianness from native-endian to 696 * little-endian. See vtmmio_read_dev_config. 697 */ 698 if (sc->vtmmio_version > 1) { 699 switch (length) { 700 case 1: 701 vtmmio_write_config_1(sc, off, *(uint8_t *)src); 702 break; 703 case 2: 704 vtmmio_write_config_2(sc, off, 705 htole16(*(uint16_t *)src)); 706 break; 707 case 4: 708 vtmmio_write_config_4(sc, off, 709 htole32(*(uint32_t *)src)); 710 break; 711 case 8: 712 vtmmio_write_config_4(sc, off, 713 htole32(*(uint64_t *)src)); 714 vtmmio_write_config_4(sc, off + 4, 715 htole32((*(uint64_t *)src) >> 32)); 716 break; 717 default: 718 panic("%s: invalid length %d\n", __func__, length); 719 } 720 721 return; 722 } 723 724 for (s = src; length > 0; s += size, off += size, length -= size) { 725 #ifdef ALLOW_WORD_ALIGNED_ACCESS 726 if (length >= 4) { 727 size = 4; 728 vtmmio_write_config_4(sc, off, *(uint32_t *)s); 729 } else if (length >= 2) { 730 size = 2; 731 vtmmio_write_config_2(sc, off, *(uint16_t *)s); 732 } else 733 #endif 734 { 735 size = 1; 736 vtmmio_write_config_1(sc, off, *s); 737 } 738 } 739 } 740 741 static void 742 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg, 743 uint64_t features) 744 { 745 device_t dev, child; 746 747 dev = sc->dev; 748 child = sc->vtmmio_child_dev; 749 750 if (device_is_attached(child) || bootverbose == 0) 751 return; 752 753 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc); 754 } 755 756 static void 757 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc) 758 { 759 device_t dev, child; 760 761 dev = sc->dev; 762 child = sc->vtmmio_child_dev; 763 764 if (child == NULL) 765 return; 766 767 if (device_get_state(child) != DS_NOTPRESENT) { 768 return; 769 } 770 771 if (device_probe(child) != 0) { 772 return; 773 } 774 775 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 776 if (device_attach(child) != 0) { 777 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 778 vtmmio_reset(sc); 779 vtmmio_release_child_resources(sc); 780 /* Reset status for future attempt. */ 781 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 782 } else { 783 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 784 VIRTIO_ATTACH_COMPLETED(child); 785 } 786 } 787 788 static int 789 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx) 790 { 791 struct vtmmio_virtqueue *vqx; 792 struct virtqueue *vq; 793 int error; 794 uint16_t size; 795 796 vqx = &sc->vtmmio_vqs[idx]; 797 vq = vqx->vtv_vq; 798 799 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 800 801 vtmmio_select_virtqueue(sc, idx); 802 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 803 804 error = virtqueue_reinit(vq, size); 805 if (error) 806 return (error); 807 808 vtmmio_set_virtqueue(sc, vq, size); 809 810 return (0); 811 } 812 813 static void 814 vtmmio_free_interrupts(struct vtmmio_softc *sc) 815 { 816 817 if (sc->ih != NULL) 818 bus_teardown_intr(sc->dev, sc->res[1], sc->ih); 819 820 if (sc->res[1] != NULL) 821 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]); 822 } 823 824 static void 825 vtmmio_free_virtqueues(struct vtmmio_softc *sc) 826 { 827 struct vtmmio_virtqueue *vqx; 828 int idx; 829 830 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 831 vqx = &sc->vtmmio_vqs[idx]; 832 833 vtmmio_select_virtqueue(sc, idx); 834 if (sc->vtmmio_version == 1) 835 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0); 836 else 837 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 0); 838 839 virtqueue_free(vqx->vtv_vq); 840 vqx->vtv_vq = NULL; 841 } 842 843 free(sc->vtmmio_vqs, M_DEVBUF); 844 sc->vtmmio_vqs = NULL; 845 sc->vtmmio_nvqs = 0; 846 } 847 848 static void 849 vtmmio_release_child_resources(struct vtmmio_softc *sc) 850 { 851 852 vtmmio_free_interrupts(sc); 853 vtmmio_free_virtqueues(sc); 854 } 855 856 static void 857 vtmmio_reset(struct vtmmio_softc *sc) 858 { 859 860 /* 861 * Setting the status to RESET sets the host device to 862 * the original, uninitialized state. 863 */ 864 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET); 865 } 866 867 static void 868 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx) 869 { 870 871 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 872 } 873 874 static void 875 vtmmio_vq_intr(void *arg) 876 { 877 struct vtmmio_virtqueue *vqx; 878 struct vtmmio_softc *sc; 879 struct virtqueue *vq; 880 uint32_t status; 881 int idx; 882 883 sc = arg; 884 885 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS); 886 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status); 887 888 /* The config changed */ 889 if (status & VIRTIO_MMIO_INT_CONFIG) 890 if (sc->vtmmio_child_dev != NULL) 891 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev); 892 893 /* Notify all virtqueues. */ 894 if (status & VIRTIO_MMIO_INT_VRING) { 895 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 896 vqx = &sc->vtmmio_vqs[idx]; 897 if (vqx->vtv_no_intr == 0) { 898 vq = vqx->vtv_vq; 899 virtqueue_intr(vq); 900 } 901 } 902 } 903 } 904