1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Portions of this software were developed by Andrew Turner 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * VirtIO MMIO interface. 37 * This driver is heavily based on VirtIO PCI interface driver. 38 */ 39 40 #include <sys/cdefs.h> 41 __FBSDID("$FreeBSD$"); 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/kernel.h> 47 #include <sys/module.h> 48 #include <sys/malloc.h> 49 #include <sys/rman.h> 50 #include <sys/endian.h> 51 52 #include <machine/bus.h> 53 #include <machine/resource.h> 54 55 #include <dev/virtio/virtio.h> 56 #include <dev/virtio/virtqueue.h> 57 #include <dev/virtio/mmio/virtio_mmio.h> 58 59 #include "virtio_mmio_if.h" 60 #include "virtio_bus_if.h" 61 #include "virtio_if.h" 62 63 struct vtmmio_virtqueue { 64 struct virtqueue *vtv_vq; 65 int vtv_no_intr; 66 }; 67 68 static int vtmmio_detach(device_t); 69 static int vtmmio_suspend(device_t); 70 static int vtmmio_resume(device_t); 71 static int vtmmio_shutdown(device_t); 72 static void vtmmio_driver_added(device_t, driver_t *); 73 static void vtmmio_child_detached(device_t, device_t); 74 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *); 75 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t); 76 static uint64_t vtmmio_negotiate_features(device_t, uint64_t); 77 static int vtmmio_with_feature(device_t, uint64_t); 78 static void vtmmio_set_virtqueue(struct vtmmio_softc *sc, 79 struct virtqueue *vq, uint32_t size); 80 static int vtmmio_alloc_virtqueues(device_t, int, int, 81 struct vq_alloc_info *); 82 static int vtmmio_setup_intr(device_t, enum intr_type); 83 static void vtmmio_stop(device_t); 84 static void vtmmio_poll(device_t); 85 static int vtmmio_reinit(device_t, uint64_t); 86 static void vtmmio_reinit_complete(device_t); 87 static void vtmmio_notify_virtqueue(device_t, uint16_t, bus_size_t); 88 static uint8_t vtmmio_get_status(device_t); 89 static void vtmmio_set_status(device_t, uint8_t); 90 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); 91 static void vtmmio_write_dev_config(device_t, bus_size_t, const void *, int); 92 static void vtmmio_describe_features(struct vtmmio_softc *, const char *, 93 uint64_t); 94 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *); 95 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int); 96 static void vtmmio_free_interrupts(struct vtmmio_softc *); 97 static void vtmmio_free_virtqueues(struct vtmmio_softc *); 98 static void vtmmio_release_child_resources(struct vtmmio_softc *); 99 static void vtmmio_reset(struct vtmmio_softc *); 100 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int); 101 static void vtmmio_vq_intr(void *); 102 103 /* 104 * I/O port read/write wrappers. 105 */ 106 #define vtmmio_write_config_1(sc, o, v) \ 107 do { \ 108 if (sc->platform != NULL) \ 109 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 110 bus_write_1((sc)->res[0], (o), (v)); \ 111 if (sc->platform != NULL) \ 112 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 113 } while (0) 114 #define vtmmio_write_config_2(sc, o, v) \ 115 do { \ 116 if (sc->platform != NULL) \ 117 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 118 bus_write_2((sc)->res[0], (o), (v)); \ 119 if (sc->platform != NULL) \ 120 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 121 } while (0) 122 #define vtmmio_write_config_4(sc, o, v) \ 123 do { \ 124 if (sc->platform != NULL) \ 125 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 126 bus_write_4((sc)->res[0], (o), (v)); \ 127 if (sc->platform != NULL) \ 128 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 129 } while (0) 130 131 #define vtmmio_read_config_1(sc, o) \ 132 bus_read_1((sc)->res[0], (o)) 133 #define vtmmio_read_config_2(sc, o) \ 134 bus_read_2((sc)->res[0], (o)) 135 #define vtmmio_read_config_4(sc, o) \ 136 bus_read_4((sc)->res[0], (o)) 137 138 static device_method_t vtmmio_methods[] = { 139 /* Device interface. */ 140 DEVMETHOD(device_attach, vtmmio_attach), 141 DEVMETHOD(device_detach, vtmmio_detach), 142 DEVMETHOD(device_suspend, vtmmio_suspend), 143 DEVMETHOD(device_resume, vtmmio_resume), 144 DEVMETHOD(device_shutdown, vtmmio_shutdown), 145 146 /* Bus interface. */ 147 DEVMETHOD(bus_driver_added, vtmmio_driver_added), 148 DEVMETHOD(bus_child_detached, vtmmio_child_detached), 149 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), 150 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar), 151 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar), 152 153 /* VirtIO bus interface. */ 154 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features), 155 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature), 156 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), 157 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), 158 DEVMETHOD(virtio_bus_stop, vtmmio_stop), 159 DEVMETHOD(virtio_bus_poll, vtmmio_poll), 160 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), 161 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), 162 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), 163 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config), 164 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config), 165 166 DEVMETHOD_END 167 }; 168 169 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods, 170 sizeof(struct vtmmio_softc)); 171 172 MODULE_VERSION(virtio_mmio, 1); 173 174 int 175 vtmmio_probe(device_t dev) 176 { 177 struct vtmmio_softc *sc; 178 int rid; 179 uint32_t magic, version; 180 181 sc = device_get_softc(dev); 182 183 rid = 0; 184 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 185 RF_ACTIVE); 186 if (sc->res[0] == NULL) { 187 device_printf(dev, "Cannot allocate memory window.\n"); 188 return (ENXIO); 189 } 190 191 magic = vtmmio_read_config_4(sc, VIRTIO_MMIO_MAGIC_VALUE); 192 if (magic != VIRTIO_MMIO_MAGIC_VIRT) { 193 device_printf(dev, "Bad magic value %#x\n", magic); 194 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); 195 return (ENXIO); 196 } 197 198 version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION); 199 if (version < 1 || version > 2) { 200 device_printf(dev, "Unsupported version: %#x\n", version); 201 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); 202 return (ENXIO); 203 } 204 205 if (vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID) == 0) { 206 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); 207 return (ENXIO); 208 } 209 210 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); 211 212 device_set_desc(dev, "VirtIO MMIO adapter"); 213 return (BUS_PROBE_DEFAULT); 214 } 215 216 static int 217 vtmmio_setup_intr(device_t dev, enum intr_type type) 218 { 219 struct vtmmio_softc *sc; 220 int rid; 221 int err; 222 223 sc = device_get_softc(dev); 224 225 if (sc->platform != NULL) { 226 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, 227 vtmmio_vq_intr, sc); 228 if (err == 0) { 229 /* Okay we have backend-specific interrupts */ 230 return (0); 231 } 232 } 233 234 rid = 0; 235 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 236 RF_ACTIVE); 237 if (!sc->res[1]) { 238 device_printf(dev, "Can't allocate interrupt\n"); 239 return (ENXIO); 240 } 241 242 if (bus_setup_intr(dev, sc->res[1], type | INTR_MPSAFE, 243 NULL, vtmmio_vq_intr, sc, &sc->ih)) { 244 device_printf(dev, "Can't setup the interrupt\n"); 245 return (ENXIO); 246 } 247 248 return (0); 249 } 250 251 int 252 vtmmio_attach(device_t dev) 253 { 254 struct vtmmio_softc *sc; 255 device_t child; 256 int rid; 257 258 sc = device_get_softc(dev); 259 sc->dev = dev; 260 261 rid = 0; 262 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 263 RF_ACTIVE); 264 if (sc->res[0] == NULL) { 265 device_printf(dev, "Cannot allocate memory window.\n"); 266 return (ENXIO); 267 } 268 269 sc->vtmmio_version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION); 270 271 vtmmio_reset(sc); 272 273 /* Tell the host we've noticed this device. */ 274 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 275 276 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 277 device_printf(dev, "Cannot create child device.\n"); 278 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 279 vtmmio_detach(dev); 280 return (ENOMEM); 281 } 282 283 sc->vtmmio_child_dev = child; 284 vtmmio_probe_and_attach_child(sc); 285 286 return (0); 287 } 288 289 static int 290 vtmmio_detach(device_t dev) 291 { 292 struct vtmmio_softc *sc; 293 device_t child; 294 int error; 295 296 sc = device_get_softc(dev); 297 298 if ((child = sc->vtmmio_child_dev) != NULL) { 299 error = device_delete_child(dev, child); 300 if (error) 301 return (error); 302 sc->vtmmio_child_dev = NULL; 303 } 304 305 vtmmio_reset(sc); 306 307 if (sc->res[0] != NULL) { 308 bus_release_resource(dev, SYS_RES_MEMORY, 0, 309 sc->res[0]); 310 sc->res[0] = NULL; 311 } 312 313 return (0); 314 } 315 316 static int 317 vtmmio_suspend(device_t dev) 318 { 319 320 return (bus_generic_suspend(dev)); 321 } 322 323 static int 324 vtmmio_resume(device_t dev) 325 { 326 327 return (bus_generic_resume(dev)); 328 } 329 330 static int 331 vtmmio_shutdown(device_t dev) 332 { 333 334 (void) bus_generic_shutdown(dev); 335 336 /* Forcibly stop the host device. */ 337 vtmmio_stop(dev); 338 339 return (0); 340 } 341 342 static void 343 vtmmio_driver_added(device_t dev, driver_t *driver) 344 { 345 struct vtmmio_softc *sc; 346 347 sc = device_get_softc(dev); 348 349 vtmmio_probe_and_attach_child(sc); 350 } 351 352 static void 353 vtmmio_child_detached(device_t dev, device_t child) 354 { 355 struct vtmmio_softc *sc; 356 357 sc = device_get_softc(dev); 358 359 vtmmio_reset(sc); 360 vtmmio_release_child_resources(sc); 361 } 362 363 static int 364 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 365 { 366 struct vtmmio_softc *sc; 367 368 sc = device_get_softc(dev); 369 370 if (sc->vtmmio_child_dev != child) 371 return (ENOENT); 372 373 switch (index) { 374 case VIRTIO_IVAR_DEVTYPE: 375 case VIRTIO_IVAR_SUBDEVICE: 376 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID); 377 break; 378 case VIRTIO_IVAR_VENDOR: 379 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID); 380 break; 381 case VIRTIO_IVAR_SUBVENDOR: 382 case VIRTIO_IVAR_DEVICE: 383 /* 384 * Dummy value for fields not present in this bus. Used by 385 * bus-agnostic virtio_child_pnpinfo. 386 */ 387 *result = 0; 388 break; 389 case VIRTIO_IVAR_MODERN: 390 /* 391 * There are several modern (aka MMIO v2) spec compliance 392 * issues with this driver, but keep the status quo. 393 */ 394 *result = sc->vtmmio_version > 1; 395 break; 396 default: 397 return (ENOENT); 398 } 399 400 return (0); 401 } 402 403 static int 404 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 405 { 406 struct vtmmio_softc *sc; 407 408 sc = device_get_softc(dev); 409 410 if (sc->vtmmio_child_dev != child) 411 return (ENOENT); 412 413 switch (index) { 414 case VIRTIO_IVAR_FEATURE_DESC: 415 sc->vtmmio_child_feat_desc = (void *) value; 416 break; 417 default: 418 return (ENOENT); 419 } 420 421 return (0); 422 } 423 424 static uint64_t 425 vtmmio_negotiate_features(device_t dev, uint64_t child_features) 426 { 427 struct vtmmio_softc *sc; 428 uint64_t host_features, features; 429 430 sc = device_get_softc(dev); 431 432 if (sc->vtmmio_version > 1) { 433 child_features |= VIRTIO_F_VERSION_1; 434 } 435 436 vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 1); 437 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 438 host_features <<= 32; 439 440 vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 0); 441 host_features |= vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 442 443 vtmmio_describe_features(sc, "host", host_features); 444 445 /* 446 * Limit negotiated features to what the driver, virtqueue, and 447 * host all support. 448 */ 449 features = host_features & child_features; 450 features = virtio_filter_transport_features(features); 451 sc->vtmmio_features = features; 452 453 vtmmio_describe_features(sc, "negotiated", features); 454 455 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES_SEL, 1); 456 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features >> 32); 457 458 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES_SEL, 0); 459 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features); 460 461 return (features); 462 } 463 464 static int 465 vtmmio_with_feature(device_t dev, uint64_t feature) 466 { 467 struct vtmmio_softc *sc; 468 469 sc = device_get_softc(dev); 470 471 return ((sc->vtmmio_features & feature) != 0); 472 } 473 474 static void 475 vtmmio_set_virtqueue(struct vtmmio_softc *sc, struct virtqueue *vq, 476 uint32_t size) 477 { 478 vm_paddr_t paddr; 479 480 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 481 482 if (sc->vtmmio_version == 1) { 483 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 484 VIRTIO_MMIO_VRING_ALIGN); 485 paddr = virtqueue_paddr(vq); 486 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 487 paddr >> PAGE_SHIFT); 488 } else { 489 paddr = virtqueue_desc_paddr(vq); 490 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_LOW, 491 paddr); 492 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_HIGH, 493 ((uint64_t)paddr) >> 32); 494 495 paddr = virtqueue_avail_paddr(vq); 496 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_LOW, 497 paddr); 498 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_HIGH, 499 ((uint64_t)paddr) >> 32); 500 501 paddr = virtqueue_used_paddr(vq); 502 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_LOW, 503 paddr); 504 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_HIGH, 505 ((uint64_t)paddr) >> 32); 506 507 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 1); 508 } 509 } 510 511 static int 512 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, 513 struct vq_alloc_info *vq_info) 514 { 515 struct vtmmio_virtqueue *vqx; 516 struct vq_alloc_info *info; 517 struct vtmmio_softc *sc; 518 struct virtqueue *vq; 519 uint32_t size; 520 int idx, error; 521 522 sc = device_get_softc(dev); 523 524 if (sc->vtmmio_nvqs != 0) 525 return (EALREADY); 526 if (nvqs <= 0) 527 return (EINVAL); 528 529 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue), 530 M_DEVBUF, M_NOWAIT | M_ZERO); 531 if (sc->vtmmio_vqs == NULL) 532 return (ENOMEM); 533 534 if (sc->vtmmio_version == 1) { 535 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 536 (1 << PAGE_SHIFT)); 537 } 538 539 for (idx = 0; idx < nvqs; idx++) { 540 vqx = &sc->vtmmio_vqs[idx]; 541 info = &vq_info[idx]; 542 543 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 544 545 vtmmio_select_virtqueue(sc, idx); 546 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 547 548 error = virtqueue_alloc(dev, idx, size, 549 VIRTIO_MMIO_QUEUE_NOTIFY, VIRTIO_MMIO_VRING_ALIGN, 550 ~(vm_paddr_t)0, info, &vq); 551 if (error) { 552 device_printf(dev, 553 "cannot allocate virtqueue %d: %d\n", 554 idx, error); 555 break; 556 } 557 558 vtmmio_set_virtqueue(sc, vq, size); 559 560 vqx->vtv_vq = *info->vqai_vq = vq; 561 vqx->vtv_no_intr = info->vqai_intr == NULL; 562 563 sc->vtmmio_nvqs++; 564 } 565 566 if (error) 567 vtmmio_free_virtqueues(sc); 568 569 return (error); 570 } 571 572 static void 573 vtmmio_stop(device_t dev) 574 { 575 576 vtmmio_reset(device_get_softc(dev)); 577 } 578 579 static void 580 vtmmio_poll(device_t dev) 581 { 582 struct vtmmio_softc *sc; 583 584 sc = device_get_softc(dev); 585 586 if (sc->platform != NULL) 587 VIRTIO_MMIO_POLL(sc->platform); 588 } 589 590 static int 591 vtmmio_reinit(device_t dev, uint64_t features) 592 { 593 struct vtmmio_softc *sc; 594 int idx, error; 595 596 sc = device_get_softc(dev); 597 598 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 599 vtmmio_stop(dev); 600 601 /* 602 * Quickly drive the status through ACK and DRIVER. The device 603 * does not become usable again until vtmmio_reinit_complete(). 604 */ 605 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 606 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 607 608 vtmmio_negotiate_features(dev, features); 609 610 if (sc->vtmmio_version == 1) { 611 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 612 (1 << PAGE_SHIFT)); 613 } 614 615 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 616 error = vtmmio_reinit_virtqueue(sc, idx); 617 if (error) 618 return (error); 619 } 620 621 return (0); 622 } 623 624 static void 625 vtmmio_reinit_complete(device_t dev) 626 { 627 628 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 629 } 630 631 static void 632 vtmmio_notify_virtqueue(device_t dev, uint16_t queue, bus_size_t offset) 633 { 634 struct vtmmio_softc *sc; 635 636 sc = device_get_softc(dev); 637 MPASS(offset == VIRTIO_MMIO_QUEUE_NOTIFY); 638 639 vtmmio_write_config_4(sc, offset, queue); 640 } 641 642 static uint8_t 643 vtmmio_get_status(device_t dev) 644 { 645 struct vtmmio_softc *sc; 646 647 sc = device_get_softc(dev); 648 649 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS)); 650 } 651 652 static void 653 vtmmio_set_status(device_t dev, uint8_t status) 654 { 655 struct vtmmio_softc *sc; 656 657 sc = device_get_softc(dev); 658 659 if (status != VIRTIO_CONFIG_STATUS_RESET) 660 status |= vtmmio_get_status(dev); 661 662 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status); 663 } 664 665 static void 666 vtmmio_read_dev_config(device_t dev, bus_size_t offset, 667 void *dst, int length) 668 { 669 struct vtmmio_softc *sc; 670 bus_size_t off; 671 uint8_t *d; 672 int size; 673 uint64_t low32, high32; 674 675 sc = device_get_softc(dev); 676 off = VIRTIO_MMIO_CONFIG + offset; 677 678 /* 679 * The non-legacy MMIO specification adds the following restriction: 680 * 681 * 4.2.2.2: For the device-specific configuration space, the driver 682 * MUST use 8 bit wide accesses for 8 bit wide fields, 16 bit wide 683 * and aligned accesses for 16 bit wide fields and 32 bit wide and 684 * aligned accesses for 32 and 64 bit wide fields. 685 * 686 * The endianness also varies between non-legacy and legacy: 687 * 688 * 2.4: Note: The device configuration space uses the little-endian 689 * format for multi-byte fields. 690 * 691 * 2.4.3: Note that for legacy interfaces, device configuration space 692 * is generally the guest’s native endian, rather than PCI’s 693 * little-endian. The correct endian-ness is documented for each 694 * device. 695 */ 696 if (sc->vtmmio_version > 1) { 697 switch (length) { 698 case 1: 699 *(uint8_t *)dst = vtmmio_read_config_1(sc, off); 700 break; 701 case 2: 702 *(uint16_t *)dst = 703 le16toh(vtmmio_read_config_2(sc, off)); 704 break; 705 case 4: 706 *(uint32_t *)dst = 707 le32toh(vtmmio_read_config_4(sc, off)); 708 break; 709 case 8: 710 low32 = le32toh(vtmmio_read_config_4(sc, off)); 711 high32 = le32toh(vtmmio_read_config_4(sc, off + 4)); 712 *(uint64_t *)dst = (high32 << 32) | low32; 713 break; 714 default: 715 panic("%s: invalid length %d\n", __func__, length); 716 } 717 718 return; 719 } 720 721 for (d = dst; length > 0; d += size, off += size, length -= size) { 722 #ifdef ALLOW_WORD_ALIGNED_ACCESS 723 if (length >= 4) { 724 size = 4; 725 *(uint32_t *)d = vtmmio_read_config_4(sc, off); 726 } else if (length >= 2) { 727 size = 2; 728 *(uint16_t *)d = vtmmio_read_config_2(sc, off); 729 } else 730 #endif 731 { 732 size = 1; 733 *d = vtmmio_read_config_1(sc, off); 734 } 735 } 736 } 737 738 static void 739 vtmmio_write_dev_config(device_t dev, bus_size_t offset, 740 const void *src, int length) 741 { 742 struct vtmmio_softc *sc; 743 bus_size_t off; 744 const uint8_t *s; 745 int size; 746 747 sc = device_get_softc(dev); 748 off = VIRTIO_MMIO_CONFIG + offset; 749 750 /* 751 * The non-legacy MMIO specification adds size and alignment 752 * restrctions. It also changes the endianness from native-endian to 753 * little-endian. See vtmmio_read_dev_config. 754 */ 755 if (sc->vtmmio_version > 1) { 756 switch (length) { 757 case 1: 758 vtmmio_write_config_1(sc, off, *(const uint8_t *)src); 759 break; 760 case 2: 761 vtmmio_write_config_2(sc, off, 762 htole16(*(const uint16_t *)src)); 763 break; 764 case 4: 765 vtmmio_write_config_4(sc, off, 766 htole32(*(const uint32_t *)src)); 767 break; 768 case 8: 769 vtmmio_write_config_4(sc, off, 770 htole32(*(const uint64_t *)src)); 771 vtmmio_write_config_4(sc, off + 4, 772 htole32((*(const uint64_t *)src) >> 32)); 773 break; 774 default: 775 panic("%s: invalid length %d\n", __func__, length); 776 } 777 778 return; 779 } 780 781 for (s = src; length > 0; s += size, off += size, length -= size) { 782 #ifdef ALLOW_WORD_ALIGNED_ACCESS 783 if (length >= 4) { 784 size = 4; 785 vtmmio_write_config_4(sc, off, *(uint32_t *)s); 786 } else if (length >= 2) { 787 size = 2; 788 vtmmio_write_config_2(sc, off, *(uint16_t *)s); 789 } else 790 #endif 791 { 792 size = 1; 793 vtmmio_write_config_1(sc, off, *s); 794 } 795 } 796 } 797 798 static void 799 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg, 800 uint64_t features) 801 { 802 device_t dev, child; 803 804 dev = sc->dev; 805 child = sc->vtmmio_child_dev; 806 807 if (device_is_attached(child) || bootverbose == 0) 808 return; 809 810 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc); 811 } 812 813 static void 814 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc) 815 { 816 device_t dev, child; 817 818 dev = sc->dev; 819 child = sc->vtmmio_child_dev; 820 821 if (child == NULL) 822 return; 823 824 if (device_get_state(child) != DS_NOTPRESENT) { 825 return; 826 } 827 828 if (device_probe(child) != 0) { 829 return; 830 } 831 832 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 833 if (device_attach(child) != 0) { 834 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 835 vtmmio_reset(sc); 836 vtmmio_release_child_resources(sc); 837 /* Reset status for future attempt. */ 838 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 839 } else { 840 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 841 VIRTIO_ATTACH_COMPLETED(child); 842 } 843 } 844 845 static int 846 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx) 847 { 848 struct vtmmio_virtqueue *vqx; 849 struct virtqueue *vq; 850 int error; 851 uint16_t size; 852 853 vqx = &sc->vtmmio_vqs[idx]; 854 vq = vqx->vtv_vq; 855 856 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 857 858 vtmmio_select_virtqueue(sc, idx); 859 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 860 861 error = virtqueue_reinit(vq, size); 862 if (error) 863 return (error); 864 865 vtmmio_set_virtqueue(sc, vq, size); 866 867 return (0); 868 } 869 870 static void 871 vtmmio_free_interrupts(struct vtmmio_softc *sc) 872 { 873 874 if (sc->ih != NULL) 875 bus_teardown_intr(sc->dev, sc->res[1], sc->ih); 876 877 if (sc->res[1] != NULL) 878 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]); 879 } 880 881 static void 882 vtmmio_free_virtqueues(struct vtmmio_softc *sc) 883 { 884 struct vtmmio_virtqueue *vqx; 885 int idx; 886 887 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 888 vqx = &sc->vtmmio_vqs[idx]; 889 890 vtmmio_select_virtqueue(sc, idx); 891 if (sc->vtmmio_version == 1) 892 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0); 893 else 894 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 0); 895 896 virtqueue_free(vqx->vtv_vq); 897 vqx->vtv_vq = NULL; 898 } 899 900 free(sc->vtmmio_vqs, M_DEVBUF); 901 sc->vtmmio_vqs = NULL; 902 sc->vtmmio_nvqs = 0; 903 } 904 905 static void 906 vtmmio_release_child_resources(struct vtmmio_softc *sc) 907 { 908 909 vtmmio_free_interrupts(sc); 910 vtmmio_free_virtqueues(sc); 911 } 912 913 static void 914 vtmmio_reset(struct vtmmio_softc *sc) 915 { 916 917 /* 918 * Setting the status to RESET sets the host device to 919 * the original, uninitialized state. 920 */ 921 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET); 922 } 923 924 static void 925 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx) 926 { 927 928 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 929 } 930 931 static void 932 vtmmio_vq_intr(void *arg) 933 { 934 struct vtmmio_virtqueue *vqx; 935 struct vtmmio_softc *sc; 936 struct virtqueue *vq; 937 uint32_t status; 938 int idx; 939 940 sc = arg; 941 942 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS); 943 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status); 944 945 /* The config changed */ 946 if (status & VIRTIO_MMIO_INT_CONFIG) 947 if (sc->vtmmio_child_dev != NULL) 948 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev); 949 950 /* Notify all virtqueues. */ 951 if (status & VIRTIO_MMIO_INT_VRING) { 952 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 953 vqx = &sc->vtmmio_vqs[idx]; 954 if (vqx->vtv_no_intr == 0) { 955 vq = vqx->vtv_vq; 956 virtqueue_intr(vq); 957 } 958 } 959 } 960 } 961