1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for the modern VirtIO PCI interface. */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/bus.h> 37 #include <sys/lock.h> 38 #include <sys/kernel.h> 39 #include <sys/module.h> 40 41 #include <machine/bus.h> 42 #include <machine/cpu.h> 43 #include <machine/resource.h> 44 #include <sys/bus.h> 45 #include <sys/rman.h> 46 47 #include <dev/pci/pcivar.h> 48 #include <dev/pci/pcireg.h> 49 50 #include <dev/virtio/virtio.h> 51 #include <dev/virtio/virtqueue.h> 52 #include <dev/virtio/pci/virtio_pci.h> 53 #include <dev/virtio/pci/virtio_pci_modern_var.h> 54 55 #include "virtio_bus_if.h" 56 #include "virtio_pci_if.h" 57 #include "virtio_if.h" 58 59 struct vtpci_modern_resource_map { 60 struct resource_map vtrm_map; 61 int vtrm_cap_offset; 62 int vtrm_bar; 63 int vtrm_offset; 64 int vtrm_length; 65 int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */ 66 }; 67 68 struct vtpci_modern_bar_resource { 69 struct resource *vtbr_res; 70 int vtbr_type; 71 }; 72 73 struct vtpci_modern_softc { 74 device_t vtpci_dev; 75 struct vtpci_common vtpci_common; 76 uint32_t vtpci_notify_offset_multiplier; 77 uint16_t vtpci_devid; 78 int vtpci_msix_bar; 79 struct resource *vtpci_msix_res; 80 81 struct vtpci_modern_resource_map vtpci_common_res_map; 82 struct vtpci_modern_resource_map vtpci_notify_res_map; 83 struct vtpci_modern_resource_map vtpci_isr_res_map; 84 struct vtpci_modern_resource_map vtpci_device_res_map; 85 86 #define VTPCI_MODERN_MAX_BARS 6 87 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS]; 88 }; 89 90 static int vtpci_modern_probe(device_t); 91 static int vtpci_modern_attach(device_t); 92 static int vtpci_modern_detach(device_t); 93 static int vtpci_modern_suspend(device_t); 94 static int vtpci_modern_resume(device_t); 95 static int vtpci_modern_shutdown(device_t); 96 97 static void vtpci_modern_driver_added(device_t, driver_t *); 98 static void vtpci_modern_child_detached(device_t, device_t); 99 static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *); 100 static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t); 101 102 static uint8_t vtpci_modern_read_isr(device_t); 103 static uint16_t vtpci_modern_get_vq_size(device_t, int); 104 static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int); 105 static void vtpci_modern_set_vq(device_t, struct virtqueue *); 106 static void vtpci_modern_disable_vq(device_t, int); 107 static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int, 108 struct vtpci_interrupt *); 109 static int vtpci_modern_register_cfg_msix(device_t, 110 struct vtpci_interrupt *); 111 static int vtpci_modern_register_vq_msix(device_t, int idx, 112 struct vtpci_interrupt *); 113 114 static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t); 115 static int vtpci_modern_finalize_features(device_t); 116 static int vtpci_modern_with_feature(device_t, uint64_t); 117 static int vtpci_modern_alloc_virtqueues(device_t, int, int, 118 struct vq_alloc_info *); 119 static int vtpci_modern_setup_interrupts(device_t, enum intr_type); 120 static void vtpci_modern_stop(device_t); 121 static int vtpci_modern_reinit(device_t, uint64_t); 122 static void vtpci_modern_reinit_complete(device_t); 123 static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t); 124 static int vtpci_modern_config_generation(device_t); 125 static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int); 126 static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int); 127 128 static int vtpci_modern_probe_configs(device_t); 129 static int vtpci_modern_find_cap(device_t, uint8_t, int *); 130 static int vtpci_modern_map_configs(struct vtpci_modern_softc *); 131 static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *); 132 static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *, 133 uint8_t, int, int, struct vtpci_modern_resource_map *); 134 static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int); 135 static struct resource *vtpci_modern_get_bar_resource( 136 struct vtpci_modern_softc *, int, int); 137 static struct resource *vtpci_modern_alloc_bar_resource( 138 struct vtpci_modern_softc *, int, int); 139 static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *); 140 static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *, 141 struct vtpci_modern_resource_map *); 142 static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *, 143 struct vtpci_modern_resource_map *); 144 static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *); 145 static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *); 146 147 static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *); 148 149 static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *); 150 static void vtpci_modern_write_features(struct vtpci_modern_softc *, 151 uint64_t); 152 static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int); 153 static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *); 154 static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t); 155 static void vtpci_modern_reset(struct vtpci_modern_softc *); 156 static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *); 157 158 static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *, 159 bus_size_t); 160 static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *, 161 bus_size_t); 162 static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *, 163 bus_size_t); 164 static void vtpci_modern_write_common_1(struct vtpci_modern_softc *, 165 bus_size_t, uint8_t); 166 static void vtpci_modern_write_common_2(struct vtpci_modern_softc *, 167 bus_size_t, uint16_t); 168 static void vtpci_modern_write_common_4(struct vtpci_modern_softc *, 169 bus_size_t, uint32_t); 170 static void vtpci_modern_write_common_8(struct vtpci_modern_softc *, 171 bus_size_t, uint64_t); 172 static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *, 173 bus_size_t, uint16_t); 174 static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *, 175 bus_size_t); 176 static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *, 177 bus_size_t); 178 static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *, 179 bus_size_t); 180 static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *, 181 bus_size_t); 182 static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *, 183 bus_size_t); 184 static void vtpci_modern_write_device_1(struct vtpci_modern_softc *, 185 bus_size_t, uint8_t); 186 static void vtpci_modern_write_device_2(struct vtpci_modern_softc *, 187 bus_size_t, uint16_t); 188 static void vtpci_modern_write_device_4(struct vtpci_modern_softc *, 189 bus_size_t, uint32_t); 190 static void vtpci_modern_write_device_8(struct vtpci_modern_softc *, 191 bus_size_t, uint64_t); 192 193 /* Tunables. */ 194 static int vtpci_modern_transitional = 0; 195 TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional); 196 197 static device_method_t vtpci_modern_methods[] = { 198 /* Device interface. */ 199 DEVMETHOD(device_probe, vtpci_modern_probe), 200 DEVMETHOD(device_attach, vtpci_modern_attach), 201 DEVMETHOD(device_detach, vtpci_modern_detach), 202 DEVMETHOD(device_suspend, vtpci_modern_suspend), 203 DEVMETHOD(device_resume, vtpci_modern_resume), 204 DEVMETHOD(device_shutdown, vtpci_modern_shutdown), 205 206 /* Bus interface. */ 207 DEVMETHOD(bus_driver_added, vtpci_modern_driver_added), 208 DEVMETHOD(bus_child_detached, vtpci_modern_child_detached), 209 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), 210 DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar), 211 DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar), 212 213 /* VirtIO PCI interface. */ 214 DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr), 215 DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size), 216 DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off), 217 DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq), 218 DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq), 219 DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix), 220 DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix), 221 222 /* VirtIO bus interface. */ 223 DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features), 224 DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features), 225 DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature), 226 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues), 227 DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts), 228 DEVMETHOD(virtio_bus_stop, vtpci_modern_stop), 229 DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit), 230 DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete), 231 DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq), 232 DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation), 233 DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config), 234 DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config), 235 236 DEVMETHOD_END 237 }; 238 239 static driver_t vtpci_modern_driver = { 240 .name = "virtio_pci", 241 .methods = vtpci_modern_methods, 242 .size = sizeof(struct vtpci_modern_softc) 243 }; 244 245 devclass_t vtpci_modern_devclass; 246 247 DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 248 vtpci_modern_devclass, 0, 0); 249 250 static int 251 vtpci_modern_probe(device_t dev) 252 { 253 char desc[64]; 254 const char *name; 255 uint16_t devid; 256 257 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 258 return (ENXIO); 259 260 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 261 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX) 262 return (ENXIO); 263 264 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) { 265 if (!vtpci_modern_transitional) 266 return (ENXIO); 267 devid = pci_get_subdevice(dev); 268 } else 269 devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; 270 271 if (vtpci_modern_probe_configs(dev) != 0) 272 return (ENXIO); 273 274 name = virtio_device_name(devid); 275 if (name == NULL) 276 name = "Unknown"; 277 278 snprintf(desc, sizeof(desc), "VirtIO PCI (modern) %s adapter", name); 279 device_set_desc_copy(dev, desc); 280 281 return (BUS_PROBE_DEFAULT); 282 } 283 284 static int 285 vtpci_modern_attach(device_t dev) 286 { 287 struct vtpci_modern_softc *sc; 288 int error; 289 290 sc = device_get_softc(dev); 291 sc->vtpci_dev = dev; 292 vtpci_init(&sc->vtpci_common, dev, true); 293 294 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) 295 sc->vtpci_devid = pci_get_subdevice(dev); 296 else 297 sc->vtpci_devid = pci_get_device(dev) - 298 VIRTIO_PCI_DEVICEID_MODERN_MIN; 299 300 error = vtpci_modern_map_configs(sc); 301 if (error) { 302 device_printf(dev, "cannot map configs\n"); 303 vtpci_modern_unmap_configs(sc); 304 return (error); 305 } 306 307 vtpci_modern_reset(sc); 308 309 /* Tell the host we've noticed this device. */ 310 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 311 312 error = vtpci_add_child(&sc->vtpci_common); 313 if (error) 314 goto fail; 315 316 vtpci_modern_probe_and_attach_child(sc); 317 318 return (0); 319 320 fail: 321 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 322 vtpci_modern_detach(dev); 323 324 return (error); 325 } 326 327 static int 328 vtpci_modern_detach(device_t dev) 329 { 330 struct vtpci_modern_softc *sc; 331 int error; 332 333 sc = device_get_softc(dev); 334 335 error = vtpci_delete_child(&sc->vtpci_common); 336 if (error) 337 return (error); 338 339 vtpci_modern_reset(sc); 340 vtpci_modern_unmap_configs(sc); 341 342 return (0); 343 } 344 345 static int 346 vtpci_modern_suspend(device_t dev) 347 { 348 return (bus_generic_suspend(dev)); 349 } 350 351 static int 352 vtpci_modern_resume(device_t dev) 353 { 354 return (bus_generic_resume(dev)); 355 } 356 357 static int 358 vtpci_modern_shutdown(device_t dev) 359 { 360 (void) bus_generic_shutdown(dev); 361 /* Forcibly stop the host device. */ 362 vtpci_modern_stop(dev); 363 364 return (0); 365 } 366 367 static void 368 vtpci_modern_driver_added(device_t dev, driver_t *driver) 369 { 370 vtpci_modern_probe_and_attach_child(device_get_softc(dev)); 371 } 372 373 static void 374 vtpci_modern_child_detached(device_t dev, device_t child) 375 { 376 struct vtpci_modern_softc *sc; 377 378 sc = device_get_softc(dev); 379 380 vtpci_modern_reset(sc); 381 vtpci_child_detached(&sc->vtpci_common); 382 383 /* After the reset, retell the host we've noticed this device. */ 384 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 385 } 386 387 static int 388 vtpci_modern_read_ivar(device_t dev, device_t child, int index, 389 uintptr_t *result) 390 { 391 struct vtpci_modern_softc *sc; 392 struct vtpci_common *cn; 393 394 sc = device_get_softc(dev); 395 cn = &sc->vtpci_common; 396 397 if (vtpci_child_device(cn) != child) 398 return (ENOENT); 399 400 switch (index) { 401 case VIRTIO_IVAR_DEVTYPE: 402 *result = sc->vtpci_devid; 403 break; 404 default: 405 return (vtpci_read_ivar(cn, index, result)); 406 } 407 408 return (0); 409 } 410 411 static int 412 vtpci_modern_write_ivar(device_t dev, device_t child, int index, 413 uintptr_t value) 414 { 415 struct vtpci_modern_softc *sc; 416 struct vtpci_common *cn; 417 418 sc = device_get_softc(dev); 419 cn = &sc->vtpci_common; 420 421 if (vtpci_child_device(cn) != child) 422 return (ENOENT); 423 424 switch (index) { 425 default: 426 return (vtpci_write_ivar(cn, index, value)); 427 } 428 429 return (0); 430 } 431 432 static uint64_t 433 vtpci_modern_negotiate_features(device_t dev, uint64_t child_features) 434 { 435 struct vtpci_modern_softc *sc; 436 uint64_t host_features, features; 437 438 sc = device_get_softc(dev); 439 host_features = vtpci_modern_read_features(sc); 440 441 /* 442 * Since the driver was added as a child of the modern PCI bus, 443 * always add the V1 flag. 444 */ 445 child_features |= VIRTIO_F_VERSION_1; 446 447 features = vtpci_negotiate_features(&sc->vtpci_common, 448 child_features, host_features); 449 vtpci_modern_write_features(sc, features); 450 451 return (features); 452 } 453 454 static int 455 vtpci_modern_finalize_features(device_t dev) 456 { 457 struct vtpci_modern_softc *sc; 458 uint8_t status; 459 460 sc = device_get_softc(dev); 461 462 /* 463 * Must re-read the status after setting it to verify the negotiated 464 * features were accepted by the device. 465 */ 466 vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK); 467 468 status = vtpci_modern_get_status(sc); 469 if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { 470 device_printf(dev, "desired features were not accepted\n"); 471 return (ENOTSUP); 472 } 473 474 return (0); 475 } 476 477 static int 478 vtpci_modern_with_feature(device_t dev, uint64_t feature) 479 { 480 struct vtpci_modern_softc *sc; 481 482 sc = device_get_softc(dev); 483 484 return (vtpci_with_feature(&sc->vtpci_common, feature)); 485 } 486 487 static uint64_t 488 vtpci_modern_read_features(struct vtpci_modern_softc *sc) 489 { 490 uint32_t features0, features1; 491 492 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0); 493 features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 494 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1); 495 features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 496 497 return (((uint64_t) features1 << 32) | features0); 498 } 499 500 static void 501 vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features) 502 { 503 uint32_t features0, features1; 504 505 features0 = features; 506 features1 = features >> 32; 507 508 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0); 509 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0); 510 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1); 511 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1); 512 } 513 514 static int 515 vtpci_modern_alloc_virtqueues(device_t dev, int flags, int nvqs, 516 struct vq_alloc_info *vq_info) 517 { 518 struct vtpci_modern_softc *sc; 519 struct vtpci_common *cn; 520 uint16_t max_nvqs; 521 522 sc = device_get_softc(dev); 523 cn = &sc->vtpci_common; 524 525 max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ); 526 if (nvqs > max_nvqs) { 527 device_printf(sc->vtpci_dev, "requested virtqueue count %d " 528 "exceeds max %d\n", nvqs, max_nvqs); 529 return (E2BIG); 530 } 531 532 return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info)); 533 } 534 535 static int 536 vtpci_modern_setup_interrupts(device_t dev, enum intr_type type) 537 { 538 struct vtpci_modern_softc *sc; 539 int error; 540 541 sc = device_get_softc(dev); 542 543 error = vtpci_setup_interrupts(&sc->vtpci_common, type); 544 if (error == 0) 545 vtpci_modern_enable_virtqueues(sc); 546 547 return (error); 548 } 549 550 static void 551 vtpci_modern_stop(device_t dev) 552 { 553 vtpci_modern_reset(device_get_softc(dev)); 554 } 555 556 static int 557 vtpci_modern_reinit(device_t dev, uint64_t features) 558 { 559 struct vtpci_modern_softc *sc; 560 struct vtpci_common *cn; 561 int error; 562 563 sc = device_get_softc(dev); 564 cn = &sc->vtpci_common; 565 566 /* 567 * Redrive the device initialization. This is a bit of an abuse of 568 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 569 * play nice. 570 * 571 * We do not allow the host device to change from what was originally 572 * negotiated beyond what the guest driver changed. MSIX state should 573 * not change, number of virtqueues and their size remain the same, etc. 574 * This will need to be rethought when we want to support migration. 575 */ 576 577 if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 578 vtpci_modern_stop(dev); 579 580 /* 581 * Quickly drive the status through ACK and DRIVER. The device does 582 * not become usable again until DRIVER_OK in reinit complete. 583 */ 584 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 585 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 586 587 /* 588 * TODO: Check that features are not added as to what was 589 * originally negotiated. 590 */ 591 vtpci_modern_negotiate_features(dev, features); 592 error = vtpci_modern_finalize_features(dev); 593 if (error) { 594 device_printf(dev, "cannot finalize features during reinit\n"); 595 return (error); 596 } 597 598 error = vtpci_reinit(cn); 599 if (error) 600 return (error); 601 602 return (0); 603 } 604 605 static void 606 vtpci_modern_reinit_complete(device_t dev) 607 { 608 struct vtpci_modern_softc *sc; 609 610 sc = device_get_softc(dev); 611 612 vtpci_modern_enable_virtqueues(sc); 613 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 614 } 615 616 static void 617 vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) 618 { 619 struct vtpci_modern_softc *sc; 620 621 sc = device_get_softc(dev); 622 623 vtpci_modern_write_notify_2(sc, offset, queue); 624 } 625 626 static uint8_t 627 vtpci_modern_get_status(struct vtpci_modern_softc *sc) 628 { 629 return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS)); 630 } 631 632 static void 633 vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status) 634 { 635 if (status != VIRTIO_CONFIG_STATUS_RESET) 636 status |= vtpci_modern_get_status(sc); 637 638 vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status); 639 } 640 641 static int 642 vtpci_modern_config_generation(device_t dev) 643 { 644 struct vtpci_modern_softc *sc; 645 uint8_t gen; 646 647 sc = device_get_softc(dev); 648 gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION); 649 650 return (gen); 651 } 652 653 static void 654 vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst, 655 int length) 656 { 657 struct vtpci_modern_softc *sc; 658 659 sc = device_get_softc(dev); 660 661 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 662 panic("%s: attempt to read dev config but not present", 663 __func__); 664 } 665 666 switch (length) { 667 case 1: 668 *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset); 669 break; 670 case 2: 671 *(uint16_t *) dst = virtio_htog16(true, 672 vtpci_modern_read_device_2(sc, offset)); 673 break; 674 case 4: 675 *(uint32_t *) dst = virtio_htog32(true, 676 vtpci_modern_read_device_4(sc, offset)); 677 break; 678 case 8: 679 *(uint64_t *) dst = virtio_htog64(true, 680 vtpci_modern_read_device_8(sc, offset)); 681 break; 682 default: 683 panic("%s: device %s invalid device read length %d offset %d", 684 __func__, device_get_nameunit(dev), length, (int) offset); 685 } 686 } 687 688 static void 689 vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src, 690 int length) 691 { 692 struct vtpci_modern_softc *sc; 693 694 sc = device_get_softc(dev); 695 696 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 697 panic("%s: attempt to write dev config but not present", 698 __func__); 699 } 700 701 switch (length) { 702 case 1: 703 vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src); 704 break; 705 case 2: { 706 uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src); 707 vtpci_modern_write_device_2(sc, offset, val); 708 break; 709 } 710 case 4: { 711 uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src); 712 vtpci_modern_write_device_4(sc, offset, val); 713 break; 714 } 715 case 8: { 716 uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src); 717 vtpci_modern_write_device_8(sc, offset, val); 718 break; 719 } 720 default: 721 panic("%s: device %s invalid device write length %d offset %d", 722 __func__, device_get_nameunit(dev), length, (int) offset); 723 } 724 } 725 726 static int 727 vtpci_modern_probe_configs(device_t dev) 728 { 729 int error; 730 731 /* 732 * These config capabilities must be present. The DEVICE_CFG 733 * capability is only present if the device requires it. 734 */ 735 736 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL); 737 if (error) { 738 device_printf(dev, "cannot find COMMON_CFG capability\n"); 739 return (error); 740 } 741 742 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL); 743 if (error) { 744 device_printf(dev, "cannot find NOTIFY_CFG capability\n"); 745 return (error); 746 } 747 748 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL); 749 if (error) { 750 device_printf(dev, "cannot find ISR_CFG capability\n"); 751 return (error); 752 } 753 754 return (0); 755 } 756 757 static int 758 vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset) 759 { 760 uint32_t type, bar; 761 int capreg, error; 762 763 for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg); 764 error == 0; 765 error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) { 766 767 type = pci_read_config(dev, capreg + 768 offsetof(struct virtio_pci_cap, cfg_type), 1); 769 bar = pci_read_config(dev, capreg + 770 offsetof(struct virtio_pci_cap, bar), 1); 771 772 /* Must ignore reserved BARs. */ 773 if (bar >= VTPCI_MODERN_MAX_BARS) 774 continue; 775 776 if (type == cfg_type) { 777 if (cap_offset != NULL) 778 *cap_offset = capreg; 779 break; 780 } 781 } 782 783 return (error); 784 } 785 786 static int 787 vtpci_modern_map_common_config(struct vtpci_modern_softc *sc) 788 { 789 device_t dev; 790 int error; 791 792 dev = sc->vtpci_dev; 793 794 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG, 795 sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map); 796 if (error) { 797 device_printf(dev, "cannot find cap COMMON_CFG resource\n"); 798 return (error); 799 } 800 801 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map); 802 if (error) { 803 device_printf(dev, "cannot alloc resource for COMMON_CFG\n"); 804 return (error); 805 } 806 807 return (0); 808 } 809 810 static int 811 vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc) 812 { 813 device_t dev; 814 int cap_offset, error; 815 816 dev = sc->vtpci_dev; 817 818 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG, 819 -1, 2, &sc->vtpci_notify_res_map); 820 if (error) { 821 device_printf(dev, "cannot find cap NOTIFY_CFG resource\n"); 822 return (error); 823 } 824 825 cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset; 826 827 sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset + 828 offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4); 829 830 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map); 831 if (error) { 832 device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n"); 833 return (error); 834 } 835 836 return (0); 837 } 838 839 static int 840 vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc) 841 { 842 device_t dev; 843 int error; 844 845 dev = sc->vtpci_dev; 846 847 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG, 848 sizeof(uint8_t), 1, &sc->vtpci_isr_res_map); 849 if (error) { 850 device_printf(dev, "cannot find cap ISR_CFG resource\n"); 851 return (error); 852 } 853 854 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map); 855 if (error) { 856 device_printf(dev, "cannot alloc resource for ISR_CFG\n"); 857 return (error); 858 } 859 860 return (0); 861 } 862 863 static int 864 vtpci_modern_map_device_config(struct vtpci_modern_softc *sc) 865 { 866 device_t dev; 867 int error; 868 869 dev = sc->vtpci_dev; 870 871 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG, 872 -1, 4, &sc->vtpci_device_res_map); 873 if (error == ENOENT) { 874 /* Device configuration is optional depending on device. */ 875 return (0); 876 } else if (error) { 877 device_printf(dev, "cannot find cap DEVICE_CFG resource\n"); 878 return (error); 879 } 880 881 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map); 882 if (error) { 883 device_printf(dev, "cannot alloc resource for DEVICE_CFG\n"); 884 return (error); 885 } 886 887 return (0); 888 } 889 890 static int 891 vtpci_modern_map_configs(struct vtpci_modern_softc *sc) 892 { 893 int error; 894 895 error = vtpci_modern_map_common_config(sc); 896 if (error) 897 return (error); 898 899 error = vtpci_modern_map_notify_config(sc); 900 if (error) 901 return (error); 902 903 error = vtpci_modern_map_isr_config(sc); 904 if (error) 905 return (error); 906 907 error = vtpci_modern_map_device_config(sc); 908 if (error) 909 return (error); 910 911 vtpci_modern_alloc_msix_resource(sc); 912 913 return (0); 914 } 915 916 static void 917 vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc) 918 { 919 920 vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map); 921 vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map); 922 vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map); 923 vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map); 924 925 vtpci_modern_free_bar_resources(sc); 926 vtpci_modern_free_msix_resource(sc); 927 928 sc->vtpci_notify_offset_multiplier = 0; 929 } 930 931 static int 932 vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type, 933 int min_size, int alignment, struct vtpci_modern_resource_map *res) 934 { 935 device_t dev; 936 int cap_offset, offset, length, error; 937 uint8_t bar, cap_length; 938 939 dev = sc->vtpci_dev; 940 941 error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset); 942 if (error) 943 return (error); 944 945 cap_length = pci_read_config(dev, 946 cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1); 947 948 if (cap_length < sizeof(struct virtio_pci_cap)) { 949 device_printf(dev, "cap %u length %d less than expected\n", 950 cfg_type, cap_length); 951 return (ENXIO); 952 } 953 954 bar = pci_read_config(dev, 955 cap_offset + offsetof(struct virtio_pci_cap, bar), 1); 956 offset = pci_read_config(dev, 957 cap_offset + offsetof(struct virtio_pci_cap, offset), 4); 958 length = pci_read_config(dev, 959 cap_offset + offsetof(struct virtio_pci_cap, length), 4); 960 961 if (min_size != -1 && length < min_size) { 962 device_printf(dev, "cap %u struct length %d less than min %d\n", 963 cfg_type, length, min_size); 964 return (ENXIO); 965 } 966 967 if (offset % alignment) { 968 device_printf(dev, "cap %u struct offset %d not aligned to %d\n", 969 cfg_type, offset, alignment); 970 return (ENXIO); 971 } 972 973 /* BMV: TODO Can we determine the size of the BAR here? */ 974 975 res->vtrm_cap_offset = cap_offset; 976 res->vtrm_bar = bar; 977 res->vtrm_offset = offset; 978 res->vtrm_length = length; 979 res->vtrm_type = vtpci_modern_bar_type(sc, bar); 980 981 return (0); 982 } 983 984 static int 985 vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar) 986 { 987 uint32_t val; 988 989 /* 990 * The BAR described by a config capability may be either an IOPORT or 991 * MEM, but we must know the type when calling bus_alloc_resource(). 992 */ 993 val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4); 994 if (PCI_BAR_IO(val)) 995 return (SYS_RES_IOPORT); 996 else 997 return (SYS_RES_MEMORY); 998 } 999 1000 static struct resource * 1001 vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) 1002 { 1003 struct resource *res; 1004 1005 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1006 res = sc->vtpci_bar_res[bar].vtbr_res; 1007 MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type); 1008 1009 return (res); 1010 } 1011 1012 static struct resource * 1013 vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar, 1014 int type) 1015 { 1016 struct resource *res; 1017 int rid; 1018 1019 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1020 MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT); 1021 1022 res = sc->vtpci_bar_res[bar].vtbr_res; 1023 if (res != NULL) { 1024 MPASS(sc->vtpci_bar_res[bar].vtbr_type == type); 1025 return (res); 1026 } 1027 1028 rid = PCIR_BAR(bar); 1029 res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid, 1030 RF_ACTIVE | RF_UNMAPPED); 1031 if (res != NULL) { 1032 sc->vtpci_bar_res[bar].vtbr_res = res; 1033 sc->vtpci_bar_res[bar].vtbr_type = type; 1034 } 1035 1036 return (res); 1037 } 1038 1039 static void 1040 vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc) 1041 { 1042 device_t dev; 1043 struct resource *res; 1044 int bar, rid, type; 1045 1046 dev = sc->vtpci_dev; 1047 1048 for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) { 1049 res = sc->vtpci_bar_res[bar].vtbr_res; 1050 type = sc->vtpci_bar_res[bar].vtbr_type; 1051 1052 if (res != NULL) { 1053 rid = PCIR_BAR(bar); 1054 bus_release_resource(dev, type, rid, res); 1055 sc->vtpci_bar_res[bar].vtbr_res = NULL; 1056 sc->vtpci_bar_res[bar].vtbr_type = 0; 1057 } 1058 } 1059 } 1060 1061 static int 1062 vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc, 1063 struct vtpci_modern_resource_map *map) 1064 { 1065 struct resource_map_request req; 1066 struct resource *res; 1067 int type; 1068 1069 type = map->vtrm_type; 1070 1071 res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type); 1072 if (res == NULL) 1073 return (ENXIO); 1074 1075 resource_init_map_request(&req); 1076 req.offset = map->vtrm_offset; 1077 req.length = map->vtrm_length; 1078 1079 return (bus_map_resource(sc->vtpci_dev, type, res, &req, 1080 &map->vtrm_map)); 1081 } 1082 1083 static void 1084 vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc, 1085 struct vtpci_modern_resource_map *map) 1086 { 1087 struct resource *res; 1088 int type; 1089 1090 type = map->vtrm_type; 1091 res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type); 1092 1093 if (res != NULL && map->vtrm_map.r_size != 0) { 1094 bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map); 1095 bzero(map, sizeof(struct vtpci_modern_resource_map)); 1096 } 1097 } 1098 1099 static void 1100 vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc) 1101 { 1102 device_t dev; 1103 int bar; 1104 1105 dev = sc->vtpci_dev; 1106 1107 if (!vtpci_is_msix_available(&sc->vtpci_common) || 1108 (bar = pci_msix_table_bar(dev)) == -1) 1109 return; 1110 1111 /* TODO: Can this BAR be in the 0-5 range? */ 1112 sc->vtpci_msix_bar = bar; 1113 if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1114 &bar, RF_ACTIVE)) == NULL) 1115 device_printf(dev, "Unable to map MSIX table\n"); 1116 } 1117 1118 static void 1119 vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc) 1120 { 1121 device_t dev; 1122 1123 dev = sc->vtpci_dev; 1124 1125 if (sc->vtpci_msix_res != NULL) { 1126 bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar, 1127 sc->vtpci_msix_res); 1128 sc->vtpci_msix_bar = 0; 1129 sc->vtpci_msix_res = NULL; 1130 } 1131 } 1132 1133 static void 1134 vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc) 1135 { 1136 device_t dev, child; 1137 1138 dev = sc->vtpci_dev; 1139 child = vtpci_child_device(&sc->vtpci_common); 1140 1141 if (child == NULL || device_get_state(child) != DS_NOTPRESENT) 1142 return; 1143 1144 if (device_probe(child) != 0) 1145 return; 1146 1147 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 1148 1149 if (device_attach(child) != 0) { 1150 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 1151 /* Reset state for later attempt. */ 1152 vtpci_modern_child_detached(dev, child); 1153 } else { 1154 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1155 VIRTIO_ATTACH_COMPLETED(child); 1156 } 1157 } 1158 1159 static int 1160 vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset, 1161 struct vtpci_interrupt *intr) 1162 { 1163 uint16_t vector; 1164 1165 if (intr != NULL) { 1166 /* Map from guest rid to host vector. */ 1167 vector = intr->vti_rid - 1; 1168 } else 1169 vector = VIRTIO_MSI_NO_VECTOR; 1170 1171 vtpci_modern_write_common_2(sc, offset, vector); 1172 return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV); 1173 } 1174 1175 static int 1176 vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) 1177 { 1178 struct vtpci_modern_softc *sc; 1179 int error; 1180 1181 sc = device_get_softc(dev); 1182 1183 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr); 1184 if (error) { 1185 device_printf(dev, 1186 "unable to register config MSIX interrupt\n"); 1187 return (error); 1188 } 1189 1190 return (0); 1191 } 1192 1193 static int 1194 vtpci_modern_register_vq_msix(device_t dev, int idx, 1195 struct vtpci_interrupt *intr) 1196 { 1197 struct vtpci_modern_softc *sc; 1198 int error; 1199 1200 sc = device_get_softc(dev); 1201 1202 vtpci_modern_select_virtqueue(sc, idx); 1203 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr); 1204 if (error) { 1205 device_printf(dev, 1206 "unable to register virtqueue MSIX interrupt\n"); 1207 return (error); 1208 } 1209 1210 return (0); 1211 } 1212 1213 static void 1214 vtpci_modern_reset(struct vtpci_modern_softc *sc) 1215 { 1216 /* 1217 * Setting the status to RESET sets the host device to the 1218 * original, uninitialized state. Must poll the status until 1219 * the reset is complete. 1220 */ 1221 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); 1222 1223 while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 1224 cpu_spinwait(); 1225 } 1226 1227 static void 1228 vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx) 1229 { 1230 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx); 1231 } 1232 1233 static uint8_t 1234 vtpci_modern_read_isr(device_t dev) 1235 { 1236 return (vtpci_modern_read_isr_1(device_get_softc(dev), 0)); 1237 } 1238 1239 static uint16_t 1240 vtpci_modern_get_vq_size(device_t dev, int idx) 1241 { 1242 struct vtpci_modern_softc *sc; 1243 1244 sc = device_get_softc(dev); 1245 1246 vtpci_modern_select_virtqueue(sc, idx); 1247 return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE)); 1248 } 1249 1250 static bus_size_t 1251 vtpci_modern_get_vq_notify_off(device_t dev, int idx) 1252 { 1253 struct vtpci_modern_softc *sc; 1254 uint16_t q_notify_off; 1255 1256 sc = device_get_softc(dev); 1257 1258 vtpci_modern_select_virtqueue(sc, idx); 1259 q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF); 1260 1261 return (q_notify_off * sc->vtpci_notify_offset_multiplier); 1262 } 1263 1264 static void 1265 vtpci_modern_set_vq(device_t dev, struct virtqueue *vq) 1266 { 1267 struct vtpci_modern_softc *sc; 1268 1269 sc = device_get_softc(dev); 1270 1271 vtpci_modern_select_virtqueue(sc, virtqueue_index(vq)); 1272 1273 /* BMV: Currently we never adjust the device's proposed VQ size. */ 1274 vtpci_modern_write_common_2(sc, 1275 VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq)); 1276 1277 vtpci_modern_write_common_8(sc, 1278 VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq)); 1279 vtpci_modern_write_common_8(sc, 1280 VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq)); 1281 vtpci_modern_write_common_8(sc, 1282 VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq)); 1283 } 1284 1285 static void 1286 vtpci_modern_disable_vq(device_t dev, int idx) 1287 { 1288 struct vtpci_modern_softc *sc; 1289 1290 sc = device_get_softc(dev); 1291 1292 vtpci_modern_select_virtqueue(sc, idx); 1293 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL); 1294 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL); 1295 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL); 1296 } 1297 1298 static void 1299 vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc) 1300 { 1301 int idx; 1302 1303 for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) { 1304 vtpci_modern_select_virtqueue(sc, idx); 1305 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1); 1306 } 1307 } 1308 1309 static uint8_t 1310 vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off) 1311 { 1312 return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off)); 1313 } 1314 1315 static uint16_t 1316 vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off) 1317 { 1318 return virtio_htog16(true, 1319 bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off)); 1320 } 1321 1322 static uint32_t 1323 vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off) 1324 { 1325 return virtio_htog32(true, 1326 bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off)); 1327 } 1328 1329 static void 1330 vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off, 1331 uint8_t val) 1332 { 1333 bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val); 1334 } 1335 1336 static void 1337 vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off, 1338 uint16_t val) 1339 { 1340 bus_write_2(&sc->vtpci_common_res_map.vtrm_map, 1341 off, virtio_gtoh16(true, val)); 1342 } 1343 1344 static void 1345 vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off, 1346 uint32_t val) 1347 { 1348 bus_write_4(&sc->vtpci_common_res_map.vtrm_map, 1349 off, virtio_gtoh32(true, val)); 1350 } 1351 1352 static void 1353 vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off, 1354 uint64_t val) 1355 { 1356 uint32_t val0, val1; 1357 1358 val0 = (uint32_t) val; 1359 val1 = val >> 32; 1360 1361 vtpci_modern_write_common_4(sc, off, val0); 1362 vtpci_modern_write_common_4(sc, off + 4, val1); 1363 } 1364 1365 static void 1366 vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off, 1367 uint16_t val) 1368 { 1369 bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val); 1370 } 1371 1372 static uint8_t 1373 vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off) 1374 { 1375 return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off)); 1376 } 1377 1378 static uint8_t 1379 vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off) 1380 { 1381 return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off)); 1382 } 1383 1384 static uint16_t 1385 vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off) 1386 { 1387 return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off)); 1388 } 1389 1390 static uint32_t 1391 vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off) 1392 { 1393 return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off)); 1394 } 1395 1396 static uint64_t 1397 vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off) 1398 { 1399 device_t dev; 1400 int gen; 1401 uint32_t val0, val1; 1402 1403 dev = sc->vtpci_dev; 1404 1405 /* 1406 * Treat the 64-bit field as two 32-bit fields. Use the generation 1407 * to ensure a consistent read. 1408 */ 1409 do { 1410 gen = vtpci_modern_config_generation(dev); 1411 val0 = vtpci_modern_read_device_4(sc, off); 1412 val1 = vtpci_modern_read_device_4(sc, off + 4); 1413 } while (gen != vtpci_modern_config_generation(dev)); 1414 1415 return (((uint64_t) val1 << 32) | val0); 1416 } 1417 1418 static void 1419 vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off, 1420 uint8_t val) 1421 { 1422 bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val); 1423 } 1424 1425 static void 1426 vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off, 1427 uint16_t val) 1428 { 1429 bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val); 1430 } 1431 1432 static void 1433 vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off, 1434 uint32_t val) 1435 { 1436 bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val); 1437 } 1438 1439 static void 1440 vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off, 1441 uint64_t val) 1442 { 1443 uint32_t val0, val1; 1444 1445 val0 = (uint32_t) val; 1446 val1 = val >> 32; 1447 1448 vtpci_modern_write_device_4(sc, off, val0); 1449 vtpci_modern_write_device_4(sc, off + 4, val1); 1450 } 1451