1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for the modern VirtIO PCI interface. */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/bus.h> 37 #include <sys/lock.h> 38 #include <sys/kernel.h> 39 #include <sys/module.h> 40 41 #include <machine/bus.h> 42 #include <machine/cpu.h> 43 #include <machine/resource.h> 44 #include <sys/bus.h> 45 #include <sys/rman.h> 46 47 #include <dev/pci/pcivar.h> 48 #include <dev/pci/pcireg.h> 49 50 #include <dev/virtio/virtio.h> 51 #include <dev/virtio/virtqueue.h> 52 #include <dev/virtio/pci/virtio_pci.h> 53 #include <dev/virtio/pci/virtio_pci_modern_var.h> 54 55 #include "virtio_bus_if.h" 56 #include "virtio_pci_if.h" 57 #include "virtio_if.h" 58 59 struct vtpci_modern_resource_map { 60 struct resource_map vtrm_map; 61 int vtrm_cap_offset; 62 int vtrm_bar; 63 int vtrm_offset; 64 int vtrm_length; 65 int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */ 66 }; 67 68 struct vtpci_modern_bar_resource { 69 struct resource *vtbr_res; 70 int vtbr_type; 71 }; 72 73 struct vtpci_modern_softc { 74 device_t vtpci_dev; 75 struct vtpci_common vtpci_common; 76 uint32_t vtpci_notify_offset_multiplier; 77 uint16_t vtpci_devid; 78 int vtpci_msix_bar; 79 struct resource *vtpci_msix_res; 80 81 struct vtpci_modern_resource_map vtpci_common_res_map; 82 struct vtpci_modern_resource_map vtpci_notify_res_map; 83 struct vtpci_modern_resource_map vtpci_isr_res_map; 84 struct vtpci_modern_resource_map vtpci_device_res_map; 85 86 #define VTPCI_MODERN_MAX_BARS 6 87 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS]; 88 }; 89 90 static int vtpci_modern_probe(device_t); 91 static int vtpci_modern_attach(device_t); 92 static int vtpci_modern_detach(device_t); 93 static int vtpci_modern_suspend(device_t); 94 static int vtpci_modern_resume(device_t); 95 static int vtpci_modern_shutdown(device_t); 96 97 static void vtpci_modern_driver_added(device_t, driver_t *); 98 static void vtpci_modern_child_detached(device_t, device_t); 99 static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *); 100 static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t); 101 102 static uint8_t vtpci_modern_read_isr(device_t); 103 static uint16_t vtpci_modern_get_vq_size(device_t, int); 104 static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int); 105 static void vtpci_modern_set_vq(device_t, struct virtqueue *); 106 static void vtpci_modern_disable_vq(device_t, int); 107 static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int, 108 struct vtpci_interrupt *); 109 static int vtpci_modern_register_cfg_msix(device_t, 110 struct vtpci_interrupt *); 111 static int vtpci_modern_register_vq_msix(device_t, int idx, 112 struct vtpci_interrupt *); 113 114 static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t); 115 static int vtpci_modern_finalize_features(device_t); 116 static int vtpci_modern_with_feature(device_t, uint64_t); 117 static int vtpci_modern_alloc_virtqueues(device_t, int, int, 118 struct vq_alloc_info *); 119 static int vtpci_modern_setup_interrupts(device_t, enum intr_type); 120 static void vtpci_modern_stop(device_t); 121 static int vtpci_modern_reinit(device_t, uint64_t); 122 static void vtpci_modern_reinit_complete(device_t); 123 static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t); 124 static int vtpci_modern_config_generation(device_t); 125 static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int); 126 static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int); 127 128 static int vtpci_modern_probe_configs(device_t); 129 static int vtpci_modern_find_cap(device_t, uint8_t, int *); 130 static int vtpci_modern_map_configs(struct vtpci_modern_softc *); 131 static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *); 132 static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *, 133 uint8_t, int, int, struct vtpci_modern_resource_map *); 134 static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int); 135 static struct resource *vtpci_modern_get_bar_resource( 136 struct vtpci_modern_softc *, int, int); 137 static struct resource *vtpci_modern_alloc_bar_resource( 138 struct vtpci_modern_softc *, int, int); 139 static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *); 140 static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *, 141 struct vtpci_modern_resource_map *); 142 static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *, 143 struct vtpci_modern_resource_map *); 144 static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *); 145 static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *); 146 147 static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *); 148 149 static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *); 150 static void vtpci_modern_write_features(struct vtpci_modern_softc *, 151 uint64_t); 152 static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int); 153 static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *); 154 static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t); 155 static void vtpci_modern_reset(struct vtpci_modern_softc *); 156 static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *); 157 158 static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *, 159 bus_size_t); 160 static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *, 161 bus_size_t); 162 static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *, 163 bus_size_t); 164 static void vtpci_modern_write_common_1(struct vtpci_modern_softc *, 165 bus_size_t, uint8_t); 166 static void vtpci_modern_write_common_2(struct vtpci_modern_softc *, 167 bus_size_t, uint16_t); 168 static void vtpci_modern_write_common_4(struct vtpci_modern_softc *, 169 bus_size_t, uint32_t); 170 static void vtpci_modern_write_common_8(struct vtpci_modern_softc *, 171 bus_size_t, uint64_t); 172 static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *, 173 bus_size_t, uint16_t); 174 static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *, 175 bus_size_t); 176 static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *, 177 bus_size_t); 178 static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *, 179 bus_size_t); 180 static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *, 181 bus_size_t); 182 static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *, 183 bus_size_t); 184 static void vtpci_modern_write_device_1(struct vtpci_modern_softc *, 185 bus_size_t, uint8_t); 186 static void vtpci_modern_write_device_2(struct vtpci_modern_softc *, 187 bus_size_t, uint16_t); 188 static void vtpci_modern_write_device_4(struct vtpci_modern_softc *, 189 bus_size_t, uint32_t); 190 static void vtpci_modern_write_device_8(struct vtpci_modern_softc *, 191 bus_size_t, uint64_t); 192 193 /* Tunables. */ 194 static int vtpci_modern_transitional = 0; 195 TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional); 196 197 static device_method_t vtpci_modern_methods[] = { 198 /* Device interface. */ 199 DEVMETHOD(device_probe, vtpci_modern_probe), 200 DEVMETHOD(device_attach, vtpci_modern_attach), 201 DEVMETHOD(device_detach, vtpci_modern_detach), 202 DEVMETHOD(device_suspend, vtpci_modern_suspend), 203 DEVMETHOD(device_resume, vtpci_modern_resume), 204 DEVMETHOD(device_shutdown, vtpci_modern_shutdown), 205 206 /* Bus interface. */ 207 DEVMETHOD(bus_driver_added, vtpci_modern_driver_added), 208 DEVMETHOD(bus_child_detached, vtpci_modern_child_detached), 209 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), 210 DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar), 211 DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar), 212 213 /* VirtIO PCI interface. */ 214 DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr), 215 DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size), 216 DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off), 217 DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq), 218 DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq), 219 DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix), 220 DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix), 221 222 /* VirtIO bus interface. */ 223 DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features), 224 DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features), 225 DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature), 226 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues), 227 DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts), 228 DEVMETHOD(virtio_bus_stop, vtpci_modern_stop), 229 DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit), 230 DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete), 231 DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq), 232 DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation), 233 DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config), 234 DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config), 235 236 DEVMETHOD_END 237 }; 238 239 static driver_t vtpci_modern_driver = { 240 .name = "virtio_pci", 241 .methods = vtpci_modern_methods, 242 .size = sizeof(struct vtpci_modern_softc) 243 }; 244 245 DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0); 246 247 static int 248 vtpci_modern_probe(device_t dev) 249 { 250 char desc[64]; 251 const char *name; 252 uint16_t devid; 253 254 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 255 return (ENXIO); 256 257 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 258 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX) 259 return (ENXIO); 260 261 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) { 262 if (!vtpci_modern_transitional) 263 return (ENXIO); 264 devid = pci_get_subdevice(dev); 265 } else 266 devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; 267 268 if (vtpci_modern_probe_configs(dev) != 0) 269 return (ENXIO); 270 271 name = virtio_device_name(devid); 272 if (name == NULL) 273 name = "Unknown"; 274 275 snprintf(desc, sizeof(desc), "VirtIO PCI (modern) %s adapter", name); 276 device_set_desc_copy(dev, desc); 277 278 return (BUS_PROBE_DEFAULT); 279 } 280 281 static int 282 vtpci_modern_attach(device_t dev) 283 { 284 struct vtpci_modern_softc *sc; 285 int error; 286 287 sc = device_get_softc(dev); 288 sc->vtpci_dev = dev; 289 vtpci_init(&sc->vtpci_common, dev, true); 290 291 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) 292 sc->vtpci_devid = pci_get_subdevice(dev); 293 else 294 sc->vtpci_devid = pci_get_device(dev) - 295 VIRTIO_PCI_DEVICEID_MODERN_MIN; 296 297 error = vtpci_modern_map_configs(sc); 298 if (error) { 299 device_printf(dev, "cannot map configs\n"); 300 vtpci_modern_unmap_configs(sc); 301 return (error); 302 } 303 304 vtpci_modern_reset(sc); 305 306 /* Tell the host we've noticed this device. */ 307 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 308 309 error = vtpci_add_child(&sc->vtpci_common); 310 if (error) 311 goto fail; 312 313 vtpci_modern_probe_and_attach_child(sc); 314 315 return (0); 316 317 fail: 318 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 319 vtpci_modern_detach(dev); 320 321 return (error); 322 } 323 324 static int 325 vtpci_modern_detach(device_t dev) 326 { 327 struct vtpci_modern_softc *sc; 328 int error; 329 330 sc = device_get_softc(dev); 331 332 error = vtpci_delete_child(&sc->vtpci_common); 333 if (error) 334 return (error); 335 336 vtpci_modern_reset(sc); 337 vtpci_modern_unmap_configs(sc); 338 339 return (0); 340 } 341 342 static int 343 vtpci_modern_suspend(device_t dev) 344 { 345 return (bus_generic_suspend(dev)); 346 } 347 348 static int 349 vtpci_modern_resume(device_t dev) 350 { 351 return (bus_generic_resume(dev)); 352 } 353 354 static int 355 vtpci_modern_shutdown(device_t dev) 356 { 357 (void) bus_generic_shutdown(dev); 358 /* Forcibly stop the host device. */ 359 vtpci_modern_stop(dev); 360 361 return (0); 362 } 363 364 static void 365 vtpci_modern_driver_added(device_t dev, driver_t *driver) 366 { 367 vtpci_modern_probe_and_attach_child(device_get_softc(dev)); 368 } 369 370 static void 371 vtpci_modern_child_detached(device_t dev, device_t child) 372 { 373 struct vtpci_modern_softc *sc; 374 375 sc = device_get_softc(dev); 376 377 vtpci_modern_reset(sc); 378 vtpci_child_detached(&sc->vtpci_common); 379 380 /* After the reset, retell the host we've noticed this device. */ 381 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 382 } 383 384 static int 385 vtpci_modern_read_ivar(device_t dev, device_t child, int index, 386 uintptr_t *result) 387 { 388 struct vtpci_modern_softc *sc; 389 struct vtpci_common *cn; 390 391 sc = device_get_softc(dev); 392 cn = &sc->vtpci_common; 393 394 if (vtpci_child_device(cn) != child) 395 return (ENOENT); 396 397 switch (index) { 398 case VIRTIO_IVAR_DEVTYPE: 399 *result = sc->vtpci_devid; 400 break; 401 default: 402 return (vtpci_read_ivar(cn, index, result)); 403 } 404 405 return (0); 406 } 407 408 static int 409 vtpci_modern_write_ivar(device_t dev, device_t child, int index, 410 uintptr_t value) 411 { 412 struct vtpci_modern_softc *sc; 413 struct vtpci_common *cn; 414 415 sc = device_get_softc(dev); 416 cn = &sc->vtpci_common; 417 418 if (vtpci_child_device(cn) != child) 419 return (ENOENT); 420 421 switch (index) { 422 default: 423 return (vtpci_write_ivar(cn, index, value)); 424 } 425 426 return (0); 427 } 428 429 static uint64_t 430 vtpci_modern_negotiate_features(device_t dev, uint64_t child_features) 431 { 432 struct vtpci_modern_softc *sc; 433 uint64_t host_features, features; 434 435 sc = device_get_softc(dev); 436 host_features = vtpci_modern_read_features(sc); 437 438 /* 439 * Since the driver was added as a child of the modern PCI bus, 440 * always add the V1 flag. 441 */ 442 child_features |= VIRTIO_F_VERSION_1; 443 444 features = vtpci_negotiate_features(&sc->vtpci_common, 445 child_features, host_features); 446 vtpci_modern_write_features(sc, features); 447 448 return (features); 449 } 450 451 static int 452 vtpci_modern_finalize_features(device_t dev) 453 { 454 struct vtpci_modern_softc *sc; 455 uint8_t status; 456 457 sc = device_get_softc(dev); 458 459 /* 460 * Must re-read the status after setting it to verify the negotiated 461 * features were accepted by the device. 462 */ 463 vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK); 464 465 status = vtpci_modern_get_status(sc); 466 if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { 467 device_printf(dev, "desired features were not accepted\n"); 468 return (ENOTSUP); 469 } 470 471 return (0); 472 } 473 474 static int 475 vtpci_modern_with_feature(device_t dev, uint64_t feature) 476 { 477 struct vtpci_modern_softc *sc; 478 479 sc = device_get_softc(dev); 480 481 return (vtpci_with_feature(&sc->vtpci_common, feature)); 482 } 483 484 static uint64_t 485 vtpci_modern_read_features(struct vtpci_modern_softc *sc) 486 { 487 uint32_t features0, features1; 488 489 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0); 490 features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 491 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1); 492 features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 493 494 return (((uint64_t) features1 << 32) | features0); 495 } 496 497 static void 498 vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features) 499 { 500 uint32_t features0, features1; 501 502 features0 = features; 503 features1 = features >> 32; 504 505 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0); 506 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0); 507 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1); 508 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1); 509 } 510 511 static int 512 vtpci_modern_alloc_virtqueues(device_t dev, int flags, int nvqs, 513 struct vq_alloc_info *vq_info) 514 { 515 struct vtpci_modern_softc *sc; 516 struct vtpci_common *cn; 517 uint16_t max_nvqs; 518 519 sc = device_get_softc(dev); 520 cn = &sc->vtpci_common; 521 522 max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ); 523 if (nvqs > max_nvqs) { 524 device_printf(sc->vtpci_dev, "requested virtqueue count %d " 525 "exceeds max %d\n", nvqs, max_nvqs); 526 return (E2BIG); 527 } 528 529 return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info)); 530 } 531 532 static int 533 vtpci_modern_setup_interrupts(device_t dev, enum intr_type type) 534 { 535 struct vtpci_modern_softc *sc; 536 int error; 537 538 sc = device_get_softc(dev); 539 540 error = vtpci_setup_interrupts(&sc->vtpci_common, type); 541 if (error == 0) 542 vtpci_modern_enable_virtqueues(sc); 543 544 return (error); 545 } 546 547 static void 548 vtpci_modern_stop(device_t dev) 549 { 550 vtpci_modern_reset(device_get_softc(dev)); 551 } 552 553 static int 554 vtpci_modern_reinit(device_t dev, uint64_t features) 555 { 556 struct vtpci_modern_softc *sc; 557 struct vtpci_common *cn; 558 int error; 559 560 sc = device_get_softc(dev); 561 cn = &sc->vtpci_common; 562 563 /* 564 * Redrive the device initialization. This is a bit of an abuse of 565 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 566 * play nice. 567 * 568 * We do not allow the host device to change from what was originally 569 * negotiated beyond what the guest driver changed. MSIX state should 570 * not change, number of virtqueues and their size remain the same, etc. 571 * This will need to be rethought when we want to support migration. 572 */ 573 574 if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 575 vtpci_modern_stop(dev); 576 577 /* 578 * Quickly drive the status through ACK and DRIVER. The device does 579 * not become usable again until DRIVER_OK in reinit complete. 580 */ 581 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 582 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 583 584 /* 585 * TODO: Check that features are not added as to what was 586 * originally negotiated. 587 */ 588 vtpci_modern_negotiate_features(dev, features); 589 error = vtpci_modern_finalize_features(dev); 590 if (error) { 591 device_printf(dev, "cannot finalize features during reinit\n"); 592 return (error); 593 } 594 595 error = vtpci_reinit(cn); 596 if (error) 597 return (error); 598 599 return (0); 600 } 601 602 static void 603 vtpci_modern_reinit_complete(device_t dev) 604 { 605 struct vtpci_modern_softc *sc; 606 607 sc = device_get_softc(dev); 608 609 vtpci_modern_enable_virtqueues(sc); 610 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 611 } 612 613 static void 614 vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) 615 { 616 struct vtpci_modern_softc *sc; 617 618 sc = device_get_softc(dev); 619 620 vtpci_modern_write_notify_2(sc, offset, queue); 621 } 622 623 static uint8_t 624 vtpci_modern_get_status(struct vtpci_modern_softc *sc) 625 { 626 return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS)); 627 } 628 629 static void 630 vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status) 631 { 632 if (status != VIRTIO_CONFIG_STATUS_RESET) 633 status |= vtpci_modern_get_status(sc); 634 635 vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status); 636 } 637 638 static int 639 vtpci_modern_config_generation(device_t dev) 640 { 641 struct vtpci_modern_softc *sc; 642 uint8_t gen; 643 644 sc = device_get_softc(dev); 645 gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION); 646 647 return (gen); 648 } 649 650 static void 651 vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst, 652 int length) 653 { 654 struct vtpci_modern_softc *sc; 655 656 sc = device_get_softc(dev); 657 658 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 659 panic("%s: attempt to read dev config but not present", 660 __func__); 661 } 662 663 switch (length) { 664 case 1: 665 *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset); 666 break; 667 case 2: 668 *(uint16_t *) dst = virtio_htog16(true, 669 vtpci_modern_read_device_2(sc, offset)); 670 break; 671 case 4: 672 *(uint32_t *) dst = virtio_htog32(true, 673 vtpci_modern_read_device_4(sc, offset)); 674 break; 675 case 8: 676 *(uint64_t *) dst = virtio_htog64(true, 677 vtpci_modern_read_device_8(sc, offset)); 678 break; 679 default: 680 panic("%s: device %s invalid device read length %d offset %d", 681 __func__, device_get_nameunit(dev), length, (int) offset); 682 } 683 } 684 685 static void 686 vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src, 687 int length) 688 { 689 struct vtpci_modern_softc *sc; 690 691 sc = device_get_softc(dev); 692 693 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 694 panic("%s: attempt to write dev config but not present", 695 __func__); 696 } 697 698 switch (length) { 699 case 1: 700 vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src); 701 break; 702 case 2: { 703 uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src); 704 vtpci_modern_write_device_2(sc, offset, val); 705 break; 706 } 707 case 4: { 708 uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src); 709 vtpci_modern_write_device_4(sc, offset, val); 710 break; 711 } 712 case 8: { 713 uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src); 714 vtpci_modern_write_device_8(sc, offset, val); 715 break; 716 } 717 default: 718 panic("%s: device %s invalid device write length %d offset %d", 719 __func__, device_get_nameunit(dev), length, (int) offset); 720 } 721 } 722 723 static int 724 vtpci_modern_probe_configs(device_t dev) 725 { 726 int error; 727 728 /* 729 * These config capabilities must be present. The DEVICE_CFG 730 * capability is only present if the device requires it. 731 */ 732 733 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL); 734 if (error) { 735 device_printf(dev, "cannot find COMMON_CFG capability\n"); 736 return (error); 737 } 738 739 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL); 740 if (error) { 741 device_printf(dev, "cannot find NOTIFY_CFG capability\n"); 742 return (error); 743 } 744 745 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL); 746 if (error) { 747 device_printf(dev, "cannot find ISR_CFG capability\n"); 748 return (error); 749 } 750 751 return (0); 752 } 753 754 static int 755 vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset) 756 { 757 uint32_t type, bar; 758 int capreg, error; 759 760 for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg); 761 error == 0; 762 error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) { 763 764 type = pci_read_config(dev, capreg + 765 offsetof(struct virtio_pci_cap, cfg_type), 1); 766 bar = pci_read_config(dev, capreg + 767 offsetof(struct virtio_pci_cap, bar), 1); 768 769 /* Must ignore reserved BARs. */ 770 if (bar >= VTPCI_MODERN_MAX_BARS) 771 continue; 772 773 if (type == cfg_type) { 774 if (cap_offset != NULL) 775 *cap_offset = capreg; 776 break; 777 } 778 } 779 780 return (error); 781 } 782 783 static int 784 vtpci_modern_map_common_config(struct vtpci_modern_softc *sc) 785 { 786 device_t dev; 787 int error; 788 789 dev = sc->vtpci_dev; 790 791 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG, 792 sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map); 793 if (error) { 794 device_printf(dev, "cannot find cap COMMON_CFG resource\n"); 795 return (error); 796 } 797 798 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map); 799 if (error) { 800 device_printf(dev, "cannot alloc resource for COMMON_CFG\n"); 801 return (error); 802 } 803 804 return (0); 805 } 806 807 static int 808 vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc) 809 { 810 device_t dev; 811 int cap_offset, error; 812 813 dev = sc->vtpci_dev; 814 815 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG, 816 -1, 2, &sc->vtpci_notify_res_map); 817 if (error) { 818 device_printf(dev, "cannot find cap NOTIFY_CFG resource\n"); 819 return (error); 820 } 821 822 cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset; 823 824 sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset + 825 offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4); 826 827 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map); 828 if (error) { 829 device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n"); 830 return (error); 831 } 832 833 return (0); 834 } 835 836 static int 837 vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc) 838 { 839 device_t dev; 840 int error; 841 842 dev = sc->vtpci_dev; 843 844 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG, 845 sizeof(uint8_t), 1, &sc->vtpci_isr_res_map); 846 if (error) { 847 device_printf(dev, "cannot find cap ISR_CFG resource\n"); 848 return (error); 849 } 850 851 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map); 852 if (error) { 853 device_printf(dev, "cannot alloc resource for ISR_CFG\n"); 854 return (error); 855 } 856 857 return (0); 858 } 859 860 static int 861 vtpci_modern_map_device_config(struct vtpci_modern_softc *sc) 862 { 863 device_t dev; 864 int error; 865 866 dev = sc->vtpci_dev; 867 868 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG, 869 -1, 4, &sc->vtpci_device_res_map); 870 if (error == ENOENT) { 871 /* Device configuration is optional depending on device. */ 872 return (0); 873 } else if (error) { 874 device_printf(dev, "cannot find cap DEVICE_CFG resource\n"); 875 return (error); 876 } 877 878 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map); 879 if (error) { 880 device_printf(dev, "cannot alloc resource for DEVICE_CFG\n"); 881 return (error); 882 } 883 884 return (0); 885 } 886 887 static int 888 vtpci_modern_map_configs(struct vtpci_modern_softc *sc) 889 { 890 int error; 891 892 error = vtpci_modern_map_common_config(sc); 893 if (error) 894 return (error); 895 896 error = vtpci_modern_map_notify_config(sc); 897 if (error) 898 return (error); 899 900 error = vtpci_modern_map_isr_config(sc); 901 if (error) 902 return (error); 903 904 error = vtpci_modern_map_device_config(sc); 905 if (error) 906 return (error); 907 908 vtpci_modern_alloc_msix_resource(sc); 909 910 return (0); 911 } 912 913 static void 914 vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc) 915 { 916 917 vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map); 918 vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map); 919 vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map); 920 vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map); 921 922 vtpci_modern_free_bar_resources(sc); 923 vtpci_modern_free_msix_resource(sc); 924 925 sc->vtpci_notify_offset_multiplier = 0; 926 } 927 928 static int 929 vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type, 930 int min_size, int alignment, struct vtpci_modern_resource_map *res) 931 { 932 device_t dev; 933 int cap_offset, offset, length, error; 934 uint8_t bar, cap_length; 935 936 dev = sc->vtpci_dev; 937 938 error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset); 939 if (error) 940 return (error); 941 942 cap_length = pci_read_config(dev, 943 cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1); 944 945 if (cap_length < sizeof(struct virtio_pci_cap)) { 946 device_printf(dev, "cap %u length %d less than expected\n", 947 cfg_type, cap_length); 948 return (ENXIO); 949 } 950 951 bar = pci_read_config(dev, 952 cap_offset + offsetof(struct virtio_pci_cap, bar), 1); 953 offset = pci_read_config(dev, 954 cap_offset + offsetof(struct virtio_pci_cap, offset), 4); 955 length = pci_read_config(dev, 956 cap_offset + offsetof(struct virtio_pci_cap, length), 4); 957 958 if (min_size != -1 && length < min_size) { 959 device_printf(dev, "cap %u struct length %d less than min %d\n", 960 cfg_type, length, min_size); 961 return (ENXIO); 962 } 963 964 if (offset % alignment) { 965 device_printf(dev, "cap %u struct offset %d not aligned to %d\n", 966 cfg_type, offset, alignment); 967 return (ENXIO); 968 } 969 970 /* BMV: TODO Can we determine the size of the BAR here? */ 971 972 res->vtrm_cap_offset = cap_offset; 973 res->vtrm_bar = bar; 974 res->vtrm_offset = offset; 975 res->vtrm_length = length; 976 res->vtrm_type = vtpci_modern_bar_type(sc, bar); 977 978 return (0); 979 } 980 981 static int 982 vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar) 983 { 984 uint32_t val; 985 986 /* 987 * The BAR described by a config capability may be either an IOPORT or 988 * MEM, but we must know the type when calling bus_alloc_resource(). 989 */ 990 val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4); 991 if (PCI_BAR_IO(val)) 992 return (SYS_RES_IOPORT); 993 else 994 return (SYS_RES_MEMORY); 995 } 996 997 static struct resource * 998 vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) 999 { 1000 struct resource *res; 1001 1002 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1003 res = sc->vtpci_bar_res[bar].vtbr_res; 1004 MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type); 1005 1006 return (res); 1007 } 1008 1009 static struct resource * 1010 vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar, 1011 int type) 1012 { 1013 struct resource *res; 1014 int rid; 1015 1016 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1017 MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT); 1018 1019 res = sc->vtpci_bar_res[bar].vtbr_res; 1020 if (res != NULL) { 1021 MPASS(sc->vtpci_bar_res[bar].vtbr_type == type); 1022 return (res); 1023 } 1024 1025 rid = PCIR_BAR(bar); 1026 res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid, 1027 RF_ACTIVE | RF_UNMAPPED); 1028 if (res != NULL) { 1029 sc->vtpci_bar_res[bar].vtbr_res = res; 1030 sc->vtpci_bar_res[bar].vtbr_type = type; 1031 } 1032 1033 return (res); 1034 } 1035 1036 static void 1037 vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc) 1038 { 1039 device_t dev; 1040 struct resource *res; 1041 int bar, rid, type; 1042 1043 dev = sc->vtpci_dev; 1044 1045 for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) { 1046 res = sc->vtpci_bar_res[bar].vtbr_res; 1047 type = sc->vtpci_bar_res[bar].vtbr_type; 1048 1049 if (res != NULL) { 1050 rid = PCIR_BAR(bar); 1051 bus_release_resource(dev, type, rid, res); 1052 sc->vtpci_bar_res[bar].vtbr_res = NULL; 1053 sc->vtpci_bar_res[bar].vtbr_type = 0; 1054 } 1055 } 1056 } 1057 1058 static int 1059 vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc, 1060 struct vtpci_modern_resource_map *map) 1061 { 1062 struct resource_map_request req; 1063 struct resource *res; 1064 int type; 1065 1066 type = map->vtrm_type; 1067 1068 res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type); 1069 if (res == NULL) 1070 return (ENXIO); 1071 1072 resource_init_map_request(&req); 1073 req.offset = map->vtrm_offset; 1074 req.length = map->vtrm_length; 1075 1076 return (bus_map_resource(sc->vtpci_dev, type, res, &req, 1077 &map->vtrm_map)); 1078 } 1079 1080 static void 1081 vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc, 1082 struct vtpci_modern_resource_map *map) 1083 { 1084 struct resource *res; 1085 int type; 1086 1087 type = map->vtrm_type; 1088 res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type); 1089 1090 if (res != NULL && map->vtrm_map.r_size != 0) { 1091 bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map); 1092 bzero(map, sizeof(struct vtpci_modern_resource_map)); 1093 } 1094 } 1095 1096 static void 1097 vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc) 1098 { 1099 device_t dev; 1100 int bar; 1101 1102 dev = sc->vtpci_dev; 1103 1104 if (!vtpci_is_msix_available(&sc->vtpci_common) || 1105 (bar = pci_msix_table_bar(dev)) == -1) 1106 return; 1107 1108 /* TODO: Can this BAR be in the 0-5 range? */ 1109 sc->vtpci_msix_bar = bar; 1110 if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1111 &bar, RF_ACTIVE)) == NULL) 1112 device_printf(dev, "Unable to map MSIX table\n"); 1113 } 1114 1115 static void 1116 vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc) 1117 { 1118 device_t dev; 1119 1120 dev = sc->vtpci_dev; 1121 1122 if (sc->vtpci_msix_res != NULL) { 1123 bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar, 1124 sc->vtpci_msix_res); 1125 sc->vtpci_msix_bar = 0; 1126 sc->vtpci_msix_res = NULL; 1127 } 1128 } 1129 1130 static void 1131 vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc) 1132 { 1133 device_t dev, child; 1134 1135 dev = sc->vtpci_dev; 1136 child = vtpci_child_device(&sc->vtpci_common); 1137 1138 if (child == NULL || device_get_state(child) != DS_NOTPRESENT) 1139 return; 1140 1141 if (device_probe(child) != 0) 1142 return; 1143 1144 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 1145 1146 if (device_attach(child) != 0) { 1147 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 1148 /* Reset state for later attempt. */ 1149 vtpci_modern_child_detached(dev, child); 1150 } else { 1151 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1152 VIRTIO_ATTACH_COMPLETED(child); 1153 } 1154 } 1155 1156 static int 1157 vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset, 1158 struct vtpci_interrupt *intr) 1159 { 1160 uint16_t vector; 1161 1162 if (intr != NULL) { 1163 /* Map from guest rid to host vector. */ 1164 vector = intr->vti_rid - 1; 1165 } else 1166 vector = VIRTIO_MSI_NO_VECTOR; 1167 1168 vtpci_modern_write_common_2(sc, offset, vector); 1169 return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV); 1170 } 1171 1172 static int 1173 vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) 1174 { 1175 struct vtpci_modern_softc *sc; 1176 int error; 1177 1178 sc = device_get_softc(dev); 1179 1180 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr); 1181 if (error) { 1182 device_printf(dev, 1183 "unable to register config MSIX interrupt\n"); 1184 return (error); 1185 } 1186 1187 return (0); 1188 } 1189 1190 static int 1191 vtpci_modern_register_vq_msix(device_t dev, int idx, 1192 struct vtpci_interrupt *intr) 1193 { 1194 struct vtpci_modern_softc *sc; 1195 int error; 1196 1197 sc = device_get_softc(dev); 1198 1199 vtpci_modern_select_virtqueue(sc, idx); 1200 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr); 1201 if (error) { 1202 device_printf(dev, 1203 "unable to register virtqueue MSIX interrupt\n"); 1204 return (error); 1205 } 1206 1207 return (0); 1208 } 1209 1210 static void 1211 vtpci_modern_reset(struct vtpci_modern_softc *sc) 1212 { 1213 /* 1214 * Setting the status to RESET sets the host device to the 1215 * original, uninitialized state. Must poll the status until 1216 * the reset is complete. 1217 */ 1218 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); 1219 1220 while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 1221 cpu_spinwait(); 1222 } 1223 1224 static void 1225 vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx) 1226 { 1227 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx); 1228 } 1229 1230 static uint8_t 1231 vtpci_modern_read_isr(device_t dev) 1232 { 1233 return (vtpci_modern_read_isr_1(device_get_softc(dev), 0)); 1234 } 1235 1236 static uint16_t 1237 vtpci_modern_get_vq_size(device_t dev, int idx) 1238 { 1239 struct vtpci_modern_softc *sc; 1240 1241 sc = device_get_softc(dev); 1242 1243 vtpci_modern_select_virtqueue(sc, idx); 1244 return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE)); 1245 } 1246 1247 static bus_size_t 1248 vtpci_modern_get_vq_notify_off(device_t dev, int idx) 1249 { 1250 struct vtpci_modern_softc *sc; 1251 uint16_t q_notify_off; 1252 1253 sc = device_get_softc(dev); 1254 1255 vtpci_modern_select_virtqueue(sc, idx); 1256 q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF); 1257 1258 return (q_notify_off * sc->vtpci_notify_offset_multiplier); 1259 } 1260 1261 static void 1262 vtpci_modern_set_vq(device_t dev, struct virtqueue *vq) 1263 { 1264 struct vtpci_modern_softc *sc; 1265 1266 sc = device_get_softc(dev); 1267 1268 vtpci_modern_select_virtqueue(sc, virtqueue_index(vq)); 1269 1270 /* BMV: Currently we never adjust the device's proposed VQ size. */ 1271 vtpci_modern_write_common_2(sc, 1272 VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq)); 1273 1274 vtpci_modern_write_common_8(sc, 1275 VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq)); 1276 vtpci_modern_write_common_8(sc, 1277 VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq)); 1278 vtpci_modern_write_common_8(sc, 1279 VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq)); 1280 } 1281 1282 static void 1283 vtpci_modern_disable_vq(device_t dev, int idx) 1284 { 1285 struct vtpci_modern_softc *sc; 1286 1287 sc = device_get_softc(dev); 1288 1289 vtpci_modern_select_virtqueue(sc, idx); 1290 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL); 1291 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL); 1292 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL); 1293 } 1294 1295 static void 1296 vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc) 1297 { 1298 int idx; 1299 1300 for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) { 1301 vtpci_modern_select_virtqueue(sc, idx); 1302 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1); 1303 } 1304 } 1305 1306 static uint8_t 1307 vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off) 1308 { 1309 return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off)); 1310 } 1311 1312 static uint16_t 1313 vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off) 1314 { 1315 return virtio_htog16(true, 1316 bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off)); 1317 } 1318 1319 static uint32_t 1320 vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off) 1321 { 1322 return virtio_htog32(true, 1323 bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off)); 1324 } 1325 1326 static void 1327 vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off, 1328 uint8_t val) 1329 { 1330 bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val); 1331 } 1332 1333 static void 1334 vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off, 1335 uint16_t val) 1336 { 1337 bus_write_2(&sc->vtpci_common_res_map.vtrm_map, 1338 off, virtio_gtoh16(true, val)); 1339 } 1340 1341 static void 1342 vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off, 1343 uint32_t val) 1344 { 1345 bus_write_4(&sc->vtpci_common_res_map.vtrm_map, 1346 off, virtio_gtoh32(true, val)); 1347 } 1348 1349 static void 1350 vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off, 1351 uint64_t val) 1352 { 1353 uint32_t val0, val1; 1354 1355 val0 = (uint32_t) val; 1356 val1 = val >> 32; 1357 1358 vtpci_modern_write_common_4(sc, off, val0); 1359 vtpci_modern_write_common_4(sc, off + 4, val1); 1360 } 1361 1362 static void 1363 vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off, 1364 uint16_t val) 1365 { 1366 bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val); 1367 } 1368 1369 static uint8_t 1370 vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off) 1371 { 1372 return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off)); 1373 } 1374 1375 static uint8_t 1376 vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off) 1377 { 1378 return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off)); 1379 } 1380 1381 static uint16_t 1382 vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off) 1383 { 1384 return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off)); 1385 } 1386 1387 static uint32_t 1388 vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off) 1389 { 1390 return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off)); 1391 } 1392 1393 static uint64_t 1394 vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off) 1395 { 1396 device_t dev; 1397 int gen; 1398 uint32_t val0, val1; 1399 1400 dev = sc->vtpci_dev; 1401 1402 /* 1403 * Treat the 64-bit field as two 32-bit fields. Use the generation 1404 * to ensure a consistent read. 1405 */ 1406 do { 1407 gen = vtpci_modern_config_generation(dev); 1408 val0 = vtpci_modern_read_device_4(sc, off); 1409 val1 = vtpci_modern_read_device_4(sc, off + 4); 1410 } while (gen != vtpci_modern_config_generation(dev)); 1411 1412 return (((uint64_t) val1 << 32) | val0); 1413 } 1414 1415 static void 1416 vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off, 1417 uint8_t val) 1418 { 1419 bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val); 1420 } 1421 1422 static void 1423 vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off, 1424 uint16_t val) 1425 { 1426 bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val); 1427 } 1428 1429 static void 1430 vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off, 1431 uint32_t val) 1432 { 1433 bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val); 1434 } 1435 1436 static void 1437 vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off, 1438 uint64_t val) 1439 { 1440 uint32_t val0, val1; 1441 1442 val0 = (uint32_t) val; 1443 val1 = val >> 32; 1444 1445 vtpci_modern_write_device_4(sc, off, val0); 1446 vtpci_modern_write_device_4(sc, off + 4, val1); 1447 } 1448