1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for the modern VirtIO PCI interface. */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/bus.h> 34 #include <sys/lock.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 38 #include <machine/bus.h> 39 #include <machine/cpu.h> 40 #include <machine/resource.h> 41 #include <sys/bus.h> 42 #include <sys/rman.h> 43 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pcireg.h> 46 47 #include <dev/virtio/virtio.h> 48 #include <dev/virtio/virtqueue.h> 49 #include <dev/virtio/pci/virtio_pci.h> 50 #include <dev/virtio/pci/virtio_pci_modern_var.h> 51 52 #include "virtio_bus_if.h" 53 #include "virtio_pci_if.h" 54 #include "virtio_if.h" 55 56 struct vtpci_modern_resource_map { 57 struct resource_map vtrm_map; 58 int vtrm_cap_offset; 59 int vtrm_bar; 60 int vtrm_offset; 61 int vtrm_length; 62 int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */ 63 }; 64 65 struct vtpci_modern_bar_resource { 66 struct resource *vtbr_res; 67 int vtbr_type; 68 }; 69 70 struct vtpci_modern_softc { 71 device_t vtpci_dev; 72 struct vtpci_common vtpci_common; 73 uint32_t vtpci_notify_offset_multiplier; 74 uint16_t vtpci_devid; 75 int vtpci_msix_bar; 76 struct resource *vtpci_msix_res; 77 78 struct vtpci_modern_resource_map vtpci_common_res_map; 79 struct vtpci_modern_resource_map vtpci_notify_res_map; 80 struct vtpci_modern_resource_map vtpci_isr_res_map; 81 struct vtpci_modern_resource_map vtpci_device_res_map; 82 83 #define VTPCI_MODERN_MAX_BARS 6 84 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS]; 85 }; 86 87 static int vtpci_modern_probe(device_t); 88 static int vtpci_modern_attach(device_t); 89 static int vtpci_modern_detach(device_t); 90 static int vtpci_modern_suspend(device_t); 91 static int vtpci_modern_resume(device_t); 92 static int vtpci_modern_shutdown(device_t); 93 94 static void vtpci_modern_driver_added(device_t, driver_t *); 95 static void vtpci_modern_child_detached(device_t, device_t); 96 static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *); 97 static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t); 98 99 static uint8_t vtpci_modern_read_isr(device_t); 100 static uint16_t vtpci_modern_get_vq_size(device_t, int); 101 static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int); 102 static void vtpci_modern_set_vq(device_t, struct virtqueue *); 103 static void vtpci_modern_disable_vq(device_t, int); 104 static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int, 105 struct vtpci_interrupt *); 106 static int vtpci_modern_register_cfg_msix(device_t, 107 struct vtpci_interrupt *); 108 static int vtpci_modern_register_vq_msix(device_t, int idx, 109 struct vtpci_interrupt *); 110 111 static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t); 112 static int vtpci_modern_finalize_features(device_t); 113 static bool vtpci_modern_with_feature(device_t, uint64_t); 114 static int vtpci_modern_alloc_virtqueues(device_t, int, 115 struct vq_alloc_info *); 116 static int vtpci_modern_setup_interrupts(device_t, enum intr_type); 117 static void vtpci_modern_stop(device_t); 118 static int vtpci_modern_reinit(device_t, uint64_t); 119 static void vtpci_modern_reinit_complete(device_t); 120 static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t); 121 static int vtpci_modern_config_generation(device_t); 122 static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int); 123 static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int); 124 125 static int vtpci_modern_probe_configs(device_t); 126 static int vtpci_modern_find_cap(device_t, uint8_t, int *); 127 static int vtpci_modern_map_configs(struct vtpci_modern_softc *); 128 static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *); 129 static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *, 130 uint8_t, int, int, struct vtpci_modern_resource_map *); 131 static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int); 132 static struct resource *vtpci_modern_get_bar_resource( 133 struct vtpci_modern_softc *, int, int); 134 static struct resource *vtpci_modern_alloc_bar_resource( 135 struct vtpci_modern_softc *, int, int); 136 static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *); 137 static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *, 138 struct vtpci_modern_resource_map *); 139 static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *, 140 struct vtpci_modern_resource_map *); 141 static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *); 142 static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *); 143 144 static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *); 145 146 static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *); 147 static void vtpci_modern_write_features(struct vtpci_modern_softc *, 148 uint64_t); 149 static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int); 150 static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *); 151 static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t); 152 static void vtpci_modern_reset(struct vtpci_modern_softc *); 153 static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *); 154 155 static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *, 156 bus_size_t); 157 static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *, 158 bus_size_t); 159 static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *, 160 bus_size_t); 161 static void vtpci_modern_write_common_1(struct vtpci_modern_softc *, 162 bus_size_t, uint8_t); 163 static void vtpci_modern_write_common_2(struct vtpci_modern_softc *, 164 bus_size_t, uint16_t); 165 static void vtpci_modern_write_common_4(struct vtpci_modern_softc *, 166 bus_size_t, uint32_t); 167 static void vtpci_modern_write_common_8(struct vtpci_modern_softc *, 168 bus_size_t, uint64_t); 169 static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *, 170 bus_size_t, uint16_t); 171 static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *, 172 bus_size_t); 173 static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *, 174 bus_size_t); 175 static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *, 176 bus_size_t); 177 static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *, 178 bus_size_t); 179 static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *, 180 bus_size_t); 181 static void vtpci_modern_write_device_1(struct vtpci_modern_softc *, 182 bus_size_t, uint8_t); 183 static void vtpci_modern_write_device_2(struct vtpci_modern_softc *, 184 bus_size_t, uint16_t); 185 static void vtpci_modern_write_device_4(struct vtpci_modern_softc *, 186 bus_size_t, uint32_t); 187 static void vtpci_modern_write_device_8(struct vtpci_modern_softc *, 188 bus_size_t, uint64_t); 189 190 /* Tunables. */ 191 static int vtpci_modern_transitional = 0; 192 TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional); 193 194 static device_method_t vtpci_modern_methods[] = { 195 /* Device interface. */ 196 DEVMETHOD(device_probe, vtpci_modern_probe), 197 DEVMETHOD(device_attach, vtpci_modern_attach), 198 DEVMETHOD(device_detach, vtpci_modern_detach), 199 DEVMETHOD(device_suspend, vtpci_modern_suspend), 200 DEVMETHOD(device_resume, vtpci_modern_resume), 201 DEVMETHOD(device_shutdown, vtpci_modern_shutdown), 202 203 /* Bus interface. */ 204 DEVMETHOD(bus_driver_added, vtpci_modern_driver_added), 205 DEVMETHOD(bus_child_detached, vtpci_modern_child_detached), 206 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), 207 DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar), 208 DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar), 209 210 /* VirtIO PCI interface. */ 211 DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr), 212 DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size), 213 DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off), 214 DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq), 215 DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq), 216 DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix), 217 DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix), 218 219 /* VirtIO bus interface. */ 220 DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features), 221 DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features), 222 DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature), 223 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues), 224 DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts), 225 DEVMETHOD(virtio_bus_stop, vtpci_modern_stop), 226 DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit), 227 DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete), 228 DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq), 229 DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation), 230 DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config), 231 DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config), 232 233 DEVMETHOD_END 234 }; 235 236 static driver_t vtpci_modern_driver = { 237 .name = "virtio_pci", 238 .methods = vtpci_modern_methods, 239 .size = sizeof(struct vtpci_modern_softc) 240 }; 241 242 DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0); 243 244 static int 245 vtpci_modern_probe(device_t dev) 246 { 247 const char *name; 248 uint16_t devid; 249 250 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 251 return (ENXIO); 252 253 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 254 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX) 255 return (ENXIO); 256 257 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) { 258 if (!vtpci_modern_transitional) 259 return (ENXIO); 260 devid = pci_get_subdevice(dev); 261 } else 262 devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; 263 264 if (vtpci_modern_probe_configs(dev) != 0) 265 return (ENXIO); 266 267 name = virtio_device_name(devid); 268 if (name == NULL) 269 name = "Unknown"; 270 271 device_set_descf(dev, "VirtIO PCI (modern) %s adapter", name); 272 273 return (BUS_PROBE_DEFAULT); 274 } 275 276 static int 277 vtpci_modern_attach(device_t dev) 278 { 279 struct vtpci_modern_softc *sc; 280 int error; 281 282 sc = device_get_softc(dev); 283 sc->vtpci_dev = dev; 284 vtpci_init(&sc->vtpci_common, dev, true); 285 286 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) 287 sc->vtpci_devid = pci_get_subdevice(dev); 288 else 289 sc->vtpci_devid = pci_get_device(dev) - 290 VIRTIO_PCI_DEVICEID_MODERN_MIN; 291 292 error = vtpci_modern_map_configs(sc); 293 if (error) { 294 device_printf(dev, "cannot map configs\n"); 295 vtpci_modern_unmap_configs(sc); 296 return (error); 297 } 298 299 vtpci_modern_reset(sc); 300 301 /* Tell the host we've noticed this device. */ 302 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 303 304 error = vtpci_add_child(&sc->vtpci_common); 305 if (error) 306 goto fail; 307 308 vtpci_modern_probe_and_attach_child(sc); 309 310 return (0); 311 312 fail: 313 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 314 vtpci_modern_detach(dev); 315 316 return (error); 317 } 318 319 static int 320 vtpci_modern_detach(device_t dev) 321 { 322 struct vtpci_modern_softc *sc; 323 int error; 324 325 sc = device_get_softc(dev); 326 327 error = vtpci_delete_child(&sc->vtpci_common); 328 if (error) 329 return (error); 330 331 vtpci_modern_reset(sc); 332 vtpci_modern_unmap_configs(sc); 333 334 return (0); 335 } 336 337 static int 338 vtpci_modern_suspend(device_t dev) 339 { 340 return (bus_generic_suspend(dev)); 341 } 342 343 static int 344 vtpci_modern_resume(device_t dev) 345 { 346 return (bus_generic_resume(dev)); 347 } 348 349 static int 350 vtpci_modern_shutdown(device_t dev) 351 { 352 (void) bus_generic_shutdown(dev); 353 /* Forcibly stop the host device. */ 354 vtpci_modern_stop(dev); 355 356 return (0); 357 } 358 359 static void 360 vtpci_modern_driver_added(device_t dev, driver_t *driver) 361 { 362 vtpci_modern_probe_and_attach_child(device_get_softc(dev)); 363 } 364 365 static void 366 vtpci_modern_child_detached(device_t dev, device_t child) 367 { 368 struct vtpci_modern_softc *sc; 369 370 sc = device_get_softc(dev); 371 372 vtpci_modern_reset(sc); 373 vtpci_child_detached(&sc->vtpci_common); 374 375 /* After the reset, retell the host we've noticed this device. */ 376 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 377 } 378 379 static int 380 vtpci_modern_read_ivar(device_t dev, device_t child, int index, 381 uintptr_t *result) 382 { 383 struct vtpci_modern_softc *sc; 384 struct vtpci_common *cn; 385 386 sc = device_get_softc(dev); 387 cn = &sc->vtpci_common; 388 389 if (vtpci_child_device(cn) != child) 390 return (ENOENT); 391 392 switch (index) { 393 case VIRTIO_IVAR_DEVTYPE: 394 *result = sc->vtpci_devid; 395 break; 396 default: 397 return (vtpci_read_ivar(cn, index, result)); 398 } 399 400 return (0); 401 } 402 403 static int 404 vtpci_modern_write_ivar(device_t dev, device_t child, int index, 405 uintptr_t value) 406 { 407 struct vtpci_modern_softc *sc; 408 struct vtpci_common *cn; 409 410 sc = device_get_softc(dev); 411 cn = &sc->vtpci_common; 412 413 if (vtpci_child_device(cn) != child) 414 return (ENOENT); 415 416 switch (index) { 417 default: 418 return (vtpci_write_ivar(cn, index, value)); 419 } 420 421 return (0); 422 } 423 424 static uint64_t 425 vtpci_modern_negotiate_features(device_t dev, uint64_t child_features) 426 { 427 struct vtpci_modern_softc *sc; 428 uint64_t host_features, features; 429 430 sc = device_get_softc(dev); 431 host_features = vtpci_modern_read_features(sc); 432 433 /* 434 * Since the driver was added as a child of the modern PCI bus, 435 * always add the V1 flag. 436 */ 437 child_features |= VIRTIO_F_VERSION_1; 438 439 features = vtpci_negotiate_features(&sc->vtpci_common, 440 child_features, host_features); 441 vtpci_modern_write_features(sc, features); 442 443 return (features); 444 } 445 446 static int 447 vtpci_modern_finalize_features(device_t dev) 448 { 449 struct vtpci_modern_softc *sc; 450 uint8_t status; 451 452 sc = device_get_softc(dev); 453 454 /* 455 * Must re-read the status after setting it to verify the negotiated 456 * features were accepted by the device. 457 */ 458 vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK); 459 460 status = vtpci_modern_get_status(sc); 461 if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { 462 device_printf(dev, "desired features were not accepted\n"); 463 return (ENOTSUP); 464 } 465 466 return (0); 467 } 468 469 static bool 470 vtpci_modern_with_feature(device_t dev, uint64_t feature) 471 { 472 struct vtpci_modern_softc *sc; 473 474 sc = device_get_softc(dev); 475 476 return (vtpci_with_feature(&sc->vtpci_common, feature)); 477 } 478 479 static uint64_t 480 vtpci_modern_read_features(struct vtpci_modern_softc *sc) 481 { 482 uint32_t features0, features1; 483 484 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0); 485 features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 486 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1); 487 features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 488 489 return (((uint64_t) features1 << 32) | features0); 490 } 491 492 static void 493 vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features) 494 { 495 uint32_t features0, features1; 496 497 features0 = features; 498 features1 = features >> 32; 499 500 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0); 501 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0); 502 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1); 503 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1); 504 } 505 506 static int 507 vtpci_modern_alloc_virtqueues(device_t dev, int nvqs, 508 struct vq_alloc_info *vq_info) 509 { 510 struct vtpci_modern_softc *sc; 511 struct vtpci_common *cn; 512 uint16_t max_nvqs; 513 514 sc = device_get_softc(dev); 515 cn = &sc->vtpci_common; 516 517 max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ); 518 if (nvqs > max_nvqs) { 519 device_printf(sc->vtpci_dev, "requested virtqueue count %d " 520 "exceeds max %d\n", nvqs, max_nvqs); 521 return (E2BIG); 522 } 523 524 return (vtpci_alloc_virtqueues(cn, nvqs, vq_info)); 525 } 526 527 static int 528 vtpci_modern_setup_interrupts(device_t dev, enum intr_type type) 529 { 530 struct vtpci_modern_softc *sc; 531 int error; 532 533 sc = device_get_softc(dev); 534 535 error = vtpci_setup_interrupts(&sc->vtpci_common, type); 536 if (error == 0) 537 vtpci_modern_enable_virtqueues(sc); 538 539 return (error); 540 } 541 542 static void 543 vtpci_modern_stop(device_t dev) 544 { 545 vtpci_modern_reset(device_get_softc(dev)); 546 } 547 548 static int 549 vtpci_modern_reinit(device_t dev, uint64_t features) 550 { 551 struct vtpci_modern_softc *sc; 552 struct vtpci_common *cn; 553 int error; 554 555 sc = device_get_softc(dev); 556 cn = &sc->vtpci_common; 557 558 /* 559 * Redrive the device initialization. This is a bit of an abuse of 560 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 561 * play nice. 562 * 563 * We do not allow the host device to change from what was originally 564 * negotiated beyond what the guest driver changed. MSIX state should 565 * not change, number of virtqueues and their size remain the same, etc. 566 * This will need to be rethought when we want to support migration. 567 */ 568 569 if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 570 vtpci_modern_stop(dev); 571 572 /* 573 * Quickly drive the status through ACK and DRIVER. The device does 574 * not become usable again until DRIVER_OK in reinit complete. 575 */ 576 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 577 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 578 579 /* 580 * TODO: Check that features are not added as to what was 581 * originally negotiated. 582 */ 583 vtpci_modern_negotiate_features(dev, features); 584 error = vtpci_modern_finalize_features(dev); 585 if (error) { 586 device_printf(dev, "cannot finalize features during reinit\n"); 587 return (error); 588 } 589 590 error = vtpci_reinit(cn); 591 if (error) 592 return (error); 593 594 return (0); 595 } 596 597 static void 598 vtpci_modern_reinit_complete(device_t dev) 599 { 600 struct vtpci_modern_softc *sc; 601 602 sc = device_get_softc(dev); 603 604 vtpci_modern_enable_virtqueues(sc); 605 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 606 } 607 608 static void 609 vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) 610 { 611 struct vtpci_modern_softc *sc; 612 613 sc = device_get_softc(dev); 614 615 vtpci_modern_write_notify_2(sc, offset, queue); 616 } 617 618 static uint8_t 619 vtpci_modern_get_status(struct vtpci_modern_softc *sc) 620 { 621 return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS)); 622 } 623 624 static void 625 vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status) 626 { 627 if (status != VIRTIO_CONFIG_STATUS_RESET) 628 status |= vtpci_modern_get_status(sc); 629 630 vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status); 631 } 632 633 static int 634 vtpci_modern_config_generation(device_t dev) 635 { 636 struct vtpci_modern_softc *sc; 637 uint8_t gen; 638 639 sc = device_get_softc(dev); 640 gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION); 641 642 return (gen); 643 } 644 645 static void 646 vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst, 647 int length) 648 { 649 struct vtpci_modern_softc *sc; 650 651 sc = device_get_softc(dev); 652 653 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 654 panic("%s: attempt to read dev config but not present", 655 __func__); 656 } 657 658 switch (length) { 659 case 1: 660 *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset); 661 break; 662 case 2: 663 *(uint16_t *) dst = virtio_htog16(true, 664 vtpci_modern_read_device_2(sc, offset)); 665 break; 666 case 4: 667 *(uint32_t *) dst = virtio_htog32(true, 668 vtpci_modern_read_device_4(sc, offset)); 669 break; 670 case 8: 671 *(uint64_t *) dst = virtio_htog64(true, 672 vtpci_modern_read_device_8(sc, offset)); 673 break; 674 default: 675 panic("%s: device %s invalid device read length %d offset %d", 676 __func__, device_get_nameunit(dev), length, (int) offset); 677 } 678 } 679 680 static void 681 vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src, 682 int length) 683 { 684 struct vtpci_modern_softc *sc; 685 686 sc = device_get_softc(dev); 687 688 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 689 panic("%s: attempt to write dev config but not present", 690 __func__); 691 } 692 693 switch (length) { 694 case 1: 695 vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src); 696 break; 697 case 2: { 698 uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src); 699 vtpci_modern_write_device_2(sc, offset, val); 700 break; 701 } 702 case 4: { 703 uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src); 704 vtpci_modern_write_device_4(sc, offset, val); 705 break; 706 } 707 case 8: { 708 uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src); 709 vtpci_modern_write_device_8(sc, offset, val); 710 break; 711 } 712 default: 713 panic("%s: device %s invalid device write length %d offset %d", 714 __func__, device_get_nameunit(dev), length, (int) offset); 715 } 716 } 717 718 static int 719 vtpci_modern_probe_configs(device_t dev) 720 { 721 int error; 722 723 /* 724 * These config capabilities must be present. The DEVICE_CFG 725 * capability is only present if the device requires it. 726 */ 727 728 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL); 729 if (error) { 730 device_printf(dev, "cannot find COMMON_CFG capability\n"); 731 return (error); 732 } 733 734 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL); 735 if (error) { 736 device_printf(dev, "cannot find NOTIFY_CFG capability\n"); 737 return (error); 738 } 739 740 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL); 741 if (error) { 742 device_printf(dev, "cannot find ISR_CFG capability\n"); 743 return (error); 744 } 745 746 return (0); 747 } 748 749 static int 750 vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset) 751 { 752 uint32_t type, bar; 753 int capreg, error; 754 755 for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg); 756 error == 0; 757 error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) { 758 759 type = pci_read_config(dev, capreg + 760 offsetof(struct virtio_pci_cap, cfg_type), 1); 761 bar = pci_read_config(dev, capreg + 762 offsetof(struct virtio_pci_cap, bar), 1); 763 764 /* Must ignore reserved BARs. */ 765 if (bar >= VTPCI_MODERN_MAX_BARS) 766 continue; 767 768 if (type == cfg_type) { 769 if (cap_offset != NULL) 770 *cap_offset = capreg; 771 break; 772 } 773 } 774 775 return (error); 776 } 777 778 static int 779 vtpci_modern_map_common_config(struct vtpci_modern_softc *sc) 780 { 781 device_t dev; 782 int error; 783 784 dev = sc->vtpci_dev; 785 786 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG, 787 sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map); 788 if (error) { 789 device_printf(dev, "cannot find cap COMMON_CFG resource\n"); 790 return (error); 791 } 792 793 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map); 794 if (error) { 795 device_printf(dev, "cannot alloc resource for COMMON_CFG\n"); 796 return (error); 797 } 798 799 return (0); 800 } 801 802 static int 803 vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc) 804 { 805 device_t dev; 806 int cap_offset, error; 807 808 dev = sc->vtpci_dev; 809 810 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG, 811 -1, 2, &sc->vtpci_notify_res_map); 812 if (error) { 813 device_printf(dev, "cannot find cap NOTIFY_CFG resource\n"); 814 return (error); 815 } 816 817 cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset; 818 819 sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset + 820 offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4); 821 822 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map); 823 if (error) { 824 device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n"); 825 return (error); 826 } 827 828 return (0); 829 } 830 831 static int 832 vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc) 833 { 834 device_t dev; 835 int error; 836 837 dev = sc->vtpci_dev; 838 839 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG, 840 sizeof(uint8_t), 1, &sc->vtpci_isr_res_map); 841 if (error) { 842 device_printf(dev, "cannot find cap ISR_CFG resource\n"); 843 return (error); 844 } 845 846 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map); 847 if (error) { 848 device_printf(dev, "cannot alloc resource for ISR_CFG\n"); 849 return (error); 850 } 851 852 return (0); 853 } 854 855 static int 856 vtpci_modern_map_device_config(struct vtpci_modern_softc *sc) 857 { 858 device_t dev; 859 int error; 860 861 dev = sc->vtpci_dev; 862 863 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG, 864 -1, 4, &sc->vtpci_device_res_map); 865 if (error == ENOENT) { 866 /* Device configuration is optional depending on device. */ 867 return (0); 868 } else if (error) { 869 device_printf(dev, "cannot find cap DEVICE_CFG resource\n"); 870 return (error); 871 } 872 873 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map); 874 if (error) { 875 device_printf(dev, "cannot alloc resource for DEVICE_CFG\n"); 876 return (error); 877 } 878 879 return (0); 880 } 881 882 static int 883 vtpci_modern_map_configs(struct vtpci_modern_softc *sc) 884 { 885 int error; 886 887 error = vtpci_modern_map_common_config(sc); 888 if (error) 889 return (error); 890 891 error = vtpci_modern_map_notify_config(sc); 892 if (error) 893 return (error); 894 895 error = vtpci_modern_map_isr_config(sc); 896 if (error) 897 return (error); 898 899 error = vtpci_modern_map_device_config(sc); 900 if (error) 901 return (error); 902 903 vtpci_modern_alloc_msix_resource(sc); 904 905 return (0); 906 } 907 908 static void 909 vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc) 910 { 911 912 vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map); 913 vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map); 914 vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map); 915 vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map); 916 917 vtpci_modern_free_bar_resources(sc); 918 vtpci_modern_free_msix_resource(sc); 919 920 sc->vtpci_notify_offset_multiplier = 0; 921 } 922 923 static int 924 vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type, 925 int min_size, int alignment, struct vtpci_modern_resource_map *res) 926 { 927 device_t dev; 928 int cap_offset, offset, length, error; 929 uint8_t bar, cap_length; 930 931 dev = sc->vtpci_dev; 932 933 error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset); 934 if (error) 935 return (error); 936 937 cap_length = pci_read_config(dev, 938 cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1); 939 940 if (cap_length < sizeof(struct virtio_pci_cap)) { 941 device_printf(dev, "cap %u length %d less than expected\n", 942 cfg_type, cap_length); 943 return (ENXIO); 944 } 945 946 bar = pci_read_config(dev, 947 cap_offset + offsetof(struct virtio_pci_cap, bar), 1); 948 offset = pci_read_config(dev, 949 cap_offset + offsetof(struct virtio_pci_cap, offset), 4); 950 length = pci_read_config(dev, 951 cap_offset + offsetof(struct virtio_pci_cap, length), 4); 952 953 if (min_size != -1 && length < min_size) { 954 device_printf(dev, "cap %u struct length %d less than min %d\n", 955 cfg_type, length, min_size); 956 return (ENXIO); 957 } 958 959 if (offset % alignment) { 960 device_printf(dev, "cap %u struct offset %d not aligned to %d\n", 961 cfg_type, offset, alignment); 962 return (ENXIO); 963 } 964 965 /* BMV: TODO Can we determine the size of the BAR here? */ 966 967 res->vtrm_cap_offset = cap_offset; 968 res->vtrm_bar = bar; 969 res->vtrm_offset = offset; 970 res->vtrm_length = length; 971 res->vtrm_type = vtpci_modern_bar_type(sc, bar); 972 973 return (0); 974 } 975 976 static int 977 vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar) 978 { 979 uint32_t val; 980 981 /* 982 * The BAR described by a config capability may be either an IOPORT or 983 * MEM, but we must know the type when calling bus_alloc_resource(). 984 */ 985 val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4); 986 if (PCI_BAR_IO(val)) 987 return (SYS_RES_IOPORT); 988 else 989 return (SYS_RES_MEMORY); 990 } 991 992 static struct resource * 993 vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) 994 { 995 struct resource *res; 996 997 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 998 res = sc->vtpci_bar_res[bar].vtbr_res; 999 MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type); 1000 1001 return (res); 1002 } 1003 1004 static struct resource * 1005 vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar, 1006 int type) 1007 { 1008 struct resource *res; 1009 int rid; 1010 1011 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1012 MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT); 1013 1014 res = sc->vtpci_bar_res[bar].vtbr_res; 1015 if (res != NULL) { 1016 MPASS(sc->vtpci_bar_res[bar].vtbr_type == type); 1017 return (res); 1018 } 1019 1020 rid = PCIR_BAR(bar); 1021 res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid, 1022 RF_ACTIVE | RF_UNMAPPED); 1023 if (res != NULL) { 1024 sc->vtpci_bar_res[bar].vtbr_res = res; 1025 sc->vtpci_bar_res[bar].vtbr_type = type; 1026 } 1027 1028 return (res); 1029 } 1030 1031 static void 1032 vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc) 1033 { 1034 device_t dev; 1035 struct resource *res; 1036 int bar, rid, type; 1037 1038 dev = sc->vtpci_dev; 1039 1040 for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) { 1041 res = sc->vtpci_bar_res[bar].vtbr_res; 1042 type = sc->vtpci_bar_res[bar].vtbr_type; 1043 1044 if (res != NULL) { 1045 rid = PCIR_BAR(bar); 1046 bus_release_resource(dev, type, rid, res); 1047 sc->vtpci_bar_res[bar].vtbr_res = NULL; 1048 sc->vtpci_bar_res[bar].vtbr_type = 0; 1049 } 1050 } 1051 } 1052 1053 static int 1054 vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc, 1055 struct vtpci_modern_resource_map *map) 1056 { 1057 struct resource_map_request req; 1058 struct resource *res; 1059 int type; 1060 1061 type = map->vtrm_type; 1062 1063 res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type); 1064 if (res == NULL) 1065 return (ENXIO); 1066 1067 resource_init_map_request(&req); 1068 req.offset = map->vtrm_offset; 1069 req.length = map->vtrm_length; 1070 1071 return (bus_map_resource(sc->vtpci_dev, type, res, &req, 1072 &map->vtrm_map)); 1073 } 1074 1075 static void 1076 vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc, 1077 struct vtpci_modern_resource_map *map) 1078 { 1079 struct resource *res; 1080 int type; 1081 1082 type = map->vtrm_type; 1083 res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type); 1084 1085 if (res != NULL && map->vtrm_map.r_size != 0) { 1086 bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map); 1087 bzero(map, sizeof(struct vtpci_modern_resource_map)); 1088 } 1089 } 1090 1091 static void 1092 vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc) 1093 { 1094 device_t dev; 1095 int bar; 1096 1097 dev = sc->vtpci_dev; 1098 1099 if (!vtpci_is_msix_available(&sc->vtpci_common) || 1100 (bar = pci_msix_table_bar(dev)) == -1) 1101 return; 1102 1103 /* TODO: Can this BAR be in the 0-5 range? */ 1104 sc->vtpci_msix_bar = bar; 1105 if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1106 &bar, RF_ACTIVE)) == NULL) 1107 device_printf(dev, "Unable to map MSIX table\n"); 1108 } 1109 1110 static void 1111 vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc) 1112 { 1113 device_t dev; 1114 1115 dev = sc->vtpci_dev; 1116 1117 if (sc->vtpci_msix_res != NULL) { 1118 bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar, 1119 sc->vtpci_msix_res); 1120 sc->vtpci_msix_bar = 0; 1121 sc->vtpci_msix_res = NULL; 1122 } 1123 } 1124 1125 static void 1126 vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc) 1127 { 1128 device_t dev, child; 1129 1130 dev = sc->vtpci_dev; 1131 child = vtpci_child_device(&sc->vtpci_common); 1132 1133 if (child == NULL || device_get_state(child) != DS_NOTPRESENT) 1134 return; 1135 1136 if (device_probe(child) != 0) 1137 return; 1138 1139 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 1140 1141 if (device_attach(child) != 0) { 1142 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 1143 /* Reset state for later attempt. */ 1144 vtpci_modern_child_detached(dev, child); 1145 } else { 1146 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1147 VIRTIO_ATTACH_COMPLETED(child); 1148 } 1149 } 1150 1151 static int 1152 vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset, 1153 struct vtpci_interrupt *intr) 1154 { 1155 uint16_t vector; 1156 1157 if (intr != NULL) { 1158 /* Map from guest rid to host vector. */ 1159 vector = intr->vti_rid - 1; 1160 } else 1161 vector = VIRTIO_MSI_NO_VECTOR; 1162 1163 vtpci_modern_write_common_2(sc, offset, vector); 1164 return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV); 1165 } 1166 1167 static int 1168 vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) 1169 { 1170 struct vtpci_modern_softc *sc; 1171 int error; 1172 1173 sc = device_get_softc(dev); 1174 1175 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr); 1176 if (error) { 1177 device_printf(dev, 1178 "unable to register config MSIX interrupt\n"); 1179 return (error); 1180 } 1181 1182 return (0); 1183 } 1184 1185 static int 1186 vtpci_modern_register_vq_msix(device_t dev, int idx, 1187 struct vtpci_interrupt *intr) 1188 { 1189 struct vtpci_modern_softc *sc; 1190 int error; 1191 1192 sc = device_get_softc(dev); 1193 1194 vtpci_modern_select_virtqueue(sc, idx); 1195 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr); 1196 if (error) { 1197 device_printf(dev, 1198 "unable to register virtqueue MSIX interrupt\n"); 1199 return (error); 1200 } 1201 1202 return (0); 1203 } 1204 1205 static void 1206 vtpci_modern_reset(struct vtpci_modern_softc *sc) 1207 { 1208 /* 1209 * Setting the status to RESET sets the host device to the 1210 * original, uninitialized state. Must poll the status until 1211 * the reset is complete. 1212 */ 1213 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); 1214 1215 while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 1216 cpu_spinwait(); 1217 } 1218 1219 static void 1220 vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx) 1221 { 1222 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx); 1223 } 1224 1225 static uint8_t 1226 vtpci_modern_read_isr(device_t dev) 1227 { 1228 return (vtpci_modern_read_isr_1(device_get_softc(dev), 0)); 1229 } 1230 1231 static uint16_t 1232 vtpci_modern_get_vq_size(device_t dev, int idx) 1233 { 1234 struct vtpci_modern_softc *sc; 1235 1236 sc = device_get_softc(dev); 1237 1238 vtpci_modern_select_virtqueue(sc, idx); 1239 return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE)); 1240 } 1241 1242 static bus_size_t 1243 vtpci_modern_get_vq_notify_off(device_t dev, int idx) 1244 { 1245 struct vtpci_modern_softc *sc; 1246 uint16_t q_notify_off; 1247 1248 sc = device_get_softc(dev); 1249 1250 vtpci_modern_select_virtqueue(sc, idx); 1251 q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF); 1252 1253 return (q_notify_off * sc->vtpci_notify_offset_multiplier); 1254 } 1255 1256 static void 1257 vtpci_modern_set_vq(device_t dev, struct virtqueue *vq) 1258 { 1259 struct vtpci_modern_softc *sc; 1260 1261 sc = device_get_softc(dev); 1262 1263 vtpci_modern_select_virtqueue(sc, virtqueue_index(vq)); 1264 1265 /* BMV: Currently we never adjust the device's proposed VQ size. */ 1266 vtpci_modern_write_common_2(sc, 1267 VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq)); 1268 1269 vtpci_modern_write_common_8(sc, 1270 VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq)); 1271 vtpci_modern_write_common_8(sc, 1272 VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq)); 1273 vtpci_modern_write_common_8(sc, 1274 VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq)); 1275 } 1276 1277 static void 1278 vtpci_modern_disable_vq(device_t dev, int idx) 1279 { 1280 struct vtpci_modern_softc *sc; 1281 1282 sc = device_get_softc(dev); 1283 1284 vtpci_modern_select_virtqueue(sc, idx); 1285 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL); 1286 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL); 1287 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL); 1288 } 1289 1290 static void 1291 vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc) 1292 { 1293 int idx; 1294 1295 for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) { 1296 vtpci_modern_select_virtqueue(sc, idx); 1297 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1); 1298 } 1299 } 1300 1301 static uint8_t 1302 vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off) 1303 { 1304 return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off)); 1305 } 1306 1307 static uint16_t 1308 vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off) 1309 { 1310 return virtio_htog16(true, 1311 bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off)); 1312 } 1313 1314 static uint32_t 1315 vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off) 1316 { 1317 return virtio_htog32(true, 1318 bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off)); 1319 } 1320 1321 static void 1322 vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off, 1323 uint8_t val) 1324 { 1325 bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val); 1326 } 1327 1328 static void 1329 vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off, 1330 uint16_t val) 1331 { 1332 bus_write_2(&sc->vtpci_common_res_map.vtrm_map, 1333 off, virtio_gtoh16(true, val)); 1334 } 1335 1336 static void 1337 vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off, 1338 uint32_t val) 1339 { 1340 bus_write_4(&sc->vtpci_common_res_map.vtrm_map, 1341 off, virtio_gtoh32(true, val)); 1342 } 1343 1344 static void 1345 vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off, 1346 uint64_t val) 1347 { 1348 uint32_t val0, val1; 1349 1350 val0 = (uint32_t) val; 1351 val1 = val >> 32; 1352 1353 vtpci_modern_write_common_4(sc, off, val0); 1354 vtpci_modern_write_common_4(sc, off + 4, val1); 1355 } 1356 1357 static void 1358 vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off, 1359 uint16_t val) 1360 { 1361 bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val); 1362 } 1363 1364 static uint8_t 1365 vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off) 1366 { 1367 return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off)); 1368 } 1369 1370 static uint8_t 1371 vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off) 1372 { 1373 return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off)); 1374 } 1375 1376 static uint16_t 1377 vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off) 1378 { 1379 return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off)); 1380 } 1381 1382 static uint32_t 1383 vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off) 1384 { 1385 return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off)); 1386 } 1387 1388 static uint64_t 1389 vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off) 1390 { 1391 device_t dev; 1392 int gen; 1393 uint32_t val0, val1; 1394 1395 dev = sc->vtpci_dev; 1396 1397 /* 1398 * Treat the 64-bit field as two 32-bit fields. Use the generation 1399 * to ensure a consistent read. 1400 */ 1401 do { 1402 gen = vtpci_modern_config_generation(dev); 1403 val0 = vtpci_modern_read_device_4(sc, off); 1404 val1 = vtpci_modern_read_device_4(sc, off + 4); 1405 } while (gen != vtpci_modern_config_generation(dev)); 1406 1407 return (((uint64_t) val1 << 32) | val0); 1408 } 1409 1410 static void 1411 vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off, 1412 uint8_t val) 1413 { 1414 bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val); 1415 } 1416 1417 static void 1418 vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off, 1419 uint16_t val) 1420 { 1421 bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val); 1422 } 1423 1424 static void 1425 vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off, 1426 uint32_t val) 1427 { 1428 bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val); 1429 } 1430 1431 static void 1432 vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off, 1433 uint64_t val) 1434 { 1435 uint32_t val0, val1; 1436 1437 val0 = (uint32_t) val; 1438 val1 = val >> 32; 1439 1440 vtpci_modern_write_device_4(sc, off, val0); 1441 vtpci_modern_write_device_4(sc, off + 4, val1); 1442 } 1443