1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for the modern VirtIO PCI interface. */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/bus.h> 34 #include <sys/lock.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 38 #include <machine/bus.h> 39 #include <machine/cpu.h> 40 #include <machine/resource.h> 41 #include <sys/bus.h> 42 #include <sys/rman.h> 43 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pcireg.h> 46 47 #include <dev/virtio/virtio.h> 48 #include <dev/virtio/virtqueue.h> 49 #include <dev/virtio/pci/virtio_pci.h> 50 #include <dev/virtio/pci/virtio_pci_modern_var.h> 51 52 #include "virtio_bus_if.h" 53 #include "virtio_pci_if.h" 54 #include "virtio_if.h" 55 56 struct vtpci_modern_resource_map { 57 struct resource_map vtrm_map; 58 int vtrm_cap_offset; 59 int vtrm_bar; 60 int vtrm_offset; 61 int vtrm_length; 62 int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */ 63 }; 64 65 struct vtpci_modern_bar_resource { 66 struct resource *vtbr_res; 67 int vtbr_type; 68 }; 69 70 struct vtpci_modern_softc { 71 device_t vtpci_dev; 72 struct vtpci_common vtpci_common; 73 uint32_t vtpci_notify_offset_multiplier; 74 uint16_t vtpci_devid; 75 int vtpci_msix_bar; 76 struct resource *vtpci_msix_res; 77 78 struct vtpci_modern_resource_map vtpci_common_res_map; 79 struct vtpci_modern_resource_map vtpci_notify_res_map; 80 struct vtpci_modern_resource_map vtpci_isr_res_map; 81 struct vtpci_modern_resource_map vtpci_device_res_map; 82 83 #define VTPCI_MODERN_MAX_BARS 6 84 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS]; 85 }; 86 87 static int vtpci_modern_probe(device_t); 88 static int vtpci_modern_attach(device_t); 89 static int vtpci_modern_detach(device_t); 90 static int vtpci_modern_suspend(device_t); 91 static int vtpci_modern_resume(device_t); 92 static int vtpci_modern_shutdown(device_t); 93 94 static void vtpci_modern_driver_added(device_t, driver_t *); 95 static void vtpci_modern_child_detached(device_t, device_t); 96 static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *); 97 static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t); 98 99 static uint8_t vtpci_modern_read_isr(device_t); 100 static uint16_t vtpci_modern_get_vq_size(device_t, int); 101 static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int); 102 static void vtpci_modern_set_vq(device_t, struct virtqueue *); 103 static void vtpci_modern_disable_vq(device_t, int); 104 static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int, 105 struct vtpci_interrupt *); 106 static int vtpci_modern_register_cfg_msix(device_t, 107 struct vtpci_interrupt *); 108 static int vtpci_modern_register_vq_msix(device_t, int idx, 109 struct vtpci_interrupt *); 110 111 static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t); 112 static int vtpci_modern_finalize_features(device_t); 113 static bool vtpci_modern_with_feature(device_t, uint64_t); 114 static int vtpci_modern_alloc_virtqueues(device_t, int, 115 struct vq_alloc_info *); 116 static int vtpci_modern_setup_interrupts(device_t, enum intr_type); 117 static void vtpci_modern_stop(device_t); 118 static int vtpci_modern_reinit(device_t, uint64_t); 119 static void vtpci_modern_reinit_complete(device_t); 120 static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t); 121 static int vtpci_modern_config_generation(device_t); 122 static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int); 123 static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int); 124 125 static int vtpci_modern_probe_configs(device_t); 126 static int vtpci_modern_find_cap(device_t, uint8_t, int *); 127 static int vtpci_modern_map_configs(struct vtpci_modern_softc *); 128 static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *); 129 static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *, 130 uint8_t, int, int, struct vtpci_modern_resource_map *); 131 static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int); 132 static struct resource *vtpci_modern_get_bar_resource( 133 struct vtpci_modern_softc *, int, int); 134 static struct resource *vtpci_modern_alloc_bar_resource( 135 struct vtpci_modern_softc *, int, int); 136 static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *); 137 static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *, 138 struct vtpci_modern_resource_map *); 139 static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *, 140 struct vtpci_modern_resource_map *); 141 static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *); 142 static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *); 143 144 static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *); 145 146 static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *); 147 static void vtpci_modern_write_features(struct vtpci_modern_softc *, 148 uint64_t); 149 static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int); 150 static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *); 151 static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t); 152 static void vtpci_modern_reset(struct vtpci_modern_softc *); 153 static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *); 154 155 static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *, 156 bus_size_t); 157 static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *, 158 bus_size_t); 159 static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *, 160 bus_size_t); 161 static void vtpci_modern_write_common_1(struct vtpci_modern_softc *, 162 bus_size_t, uint8_t); 163 static void vtpci_modern_write_common_2(struct vtpci_modern_softc *, 164 bus_size_t, uint16_t); 165 static void vtpci_modern_write_common_4(struct vtpci_modern_softc *, 166 bus_size_t, uint32_t); 167 static void vtpci_modern_write_common_8(struct vtpci_modern_softc *, 168 bus_size_t, uint64_t); 169 static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *, 170 bus_size_t, uint16_t); 171 static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *, 172 bus_size_t); 173 static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *, 174 bus_size_t); 175 static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *, 176 bus_size_t); 177 static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *, 178 bus_size_t); 179 static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *, 180 bus_size_t); 181 static void vtpci_modern_write_device_1(struct vtpci_modern_softc *, 182 bus_size_t, uint8_t); 183 static void vtpci_modern_write_device_2(struct vtpci_modern_softc *, 184 bus_size_t, uint16_t); 185 static void vtpci_modern_write_device_4(struct vtpci_modern_softc *, 186 bus_size_t, uint32_t); 187 static void vtpci_modern_write_device_8(struct vtpci_modern_softc *, 188 bus_size_t, uint64_t); 189 190 /* Tunables. */ 191 static int vtpci_modern_transitional = 0; 192 TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional); 193 194 static device_method_t vtpci_modern_methods[] = { 195 /* Device interface. */ 196 DEVMETHOD(device_probe, vtpci_modern_probe), 197 DEVMETHOD(device_attach, vtpci_modern_attach), 198 DEVMETHOD(device_detach, vtpci_modern_detach), 199 DEVMETHOD(device_suspend, vtpci_modern_suspend), 200 DEVMETHOD(device_resume, vtpci_modern_resume), 201 DEVMETHOD(device_shutdown, vtpci_modern_shutdown), 202 203 /* Bus interface. */ 204 DEVMETHOD(bus_driver_added, vtpci_modern_driver_added), 205 DEVMETHOD(bus_child_detached, vtpci_modern_child_detached), 206 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), 207 DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar), 208 DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar), 209 210 /* VirtIO PCI interface. */ 211 DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr), 212 DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size), 213 DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off), 214 DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq), 215 DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq), 216 DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix), 217 DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix), 218 219 /* VirtIO bus interface. */ 220 DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features), 221 DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features), 222 DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature), 223 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues), 224 DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts), 225 DEVMETHOD(virtio_bus_stop, vtpci_modern_stop), 226 DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit), 227 DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete), 228 DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq), 229 DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation), 230 DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config), 231 DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config), 232 233 DEVMETHOD_END 234 }; 235 236 static driver_t vtpci_modern_driver = { 237 .name = "virtio_pci", 238 .methods = vtpci_modern_methods, 239 .size = sizeof(struct vtpci_modern_softc) 240 }; 241 242 DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0); 243 244 static int 245 vtpci_modern_probe(device_t dev) 246 { 247 char desc[64]; 248 const char *name; 249 uint16_t devid; 250 251 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 252 return (ENXIO); 253 254 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 255 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX) 256 return (ENXIO); 257 258 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) { 259 if (!vtpci_modern_transitional) 260 return (ENXIO); 261 devid = pci_get_subdevice(dev); 262 } else 263 devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; 264 265 if (vtpci_modern_probe_configs(dev) != 0) 266 return (ENXIO); 267 268 name = virtio_device_name(devid); 269 if (name == NULL) 270 name = "Unknown"; 271 272 snprintf(desc, sizeof(desc), "VirtIO PCI (modern) %s adapter", name); 273 device_set_desc_copy(dev, desc); 274 275 return (BUS_PROBE_DEFAULT); 276 } 277 278 static int 279 vtpci_modern_attach(device_t dev) 280 { 281 struct vtpci_modern_softc *sc; 282 int error; 283 284 sc = device_get_softc(dev); 285 sc->vtpci_dev = dev; 286 vtpci_init(&sc->vtpci_common, dev, true); 287 288 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) 289 sc->vtpci_devid = pci_get_subdevice(dev); 290 else 291 sc->vtpci_devid = pci_get_device(dev) - 292 VIRTIO_PCI_DEVICEID_MODERN_MIN; 293 294 error = vtpci_modern_map_configs(sc); 295 if (error) { 296 device_printf(dev, "cannot map configs\n"); 297 vtpci_modern_unmap_configs(sc); 298 return (error); 299 } 300 301 vtpci_modern_reset(sc); 302 303 /* Tell the host we've noticed this device. */ 304 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 305 306 error = vtpci_add_child(&sc->vtpci_common); 307 if (error) 308 goto fail; 309 310 vtpci_modern_probe_and_attach_child(sc); 311 312 return (0); 313 314 fail: 315 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 316 vtpci_modern_detach(dev); 317 318 return (error); 319 } 320 321 static int 322 vtpci_modern_detach(device_t dev) 323 { 324 struct vtpci_modern_softc *sc; 325 int error; 326 327 sc = device_get_softc(dev); 328 329 error = vtpci_delete_child(&sc->vtpci_common); 330 if (error) 331 return (error); 332 333 vtpci_modern_reset(sc); 334 vtpci_modern_unmap_configs(sc); 335 336 return (0); 337 } 338 339 static int 340 vtpci_modern_suspend(device_t dev) 341 { 342 return (bus_generic_suspend(dev)); 343 } 344 345 static int 346 vtpci_modern_resume(device_t dev) 347 { 348 return (bus_generic_resume(dev)); 349 } 350 351 static int 352 vtpci_modern_shutdown(device_t dev) 353 { 354 (void) bus_generic_shutdown(dev); 355 /* Forcibly stop the host device. */ 356 vtpci_modern_stop(dev); 357 358 return (0); 359 } 360 361 static void 362 vtpci_modern_driver_added(device_t dev, driver_t *driver) 363 { 364 vtpci_modern_probe_and_attach_child(device_get_softc(dev)); 365 } 366 367 static void 368 vtpci_modern_child_detached(device_t dev, device_t child) 369 { 370 struct vtpci_modern_softc *sc; 371 372 sc = device_get_softc(dev); 373 374 vtpci_modern_reset(sc); 375 vtpci_child_detached(&sc->vtpci_common); 376 377 /* After the reset, retell the host we've noticed this device. */ 378 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 379 } 380 381 static int 382 vtpci_modern_read_ivar(device_t dev, device_t child, int index, 383 uintptr_t *result) 384 { 385 struct vtpci_modern_softc *sc; 386 struct vtpci_common *cn; 387 388 sc = device_get_softc(dev); 389 cn = &sc->vtpci_common; 390 391 if (vtpci_child_device(cn) != child) 392 return (ENOENT); 393 394 switch (index) { 395 case VIRTIO_IVAR_DEVTYPE: 396 *result = sc->vtpci_devid; 397 break; 398 default: 399 return (vtpci_read_ivar(cn, index, result)); 400 } 401 402 return (0); 403 } 404 405 static int 406 vtpci_modern_write_ivar(device_t dev, device_t child, int index, 407 uintptr_t value) 408 { 409 struct vtpci_modern_softc *sc; 410 struct vtpci_common *cn; 411 412 sc = device_get_softc(dev); 413 cn = &sc->vtpci_common; 414 415 if (vtpci_child_device(cn) != child) 416 return (ENOENT); 417 418 switch (index) { 419 default: 420 return (vtpci_write_ivar(cn, index, value)); 421 } 422 423 return (0); 424 } 425 426 static uint64_t 427 vtpci_modern_negotiate_features(device_t dev, uint64_t child_features) 428 { 429 struct vtpci_modern_softc *sc; 430 uint64_t host_features, features; 431 432 sc = device_get_softc(dev); 433 host_features = vtpci_modern_read_features(sc); 434 435 /* 436 * Since the driver was added as a child of the modern PCI bus, 437 * always add the V1 flag. 438 */ 439 child_features |= VIRTIO_F_VERSION_1; 440 441 features = vtpci_negotiate_features(&sc->vtpci_common, 442 child_features, host_features); 443 vtpci_modern_write_features(sc, features); 444 445 return (features); 446 } 447 448 static int 449 vtpci_modern_finalize_features(device_t dev) 450 { 451 struct vtpci_modern_softc *sc; 452 uint8_t status; 453 454 sc = device_get_softc(dev); 455 456 /* 457 * Must re-read the status after setting it to verify the negotiated 458 * features were accepted by the device. 459 */ 460 vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK); 461 462 status = vtpci_modern_get_status(sc); 463 if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { 464 device_printf(dev, "desired features were not accepted\n"); 465 return (ENOTSUP); 466 } 467 468 return (0); 469 } 470 471 static bool 472 vtpci_modern_with_feature(device_t dev, uint64_t feature) 473 { 474 struct vtpci_modern_softc *sc; 475 476 sc = device_get_softc(dev); 477 478 return (vtpci_with_feature(&sc->vtpci_common, feature)); 479 } 480 481 static uint64_t 482 vtpci_modern_read_features(struct vtpci_modern_softc *sc) 483 { 484 uint32_t features0, features1; 485 486 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0); 487 features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 488 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1); 489 features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 490 491 return (((uint64_t) features1 << 32) | features0); 492 } 493 494 static void 495 vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features) 496 { 497 uint32_t features0, features1; 498 499 features0 = features; 500 features1 = features >> 32; 501 502 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0); 503 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0); 504 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1); 505 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1); 506 } 507 508 static int 509 vtpci_modern_alloc_virtqueues(device_t dev, int nvqs, 510 struct vq_alloc_info *vq_info) 511 { 512 struct vtpci_modern_softc *sc; 513 struct vtpci_common *cn; 514 uint16_t max_nvqs; 515 516 sc = device_get_softc(dev); 517 cn = &sc->vtpci_common; 518 519 max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ); 520 if (nvqs > max_nvqs) { 521 device_printf(sc->vtpci_dev, "requested virtqueue count %d " 522 "exceeds max %d\n", nvqs, max_nvqs); 523 return (E2BIG); 524 } 525 526 return (vtpci_alloc_virtqueues(cn, nvqs, vq_info)); 527 } 528 529 static int 530 vtpci_modern_setup_interrupts(device_t dev, enum intr_type type) 531 { 532 struct vtpci_modern_softc *sc; 533 int error; 534 535 sc = device_get_softc(dev); 536 537 error = vtpci_setup_interrupts(&sc->vtpci_common, type); 538 if (error == 0) 539 vtpci_modern_enable_virtqueues(sc); 540 541 return (error); 542 } 543 544 static void 545 vtpci_modern_stop(device_t dev) 546 { 547 vtpci_modern_reset(device_get_softc(dev)); 548 } 549 550 static int 551 vtpci_modern_reinit(device_t dev, uint64_t features) 552 { 553 struct vtpci_modern_softc *sc; 554 struct vtpci_common *cn; 555 int error; 556 557 sc = device_get_softc(dev); 558 cn = &sc->vtpci_common; 559 560 /* 561 * Redrive the device initialization. This is a bit of an abuse of 562 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 563 * play nice. 564 * 565 * We do not allow the host device to change from what was originally 566 * negotiated beyond what the guest driver changed. MSIX state should 567 * not change, number of virtqueues and their size remain the same, etc. 568 * This will need to be rethought when we want to support migration. 569 */ 570 571 if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 572 vtpci_modern_stop(dev); 573 574 /* 575 * Quickly drive the status through ACK and DRIVER. The device does 576 * not become usable again until DRIVER_OK in reinit complete. 577 */ 578 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 579 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 580 581 /* 582 * TODO: Check that features are not added as to what was 583 * originally negotiated. 584 */ 585 vtpci_modern_negotiate_features(dev, features); 586 error = vtpci_modern_finalize_features(dev); 587 if (error) { 588 device_printf(dev, "cannot finalize features during reinit\n"); 589 return (error); 590 } 591 592 error = vtpci_reinit(cn); 593 if (error) 594 return (error); 595 596 return (0); 597 } 598 599 static void 600 vtpci_modern_reinit_complete(device_t dev) 601 { 602 struct vtpci_modern_softc *sc; 603 604 sc = device_get_softc(dev); 605 606 vtpci_modern_enable_virtqueues(sc); 607 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 608 } 609 610 static void 611 vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) 612 { 613 struct vtpci_modern_softc *sc; 614 615 sc = device_get_softc(dev); 616 617 vtpci_modern_write_notify_2(sc, offset, queue); 618 } 619 620 static uint8_t 621 vtpci_modern_get_status(struct vtpci_modern_softc *sc) 622 { 623 return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS)); 624 } 625 626 static void 627 vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status) 628 { 629 if (status != VIRTIO_CONFIG_STATUS_RESET) 630 status |= vtpci_modern_get_status(sc); 631 632 vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status); 633 } 634 635 static int 636 vtpci_modern_config_generation(device_t dev) 637 { 638 struct vtpci_modern_softc *sc; 639 uint8_t gen; 640 641 sc = device_get_softc(dev); 642 gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION); 643 644 return (gen); 645 } 646 647 static void 648 vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst, 649 int length) 650 { 651 struct vtpci_modern_softc *sc; 652 653 sc = device_get_softc(dev); 654 655 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 656 panic("%s: attempt to read dev config but not present", 657 __func__); 658 } 659 660 switch (length) { 661 case 1: 662 *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset); 663 break; 664 case 2: 665 *(uint16_t *) dst = virtio_htog16(true, 666 vtpci_modern_read_device_2(sc, offset)); 667 break; 668 case 4: 669 *(uint32_t *) dst = virtio_htog32(true, 670 vtpci_modern_read_device_4(sc, offset)); 671 break; 672 case 8: 673 *(uint64_t *) dst = virtio_htog64(true, 674 vtpci_modern_read_device_8(sc, offset)); 675 break; 676 default: 677 panic("%s: device %s invalid device read length %d offset %d", 678 __func__, device_get_nameunit(dev), length, (int) offset); 679 } 680 } 681 682 static void 683 vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src, 684 int length) 685 { 686 struct vtpci_modern_softc *sc; 687 688 sc = device_get_softc(dev); 689 690 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 691 panic("%s: attempt to write dev config but not present", 692 __func__); 693 } 694 695 switch (length) { 696 case 1: 697 vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src); 698 break; 699 case 2: { 700 uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src); 701 vtpci_modern_write_device_2(sc, offset, val); 702 break; 703 } 704 case 4: { 705 uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src); 706 vtpci_modern_write_device_4(sc, offset, val); 707 break; 708 } 709 case 8: { 710 uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src); 711 vtpci_modern_write_device_8(sc, offset, val); 712 break; 713 } 714 default: 715 panic("%s: device %s invalid device write length %d offset %d", 716 __func__, device_get_nameunit(dev), length, (int) offset); 717 } 718 } 719 720 static int 721 vtpci_modern_probe_configs(device_t dev) 722 { 723 int error; 724 725 /* 726 * These config capabilities must be present. The DEVICE_CFG 727 * capability is only present if the device requires it. 728 */ 729 730 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL); 731 if (error) { 732 device_printf(dev, "cannot find COMMON_CFG capability\n"); 733 return (error); 734 } 735 736 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL); 737 if (error) { 738 device_printf(dev, "cannot find NOTIFY_CFG capability\n"); 739 return (error); 740 } 741 742 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL); 743 if (error) { 744 device_printf(dev, "cannot find ISR_CFG capability\n"); 745 return (error); 746 } 747 748 return (0); 749 } 750 751 static int 752 vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset) 753 { 754 uint32_t type, bar; 755 int capreg, error; 756 757 for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg); 758 error == 0; 759 error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) { 760 761 type = pci_read_config(dev, capreg + 762 offsetof(struct virtio_pci_cap, cfg_type), 1); 763 bar = pci_read_config(dev, capreg + 764 offsetof(struct virtio_pci_cap, bar), 1); 765 766 /* Must ignore reserved BARs. */ 767 if (bar >= VTPCI_MODERN_MAX_BARS) 768 continue; 769 770 if (type == cfg_type) { 771 if (cap_offset != NULL) 772 *cap_offset = capreg; 773 break; 774 } 775 } 776 777 return (error); 778 } 779 780 static int 781 vtpci_modern_map_common_config(struct vtpci_modern_softc *sc) 782 { 783 device_t dev; 784 int error; 785 786 dev = sc->vtpci_dev; 787 788 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG, 789 sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map); 790 if (error) { 791 device_printf(dev, "cannot find cap COMMON_CFG resource\n"); 792 return (error); 793 } 794 795 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map); 796 if (error) { 797 device_printf(dev, "cannot alloc resource for COMMON_CFG\n"); 798 return (error); 799 } 800 801 return (0); 802 } 803 804 static int 805 vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc) 806 { 807 device_t dev; 808 int cap_offset, error; 809 810 dev = sc->vtpci_dev; 811 812 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG, 813 -1, 2, &sc->vtpci_notify_res_map); 814 if (error) { 815 device_printf(dev, "cannot find cap NOTIFY_CFG resource\n"); 816 return (error); 817 } 818 819 cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset; 820 821 sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset + 822 offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4); 823 824 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map); 825 if (error) { 826 device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n"); 827 return (error); 828 } 829 830 return (0); 831 } 832 833 static int 834 vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc) 835 { 836 device_t dev; 837 int error; 838 839 dev = sc->vtpci_dev; 840 841 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG, 842 sizeof(uint8_t), 1, &sc->vtpci_isr_res_map); 843 if (error) { 844 device_printf(dev, "cannot find cap ISR_CFG resource\n"); 845 return (error); 846 } 847 848 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map); 849 if (error) { 850 device_printf(dev, "cannot alloc resource for ISR_CFG\n"); 851 return (error); 852 } 853 854 return (0); 855 } 856 857 static int 858 vtpci_modern_map_device_config(struct vtpci_modern_softc *sc) 859 { 860 device_t dev; 861 int error; 862 863 dev = sc->vtpci_dev; 864 865 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG, 866 -1, 4, &sc->vtpci_device_res_map); 867 if (error == ENOENT) { 868 /* Device configuration is optional depending on device. */ 869 return (0); 870 } else if (error) { 871 device_printf(dev, "cannot find cap DEVICE_CFG resource\n"); 872 return (error); 873 } 874 875 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map); 876 if (error) { 877 device_printf(dev, "cannot alloc resource for DEVICE_CFG\n"); 878 return (error); 879 } 880 881 return (0); 882 } 883 884 static int 885 vtpci_modern_map_configs(struct vtpci_modern_softc *sc) 886 { 887 int error; 888 889 error = vtpci_modern_map_common_config(sc); 890 if (error) 891 return (error); 892 893 error = vtpci_modern_map_notify_config(sc); 894 if (error) 895 return (error); 896 897 error = vtpci_modern_map_isr_config(sc); 898 if (error) 899 return (error); 900 901 error = vtpci_modern_map_device_config(sc); 902 if (error) 903 return (error); 904 905 vtpci_modern_alloc_msix_resource(sc); 906 907 return (0); 908 } 909 910 static void 911 vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc) 912 { 913 914 vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map); 915 vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map); 916 vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map); 917 vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map); 918 919 vtpci_modern_free_bar_resources(sc); 920 vtpci_modern_free_msix_resource(sc); 921 922 sc->vtpci_notify_offset_multiplier = 0; 923 } 924 925 static int 926 vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type, 927 int min_size, int alignment, struct vtpci_modern_resource_map *res) 928 { 929 device_t dev; 930 int cap_offset, offset, length, error; 931 uint8_t bar, cap_length; 932 933 dev = sc->vtpci_dev; 934 935 error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset); 936 if (error) 937 return (error); 938 939 cap_length = pci_read_config(dev, 940 cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1); 941 942 if (cap_length < sizeof(struct virtio_pci_cap)) { 943 device_printf(dev, "cap %u length %d less than expected\n", 944 cfg_type, cap_length); 945 return (ENXIO); 946 } 947 948 bar = pci_read_config(dev, 949 cap_offset + offsetof(struct virtio_pci_cap, bar), 1); 950 offset = pci_read_config(dev, 951 cap_offset + offsetof(struct virtio_pci_cap, offset), 4); 952 length = pci_read_config(dev, 953 cap_offset + offsetof(struct virtio_pci_cap, length), 4); 954 955 if (min_size != -1 && length < min_size) { 956 device_printf(dev, "cap %u struct length %d less than min %d\n", 957 cfg_type, length, min_size); 958 return (ENXIO); 959 } 960 961 if (offset % alignment) { 962 device_printf(dev, "cap %u struct offset %d not aligned to %d\n", 963 cfg_type, offset, alignment); 964 return (ENXIO); 965 } 966 967 /* BMV: TODO Can we determine the size of the BAR here? */ 968 969 res->vtrm_cap_offset = cap_offset; 970 res->vtrm_bar = bar; 971 res->vtrm_offset = offset; 972 res->vtrm_length = length; 973 res->vtrm_type = vtpci_modern_bar_type(sc, bar); 974 975 return (0); 976 } 977 978 static int 979 vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar) 980 { 981 uint32_t val; 982 983 /* 984 * The BAR described by a config capability may be either an IOPORT or 985 * MEM, but we must know the type when calling bus_alloc_resource(). 986 */ 987 val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4); 988 if (PCI_BAR_IO(val)) 989 return (SYS_RES_IOPORT); 990 else 991 return (SYS_RES_MEMORY); 992 } 993 994 static struct resource * 995 vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) 996 { 997 struct resource *res; 998 999 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1000 res = sc->vtpci_bar_res[bar].vtbr_res; 1001 MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type); 1002 1003 return (res); 1004 } 1005 1006 static struct resource * 1007 vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar, 1008 int type) 1009 { 1010 struct resource *res; 1011 int rid; 1012 1013 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1014 MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT); 1015 1016 res = sc->vtpci_bar_res[bar].vtbr_res; 1017 if (res != NULL) { 1018 MPASS(sc->vtpci_bar_res[bar].vtbr_type == type); 1019 return (res); 1020 } 1021 1022 rid = PCIR_BAR(bar); 1023 res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid, 1024 RF_ACTIVE | RF_UNMAPPED); 1025 if (res != NULL) { 1026 sc->vtpci_bar_res[bar].vtbr_res = res; 1027 sc->vtpci_bar_res[bar].vtbr_type = type; 1028 } 1029 1030 return (res); 1031 } 1032 1033 static void 1034 vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc) 1035 { 1036 device_t dev; 1037 struct resource *res; 1038 int bar, rid, type; 1039 1040 dev = sc->vtpci_dev; 1041 1042 for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) { 1043 res = sc->vtpci_bar_res[bar].vtbr_res; 1044 type = sc->vtpci_bar_res[bar].vtbr_type; 1045 1046 if (res != NULL) { 1047 rid = PCIR_BAR(bar); 1048 bus_release_resource(dev, type, rid, res); 1049 sc->vtpci_bar_res[bar].vtbr_res = NULL; 1050 sc->vtpci_bar_res[bar].vtbr_type = 0; 1051 } 1052 } 1053 } 1054 1055 static int 1056 vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc, 1057 struct vtpci_modern_resource_map *map) 1058 { 1059 struct resource_map_request req; 1060 struct resource *res; 1061 int type; 1062 1063 type = map->vtrm_type; 1064 1065 res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type); 1066 if (res == NULL) 1067 return (ENXIO); 1068 1069 resource_init_map_request(&req); 1070 req.offset = map->vtrm_offset; 1071 req.length = map->vtrm_length; 1072 1073 return (bus_map_resource(sc->vtpci_dev, type, res, &req, 1074 &map->vtrm_map)); 1075 } 1076 1077 static void 1078 vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc, 1079 struct vtpci_modern_resource_map *map) 1080 { 1081 struct resource *res; 1082 int type; 1083 1084 type = map->vtrm_type; 1085 res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type); 1086 1087 if (res != NULL && map->vtrm_map.r_size != 0) { 1088 bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map); 1089 bzero(map, sizeof(struct vtpci_modern_resource_map)); 1090 } 1091 } 1092 1093 static void 1094 vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc) 1095 { 1096 device_t dev; 1097 int bar; 1098 1099 dev = sc->vtpci_dev; 1100 1101 if (!vtpci_is_msix_available(&sc->vtpci_common) || 1102 (bar = pci_msix_table_bar(dev)) == -1) 1103 return; 1104 1105 /* TODO: Can this BAR be in the 0-5 range? */ 1106 sc->vtpci_msix_bar = bar; 1107 if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1108 &bar, RF_ACTIVE)) == NULL) 1109 device_printf(dev, "Unable to map MSIX table\n"); 1110 } 1111 1112 static void 1113 vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc) 1114 { 1115 device_t dev; 1116 1117 dev = sc->vtpci_dev; 1118 1119 if (sc->vtpci_msix_res != NULL) { 1120 bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar, 1121 sc->vtpci_msix_res); 1122 sc->vtpci_msix_bar = 0; 1123 sc->vtpci_msix_res = NULL; 1124 } 1125 } 1126 1127 static void 1128 vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc) 1129 { 1130 device_t dev, child; 1131 1132 dev = sc->vtpci_dev; 1133 child = vtpci_child_device(&sc->vtpci_common); 1134 1135 if (child == NULL || device_get_state(child) != DS_NOTPRESENT) 1136 return; 1137 1138 if (device_probe(child) != 0) 1139 return; 1140 1141 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 1142 1143 if (device_attach(child) != 0) { 1144 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 1145 /* Reset state for later attempt. */ 1146 vtpci_modern_child_detached(dev, child); 1147 } else { 1148 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1149 VIRTIO_ATTACH_COMPLETED(child); 1150 } 1151 } 1152 1153 static int 1154 vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset, 1155 struct vtpci_interrupt *intr) 1156 { 1157 uint16_t vector; 1158 1159 if (intr != NULL) { 1160 /* Map from guest rid to host vector. */ 1161 vector = intr->vti_rid - 1; 1162 } else 1163 vector = VIRTIO_MSI_NO_VECTOR; 1164 1165 vtpci_modern_write_common_2(sc, offset, vector); 1166 return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV); 1167 } 1168 1169 static int 1170 vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) 1171 { 1172 struct vtpci_modern_softc *sc; 1173 int error; 1174 1175 sc = device_get_softc(dev); 1176 1177 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr); 1178 if (error) { 1179 device_printf(dev, 1180 "unable to register config MSIX interrupt\n"); 1181 return (error); 1182 } 1183 1184 return (0); 1185 } 1186 1187 static int 1188 vtpci_modern_register_vq_msix(device_t dev, int idx, 1189 struct vtpci_interrupt *intr) 1190 { 1191 struct vtpci_modern_softc *sc; 1192 int error; 1193 1194 sc = device_get_softc(dev); 1195 1196 vtpci_modern_select_virtqueue(sc, idx); 1197 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr); 1198 if (error) { 1199 device_printf(dev, 1200 "unable to register virtqueue MSIX interrupt\n"); 1201 return (error); 1202 } 1203 1204 return (0); 1205 } 1206 1207 static void 1208 vtpci_modern_reset(struct vtpci_modern_softc *sc) 1209 { 1210 /* 1211 * Setting the status to RESET sets the host device to the 1212 * original, uninitialized state. Must poll the status until 1213 * the reset is complete. 1214 */ 1215 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); 1216 1217 while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 1218 cpu_spinwait(); 1219 } 1220 1221 static void 1222 vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx) 1223 { 1224 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx); 1225 } 1226 1227 static uint8_t 1228 vtpci_modern_read_isr(device_t dev) 1229 { 1230 return (vtpci_modern_read_isr_1(device_get_softc(dev), 0)); 1231 } 1232 1233 static uint16_t 1234 vtpci_modern_get_vq_size(device_t dev, int idx) 1235 { 1236 struct vtpci_modern_softc *sc; 1237 1238 sc = device_get_softc(dev); 1239 1240 vtpci_modern_select_virtqueue(sc, idx); 1241 return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE)); 1242 } 1243 1244 static bus_size_t 1245 vtpci_modern_get_vq_notify_off(device_t dev, int idx) 1246 { 1247 struct vtpci_modern_softc *sc; 1248 uint16_t q_notify_off; 1249 1250 sc = device_get_softc(dev); 1251 1252 vtpci_modern_select_virtqueue(sc, idx); 1253 q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF); 1254 1255 return (q_notify_off * sc->vtpci_notify_offset_multiplier); 1256 } 1257 1258 static void 1259 vtpci_modern_set_vq(device_t dev, struct virtqueue *vq) 1260 { 1261 struct vtpci_modern_softc *sc; 1262 1263 sc = device_get_softc(dev); 1264 1265 vtpci_modern_select_virtqueue(sc, virtqueue_index(vq)); 1266 1267 /* BMV: Currently we never adjust the device's proposed VQ size. */ 1268 vtpci_modern_write_common_2(sc, 1269 VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq)); 1270 1271 vtpci_modern_write_common_8(sc, 1272 VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq)); 1273 vtpci_modern_write_common_8(sc, 1274 VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq)); 1275 vtpci_modern_write_common_8(sc, 1276 VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq)); 1277 } 1278 1279 static void 1280 vtpci_modern_disable_vq(device_t dev, int idx) 1281 { 1282 struct vtpci_modern_softc *sc; 1283 1284 sc = device_get_softc(dev); 1285 1286 vtpci_modern_select_virtqueue(sc, idx); 1287 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL); 1288 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL); 1289 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL); 1290 } 1291 1292 static void 1293 vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc) 1294 { 1295 int idx; 1296 1297 for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) { 1298 vtpci_modern_select_virtqueue(sc, idx); 1299 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1); 1300 } 1301 } 1302 1303 static uint8_t 1304 vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off) 1305 { 1306 return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off)); 1307 } 1308 1309 static uint16_t 1310 vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off) 1311 { 1312 return virtio_htog16(true, 1313 bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off)); 1314 } 1315 1316 static uint32_t 1317 vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off) 1318 { 1319 return virtio_htog32(true, 1320 bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off)); 1321 } 1322 1323 static void 1324 vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off, 1325 uint8_t val) 1326 { 1327 bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val); 1328 } 1329 1330 static void 1331 vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off, 1332 uint16_t val) 1333 { 1334 bus_write_2(&sc->vtpci_common_res_map.vtrm_map, 1335 off, virtio_gtoh16(true, val)); 1336 } 1337 1338 static void 1339 vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off, 1340 uint32_t val) 1341 { 1342 bus_write_4(&sc->vtpci_common_res_map.vtrm_map, 1343 off, virtio_gtoh32(true, val)); 1344 } 1345 1346 static void 1347 vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off, 1348 uint64_t val) 1349 { 1350 uint32_t val0, val1; 1351 1352 val0 = (uint32_t) val; 1353 val1 = val >> 32; 1354 1355 vtpci_modern_write_common_4(sc, off, val0); 1356 vtpci_modern_write_common_4(sc, off + 4, val1); 1357 } 1358 1359 static void 1360 vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off, 1361 uint16_t val) 1362 { 1363 bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val); 1364 } 1365 1366 static uint8_t 1367 vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off) 1368 { 1369 return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off)); 1370 } 1371 1372 static uint8_t 1373 vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off) 1374 { 1375 return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off)); 1376 } 1377 1378 static uint16_t 1379 vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off) 1380 { 1381 return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off)); 1382 } 1383 1384 static uint32_t 1385 vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off) 1386 { 1387 return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off)); 1388 } 1389 1390 static uint64_t 1391 vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off) 1392 { 1393 device_t dev; 1394 int gen; 1395 uint32_t val0, val1; 1396 1397 dev = sc->vtpci_dev; 1398 1399 /* 1400 * Treat the 64-bit field as two 32-bit fields. Use the generation 1401 * to ensure a consistent read. 1402 */ 1403 do { 1404 gen = vtpci_modern_config_generation(dev); 1405 val0 = vtpci_modern_read_device_4(sc, off); 1406 val1 = vtpci_modern_read_device_4(sc, off + 4); 1407 } while (gen != vtpci_modern_config_generation(dev)); 1408 1409 return (((uint64_t) val1 << 32) | val0); 1410 } 1411 1412 static void 1413 vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off, 1414 uint8_t val) 1415 { 1416 bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val); 1417 } 1418 1419 static void 1420 vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off, 1421 uint16_t val) 1422 { 1423 bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val); 1424 } 1425 1426 static void 1427 vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off, 1428 uint32_t val) 1429 { 1430 bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val); 1431 } 1432 1433 static void 1434 vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off, 1435 uint64_t val) 1436 { 1437 uint32_t val0, val1; 1438 1439 val0 = (uint32_t) val; 1440 val1 = val >> 32; 1441 1442 vtpci_modern_write_device_4(sc, off, val0); 1443 vtpci_modern_write_device_4(sc, off + 4, val1); 1444 } 1445