1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for the modern VirtIO PCI interface. */ 30 31 #include <sys/cdefs.h> 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/lock.h> 36 #include <sys/kernel.h> 37 #include <sys/module.h> 38 39 #include <machine/bus.h> 40 #include <machine/cpu.h> 41 #include <machine/resource.h> 42 #include <sys/bus.h> 43 #include <sys/rman.h> 44 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pcireg.h> 47 48 #include <dev/virtio/virtio.h> 49 #include <dev/virtio/virtqueue.h> 50 #include <dev/virtio/pci/virtio_pci.h> 51 #include <dev/virtio/pci/virtio_pci_modern_var.h> 52 53 #include "virtio_bus_if.h" 54 #include "virtio_pci_if.h" 55 #include "virtio_if.h" 56 57 struct vtpci_modern_resource_map { 58 struct resource_map vtrm_map; 59 int vtrm_cap_offset; 60 int vtrm_bar; 61 int vtrm_offset; 62 int vtrm_length; 63 int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */ 64 }; 65 66 struct vtpci_modern_bar_resource { 67 struct resource *vtbr_res; 68 int vtbr_type; 69 }; 70 71 struct vtpci_modern_softc { 72 device_t vtpci_dev; 73 struct vtpci_common vtpci_common; 74 uint32_t vtpci_notify_offset_multiplier; 75 uint16_t vtpci_devid; 76 int vtpci_msix_bar; 77 struct resource *vtpci_msix_res; 78 79 struct vtpci_modern_resource_map vtpci_common_res_map; 80 struct vtpci_modern_resource_map vtpci_notify_res_map; 81 struct vtpci_modern_resource_map vtpci_isr_res_map; 82 struct vtpci_modern_resource_map vtpci_device_res_map; 83 84 #define VTPCI_MODERN_MAX_BARS 6 85 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS]; 86 }; 87 88 static int vtpci_modern_probe(device_t); 89 static int vtpci_modern_attach(device_t); 90 static int vtpci_modern_detach(device_t); 91 static int vtpci_modern_suspend(device_t); 92 static int vtpci_modern_resume(device_t); 93 static int vtpci_modern_shutdown(device_t); 94 95 static void vtpci_modern_driver_added(device_t, driver_t *); 96 static void vtpci_modern_child_detached(device_t, device_t); 97 static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *); 98 static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t); 99 100 static uint8_t vtpci_modern_read_isr(device_t); 101 static uint16_t vtpci_modern_get_vq_size(device_t, int); 102 static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int); 103 static void vtpci_modern_set_vq(device_t, struct virtqueue *); 104 static void vtpci_modern_disable_vq(device_t, int); 105 static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int, 106 struct vtpci_interrupt *); 107 static int vtpci_modern_register_cfg_msix(device_t, 108 struct vtpci_interrupt *); 109 static int vtpci_modern_register_vq_msix(device_t, int idx, 110 struct vtpci_interrupt *); 111 112 static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t); 113 static int vtpci_modern_finalize_features(device_t); 114 static int vtpci_modern_with_feature(device_t, uint64_t); 115 static int vtpci_modern_alloc_virtqueues(device_t, int, int, 116 struct vq_alloc_info *); 117 static int vtpci_modern_setup_interrupts(device_t, enum intr_type); 118 static void vtpci_modern_stop(device_t); 119 static int vtpci_modern_reinit(device_t, uint64_t); 120 static void vtpci_modern_reinit_complete(device_t); 121 static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t); 122 static int vtpci_modern_config_generation(device_t); 123 static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int); 124 static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int); 125 126 static int vtpci_modern_probe_configs(device_t); 127 static int vtpci_modern_find_cap(device_t, uint8_t, int *); 128 static int vtpci_modern_map_configs(struct vtpci_modern_softc *); 129 static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *); 130 static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *, 131 uint8_t, int, int, struct vtpci_modern_resource_map *); 132 static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int); 133 static struct resource *vtpci_modern_get_bar_resource( 134 struct vtpci_modern_softc *, int, int); 135 static struct resource *vtpci_modern_alloc_bar_resource( 136 struct vtpci_modern_softc *, int, int); 137 static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *); 138 static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *, 139 struct vtpci_modern_resource_map *); 140 static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *, 141 struct vtpci_modern_resource_map *); 142 static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *); 143 static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *); 144 145 static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *); 146 147 static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *); 148 static void vtpci_modern_write_features(struct vtpci_modern_softc *, 149 uint64_t); 150 static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int); 151 static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *); 152 static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t); 153 static void vtpci_modern_reset(struct vtpci_modern_softc *); 154 static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *); 155 156 static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *, 157 bus_size_t); 158 static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *, 159 bus_size_t); 160 static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *, 161 bus_size_t); 162 static void vtpci_modern_write_common_1(struct vtpci_modern_softc *, 163 bus_size_t, uint8_t); 164 static void vtpci_modern_write_common_2(struct vtpci_modern_softc *, 165 bus_size_t, uint16_t); 166 static void vtpci_modern_write_common_4(struct vtpci_modern_softc *, 167 bus_size_t, uint32_t); 168 static void vtpci_modern_write_common_8(struct vtpci_modern_softc *, 169 bus_size_t, uint64_t); 170 static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *, 171 bus_size_t, uint16_t); 172 static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *, 173 bus_size_t); 174 static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *, 175 bus_size_t); 176 static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *, 177 bus_size_t); 178 static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *, 179 bus_size_t); 180 static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *, 181 bus_size_t); 182 static void vtpci_modern_write_device_1(struct vtpci_modern_softc *, 183 bus_size_t, uint8_t); 184 static void vtpci_modern_write_device_2(struct vtpci_modern_softc *, 185 bus_size_t, uint16_t); 186 static void vtpci_modern_write_device_4(struct vtpci_modern_softc *, 187 bus_size_t, uint32_t); 188 static void vtpci_modern_write_device_8(struct vtpci_modern_softc *, 189 bus_size_t, uint64_t); 190 191 /* Tunables. */ 192 static int vtpci_modern_transitional = 0; 193 TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional); 194 195 static device_method_t vtpci_modern_methods[] = { 196 /* Device interface. */ 197 DEVMETHOD(device_probe, vtpci_modern_probe), 198 DEVMETHOD(device_attach, vtpci_modern_attach), 199 DEVMETHOD(device_detach, vtpci_modern_detach), 200 DEVMETHOD(device_suspend, vtpci_modern_suspend), 201 DEVMETHOD(device_resume, vtpci_modern_resume), 202 DEVMETHOD(device_shutdown, vtpci_modern_shutdown), 203 204 /* Bus interface. */ 205 DEVMETHOD(bus_driver_added, vtpci_modern_driver_added), 206 DEVMETHOD(bus_child_detached, vtpci_modern_child_detached), 207 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), 208 DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar), 209 DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar), 210 211 /* VirtIO PCI interface. */ 212 DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr), 213 DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size), 214 DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off), 215 DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq), 216 DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq), 217 DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix), 218 DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix), 219 220 /* VirtIO bus interface. */ 221 DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features), 222 DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features), 223 DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature), 224 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues), 225 DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts), 226 DEVMETHOD(virtio_bus_stop, vtpci_modern_stop), 227 DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit), 228 DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete), 229 DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq), 230 DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation), 231 DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config), 232 DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config), 233 234 DEVMETHOD_END 235 }; 236 237 static driver_t vtpci_modern_driver = { 238 .name = "virtio_pci", 239 .methods = vtpci_modern_methods, 240 .size = sizeof(struct vtpci_modern_softc) 241 }; 242 243 DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0); 244 245 static int 246 vtpci_modern_probe(device_t dev) 247 { 248 char desc[64]; 249 const char *name; 250 uint16_t devid; 251 252 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 253 return (ENXIO); 254 255 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 256 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX) 257 return (ENXIO); 258 259 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) { 260 if (!vtpci_modern_transitional) 261 return (ENXIO); 262 devid = pci_get_subdevice(dev); 263 } else 264 devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; 265 266 if (vtpci_modern_probe_configs(dev) != 0) 267 return (ENXIO); 268 269 name = virtio_device_name(devid); 270 if (name == NULL) 271 name = "Unknown"; 272 273 snprintf(desc, sizeof(desc), "VirtIO PCI (modern) %s adapter", name); 274 device_set_desc_copy(dev, desc); 275 276 return (BUS_PROBE_DEFAULT); 277 } 278 279 static int 280 vtpci_modern_attach(device_t dev) 281 { 282 struct vtpci_modern_softc *sc; 283 int error; 284 285 sc = device_get_softc(dev); 286 sc->vtpci_dev = dev; 287 vtpci_init(&sc->vtpci_common, dev, true); 288 289 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) 290 sc->vtpci_devid = pci_get_subdevice(dev); 291 else 292 sc->vtpci_devid = pci_get_device(dev) - 293 VIRTIO_PCI_DEVICEID_MODERN_MIN; 294 295 error = vtpci_modern_map_configs(sc); 296 if (error) { 297 device_printf(dev, "cannot map configs\n"); 298 vtpci_modern_unmap_configs(sc); 299 return (error); 300 } 301 302 vtpci_modern_reset(sc); 303 304 /* Tell the host we've noticed this device. */ 305 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 306 307 error = vtpci_add_child(&sc->vtpci_common); 308 if (error) 309 goto fail; 310 311 vtpci_modern_probe_and_attach_child(sc); 312 313 return (0); 314 315 fail: 316 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 317 vtpci_modern_detach(dev); 318 319 return (error); 320 } 321 322 static int 323 vtpci_modern_detach(device_t dev) 324 { 325 struct vtpci_modern_softc *sc; 326 int error; 327 328 sc = device_get_softc(dev); 329 330 error = vtpci_delete_child(&sc->vtpci_common); 331 if (error) 332 return (error); 333 334 vtpci_modern_reset(sc); 335 vtpci_modern_unmap_configs(sc); 336 337 return (0); 338 } 339 340 static int 341 vtpci_modern_suspend(device_t dev) 342 { 343 return (bus_generic_suspend(dev)); 344 } 345 346 static int 347 vtpci_modern_resume(device_t dev) 348 { 349 return (bus_generic_resume(dev)); 350 } 351 352 static int 353 vtpci_modern_shutdown(device_t dev) 354 { 355 (void) bus_generic_shutdown(dev); 356 /* Forcibly stop the host device. */ 357 vtpci_modern_stop(dev); 358 359 return (0); 360 } 361 362 static void 363 vtpci_modern_driver_added(device_t dev, driver_t *driver) 364 { 365 vtpci_modern_probe_and_attach_child(device_get_softc(dev)); 366 } 367 368 static void 369 vtpci_modern_child_detached(device_t dev, device_t child) 370 { 371 struct vtpci_modern_softc *sc; 372 373 sc = device_get_softc(dev); 374 375 vtpci_modern_reset(sc); 376 vtpci_child_detached(&sc->vtpci_common); 377 378 /* After the reset, retell the host we've noticed this device. */ 379 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 380 } 381 382 static int 383 vtpci_modern_read_ivar(device_t dev, device_t child, int index, 384 uintptr_t *result) 385 { 386 struct vtpci_modern_softc *sc; 387 struct vtpci_common *cn; 388 389 sc = device_get_softc(dev); 390 cn = &sc->vtpci_common; 391 392 if (vtpci_child_device(cn) != child) 393 return (ENOENT); 394 395 switch (index) { 396 case VIRTIO_IVAR_DEVTYPE: 397 *result = sc->vtpci_devid; 398 break; 399 default: 400 return (vtpci_read_ivar(cn, index, result)); 401 } 402 403 return (0); 404 } 405 406 static int 407 vtpci_modern_write_ivar(device_t dev, device_t child, int index, 408 uintptr_t value) 409 { 410 struct vtpci_modern_softc *sc; 411 struct vtpci_common *cn; 412 413 sc = device_get_softc(dev); 414 cn = &sc->vtpci_common; 415 416 if (vtpci_child_device(cn) != child) 417 return (ENOENT); 418 419 switch (index) { 420 default: 421 return (vtpci_write_ivar(cn, index, value)); 422 } 423 424 return (0); 425 } 426 427 static uint64_t 428 vtpci_modern_negotiate_features(device_t dev, uint64_t child_features) 429 { 430 struct vtpci_modern_softc *sc; 431 uint64_t host_features, features; 432 433 sc = device_get_softc(dev); 434 host_features = vtpci_modern_read_features(sc); 435 436 /* 437 * Since the driver was added as a child of the modern PCI bus, 438 * always add the V1 flag. 439 */ 440 child_features |= VIRTIO_F_VERSION_1; 441 442 features = vtpci_negotiate_features(&sc->vtpci_common, 443 child_features, host_features); 444 vtpci_modern_write_features(sc, features); 445 446 return (features); 447 } 448 449 static int 450 vtpci_modern_finalize_features(device_t dev) 451 { 452 struct vtpci_modern_softc *sc; 453 uint8_t status; 454 455 sc = device_get_softc(dev); 456 457 /* 458 * Must re-read the status after setting it to verify the negotiated 459 * features were accepted by the device. 460 */ 461 vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK); 462 463 status = vtpci_modern_get_status(sc); 464 if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { 465 device_printf(dev, "desired features were not accepted\n"); 466 return (ENOTSUP); 467 } 468 469 return (0); 470 } 471 472 static int 473 vtpci_modern_with_feature(device_t dev, uint64_t feature) 474 { 475 struct vtpci_modern_softc *sc; 476 477 sc = device_get_softc(dev); 478 479 return (vtpci_with_feature(&sc->vtpci_common, feature)); 480 } 481 482 static uint64_t 483 vtpci_modern_read_features(struct vtpci_modern_softc *sc) 484 { 485 uint32_t features0, features1; 486 487 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0); 488 features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 489 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1); 490 features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); 491 492 return (((uint64_t) features1 << 32) | features0); 493 } 494 495 static void 496 vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features) 497 { 498 uint32_t features0, features1; 499 500 features0 = features; 501 features1 = features >> 32; 502 503 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0); 504 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0); 505 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1); 506 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1); 507 } 508 509 static int 510 vtpci_modern_alloc_virtqueues(device_t dev, int flags, int nvqs, 511 struct vq_alloc_info *vq_info) 512 { 513 struct vtpci_modern_softc *sc; 514 struct vtpci_common *cn; 515 uint16_t max_nvqs; 516 517 sc = device_get_softc(dev); 518 cn = &sc->vtpci_common; 519 520 max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ); 521 if (nvqs > max_nvqs) { 522 device_printf(sc->vtpci_dev, "requested virtqueue count %d " 523 "exceeds max %d\n", nvqs, max_nvqs); 524 return (E2BIG); 525 } 526 527 return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info)); 528 } 529 530 static int 531 vtpci_modern_setup_interrupts(device_t dev, enum intr_type type) 532 { 533 struct vtpci_modern_softc *sc; 534 int error; 535 536 sc = device_get_softc(dev); 537 538 error = vtpci_setup_interrupts(&sc->vtpci_common, type); 539 if (error == 0) 540 vtpci_modern_enable_virtqueues(sc); 541 542 return (error); 543 } 544 545 static void 546 vtpci_modern_stop(device_t dev) 547 { 548 vtpci_modern_reset(device_get_softc(dev)); 549 } 550 551 static int 552 vtpci_modern_reinit(device_t dev, uint64_t features) 553 { 554 struct vtpci_modern_softc *sc; 555 struct vtpci_common *cn; 556 int error; 557 558 sc = device_get_softc(dev); 559 cn = &sc->vtpci_common; 560 561 /* 562 * Redrive the device initialization. This is a bit of an abuse of 563 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 564 * play nice. 565 * 566 * We do not allow the host device to change from what was originally 567 * negotiated beyond what the guest driver changed. MSIX state should 568 * not change, number of virtqueues and their size remain the same, etc. 569 * This will need to be rethought when we want to support migration. 570 */ 571 572 if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 573 vtpci_modern_stop(dev); 574 575 /* 576 * Quickly drive the status through ACK and DRIVER. The device does 577 * not become usable again until DRIVER_OK in reinit complete. 578 */ 579 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); 580 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 581 582 /* 583 * TODO: Check that features are not added as to what was 584 * originally negotiated. 585 */ 586 vtpci_modern_negotiate_features(dev, features); 587 error = vtpci_modern_finalize_features(dev); 588 if (error) { 589 device_printf(dev, "cannot finalize features during reinit\n"); 590 return (error); 591 } 592 593 error = vtpci_reinit(cn); 594 if (error) 595 return (error); 596 597 return (0); 598 } 599 600 static void 601 vtpci_modern_reinit_complete(device_t dev) 602 { 603 struct vtpci_modern_softc *sc; 604 605 sc = device_get_softc(dev); 606 607 vtpci_modern_enable_virtqueues(sc); 608 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 609 } 610 611 static void 612 vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) 613 { 614 struct vtpci_modern_softc *sc; 615 616 sc = device_get_softc(dev); 617 618 vtpci_modern_write_notify_2(sc, offset, queue); 619 } 620 621 static uint8_t 622 vtpci_modern_get_status(struct vtpci_modern_softc *sc) 623 { 624 return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS)); 625 } 626 627 static void 628 vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status) 629 { 630 if (status != VIRTIO_CONFIG_STATUS_RESET) 631 status |= vtpci_modern_get_status(sc); 632 633 vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status); 634 } 635 636 static int 637 vtpci_modern_config_generation(device_t dev) 638 { 639 struct vtpci_modern_softc *sc; 640 uint8_t gen; 641 642 sc = device_get_softc(dev); 643 gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION); 644 645 return (gen); 646 } 647 648 static void 649 vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst, 650 int length) 651 { 652 struct vtpci_modern_softc *sc; 653 654 sc = device_get_softc(dev); 655 656 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 657 panic("%s: attempt to read dev config but not present", 658 __func__); 659 } 660 661 switch (length) { 662 case 1: 663 *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset); 664 break; 665 case 2: 666 *(uint16_t *) dst = virtio_htog16(true, 667 vtpci_modern_read_device_2(sc, offset)); 668 break; 669 case 4: 670 *(uint32_t *) dst = virtio_htog32(true, 671 vtpci_modern_read_device_4(sc, offset)); 672 break; 673 case 8: 674 *(uint64_t *) dst = virtio_htog64(true, 675 vtpci_modern_read_device_8(sc, offset)); 676 break; 677 default: 678 panic("%s: device %s invalid device read length %d offset %d", 679 __func__, device_get_nameunit(dev), length, (int) offset); 680 } 681 } 682 683 static void 684 vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src, 685 int length) 686 { 687 struct vtpci_modern_softc *sc; 688 689 sc = device_get_softc(dev); 690 691 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { 692 panic("%s: attempt to write dev config but not present", 693 __func__); 694 } 695 696 switch (length) { 697 case 1: 698 vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src); 699 break; 700 case 2: { 701 uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src); 702 vtpci_modern_write_device_2(sc, offset, val); 703 break; 704 } 705 case 4: { 706 uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src); 707 vtpci_modern_write_device_4(sc, offset, val); 708 break; 709 } 710 case 8: { 711 uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src); 712 vtpci_modern_write_device_8(sc, offset, val); 713 break; 714 } 715 default: 716 panic("%s: device %s invalid device write length %d offset %d", 717 __func__, device_get_nameunit(dev), length, (int) offset); 718 } 719 } 720 721 static int 722 vtpci_modern_probe_configs(device_t dev) 723 { 724 int error; 725 726 /* 727 * These config capabilities must be present. The DEVICE_CFG 728 * capability is only present if the device requires it. 729 */ 730 731 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL); 732 if (error) { 733 device_printf(dev, "cannot find COMMON_CFG capability\n"); 734 return (error); 735 } 736 737 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL); 738 if (error) { 739 device_printf(dev, "cannot find NOTIFY_CFG capability\n"); 740 return (error); 741 } 742 743 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL); 744 if (error) { 745 device_printf(dev, "cannot find ISR_CFG capability\n"); 746 return (error); 747 } 748 749 return (0); 750 } 751 752 static int 753 vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset) 754 { 755 uint32_t type, bar; 756 int capreg, error; 757 758 for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg); 759 error == 0; 760 error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) { 761 762 type = pci_read_config(dev, capreg + 763 offsetof(struct virtio_pci_cap, cfg_type), 1); 764 bar = pci_read_config(dev, capreg + 765 offsetof(struct virtio_pci_cap, bar), 1); 766 767 /* Must ignore reserved BARs. */ 768 if (bar >= VTPCI_MODERN_MAX_BARS) 769 continue; 770 771 if (type == cfg_type) { 772 if (cap_offset != NULL) 773 *cap_offset = capreg; 774 break; 775 } 776 } 777 778 return (error); 779 } 780 781 static int 782 vtpci_modern_map_common_config(struct vtpci_modern_softc *sc) 783 { 784 device_t dev; 785 int error; 786 787 dev = sc->vtpci_dev; 788 789 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG, 790 sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map); 791 if (error) { 792 device_printf(dev, "cannot find cap COMMON_CFG resource\n"); 793 return (error); 794 } 795 796 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map); 797 if (error) { 798 device_printf(dev, "cannot alloc resource for COMMON_CFG\n"); 799 return (error); 800 } 801 802 return (0); 803 } 804 805 static int 806 vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc) 807 { 808 device_t dev; 809 int cap_offset, error; 810 811 dev = sc->vtpci_dev; 812 813 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG, 814 -1, 2, &sc->vtpci_notify_res_map); 815 if (error) { 816 device_printf(dev, "cannot find cap NOTIFY_CFG resource\n"); 817 return (error); 818 } 819 820 cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset; 821 822 sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset + 823 offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4); 824 825 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map); 826 if (error) { 827 device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n"); 828 return (error); 829 } 830 831 return (0); 832 } 833 834 static int 835 vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc) 836 { 837 device_t dev; 838 int error; 839 840 dev = sc->vtpci_dev; 841 842 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG, 843 sizeof(uint8_t), 1, &sc->vtpci_isr_res_map); 844 if (error) { 845 device_printf(dev, "cannot find cap ISR_CFG resource\n"); 846 return (error); 847 } 848 849 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map); 850 if (error) { 851 device_printf(dev, "cannot alloc resource for ISR_CFG\n"); 852 return (error); 853 } 854 855 return (0); 856 } 857 858 static int 859 vtpci_modern_map_device_config(struct vtpci_modern_softc *sc) 860 { 861 device_t dev; 862 int error; 863 864 dev = sc->vtpci_dev; 865 866 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG, 867 -1, 4, &sc->vtpci_device_res_map); 868 if (error == ENOENT) { 869 /* Device configuration is optional depending on device. */ 870 return (0); 871 } else if (error) { 872 device_printf(dev, "cannot find cap DEVICE_CFG resource\n"); 873 return (error); 874 } 875 876 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map); 877 if (error) { 878 device_printf(dev, "cannot alloc resource for DEVICE_CFG\n"); 879 return (error); 880 } 881 882 return (0); 883 } 884 885 static int 886 vtpci_modern_map_configs(struct vtpci_modern_softc *sc) 887 { 888 int error; 889 890 error = vtpci_modern_map_common_config(sc); 891 if (error) 892 return (error); 893 894 error = vtpci_modern_map_notify_config(sc); 895 if (error) 896 return (error); 897 898 error = vtpci_modern_map_isr_config(sc); 899 if (error) 900 return (error); 901 902 error = vtpci_modern_map_device_config(sc); 903 if (error) 904 return (error); 905 906 vtpci_modern_alloc_msix_resource(sc); 907 908 return (0); 909 } 910 911 static void 912 vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc) 913 { 914 915 vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map); 916 vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map); 917 vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map); 918 vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map); 919 920 vtpci_modern_free_bar_resources(sc); 921 vtpci_modern_free_msix_resource(sc); 922 923 sc->vtpci_notify_offset_multiplier = 0; 924 } 925 926 static int 927 vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type, 928 int min_size, int alignment, struct vtpci_modern_resource_map *res) 929 { 930 device_t dev; 931 int cap_offset, offset, length, error; 932 uint8_t bar, cap_length; 933 934 dev = sc->vtpci_dev; 935 936 error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset); 937 if (error) 938 return (error); 939 940 cap_length = pci_read_config(dev, 941 cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1); 942 943 if (cap_length < sizeof(struct virtio_pci_cap)) { 944 device_printf(dev, "cap %u length %d less than expected\n", 945 cfg_type, cap_length); 946 return (ENXIO); 947 } 948 949 bar = pci_read_config(dev, 950 cap_offset + offsetof(struct virtio_pci_cap, bar), 1); 951 offset = pci_read_config(dev, 952 cap_offset + offsetof(struct virtio_pci_cap, offset), 4); 953 length = pci_read_config(dev, 954 cap_offset + offsetof(struct virtio_pci_cap, length), 4); 955 956 if (min_size != -1 && length < min_size) { 957 device_printf(dev, "cap %u struct length %d less than min %d\n", 958 cfg_type, length, min_size); 959 return (ENXIO); 960 } 961 962 if (offset % alignment) { 963 device_printf(dev, "cap %u struct offset %d not aligned to %d\n", 964 cfg_type, offset, alignment); 965 return (ENXIO); 966 } 967 968 /* BMV: TODO Can we determine the size of the BAR here? */ 969 970 res->vtrm_cap_offset = cap_offset; 971 res->vtrm_bar = bar; 972 res->vtrm_offset = offset; 973 res->vtrm_length = length; 974 res->vtrm_type = vtpci_modern_bar_type(sc, bar); 975 976 return (0); 977 } 978 979 static int 980 vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar) 981 { 982 uint32_t val; 983 984 /* 985 * The BAR described by a config capability may be either an IOPORT or 986 * MEM, but we must know the type when calling bus_alloc_resource(). 987 */ 988 val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4); 989 if (PCI_BAR_IO(val)) 990 return (SYS_RES_IOPORT); 991 else 992 return (SYS_RES_MEMORY); 993 } 994 995 static struct resource * 996 vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) 997 { 998 struct resource *res; 999 1000 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1001 res = sc->vtpci_bar_res[bar].vtbr_res; 1002 MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type); 1003 1004 return (res); 1005 } 1006 1007 static struct resource * 1008 vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar, 1009 int type) 1010 { 1011 struct resource *res; 1012 int rid; 1013 1014 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); 1015 MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT); 1016 1017 res = sc->vtpci_bar_res[bar].vtbr_res; 1018 if (res != NULL) { 1019 MPASS(sc->vtpci_bar_res[bar].vtbr_type == type); 1020 return (res); 1021 } 1022 1023 rid = PCIR_BAR(bar); 1024 res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid, 1025 RF_ACTIVE | RF_UNMAPPED); 1026 if (res != NULL) { 1027 sc->vtpci_bar_res[bar].vtbr_res = res; 1028 sc->vtpci_bar_res[bar].vtbr_type = type; 1029 } 1030 1031 return (res); 1032 } 1033 1034 static void 1035 vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc) 1036 { 1037 device_t dev; 1038 struct resource *res; 1039 int bar, rid, type; 1040 1041 dev = sc->vtpci_dev; 1042 1043 for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) { 1044 res = sc->vtpci_bar_res[bar].vtbr_res; 1045 type = sc->vtpci_bar_res[bar].vtbr_type; 1046 1047 if (res != NULL) { 1048 rid = PCIR_BAR(bar); 1049 bus_release_resource(dev, type, rid, res); 1050 sc->vtpci_bar_res[bar].vtbr_res = NULL; 1051 sc->vtpci_bar_res[bar].vtbr_type = 0; 1052 } 1053 } 1054 } 1055 1056 static int 1057 vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc, 1058 struct vtpci_modern_resource_map *map) 1059 { 1060 struct resource_map_request req; 1061 struct resource *res; 1062 int type; 1063 1064 type = map->vtrm_type; 1065 1066 res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type); 1067 if (res == NULL) 1068 return (ENXIO); 1069 1070 resource_init_map_request(&req); 1071 req.offset = map->vtrm_offset; 1072 req.length = map->vtrm_length; 1073 1074 return (bus_map_resource(sc->vtpci_dev, type, res, &req, 1075 &map->vtrm_map)); 1076 } 1077 1078 static void 1079 vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc, 1080 struct vtpci_modern_resource_map *map) 1081 { 1082 struct resource *res; 1083 int type; 1084 1085 type = map->vtrm_type; 1086 res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type); 1087 1088 if (res != NULL && map->vtrm_map.r_size != 0) { 1089 bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map); 1090 bzero(map, sizeof(struct vtpci_modern_resource_map)); 1091 } 1092 } 1093 1094 static void 1095 vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc) 1096 { 1097 device_t dev; 1098 int bar; 1099 1100 dev = sc->vtpci_dev; 1101 1102 if (!vtpci_is_msix_available(&sc->vtpci_common) || 1103 (bar = pci_msix_table_bar(dev)) == -1) 1104 return; 1105 1106 /* TODO: Can this BAR be in the 0-5 range? */ 1107 sc->vtpci_msix_bar = bar; 1108 if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1109 &bar, RF_ACTIVE)) == NULL) 1110 device_printf(dev, "Unable to map MSIX table\n"); 1111 } 1112 1113 static void 1114 vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc) 1115 { 1116 device_t dev; 1117 1118 dev = sc->vtpci_dev; 1119 1120 if (sc->vtpci_msix_res != NULL) { 1121 bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar, 1122 sc->vtpci_msix_res); 1123 sc->vtpci_msix_bar = 0; 1124 sc->vtpci_msix_res = NULL; 1125 } 1126 } 1127 1128 static void 1129 vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc) 1130 { 1131 device_t dev, child; 1132 1133 dev = sc->vtpci_dev; 1134 child = vtpci_child_device(&sc->vtpci_common); 1135 1136 if (child == NULL || device_get_state(child) != DS_NOTPRESENT) 1137 return; 1138 1139 if (device_probe(child) != 0) 1140 return; 1141 1142 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); 1143 1144 if (device_attach(child) != 0) { 1145 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); 1146 /* Reset state for later attempt. */ 1147 vtpci_modern_child_detached(dev, child); 1148 } else { 1149 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1150 VIRTIO_ATTACH_COMPLETED(child); 1151 } 1152 } 1153 1154 static int 1155 vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset, 1156 struct vtpci_interrupt *intr) 1157 { 1158 uint16_t vector; 1159 1160 if (intr != NULL) { 1161 /* Map from guest rid to host vector. */ 1162 vector = intr->vti_rid - 1; 1163 } else 1164 vector = VIRTIO_MSI_NO_VECTOR; 1165 1166 vtpci_modern_write_common_2(sc, offset, vector); 1167 return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV); 1168 } 1169 1170 static int 1171 vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) 1172 { 1173 struct vtpci_modern_softc *sc; 1174 int error; 1175 1176 sc = device_get_softc(dev); 1177 1178 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr); 1179 if (error) { 1180 device_printf(dev, 1181 "unable to register config MSIX interrupt\n"); 1182 return (error); 1183 } 1184 1185 return (0); 1186 } 1187 1188 static int 1189 vtpci_modern_register_vq_msix(device_t dev, int idx, 1190 struct vtpci_interrupt *intr) 1191 { 1192 struct vtpci_modern_softc *sc; 1193 int error; 1194 1195 sc = device_get_softc(dev); 1196 1197 vtpci_modern_select_virtqueue(sc, idx); 1198 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr); 1199 if (error) { 1200 device_printf(dev, 1201 "unable to register virtqueue MSIX interrupt\n"); 1202 return (error); 1203 } 1204 1205 return (0); 1206 } 1207 1208 static void 1209 vtpci_modern_reset(struct vtpci_modern_softc *sc) 1210 { 1211 /* 1212 * Setting the status to RESET sets the host device to the 1213 * original, uninitialized state. Must poll the status until 1214 * the reset is complete. 1215 */ 1216 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); 1217 1218 while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) 1219 cpu_spinwait(); 1220 } 1221 1222 static void 1223 vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx) 1224 { 1225 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx); 1226 } 1227 1228 static uint8_t 1229 vtpci_modern_read_isr(device_t dev) 1230 { 1231 return (vtpci_modern_read_isr_1(device_get_softc(dev), 0)); 1232 } 1233 1234 static uint16_t 1235 vtpci_modern_get_vq_size(device_t dev, int idx) 1236 { 1237 struct vtpci_modern_softc *sc; 1238 1239 sc = device_get_softc(dev); 1240 1241 vtpci_modern_select_virtqueue(sc, idx); 1242 return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE)); 1243 } 1244 1245 static bus_size_t 1246 vtpci_modern_get_vq_notify_off(device_t dev, int idx) 1247 { 1248 struct vtpci_modern_softc *sc; 1249 uint16_t q_notify_off; 1250 1251 sc = device_get_softc(dev); 1252 1253 vtpci_modern_select_virtqueue(sc, idx); 1254 q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF); 1255 1256 return (q_notify_off * sc->vtpci_notify_offset_multiplier); 1257 } 1258 1259 static void 1260 vtpci_modern_set_vq(device_t dev, struct virtqueue *vq) 1261 { 1262 struct vtpci_modern_softc *sc; 1263 1264 sc = device_get_softc(dev); 1265 1266 vtpci_modern_select_virtqueue(sc, virtqueue_index(vq)); 1267 1268 /* BMV: Currently we never adjust the device's proposed VQ size. */ 1269 vtpci_modern_write_common_2(sc, 1270 VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq)); 1271 1272 vtpci_modern_write_common_8(sc, 1273 VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq)); 1274 vtpci_modern_write_common_8(sc, 1275 VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq)); 1276 vtpci_modern_write_common_8(sc, 1277 VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq)); 1278 } 1279 1280 static void 1281 vtpci_modern_disable_vq(device_t dev, int idx) 1282 { 1283 struct vtpci_modern_softc *sc; 1284 1285 sc = device_get_softc(dev); 1286 1287 vtpci_modern_select_virtqueue(sc, idx); 1288 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL); 1289 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL); 1290 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL); 1291 } 1292 1293 static void 1294 vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc) 1295 { 1296 int idx; 1297 1298 for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) { 1299 vtpci_modern_select_virtqueue(sc, idx); 1300 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1); 1301 } 1302 } 1303 1304 static uint8_t 1305 vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off) 1306 { 1307 return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off)); 1308 } 1309 1310 static uint16_t 1311 vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off) 1312 { 1313 return virtio_htog16(true, 1314 bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off)); 1315 } 1316 1317 static uint32_t 1318 vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off) 1319 { 1320 return virtio_htog32(true, 1321 bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off)); 1322 } 1323 1324 static void 1325 vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off, 1326 uint8_t val) 1327 { 1328 bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val); 1329 } 1330 1331 static void 1332 vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off, 1333 uint16_t val) 1334 { 1335 bus_write_2(&sc->vtpci_common_res_map.vtrm_map, 1336 off, virtio_gtoh16(true, val)); 1337 } 1338 1339 static void 1340 vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off, 1341 uint32_t val) 1342 { 1343 bus_write_4(&sc->vtpci_common_res_map.vtrm_map, 1344 off, virtio_gtoh32(true, val)); 1345 } 1346 1347 static void 1348 vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off, 1349 uint64_t val) 1350 { 1351 uint32_t val0, val1; 1352 1353 val0 = (uint32_t) val; 1354 val1 = val >> 32; 1355 1356 vtpci_modern_write_common_4(sc, off, val0); 1357 vtpci_modern_write_common_4(sc, off + 4, val1); 1358 } 1359 1360 static void 1361 vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off, 1362 uint16_t val) 1363 { 1364 bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val); 1365 } 1366 1367 static uint8_t 1368 vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off) 1369 { 1370 return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off)); 1371 } 1372 1373 static uint8_t 1374 vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off) 1375 { 1376 return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off)); 1377 } 1378 1379 static uint16_t 1380 vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off) 1381 { 1382 return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off)); 1383 } 1384 1385 static uint32_t 1386 vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off) 1387 { 1388 return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off)); 1389 } 1390 1391 static uint64_t 1392 vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off) 1393 { 1394 device_t dev; 1395 int gen; 1396 uint32_t val0, val1; 1397 1398 dev = sc->vtpci_dev; 1399 1400 /* 1401 * Treat the 64-bit field as two 32-bit fields. Use the generation 1402 * to ensure a consistent read. 1403 */ 1404 do { 1405 gen = vtpci_modern_config_generation(dev); 1406 val0 = vtpci_modern_read_device_4(sc, off); 1407 val1 = vtpci_modern_read_device_4(sc, off + 4); 1408 } while (gen != vtpci_modern_config_generation(dev)); 1409 1410 return (((uint64_t) val1 << 32) | val0); 1411 } 1412 1413 static void 1414 vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off, 1415 uint8_t val) 1416 { 1417 bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val); 1418 } 1419 1420 static void 1421 vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off, 1422 uint16_t val) 1423 { 1424 bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val); 1425 } 1426 1427 static void 1428 vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off, 1429 uint32_t val) 1430 { 1431 bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val); 1432 } 1433 1434 static void 1435 vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off, 1436 uint64_t val) 1437 { 1438 uint32_t val0, val1; 1439 1440 val0 = (uint32_t) val; 1441 val1 = val >> 32; 1442 1443 vtpci_modern_write_device_4(sc, off, val0); 1444 vtpci_modern_write_device_4(sc, off + 4, val1); 1445 } 1446