1 /*- 2 * Copyright (c) 2015-2016 Mellanox Technologies, Ltd. 3 * All rights reserved. 4 * Copyright (c) 2020-2026 The FreeBSD Foundation 5 * 6 * Portions of this software were developed by Björn Zeeb 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 /* 32 * We have two ways to create a pci_dev (pdev): 33 * (1) coming from the device_attach DEVMETHOD, and 34 * (2) the other from manual creation via lkpinew_pci_dev(). 35 * 36 * Only devices from (1) end up on our LinuxKPI global pci_devices list. 37 * All others are "place fillers" -- XXX if only "place filler" was always true. 38 */ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/bus.h> 43 #include <sys/malloc.h> 44 #include <sys/kernel.h> 45 #include <sys/sysctl.h> 46 #include <sys/lock.h> 47 #include <sys/mutex.h> 48 #include <sys/fcntl.h> 49 #include <sys/file.h> 50 #include <sys/filio.h> 51 #include <sys/pciio.h> 52 #include <sys/pctrie.h> 53 #include <sys/rman.h> 54 #include <sys/rwlock.h> 55 #include <sys/stdarg.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 60 #include <machine/bus.h> 61 #include <machine/resource.h> 62 63 #include <dev/pci/pcivar.h> 64 #include <dev/pci/pci_private.h> 65 #include <dev/pci/pci_iov.h> 66 #include <dev/backlight/backlight.h> 67 68 #include <linux/kernel.h> 69 #include <linux/kobject.h> 70 #include <linux/device.h> 71 #include <linux/slab.h> 72 #include <linux/module.h> 73 #include <linux/cdev.h> 74 #include <linux/file.h> 75 #include <linux/sysfs.h> 76 #include <linux/mm.h> 77 #include <linux/io.h> 78 #include <linux/vmalloc.h> 79 #define WANT_NATIVE_PCI_GET_SLOT 80 #include <linux/pci.h> 81 #include <linux/compat.h> 82 83 #include <linux/backlight.h> 84 85 #include "backlight_if.h" 86 #include "pcib_if.h" 87 88 /* Undef the linux function macro defined in linux/pci.h */ 89 #undef pci_get_class 90 91 extern int linuxkpi_debug; 92 93 SYSCTL_DECL(_compat_linuxkpi); 94 95 static counter_u64_t lkpi_pci_nseg1_fail; 96 SYSCTL_COUNTER_U64(_compat_linuxkpi, OID_AUTO, lkpi_pci_nseg1_fail, CTLFLAG_RD, 97 &lkpi_pci_nseg1_fail, "Count of busdma mapping failures of single-segment"); 98 99 static device_probe_t linux_pci_probe; 100 static device_attach_t linux_pci_attach; 101 static device_detach_t linux_pci_detach; 102 static device_suspend_t linux_pci_suspend; 103 static device_resume_t linux_pci_resume; 104 static device_shutdown_t linux_pci_shutdown; 105 static pci_iov_init_t linux_pci_iov_init; 106 static pci_iov_uninit_t linux_pci_iov_uninit; 107 static pci_iov_add_vf_t linux_pci_iov_add_vf; 108 static int linux_backlight_get_status(device_t dev, struct backlight_props *props); 109 static int linux_backlight_update_status(device_t dev, struct backlight_props *props); 110 static int linux_backlight_get_info(device_t dev, struct backlight_info *info); 111 static void lkpi_pcim_iomap_table_release(struct device *, void *); 112 static void lkpinew_pci_dev_release(struct device *); 113 114 static device_method_t pci_methods[] = { 115 DEVMETHOD(device_probe, linux_pci_probe), 116 DEVMETHOD(device_attach, linux_pci_attach), 117 DEVMETHOD(device_detach, linux_pci_detach), 118 DEVMETHOD(device_suspend, linux_pci_suspend), 119 DEVMETHOD(device_resume, linux_pci_resume), 120 DEVMETHOD(device_shutdown, linux_pci_shutdown), 121 DEVMETHOD(pci_iov_init, linux_pci_iov_init), 122 DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit), 123 DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf), 124 125 /* Bus interface. */ 126 DEVMETHOD(bus_add_child, bus_generic_add_child), 127 128 /* backlight interface */ 129 DEVMETHOD(backlight_update_status, linux_backlight_update_status), 130 DEVMETHOD(backlight_get_status, linux_backlight_get_status), 131 DEVMETHOD(backlight_get_info, linux_backlight_get_info), 132 DEVMETHOD_END 133 }; 134 135 const char *pci_power_names[] = { 136 "UNKNOWN", "D0", "D1", "D2", "D3hot", "D3cold" 137 }; 138 139 /* We need some meta-struct to keep track of these for devres. */ 140 struct pci_devres { 141 bool enable_io; 142 /* PCIR_MAX_BAR_0 + 1 = 6 => BIT(0..5). */ 143 uint8_t region_mask; 144 struct resource *region_table[PCIR_MAX_BAR_0 + 1]; /* Not needed. */ 145 }; 146 struct pcim_iomap_devres { 147 void *mmio_table[PCIR_MAX_BAR_0 + 1]; 148 struct resource *res_table[PCIR_MAX_BAR_0 + 1]; 149 }; 150 151 struct linux_dma_priv { 152 uint64_t dma_mask; 153 bus_dma_tag_t dmat; 154 uint64_t dma_coherent_mask; 155 bus_dma_tag_t dmat_coherent; 156 struct mtx lock; 157 struct pctrie ptree; 158 }; 159 #define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock) 160 #define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock) 161 162 static void 163 lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres *dr, int bar, 164 void *res) 165 { 166 dr->mmio_table[bar] = (void *)rman_get_bushandle(res); 167 dr->res_table[bar] = res; 168 } 169 170 static bool 171 lkpi_pci_bar_id_valid(int bar) 172 { 173 if (bar < 0 || bar > PCIR_MAX_BAR_0) 174 return (false); 175 176 return (true); 177 } 178 179 static int 180 linux_pdev_dma_uninit(struct pci_dev *pdev) 181 { 182 struct linux_dma_priv *priv; 183 184 priv = pdev->dev.dma_priv; 185 if (priv->dmat) 186 bus_dma_tag_destroy(priv->dmat); 187 if (priv->dmat_coherent) 188 bus_dma_tag_destroy(priv->dmat_coherent); 189 mtx_destroy(&priv->lock); 190 pdev->dev.dma_priv = NULL; 191 free(priv, M_DEVBUF); 192 return (0); 193 } 194 195 static int 196 linux_pdev_dma_init(struct pci_dev *pdev) 197 { 198 struct linux_dma_priv *priv; 199 int error; 200 201 priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO); 202 203 mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF); 204 pctrie_init(&priv->ptree); 205 206 pdev->dev.dma_priv = priv; 207 208 /* Create a default DMA tags. */ 209 error = linux_dma_tag_init(&pdev->dev, DMA_BIT_MASK(64)); 210 if (error != 0) 211 goto err; 212 /* Coherent is lower 32bit only by default in Linux. */ 213 error = linux_dma_tag_init_coherent(&pdev->dev, DMA_BIT_MASK(32)); 214 if (error != 0) 215 goto err; 216 217 return (error); 218 219 err: 220 linux_pdev_dma_uninit(pdev); 221 return (error); 222 } 223 224 int 225 linux_dma_tag_init(struct device *dev, u64 dma_mask) 226 { 227 struct linux_dma_priv *priv; 228 int error; 229 230 priv = dev->dma_priv; 231 232 if (priv->dmat) { 233 if (priv->dma_mask == dma_mask) 234 return (0); 235 236 bus_dma_tag_destroy(priv->dmat); 237 } 238 239 priv->dma_mask = dma_mask; 240 241 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 242 1, 0, /* alignment, boundary */ 243 dma_mask, /* lowaddr */ 244 BUS_SPACE_MAXADDR, /* highaddr */ 245 NULL, NULL, /* filtfunc, filtfuncarg */ 246 BUS_SPACE_MAXSIZE, /* maxsize */ 247 1, /* nsegments */ 248 BUS_SPACE_MAXSIZE, /* maxsegsz */ 249 0, /* flags */ 250 NULL, NULL, /* lockfunc, lockfuncarg */ 251 &priv->dmat); 252 return (-error); 253 } 254 255 int 256 linux_dma_tag_init_coherent(struct device *dev, u64 dma_mask) 257 { 258 struct linux_dma_priv *priv; 259 int error; 260 261 priv = dev->dma_priv; 262 263 if (priv->dmat_coherent) { 264 if (priv->dma_coherent_mask == dma_mask) 265 return (0); 266 267 bus_dma_tag_destroy(priv->dmat_coherent); 268 } 269 270 priv->dma_coherent_mask = dma_mask; 271 272 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 273 1, 0, /* alignment, boundary */ 274 dma_mask, /* lowaddr */ 275 BUS_SPACE_MAXADDR, /* highaddr */ 276 NULL, NULL, /* filtfunc, filtfuncarg */ 277 BUS_SPACE_MAXSIZE, /* maxsize */ 278 1, /* nsegments */ 279 BUS_SPACE_MAXSIZE, /* maxsegsz */ 280 0, /* flags */ 281 NULL, NULL, /* lockfunc, lockfuncarg */ 282 &priv->dmat_coherent); 283 return (-error); 284 } 285 286 static struct pci_driver * 287 linux_pci_find(device_t dev, const struct pci_device_id **idp) 288 { 289 const struct pci_device_id *id; 290 struct pci_driver *pdrv; 291 uint16_t vendor; 292 uint16_t device; 293 uint16_t subvendor; 294 uint16_t subdevice; 295 296 vendor = pci_get_vendor(dev); 297 device = pci_get_device(dev); 298 subvendor = pci_get_subvendor(dev); 299 subdevice = pci_get_subdevice(dev); 300 301 spin_lock(&pci_lock); 302 list_for_each_entry(pdrv, &pci_drivers, node) { 303 for (id = pdrv->id_table; id->vendor != 0; id++) { 304 if (vendor == id->vendor && 305 (PCI_ANY_ID == id->device || device == id->device) && 306 (PCI_ANY_ID == id->subvendor || subvendor == id->subvendor) && 307 (PCI_ANY_ID == id->subdevice || subdevice == id->subdevice)) { 308 *idp = id; 309 spin_unlock(&pci_lock); 310 return (pdrv); 311 } 312 } 313 } 314 spin_unlock(&pci_lock); 315 return (NULL); 316 } 317 318 struct pci_dev * 319 lkpi_pci_get_device(uint32_t vendor, uint32_t device, struct pci_dev *odev) 320 { 321 struct pci_dev *pdev, *found, *odev0; 322 323 odev0 = odev; 324 found = NULL; 325 spin_lock(&pci_lock); 326 list_for_each_entry(pdev, &pci_devices, links) { 327 /* Walk until we find odev. */ 328 if (odev != NULL) { 329 if (pdev == odev) 330 odev = NULL; 331 continue; 332 } 333 334 if ((pdev->vendor == vendor || vendor == PCI_ANY_ID) && 335 (pdev->device == device || device == PCI_ANY_ID)) { 336 found = pdev; 337 break; 338 } 339 } 340 pci_dev_get(found); 341 spin_unlock(&pci_lock); 342 pci_dev_put(odev0); 343 344 return (found); 345 } 346 347 static void 348 lkpi_pci_dev_release(struct device *dev) 349 { 350 struct pci_dev *pdev; 351 352 /* 353 * Before anything else, we have to free all the dynamic 354 * resource which are on the devres list. 355 * Otherwise we risk that supporting infrastructure 356 * is gone and we panic 'randomly'. 357 */ 358 lkpi_devres_release_free_list(dev); 359 360 /* 361 * Now undo linux_pci_attach_device() in reverse-ish 362 * order. 363 */ 364 pdev = to_pci_dev(dev); 365 366 /* 367 * pdrv->remove happens before pci_put_dev() in 368 * linux_pci_detach_device(), which means the driver should have 369 * cleaned up before we get here; see irqents and mmio below. 370 */ 371 372 /* Clear the hierarchy recursively to root. */ 373 if (pdev->bus->self != pdev) { 374 pci_dev_put(pdev->bus->self); 375 pdev->bus->self = NULL; 376 } 377 378 if (pdev->root != NULL) { 379 lkpinew_pci_dev_release(&pdev->root->dev); /* pci_dev_put(pdev->root); ? */ 380 pdev->root = NULL; 381 } 382 383 spin_lock(&pci_lock); 384 list_del(&pdev->links); 385 spin_unlock(&pci_lock); 386 387 linux_pdev_dma_uninit(pdev); 388 389 /* irq? */ 390 391 /* Undo lkpifill_pci_dev(). */ 392 /* devres is gone already; went at the very top. */ 393 if (!list_empty_careful(&pdev->dev.irqents)) { 394 dev_warn(&pdev->dev, "%s: driver did not clean up; " 395 "leaking IRQs\n", __func__); 396 /* 397 * XXX add private function to interrupt.h/linux_interrupt.c 398 * to walk the list and call free_irq on each if we have to. 399 */ 400 } 401 402 spin_lock_destroy(&dev->devres_lock); 403 spin_lock_destroy(&pdev->pcie_cap_lock); 404 405 if (!TAILQ_EMPTY(&pdev->mmio)) { 406 dev_warn(&pdev->dev, "%s: driver did not clean up; " 407 "leaking mmio resources\n", __func__); 408 /* XXX we have two functions to walk and release in here. */ 409 } 410 411 if (pdev->msi_desc != NULL) { 412 for (int i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--) 413 free(pdev->msi_desc[i], M_DEVBUF); 414 free(pdev->msi_desc, M_DEVBUF); 415 } 416 417 free(pdev->bus, M_DEVBUF); 418 kfree(pdev->path_name); 419 420 /* 421 * Lastly, apply an internal hack in order to signal 422 * that this was run (device reference fully dropped). 423 * See comment in linux_pci_detach_device(). 424 */ 425 pdev->dev.release = NULL; 426 } 427 428 static int 429 lkpifill_pci_dev(device_t dev, struct pci_dev *pdev) 430 { 431 struct pci_devinfo *dinfo; 432 int error; 433 434 error = kobject_init_and_add(&pdev->dev.kobj, &linux_dev_ktype, 435 &linux_root_device.kobj, device_get_nameunit(dev)); 436 if (error != 0) { 437 printf("%s:%d: kobject_init_and_add returned %d\n", 438 __func__, __LINE__, error); 439 return (error); 440 } 441 442 pdev->devfn = PCI_DEVFN(pci_get_slot(dev), pci_get_function(dev)); 443 pdev->vendor = pci_get_vendor(dev); 444 pdev->device = pci_get_device(dev); 445 pdev->subsystem_vendor = pci_get_subvendor(dev); 446 pdev->subsystem_device = pci_get_subdevice(dev); 447 pdev->class = pci_get_class(dev); 448 pdev->revision = pci_get_revid(dev); 449 pdev->path_name = kasprintf(GFP_KERNEL, "%04d:%02d:%02d.%d", 450 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), 451 pci_get_function(dev)); 452 453 pdev->bus = malloc(sizeof(*pdev->bus), M_DEVBUF, M_WAITOK | M_ZERO); 454 pdev->bus->number = pci_get_bus(dev); 455 pdev->bus->domain = pci_get_domain(dev); 456 457 /* Check if we have reached the root to satisfy pci_is_root_bus() */ 458 dinfo = device_get_ivars(dev); 459 if (dinfo->cfg.pcie.pcie_location != 0 && 460 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) { 461 pdev->bus->self = NULL; 462 } else { 463 /* 464 * This should be the upstream bridge; pci_upstream_bridge() 465 * handles that case on demand as otherwise we'll shadow the 466 * entire PCI hierarchy. 467 */ 468 pdev->bus->self = pdev; 469 } 470 pdev->dev.bsddev = dev; 471 pdev->dev.parent = &linux_root_device; 472 pdev->dev.release = lkpi_pci_dev_release; 473 474 if (pci_msi_count(dev) > 0) 475 pdev->msi_desc = malloc(pci_msi_count(dev) * 476 sizeof(*pdev->msi_desc), M_DEVBUF, M_WAITOK | M_ZERO); 477 478 TAILQ_INIT(&pdev->mmio); 479 spin_lock_init(&pdev->pcie_cap_lock); 480 spin_lock_init(&pdev->dev.devres_lock); 481 INIT_LIST_HEAD(&pdev->dev.devres_head); 482 INIT_LIST_HEAD(&pdev->dev.irqents); 483 484 return (0); 485 } 486 487 static void 488 lkpinew_pci_dev_release(struct device *dev) 489 { 490 struct pci_dev *pdev; 491 int i; 492 493 pdev = to_pci_dev(dev); 494 if (pdev->root != NULL) 495 pci_dev_put(pdev->root); 496 if (pdev->bus->self != pdev && pdev->bus->self != NULL) 497 pci_dev_put(pdev->bus->self); 498 free(pdev->bus, M_DEVBUF); 499 if (pdev->msi_desc != NULL) { 500 for (i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--) 501 free(pdev->msi_desc[i], M_DEVBUF); 502 free(pdev->msi_desc, M_DEVBUF); 503 } 504 kfree(pdev->path_name); 505 free(pdev, M_DEVBUF); 506 } 507 508 struct pci_dev * 509 lkpinew_pci_dev(device_t dev) 510 { 511 struct pci_dev *pdev; 512 int error; 513 514 pdev = malloc(sizeof(*pdev), M_DEVBUF, M_WAITOK|M_ZERO); 515 error = lkpifill_pci_dev(dev, pdev); 516 if (error != 0) { 517 free(pdev, M_DEVBUF); 518 return (NULL); 519 } 520 pdev->dev.release = lkpinew_pci_dev_release; 521 522 return (pdev); 523 } 524 525 struct pci_dev * 526 lkpi_pci_get_class(unsigned int class, struct pci_dev *from) 527 { 528 device_t dev; 529 device_t devfrom = NULL; 530 struct pci_dev *pdev; 531 532 if (from != NULL) 533 devfrom = from->dev.bsddev; 534 535 dev = pci_find_class_from(class >> 16, (class >> 8) & 0xFF, devfrom); 536 if (dev == NULL) 537 return (NULL); 538 539 pdev = lkpinew_pci_dev(dev); 540 return (pdev); 541 } 542 543 struct pci_dev * 544 lkpi_pci_get_base_class(unsigned int baseclass, struct pci_dev *from) 545 { 546 device_t dev; 547 device_t devfrom = NULL; 548 struct pci_dev *pdev; 549 550 if (from != NULL) 551 devfrom = from->dev.bsddev; 552 553 dev = pci_find_base_class_from(baseclass, devfrom); 554 if (dev == NULL) 555 return (NULL); 556 557 pdev = lkpinew_pci_dev(dev); 558 return (pdev); 559 } 560 561 struct pci_dev * 562 lkpi_pci_get_domain_bus_and_slot(int domain, unsigned int bus, 563 unsigned int devfn) 564 { 565 device_t dev; 566 struct pci_dev *pdev; 567 568 dev = pci_find_dbsf(domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 569 if (dev == NULL) 570 return (NULL); 571 572 pdev = lkpinew_pci_dev(dev); 573 return (pdev); 574 } 575 576 struct pci_dev * 577 lkpi_pci_get_slot(struct pci_bus *pbus, unsigned int devfn) 578 { 579 device_t dev; 580 struct pci_dev *pdev; 581 582 dev = pci_find_bsf(pbus->number, PCI_SLOT(devfn), PCI_FUNC(devfn)); 583 if (dev == NULL) 584 return (NULL); 585 586 pdev = lkpinew_pci_dev(dev); 587 return (pdev); 588 } 589 590 static int 591 linux_pci_probe(device_t dev) 592 { 593 const struct pci_device_id *id; 594 struct pci_driver *pdrv; 595 596 if ((pdrv = linux_pci_find(dev, &id)) == NULL) 597 return (ENXIO); 598 if (device_get_driver(dev) != &pdrv->bsddriver) 599 return (ENXIO); 600 device_set_desc(dev, pdrv->name); 601 602 /* Assume BSS initialized (should never return BUS_PROBE_SPECIFIC). */ 603 if (pdrv->bsd_probe_return == 0) 604 return (BUS_PROBE_DEFAULT); 605 else 606 return (pdrv->bsd_probe_return); 607 } 608 609 static int 610 linux_pci_attach(device_t dev) 611 { 612 const struct pci_device_id *id; 613 struct pci_driver *pdrv; 614 struct pci_dev *pdev; 615 616 pdrv = linux_pci_find(dev, &id); 617 pdev = device_get_softc(dev); 618 619 MPASS(pdrv != NULL); 620 MPASS(pdev != NULL); 621 622 return (linux_pci_attach_device(dev, pdrv, id, pdev)); 623 } 624 625 static struct resource_list_entry * 626 linux_pci_reserve_bar(struct pci_dev *pdev, struct resource_list *rl, 627 int type, int rid) 628 { 629 device_t dev; 630 struct resource *res; 631 632 KASSERT(type == SYS_RES_IOPORT || type == SYS_RES_MEMORY, 633 ("trying to reserve non-BAR type %d", type)); 634 635 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? 636 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; 637 res = pci_reserve_map(device_get_parent(dev), dev, type, rid, 0, ~0, 638 1, 1, 0); 639 if (res == NULL) 640 return (NULL); 641 return (resource_list_find(rl, type, rid)); 642 } 643 644 static struct resource_list_entry * 645 linux_pci_get_rle(struct pci_dev *pdev, int type, int rid, bool reserve_bar) 646 { 647 struct pci_devinfo *dinfo; 648 struct resource_list *rl; 649 struct resource_list_entry *rle; 650 651 dinfo = device_get_ivars(pdev->dev.bsddev); 652 rl = &dinfo->resources; 653 rle = resource_list_find(rl, type, rid); 654 /* Reserve resources for this BAR if needed. */ 655 if (rle == NULL && reserve_bar) 656 rle = linux_pci_reserve_bar(pdev, rl, type, rid); 657 return (rle); 658 } 659 660 int 661 linux_pci_attach_device(device_t dev, struct pci_driver *pdrv, 662 const struct pci_device_id *id, struct pci_dev *pdev) 663 { 664 struct resource_list_entry *rle; 665 device_t parent; 666 struct pci_dev *pbus, *ppbus; 667 uintptr_t rid; 668 int error; 669 bool isdrm; 670 671 linux_set_current(curthread); 672 673 parent = device_get_parent(dev); 674 isdrm = pdrv != NULL && pdrv->isdrm; 675 676 if (isdrm) { 677 struct pci_devinfo *dinfo; 678 679 dinfo = device_get_ivars(parent); 680 device_set_ivars(dev, dinfo); 681 } 682 683 error = lkpifill_pci_dev(dev, pdev); 684 if (error != 0) 685 return (error); 686 687 if (isdrm) 688 PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid); 689 else 690 PCI_GET_ID(parent, dev, PCI_ID_RID, &rid); 691 pdev->devfn = rid; 692 pdev->pdrv = pdrv; 693 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0, false); 694 if (rle != NULL) 695 pdev->dev.irq = rle->start; 696 else 697 pdev->dev.irq = LINUX_IRQ_INVALID; 698 pdev->irq = pdev->dev.irq; 699 error = linux_pdev_dma_init(pdev); 700 if (error) 701 goto out_dma_init; 702 703 spin_lock(&pci_lock); 704 list_add(&pdev->links, &pci_devices); 705 spin_unlock(&pci_lock); 706 707 /* 708 * Create the hierarchy now as we cannot on demand later. 709 * Take special care of DRM as there is a non-PCI device in the chain. 710 */ 711 pbus = pdev; 712 if (isdrm) { 713 pbus = lkpinew_pci_dev(parent); 714 if (pbus == NULL) { 715 error = ENXIO; 716 goto out_dma_init; 717 } 718 } 719 pcie_find_root_port(pbus); 720 if (isdrm) 721 pdev->root = pbus->root; 722 ppbus = pci_upstream_bridge(pbus); 723 while (ppbus != NULL && ppbus != pbus) { 724 pbus = ppbus; 725 ppbus = pci_upstream_bridge(pbus); 726 } 727 728 if (pdrv != NULL) { 729 error = pdrv->probe(pdev, id); 730 if (error) 731 goto out_probe; 732 } 733 return (0); 734 735 /* XXX the cleanup does not match the allocation up there. */ 736 out_probe: 737 free(pdev->bus, M_DEVBUF); 738 spin_lock_destroy(&pdev->pcie_cap_lock); 739 linux_pdev_dma_uninit(pdev); 740 out_dma_init: 741 spin_lock(&pci_lock); 742 list_del(&pdev->links); 743 spin_unlock(&pci_lock); 744 put_device(&pdev->dev); 745 return (-error); 746 } 747 748 static int 749 linux_pci_detach(device_t dev) 750 { 751 struct pci_dev *pdev; 752 int error; 753 754 pdev = device_get_softc(dev); 755 MPASS(pdev != NULL); 756 757 error = linux_pci_detach_device(pdev); 758 if (error == 0) 759 device_set_desc(dev, NULL); 760 761 return (error); 762 } 763 764 int 765 linux_pci_detach_device(struct pci_dev *pdev) 766 { 767 768 linux_set_current(curthread); 769 770 /* 771 * We cannot do much here as almost everything will have 772 * to happen as the last reference to the LinuxKPI device 773 * goes away. That will call the release function, 774 * which lkpifill_pci_dev() set. That is were most 775 * of the cleanup will happen. But before that give 776 * the driver a chance to cleanup. 777 * The big problem is that the Linux KPI does not 778 * report back if it was the last kref (well kref 779 * does report back but then kobj, dev, pdev do not). 780 * So we have little way of knowing if the release 781 * happened or not. We have to play tricks for that 782 * and we can given the softc (pdev) is still valid 783 * until we return from here. 784 */ 785 786 if (pdev->pdrv != NULL) 787 pdev->pdrv->remove(pdev); 788 789 pci_dev_put(pdev); 790 791 /* 792 * We (ab)use the release function as a guard to 793 * know if we made it there and the device is gone. 794 */ 795 if (pdev->dev.release != lkpi_pci_dev_release) 796 return (0); 797 798 /* 799 * Detach failed. 800 * We need to re-acquire the ref and wait for 801 * the other refs to be gone... In theory this 802 * should never happen, so log it! 803 * XXX I wish there was a KPI to query the ref. 804 * 805 * If we do not error and wait, we will have a 806 * LinuxKPI device dangling active with pointers 807 * but the FreeBSD device_t will be 'gone'. 808 */ 809 device_printf(pdev->dev.bsddev, "%s failed due to %u other pending " 810 "references on the LinuxKPI device.\n", __func__, 811 kref_read(&pdev->dev.kobj.kref)); 812 pci_dev_get(pdev); 813 814 return (EBUSY); 815 } 816 817 static int 818 lkpi_pci_disable_dev(struct device *dev) 819 { 820 821 (void) pci_disable_io(dev->bsddev, SYS_RES_MEMORY); 822 (void) pci_disable_io(dev->bsddev, SYS_RES_IOPORT); 823 return (0); 824 } 825 826 static struct pci_devres * 827 lkpi_pci_devres_get_alloc(struct pci_dev *pdev) 828 { 829 struct pci_devres *dr; 830 831 dr = lkpi_devres_find(&pdev->dev, lkpi_pci_devres_release, NULL, NULL); 832 if (dr == NULL) { 833 dr = lkpi_devres_alloc(lkpi_pci_devres_release, sizeof(*dr), 834 GFP_KERNEL | __GFP_ZERO); 835 if (dr != NULL) 836 lkpi_devres_add(&pdev->dev, dr); 837 } 838 839 return (dr); 840 } 841 842 static struct pci_devres * 843 lkpi_pci_devres_find(struct pci_dev *pdev) 844 { 845 if (!pdev->managed) 846 return (NULL); 847 848 return (lkpi_pci_devres_get_alloc(pdev)); 849 } 850 851 void 852 lkpi_pci_devres_release(struct device *dev, void *p) 853 { 854 struct pci_devres *dr; 855 struct pci_dev *pdev; 856 int bar; 857 858 pdev = to_pci_dev(dev); 859 dr = p; 860 861 if (pdev->msix_enabled) 862 lkpi_pci_disable_msix(pdev); 863 if (pdev->msi_enabled) 864 lkpi_pci_disable_msi(pdev); 865 866 if (dr->enable_io && lkpi_pci_disable_dev(dev) == 0) 867 dr->enable_io = false; 868 869 if (dr->region_mask == 0) 870 return; 871 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 872 873 if ((dr->region_mask & (1 << bar)) == 0) 874 continue; 875 pci_release_region(pdev, bar); 876 } 877 } 878 879 int 880 linuxkpi_pcim_enable_device(struct pci_dev *pdev) 881 { 882 struct pci_devres *dr; 883 int error; 884 885 /* Here we cannot run through the pdev->managed check. */ 886 dr = lkpi_pci_devres_get_alloc(pdev); 887 if (dr == NULL) 888 return (-ENOMEM); 889 890 /* If resources were enabled before do not do it again. */ 891 if (dr->enable_io) 892 return (0); 893 894 error = pci_enable_device(pdev); 895 if (error == 0) 896 dr->enable_io = true; 897 898 /* This device is not managed. */ 899 pdev->managed = true; 900 901 return (error); 902 } 903 904 static struct pcim_iomap_devres * 905 lkpi_pcim_iomap_devres_find(struct pci_dev *pdev) 906 { 907 struct pcim_iomap_devres *dr; 908 909 dr = lkpi_devres_find(&pdev->dev, lkpi_pcim_iomap_table_release, 910 NULL, NULL); 911 if (dr == NULL) { 912 dr = lkpi_devres_alloc(lkpi_pcim_iomap_table_release, 913 sizeof(*dr), GFP_KERNEL | __GFP_ZERO); 914 if (dr != NULL) 915 lkpi_devres_add(&pdev->dev, dr); 916 } 917 918 if (dr == NULL) 919 device_printf(pdev->dev.bsddev, "%s: NULL\n", __func__); 920 921 return (dr); 922 } 923 924 void __iomem ** 925 linuxkpi_pcim_iomap_table(struct pci_dev *pdev) 926 { 927 struct pcim_iomap_devres *dr; 928 929 dr = lkpi_pcim_iomap_devres_find(pdev); 930 if (dr == NULL) 931 return (NULL); 932 933 /* 934 * If the driver has manually set a flag to be able to request the 935 * resource to use bus_read/write_<n>, return the shadow table. 936 */ 937 if (pdev->want_iomap_res) 938 return ((void **)dr->res_table); 939 940 /* This is the Linux default. */ 941 return (dr->mmio_table); 942 } 943 944 static struct resource * 945 _lkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen __unused) 946 { 947 struct pci_mmio_region *mmio, *p; 948 int type; 949 950 if (!lkpi_pci_bar_id_valid(bar)) 951 return (NULL); 952 953 type = pci_resource_type(pdev, bar); 954 if (type < 0) { 955 device_printf(pdev->dev.bsddev, "%s: bar %d type %d\n", 956 __func__, bar, type); 957 return (NULL); 958 } 959 960 /* 961 * Check for duplicate mappings. 962 * This can happen if a driver calls pci_request_region() first. 963 */ 964 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 965 if (mmio->type == type && mmio->rid == PCIR_BAR(bar)) { 966 return (mmio->res); 967 } 968 } 969 970 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO); 971 mmio->rid = PCIR_BAR(bar); 972 mmio->type = type; 973 mmio->res = bus_alloc_resource_any(pdev->dev.bsddev, mmio->type, 974 &mmio->rid, RF_ACTIVE|RF_SHAREABLE); 975 if (mmio->res == NULL) { 976 device_printf(pdev->dev.bsddev, "%s: failed to alloc " 977 "bar %d type %d rid %d\n", 978 __func__, bar, type, PCIR_BAR(bar)); 979 free(mmio, M_DEVBUF); 980 return (NULL); 981 } 982 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next); 983 984 return (mmio->res); 985 } 986 987 void * 988 linuxkpi_pci_iomap_range(struct pci_dev *pdev, int bar, 989 unsigned long off, unsigned long maxlen) 990 { 991 struct resource *res; 992 993 if (!lkpi_pci_bar_id_valid(bar)) 994 return (NULL); 995 996 res = _lkpi_pci_iomap(pdev, bar, maxlen); 997 if (res == NULL) 998 return (NULL); 999 /* This is a FreeBSD extension so we can use bus_*(). */ 1000 if (pdev->want_iomap_res) 1001 return (res); 1002 MPASS(off < rman_get_size(res)); 1003 return ((void *)(rman_get_bushandle(res) + off)); 1004 } 1005 1006 void * 1007 linuxkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen) 1008 { 1009 if (!lkpi_pci_bar_id_valid(bar)) 1010 return (NULL); 1011 1012 return (linuxkpi_pci_iomap_range(pdev, bar, 0, maxlen)); 1013 } 1014 1015 void * 1016 linuxkpi_pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen) 1017 { 1018 struct pcim_iomap_devres *dr; 1019 void *res; 1020 1021 if (!lkpi_pci_bar_id_valid(bar)) 1022 return (NULL); 1023 1024 dr = lkpi_pcim_iomap_devres_find(pdev); 1025 if (dr == NULL) 1026 return (NULL); 1027 1028 if (dr->res_table[bar] != NULL) 1029 return (dr->res_table[bar]); 1030 1031 res = linuxkpi_pci_iomap(pdev, bar, maxlen); 1032 if (res == NULL) { 1033 /* 1034 * Do not free the devres in case there were 1035 * other valid mappings before already. 1036 */ 1037 return (NULL); 1038 } 1039 lkpi_set_pcim_iomap_devres(dr, bar, res); 1040 1041 return (res); 1042 } 1043 1044 void 1045 linuxkpi_pci_iounmap(struct pci_dev *pdev, void *res) 1046 { 1047 struct pci_mmio_region *mmio, *p; 1048 bus_space_handle_t bh = (bus_space_handle_t)res; 1049 1050 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 1051 if (pdev->want_iomap_res) { 1052 if (res != mmio->res) 1053 continue; 1054 } else { 1055 if (bh < rman_get_bushandle(mmio->res) || 1056 bh >= rman_get_bushandle(mmio->res) + 1057 rman_get_size(mmio->res)) 1058 continue; 1059 } 1060 bus_release_resource(pdev->dev.bsddev, 1061 mmio->type, mmio->rid, mmio->res); 1062 TAILQ_REMOVE(&pdev->mmio, mmio, next); 1063 free(mmio, M_DEVBUF); 1064 return; 1065 } 1066 } 1067 1068 int 1069 linuxkpi_pcim_iomap_regions(struct pci_dev *pdev, uint32_t mask, const char *name) 1070 { 1071 struct pcim_iomap_devres *dr; 1072 void *res; 1073 uint32_t mappings; 1074 int bar; 1075 1076 dr = lkpi_pcim_iomap_devres_find(pdev); 1077 if (dr == NULL) 1078 return (-ENOMEM); 1079 1080 /* Now iomap all the requested (by "mask") ones. */ 1081 for (bar = mappings = 0; mappings != mask; bar++) { 1082 if ((mask & (1 << bar)) == 0) 1083 continue; 1084 1085 /* Request double is not allowed. */ 1086 if (dr->mmio_table[bar] != NULL) { 1087 device_printf(pdev->dev.bsddev, "%s: bar %d %p\n", 1088 __func__, bar, dr->mmio_table[bar]); 1089 goto err; 1090 } 1091 1092 res = _lkpi_pci_iomap(pdev, bar, 0); 1093 if (res == NULL) 1094 goto err; 1095 lkpi_set_pcim_iomap_devres(dr, bar, res); 1096 1097 mappings |= (1 << bar); 1098 } 1099 1100 return (0); 1101 err: 1102 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 1103 if ((mappings & (1 << bar)) != 0) { 1104 res = dr->mmio_table[bar]; 1105 if (res == NULL) 1106 continue; 1107 pci_iounmap(pdev, res); 1108 } 1109 } 1110 1111 return (-EINVAL); 1112 } 1113 1114 static void 1115 lkpi_pcim_iomap_table_release(struct device *dev, void *p) 1116 { 1117 struct pcim_iomap_devres *dr; 1118 struct pci_dev *pdev; 1119 int bar; 1120 1121 dr = p; 1122 pdev = to_pci_dev(dev); 1123 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 1124 1125 if (dr->mmio_table[bar] == NULL) 1126 continue; 1127 1128 pci_iounmap(pdev, dr->mmio_table[bar]); 1129 } 1130 } 1131 1132 static int 1133 linux_pci_suspend(device_t dev) 1134 { 1135 const struct dev_pm_ops *pmops; 1136 struct pm_message pm = { }; 1137 struct pci_dev *pdev; 1138 int error; 1139 1140 error = 0; 1141 linux_set_current(curthread); 1142 pdev = device_get_softc(dev); 1143 pmops = pdev->pdrv->driver.pm; 1144 1145 if (pdev->pdrv->suspend != NULL) 1146 error = -pdev->pdrv->suspend(pdev, pm); 1147 else if (pmops != NULL && pmops->suspend != NULL) { 1148 error = -pmops->suspend(&pdev->dev); 1149 if (error == 0 && pmops->suspend_late != NULL) 1150 error = -pmops->suspend_late(&pdev->dev); 1151 if (error == 0 && pmops->suspend_noirq != NULL) 1152 error = -pmops->suspend_noirq(&pdev->dev); 1153 } 1154 return (error); 1155 } 1156 1157 static int 1158 linux_pci_resume(device_t dev) 1159 { 1160 const struct dev_pm_ops *pmops; 1161 struct pci_dev *pdev; 1162 int error; 1163 1164 error = 0; 1165 linux_set_current(curthread); 1166 pdev = device_get_softc(dev); 1167 pmops = pdev->pdrv->driver.pm; 1168 1169 if (pdev->pdrv->resume != NULL) 1170 error = -pdev->pdrv->resume(pdev); 1171 else if (pmops != NULL && pmops->resume != NULL) { 1172 if (pmops->resume_early != NULL) 1173 error = -pmops->resume_early(&pdev->dev); 1174 if (error == 0 && pmops->resume != NULL) 1175 error = -pmops->resume(&pdev->dev); 1176 } 1177 return (error); 1178 } 1179 1180 static int 1181 linux_pci_shutdown(device_t dev) 1182 { 1183 struct pci_dev *pdev; 1184 1185 linux_set_current(curthread); 1186 pdev = device_get_softc(dev); 1187 if (pdev->pdrv->shutdown != NULL) 1188 pdev->pdrv->shutdown(pdev); 1189 return (0); 1190 } 1191 1192 static int 1193 linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config) 1194 { 1195 struct pci_dev *pdev; 1196 int error; 1197 1198 linux_set_current(curthread); 1199 pdev = device_get_softc(dev); 1200 if (pdev->pdrv->bsd_iov_init != NULL) 1201 error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config); 1202 else 1203 error = EINVAL; 1204 return (error); 1205 } 1206 1207 static void 1208 linux_pci_iov_uninit(device_t dev) 1209 { 1210 struct pci_dev *pdev; 1211 1212 linux_set_current(curthread); 1213 pdev = device_get_softc(dev); 1214 if (pdev->pdrv->bsd_iov_uninit != NULL) 1215 pdev->pdrv->bsd_iov_uninit(dev); 1216 } 1217 1218 static int 1219 linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config) 1220 { 1221 struct pci_dev *pdev; 1222 int error; 1223 1224 linux_set_current(curthread); 1225 pdev = device_get_softc(dev); 1226 if (pdev->pdrv->bsd_iov_add_vf != NULL) 1227 error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config); 1228 else 1229 error = EINVAL; 1230 return (error); 1231 } 1232 1233 static int 1234 _linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc) 1235 { 1236 int error; 1237 1238 linux_set_current(curthread); 1239 spin_lock(&pci_lock); 1240 list_add(&pdrv->node, &pci_drivers); 1241 spin_unlock(&pci_lock); 1242 if (pdrv->bsddriver.name == NULL) 1243 pdrv->bsddriver.name = pdrv->name; 1244 pdrv->bsddriver.methods = pci_methods; 1245 pdrv->bsddriver.size = sizeof(struct pci_dev); 1246 1247 bus_topo_lock(); 1248 error = devclass_add_driver(dc, &pdrv->bsddriver, 1249 BUS_PASS_DEFAULT, &pdrv->bsdclass); 1250 bus_topo_unlock(); 1251 return (-error); 1252 } 1253 1254 int 1255 linux_pci_register_driver(struct pci_driver *pdrv) 1256 { 1257 devclass_t dc; 1258 1259 pdrv->isdrm = strcmp(pdrv->name, "drmn") == 0; 1260 dc = pdrv->isdrm ? devclass_create("vgapci") : devclass_find("pci"); 1261 if (dc == NULL) 1262 return (-ENXIO); 1263 return (_linux_pci_register_driver(pdrv, dc)); 1264 } 1265 1266 static struct resource_list_entry * 1267 lkpi_pci_get_bar(struct pci_dev *pdev, int bar, bool reserve) 1268 { 1269 int type; 1270 1271 type = pci_resource_type(pdev, bar); 1272 if (type < 0) 1273 return (NULL); 1274 bar = PCIR_BAR(bar); 1275 return (linux_pci_get_rle(pdev, type, bar, reserve)); 1276 } 1277 1278 struct device * 1279 lkpi_pci_find_irq_dev(unsigned int irq) 1280 { 1281 struct pci_dev *pdev; 1282 struct device *found; 1283 1284 found = NULL; 1285 spin_lock(&pci_lock); 1286 list_for_each_entry(pdev, &pci_devices, links) { 1287 if (irq == pdev->dev.irq || 1288 (irq >= pdev->dev.irq_start && irq < pdev->dev.irq_end)) { 1289 found = &pdev->dev; 1290 break; 1291 } 1292 } 1293 spin_unlock(&pci_lock); 1294 return (found); 1295 } 1296 1297 unsigned long 1298 pci_resource_start(struct pci_dev *pdev, int bar) 1299 { 1300 struct resource_list_entry *rle; 1301 rman_res_t newstart; 1302 device_t dev; 1303 int error; 1304 1305 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL) 1306 return (0); 1307 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? 1308 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; 1309 error = bus_translate_resource(dev, rle->type, rle->start, &newstart); 1310 if (error != 0) { 1311 device_printf(pdev->dev.bsddev, 1312 "translate of %#jx failed: %d\n", 1313 (uintmax_t)rle->start, error); 1314 return (0); 1315 } 1316 return (newstart); 1317 } 1318 1319 unsigned long 1320 pci_resource_len(struct pci_dev *pdev, int bar) 1321 { 1322 struct resource_list_entry *rle; 1323 1324 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL) 1325 return (0); 1326 return (rle->count); 1327 } 1328 1329 static int 1330 lkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name, 1331 bool managed) 1332 { 1333 struct resource *res; 1334 struct pci_devres *dr; 1335 struct pci_mmio_region *mmio; 1336 int rid; 1337 int type; 1338 1339 if (!lkpi_pci_bar_id_valid(bar)) 1340 return (-EINVAL); 1341 1342 type = pci_resource_type(pdev, bar); 1343 if (type < 0) 1344 return (0); 1345 1346 rid = PCIR_BAR(bar); 1347 res = bus_alloc_resource_any(pdev->dev.bsddev, type, &rid, 1348 RF_ACTIVE|RF_SHAREABLE); 1349 if (res == NULL) { 1350 device_printf(pdev->dev.bsddev, "%s: failed to alloc " 1351 "bar %d type %d rid %d\n", 1352 __func__, bar, type, PCIR_BAR(bar)); 1353 return (-EBUSY); 1354 } 1355 1356 /* 1357 * It seems there is an implicit devres tracking on these if the device 1358 * is managed (lkpi_pci_devres_find() case); otherwise the resources are 1359 * not automatically freed on FreeBSD/LinuxKPI though they should be/are 1360 * expected to be by Linux drivers. 1361 * Otherwise if we are called from a pcim-function with the managed 1362 * argument set, we need to track devres independent of pdev->managed. 1363 */ 1364 if (managed) 1365 dr = lkpi_pci_devres_get_alloc(pdev); 1366 else 1367 dr = lkpi_pci_devres_find(pdev); 1368 if (dr != NULL) { 1369 dr->region_mask |= (1 << bar); 1370 dr->region_table[bar] = res; 1371 } 1372 1373 /* Even if the device is not managed we need to track it for iomap. */ 1374 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO); 1375 mmio->rid = PCIR_BAR(bar); 1376 mmio->type = type; 1377 mmio->res = res; 1378 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next); 1379 1380 return (0); 1381 } 1382 1383 int 1384 linuxkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) 1385 { 1386 return (lkpi_pci_request_region(pdev, bar, res_name, false)); 1387 } 1388 1389 int 1390 linuxkpi_pci_request_regions(struct pci_dev *pdev, const char *res_name) 1391 { 1392 int error; 1393 int i; 1394 1395 for (i = 0; i <= PCIR_MAX_BAR_0; i++) { 1396 error = pci_request_region(pdev, i, res_name); 1397 if (error && error != -EBUSY) { 1398 pci_release_regions(pdev); 1399 return (error); 1400 } 1401 } 1402 return (0); 1403 } 1404 1405 int 1406 linuxkpi_pcim_request_all_regions(struct pci_dev *pdev, const char *res_name) 1407 { 1408 int bar, error; 1409 1410 for (bar = 0; bar <= PCIR_MAX_BAR_0; bar++) { 1411 error = lkpi_pci_request_region(pdev, bar, res_name, true); 1412 if (error != 0 && error != -EBUSY) { 1413 device_printf(pdev->dev.bsddev, "%s: bar %d res_name '%s': " 1414 "lkpi_pci_request_region returned %d\n", __func__, 1415 bar, res_name, error); 1416 pci_release_regions(pdev); 1417 return (error); 1418 } 1419 } 1420 return (0); 1421 } 1422 1423 void 1424 linuxkpi_pci_release_region(struct pci_dev *pdev, int bar) 1425 { 1426 struct resource_list_entry *rle; 1427 struct pci_devres *dr; 1428 struct pci_mmio_region *mmio, *p; 1429 1430 if ((rle = lkpi_pci_get_bar(pdev, bar, false)) == NULL) 1431 return; 1432 1433 /* 1434 * As we implicitly track the requests we also need to clear them on 1435 * release. Do clear before resource release. 1436 */ 1437 dr = lkpi_pci_devres_find(pdev); 1438 if (dr != NULL) { 1439 KASSERT(dr->region_table[bar] == rle->res, ("%s: pdev %p bar %d" 1440 " region_table res %p != rel->res %p\n", __func__, pdev, 1441 bar, dr->region_table[bar], rle->res)); 1442 dr->region_table[bar] = NULL; 1443 dr->region_mask &= ~(1 << bar); 1444 } 1445 1446 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 1447 if (rle->res != (void *)rman_get_bushandle(mmio->res)) 1448 continue; 1449 TAILQ_REMOVE(&pdev->mmio, mmio, next); 1450 free(mmio, M_DEVBUF); 1451 } 1452 1453 bus_release_resource(pdev->dev.bsddev, rle->type, rle->rid, rle->res); 1454 } 1455 1456 void 1457 linuxkpi_pci_release_regions(struct pci_dev *pdev) 1458 { 1459 int i; 1460 1461 for (i = 0; i <= PCIR_MAX_BAR_0; i++) 1462 pci_release_region(pdev, i); 1463 } 1464 1465 int 1466 linux_pci_register_drm_driver(struct pci_driver *pdrv) 1467 { 1468 devclass_t dc; 1469 1470 dc = devclass_create("vgapci"); 1471 if (dc == NULL) 1472 return (-ENXIO); 1473 pdrv->isdrm = true; 1474 pdrv->name = "drmn"; 1475 return (_linux_pci_register_driver(pdrv, dc)); 1476 } 1477 1478 void 1479 linux_pci_unregister_driver(struct pci_driver *pdrv) 1480 { 1481 devclass_t bus; 1482 1483 bus = devclass_find(pdrv->isdrm ? "vgapci" : "pci"); 1484 1485 spin_lock(&pci_lock); 1486 list_del(&pdrv->node); 1487 spin_unlock(&pci_lock); 1488 bus_topo_lock(); 1489 if (bus != NULL) 1490 devclass_delete_driver(bus, &pdrv->bsddriver); 1491 bus_topo_unlock(); 1492 } 1493 1494 void 1495 linux_pci_unregister_drm_driver(struct pci_driver *pdrv) 1496 { 1497 devclass_t bus; 1498 1499 bus = devclass_find("vgapci"); 1500 1501 spin_lock(&pci_lock); 1502 list_del(&pdrv->node); 1503 spin_unlock(&pci_lock); 1504 bus_topo_lock(); 1505 if (bus != NULL) 1506 devclass_delete_driver(bus, &pdrv->bsddriver); 1507 bus_topo_unlock(); 1508 } 1509 1510 int 1511 linuxkpi_pci_enable_msix(struct pci_dev *pdev, struct msix_entry *entries, 1512 int nreq) 1513 { 1514 struct resource_list_entry *rle; 1515 int error; 1516 int avail; 1517 int i; 1518 1519 avail = pci_msix_count(pdev->dev.bsddev); 1520 if (avail < nreq) { 1521 if (avail == 0) 1522 return -EINVAL; 1523 return avail; 1524 } 1525 avail = nreq; 1526 if ((error = -pci_alloc_msix(pdev->dev.bsddev, &avail)) != 0) 1527 return error; 1528 /* 1529 * Handle case where "pci_alloc_msix()" may allocate less 1530 * interrupts than available and return with no error: 1531 */ 1532 if (avail < nreq) { 1533 pci_release_msi(pdev->dev.bsddev); 1534 return avail; 1535 } 1536 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); 1537 pdev->dev.irq_start = rle->start; 1538 pdev->dev.irq_end = rle->start + avail; 1539 for (i = 0; i < nreq; i++) 1540 entries[i].vector = pdev->dev.irq_start + i; 1541 pdev->msix_enabled = true; 1542 return (0); 1543 } 1544 1545 int 1546 _lkpi_pci_enable_msi_range(struct pci_dev *pdev, int minvec, int maxvec) 1547 { 1548 struct resource_list_entry *rle; 1549 int error; 1550 int nvec; 1551 1552 if (maxvec < minvec) 1553 return (-EINVAL); 1554 1555 nvec = pci_msi_count(pdev->dev.bsddev); 1556 if (nvec < 1 || nvec < minvec) 1557 return (-ENOSPC); 1558 1559 nvec = min(nvec, maxvec); 1560 if ((error = -pci_alloc_msi(pdev->dev.bsddev, &nvec)) != 0) 1561 return error; 1562 1563 /* Native PCI might only ever ask for 32 vectors. */ 1564 if (nvec < minvec) { 1565 pci_release_msi(pdev->dev.bsddev); 1566 return (-ENOSPC); 1567 } 1568 1569 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); 1570 pdev->dev.irq_start = rle->start; 1571 pdev->dev.irq_end = rle->start + nvec; 1572 pdev->irq = rle->start; 1573 pdev->msi_enabled = true; 1574 return (0); 1575 } 1576 1577 int 1578 pci_alloc_irq_vectors(struct pci_dev *pdev, int minv, int maxv, 1579 unsigned int flags) 1580 { 1581 int error; 1582 1583 if ((flags & PCI_IRQ_AFFINITY) != 0) { 1584 pr_debug("%s: TODO PCI_IRQ_AFFINITY\n", __func__); 1585 } 1586 if (flags & PCI_IRQ_MSIX) { 1587 struct msix_entry *entries; 1588 int i; 1589 1590 entries = kcalloc(maxv, sizeof(*entries), GFP_KERNEL); 1591 if (entries == NULL) { 1592 error = -ENOMEM; 1593 goto out; 1594 } 1595 for (i = 0; i < maxv; ++i) 1596 entries[i].entry = i; 1597 error = pci_enable_msix(pdev, entries, maxv); 1598 out: 1599 kfree(entries); 1600 if (error == 0 && pdev->msix_enabled) 1601 return (pdev->dev.irq_end - pdev->dev.irq_start); 1602 } 1603 if (flags & PCI_IRQ_MSI) { 1604 if (pci_msi_count(pdev->dev.bsddev) < minv) 1605 return (-ENOSPC); 1606 error = _lkpi_pci_enable_msi_range(pdev, minv, maxv); 1607 if (error == 0 && pdev->msi_enabled) 1608 return (pdev->dev.irq_end - pdev->dev.irq_start); 1609 } 1610 if (flags & PCI_IRQ_INTX) { 1611 if (pdev->irq) 1612 return (1); 1613 } 1614 1615 return (-EINVAL); 1616 } 1617 1618 struct msi_desc * 1619 lkpi_pci_msi_desc_alloc(unsigned int irq) 1620 { 1621 struct device *dev; 1622 struct pci_dev *pdev; 1623 struct msi_desc *desc; 1624 struct pci_devinfo *dinfo; 1625 struct pcicfg_msi *msi; 1626 int vec; 1627 1628 dev = lkpi_pci_find_irq_dev(irq); 1629 if (dev == NULL) 1630 return (NULL); 1631 1632 pdev = to_pci_dev(dev); 1633 1634 if (pdev->msi_desc == NULL) 1635 return (NULL); 1636 1637 if (irq < pdev->dev.irq_start || irq >= pdev->dev.irq_end) 1638 return (NULL); 1639 1640 vec = pdev->dev.irq_start - irq; 1641 1642 if (pdev->msi_desc[vec] != NULL) 1643 return (pdev->msi_desc[vec]); 1644 1645 dinfo = device_get_ivars(dev->bsddev); 1646 msi = &dinfo->cfg.msi; 1647 1648 desc = malloc(sizeof(*desc), M_DEVBUF, M_WAITOK | M_ZERO); 1649 1650 desc->pci.msi_attrib.is_64 = 1651 (msi->msi_ctrl & PCIM_MSICTRL_64BIT) ? true : false; 1652 desc->msg.data = msi->msi_data; 1653 1654 pdev->msi_desc[vec] = desc; 1655 1656 return (desc); 1657 } 1658 1659 bool 1660 pci_device_is_present(struct pci_dev *pdev) 1661 { 1662 device_t dev; 1663 1664 dev = pdev->dev.bsddev; 1665 1666 return (bus_child_present(dev)); 1667 } 1668 1669 void * 1670 linuxkpi_pci_map_rom(struct pci_dev *pdev, size_t *size) 1671 { 1672 device_t dev; 1673 1674 dev = pdev->dev.bsddev; 1675 1676 if (pci_get_class(dev) != PCIC_DISPLAY && 1677 (pci_get_class(dev) != PCIC_OLD || 1678 pci_get_subclass(dev) != PCIS_OLD_VGA)) { 1679 pr_debug("%s: TODO\n", __func__); 1680 return (NULL); 1681 } 1682 1683 return (vga_pci_map_bios(device_get_parent(dev), size)); 1684 } 1685 1686 void 1687 linuxkpi_pci_unmap_rom(struct pci_dev *pdev, void *rom) 1688 { 1689 device_t dev; 1690 1691 dev = pdev->dev.bsddev; 1692 1693 vga_pci_unmap_bios(device_get_parent(dev), rom); 1694 } 1695 1696 CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t)); 1697 1698 struct linux_dma_obj { 1699 void *vaddr; 1700 uint64_t dma_addr; 1701 bus_dmamap_t dmamap; 1702 bus_dma_tag_t dmat; 1703 }; 1704 1705 static uma_zone_t linux_dma_trie_zone; 1706 static uma_zone_t linux_dma_obj_zone; 1707 1708 static void 1709 linux_dma_init(void *arg) 1710 { 1711 1712 linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie", 1713 pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL, 1714 UMA_ALIGN_PTR, 0); 1715 linux_dma_obj_zone = uma_zcreate("linux_dma_object", 1716 sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL, 1717 UMA_ALIGN_PTR, 0); 1718 lkpi_pci_nseg1_fail = counter_u64_alloc(M_WAITOK); 1719 } 1720 SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL); 1721 1722 static void 1723 linux_dma_uninit(void *arg) 1724 { 1725 1726 counter_u64_free(lkpi_pci_nseg1_fail); 1727 uma_zdestroy(linux_dma_obj_zone); 1728 uma_zdestroy(linux_dma_trie_zone); 1729 } 1730 SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL); 1731 1732 static void * 1733 linux_dma_trie_alloc(struct pctrie *ptree) 1734 { 1735 1736 return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT)); 1737 } 1738 1739 static void 1740 linux_dma_trie_free(struct pctrie *ptree, void *node) 1741 { 1742 1743 uma_zfree(linux_dma_trie_zone, node); 1744 } 1745 1746 PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, 1747 linux_dma_trie_free); 1748 1749 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 1750 static dma_addr_t 1751 linux_dma_map_phys_common(struct device *dev, vm_paddr_t phys, size_t len, 1752 bus_dma_tag_t dmat) 1753 { 1754 struct linux_dma_priv *priv; 1755 struct linux_dma_obj *obj; 1756 int error, nseg; 1757 bus_dma_segment_t seg; 1758 1759 priv = dev->dma_priv; 1760 1761 /* 1762 * If the resultant mapping will be entirely 1:1 with the 1763 * physical address, short-circuit the remainder of the 1764 * bus_dma API. This avoids tracking collisions in the pctrie 1765 * with the additional benefit of reducing overhead. 1766 */ 1767 if (bus_dma_id_mapped(dmat, phys, len)) 1768 return (phys); 1769 1770 obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT); 1771 if (obj == NULL) { 1772 return (0); 1773 } 1774 obj->dmat = dmat; 1775 1776 DMA_PRIV_LOCK(priv); 1777 if (bus_dmamap_create(obj->dmat, 0, &obj->dmamap) != 0) { 1778 DMA_PRIV_UNLOCK(priv); 1779 uma_zfree(linux_dma_obj_zone, obj); 1780 return (0); 1781 } 1782 1783 nseg = -1; 1784 error = _bus_dmamap_load_phys(obj->dmat, obj->dmamap, phys, len, 1785 BUS_DMA_NOWAIT, &seg, &nseg); 1786 if (error != 0) { 1787 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1788 DMA_PRIV_UNLOCK(priv); 1789 uma_zfree(linux_dma_obj_zone, obj); 1790 counter_u64_add(lkpi_pci_nseg1_fail, 1); 1791 if (linuxkpi_debug) { 1792 device_printf(dev->bsddev, "%s: _bus_dmamap_load_phys " 1793 "error %d, phys %#018jx len %zu\n", __func__, 1794 error, (uintmax_t)phys, len); 1795 dump_stack(); 1796 } 1797 return (0); 1798 } 1799 1800 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 1801 obj->dma_addr = seg.ds_addr; 1802 1803 error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); 1804 if (error != 0) { 1805 bus_dmamap_unload(obj->dmat, obj->dmamap); 1806 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1807 DMA_PRIV_UNLOCK(priv); 1808 uma_zfree(linux_dma_obj_zone, obj); 1809 return (0); 1810 } 1811 DMA_PRIV_UNLOCK(priv); 1812 return (obj->dma_addr); 1813 } 1814 #else 1815 static dma_addr_t 1816 linux_dma_map_phys_common(struct device *dev __unused, vm_paddr_t phys, 1817 size_t len __unused, bus_dma_tag_t dmat __unused) 1818 { 1819 return (phys); 1820 } 1821 #endif 1822 1823 dma_addr_t 1824 lkpi_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len, 1825 enum dma_data_direction direction, unsigned long attrs) 1826 { 1827 struct linux_dma_priv *priv; 1828 dma_addr_t dma; 1829 1830 priv = dev->dma_priv; 1831 dma = linux_dma_map_phys_common(dev, phys, len, priv->dmat); 1832 if (dma_mapping_error(dev, dma)) 1833 return (dma); 1834 1835 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 1836 dma_sync_single_for_device(dev, dma, len, direction); 1837 1838 return (dma); 1839 } 1840 1841 /* For backward compat only so we can MFC this. Remove before 15. */ 1842 dma_addr_t 1843 linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) 1844 { 1845 return (lkpi_dma_map_phys(dev, phys, len, DMA_NONE, 0)); 1846 } 1847 1848 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 1849 void 1850 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, 1851 enum dma_data_direction direction, unsigned long attrs) 1852 { 1853 struct linux_dma_priv *priv; 1854 struct linux_dma_obj *obj; 1855 1856 priv = dev->dma_priv; 1857 1858 if (pctrie_is_empty(&priv->ptree)) 1859 return; 1860 1861 DMA_PRIV_LOCK(priv); 1862 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); 1863 if (obj == NULL) { 1864 DMA_PRIV_UNLOCK(priv); 1865 return; 1866 } 1867 LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); 1868 1869 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 1870 goto skip_sync; 1871 1872 /* dma_sync_single_for_cpu() unrolled to avoid lock recursicn. */ 1873 switch (direction) { 1874 case DMA_BIDIRECTIONAL: 1875 bus_dmamap_sync(obj->dmat, obj->dmamap, 1876 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1877 break; 1878 case DMA_TO_DEVICE: 1879 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_POSTWRITE); 1880 break; 1881 case DMA_FROM_DEVICE: 1882 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_POSTREAD); 1883 break; 1884 default: 1885 break; 1886 } 1887 1888 skip_sync: 1889 bus_dmamap_unload(obj->dmat, obj->dmamap); 1890 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1891 DMA_PRIV_UNLOCK(priv); 1892 1893 uma_zfree(linux_dma_obj_zone, obj); 1894 } 1895 #else 1896 void 1897 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, 1898 enum dma_data_direction direction, unsigned long attrs) 1899 { 1900 } 1901 #endif 1902 1903 /* For backward compat only so we can MFC this. Remove before 15. */ 1904 void 1905 linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) 1906 { 1907 lkpi_dma_unmap(dev, dma_addr, len, DMA_NONE, 0); 1908 } 1909 1910 void * 1911 linux_dma_alloc_coherent(struct device *dev, size_t size, 1912 dma_addr_t *dma_handle, gfp_t flag) 1913 { 1914 struct linux_dma_priv *priv; 1915 vm_paddr_t high; 1916 size_t align; 1917 void *mem; 1918 1919 if (dev == NULL || dev->dma_priv == NULL) { 1920 *dma_handle = 0; 1921 return (NULL); 1922 } 1923 priv = dev->dma_priv; 1924 if (priv->dma_coherent_mask) 1925 high = priv->dma_coherent_mask; 1926 else 1927 /* Coherent is lower 32bit only by default in Linux. */ 1928 high = BUS_SPACE_MAXADDR_32BIT; 1929 align = PAGE_SIZE << get_order(size); 1930 /* Always zero the allocation. */ 1931 flag |= M_ZERO; 1932 mem = kmem_alloc_contig(size, flag & GFP_NATIVE_MASK, 0, high, 1933 align, 0, VM_MEMATTR_DEFAULT); 1934 if (mem != NULL) { 1935 *dma_handle = linux_dma_map_phys_common(dev, vtophys(mem), size, 1936 priv->dmat_coherent); 1937 if (*dma_handle == 0) { 1938 kmem_free(mem, size); 1939 mem = NULL; 1940 } 1941 } else { 1942 *dma_handle = 0; 1943 } 1944 return (mem); 1945 } 1946 1947 struct lkpi_devres_dmam_coherent { 1948 size_t size; 1949 dma_addr_t handle; 1950 void *mem; 1951 }; 1952 1953 static void 1954 lkpi_dmam_free_coherent(struct device *dev, void *p) 1955 { 1956 struct lkpi_devres_dmam_coherent *dr; 1957 1958 dr = p; 1959 dma_free_coherent(dev, dr->size, dr->mem, dr->handle); 1960 } 1961 1962 static int 1963 lkpi_dmam_coherent_match(struct device *dev, void *dr, void *mp) 1964 { 1965 struct lkpi_devres_dmam_coherent *a, *b; 1966 1967 a = dr; 1968 b = mp; 1969 1970 if (a->mem != b->mem) 1971 return (0); 1972 if (a->size != b->size || a->handle != b->handle) 1973 dev_WARN(dev, "for mem %p: size %zu != %zu || handle %#jx != %#jx\n", 1974 a->mem, a->size, b->size, 1975 (uintmax_t)a->handle, (uintmax_t)b->handle); 1976 return (1); 1977 } 1978 1979 void 1980 linuxkpi_dmam_free_coherent(struct device *dev, size_t size, 1981 void *addr, dma_addr_t dma_handle) 1982 { 1983 struct lkpi_devres_dmam_coherent match = { 1984 .size = size, 1985 .handle = dma_handle, 1986 .mem = addr 1987 }; 1988 int error; 1989 1990 error = devres_destroy(dev, lkpi_dmam_free_coherent, 1991 lkpi_dmam_coherent_match, &match); 1992 if (error != 0) 1993 dev_WARN(dev, "devres_destroy returned %d, size %zu addr %p " 1994 "dma_handle %#jx\n", error, size, addr, (uintmax_t)dma_handle); 1995 dma_free_coherent(dev, size, addr, dma_handle); 1996 } 1997 1998 void * 1999 linuxkpi_dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, 2000 gfp_t flag) 2001 { 2002 struct lkpi_devres_dmam_coherent *dr; 2003 2004 dr = lkpi_devres_alloc(lkpi_dmam_free_coherent, 2005 sizeof(*dr), GFP_KERNEL | __GFP_ZERO); 2006 2007 if (dr == NULL) 2008 return (NULL); 2009 2010 dr->size = size; 2011 dr->mem = linux_dma_alloc_coherent(dev, size, dma_handle, flag); 2012 dr->handle = *dma_handle; 2013 if (dr->mem == NULL) { 2014 lkpi_devres_free(dr); 2015 return (NULL); 2016 } 2017 2018 lkpi_devres_add(dev, dr); 2019 return (dr->mem); 2020 } 2021 2022 void * 2023 linuxkpi_dma_alloc_noncoherent(struct device *dev, size_t size, dma_addr_t *dma_handle, 2024 enum dma_data_direction direction, gfp_t gfp) 2025 { 2026 struct linux_dma_priv *priv; 2027 size_t align; 2028 void *mem; 2029 2030 size = PAGE_ALIGN(size); 2031 align = PAGE_SIZE << get_order(size); 2032 mem = kmem_alloc_contig(size, gfp & GFP_NATIVE_MASK, 0, BUS_SPACE_MAXADDR, 2033 align, 0, VM_MEMATTR_DEFAULT); 2034 if (mem == NULL) { 2035 *dma_handle = 0; 2036 return (NULL); 2037 } 2038 2039 priv = dev->dma_priv; 2040 *dma_handle = linux_dma_map_phys_common(dev, vtophys(mem), size, priv->dmat); 2041 if (*dma_handle == 0) { 2042 kmem_free(mem, size); 2043 mem = NULL; 2044 } 2045 return (mem); 2046 } 2047 2048 void 2049 linuxkpi_dma_free_noncoherent(struct device *dev, size_t size, void *vaddr, 2050 dma_addr_t dma_handle, enum dma_data_direction direction) 2051 { 2052 lkpi_dma_unmap(dev, dma_handle, size, direction, 0); 2053 kmem_free(vaddr, size); 2054 } 2055 2056 void * 2057 linuxkpi_dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 2058 gfp_t gfp, unsigned long attrs) 2059 { 2060 return (linuxkpi_dma_alloc_noncoherent(dev, size, dma_handle, 2061 DMA_BIDIRECTIONAL, gfp)); 2062 } 2063 2064 void 2065 linuxkpi_dma_free_attrs(struct device *dev, size_t size, void *vaddr, 2066 dma_addr_t dma_handle, unsigned long attrs) 2067 { 2068 linuxkpi_dma_free_noncoherent(dev, size, vaddr, dma_handle, 2069 DMA_BIDIRECTIONAL); 2070 } 2071 2072 void 2073 linuxkpi_dma_sync(struct device *dev, dma_addr_t dma_addr, size_t size, 2074 bus_dmasync_op_t op) 2075 { 2076 struct linux_dma_priv *priv; 2077 struct linux_dma_obj *obj; 2078 2079 priv = dev->dma_priv; 2080 2081 if (pctrie_is_empty(&priv->ptree)) 2082 return; 2083 2084 DMA_PRIV_LOCK(priv); 2085 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); 2086 if (obj == NULL) { 2087 DMA_PRIV_UNLOCK(priv); 2088 return; 2089 } 2090 2091 bus_dmamap_sync(obj->dmat, obj->dmamap, op); 2092 DMA_PRIV_UNLOCK(priv); 2093 } 2094 2095 void 2096 lkpi_dma_sync_sg(struct device *dev, struct scatterlist *sgl, bus_dmasync_op_t op) 2097 { 2098 struct linux_dma_priv *priv; 2099 2100 priv = dev->dma_priv; 2101 DMA_PRIV_LOCK(priv); 2102 bus_dmamap_sync(priv->dmat, sgl->dma_map, op); 2103 DMA_PRIV_UNLOCK(priv); 2104 } 2105 2106 int 2107 linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, 2108 enum dma_data_direction direction, unsigned long attrs) 2109 { 2110 struct linux_dma_priv *priv; 2111 struct scatterlist *sg; 2112 int i, nseg; 2113 bus_dma_segment_t seg; 2114 2115 priv = dev->dma_priv; 2116 2117 DMA_PRIV_LOCK(priv); 2118 2119 /* create common DMA map in the first S/G entry */ 2120 if (bus_dmamap_create(priv->dmat, 0, &sgl->dma_map) != 0) { 2121 DMA_PRIV_UNLOCK(priv); 2122 return (0); 2123 } 2124 2125 /* load all S/G list entries */ 2126 for_each_sg(sgl, sg, nents, i) { 2127 nseg = -1; 2128 if (_bus_dmamap_load_phys(priv->dmat, sgl->dma_map, 2129 sg_phys(sg), sg->length, BUS_DMA_NOWAIT, 2130 &seg, &nseg) != 0) { 2131 bus_dmamap_unload(priv->dmat, sgl->dma_map); 2132 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 2133 DMA_PRIV_UNLOCK(priv); 2134 return (0); 2135 } 2136 KASSERT(nseg == 0, 2137 ("More than one segment (nseg=%d)", nseg + 1)); 2138 2139 sg_dma_address(sg) = seg.ds_addr; 2140 sg->dma_length = sg->length; 2141 } 2142 2143 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 2144 goto skip_sync; 2145 2146 switch (direction) { 2147 case DMA_BIDIRECTIONAL: 2148 bus_dmamap_sync(priv->dmat, sgl->dma_map, 2149 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2150 break; 2151 case DMA_TO_DEVICE: 2152 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE); 2153 break; 2154 case DMA_FROM_DEVICE: 2155 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREREAD); 2156 break; 2157 default: 2158 break; 2159 } 2160 skip_sync: 2161 2162 DMA_PRIV_UNLOCK(priv); 2163 2164 return (nents); 2165 } 2166 2167 void 2168 linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, 2169 int nents __unused, enum dma_data_direction direction, 2170 unsigned long attrs) 2171 { 2172 struct linux_dma_priv *priv; 2173 2174 priv = dev->dma_priv; 2175 2176 DMA_PRIV_LOCK(priv); 2177 2178 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 2179 goto skip_sync; 2180 2181 switch (direction) { 2182 case DMA_BIDIRECTIONAL: 2183 bus_dmamap_sync(priv->dmat, sgl->dma_map, 2184 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2185 break; 2186 case DMA_TO_DEVICE: 2187 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTWRITE); 2188 break; 2189 case DMA_FROM_DEVICE: 2190 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD); 2191 break; 2192 default: 2193 break; 2194 } 2195 skip_sync: 2196 2197 bus_dmamap_unload(priv->dmat, sgl->dma_map); 2198 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 2199 DMA_PRIV_UNLOCK(priv); 2200 } 2201 2202 struct dma_pool { 2203 struct device *pool_device; 2204 uma_zone_t pool_zone; 2205 struct mtx pool_lock; 2206 bus_dma_tag_t pool_dmat; 2207 size_t pool_entry_size; 2208 struct pctrie pool_ptree; 2209 }; 2210 2211 #define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock) 2212 #define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock) 2213 2214 static inline int 2215 dma_pool_obj_ctor(void *mem, int size, void *arg, int flags) 2216 { 2217 struct linux_dma_obj *obj = mem; 2218 struct dma_pool *pool = arg; 2219 int error, nseg; 2220 bus_dma_segment_t seg; 2221 2222 nseg = -1; 2223 DMA_POOL_LOCK(pool); 2224 error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap, 2225 vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT, 2226 &seg, &nseg); 2227 DMA_POOL_UNLOCK(pool); 2228 if (error != 0) { 2229 return (error); 2230 } 2231 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 2232 obj->dma_addr = seg.ds_addr; 2233 2234 return (0); 2235 } 2236 2237 static void 2238 dma_pool_obj_dtor(void *mem, int size, void *arg) 2239 { 2240 struct linux_dma_obj *obj = mem; 2241 struct dma_pool *pool = arg; 2242 2243 DMA_POOL_LOCK(pool); 2244 bus_dmamap_unload(pool->pool_dmat, obj->dmamap); 2245 DMA_POOL_UNLOCK(pool); 2246 } 2247 2248 static int 2249 dma_pool_obj_import(void *arg, void **store, int count, int domain __unused, 2250 int flags) 2251 { 2252 struct dma_pool *pool = arg; 2253 struct linux_dma_obj *obj; 2254 int error, i; 2255 2256 for (i = 0; i < count; i++) { 2257 obj = uma_zalloc(linux_dma_obj_zone, flags); 2258 if (obj == NULL) 2259 break; 2260 2261 error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr, 2262 BUS_DMA_NOWAIT, &obj->dmamap); 2263 if (error!= 0) { 2264 uma_zfree(linux_dma_obj_zone, obj); 2265 break; 2266 } 2267 2268 store[i] = obj; 2269 } 2270 2271 return (i); 2272 } 2273 2274 static void 2275 dma_pool_obj_release(void *arg, void **store, int count) 2276 { 2277 struct dma_pool *pool = arg; 2278 struct linux_dma_obj *obj; 2279 int i; 2280 2281 for (i = 0; i < count; i++) { 2282 obj = store[i]; 2283 bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap); 2284 uma_zfree(linux_dma_obj_zone, obj); 2285 } 2286 } 2287 2288 struct dma_pool * 2289 linux_dma_pool_create(char *name, struct device *dev, size_t size, 2290 size_t align, size_t boundary) 2291 { 2292 struct linux_dma_priv *priv; 2293 struct dma_pool *pool; 2294 2295 priv = dev->dma_priv; 2296 2297 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 2298 pool->pool_device = dev; 2299 pool->pool_entry_size = size; 2300 2301 if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 2302 align, boundary, /* alignment, boundary */ 2303 priv->dma_mask, /* lowaddr */ 2304 BUS_SPACE_MAXADDR, /* highaddr */ 2305 NULL, NULL, /* filtfunc, filtfuncarg */ 2306 size, /* maxsize */ 2307 1, /* nsegments */ 2308 size, /* maxsegsz */ 2309 0, /* flags */ 2310 NULL, NULL, /* lockfunc, lockfuncarg */ 2311 &pool->pool_dmat)) { 2312 kfree(pool); 2313 return (NULL); 2314 } 2315 2316 pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor, 2317 dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import, 2318 dma_pool_obj_release, pool, 0); 2319 2320 mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF); 2321 pctrie_init(&pool->pool_ptree); 2322 2323 return (pool); 2324 } 2325 2326 void 2327 linux_dma_pool_destroy(struct dma_pool *pool) 2328 { 2329 2330 uma_zdestroy(pool->pool_zone); 2331 bus_dma_tag_destroy(pool->pool_dmat); 2332 mtx_destroy(&pool->pool_lock); 2333 kfree(pool); 2334 } 2335 2336 void 2337 lkpi_dmam_pool_destroy(struct device *dev, void *p) 2338 { 2339 struct dma_pool *pool; 2340 2341 pool = *(struct dma_pool **)p; 2342 LINUX_DMA_PCTRIE_RECLAIM(&pool->pool_ptree); 2343 linux_dma_pool_destroy(pool); 2344 } 2345 2346 void * 2347 linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, 2348 dma_addr_t *handle) 2349 { 2350 struct linux_dma_obj *obj; 2351 2352 obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags & GFP_NATIVE_MASK); 2353 if (obj == NULL) 2354 return (NULL); 2355 2356 DMA_POOL_LOCK(pool); 2357 if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) { 2358 DMA_POOL_UNLOCK(pool); 2359 uma_zfree_arg(pool->pool_zone, obj, pool); 2360 return (NULL); 2361 } 2362 DMA_POOL_UNLOCK(pool); 2363 2364 *handle = obj->dma_addr; 2365 return (obj->vaddr); 2366 } 2367 2368 void 2369 linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr) 2370 { 2371 struct linux_dma_obj *obj; 2372 2373 DMA_POOL_LOCK(pool); 2374 obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr); 2375 if (obj == NULL) { 2376 DMA_POOL_UNLOCK(pool); 2377 return; 2378 } 2379 LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr); 2380 DMA_POOL_UNLOCK(pool); 2381 2382 uma_zfree_arg(pool->pool_zone, obj, pool); 2383 } 2384 2385 static int 2386 linux_backlight_get_status(device_t dev, struct backlight_props *props) 2387 { 2388 struct pci_dev *pdev; 2389 2390 linux_set_current(curthread); 2391 pdev = device_get_softc(dev); 2392 2393 props->brightness = pdev->dev.bd->props.brightness; 2394 props->brightness = props->brightness * 100 / pdev->dev.bd->props.max_brightness; 2395 props->nlevels = 0; 2396 2397 return (0); 2398 } 2399 2400 static int 2401 linux_backlight_get_info(device_t dev, struct backlight_info *info) 2402 { 2403 struct pci_dev *pdev; 2404 2405 linux_set_current(curthread); 2406 pdev = device_get_softc(dev); 2407 2408 info->type = BACKLIGHT_TYPE_PANEL; 2409 strlcpy(info->name, pdev->dev.bd->name, BACKLIGHTMAXNAMELENGTH); 2410 return (0); 2411 } 2412 2413 static int 2414 linux_backlight_update_status(device_t dev, struct backlight_props *props) 2415 { 2416 struct pci_dev *pdev; 2417 2418 linux_set_current(curthread); 2419 pdev = device_get_softc(dev); 2420 2421 pdev->dev.bd->props.brightness = pdev->dev.bd->props.max_brightness * 2422 props->brightness / 100; 2423 pdev->dev.bd->props.power = props->brightness == 0 ? 2424 4/* FB_BLANK_POWERDOWN */ : 0/* FB_BLANK_UNBLANK */; 2425 return (pdev->dev.bd->ops->update_status(pdev->dev.bd)); 2426 } 2427 2428 struct backlight_device * 2429 linux_backlight_device_register(const char *name, struct device *dev, 2430 void *data, const struct backlight_ops *ops, struct backlight_properties *props) 2431 { 2432 2433 dev->bd = malloc(sizeof(*dev->bd), M_DEVBUF, M_WAITOK | M_ZERO); 2434 dev->bd->ops = ops; 2435 dev->bd->props.type = props->type; 2436 dev->bd->props.max_brightness = props->max_brightness; 2437 dev->bd->props.brightness = props->brightness; 2438 dev->bd->props.power = props->power; 2439 dev->bd->data = data; 2440 dev->bd->dev = dev; 2441 dev->bd->name = strdup(name, M_DEVBUF); 2442 2443 dev->backlight_dev = backlight_register(name, dev->bsddev); 2444 2445 return (dev->bd); 2446 } 2447 2448 void 2449 linux_backlight_device_unregister(struct backlight_device *bd) 2450 { 2451 2452 backlight_destroy(bd->dev->backlight_dev); 2453 free(bd->name, M_DEVBUF); 2454 free(bd, M_DEVBUF); 2455 } 2456