1 /*- 2 * Copyright (c) 2015-2016 Mellanox Technologies, Ltd. 3 * All rights reserved. 4 * Copyright (c) 2020-2026 The FreeBSD Foundation 5 * 6 * Portions of this software were developed by Björn Zeeb 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 /* 32 * We have two ways to create a pci_dev (pdev): 33 * (1) coming from the device_attach DEVMETHOD, and 34 * (2) the other from manual creation via lkpinew_pci_dev(). 35 * 36 * Only devices from (1) end up on our LinuxKPI global pci_devices list. 37 * All others are "place fillers" -- XXX if only "place filler" was always true. 38 */ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/bus.h> 43 #include <sys/malloc.h> 44 #include <sys/kernel.h> 45 #include <sys/sysctl.h> 46 #include <sys/lock.h> 47 #include <sys/mutex.h> 48 #include <sys/fcntl.h> 49 #include <sys/file.h> 50 #include <sys/filio.h> 51 #include <sys/pciio.h> 52 #include <sys/pctrie.h> 53 #include <sys/rman.h> 54 #include <sys/rwlock.h> 55 #include <sys/stdarg.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 60 #include <machine/bus.h> 61 #include <machine/resource.h> 62 63 #include <dev/pci/pcivar.h> 64 #include <dev/pci/pci_private.h> 65 #include <dev/pci/pci_iov.h> 66 #include <dev/backlight/backlight.h> 67 68 #include <linux/kernel.h> 69 #include <linux/kobject.h> 70 #include <linux/device.h> 71 #include <linux/slab.h> 72 #include <linux/module.h> 73 #include <linux/cdev.h> 74 #include <linux/file.h> 75 #include <linux/sysfs.h> 76 #include <linux/mm.h> 77 #include <linux/io.h> 78 #include <linux/vmalloc.h> 79 #define WANT_NATIVE_PCI_GET_SLOT 80 #include <linux/pci.h> 81 #include <linux/compat.h> 82 83 #include <linux/backlight.h> 84 85 #include "backlight_if.h" 86 #include "pcib_if.h" 87 88 /* Undef the linux function macro defined in linux/pci.h */ 89 #undef pci_get_class 90 91 extern int linuxkpi_debug; 92 93 SYSCTL_DECL(_compat_linuxkpi); 94 95 static counter_u64_t lkpi_pci_nseg1_fail; 96 SYSCTL_COUNTER_U64(_compat_linuxkpi, OID_AUTO, lkpi_pci_nseg1_fail, CTLFLAG_RD, 97 &lkpi_pci_nseg1_fail, "Count of busdma mapping failures of single-segment"); 98 99 static device_probe_t linux_pci_probe; 100 static device_attach_t linux_pci_attach; 101 static device_detach_t linux_pci_detach; 102 static device_suspend_t linux_pci_suspend; 103 static device_resume_t linux_pci_resume; 104 static device_shutdown_t linux_pci_shutdown; 105 static pci_iov_init_t linux_pci_iov_init; 106 static pci_iov_uninit_t linux_pci_iov_uninit; 107 static pci_iov_add_vf_t linux_pci_iov_add_vf; 108 static int linux_backlight_get_status(device_t dev, struct backlight_props *props); 109 static int linux_backlight_update_status(device_t dev, struct backlight_props *props); 110 static int linux_backlight_get_info(device_t dev, struct backlight_info *info); 111 static void lkpi_pcim_iomap_table_release(struct device *, void *); 112 static void lkpinew_pci_dev_release(struct device *); 113 114 static device_method_t pci_methods[] = { 115 DEVMETHOD(device_probe, linux_pci_probe), 116 DEVMETHOD(device_attach, linux_pci_attach), 117 DEVMETHOD(device_detach, linux_pci_detach), 118 DEVMETHOD(device_suspend, linux_pci_suspend), 119 DEVMETHOD(device_resume, linux_pci_resume), 120 DEVMETHOD(device_shutdown, linux_pci_shutdown), 121 DEVMETHOD(pci_iov_init, linux_pci_iov_init), 122 DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit), 123 DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf), 124 125 /* Bus interface. */ 126 DEVMETHOD(bus_add_child, bus_generic_add_child), 127 128 /* backlight interface */ 129 DEVMETHOD(backlight_update_status, linux_backlight_update_status), 130 DEVMETHOD(backlight_get_status, linux_backlight_get_status), 131 DEVMETHOD(backlight_get_info, linux_backlight_get_info), 132 DEVMETHOD_END 133 }; 134 135 const char *pci_power_names[] = { 136 "UNKNOWN", "D0", "D1", "D2", "D3hot", "D3cold" 137 }; 138 139 /* We need some meta-struct to keep track of these for devres. */ 140 struct pci_devres { 141 bool enable_io; 142 /* PCIR_MAX_BAR_0 + 1 = 6 => BIT(0..5). */ 143 uint8_t region_mask; 144 struct resource *region_table[PCIR_MAX_BAR_0 + 1]; /* Not needed. */ 145 }; 146 struct pcim_iomap_devres { 147 void *mmio_table[PCIR_MAX_BAR_0 + 1]; 148 struct resource *res_table[PCIR_MAX_BAR_0 + 1]; 149 }; 150 151 struct linux_dma_priv { 152 uint64_t dma_mask; 153 bus_dma_tag_t dmat; 154 uint64_t dma_coherent_mask; 155 bus_dma_tag_t dmat_coherent; 156 struct mtx lock; 157 struct pctrie ptree; 158 }; 159 #define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock) 160 #define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock) 161 162 static void 163 lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres *dr, int bar, 164 void *res) 165 { 166 dr->mmio_table[bar] = (void *)rman_get_bushandle(res); 167 dr->res_table[bar] = res; 168 } 169 170 static bool 171 lkpi_pci_bar_id_valid(int bar) 172 { 173 if (bar < 0 || bar > PCIR_MAX_BAR_0) 174 return (false); 175 176 return (true); 177 } 178 179 static int 180 linux_pdev_dma_uninit(struct pci_dev *pdev) 181 { 182 struct linux_dma_priv *priv; 183 184 priv = pdev->dev.dma_priv; 185 if (priv->dmat) 186 bus_dma_tag_destroy(priv->dmat); 187 if (priv->dmat_coherent) 188 bus_dma_tag_destroy(priv->dmat_coherent); 189 mtx_destroy(&priv->lock); 190 pdev->dev.dma_priv = NULL; 191 free(priv, M_DEVBUF); 192 return (0); 193 } 194 195 static int 196 linux_pdev_dma_init(struct pci_dev *pdev) 197 { 198 struct linux_dma_priv *priv; 199 int error; 200 201 priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO); 202 203 mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF); 204 pctrie_init(&priv->ptree); 205 206 pdev->dev.dma_priv = priv; 207 208 /* Create a default DMA tags. */ 209 error = linux_dma_tag_init(&pdev->dev, DMA_BIT_MASK(64)); 210 if (error != 0) 211 goto err; 212 /* Coherent is lower 32bit only by default in Linux. */ 213 error = linux_dma_tag_init_coherent(&pdev->dev, DMA_BIT_MASK(32)); 214 if (error != 0) 215 goto err; 216 217 return (error); 218 219 err: 220 linux_pdev_dma_uninit(pdev); 221 return (error); 222 } 223 224 int 225 linux_dma_tag_init(struct device *dev, u64 dma_mask) 226 { 227 struct linux_dma_priv *priv; 228 int error; 229 230 priv = dev->dma_priv; 231 232 if (priv->dmat) { 233 if (priv->dma_mask == dma_mask) 234 return (0); 235 236 bus_dma_tag_destroy(priv->dmat); 237 } 238 239 priv->dma_mask = dma_mask; 240 241 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 242 1, 0, /* alignment, boundary */ 243 dma_mask, /* lowaddr */ 244 BUS_SPACE_MAXADDR, /* highaddr */ 245 NULL, NULL, /* filtfunc, filtfuncarg */ 246 BUS_SPACE_MAXSIZE, /* maxsize */ 247 1, /* nsegments */ 248 BUS_SPACE_MAXSIZE, /* maxsegsz */ 249 0, /* flags */ 250 NULL, NULL, /* lockfunc, lockfuncarg */ 251 &priv->dmat); 252 return (-error); 253 } 254 255 int 256 linux_dma_tag_init_coherent(struct device *dev, u64 dma_mask) 257 { 258 struct linux_dma_priv *priv; 259 int error; 260 261 priv = dev->dma_priv; 262 263 if (priv->dmat_coherent) { 264 if (priv->dma_coherent_mask == dma_mask) 265 return (0); 266 267 bus_dma_tag_destroy(priv->dmat_coherent); 268 } 269 270 priv->dma_coherent_mask = dma_mask; 271 272 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 273 1, 0, /* alignment, boundary */ 274 dma_mask, /* lowaddr */ 275 BUS_SPACE_MAXADDR, /* highaddr */ 276 NULL, NULL, /* filtfunc, filtfuncarg */ 277 BUS_SPACE_MAXSIZE, /* maxsize */ 278 1, /* nsegments */ 279 BUS_SPACE_MAXSIZE, /* maxsegsz */ 280 0, /* flags */ 281 NULL, NULL, /* lockfunc, lockfuncarg */ 282 &priv->dmat_coherent); 283 return (-error); 284 } 285 286 static struct pci_driver * 287 linux_pci_find(device_t dev, const struct pci_device_id **idp) 288 { 289 const struct pci_device_id *id; 290 struct pci_driver *pdrv; 291 uint16_t vendor; 292 uint16_t device; 293 uint16_t subvendor; 294 uint16_t subdevice; 295 296 vendor = pci_get_vendor(dev); 297 device = pci_get_device(dev); 298 subvendor = pci_get_subvendor(dev); 299 subdevice = pci_get_subdevice(dev); 300 301 spin_lock(&pci_lock); 302 list_for_each_entry(pdrv, &pci_drivers, node) { 303 for (id = pdrv->id_table; id->vendor != 0; id++) { 304 if (vendor == id->vendor && 305 (PCI_ANY_ID == id->device || device == id->device) && 306 (PCI_ANY_ID == id->subvendor || subvendor == id->subvendor) && 307 (PCI_ANY_ID == id->subdevice || subdevice == id->subdevice)) { 308 *idp = id; 309 spin_unlock(&pci_lock); 310 return (pdrv); 311 } 312 } 313 } 314 spin_unlock(&pci_lock); 315 return (NULL); 316 } 317 318 struct pci_dev * 319 lkpi_pci_get_device(uint32_t vendor, uint32_t device, struct pci_dev *odev) 320 { 321 struct pci_dev *pdev, *found, *odev0; 322 323 odev0 = odev; 324 found = NULL; 325 spin_lock(&pci_lock); 326 list_for_each_entry(pdev, &pci_devices, links) { 327 /* Walk until we find odev. */ 328 if (odev != NULL) { 329 if (pdev == odev) 330 odev = NULL; 331 continue; 332 } 333 334 if ((pdev->vendor == vendor || vendor == PCI_ANY_ID) && 335 (pdev->device == device || device == PCI_ANY_ID)) { 336 found = pdev; 337 break; 338 } 339 } 340 pci_dev_get(found); 341 spin_unlock(&pci_lock); 342 pci_dev_put(odev0); 343 344 return (found); 345 } 346 347 static void 348 lkpi_pci_dev_release(struct device *dev) 349 { 350 struct pci_dev *pdev; 351 352 /* 353 * Before anything else, we have to free all the dynamic 354 * resource which are on the devres list. 355 * Otherwise we risk that supporting infrastructure 356 * is gone and we panic 'randomly'. 357 */ 358 lkpi_devres_release_free_list(dev); 359 360 /* 361 * Now undo linux_pci_attach_device() in reverse-ish 362 * order. 363 */ 364 pdev = to_pci_dev(dev); 365 366 /* 367 * pdrv->remove happens before pci_put_dev() in 368 * linux_pci_detach_device(), which means the driver should have 369 * cleaned up before we get here; see irqents and mmio below. 370 */ 371 372 /* Clear the hierarchy recursively to root. */ 373 if (pdev->bus->self != pdev) { 374 pci_dev_put(pdev->bus->self); 375 pdev->bus->self = NULL; 376 } 377 378 if (pdev->root != NULL) { 379 lkpinew_pci_dev_release(&pdev->root->dev); /* pci_dev_put(pdev->root); ? */ 380 pdev->root = NULL; 381 } 382 383 spin_lock(&pci_lock); 384 list_del(&pdev->links); 385 spin_unlock(&pci_lock); 386 387 linux_pdev_dma_uninit(pdev); 388 389 /* irq? */ 390 391 /* Undo lkpifill_pci_dev(). */ 392 /* devres is gone already; went at the very top. */ 393 if (!list_empty_careful(&pdev->dev.irqents)) { 394 dev_warn(&pdev->dev, "%s: driver did not clean up; " 395 "leaking IRQs\n", __func__); 396 /* 397 * XXX add private function to interrupt.h/linux_interrupt.c 398 * to walk the list and call free_irq on each if we have to. 399 */ 400 } 401 402 spin_lock_destroy(&dev->devres_lock); 403 spin_lock_destroy(&pdev->pcie_cap_lock); 404 405 if (!TAILQ_EMPTY(&pdev->mmio)) { 406 dev_warn(&pdev->dev, "%s: driver did not clean up; " 407 "leaking mmio resources\n", __func__); 408 /* XXX we have two functions to walk and release in here. */ 409 } 410 411 if (pdev->msi_desc != NULL) { 412 for (int i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--) 413 free(pdev->msi_desc[i], M_DEVBUF); 414 free(pdev->msi_desc, M_DEVBUF); 415 } 416 417 free(pdev->bus, M_DEVBUF); 418 kfree(pdev->path_name); 419 420 /* 421 * Lastly, apply an internal hack in order to signal 422 * that this was run (device reference fully dropped). 423 * See comment in linux_pci_detach_device(). 424 */ 425 pdev->dev.release = NULL; 426 } 427 428 static int 429 lkpifill_pci_dev(device_t dev, struct pci_dev *pdev) 430 { 431 struct pci_devinfo *dinfo; 432 int error; 433 434 error = kobject_init_and_add(&pdev->dev.kobj, &linux_dev_ktype, 435 &linux_root_device.kobj, device_get_nameunit(dev)); 436 if (error != 0) { 437 printf("%s:%d: kobject_init_and_add returned %d\n", 438 __func__, __LINE__, error); 439 return (error); 440 } 441 442 pdev->devfn = PCI_DEVFN(pci_get_slot(dev), pci_get_function(dev)); 443 pdev->vendor = pci_get_vendor(dev); 444 pdev->device = pci_get_device(dev); 445 pdev->subsystem_vendor = pci_get_subvendor(dev); 446 pdev->subsystem_device = pci_get_subdevice(dev); 447 pdev->class = pci_get_class(dev); 448 pdev->revision = pci_get_revid(dev); 449 pdev->path_name = kasprintf(GFP_KERNEL, "%04d:%02d:%02d.%d", 450 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), 451 pci_get_function(dev)); 452 453 pdev->bus = malloc(sizeof(*pdev->bus), M_DEVBUF, M_WAITOK | M_ZERO); 454 pdev->bus->number = pci_get_bus(dev); 455 pdev->bus->domain = pci_get_domain(dev); 456 457 /* Check if we have reached the root to satisfy pci_is_root_bus() */ 458 dinfo = device_get_ivars(dev); 459 if (dinfo->cfg.pcie.pcie_location != 0 && 460 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) { 461 pdev->bus->self = NULL; 462 } else { 463 /* 464 * This should be the upstream bridge; pci_upstream_bridge() 465 * handles that case on demand as otherwise we'll shadow the 466 * entire PCI hierarchy. 467 */ 468 pdev->bus->self = pdev; 469 } 470 pdev->dev.bsddev = dev; 471 pdev->dev.parent = &linux_root_device; 472 pdev->dev.release = lkpi_pci_dev_release; 473 474 if (pci_msi_count(dev) > 0) 475 pdev->msi_desc = malloc(pci_msi_count(dev) * 476 sizeof(*pdev->msi_desc), M_DEVBUF, M_WAITOK | M_ZERO); 477 478 TAILQ_INIT(&pdev->mmio); 479 spin_lock_init(&pdev->pcie_cap_lock); 480 spin_lock_init(&pdev->dev.devres_lock); 481 INIT_LIST_HEAD(&pdev->dev.devres_head); 482 INIT_LIST_HEAD(&pdev->dev.irqents); 483 484 return (0); 485 } 486 487 static void 488 lkpinew_pci_dev_release(struct device *dev) 489 { 490 struct pci_dev *pdev; 491 int i; 492 493 pdev = to_pci_dev(dev); 494 if (pdev->root != NULL) 495 pci_dev_put(pdev->root); 496 if (pdev->bus->self != pdev && pdev->bus->self != NULL) 497 pci_dev_put(pdev->bus->self); 498 free(pdev->bus, M_DEVBUF); 499 if (pdev->msi_desc != NULL) { 500 for (i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--) 501 free(pdev->msi_desc[i], M_DEVBUF); 502 free(pdev->msi_desc, M_DEVBUF); 503 } 504 kfree(pdev->path_name); 505 free(pdev, M_DEVBUF); 506 } 507 508 struct pci_dev * 509 lkpinew_pci_dev(device_t dev) 510 { 511 struct pci_dev *pdev; 512 int error; 513 514 pdev = malloc(sizeof(*pdev), M_DEVBUF, M_WAITOK|M_ZERO); 515 error = lkpifill_pci_dev(dev, pdev); 516 if (error != 0) { 517 free(pdev, M_DEVBUF); 518 return (NULL); 519 } 520 pdev->dev.release = lkpinew_pci_dev_release; 521 522 return (pdev); 523 } 524 525 struct pci_dev * 526 lkpi_pci_get_class(unsigned int class, struct pci_dev *from) 527 { 528 device_t dev; 529 device_t devfrom = NULL; 530 struct pci_dev *pdev; 531 532 if (from != NULL) 533 devfrom = from->dev.bsddev; 534 535 dev = pci_find_class_from(class >> 16, (class >> 8) & 0xFF, devfrom); 536 if (dev == NULL) 537 return (NULL); 538 539 pdev = lkpinew_pci_dev(dev); 540 return (pdev); 541 } 542 543 struct pci_dev * 544 lkpi_pci_get_base_class(unsigned int baseclass, struct pci_dev *from) 545 { 546 device_t dev; 547 device_t devfrom = NULL; 548 struct pci_dev *pdev; 549 550 if (from != NULL) 551 devfrom = from->dev.bsddev; 552 553 dev = pci_find_base_class_from(baseclass, devfrom); 554 if (dev == NULL) 555 return (NULL); 556 557 pdev = lkpinew_pci_dev(dev); 558 return (pdev); 559 } 560 561 struct pci_dev * 562 lkpi_pci_get_domain_bus_and_slot(int domain, unsigned int bus, 563 unsigned int devfn) 564 { 565 device_t dev; 566 struct pci_dev *pdev; 567 568 dev = pci_find_dbsf(domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 569 if (dev == NULL) 570 return (NULL); 571 572 pdev = lkpinew_pci_dev(dev); 573 return (pdev); 574 } 575 576 struct pci_dev * 577 lkpi_pci_get_slot(struct pci_bus *pbus, unsigned int devfn) 578 { 579 device_t dev; 580 struct pci_dev *pdev; 581 582 dev = pci_find_bsf(pbus->number, PCI_SLOT(devfn), PCI_FUNC(devfn)); 583 if (dev == NULL) 584 return (NULL); 585 586 pdev = lkpinew_pci_dev(dev); 587 return (pdev); 588 } 589 590 static int 591 linux_pci_probe(device_t dev) 592 { 593 const struct pci_device_id *id; 594 struct pci_driver *pdrv; 595 596 if ((pdrv = linux_pci_find(dev, &id)) == NULL) 597 return (ENXIO); 598 if (device_get_driver(dev) != &pdrv->bsddriver) 599 return (ENXIO); 600 device_set_desc(dev, pdrv->name); 601 602 /* Assume BSS initialized (should never return BUS_PROBE_SPECIFIC). */ 603 if (pdrv->bsd_probe_return == 0) 604 return (BUS_PROBE_DEFAULT); 605 else 606 return (pdrv->bsd_probe_return); 607 } 608 609 static int 610 linux_pci_attach(device_t dev) 611 { 612 const struct pci_device_id *id; 613 struct pci_driver *pdrv; 614 struct pci_dev *pdev; 615 616 pdrv = linux_pci_find(dev, &id); 617 pdev = device_get_softc(dev); 618 619 MPASS(pdrv != NULL); 620 MPASS(pdev != NULL); 621 622 return (linux_pci_attach_device(dev, pdrv, id, pdev)); 623 } 624 625 static struct resource_list_entry * 626 linux_pci_reserve_bar(struct pci_dev *pdev, struct resource_list *rl, 627 int type, int rid) 628 { 629 device_t dev; 630 struct resource *res; 631 632 KASSERT(type == SYS_RES_IOPORT || type == SYS_RES_MEMORY, 633 ("trying to reserve non-BAR type %d", type)); 634 635 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? 636 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; 637 res = pci_reserve_map(device_get_parent(dev), dev, type, rid, 0, ~0, 638 1, 1, 0); 639 if (res == NULL) 640 return (NULL); 641 return (resource_list_find(rl, type, rid)); 642 } 643 644 static struct resource_list_entry * 645 linux_pci_get_rle(struct pci_dev *pdev, int type, int rid, bool reserve_bar) 646 { 647 struct pci_devinfo *dinfo; 648 struct resource_list *rl; 649 struct resource_list_entry *rle; 650 651 dinfo = device_get_ivars(pdev->dev.bsddev); 652 rl = &dinfo->resources; 653 rle = resource_list_find(rl, type, rid); 654 /* Reserve resources for this BAR if needed. */ 655 if (rle == NULL && reserve_bar) 656 rle = linux_pci_reserve_bar(pdev, rl, type, rid); 657 return (rle); 658 } 659 660 int 661 linux_pci_attach_device(device_t dev, struct pci_driver *pdrv, 662 const struct pci_device_id *id, struct pci_dev *pdev) 663 { 664 struct resource_list_entry *rle; 665 device_t parent; 666 struct pci_dev *pbus, *ppbus; 667 uintptr_t rid; 668 int error; 669 bool isdrm; 670 671 linux_set_current(curthread); 672 673 parent = device_get_parent(dev); 674 isdrm = pdrv != NULL && pdrv->isdrm; 675 676 if (isdrm) { 677 struct pci_devinfo *dinfo; 678 679 dinfo = device_get_ivars(parent); 680 device_set_ivars(dev, dinfo); 681 } 682 683 error = lkpifill_pci_dev(dev, pdev); 684 if (error != 0) 685 return (error); 686 687 if (isdrm) 688 PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid); 689 else 690 PCI_GET_ID(parent, dev, PCI_ID_RID, &rid); 691 pdev->devfn = rid; 692 pdev->pdrv = pdrv; 693 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0, false); 694 if (rle != NULL) 695 pdev->dev.irq = rle->start; 696 else 697 pdev->dev.irq = LINUX_IRQ_INVALID; 698 pdev->irq = pdev->dev.irq; 699 error = linux_pdev_dma_init(pdev); 700 if (error) 701 goto out_dma_init; 702 703 spin_lock(&pci_lock); 704 list_add(&pdev->links, &pci_devices); 705 spin_unlock(&pci_lock); 706 707 /* 708 * Create the hierarchy now as we cannot on demand later. 709 * Take special care of DRM as there is a non-PCI device in the chain. 710 */ 711 pbus = pdev; 712 if (isdrm) { 713 pbus = lkpinew_pci_dev(parent); 714 if (pbus == NULL) { 715 error = ENXIO; 716 goto out_dma_init; 717 } 718 } 719 pcie_find_root_port(pbus); 720 if (isdrm) 721 pdev->root = pbus->root; 722 ppbus = pci_upstream_bridge(pbus); 723 while (ppbus != NULL && ppbus != pbus) { 724 pbus = ppbus; 725 ppbus = pci_upstream_bridge(pbus); 726 } 727 728 if (pdrv != NULL) { 729 error = pdrv->probe(pdev, id); 730 if (error) 731 goto out_probe; 732 } 733 return (0); 734 735 /* XXX the cleanup does not match the allocation up there. */ 736 out_probe: 737 free(pdev->bus, M_DEVBUF); 738 spin_lock_destroy(&pdev->pcie_cap_lock); 739 linux_pdev_dma_uninit(pdev); 740 out_dma_init: 741 spin_lock(&pci_lock); 742 list_del(&pdev->links); 743 spin_unlock(&pci_lock); 744 put_device(&pdev->dev); 745 return (-error); 746 } 747 748 static int 749 linux_pci_detach(device_t dev) 750 { 751 struct pci_dev *pdev; 752 int error; 753 754 pdev = device_get_softc(dev); 755 MPASS(pdev != NULL); 756 757 error = linux_pci_detach_device(pdev); 758 if (error == 0) 759 device_set_desc(dev, NULL); 760 761 return (error); 762 } 763 764 int 765 linux_pci_detach_device(struct pci_dev *pdev) 766 { 767 768 linux_set_current(curthread); 769 770 /* 771 * We cannot do much here as almost everything will have 772 * to happen as the last reference to the LinuxKPI device 773 * goes away. That will call the release function, 774 * which lkpifill_pci_dev() set. That is were most 775 * of the cleanup will happen. But before that give 776 * the driver a chance to cleanup. 777 * The big problem is that the Linux KPI does not 778 * report back if it was the last kref (well kref 779 * does report back but then kobj, dev, pdev do not). 780 * So we have little way of knowing if the release 781 * happened or not. We have to play tricks for that 782 * and we can given the softc (pdev) is still valid 783 * until we return from here. 784 */ 785 786 if (pdev->pdrv != NULL) 787 pdev->pdrv->remove(pdev); 788 789 pci_dev_put(pdev); 790 791 /* 792 * We (ab)use the release function as a guard to 793 * know if we made it there and the device is gone. 794 */ 795 if (pdev->dev.release != lkpi_pci_dev_release) 796 return (0); 797 798 /* 799 * Detach failed. 800 * We need to re-acquire the ref and wait for 801 * the other refs to be gone... In theory this 802 * should never happen, so log it! 803 * XXX I wish there was a KPI to query the ref. 804 * 805 * If we do not error and wait, we will have a 806 * LinuxKPI device dangling active with pointers 807 * but the FreeBSD device_t will be 'gone'. 808 */ 809 device_printf(pdev->dev.bsddev, "%s failed due to %u other pending " 810 "references on the LinuxKPI device.\n", __func__, 811 kref_read(&pdev->dev.kobj.kref)); 812 pci_dev_get(pdev); 813 814 return (EBUSY); 815 } 816 817 static int 818 lkpi_pci_disable_dev(struct device *dev) 819 { 820 821 (void) pci_disable_io(dev->bsddev, SYS_RES_MEMORY); 822 (void) pci_disable_io(dev->bsddev, SYS_RES_IOPORT); 823 return (0); 824 } 825 826 static struct pci_devres * 827 lkpi_pci_devres_get_alloc(struct pci_dev *pdev) 828 { 829 struct pci_devres *dr; 830 831 dr = lkpi_devres_find(&pdev->dev, lkpi_pci_devres_release, NULL, NULL); 832 if (dr == NULL) { 833 dr = lkpi_devres_alloc(lkpi_pci_devres_release, sizeof(*dr), 834 GFP_KERNEL | __GFP_ZERO); 835 if (dr != NULL) 836 lkpi_devres_add(&pdev->dev, dr); 837 } 838 839 return (dr); 840 } 841 842 static struct pci_devres * 843 lkpi_pci_devres_find(struct pci_dev *pdev) 844 { 845 if (!pdev->managed) 846 return (NULL); 847 848 return (lkpi_pci_devres_get_alloc(pdev)); 849 } 850 851 void 852 lkpi_pci_devres_release(struct device *dev, void *p) 853 { 854 struct pci_devres *dr; 855 struct pci_dev *pdev; 856 int bar; 857 858 pdev = to_pci_dev(dev); 859 dr = p; 860 861 if (pdev->msix_enabled) 862 lkpi_pci_disable_msix(pdev); 863 if (pdev->msi_enabled) 864 lkpi_pci_disable_msi(pdev); 865 866 if (dr->enable_io && lkpi_pci_disable_dev(dev) == 0) 867 dr->enable_io = false; 868 869 if (dr->region_mask == 0) 870 return; 871 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 872 873 if ((dr->region_mask & (1 << bar)) == 0) 874 continue; 875 pci_release_region(pdev, bar); 876 } 877 } 878 879 int 880 linuxkpi_pcim_enable_device(struct pci_dev *pdev) 881 { 882 struct pci_devres *dr; 883 int error; 884 885 /* Here we cannot run through the pdev->managed check. */ 886 dr = lkpi_pci_devres_get_alloc(pdev); 887 if (dr == NULL) 888 return (-ENOMEM); 889 890 /* If resources were enabled before do not do it again. */ 891 if (dr->enable_io) 892 return (0); 893 894 error = pci_enable_device(pdev); 895 if (error == 0) 896 dr->enable_io = true; 897 898 /* This device is not managed. */ 899 pdev->managed = true; 900 901 return (error); 902 } 903 904 static struct pcim_iomap_devres * 905 lkpi_pcim_iomap_devres_find(struct pci_dev *pdev) 906 { 907 struct pcim_iomap_devres *dr; 908 909 dr = lkpi_devres_find(&pdev->dev, lkpi_pcim_iomap_table_release, 910 NULL, NULL); 911 if (dr == NULL) { 912 dr = lkpi_devres_alloc(lkpi_pcim_iomap_table_release, 913 sizeof(*dr), GFP_KERNEL | __GFP_ZERO); 914 if (dr != NULL) 915 lkpi_devres_add(&pdev->dev, dr); 916 } 917 918 if (dr == NULL) 919 device_printf(pdev->dev.bsddev, "%s: NULL\n", __func__); 920 921 return (dr); 922 } 923 924 void __iomem ** 925 linuxkpi_pcim_iomap_table(struct pci_dev *pdev) 926 { 927 struct pcim_iomap_devres *dr; 928 929 dr = lkpi_pcim_iomap_devres_find(pdev); 930 if (dr == NULL) 931 return (NULL); 932 933 /* 934 * If the driver has manually set a flag to be able to request the 935 * resource to use bus_read/write_<n>, return the shadow table. 936 */ 937 if (pdev->want_iomap_res) 938 return ((void **)dr->res_table); 939 940 /* This is the Linux default. */ 941 return (dr->mmio_table); 942 } 943 944 static struct resource * 945 _lkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen __unused) 946 { 947 struct pci_mmio_region *mmio, *p; 948 int type; 949 950 if (!lkpi_pci_bar_id_valid(bar)) 951 return (NULL); 952 953 type = pci_resource_type(pdev, bar); 954 if (type < 0) { 955 device_printf(pdev->dev.bsddev, "%s: bar %d type %d\n", 956 __func__, bar, type); 957 return (NULL); 958 } 959 960 /* 961 * Check for duplicate mappings. 962 * This can happen if a driver calls pci_request_region() first. 963 */ 964 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 965 if (mmio->type == type && mmio->rid == PCIR_BAR(bar)) { 966 return (mmio->res); 967 } 968 } 969 970 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO); 971 mmio->rid = PCIR_BAR(bar); 972 mmio->type = type; 973 mmio->res = bus_alloc_resource_any(pdev->dev.bsddev, mmio->type, 974 &mmio->rid, RF_ACTIVE|RF_SHAREABLE); 975 if (mmio->res == NULL) { 976 device_printf(pdev->dev.bsddev, "%s: failed to alloc " 977 "bar %d type %d rid %d\n", 978 __func__, bar, type, PCIR_BAR(bar)); 979 free(mmio, M_DEVBUF); 980 return (NULL); 981 } 982 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next); 983 984 return (mmio->res); 985 } 986 987 void * 988 linuxkpi_pci_iomap_range(struct pci_dev *pdev, int bar, 989 unsigned long off, unsigned long maxlen) 990 { 991 struct resource *res; 992 993 if (!lkpi_pci_bar_id_valid(bar)) 994 return (NULL); 995 996 res = _lkpi_pci_iomap(pdev, bar, maxlen); 997 if (res == NULL) 998 return (NULL); 999 /* This is a FreeBSD extension so we can use bus_*(). */ 1000 if (pdev->want_iomap_res) 1001 return (res); 1002 MPASS(off < rman_get_size(res)); 1003 return ((void *)(rman_get_bushandle(res) + off)); 1004 } 1005 1006 void * 1007 linuxkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen) 1008 { 1009 if (!lkpi_pci_bar_id_valid(bar)) 1010 return (NULL); 1011 1012 return (linuxkpi_pci_iomap_range(pdev, bar, 0, maxlen)); 1013 } 1014 1015 void * 1016 linuxkpi_pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen) 1017 { 1018 struct pcim_iomap_devres *dr; 1019 void *res; 1020 1021 if (!lkpi_pci_bar_id_valid(bar)) 1022 return (NULL); 1023 1024 dr = lkpi_pcim_iomap_devres_find(pdev); 1025 if (dr == NULL) 1026 return (NULL); 1027 1028 if (dr->res_table[bar] != NULL) 1029 return (dr->res_table[bar]); 1030 1031 res = linuxkpi_pci_iomap(pdev, bar, maxlen); 1032 if (res == NULL) { 1033 /* 1034 * Do not free the devres in case there were 1035 * other valid mappings before already. 1036 */ 1037 return (NULL); 1038 } 1039 lkpi_set_pcim_iomap_devres(dr, bar, res); 1040 1041 return (res); 1042 } 1043 1044 void 1045 linuxkpi_pci_iounmap(struct pci_dev *pdev, void *res) 1046 { 1047 struct pci_mmio_region *mmio, *p; 1048 bus_space_handle_t bh = (bus_space_handle_t)res; 1049 1050 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 1051 if (pdev->want_iomap_res) { 1052 if (res != mmio->res) 1053 continue; 1054 } else { 1055 if (bh < rman_get_bushandle(mmio->res) || 1056 bh >= rman_get_bushandle(mmio->res) + 1057 rman_get_size(mmio->res)) 1058 continue; 1059 } 1060 bus_release_resource(pdev->dev.bsddev, 1061 mmio->type, mmio->rid, mmio->res); 1062 TAILQ_REMOVE(&pdev->mmio, mmio, next); 1063 free(mmio, M_DEVBUF); 1064 return; 1065 } 1066 } 1067 1068 int 1069 linuxkpi_pcim_iomap_regions(struct pci_dev *pdev, uint32_t mask, const char *name) 1070 { 1071 struct pcim_iomap_devres *dr; 1072 void *res; 1073 uint32_t mappings; 1074 int bar; 1075 1076 dr = lkpi_pcim_iomap_devres_find(pdev); 1077 if (dr == NULL) 1078 return (-ENOMEM); 1079 1080 /* Now iomap all the requested (by "mask") ones. */ 1081 for (bar = mappings = 0; mappings != mask; bar++) { 1082 if ((mask & (1 << bar)) == 0) 1083 continue; 1084 1085 /* Request double is not allowed. */ 1086 if (dr->mmio_table[bar] != NULL) { 1087 device_printf(pdev->dev.bsddev, "%s: bar %d %p\n", 1088 __func__, bar, dr->mmio_table[bar]); 1089 goto err; 1090 } 1091 1092 res = _lkpi_pci_iomap(pdev, bar, 0); 1093 if (res == NULL) 1094 goto err; 1095 lkpi_set_pcim_iomap_devres(dr, bar, res); 1096 1097 mappings |= (1 << bar); 1098 } 1099 1100 return (0); 1101 err: 1102 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 1103 if ((mappings & (1 << bar)) != 0) { 1104 res = dr->mmio_table[bar]; 1105 if (res == NULL) 1106 continue; 1107 pci_iounmap(pdev, res); 1108 } 1109 } 1110 1111 return (-EINVAL); 1112 } 1113 1114 static void 1115 lkpi_pcim_iomap_table_release(struct device *dev, void *p) 1116 { 1117 struct pcim_iomap_devres *dr; 1118 struct pci_dev *pdev; 1119 int bar; 1120 1121 dr = p; 1122 pdev = to_pci_dev(dev); 1123 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 1124 1125 if (dr->mmio_table[bar] == NULL) 1126 continue; 1127 1128 pci_iounmap(pdev, dr->mmio_table[bar]); 1129 } 1130 } 1131 1132 static int 1133 linux_pci_suspend(device_t dev) 1134 { 1135 const struct dev_pm_ops *pmops; 1136 struct pm_message pm = { }; 1137 struct pci_dev *pdev; 1138 int error; 1139 1140 error = 0; 1141 linux_set_current(curthread); 1142 pdev = device_get_softc(dev); 1143 pmops = pdev->pdrv->driver.pm; 1144 1145 if (pdev->pdrv->suspend != NULL) 1146 error = -pdev->pdrv->suspend(pdev, pm); 1147 else if (pmops != NULL && pmops->suspend != NULL) { 1148 error = -pmops->suspend(&pdev->dev); 1149 if (error == 0 && pmops->suspend_late != NULL) 1150 error = -pmops->suspend_late(&pdev->dev); 1151 if (error == 0 && pmops->suspend_noirq != NULL) 1152 error = -pmops->suspend_noirq(&pdev->dev); 1153 } 1154 return (error); 1155 } 1156 1157 static int 1158 linux_pci_resume(device_t dev) 1159 { 1160 const struct dev_pm_ops *pmops; 1161 struct pci_dev *pdev; 1162 int error; 1163 1164 error = 0; 1165 linux_set_current(curthread); 1166 pdev = device_get_softc(dev); 1167 pmops = pdev->pdrv->driver.pm; 1168 1169 if (pdev->pdrv->resume != NULL) 1170 error = -pdev->pdrv->resume(pdev); 1171 else if (pmops != NULL && pmops->resume != NULL) { 1172 if (pmops->resume_early != NULL) 1173 error = -pmops->resume_early(&pdev->dev); 1174 if (error == 0 && pmops->resume != NULL) 1175 error = -pmops->resume(&pdev->dev); 1176 } 1177 return (error); 1178 } 1179 1180 static int 1181 linux_pci_shutdown(device_t dev) 1182 { 1183 struct pci_dev *pdev; 1184 1185 linux_set_current(curthread); 1186 pdev = device_get_softc(dev); 1187 if (pdev->pdrv->shutdown != NULL) 1188 pdev->pdrv->shutdown(pdev); 1189 return (0); 1190 } 1191 1192 static int 1193 linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config) 1194 { 1195 struct pci_dev *pdev; 1196 int error; 1197 1198 linux_set_current(curthread); 1199 pdev = device_get_softc(dev); 1200 if (pdev->pdrv->bsd_iov_init != NULL) 1201 error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config); 1202 else 1203 error = EINVAL; 1204 return (error); 1205 } 1206 1207 static void 1208 linux_pci_iov_uninit(device_t dev) 1209 { 1210 struct pci_dev *pdev; 1211 1212 linux_set_current(curthread); 1213 pdev = device_get_softc(dev); 1214 if (pdev->pdrv->bsd_iov_uninit != NULL) 1215 pdev->pdrv->bsd_iov_uninit(dev); 1216 } 1217 1218 static int 1219 linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config) 1220 { 1221 struct pci_dev *pdev; 1222 int error; 1223 1224 linux_set_current(curthread); 1225 pdev = device_get_softc(dev); 1226 if (pdev->pdrv->bsd_iov_add_vf != NULL) 1227 error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config); 1228 else 1229 error = EINVAL; 1230 return (error); 1231 } 1232 1233 static int 1234 _linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc) 1235 { 1236 int error; 1237 1238 linux_set_current(curthread); 1239 spin_lock(&pci_lock); 1240 list_add(&pdrv->node, &pci_drivers); 1241 spin_unlock(&pci_lock); 1242 if (pdrv->bsddriver.name == NULL) 1243 pdrv->bsddriver.name = pdrv->name; 1244 pdrv->bsddriver.methods = pci_methods; 1245 pdrv->bsddriver.size = sizeof(struct pci_dev); 1246 1247 bus_topo_lock(); 1248 error = devclass_add_driver(dc, &pdrv->bsddriver, 1249 BUS_PASS_DEFAULT, &pdrv->bsdclass); 1250 bus_topo_unlock(); 1251 return (-error); 1252 } 1253 1254 int 1255 linux_pci_register_driver(struct pci_driver *pdrv) 1256 { 1257 devclass_t dc; 1258 1259 pdrv->isdrm = strcmp(pdrv->name, "drmn") == 0; 1260 dc = pdrv->isdrm ? devclass_create("vgapci") : devclass_find("pci"); 1261 if (dc == NULL) 1262 return (-ENXIO); 1263 return (_linux_pci_register_driver(pdrv, dc)); 1264 } 1265 1266 static struct resource_list_entry * 1267 lkpi_pci_get_bar(struct pci_dev *pdev, int bar, bool reserve) 1268 { 1269 int type; 1270 1271 type = pci_resource_type(pdev, bar); 1272 if (type < 0) 1273 return (NULL); 1274 bar = PCIR_BAR(bar); 1275 return (linux_pci_get_rle(pdev, type, bar, reserve)); 1276 } 1277 1278 struct device * 1279 lkpi_pci_find_irq_dev(unsigned int irq) 1280 { 1281 struct pci_dev *pdev; 1282 struct device *found; 1283 1284 found = NULL; 1285 spin_lock(&pci_lock); 1286 list_for_each_entry(pdev, &pci_devices, links) { 1287 if (irq == pdev->dev.irq || 1288 (irq >= pdev->dev.irq_start && irq < pdev->dev.irq_end)) { 1289 found = &pdev->dev; 1290 break; 1291 } 1292 } 1293 spin_unlock(&pci_lock); 1294 return (found); 1295 } 1296 1297 unsigned long 1298 pci_resource_start(struct pci_dev *pdev, int bar) 1299 { 1300 struct resource_list_entry *rle; 1301 rman_res_t newstart; 1302 device_t dev; 1303 int error; 1304 1305 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL) 1306 return (0); 1307 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? 1308 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; 1309 error = bus_translate_resource(dev, rle->type, rle->start, &newstart); 1310 if (error != 0) { 1311 device_printf(pdev->dev.bsddev, 1312 "translate of %#jx failed: %d\n", 1313 (uintmax_t)rle->start, error); 1314 return (0); 1315 } 1316 return (newstart); 1317 } 1318 1319 unsigned long 1320 pci_resource_len(struct pci_dev *pdev, int bar) 1321 { 1322 struct resource_list_entry *rle; 1323 1324 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL) 1325 return (0); 1326 return (rle->count); 1327 } 1328 1329 static int 1330 lkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name, 1331 bool managed) 1332 { 1333 struct resource *res; 1334 struct pci_devres *dr; 1335 struct pci_mmio_region *mmio; 1336 int rid; 1337 int type; 1338 1339 if (!lkpi_pci_bar_id_valid(bar)) 1340 return (-EINVAL); 1341 1342 type = pci_resource_type(pdev, bar); 1343 if (type < 0) 1344 return (0); 1345 1346 rid = PCIR_BAR(bar); 1347 res = bus_alloc_resource_any(pdev->dev.bsddev, type, &rid, 1348 RF_ACTIVE|RF_SHAREABLE); 1349 if (res == NULL) { 1350 device_printf(pdev->dev.bsddev, "%s: failed to alloc " 1351 "bar %d type %d rid %d\n", 1352 __func__, bar, type, PCIR_BAR(bar)); 1353 return (-EBUSY); 1354 } 1355 1356 /* 1357 * It seems there is an implicit devres tracking on these if the device 1358 * is managed (lkpi_pci_devres_find() case); otherwise the resources are 1359 * not automatically freed on FreeBSD/LinuxKPI though they should be/are 1360 * expected to be by Linux drivers. 1361 * Otherwise if we are called from a pcim-function with the managed 1362 * argument set, we need to track devres independent of pdev->managed. 1363 */ 1364 if (managed) 1365 dr = lkpi_pci_devres_get_alloc(pdev); 1366 else 1367 dr = lkpi_pci_devres_find(pdev); 1368 if (dr != NULL) { 1369 dr->region_mask |= (1 << bar); 1370 dr->region_table[bar] = res; 1371 } 1372 1373 /* Even if the device is not managed we need to track it for iomap. */ 1374 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO); 1375 mmio->rid = PCIR_BAR(bar); 1376 mmio->type = type; 1377 mmio->res = res; 1378 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next); 1379 1380 return (0); 1381 } 1382 1383 int 1384 linuxkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) 1385 { 1386 return (lkpi_pci_request_region(pdev, bar, res_name, false)); 1387 } 1388 1389 int 1390 linuxkpi_pci_request_regions(struct pci_dev *pdev, const char *res_name) 1391 { 1392 int error; 1393 int i; 1394 1395 for (i = 0; i <= PCIR_MAX_BAR_0; i++) { 1396 error = pci_request_region(pdev, i, res_name); 1397 if (error && error != -EBUSY) { 1398 pci_release_regions(pdev); 1399 return (error); 1400 } 1401 } 1402 return (0); 1403 } 1404 1405 int 1406 linuxkpi_pcim_request_all_regions(struct pci_dev *pdev, const char *res_name) 1407 { 1408 int bar, error; 1409 1410 for (bar = 0; bar <= PCIR_MAX_BAR_0; bar++) { 1411 error = lkpi_pci_request_region(pdev, bar, res_name, true); 1412 if (error != 0 && error != -EBUSY) { 1413 device_printf(pdev->dev.bsddev, "%s: bar %d res_name '%s': " 1414 "lkpi_pci_request_region returned %d\n", __func__, 1415 bar, res_name, error); 1416 pci_release_regions(pdev); 1417 return (error); 1418 } 1419 } 1420 return (0); 1421 } 1422 1423 void 1424 linuxkpi_pci_release_region(struct pci_dev *pdev, int bar) 1425 { 1426 struct resource_list_entry *rle; 1427 struct pci_devres *dr; 1428 struct pci_mmio_region *mmio, *p; 1429 1430 if ((rle = lkpi_pci_get_bar(pdev, bar, false)) == NULL) 1431 return; 1432 1433 /* 1434 * As we implicitly track the requests we also need to clear them on 1435 * release. Do clear before resource release. 1436 */ 1437 dr = lkpi_pci_devres_find(pdev); 1438 if (dr != NULL) { 1439 KASSERT(dr->region_table[bar] == rle->res, ("%s: pdev %p bar %d" 1440 " region_table res %p != rel->res %p\n", __func__, pdev, 1441 bar, dr->region_table[bar], rle->res)); 1442 dr->region_table[bar] = NULL; 1443 dr->region_mask &= ~(1 << bar); 1444 } 1445 1446 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 1447 if (rle->res != (void *)rman_get_bushandle(mmio->res)) 1448 continue; 1449 TAILQ_REMOVE(&pdev->mmio, mmio, next); 1450 free(mmio, M_DEVBUF); 1451 } 1452 1453 bus_release_resource(pdev->dev.bsddev, rle->type, rle->rid, rle->res); 1454 } 1455 1456 void 1457 linuxkpi_pci_release_regions(struct pci_dev *pdev) 1458 { 1459 int i; 1460 1461 for (i = 0; i <= PCIR_MAX_BAR_0; i++) 1462 pci_release_region(pdev, i); 1463 } 1464 1465 int 1466 linux_pci_register_drm_driver(struct pci_driver *pdrv) 1467 { 1468 devclass_t dc; 1469 1470 dc = devclass_create("vgapci"); 1471 if (dc == NULL) 1472 return (-ENXIO); 1473 pdrv->isdrm = true; 1474 pdrv->name = "drmn"; 1475 return (_linux_pci_register_driver(pdrv, dc)); 1476 } 1477 1478 void 1479 linux_pci_unregister_driver(struct pci_driver *pdrv) 1480 { 1481 devclass_t bus; 1482 1483 bus = devclass_find(pdrv->isdrm ? "vgapci" : "pci"); 1484 1485 spin_lock(&pci_lock); 1486 list_del(&pdrv->node); 1487 spin_unlock(&pci_lock); 1488 bus_topo_lock(); 1489 if (bus != NULL) 1490 devclass_delete_driver(bus, &pdrv->bsddriver); 1491 bus_topo_unlock(); 1492 } 1493 1494 void 1495 linux_pci_unregister_drm_driver(struct pci_driver *pdrv) 1496 { 1497 devclass_t bus; 1498 1499 bus = devclass_find("vgapci"); 1500 1501 spin_lock(&pci_lock); 1502 list_del(&pdrv->node); 1503 spin_unlock(&pci_lock); 1504 bus_topo_lock(); 1505 if (bus != NULL) 1506 devclass_delete_driver(bus, &pdrv->bsddriver); 1507 bus_topo_unlock(); 1508 } 1509 1510 int 1511 linuxkpi_pci_enable_msix(struct pci_dev *pdev, struct msix_entry *entries, 1512 int nreq) 1513 { 1514 struct resource_list_entry *rle; 1515 int error; 1516 int avail; 1517 int i; 1518 1519 avail = pci_msix_count(pdev->dev.bsddev); 1520 if (avail < nreq) { 1521 if (avail == 0) 1522 return -EINVAL; 1523 return avail; 1524 } 1525 avail = nreq; 1526 if ((error = -pci_alloc_msix(pdev->dev.bsddev, &avail)) != 0) 1527 return error; 1528 /* 1529 * Handle case where "pci_alloc_msix()" may allocate less 1530 * interrupts than available and return with no error: 1531 */ 1532 if (avail < nreq) { 1533 pci_release_msi(pdev->dev.bsddev); 1534 return avail; 1535 } 1536 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); 1537 pdev->dev.irq_start = rle->start; 1538 pdev->dev.irq_end = rle->start + avail; 1539 for (i = 0; i < nreq; i++) 1540 entries[i].vector = pdev->dev.irq_start + i; 1541 pdev->msix_enabled = true; 1542 return (0); 1543 } 1544 1545 int 1546 _lkpi_pci_enable_msi_range(struct pci_dev *pdev, int minvec, int maxvec) 1547 { 1548 struct resource_list_entry *rle; 1549 int error; 1550 int nvec; 1551 1552 if (maxvec < minvec) 1553 return (-EINVAL); 1554 1555 nvec = pci_msi_count(pdev->dev.bsddev); 1556 if (nvec < 1 || nvec < minvec) 1557 return (-ENOSPC); 1558 1559 nvec = min(nvec, maxvec); 1560 if ((error = -pci_alloc_msi(pdev->dev.bsddev, &nvec)) != 0) 1561 return error; 1562 1563 /* Native PCI might only ever ask for 32 vectors. */ 1564 if (nvec < minvec) { 1565 pci_release_msi(pdev->dev.bsddev); 1566 return (-ENOSPC); 1567 } 1568 1569 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); 1570 pdev->dev.irq_start = rle->start; 1571 pdev->dev.irq_end = rle->start + nvec; 1572 pdev->irq = rle->start; 1573 pdev->msi_enabled = true; 1574 return (0); 1575 } 1576 1577 int 1578 pci_alloc_irq_vectors(struct pci_dev *pdev, int minv, int maxv, 1579 unsigned int flags) 1580 { 1581 int error; 1582 1583 if (flags & PCI_IRQ_MSIX) { 1584 struct msix_entry *entries; 1585 int i; 1586 1587 entries = kcalloc(maxv, sizeof(*entries), GFP_KERNEL); 1588 if (entries == NULL) { 1589 error = -ENOMEM; 1590 goto out; 1591 } 1592 for (i = 0; i < maxv; ++i) 1593 entries[i].entry = i; 1594 error = pci_enable_msix(pdev, entries, maxv); 1595 out: 1596 kfree(entries); 1597 if (error == 0 && pdev->msix_enabled) 1598 return (pdev->dev.irq_end - pdev->dev.irq_start); 1599 } 1600 if (flags & PCI_IRQ_MSI) { 1601 if (pci_msi_count(pdev->dev.bsddev) < minv) 1602 return (-ENOSPC); 1603 error = _lkpi_pci_enable_msi_range(pdev, minv, maxv); 1604 if (error == 0 && pdev->msi_enabled) 1605 return (pdev->dev.irq_end - pdev->dev.irq_start); 1606 } 1607 if (flags & PCI_IRQ_INTX) { 1608 if (pdev->irq) 1609 return (1); 1610 } 1611 1612 return (-EINVAL); 1613 } 1614 1615 struct msi_desc * 1616 lkpi_pci_msi_desc_alloc(int irq) 1617 { 1618 struct device *dev; 1619 struct pci_dev *pdev; 1620 struct msi_desc *desc; 1621 struct pci_devinfo *dinfo; 1622 struct pcicfg_msi *msi; 1623 int vec; 1624 1625 dev = lkpi_pci_find_irq_dev(irq); 1626 if (dev == NULL) 1627 return (NULL); 1628 1629 pdev = to_pci_dev(dev); 1630 1631 if (pdev->msi_desc == NULL) 1632 return (NULL); 1633 1634 if (irq < pdev->dev.irq_start || irq >= pdev->dev.irq_end) 1635 return (NULL); 1636 1637 vec = pdev->dev.irq_start - irq; 1638 1639 if (pdev->msi_desc[vec] != NULL) 1640 return (pdev->msi_desc[vec]); 1641 1642 dinfo = device_get_ivars(dev->bsddev); 1643 msi = &dinfo->cfg.msi; 1644 1645 desc = malloc(sizeof(*desc), M_DEVBUF, M_WAITOK | M_ZERO); 1646 1647 desc->pci.msi_attrib.is_64 = 1648 (msi->msi_ctrl & PCIM_MSICTRL_64BIT) ? true : false; 1649 desc->msg.data = msi->msi_data; 1650 1651 pdev->msi_desc[vec] = desc; 1652 1653 return (desc); 1654 } 1655 1656 bool 1657 pci_device_is_present(struct pci_dev *pdev) 1658 { 1659 device_t dev; 1660 1661 dev = pdev->dev.bsddev; 1662 1663 return (bus_child_present(dev)); 1664 } 1665 1666 CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t)); 1667 1668 struct linux_dma_obj { 1669 void *vaddr; 1670 uint64_t dma_addr; 1671 bus_dmamap_t dmamap; 1672 bus_dma_tag_t dmat; 1673 }; 1674 1675 static uma_zone_t linux_dma_trie_zone; 1676 static uma_zone_t linux_dma_obj_zone; 1677 1678 static void 1679 linux_dma_init(void *arg) 1680 { 1681 1682 linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie", 1683 pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL, 1684 UMA_ALIGN_PTR, 0); 1685 linux_dma_obj_zone = uma_zcreate("linux_dma_object", 1686 sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL, 1687 UMA_ALIGN_PTR, 0); 1688 lkpi_pci_nseg1_fail = counter_u64_alloc(M_WAITOK); 1689 } 1690 SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL); 1691 1692 static void 1693 linux_dma_uninit(void *arg) 1694 { 1695 1696 counter_u64_free(lkpi_pci_nseg1_fail); 1697 uma_zdestroy(linux_dma_obj_zone); 1698 uma_zdestroy(linux_dma_trie_zone); 1699 } 1700 SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL); 1701 1702 static void * 1703 linux_dma_trie_alloc(struct pctrie *ptree) 1704 { 1705 1706 return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT)); 1707 } 1708 1709 static void 1710 linux_dma_trie_free(struct pctrie *ptree, void *node) 1711 { 1712 1713 uma_zfree(linux_dma_trie_zone, node); 1714 } 1715 1716 PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, 1717 linux_dma_trie_free); 1718 1719 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 1720 static dma_addr_t 1721 linux_dma_map_phys_common(struct device *dev, vm_paddr_t phys, size_t len, 1722 bus_dma_tag_t dmat) 1723 { 1724 struct linux_dma_priv *priv; 1725 struct linux_dma_obj *obj; 1726 int error, nseg; 1727 bus_dma_segment_t seg; 1728 1729 priv = dev->dma_priv; 1730 1731 /* 1732 * If the resultant mapping will be entirely 1:1 with the 1733 * physical address, short-circuit the remainder of the 1734 * bus_dma API. This avoids tracking collisions in the pctrie 1735 * with the additional benefit of reducing overhead. 1736 */ 1737 if (bus_dma_id_mapped(dmat, phys, len)) 1738 return (phys); 1739 1740 obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT); 1741 if (obj == NULL) { 1742 return (0); 1743 } 1744 obj->dmat = dmat; 1745 1746 DMA_PRIV_LOCK(priv); 1747 if (bus_dmamap_create(obj->dmat, 0, &obj->dmamap) != 0) { 1748 DMA_PRIV_UNLOCK(priv); 1749 uma_zfree(linux_dma_obj_zone, obj); 1750 return (0); 1751 } 1752 1753 nseg = -1; 1754 error = _bus_dmamap_load_phys(obj->dmat, obj->dmamap, phys, len, 1755 BUS_DMA_NOWAIT, &seg, &nseg); 1756 if (error != 0) { 1757 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1758 DMA_PRIV_UNLOCK(priv); 1759 uma_zfree(linux_dma_obj_zone, obj); 1760 counter_u64_add(lkpi_pci_nseg1_fail, 1); 1761 if (linuxkpi_debug) { 1762 device_printf(dev->bsddev, "%s: _bus_dmamap_load_phys " 1763 "error %d, phys %#018jx len %zu\n", __func__, 1764 error, (uintmax_t)phys, len); 1765 dump_stack(); 1766 } 1767 return (0); 1768 } 1769 1770 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 1771 obj->dma_addr = seg.ds_addr; 1772 1773 error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); 1774 if (error != 0) { 1775 bus_dmamap_unload(obj->dmat, obj->dmamap); 1776 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1777 DMA_PRIV_UNLOCK(priv); 1778 uma_zfree(linux_dma_obj_zone, obj); 1779 return (0); 1780 } 1781 DMA_PRIV_UNLOCK(priv); 1782 return (obj->dma_addr); 1783 } 1784 #else 1785 static dma_addr_t 1786 linux_dma_map_phys_common(struct device *dev __unused, vm_paddr_t phys, 1787 size_t len __unused, bus_dma_tag_t dmat __unused) 1788 { 1789 return (phys); 1790 } 1791 #endif 1792 1793 dma_addr_t 1794 lkpi_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len, 1795 enum dma_data_direction direction, unsigned long attrs) 1796 { 1797 struct linux_dma_priv *priv; 1798 dma_addr_t dma; 1799 1800 priv = dev->dma_priv; 1801 dma = linux_dma_map_phys_common(dev, phys, len, priv->dmat); 1802 if (dma_mapping_error(dev, dma)) 1803 return (dma); 1804 1805 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 1806 dma_sync_single_for_device(dev, dma, len, direction); 1807 1808 return (dma); 1809 } 1810 1811 /* For backward compat only so we can MFC this. Remove before 15. */ 1812 dma_addr_t 1813 linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) 1814 { 1815 return (lkpi_dma_map_phys(dev, phys, len, DMA_NONE, 0)); 1816 } 1817 1818 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 1819 void 1820 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, 1821 enum dma_data_direction direction, unsigned long attrs) 1822 { 1823 struct linux_dma_priv *priv; 1824 struct linux_dma_obj *obj; 1825 1826 priv = dev->dma_priv; 1827 1828 if (pctrie_is_empty(&priv->ptree)) 1829 return; 1830 1831 DMA_PRIV_LOCK(priv); 1832 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); 1833 if (obj == NULL) { 1834 DMA_PRIV_UNLOCK(priv); 1835 return; 1836 } 1837 LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); 1838 1839 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 1840 goto skip_sync; 1841 1842 /* dma_sync_single_for_cpu() unrolled to avoid lock recursicn. */ 1843 switch (direction) { 1844 case DMA_BIDIRECTIONAL: 1845 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_POSTREAD); 1846 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_PREREAD); 1847 break; 1848 case DMA_TO_DEVICE: 1849 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_POSTWRITE); 1850 break; 1851 case DMA_FROM_DEVICE: 1852 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_POSTREAD); 1853 break; 1854 default: 1855 break; 1856 } 1857 1858 skip_sync: 1859 bus_dmamap_unload(obj->dmat, obj->dmamap); 1860 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1861 DMA_PRIV_UNLOCK(priv); 1862 1863 uma_zfree(linux_dma_obj_zone, obj); 1864 } 1865 #else 1866 void 1867 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, 1868 enum dma_data_direction direction, unsigned long attrs) 1869 { 1870 } 1871 #endif 1872 1873 /* For backward compat only so we can MFC this. Remove before 15. */ 1874 void 1875 linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) 1876 { 1877 lkpi_dma_unmap(dev, dma_addr, len, DMA_NONE, 0); 1878 } 1879 1880 void * 1881 linux_dma_alloc_coherent(struct device *dev, size_t size, 1882 dma_addr_t *dma_handle, gfp_t flag) 1883 { 1884 struct linux_dma_priv *priv; 1885 vm_paddr_t high; 1886 size_t align; 1887 void *mem; 1888 1889 if (dev == NULL || dev->dma_priv == NULL) { 1890 *dma_handle = 0; 1891 return (NULL); 1892 } 1893 priv = dev->dma_priv; 1894 if (priv->dma_coherent_mask) 1895 high = priv->dma_coherent_mask; 1896 else 1897 /* Coherent is lower 32bit only by default in Linux. */ 1898 high = BUS_SPACE_MAXADDR_32BIT; 1899 align = PAGE_SIZE << get_order(size); 1900 /* Always zero the allocation. */ 1901 flag |= M_ZERO; 1902 mem = kmem_alloc_contig(size, flag & GFP_NATIVE_MASK, 0, high, 1903 align, 0, VM_MEMATTR_DEFAULT); 1904 if (mem != NULL) { 1905 *dma_handle = linux_dma_map_phys_common(dev, vtophys(mem), size, 1906 priv->dmat_coherent); 1907 if (*dma_handle == 0) { 1908 kmem_free(mem, size); 1909 mem = NULL; 1910 } 1911 } else { 1912 *dma_handle = 0; 1913 } 1914 return (mem); 1915 } 1916 1917 struct lkpi_devres_dmam_coherent { 1918 size_t size; 1919 dma_addr_t handle; 1920 void *mem; 1921 }; 1922 1923 static void 1924 lkpi_dmam_free_coherent(struct device *dev, void *p) 1925 { 1926 struct lkpi_devres_dmam_coherent *dr; 1927 1928 dr = p; 1929 dma_free_coherent(dev, dr->size, dr->mem, dr->handle); 1930 } 1931 1932 static int 1933 lkpi_dmam_coherent_match(struct device *dev, void *dr, void *mp) 1934 { 1935 struct lkpi_devres_dmam_coherent *a, *b; 1936 1937 a = dr; 1938 b = mp; 1939 1940 if (a->mem != b->mem) 1941 return (0); 1942 if (a->size != b->size || a->handle != b->handle) 1943 dev_WARN(dev, "for mem %p: size %zu != %zu || handle %#jx != %#jx\n", 1944 a->mem, a->size, b->size, 1945 (uintmax_t)a->handle, (uintmax_t)b->handle); 1946 return (1); 1947 } 1948 1949 void 1950 linuxkpi_dmam_free_coherent(struct device *dev, size_t size, 1951 void *addr, dma_addr_t dma_handle) 1952 { 1953 struct lkpi_devres_dmam_coherent match = { 1954 .size = size, 1955 .handle = dma_handle, 1956 .mem = addr 1957 }; 1958 int error; 1959 1960 error = devres_destroy(dev, lkpi_dmam_free_coherent, 1961 lkpi_dmam_coherent_match, &match); 1962 if (error != 0) 1963 dev_WARN(dev, "devres_destroy returned %d, size %zu addr %p " 1964 "dma_handle %#jx\n", error, size, addr, (uintmax_t)dma_handle); 1965 dma_free_coherent(dev, size, addr, dma_handle); 1966 } 1967 1968 void * 1969 linuxkpi_dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, 1970 gfp_t flag) 1971 { 1972 struct lkpi_devres_dmam_coherent *dr; 1973 1974 dr = lkpi_devres_alloc(lkpi_dmam_free_coherent, 1975 sizeof(*dr), GFP_KERNEL | __GFP_ZERO); 1976 1977 if (dr == NULL) 1978 return (NULL); 1979 1980 dr->size = size; 1981 dr->mem = linux_dma_alloc_coherent(dev, size, dma_handle, flag); 1982 dr->handle = *dma_handle; 1983 if (dr->mem == NULL) { 1984 lkpi_devres_free(dr); 1985 return (NULL); 1986 } 1987 1988 lkpi_devres_add(dev, dr); 1989 return (dr->mem); 1990 } 1991 1992 void 1993 linuxkpi_dma_sync(struct device *dev, dma_addr_t dma_addr, size_t size, 1994 bus_dmasync_op_t op) 1995 { 1996 struct linux_dma_priv *priv; 1997 struct linux_dma_obj *obj; 1998 1999 priv = dev->dma_priv; 2000 2001 if (pctrie_is_empty(&priv->ptree)) 2002 return; 2003 2004 DMA_PRIV_LOCK(priv); 2005 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); 2006 if (obj == NULL) { 2007 DMA_PRIV_UNLOCK(priv); 2008 return; 2009 } 2010 2011 bus_dmamap_sync(obj->dmat, obj->dmamap, op); 2012 DMA_PRIV_UNLOCK(priv); 2013 } 2014 2015 int 2016 linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, 2017 enum dma_data_direction direction, unsigned long attrs) 2018 { 2019 struct linux_dma_priv *priv; 2020 struct scatterlist *sg; 2021 int i, nseg; 2022 bus_dma_segment_t seg; 2023 2024 priv = dev->dma_priv; 2025 2026 DMA_PRIV_LOCK(priv); 2027 2028 /* create common DMA map in the first S/G entry */ 2029 if (bus_dmamap_create(priv->dmat, 0, &sgl->dma_map) != 0) { 2030 DMA_PRIV_UNLOCK(priv); 2031 return (0); 2032 } 2033 2034 /* load all S/G list entries */ 2035 for_each_sg(sgl, sg, nents, i) { 2036 nseg = -1; 2037 if (_bus_dmamap_load_phys(priv->dmat, sgl->dma_map, 2038 sg_phys(sg), sg->length, BUS_DMA_NOWAIT, 2039 &seg, &nseg) != 0) { 2040 bus_dmamap_unload(priv->dmat, sgl->dma_map); 2041 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 2042 DMA_PRIV_UNLOCK(priv); 2043 return (0); 2044 } 2045 KASSERT(nseg == 0, 2046 ("More than one segment (nseg=%d)", nseg + 1)); 2047 2048 sg_dma_address(sg) = seg.ds_addr; 2049 } 2050 2051 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 2052 goto skip_sync; 2053 2054 switch (direction) { 2055 case DMA_BIDIRECTIONAL: 2056 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE); 2057 break; 2058 case DMA_TO_DEVICE: 2059 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREREAD); 2060 break; 2061 case DMA_FROM_DEVICE: 2062 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE); 2063 break; 2064 default: 2065 break; 2066 } 2067 skip_sync: 2068 2069 DMA_PRIV_UNLOCK(priv); 2070 2071 return (nents); 2072 } 2073 2074 void 2075 linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, 2076 int nents __unused, enum dma_data_direction direction, 2077 unsigned long attrs) 2078 { 2079 struct linux_dma_priv *priv; 2080 2081 priv = dev->dma_priv; 2082 2083 DMA_PRIV_LOCK(priv); 2084 2085 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 2086 goto skip_sync; 2087 2088 switch (direction) { 2089 case DMA_BIDIRECTIONAL: 2090 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD); 2091 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREREAD); 2092 break; 2093 case DMA_TO_DEVICE: 2094 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTWRITE); 2095 break; 2096 case DMA_FROM_DEVICE: 2097 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD); 2098 break; 2099 default: 2100 break; 2101 } 2102 skip_sync: 2103 2104 bus_dmamap_unload(priv->dmat, sgl->dma_map); 2105 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 2106 DMA_PRIV_UNLOCK(priv); 2107 } 2108 2109 struct dma_pool { 2110 struct device *pool_device; 2111 uma_zone_t pool_zone; 2112 struct mtx pool_lock; 2113 bus_dma_tag_t pool_dmat; 2114 size_t pool_entry_size; 2115 struct pctrie pool_ptree; 2116 }; 2117 2118 #define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock) 2119 #define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock) 2120 2121 static inline int 2122 dma_pool_obj_ctor(void *mem, int size, void *arg, int flags) 2123 { 2124 struct linux_dma_obj *obj = mem; 2125 struct dma_pool *pool = arg; 2126 int error, nseg; 2127 bus_dma_segment_t seg; 2128 2129 nseg = -1; 2130 DMA_POOL_LOCK(pool); 2131 error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap, 2132 vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT, 2133 &seg, &nseg); 2134 DMA_POOL_UNLOCK(pool); 2135 if (error != 0) { 2136 return (error); 2137 } 2138 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 2139 obj->dma_addr = seg.ds_addr; 2140 2141 return (0); 2142 } 2143 2144 static void 2145 dma_pool_obj_dtor(void *mem, int size, void *arg) 2146 { 2147 struct linux_dma_obj *obj = mem; 2148 struct dma_pool *pool = arg; 2149 2150 DMA_POOL_LOCK(pool); 2151 bus_dmamap_unload(pool->pool_dmat, obj->dmamap); 2152 DMA_POOL_UNLOCK(pool); 2153 } 2154 2155 static int 2156 dma_pool_obj_import(void *arg, void **store, int count, int domain __unused, 2157 int flags) 2158 { 2159 struct dma_pool *pool = arg; 2160 struct linux_dma_obj *obj; 2161 int error, i; 2162 2163 for (i = 0; i < count; i++) { 2164 obj = uma_zalloc(linux_dma_obj_zone, flags); 2165 if (obj == NULL) 2166 break; 2167 2168 error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr, 2169 BUS_DMA_NOWAIT, &obj->dmamap); 2170 if (error!= 0) { 2171 uma_zfree(linux_dma_obj_zone, obj); 2172 break; 2173 } 2174 2175 store[i] = obj; 2176 } 2177 2178 return (i); 2179 } 2180 2181 static void 2182 dma_pool_obj_release(void *arg, void **store, int count) 2183 { 2184 struct dma_pool *pool = arg; 2185 struct linux_dma_obj *obj; 2186 int i; 2187 2188 for (i = 0; i < count; i++) { 2189 obj = store[i]; 2190 bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap); 2191 uma_zfree(linux_dma_obj_zone, obj); 2192 } 2193 } 2194 2195 struct dma_pool * 2196 linux_dma_pool_create(char *name, struct device *dev, size_t size, 2197 size_t align, size_t boundary) 2198 { 2199 struct linux_dma_priv *priv; 2200 struct dma_pool *pool; 2201 2202 priv = dev->dma_priv; 2203 2204 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 2205 pool->pool_device = dev; 2206 pool->pool_entry_size = size; 2207 2208 if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 2209 align, boundary, /* alignment, boundary */ 2210 priv->dma_mask, /* lowaddr */ 2211 BUS_SPACE_MAXADDR, /* highaddr */ 2212 NULL, NULL, /* filtfunc, filtfuncarg */ 2213 size, /* maxsize */ 2214 1, /* nsegments */ 2215 size, /* maxsegsz */ 2216 0, /* flags */ 2217 NULL, NULL, /* lockfunc, lockfuncarg */ 2218 &pool->pool_dmat)) { 2219 kfree(pool); 2220 return (NULL); 2221 } 2222 2223 pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor, 2224 dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import, 2225 dma_pool_obj_release, pool, 0); 2226 2227 mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF); 2228 pctrie_init(&pool->pool_ptree); 2229 2230 return (pool); 2231 } 2232 2233 void 2234 linux_dma_pool_destroy(struct dma_pool *pool) 2235 { 2236 2237 uma_zdestroy(pool->pool_zone); 2238 bus_dma_tag_destroy(pool->pool_dmat); 2239 mtx_destroy(&pool->pool_lock); 2240 kfree(pool); 2241 } 2242 2243 void 2244 lkpi_dmam_pool_destroy(struct device *dev, void *p) 2245 { 2246 struct dma_pool *pool; 2247 2248 pool = *(struct dma_pool **)p; 2249 LINUX_DMA_PCTRIE_RECLAIM(&pool->pool_ptree); 2250 linux_dma_pool_destroy(pool); 2251 } 2252 2253 void * 2254 linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, 2255 dma_addr_t *handle) 2256 { 2257 struct linux_dma_obj *obj; 2258 2259 obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags & GFP_NATIVE_MASK); 2260 if (obj == NULL) 2261 return (NULL); 2262 2263 DMA_POOL_LOCK(pool); 2264 if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) { 2265 DMA_POOL_UNLOCK(pool); 2266 uma_zfree_arg(pool->pool_zone, obj, pool); 2267 return (NULL); 2268 } 2269 DMA_POOL_UNLOCK(pool); 2270 2271 *handle = obj->dma_addr; 2272 return (obj->vaddr); 2273 } 2274 2275 void 2276 linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr) 2277 { 2278 struct linux_dma_obj *obj; 2279 2280 DMA_POOL_LOCK(pool); 2281 obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr); 2282 if (obj == NULL) { 2283 DMA_POOL_UNLOCK(pool); 2284 return; 2285 } 2286 LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr); 2287 DMA_POOL_UNLOCK(pool); 2288 2289 uma_zfree_arg(pool->pool_zone, obj, pool); 2290 } 2291 2292 static int 2293 linux_backlight_get_status(device_t dev, struct backlight_props *props) 2294 { 2295 struct pci_dev *pdev; 2296 2297 linux_set_current(curthread); 2298 pdev = device_get_softc(dev); 2299 2300 props->brightness = pdev->dev.bd->props.brightness; 2301 props->brightness = props->brightness * 100 / pdev->dev.bd->props.max_brightness; 2302 props->nlevels = 0; 2303 2304 return (0); 2305 } 2306 2307 static int 2308 linux_backlight_get_info(device_t dev, struct backlight_info *info) 2309 { 2310 struct pci_dev *pdev; 2311 2312 linux_set_current(curthread); 2313 pdev = device_get_softc(dev); 2314 2315 info->type = BACKLIGHT_TYPE_PANEL; 2316 strlcpy(info->name, pdev->dev.bd->name, BACKLIGHTMAXNAMELENGTH); 2317 return (0); 2318 } 2319 2320 static int 2321 linux_backlight_update_status(device_t dev, struct backlight_props *props) 2322 { 2323 struct pci_dev *pdev; 2324 2325 linux_set_current(curthread); 2326 pdev = device_get_softc(dev); 2327 2328 pdev->dev.bd->props.brightness = pdev->dev.bd->props.max_brightness * 2329 props->brightness / 100; 2330 pdev->dev.bd->props.power = props->brightness == 0 ? 2331 4/* FB_BLANK_POWERDOWN */ : 0/* FB_BLANK_UNBLANK */; 2332 return (pdev->dev.bd->ops->update_status(pdev->dev.bd)); 2333 } 2334 2335 struct backlight_device * 2336 linux_backlight_device_register(const char *name, struct device *dev, 2337 void *data, const struct backlight_ops *ops, struct backlight_properties *props) 2338 { 2339 2340 dev->bd = malloc(sizeof(*dev->bd), M_DEVBUF, M_WAITOK | M_ZERO); 2341 dev->bd->ops = ops; 2342 dev->bd->props.type = props->type; 2343 dev->bd->props.max_brightness = props->max_brightness; 2344 dev->bd->props.brightness = props->brightness; 2345 dev->bd->props.power = props->power; 2346 dev->bd->data = data; 2347 dev->bd->dev = dev; 2348 dev->bd->name = strdup(name, M_DEVBUF); 2349 2350 dev->backlight_dev = backlight_register(name, dev->bsddev); 2351 2352 return (dev->bd); 2353 } 2354 2355 void 2356 linux_backlight_device_unregister(struct backlight_device *bd) 2357 { 2358 2359 backlight_destroy(bd->dev->backlight_dev); 2360 free(bd->name, M_DEVBUF); 2361 free(bd, M_DEVBUF); 2362 } 2363