1 /*- 2 * Copyright (c) 2015-2016 Mellanox Technologies, Ltd. 3 * All rights reserved. 4 * Copyright (c) 2020-2026 The FreeBSD Foundation 5 * 6 * Portions of this software were developed by Björn Zeeb 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 /* 32 * We have two ways to create a pci_dev (pdev): 33 * (1) coming from the device_attach DEVMETHOD, and 34 * (2) the other from manual creation via lkpinew_pci_dev(). 35 * 36 * Only devices from (1) end up on our LinuxKPI global pci_devices list. 37 * All others are "place fillers" -- XXX if only "place filler" was always true. 38 */ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/bus.h> 43 #include <sys/malloc.h> 44 #include <sys/kernel.h> 45 #include <sys/sysctl.h> 46 #include <sys/lock.h> 47 #include <sys/mutex.h> 48 #include <sys/fcntl.h> 49 #include <sys/file.h> 50 #include <sys/filio.h> 51 #include <sys/pciio.h> 52 #include <sys/pctrie.h> 53 #include <sys/rman.h> 54 #include <sys/rwlock.h> 55 #include <sys/stdarg.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 60 #include <machine/bus.h> 61 #include <machine/resource.h> 62 63 #include <dev/pci/pcivar.h> 64 #include <dev/pci/pci_private.h> 65 #include <dev/pci/pci_iov.h> 66 #include <dev/backlight/backlight.h> 67 68 #include <linux/kernel.h> 69 #include <linux/kobject.h> 70 #include <linux/device.h> 71 #include <linux/slab.h> 72 #include <linux/module.h> 73 #include <linux/cdev.h> 74 #include <linux/file.h> 75 #include <linux/sysfs.h> 76 #include <linux/mm.h> 77 #include <linux/io.h> 78 #include <linux/vmalloc.h> 79 #define WANT_NATIVE_PCI_GET_SLOT 80 #include <linux/pci.h> 81 #include <linux/compat.h> 82 83 #include <linux/backlight.h> 84 85 #include "backlight_if.h" 86 #include "pcib_if.h" 87 88 /* Undef the linux function macro defined in linux/pci.h */ 89 #undef pci_get_class 90 91 extern int linuxkpi_debug; 92 93 SYSCTL_DECL(_compat_linuxkpi); 94 95 static counter_u64_t lkpi_pci_nseg1_fail; 96 SYSCTL_COUNTER_U64(_compat_linuxkpi, OID_AUTO, lkpi_pci_nseg1_fail, CTLFLAG_RD, 97 &lkpi_pci_nseg1_fail, "Count of busdma mapping failures of single-segment"); 98 99 static device_probe_t linux_pci_probe; 100 static device_attach_t linux_pci_attach; 101 static device_detach_t linux_pci_detach; 102 static device_suspend_t linux_pci_suspend; 103 static device_resume_t linux_pci_resume; 104 static device_shutdown_t linux_pci_shutdown; 105 static pci_iov_init_t linux_pci_iov_init; 106 static pci_iov_uninit_t linux_pci_iov_uninit; 107 static pci_iov_add_vf_t linux_pci_iov_add_vf; 108 static int linux_backlight_get_status(device_t dev, struct backlight_props *props); 109 static int linux_backlight_update_status(device_t dev, struct backlight_props *props); 110 static int linux_backlight_get_info(device_t dev, struct backlight_info *info); 111 static void lkpi_pcim_iomap_table_release(struct device *, void *); 112 static void lkpinew_pci_dev_release(struct device *); 113 114 static device_method_t pci_methods[] = { 115 DEVMETHOD(device_probe, linux_pci_probe), 116 DEVMETHOD(device_attach, linux_pci_attach), 117 DEVMETHOD(device_detach, linux_pci_detach), 118 DEVMETHOD(device_suspend, linux_pci_suspend), 119 DEVMETHOD(device_resume, linux_pci_resume), 120 DEVMETHOD(device_shutdown, linux_pci_shutdown), 121 DEVMETHOD(pci_iov_init, linux_pci_iov_init), 122 DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit), 123 DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf), 124 125 /* Bus interface. */ 126 DEVMETHOD(bus_add_child, bus_generic_add_child), 127 128 /* backlight interface */ 129 DEVMETHOD(backlight_update_status, linux_backlight_update_status), 130 DEVMETHOD(backlight_get_status, linux_backlight_get_status), 131 DEVMETHOD(backlight_get_info, linux_backlight_get_info), 132 DEVMETHOD_END 133 }; 134 135 const char *pci_power_names[] = { 136 "UNKNOWN", "D0", "D1", "D2", "D3hot", "D3cold" 137 }; 138 139 /* We need some meta-struct to keep track of these for devres. */ 140 struct pci_devres { 141 bool enable_io; 142 /* PCIR_MAX_BAR_0 + 1 = 6 => BIT(0..5). */ 143 uint8_t region_mask; 144 struct resource *region_table[PCIR_MAX_BAR_0 + 1]; /* Not needed. */ 145 }; 146 struct pcim_iomap_devres { 147 void *mmio_table[PCIR_MAX_BAR_0 + 1]; 148 struct resource *res_table[PCIR_MAX_BAR_0 + 1]; 149 }; 150 151 struct linux_dma_priv { 152 uint64_t dma_mask; 153 bus_dma_tag_t dmat; 154 uint64_t dma_coherent_mask; 155 bus_dma_tag_t dmat_coherent; 156 struct mtx lock; 157 struct pctrie ptree; 158 }; 159 #define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock) 160 #define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock) 161 162 static void 163 lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres *dr, int bar, 164 void *res) 165 { 166 dr->mmio_table[bar] = (void *)rman_get_bushandle(res); 167 dr->res_table[bar] = res; 168 } 169 170 static bool 171 lkpi_pci_bar_id_valid(int bar) 172 { 173 if (bar < 0 || bar > PCIR_MAX_BAR_0) 174 return (false); 175 176 return (true); 177 } 178 179 static int 180 linux_pdev_dma_uninit(struct pci_dev *pdev) 181 { 182 struct linux_dma_priv *priv; 183 184 priv = pdev->dev.dma_priv; 185 if (priv->dmat) 186 bus_dma_tag_destroy(priv->dmat); 187 if (priv->dmat_coherent) 188 bus_dma_tag_destroy(priv->dmat_coherent); 189 mtx_destroy(&priv->lock); 190 pdev->dev.dma_priv = NULL; 191 free(priv, M_DEVBUF); 192 return (0); 193 } 194 195 static int 196 linux_pdev_dma_init(struct pci_dev *pdev) 197 { 198 struct linux_dma_priv *priv; 199 int error; 200 201 priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO); 202 203 mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF); 204 pctrie_init(&priv->ptree); 205 206 pdev->dev.dma_priv = priv; 207 208 /* Create a default DMA tags. */ 209 error = linux_dma_tag_init(&pdev->dev, DMA_BIT_MASK(64)); 210 if (error != 0) 211 goto err; 212 /* Coherent is lower 32bit only by default in Linux. */ 213 error = linux_dma_tag_init_coherent(&pdev->dev, DMA_BIT_MASK(32)); 214 if (error != 0) 215 goto err; 216 217 return (error); 218 219 err: 220 linux_pdev_dma_uninit(pdev); 221 return (error); 222 } 223 224 int 225 linux_dma_tag_init(struct device *dev, u64 dma_mask) 226 { 227 struct linux_dma_priv *priv; 228 int error; 229 230 priv = dev->dma_priv; 231 232 if (priv->dmat) { 233 if (priv->dma_mask == dma_mask) 234 return (0); 235 236 bus_dma_tag_destroy(priv->dmat); 237 } 238 239 priv->dma_mask = dma_mask; 240 241 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 242 1, 0, /* alignment, boundary */ 243 dma_mask, /* lowaddr */ 244 BUS_SPACE_MAXADDR, /* highaddr */ 245 NULL, NULL, /* filtfunc, filtfuncarg */ 246 BUS_SPACE_MAXSIZE, /* maxsize */ 247 1, /* nsegments */ 248 BUS_SPACE_MAXSIZE, /* maxsegsz */ 249 0, /* flags */ 250 NULL, NULL, /* lockfunc, lockfuncarg */ 251 &priv->dmat); 252 return (-error); 253 } 254 255 int 256 linux_dma_tag_init_coherent(struct device *dev, u64 dma_mask) 257 { 258 struct linux_dma_priv *priv; 259 int error; 260 261 priv = dev->dma_priv; 262 263 if (priv->dmat_coherent) { 264 if (priv->dma_coherent_mask == dma_mask) 265 return (0); 266 267 bus_dma_tag_destroy(priv->dmat_coherent); 268 } 269 270 priv->dma_coherent_mask = dma_mask; 271 272 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 273 1, 0, /* alignment, boundary */ 274 dma_mask, /* lowaddr */ 275 BUS_SPACE_MAXADDR, /* highaddr */ 276 NULL, NULL, /* filtfunc, filtfuncarg */ 277 BUS_SPACE_MAXSIZE, /* maxsize */ 278 1, /* nsegments */ 279 BUS_SPACE_MAXSIZE, /* maxsegsz */ 280 0, /* flags */ 281 NULL, NULL, /* lockfunc, lockfuncarg */ 282 &priv->dmat_coherent); 283 return (-error); 284 } 285 286 static struct pci_driver * 287 linux_pci_find(device_t dev, const struct pci_device_id **idp) 288 { 289 const struct pci_device_id *id; 290 struct pci_driver *pdrv; 291 uint16_t vendor; 292 uint16_t device; 293 uint16_t subvendor; 294 uint16_t subdevice; 295 296 vendor = pci_get_vendor(dev); 297 device = pci_get_device(dev); 298 subvendor = pci_get_subvendor(dev); 299 subdevice = pci_get_subdevice(dev); 300 301 spin_lock(&pci_lock); 302 list_for_each_entry(pdrv, &pci_drivers, node) { 303 for (id = pdrv->id_table; id->vendor != 0; id++) { 304 if (vendor == id->vendor && 305 (PCI_ANY_ID == id->device || device == id->device) && 306 (PCI_ANY_ID == id->subvendor || subvendor == id->subvendor) && 307 (PCI_ANY_ID == id->subdevice || subdevice == id->subdevice)) { 308 *idp = id; 309 spin_unlock(&pci_lock); 310 return (pdrv); 311 } 312 } 313 } 314 spin_unlock(&pci_lock); 315 return (NULL); 316 } 317 318 struct pci_dev * 319 lkpi_pci_get_device(uint32_t vendor, uint32_t device, struct pci_dev *odev) 320 { 321 struct pci_dev *pdev, *found, *odev0; 322 323 odev0 = odev; 324 found = NULL; 325 spin_lock(&pci_lock); 326 list_for_each_entry(pdev, &pci_devices, links) { 327 /* Walk until we find odev. */ 328 if (odev != NULL) { 329 if (pdev == odev) 330 odev = NULL; 331 continue; 332 } 333 334 if ((pdev->vendor == vendor || vendor == PCI_ANY_ID) && 335 (pdev->device == device || device == PCI_ANY_ID)) { 336 found = pdev; 337 break; 338 } 339 } 340 pci_dev_get(found); 341 spin_unlock(&pci_lock); 342 pci_dev_put(odev0); 343 344 return (found); 345 } 346 347 static void 348 lkpi_pci_dev_release(struct device *dev) 349 { 350 struct pci_dev *pdev; 351 352 /* 353 * Before anything else, we have to free all the dynamic 354 * resource which are on the devres list. 355 * Otherwise we risk that supporting infrastructure 356 * is gone and we panic 'randomly'. 357 */ 358 lkpi_devres_release_free_list(dev); 359 360 /* 361 * Now undo linux_pci_attach_device() in reverse-ish 362 * order. 363 */ 364 pdev = to_pci_dev(dev); 365 366 /* 367 * pdrv->remove happens before pci_put_dev() in 368 * linux_pci_detach_device(), which means the driver should have 369 * cleaned up before we get here; see irqents and mmio below. 370 */ 371 372 /* Clear the hierarchy recursively to root. */ 373 if (pdev->bus->self != pdev) { 374 pci_dev_put(pdev->bus->self); 375 pdev->bus->self = NULL; 376 } 377 378 if (pdev->root != NULL) { 379 lkpinew_pci_dev_release(&pdev->root->dev); /* pci_dev_put(pdev->root); ? */ 380 pdev->root = NULL; 381 } 382 383 spin_lock(&pci_lock); 384 list_del(&pdev->links); 385 spin_unlock(&pci_lock); 386 387 linux_pdev_dma_uninit(pdev); 388 389 /* irq? */ 390 391 /* Undo lkpifill_pci_dev(). */ 392 /* devres is gone already; went at the very top. */ 393 if (!list_empty_careful(&pdev->dev.irqents)) { 394 dev_warn(&pdev->dev, "%s: driver did not clean up; " 395 "leaking IRQs\n", __func__); 396 /* 397 * XXX add private function to interrupt.h/linux_interrupt.c 398 * to walk the list and call free_irq on each if we have to. 399 */ 400 } 401 402 spin_lock_destroy(&dev->devres_lock); 403 spin_lock_destroy(&pdev->pcie_cap_lock); 404 405 if (!TAILQ_EMPTY(&pdev->mmio)) { 406 dev_warn(&pdev->dev, "%s: driver did not clean up; " 407 "leaking mmio resources\n", __func__); 408 /* XXX we have two functions to walk and release in here. */ 409 } 410 411 if (pdev->msi_desc != NULL) { 412 for (int i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--) 413 free(pdev->msi_desc[i], M_DEVBUF); 414 free(pdev->msi_desc, M_DEVBUF); 415 } 416 417 free(pdev->bus, M_DEVBUF); 418 kfree(pdev->path_name); 419 420 /* 421 * Lastly, apply an internal hack in order to signal 422 * that this was run (device reference fully dropped). 423 * See comment in linux_pci_detach_device(). 424 */ 425 pdev->dev.release = NULL; 426 } 427 428 static int 429 lkpifill_pci_dev(device_t dev, struct pci_dev *pdev) 430 { 431 struct pci_devinfo *dinfo; 432 int error; 433 434 error = kobject_init_and_add(&pdev->dev.kobj, &linux_dev_ktype, 435 &linux_root_device.kobj, device_get_nameunit(dev)); 436 if (error != 0) { 437 printf("%s:%d: kobject_init_and_add returned %d\n", 438 __func__, __LINE__, error); 439 return (error); 440 } 441 442 pdev->devfn = PCI_DEVFN(pci_get_slot(dev), pci_get_function(dev)); 443 pdev->vendor = pci_get_vendor(dev); 444 pdev->device = pci_get_device(dev); 445 pdev->subsystem_vendor = pci_get_subvendor(dev); 446 pdev->subsystem_device = pci_get_subdevice(dev); 447 pdev->class = pci_get_class(dev); 448 pdev->revision = pci_get_revid(dev); 449 pdev->path_name = kasprintf(GFP_KERNEL, "%04d:%02d:%02d.%d", 450 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), 451 pci_get_function(dev)); 452 453 pdev->bus = malloc(sizeof(*pdev->bus), M_DEVBUF, M_WAITOK | M_ZERO); 454 pdev->bus->number = pci_get_bus(dev); 455 pdev->bus->domain = pci_get_domain(dev); 456 457 /* Check if we have reached the root to satisfy pci_is_root_bus() */ 458 dinfo = device_get_ivars(dev); 459 if (dinfo->cfg.pcie.pcie_location != 0 && 460 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) { 461 pdev->bus->self = NULL; 462 } else { 463 /* 464 * This should be the upstream bridge; pci_upstream_bridge() 465 * handles that case on demand as otherwise we'll shadow the 466 * entire PCI hierarchy. 467 */ 468 pdev->bus->self = pdev; 469 } 470 pdev->dev.bsddev = dev; 471 pdev->dev.parent = &linux_root_device; 472 pdev->dev.release = lkpi_pci_dev_release; 473 474 if (pci_msi_count(dev) > 0) 475 pdev->msi_desc = malloc(pci_msi_count(dev) * 476 sizeof(*pdev->msi_desc), M_DEVBUF, M_WAITOK | M_ZERO); 477 478 TAILQ_INIT(&pdev->mmio); 479 spin_lock_init(&pdev->pcie_cap_lock); 480 spin_lock_init(&pdev->dev.devres_lock); 481 INIT_LIST_HEAD(&pdev->dev.devres_head); 482 INIT_LIST_HEAD(&pdev->dev.irqents); 483 484 return (0); 485 } 486 487 static void 488 lkpinew_pci_dev_release(struct device *dev) 489 { 490 struct pci_dev *pdev; 491 int i; 492 493 pdev = to_pci_dev(dev); 494 if (pdev->root != NULL) 495 pci_dev_put(pdev->root); 496 if (pdev->bus->self != pdev && pdev->bus->self != NULL) 497 pci_dev_put(pdev->bus->self); 498 free(pdev->bus, M_DEVBUF); 499 if (pdev->msi_desc != NULL) { 500 for (i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--) 501 free(pdev->msi_desc[i], M_DEVBUF); 502 free(pdev->msi_desc, M_DEVBUF); 503 } 504 kfree(pdev->path_name); 505 free(pdev, M_DEVBUF); 506 } 507 508 struct pci_dev * 509 lkpinew_pci_dev(device_t dev) 510 { 511 struct pci_dev *pdev; 512 int error; 513 514 pdev = malloc(sizeof(*pdev), M_DEVBUF, M_WAITOK|M_ZERO); 515 error = lkpifill_pci_dev(dev, pdev); 516 if (error != 0) { 517 free(pdev, M_DEVBUF); 518 return (NULL); 519 } 520 pdev->dev.release = lkpinew_pci_dev_release; 521 522 return (pdev); 523 } 524 525 struct pci_dev * 526 lkpi_pci_get_class(unsigned int class, struct pci_dev *from) 527 { 528 device_t dev; 529 device_t devfrom = NULL; 530 struct pci_dev *pdev; 531 532 if (from != NULL) 533 devfrom = from->dev.bsddev; 534 535 dev = pci_find_class_from(class >> 16, (class >> 8) & 0xFF, devfrom); 536 if (dev == NULL) 537 return (NULL); 538 539 pdev = lkpinew_pci_dev(dev); 540 return (pdev); 541 } 542 543 struct pci_dev * 544 lkpi_pci_get_base_class(unsigned int baseclass, struct pci_dev *from) 545 { 546 device_t dev; 547 device_t devfrom = NULL; 548 struct pci_dev *pdev; 549 550 if (from != NULL) 551 devfrom = from->dev.bsddev; 552 553 dev = pci_find_base_class_from(baseclass, devfrom); 554 if (dev == NULL) 555 return (NULL); 556 557 pdev = lkpinew_pci_dev(dev); 558 return (pdev); 559 } 560 561 struct pci_dev * 562 lkpi_pci_get_domain_bus_and_slot(int domain, unsigned int bus, 563 unsigned int devfn) 564 { 565 device_t dev; 566 struct pci_dev *pdev; 567 568 dev = pci_find_dbsf(domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 569 if (dev == NULL) 570 return (NULL); 571 572 pdev = lkpinew_pci_dev(dev); 573 return (pdev); 574 } 575 576 struct pci_dev * 577 lkpi_pci_get_slot(struct pci_bus *pbus, unsigned int devfn) 578 { 579 device_t dev; 580 struct pci_dev *pdev; 581 582 dev = pci_find_bsf(pbus->number, PCI_SLOT(devfn), PCI_FUNC(devfn)); 583 if (dev == NULL) 584 return (NULL); 585 586 pdev = lkpinew_pci_dev(dev); 587 return (pdev); 588 } 589 590 static int 591 linux_pci_probe(device_t dev) 592 { 593 const struct pci_device_id *id; 594 struct pci_driver *pdrv; 595 596 if ((pdrv = linux_pci_find(dev, &id)) == NULL) 597 return (ENXIO); 598 if (device_get_driver(dev) != &pdrv->bsddriver) 599 return (ENXIO); 600 device_set_desc(dev, pdrv->name); 601 602 /* Assume BSS initialized (should never return BUS_PROBE_SPECIFIC). */ 603 if (pdrv->bsd_probe_return == 0) 604 return (BUS_PROBE_DEFAULT); 605 else 606 return (pdrv->bsd_probe_return); 607 } 608 609 static int 610 linux_pci_attach(device_t dev) 611 { 612 const struct pci_device_id *id; 613 struct pci_driver *pdrv; 614 struct pci_dev *pdev; 615 616 pdrv = linux_pci_find(dev, &id); 617 pdev = device_get_softc(dev); 618 619 MPASS(pdrv != NULL); 620 MPASS(pdev != NULL); 621 622 return (linux_pci_attach_device(dev, pdrv, id, pdev)); 623 } 624 625 static struct resource_list_entry * 626 linux_pci_reserve_bar(struct pci_dev *pdev, struct resource_list *rl, 627 int type, int rid) 628 { 629 device_t dev; 630 struct resource *res; 631 632 KASSERT(type == SYS_RES_IOPORT || type == SYS_RES_MEMORY, 633 ("trying to reserve non-BAR type %d", type)); 634 635 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? 636 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; 637 res = pci_reserve_map(device_get_parent(dev), dev, type, rid, 0, ~0, 638 1, 1, 0); 639 if (res == NULL) 640 return (NULL); 641 return (resource_list_find(rl, type, rid)); 642 } 643 644 static struct resource_list_entry * 645 linux_pci_get_rle(struct pci_dev *pdev, int type, int rid, bool reserve_bar) 646 { 647 struct pci_devinfo *dinfo; 648 struct resource_list *rl; 649 struct resource_list_entry *rle; 650 651 dinfo = device_get_ivars(pdev->dev.bsddev); 652 rl = &dinfo->resources; 653 rle = resource_list_find(rl, type, rid); 654 /* Reserve resources for this BAR if needed. */ 655 if (rle == NULL && reserve_bar) 656 rle = linux_pci_reserve_bar(pdev, rl, type, rid); 657 return (rle); 658 } 659 660 int 661 linux_pci_attach_device(device_t dev, struct pci_driver *pdrv, 662 const struct pci_device_id *id, struct pci_dev *pdev) 663 { 664 struct resource_list_entry *rle; 665 device_t parent; 666 struct pci_dev *pbus, *ppbus; 667 uintptr_t rid; 668 int error; 669 bool isdrm; 670 671 linux_set_current(curthread); 672 673 parent = device_get_parent(dev); 674 isdrm = pdrv != NULL && pdrv->isdrm; 675 676 if (isdrm) { 677 struct pci_devinfo *dinfo; 678 679 dinfo = device_get_ivars(parent); 680 device_set_ivars(dev, dinfo); 681 } 682 683 error = lkpifill_pci_dev(dev, pdev); 684 if (error != 0) 685 return (error); 686 687 if (isdrm) 688 PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid); 689 else 690 PCI_GET_ID(parent, dev, PCI_ID_RID, &rid); 691 pdev->devfn = rid; 692 pdev->pdrv = pdrv; 693 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0, false); 694 if (rle != NULL) 695 pdev->dev.irq = rle->start; 696 else 697 pdev->dev.irq = LINUX_IRQ_INVALID; 698 pdev->irq = pdev->dev.irq; 699 error = linux_pdev_dma_init(pdev); 700 if (error) 701 goto out_dma_init; 702 703 spin_lock(&pci_lock); 704 list_add(&pdev->links, &pci_devices); 705 spin_unlock(&pci_lock); 706 707 /* 708 * Create the hierarchy now as we cannot on demand later. 709 * Take special care of DRM as there is a non-PCI device in the chain. 710 */ 711 pbus = pdev; 712 if (isdrm) { 713 pbus = lkpinew_pci_dev(parent); 714 if (pbus == NULL) { 715 error = ENXIO; 716 goto out_dma_init; 717 } 718 } 719 pcie_find_root_port(pbus); 720 if (isdrm) 721 pdev->root = pbus->root; 722 ppbus = pci_upstream_bridge(pbus); 723 while (ppbus != NULL && ppbus != pbus) { 724 pbus = ppbus; 725 ppbus = pci_upstream_bridge(pbus); 726 } 727 728 if (pdrv != NULL) { 729 error = pdrv->probe(pdev, id); 730 if (error) 731 goto out_probe; 732 } 733 return (0); 734 735 /* XXX the cleanup does not match the allocation up there. */ 736 out_probe: 737 free(pdev->bus, M_DEVBUF); 738 spin_lock_destroy(&pdev->pcie_cap_lock); 739 linux_pdev_dma_uninit(pdev); 740 out_dma_init: 741 spin_lock(&pci_lock); 742 list_del(&pdev->links); 743 spin_unlock(&pci_lock); 744 put_device(&pdev->dev); 745 return (-error); 746 } 747 748 static int 749 linux_pci_detach(device_t dev) 750 { 751 struct pci_dev *pdev; 752 int error; 753 754 pdev = device_get_softc(dev); 755 MPASS(pdev != NULL); 756 757 error = linux_pci_detach_device(pdev); 758 if (error == 0) 759 device_set_desc(dev, NULL); 760 761 return (error); 762 } 763 764 int 765 linux_pci_detach_device(struct pci_dev *pdev) 766 { 767 768 linux_set_current(curthread); 769 770 /* 771 * We cannot do much here as almost everything will have 772 * to happen as the last reference to the LinuxKPI device 773 * goes away. That will call the release function, 774 * which lkpifill_pci_dev() set. That is were most 775 * of the cleanup will happen. But before that give 776 * the driver a chance to cleanup. 777 * The big problem is that the Linux KPI does not 778 * report back if it was the last kref (well kref 779 * does report back but then kobj, dev, pdev do not). 780 * So we have little way of knowing if the release 781 * happened or not. We have to play tricks for that 782 * and we can given the softc (pdev) is still valid 783 * until we return from here. 784 */ 785 786 if (pdev->pdrv != NULL) 787 pdev->pdrv->remove(pdev); 788 789 pci_dev_put(pdev); 790 791 /* 792 * We (ab)use the release function as a guard to 793 * know if we made it there and the device is gone. 794 */ 795 if (pdev->dev.release != lkpi_pci_dev_release) 796 return (0); 797 798 /* 799 * Detach failed. 800 * We need to re-acquire the ref and wait for 801 * the other refs to be gone... In theory this 802 * should never happen, so log it! 803 * XXX I wish there was a KPI to query the ref. 804 * 805 * If we do not error and wait, we will have a 806 * LinuxKPI device dangling active with pointers 807 * but the FreeBSD device_t will be 'gone'. 808 */ 809 device_printf(pdev->dev.bsddev, "%s failed due to %u other pending " 810 "references on the LinuxKPI device.\n", __func__, 811 kref_read(&pdev->dev.kobj.kref)); 812 pci_dev_get(pdev); 813 814 return (EBUSY); 815 } 816 817 static int 818 lkpi_pci_disable_dev(struct device *dev) 819 { 820 821 (void) pci_disable_io(dev->bsddev, SYS_RES_MEMORY); 822 (void) pci_disable_io(dev->bsddev, SYS_RES_IOPORT); 823 return (0); 824 } 825 826 static struct pci_devres * 827 lkpi_pci_devres_get_alloc(struct pci_dev *pdev) 828 { 829 struct pci_devres *dr; 830 831 dr = lkpi_devres_find(&pdev->dev, lkpi_pci_devres_release, NULL, NULL); 832 if (dr == NULL) { 833 dr = lkpi_devres_alloc(lkpi_pci_devres_release, sizeof(*dr), 834 GFP_KERNEL | __GFP_ZERO); 835 if (dr != NULL) 836 lkpi_devres_add(&pdev->dev, dr); 837 } 838 839 return (dr); 840 } 841 842 static struct pci_devres * 843 lkpi_pci_devres_find(struct pci_dev *pdev) 844 { 845 if (!pdev->managed) 846 return (NULL); 847 848 return (lkpi_pci_devres_get_alloc(pdev)); 849 } 850 851 void 852 lkpi_pci_devres_release(struct device *dev, void *p) 853 { 854 struct pci_devres *dr; 855 struct pci_dev *pdev; 856 int bar; 857 858 pdev = to_pci_dev(dev); 859 dr = p; 860 861 if (pdev->msix_enabled) 862 lkpi_pci_disable_msix(pdev); 863 if (pdev->msi_enabled) 864 lkpi_pci_disable_msi(pdev); 865 866 if (dr->enable_io && lkpi_pci_disable_dev(dev) == 0) 867 dr->enable_io = false; 868 869 if (dr->region_mask == 0) 870 return; 871 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 872 873 if ((dr->region_mask & (1 << bar)) == 0) 874 continue; 875 pci_release_region(pdev, bar); 876 } 877 } 878 879 int 880 linuxkpi_pcim_enable_device(struct pci_dev *pdev) 881 { 882 struct pci_devres *dr; 883 int error; 884 885 /* Here we cannot run through the pdev->managed check. */ 886 dr = lkpi_pci_devres_get_alloc(pdev); 887 if (dr == NULL) 888 return (-ENOMEM); 889 890 /* If resources were enabled before do not do it again. */ 891 if (dr->enable_io) 892 return (0); 893 894 error = pci_enable_device(pdev); 895 if (error == 0) 896 dr->enable_io = true; 897 898 /* This device is not managed. */ 899 pdev->managed = true; 900 901 return (error); 902 } 903 904 static struct pcim_iomap_devres * 905 lkpi_pcim_iomap_devres_find(struct pci_dev *pdev) 906 { 907 struct pcim_iomap_devres *dr; 908 909 dr = lkpi_devres_find(&pdev->dev, lkpi_pcim_iomap_table_release, 910 NULL, NULL); 911 if (dr == NULL) { 912 dr = lkpi_devres_alloc(lkpi_pcim_iomap_table_release, 913 sizeof(*dr), GFP_KERNEL | __GFP_ZERO); 914 if (dr != NULL) 915 lkpi_devres_add(&pdev->dev, dr); 916 } 917 918 if (dr == NULL) 919 device_printf(pdev->dev.bsddev, "%s: NULL\n", __func__); 920 921 return (dr); 922 } 923 924 void __iomem ** 925 linuxkpi_pcim_iomap_table(struct pci_dev *pdev) 926 { 927 struct pcim_iomap_devres *dr; 928 929 dr = lkpi_pcim_iomap_devres_find(pdev); 930 if (dr == NULL) 931 return (NULL); 932 933 /* 934 * If the driver has manually set a flag to be able to request the 935 * resource to use bus_read/write_<n>, return the shadow table. 936 */ 937 if (pdev->want_iomap_res) 938 return ((void **)dr->res_table); 939 940 /* This is the Linux default. */ 941 return (dr->mmio_table); 942 } 943 944 static struct resource * 945 _lkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen __unused) 946 { 947 struct pci_mmio_region *mmio, *p; 948 int type; 949 950 if (!lkpi_pci_bar_id_valid(bar)) 951 return (NULL); 952 953 type = pci_resource_type(pdev, bar); 954 if (type < 0) { 955 device_printf(pdev->dev.bsddev, "%s: bar %d type %d\n", 956 __func__, bar, type); 957 return (NULL); 958 } 959 960 /* 961 * Check for duplicate mappings. 962 * This can happen if a driver calls pci_request_region() first. 963 */ 964 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 965 if (mmio->type == type && mmio->rid == PCIR_BAR(bar)) { 966 return (mmio->res); 967 } 968 } 969 970 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO); 971 mmio->rid = PCIR_BAR(bar); 972 mmio->type = type; 973 mmio->res = bus_alloc_resource_any(pdev->dev.bsddev, mmio->type, 974 &mmio->rid, RF_ACTIVE|RF_SHAREABLE); 975 if (mmio->res == NULL) { 976 device_printf(pdev->dev.bsddev, "%s: failed to alloc " 977 "bar %d type %d rid %d\n", 978 __func__, bar, type, PCIR_BAR(bar)); 979 free(mmio, M_DEVBUF); 980 return (NULL); 981 } 982 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next); 983 984 return (mmio->res); 985 } 986 987 void * 988 linuxkpi_pci_iomap_range(struct pci_dev *pdev, int bar, 989 unsigned long off, unsigned long maxlen) 990 { 991 struct resource *res; 992 993 if (!lkpi_pci_bar_id_valid(bar)) 994 return (NULL); 995 996 res = _lkpi_pci_iomap(pdev, bar, maxlen); 997 if (res == NULL) 998 return (NULL); 999 /* This is a FreeBSD extension so we can use bus_*(). */ 1000 if (pdev->want_iomap_res) 1001 return (res); 1002 MPASS(off < rman_get_size(res)); 1003 return ((void *)(rman_get_bushandle(res) + off)); 1004 } 1005 1006 void * 1007 linuxkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen) 1008 { 1009 if (!lkpi_pci_bar_id_valid(bar)) 1010 return (NULL); 1011 1012 return (linuxkpi_pci_iomap_range(pdev, bar, 0, maxlen)); 1013 } 1014 1015 void * 1016 linuxkpi_pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen) 1017 { 1018 struct pcim_iomap_devres *dr; 1019 void *res; 1020 1021 if (!lkpi_pci_bar_id_valid(bar)) 1022 return (NULL); 1023 1024 dr = lkpi_pcim_iomap_devres_find(pdev); 1025 if (dr == NULL) 1026 return (NULL); 1027 1028 if (dr->res_table[bar] != NULL) 1029 return (dr->res_table[bar]); 1030 1031 res = linuxkpi_pci_iomap(pdev, bar, maxlen); 1032 if (res == NULL) { 1033 /* 1034 * Do not free the devres in case there were 1035 * other valid mappings before already. 1036 */ 1037 return (NULL); 1038 } 1039 lkpi_set_pcim_iomap_devres(dr, bar, res); 1040 1041 return (res); 1042 } 1043 1044 void 1045 linuxkpi_pci_iounmap(struct pci_dev *pdev, void *res) 1046 { 1047 struct pci_mmio_region *mmio, *p; 1048 bus_space_handle_t bh = (bus_space_handle_t)res; 1049 1050 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 1051 if (pdev->want_iomap_res) { 1052 if (res != mmio->res) 1053 continue; 1054 } else { 1055 if (bh < rman_get_bushandle(mmio->res) || 1056 bh >= rman_get_bushandle(mmio->res) + 1057 rman_get_size(mmio->res)) 1058 continue; 1059 } 1060 bus_release_resource(pdev->dev.bsddev, 1061 mmio->type, mmio->rid, mmio->res); 1062 TAILQ_REMOVE(&pdev->mmio, mmio, next); 1063 free(mmio, M_DEVBUF); 1064 return; 1065 } 1066 } 1067 1068 int 1069 linuxkpi_pcim_iomap_regions(struct pci_dev *pdev, uint32_t mask, const char *name) 1070 { 1071 struct pcim_iomap_devres *dr; 1072 void *res; 1073 uint32_t mappings; 1074 int bar; 1075 1076 dr = lkpi_pcim_iomap_devres_find(pdev); 1077 if (dr == NULL) 1078 return (-ENOMEM); 1079 1080 /* Now iomap all the requested (by "mask") ones. */ 1081 for (bar = mappings = 0; mappings != mask; bar++) { 1082 if ((mask & (1 << bar)) == 0) 1083 continue; 1084 1085 /* Request double is not allowed. */ 1086 if (dr->mmio_table[bar] != NULL) { 1087 device_printf(pdev->dev.bsddev, "%s: bar %d %p\n", 1088 __func__, bar, dr->mmio_table[bar]); 1089 goto err; 1090 } 1091 1092 res = _lkpi_pci_iomap(pdev, bar, 0); 1093 if (res == NULL) 1094 goto err; 1095 lkpi_set_pcim_iomap_devres(dr, bar, res); 1096 1097 mappings |= (1 << bar); 1098 } 1099 1100 return (0); 1101 err: 1102 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 1103 if ((mappings & (1 << bar)) != 0) { 1104 res = dr->mmio_table[bar]; 1105 if (res == NULL) 1106 continue; 1107 pci_iounmap(pdev, res); 1108 } 1109 } 1110 1111 return (-EINVAL); 1112 } 1113 1114 static void 1115 lkpi_pcim_iomap_table_release(struct device *dev, void *p) 1116 { 1117 struct pcim_iomap_devres *dr; 1118 struct pci_dev *pdev; 1119 int bar; 1120 1121 dr = p; 1122 pdev = to_pci_dev(dev); 1123 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 1124 1125 if (dr->mmio_table[bar] == NULL) 1126 continue; 1127 1128 pci_iounmap(pdev, dr->mmio_table[bar]); 1129 } 1130 } 1131 1132 static int 1133 linux_pci_suspend(device_t dev) 1134 { 1135 const struct dev_pm_ops *pmops; 1136 struct pm_message pm = { }; 1137 struct pci_dev *pdev; 1138 int error; 1139 1140 error = 0; 1141 linux_set_current(curthread); 1142 pdev = device_get_softc(dev); 1143 pmops = pdev->pdrv->driver.pm; 1144 1145 if (pdev->pdrv->suspend != NULL) 1146 error = -pdev->pdrv->suspend(pdev, pm); 1147 else if (pmops != NULL && pmops->suspend != NULL) { 1148 error = -pmops->suspend(&pdev->dev); 1149 if (error == 0 && pmops->suspend_late != NULL) 1150 error = -pmops->suspend_late(&pdev->dev); 1151 if (error == 0 && pmops->suspend_noirq != NULL) 1152 error = -pmops->suspend_noirq(&pdev->dev); 1153 } 1154 return (error); 1155 } 1156 1157 static int 1158 linux_pci_resume(device_t dev) 1159 { 1160 const struct dev_pm_ops *pmops; 1161 struct pci_dev *pdev; 1162 int error; 1163 1164 error = 0; 1165 linux_set_current(curthread); 1166 pdev = device_get_softc(dev); 1167 pmops = pdev->pdrv->driver.pm; 1168 1169 if (pdev->pdrv->resume != NULL) 1170 error = -pdev->pdrv->resume(pdev); 1171 else if (pmops != NULL && pmops->resume != NULL) { 1172 if (pmops->resume_early != NULL) 1173 error = -pmops->resume_early(&pdev->dev); 1174 if (error == 0 && pmops->resume != NULL) 1175 error = -pmops->resume(&pdev->dev); 1176 } 1177 return (error); 1178 } 1179 1180 static int 1181 linux_pci_shutdown(device_t dev) 1182 { 1183 struct pci_dev *pdev; 1184 1185 linux_set_current(curthread); 1186 pdev = device_get_softc(dev); 1187 if (pdev->pdrv->shutdown != NULL) 1188 pdev->pdrv->shutdown(pdev); 1189 return (0); 1190 } 1191 1192 static int 1193 linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config) 1194 { 1195 struct pci_dev *pdev; 1196 int error; 1197 1198 linux_set_current(curthread); 1199 pdev = device_get_softc(dev); 1200 if (pdev->pdrv->bsd_iov_init != NULL) 1201 error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config); 1202 else 1203 error = EINVAL; 1204 return (error); 1205 } 1206 1207 static void 1208 linux_pci_iov_uninit(device_t dev) 1209 { 1210 struct pci_dev *pdev; 1211 1212 linux_set_current(curthread); 1213 pdev = device_get_softc(dev); 1214 if (pdev->pdrv->bsd_iov_uninit != NULL) 1215 pdev->pdrv->bsd_iov_uninit(dev); 1216 } 1217 1218 static int 1219 linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config) 1220 { 1221 struct pci_dev *pdev; 1222 int error; 1223 1224 linux_set_current(curthread); 1225 pdev = device_get_softc(dev); 1226 if (pdev->pdrv->bsd_iov_add_vf != NULL) 1227 error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config); 1228 else 1229 error = EINVAL; 1230 return (error); 1231 } 1232 1233 static int 1234 _linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc) 1235 { 1236 int error; 1237 1238 linux_set_current(curthread); 1239 spin_lock(&pci_lock); 1240 list_add(&pdrv->node, &pci_drivers); 1241 spin_unlock(&pci_lock); 1242 if (pdrv->bsddriver.name == NULL) 1243 pdrv->bsddriver.name = pdrv->name; 1244 pdrv->bsddriver.methods = pci_methods; 1245 pdrv->bsddriver.size = sizeof(struct pci_dev); 1246 1247 bus_topo_lock(); 1248 error = devclass_add_driver(dc, &pdrv->bsddriver, 1249 BUS_PASS_DEFAULT, &pdrv->bsdclass); 1250 bus_topo_unlock(); 1251 return (-error); 1252 } 1253 1254 int 1255 linux_pci_register_driver(struct pci_driver *pdrv) 1256 { 1257 devclass_t dc; 1258 1259 pdrv->isdrm = strcmp(pdrv->name, "drmn") == 0; 1260 dc = pdrv->isdrm ? devclass_create("vgapci") : devclass_find("pci"); 1261 if (dc == NULL) 1262 return (-ENXIO); 1263 return (_linux_pci_register_driver(pdrv, dc)); 1264 } 1265 1266 static struct resource_list_entry * 1267 lkpi_pci_get_bar(struct pci_dev *pdev, int bar, bool reserve) 1268 { 1269 int type; 1270 1271 type = pci_resource_type(pdev, bar); 1272 if (type < 0) 1273 return (NULL); 1274 bar = PCIR_BAR(bar); 1275 return (linux_pci_get_rle(pdev, type, bar, reserve)); 1276 } 1277 1278 struct device * 1279 lkpi_pci_find_irq_dev(unsigned int irq) 1280 { 1281 struct pci_dev *pdev; 1282 struct device *found; 1283 1284 found = NULL; 1285 spin_lock(&pci_lock); 1286 list_for_each_entry(pdev, &pci_devices, links) { 1287 if (irq == pdev->dev.irq || 1288 (irq >= pdev->dev.irq_start && irq < pdev->dev.irq_end)) { 1289 found = &pdev->dev; 1290 break; 1291 } 1292 } 1293 spin_unlock(&pci_lock); 1294 return (found); 1295 } 1296 1297 unsigned long 1298 pci_resource_start(struct pci_dev *pdev, int bar) 1299 { 1300 struct resource_list_entry *rle; 1301 rman_res_t newstart; 1302 device_t dev; 1303 int error; 1304 1305 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL) 1306 return (0); 1307 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? 1308 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; 1309 error = bus_translate_resource(dev, rle->type, rle->start, &newstart); 1310 if (error != 0) { 1311 device_printf(pdev->dev.bsddev, 1312 "translate of %#jx failed: %d\n", 1313 (uintmax_t)rle->start, error); 1314 return (0); 1315 } 1316 return (newstart); 1317 } 1318 1319 unsigned long 1320 pci_resource_len(struct pci_dev *pdev, int bar) 1321 { 1322 struct resource_list_entry *rle; 1323 1324 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL) 1325 return (0); 1326 return (rle->count); 1327 } 1328 1329 static int 1330 lkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name, 1331 bool managed) 1332 { 1333 struct resource *res; 1334 struct pci_devres *dr; 1335 struct pci_mmio_region *mmio; 1336 int rid; 1337 int type; 1338 1339 if (!lkpi_pci_bar_id_valid(bar)) 1340 return (-EINVAL); 1341 1342 type = pci_resource_type(pdev, bar); 1343 if (type < 0) 1344 return (0); 1345 1346 rid = PCIR_BAR(bar); 1347 res = bus_alloc_resource_any(pdev->dev.bsddev, type, &rid, 1348 RF_ACTIVE|RF_SHAREABLE); 1349 if (res == NULL) { 1350 device_printf(pdev->dev.bsddev, "%s: failed to alloc " 1351 "bar %d type %d rid %d\n", 1352 __func__, bar, type, PCIR_BAR(bar)); 1353 return (-EBUSY); 1354 } 1355 1356 /* 1357 * It seems there is an implicit devres tracking on these if the device 1358 * is managed (lkpi_pci_devres_find() case); otherwise the resources are 1359 * not automatically freed on FreeBSD/LinuxKPI though they should be/are 1360 * expected to be by Linux drivers. 1361 * Otherwise if we are called from a pcim-function with the managed 1362 * argument set, we need to track devres independent of pdev->managed. 1363 */ 1364 if (managed) 1365 dr = lkpi_pci_devres_get_alloc(pdev); 1366 else 1367 dr = lkpi_pci_devres_find(pdev); 1368 if (dr != NULL) { 1369 dr->region_mask |= (1 << bar); 1370 dr->region_table[bar] = res; 1371 } 1372 1373 /* Even if the device is not managed we need to track it for iomap. */ 1374 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO); 1375 mmio->rid = PCIR_BAR(bar); 1376 mmio->type = type; 1377 mmio->res = res; 1378 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next); 1379 1380 return (0); 1381 } 1382 1383 int 1384 linuxkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) 1385 { 1386 return (lkpi_pci_request_region(pdev, bar, res_name, false)); 1387 } 1388 1389 int 1390 linuxkpi_pci_request_regions(struct pci_dev *pdev, const char *res_name) 1391 { 1392 int error; 1393 int i; 1394 1395 for (i = 0; i <= PCIR_MAX_BAR_0; i++) { 1396 error = pci_request_region(pdev, i, res_name); 1397 if (error && error != -EBUSY) { 1398 pci_release_regions(pdev); 1399 return (error); 1400 } 1401 } 1402 return (0); 1403 } 1404 1405 int 1406 linuxkpi_pcim_request_all_regions(struct pci_dev *pdev, const char *res_name) 1407 { 1408 int bar, error; 1409 1410 for (bar = 0; bar <= PCIR_MAX_BAR_0; bar++) { 1411 error = lkpi_pci_request_region(pdev, bar, res_name, true); 1412 if (error != 0 && error != -EBUSY) { 1413 device_printf(pdev->dev.bsddev, "%s: bar %d res_name '%s': " 1414 "lkpi_pci_request_region returned %d\n", __func__, 1415 bar, res_name, error); 1416 pci_release_regions(pdev); 1417 return (error); 1418 } 1419 } 1420 return (0); 1421 } 1422 1423 void 1424 linuxkpi_pci_release_region(struct pci_dev *pdev, int bar) 1425 { 1426 struct resource_list_entry *rle; 1427 struct pci_devres *dr; 1428 struct pci_mmio_region *mmio, *p; 1429 1430 if ((rle = lkpi_pci_get_bar(pdev, bar, false)) == NULL) 1431 return; 1432 1433 /* 1434 * As we implicitly track the requests we also need to clear them on 1435 * release. Do clear before resource release. 1436 */ 1437 dr = lkpi_pci_devres_find(pdev); 1438 if (dr != NULL) { 1439 KASSERT(dr->region_table[bar] == rle->res, ("%s: pdev %p bar %d" 1440 " region_table res %p != rel->res %p\n", __func__, pdev, 1441 bar, dr->region_table[bar], rle->res)); 1442 dr->region_table[bar] = NULL; 1443 dr->region_mask &= ~(1 << bar); 1444 } 1445 1446 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 1447 if (rle->res != (void *)rman_get_bushandle(mmio->res)) 1448 continue; 1449 TAILQ_REMOVE(&pdev->mmio, mmio, next); 1450 free(mmio, M_DEVBUF); 1451 } 1452 1453 bus_release_resource(pdev->dev.bsddev, rle->type, rle->rid, rle->res); 1454 } 1455 1456 void 1457 linuxkpi_pci_release_regions(struct pci_dev *pdev) 1458 { 1459 int i; 1460 1461 for (i = 0; i <= PCIR_MAX_BAR_0; i++) 1462 pci_release_region(pdev, i); 1463 } 1464 1465 int 1466 linux_pci_register_drm_driver(struct pci_driver *pdrv) 1467 { 1468 devclass_t dc; 1469 1470 dc = devclass_create("vgapci"); 1471 if (dc == NULL) 1472 return (-ENXIO); 1473 pdrv->isdrm = true; 1474 pdrv->name = "drmn"; 1475 return (_linux_pci_register_driver(pdrv, dc)); 1476 } 1477 1478 void 1479 linux_pci_unregister_driver(struct pci_driver *pdrv) 1480 { 1481 devclass_t bus; 1482 1483 bus = devclass_find(pdrv->isdrm ? "vgapci" : "pci"); 1484 1485 spin_lock(&pci_lock); 1486 list_del(&pdrv->node); 1487 spin_unlock(&pci_lock); 1488 bus_topo_lock(); 1489 if (bus != NULL) 1490 devclass_delete_driver(bus, &pdrv->bsddriver); 1491 bus_topo_unlock(); 1492 } 1493 1494 void 1495 linux_pci_unregister_drm_driver(struct pci_driver *pdrv) 1496 { 1497 devclass_t bus; 1498 1499 bus = devclass_find("vgapci"); 1500 1501 spin_lock(&pci_lock); 1502 list_del(&pdrv->node); 1503 spin_unlock(&pci_lock); 1504 bus_topo_lock(); 1505 if (bus != NULL) 1506 devclass_delete_driver(bus, &pdrv->bsddriver); 1507 bus_topo_unlock(); 1508 } 1509 1510 int 1511 linuxkpi_pci_enable_msix(struct pci_dev *pdev, struct msix_entry *entries, 1512 int nreq) 1513 { 1514 struct resource_list_entry *rle; 1515 int error; 1516 int avail; 1517 int i; 1518 1519 avail = pci_msix_count(pdev->dev.bsddev); 1520 if (avail < nreq) { 1521 if (avail == 0) 1522 return -EINVAL; 1523 return avail; 1524 } 1525 avail = nreq; 1526 if ((error = -pci_alloc_msix(pdev->dev.bsddev, &avail)) != 0) 1527 return error; 1528 /* 1529 * Handle case where "pci_alloc_msix()" may allocate less 1530 * interrupts than available and return with no error: 1531 */ 1532 if (avail < nreq) { 1533 pci_release_msi(pdev->dev.bsddev); 1534 return avail; 1535 } 1536 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); 1537 pdev->dev.irq_start = rle->start; 1538 pdev->dev.irq_end = rle->start + avail; 1539 for (i = 0; i < nreq; i++) 1540 entries[i].vector = pdev->dev.irq_start + i; 1541 pdev->msix_enabled = true; 1542 return (0); 1543 } 1544 1545 int 1546 _lkpi_pci_enable_msi_range(struct pci_dev *pdev, int minvec, int maxvec) 1547 { 1548 struct resource_list_entry *rle; 1549 int error; 1550 int nvec; 1551 1552 if (maxvec < minvec) 1553 return (-EINVAL); 1554 1555 nvec = pci_msi_count(pdev->dev.bsddev); 1556 if (nvec < 1 || nvec < minvec) 1557 return (-ENOSPC); 1558 1559 nvec = min(nvec, maxvec); 1560 if ((error = -pci_alloc_msi(pdev->dev.bsddev, &nvec)) != 0) 1561 return error; 1562 1563 /* Native PCI might only ever ask for 32 vectors. */ 1564 if (nvec < minvec) { 1565 pci_release_msi(pdev->dev.bsddev); 1566 return (-ENOSPC); 1567 } 1568 1569 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); 1570 pdev->dev.irq_start = rle->start; 1571 pdev->dev.irq_end = rle->start + nvec; 1572 pdev->irq = rle->start; 1573 pdev->msi_enabled = true; 1574 return (0); 1575 } 1576 1577 int 1578 pci_alloc_irq_vectors(struct pci_dev *pdev, int minv, int maxv, 1579 unsigned int flags) 1580 { 1581 int error; 1582 1583 if (flags & PCI_IRQ_MSIX) { 1584 struct msix_entry *entries; 1585 int i; 1586 1587 entries = kcalloc(maxv, sizeof(*entries), GFP_KERNEL); 1588 if (entries == NULL) { 1589 error = -ENOMEM; 1590 goto out; 1591 } 1592 for (i = 0; i < maxv; ++i) 1593 entries[i].entry = i; 1594 error = pci_enable_msix(pdev, entries, maxv); 1595 out: 1596 kfree(entries); 1597 if (error == 0 && pdev->msix_enabled) 1598 return (pdev->dev.irq_end - pdev->dev.irq_start); 1599 } 1600 if (flags & PCI_IRQ_MSI) { 1601 if (pci_msi_count(pdev->dev.bsddev) < minv) 1602 return (-ENOSPC); 1603 error = _lkpi_pci_enable_msi_range(pdev, minv, maxv); 1604 if (error == 0 && pdev->msi_enabled) 1605 return (pdev->dev.irq_end - pdev->dev.irq_start); 1606 } 1607 if (flags & PCI_IRQ_INTX) { 1608 if (pdev->irq) 1609 return (1); 1610 } 1611 1612 return (-EINVAL); 1613 } 1614 1615 struct msi_desc * 1616 lkpi_pci_msi_desc_alloc(int irq) 1617 { 1618 struct device *dev; 1619 struct pci_dev *pdev; 1620 struct msi_desc *desc; 1621 struct pci_devinfo *dinfo; 1622 struct pcicfg_msi *msi; 1623 int vec; 1624 1625 dev = lkpi_pci_find_irq_dev(irq); 1626 if (dev == NULL) 1627 return (NULL); 1628 1629 pdev = to_pci_dev(dev); 1630 1631 if (pdev->msi_desc == NULL) 1632 return (NULL); 1633 1634 if (irq < pdev->dev.irq_start || irq >= pdev->dev.irq_end) 1635 return (NULL); 1636 1637 vec = pdev->dev.irq_start - irq; 1638 1639 if (pdev->msi_desc[vec] != NULL) 1640 return (pdev->msi_desc[vec]); 1641 1642 dinfo = device_get_ivars(dev->bsddev); 1643 msi = &dinfo->cfg.msi; 1644 1645 desc = malloc(sizeof(*desc), M_DEVBUF, M_WAITOK | M_ZERO); 1646 1647 desc->pci.msi_attrib.is_64 = 1648 (msi->msi_ctrl & PCIM_MSICTRL_64BIT) ? true : false; 1649 desc->msg.data = msi->msi_data; 1650 1651 pdev->msi_desc[vec] = desc; 1652 1653 return (desc); 1654 } 1655 1656 bool 1657 pci_device_is_present(struct pci_dev *pdev) 1658 { 1659 device_t dev; 1660 1661 dev = pdev->dev.bsddev; 1662 1663 return (bus_child_present(dev)); 1664 } 1665 1666 void * 1667 linuxkpi_pci_map_rom(struct pci_dev *pdev, size_t *size) 1668 { 1669 device_t dev; 1670 1671 dev = pdev->dev.bsddev; 1672 1673 if (pci_get_class(dev) != PCIC_DISPLAY && 1674 (pci_get_class(dev) != PCIC_OLD || 1675 pci_get_subclass(dev) != PCIS_OLD_VGA)) { 1676 pr_debug("%s: TODO\n", __func__); 1677 return (NULL); 1678 } 1679 1680 return (vga_pci_map_bios(device_get_parent(dev), size)); 1681 } 1682 1683 void 1684 linuxkpi_pci_unmap_rom(struct pci_dev *pdev, void *rom) 1685 { 1686 device_t dev; 1687 1688 dev = pdev->dev.bsddev; 1689 1690 vga_pci_unmap_bios(device_get_parent(dev), rom); 1691 } 1692 1693 CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t)); 1694 1695 struct linux_dma_obj { 1696 void *vaddr; 1697 uint64_t dma_addr; 1698 bus_dmamap_t dmamap; 1699 bus_dma_tag_t dmat; 1700 }; 1701 1702 static uma_zone_t linux_dma_trie_zone; 1703 static uma_zone_t linux_dma_obj_zone; 1704 1705 static void 1706 linux_dma_init(void *arg) 1707 { 1708 1709 linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie", 1710 pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL, 1711 UMA_ALIGN_PTR, 0); 1712 linux_dma_obj_zone = uma_zcreate("linux_dma_object", 1713 sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL, 1714 UMA_ALIGN_PTR, 0); 1715 lkpi_pci_nseg1_fail = counter_u64_alloc(M_WAITOK); 1716 } 1717 SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL); 1718 1719 static void 1720 linux_dma_uninit(void *arg) 1721 { 1722 1723 counter_u64_free(lkpi_pci_nseg1_fail); 1724 uma_zdestroy(linux_dma_obj_zone); 1725 uma_zdestroy(linux_dma_trie_zone); 1726 } 1727 SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL); 1728 1729 static void * 1730 linux_dma_trie_alloc(struct pctrie *ptree) 1731 { 1732 1733 return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT)); 1734 } 1735 1736 static void 1737 linux_dma_trie_free(struct pctrie *ptree, void *node) 1738 { 1739 1740 uma_zfree(linux_dma_trie_zone, node); 1741 } 1742 1743 PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, 1744 linux_dma_trie_free); 1745 1746 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 1747 static dma_addr_t 1748 linux_dma_map_phys_common(struct device *dev, vm_paddr_t phys, size_t len, 1749 bus_dma_tag_t dmat) 1750 { 1751 struct linux_dma_priv *priv; 1752 struct linux_dma_obj *obj; 1753 int error, nseg; 1754 bus_dma_segment_t seg; 1755 1756 priv = dev->dma_priv; 1757 1758 /* 1759 * If the resultant mapping will be entirely 1:1 with the 1760 * physical address, short-circuit the remainder of the 1761 * bus_dma API. This avoids tracking collisions in the pctrie 1762 * with the additional benefit of reducing overhead. 1763 */ 1764 if (bus_dma_id_mapped(dmat, phys, len)) 1765 return (phys); 1766 1767 obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT); 1768 if (obj == NULL) { 1769 return (0); 1770 } 1771 obj->dmat = dmat; 1772 1773 DMA_PRIV_LOCK(priv); 1774 if (bus_dmamap_create(obj->dmat, 0, &obj->dmamap) != 0) { 1775 DMA_PRIV_UNLOCK(priv); 1776 uma_zfree(linux_dma_obj_zone, obj); 1777 return (0); 1778 } 1779 1780 nseg = -1; 1781 error = _bus_dmamap_load_phys(obj->dmat, obj->dmamap, phys, len, 1782 BUS_DMA_NOWAIT, &seg, &nseg); 1783 if (error != 0) { 1784 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1785 DMA_PRIV_UNLOCK(priv); 1786 uma_zfree(linux_dma_obj_zone, obj); 1787 counter_u64_add(lkpi_pci_nseg1_fail, 1); 1788 if (linuxkpi_debug) { 1789 device_printf(dev->bsddev, "%s: _bus_dmamap_load_phys " 1790 "error %d, phys %#018jx len %zu\n", __func__, 1791 error, (uintmax_t)phys, len); 1792 dump_stack(); 1793 } 1794 return (0); 1795 } 1796 1797 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 1798 obj->dma_addr = seg.ds_addr; 1799 1800 error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); 1801 if (error != 0) { 1802 bus_dmamap_unload(obj->dmat, obj->dmamap); 1803 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1804 DMA_PRIV_UNLOCK(priv); 1805 uma_zfree(linux_dma_obj_zone, obj); 1806 return (0); 1807 } 1808 DMA_PRIV_UNLOCK(priv); 1809 return (obj->dma_addr); 1810 } 1811 #else 1812 static dma_addr_t 1813 linux_dma_map_phys_common(struct device *dev __unused, vm_paddr_t phys, 1814 size_t len __unused, bus_dma_tag_t dmat __unused) 1815 { 1816 return (phys); 1817 } 1818 #endif 1819 1820 dma_addr_t 1821 lkpi_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len, 1822 enum dma_data_direction direction, unsigned long attrs) 1823 { 1824 struct linux_dma_priv *priv; 1825 dma_addr_t dma; 1826 1827 priv = dev->dma_priv; 1828 dma = linux_dma_map_phys_common(dev, phys, len, priv->dmat); 1829 if (dma_mapping_error(dev, dma)) 1830 return (dma); 1831 1832 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 1833 dma_sync_single_for_device(dev, dma, len, direction); 1834 1835 return (dma); 1836 } 1837 1838 /* For backward compat only so we can MFC this. Remove before 15. */ 1839 dma_addr_t 1840 linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) 1841 { 1842 return (lkpi_dma_map_phys(dev, phys, len, DMA_NONE, 0)); 1843 } 1844 1845 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 1846 void 1847 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, 1848 enum dma_data_direction direction, unsigned long attrs) 1849 { 1850 struct linux_dma_priv *priv; 1851 struct linux_dma_obj *obj; 1852 1853 priv = dev->dma_priv; 1854 1855 if (pctrie_is_empty(&priv->ptree)) 1856 return; 1857 1858 DMA_PRIV_LOCK(priv); 1859 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); 1860 if (obj == NULL) { 1861 DMA_PRIV_UNLOCK(priv); 1862 return; 1863 } 1864 LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); 1865 1866 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 1867 goto skip_sync; 1868 1869 /* dma_sync_single_for_cpu() unrolled to avoid lock recursicn. */ 1870 switch (direction) { 1871 case DMA_BIDIRECTIONAL: 1872 bus_dmamap_sync(obj->dmat, obj->dmamap, 1873 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1874 break; 1875 case DMA_TO_DEVICE: 1876 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_POSTWRITE); 1877 break; 1878 case DMA_FROM_DEVICE: 1879 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_POSTREAD); 1880 break; 1881 default: 1882 break; 1883 } 1884 1885 skip_sync: 1886 bus_dmamap_unload(obj->dmat, obj->dmamap); 1887 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1888 DMA_PRIV_UNLOCK(priv); 1889 1890 uma_zfree(linux_dma_obj_zone, obj); 1891 } 1892 #else 1893 void 1894 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, 1895 enum dma_data_direction direction, unsigned long attrs) 1896 { 1897 } 1898 #endif 1899 1900 /* For backward compat only so we can MFC this. Remove before 15. */ 1901 void 1902 linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) 1903 { 1904 lkpi_dma_unmap(dev, dma_addr, len, DMA_NONE, 0); 1905 } 1906 1907 void * 1908 linux_dma_alloc_coherent(struct device *dev, size_t size, 1909 dma_addr_t *dma_handle, gfp_t flag) 1910 { 1911 struct linux_dma_priv *priv; 1912 vm_paddr_t high; 1913 size_t align; 1914 void *mem; 1915 1916 if (dev == NULL || dev->dma_priv == NULL) { 1917 *dma_handle = 0; 1918 return (NULL); 1919 } 1920 priv = dev->dma_priv; 1921 if (priv->dma_coherent_mask) 1922 high = priv->dma_coherent_mask; 1923 else 1924 /* Coherent is lower 32bit only by default in Linux. */ 1925 high = BUS_SPACE_MAXADDR_32BIT; 1926 align = PAGE_SIZE << get_order(size); 1927 /* Always zero the allocation. */ 1928 flag |= M_ZERO; 1929 mem = kmem_alloc_contig(size, flag & GFP_NATIVE_MASK, 0, high, 1930 align, 0, VM_MEMATTR_DEFAULT); 1931 if (mem != NULL) { 1932 *dma_handle = linux_dma_map_phys_common(dev, vtophys(mem), size, 1933 priv->dmat_coherent); 1934 if (*dma_handle == 0) { 1935 kmem_free(mem, size); 1936 mem = NULL; 1937 } 1938 } else { 1939 *dma_handle = 0; 1940 } 1941 return (mem); 1942 } 1943 1944 struct lkpi_devres_dmam_coherent { 1945 size_t size; 1946 dma_addr_t handle; 1947 void *mem; 1948 }; 1949 1950 static void 1951 lkpi_dmam_free_coherent(struct device *dev, void *p) 1952 { 1953 struct lkpi_devres_dmam_coherent *dr; 1954 1955 dr = p; 1956 dma_free_coherent(dev, dr->size, dr->mem, dr->handle); 1957 } 1958 1959 static int 1960 lkpi_dmam_coherent_match(struct device *dev, void *dr, void *mp) 1961 { 1962 struct lkpi_devres_dmam_coherent *a, *b; 1963 1964 a = dr; 1965 b = mp; 1966 1967 if (a->mem != b->mem) 1968 return (0); 1969 if (a->size != b->size || a->handle != b->handle) 1970 dev_WARN(dev, "for mem %p: size %zu != %zu || handle %#jx != %#jx\n", 1971 a->mem, a->size, b->size, 1972 (uintmax_t)a->handle, (uintmax_t)b->handle); 1973 return (1); 1974 } 1975 1976 void 1977 linuxkpi_dmam_free_coherent(struct device *dev, size_t size, 1978 void *addr, dma_addr_t dma_handle) 1979 { 1980 struct lkpi_devres_dmam_coherent match = { 1981 .size = size, 1982 .handle = dma_handle, 1983 .mem = addr 1984 }; 1985 int error; 1986 1987 error = devres_destroy(dev, lkpi_dmam_free_coherent, 1988 lkpi_dmam_coherent_match, &match); 1989 if (error != 0) 1990 dev_WARN(dev, "devres_destroy returned %d, size %zu addr %p " 1991 "dma_handle %#jx\n", error, size, addr, (uintmax_t)dma_handle); 1992 dma_free_coherent(dev, size, addr, dma_handle); 1993 } 1994 1995 void * 1996 linuxkpi_dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, 1997 gfp_t flag) 1998 { 1999 struct lkpi_devres_dmam_coherent *dr; 2000 2001 dr = lkpi_devres_alloc(lkpi_dmam_free_coherent, 2002 sizeof(*dr), GFP_KERNEL | __GFP_ZERO); 2003 2004 if (dr == NULL) 2005 return (NULL); 2006 2007 dr->size = size; 2008 dr->mem = linux_dma_alloc_coherent(dev, size, dma_handle, flag); 2009 dr->handle = *dma_handle; 2010 if (dr->mem == NULL) { 2011 lkpi_devres_free(dr); 2012 return (NULL); 2013 } 2014 2015 lkpi_devres_add(dev, dr); 2016 return (dr->mem); 2017 } 2018 2019 void 2020 linuxkpi_dma_sync(struct device *dev, dma_addr_t dma_addr, size_t size, 2021 bus_dmasync_op_t op) 2022 { 2023 struct linux_dma_priv *priv; 2024 struct linux_dma_obj *obj; 2025 2026 priv = dev->dma_priv; 2027 2028 if (pctrie_is_empty(&priv->ptree)) 2029 return; 2030 2031 DMA_PRIV_LOCK(priv); 2032 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); 2033 if (obj == NULL) { 2034 DMA_PRIV_UNLOCK(priv); 2035 return; 2036 } 2037 2038 bus_dmamap_sync(obj->dmat, obj->dmamap, op); 2039 DMA_PRIV_UNLOCK(priv); 2040 } 2041 2042 void 2043 lkpi_dma_sync_sg(struct device *dev, struct scatterlist *sgl, bus_dmasync_op_t op) 2044 { 2045 struct linux_dma_priv *priv; 2046 2047 priv = dev->dma_priv; 2048 DMA_PRIV_LOCK(priv); 2049 bus_dmamap_sync(priv->dmat, sgl->dma_map, op); 2050 DMA_PRIV_UNLOCK(priv); 2051 } 2052 2053 int 2054 linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, 2055 enum dma_data_direction direction, unsigned long attrs) 2056 { 2057 struct linux_dma_priv *priv; 2058 struct scatterlist *sg; 2059 int i, nseg; 2060 bus_dma_segment_t seg; 2061 2062 priv = dev->dma_priv; 2063 2064 DMA_PRIV_LOCK(priv); 2065 2066 /* create common DMA map in the first S/G entry */ 2067 if (bus_dmamap_create(priv->dmat, 0, &sgl->dma_map) != 0) { 2068 DMA_PRIV_UNLOCK(priv); 2069 return (0); 2070 } 2071 2072 /* load all S/G list entries */ 2073 for_each_sg(sgl, sg, nents, i) { 2074 nseg = -1; 2075 if (_bus_dmamap_load_phys(priv->dmat, sgl->dma_map, 2076 sg_phys(sg), sg->length, BUS_DMA_NOWAIT, 2077 &seg, &nseg) != 0) { 2078 bus_dmamap_unload(priv->dmat, sgl->dma_map); 2079 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 2080 DMA_PRIV_UNLOCK(priv); 2081 return (0); 2082 } 2083 KASSERT(nseg == 0, 2084 ("More than one segment (nseg=%d)", nseg + 1)); 2085 2086 sg_dma_address(sg) = seg.ds_addr; 2087 } 2088 2089 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 2090 goto skip_sync; 2091 2092 switch (direction) { 2093 case DMA_BIDIRECTIONAL: 2094 bus_dmamap_sync(priv->dmat, sgl->dma_map, 2095 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2096 break; 2097 case DMA_TO_DEVICE: 2098 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE); 2099 break; 2100 case DMA_FROM_DEVICE: 2101 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREREAD); 2102 break; 2103 default: 2104 break; 2105 } 2106 skip_sync: 2107 2108 DMA_PRIV_UNLOCK(priv); 2109 2110 return (nents); 2111 } 2112 2113 void 2114 linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, 2115 int nents __unused, enum dma_data_direction direction, 2116 unsigned long attrs) 2117 { 2118 struct linux_dma_priv *priv; 2119 2120 priv = dev->dma_priv; 2121 2122 DMA_PRIV_LOCK(priv); 2123 2124 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 2125 goto skip_sync; 2126 2127 switch (direction) { 2128 case DMA_BIDIRECTIONAL: 2129 bus_dmamap_sync(priv->dmat, sgl->dma_map, 2130 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2131 break; 2132 case DMA_TO_DEVICE: 2133 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTWRITE); 2134 break; 2135 case DMA_FROM_DEVICE: 2136 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD); 2137 break; 2138 default: 2139 break; 2140 } 2141 skip_sync: 2142 2143 bus_dmamap_unload(priv->dmat, sgl->dma_map); 2144 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 2145 DMA_PRIV_UNLOCK(priv); 2146 } 2147 2148 struct dma_pool { 2149 struct device *pool_device; 2150 uma_zone_t pool_zone; 2151 struct mtx pool_lock; 2152 bus_dma_tag_t pool_dmat; 2153 size_t pool_entry_size; 2154 struct pctrie pool_ptree; 2155 }; 2156 2157 #define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock) 2158 #define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock) 2159 2160 static inline int 2161 dma_pool_obj_ctor(void *mem, int size, void *arg, int flags) 2162 { 2163 struct linux_dma_obj *obj = mem; 2164 struct dma_pool *pool = arg; 2165 int error, nseg; 2166 bus_dma_segment_t seg; 2167 2168 nseg = -1; 2169 DMA_POOL_LOCK(pool); 2170 error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap, 2171 vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT, 2172 &seg, &nseg); 2173 DMA_POOL_UNLOCK(pool); 2174 if (error != 0) { 2175 return (error); 2176 } 2177 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 2178 obj->dma_addr = seg.ds_addr; 2179 2180 return (0); 2181 } 2182 2183 static void 2184 dma_pool_obj_dtor(void *mem, int size, void *arg) 2185 { 2186 struct linux_dma_obj *obj = mem; 2187 struct dma_pool *pool = arg; 2188 2189 DMA_POOL_LOCK(pool); 2190 bus_dmamap_unload(pool->pool_dmat, obj->dmamap); 2191 DMA_POOL_UNLOCK(pool); 2192 } 2193 2194 static int 2195 dma_pool_obj_import(void *arg, void **store, int count, int domain __unused, 2196 int flags) 2197 { 2198 struct dma_pool *pool = arg; 2199 struct linux_dma_obj *obj; 2200 int error, i; 2201 2202 for (i = 0; i < count; i++) { 2203 obj = uma_zalloc(linux_dma_obj_zone, flags); 2204 if (obj == NULL) 2205 break; 2206 2207 error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr, 2208 BUS_DMA_NOWAIT, &obj->dmamap); 2209 if (error!= 0) { 2210 uma_zfree(linux_dma_obj_zone, obj); 2211 break; 2212 } 2213 2214 store[i] = obj; 2215 } 2216 2217 return (i); 2218 } 2219 2220 static void 2221 dma_pool_obj_release(void *arg, void **store, int count) 2222 { 2223 struct dma_pool *pool = arg; 2224 struct linux_dma_obj *obj; 2225 int i; 2226 2227 for (i = 0; i < count; i++) { 2228 obj = store[i]; 2229 bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap); 2230 uma_zfree(linux_dma_obj_zone, obj); 2231 } 2232 } 2233 2234 struct dma_pool * 2235 linux_dma_pool_create(char *name, struct device *dev, size_t size, 2236 size_t align, size_t boundary) 2237 { 2238 struct linux_dma_priv *priv; 2239 struct dma_pool *pool; 2240 2241 priv = dev->dma_priv; 2242 2243 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 2244 pool->pool_device = dev; 2245 pool->pool_entry_size = size; 2246 2247 if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 2248 align, boundary, /* alignment, boundary */ 2249 priv->dma_mask, /* lowaddr */ 2250 BUS_SPACE_MAXADDR, /* highaddr */ 2251 NULL, NULL, /* filtfunc, filtfuncarg */ 2252 size, /* maxsize */ 2253 1, /* nsegments */ 2254 size, /* maxsegsz */ 2255 0, /* flags */ 2256 NULL, NULL, /* lockfunc, lockfuncarg */ 2257 &pool->pool_dmat)) { 2258 kfree(pool); 2259 return (NULL); 2260 } 2261 2262 pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor, 2263 dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import, 2264 dma_pool_obj_release, pool, 0); 2265 2266 mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF); 2267 pctrie_init(&pool->pool_ptree); 2268 2269 return (pool); 2270 } 2271 2272 void 2273 linux_dma_pool_destroy(struct dma_pool *pool) 2274 { 2275 2276 uma_zdestroy(pool->pool_zone); 2277 bus_dma_tag_destroy(pool->pool_dmat); 2278 mtx_destroy(&pool->pool_lock); 2279 kfree(pool); 2280 } 2281 2282 void 2283 lkpi_dmam_pool_destroy(struct device *dev, void *p) 2284 { 2285 struct dma_pool *pool; 2286 2287 pool = *(struct dma_pool **)p; 2288 LINUX_DMA_PCTRIE_RECLAIM(&pool->pool_ptree); 2289 linux_dma_pool_destroy(pool); 2290 } 2291 2292 void * 2293 linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, 2294 dma_addr_t *handle) 2295 { 2296 struct linux_dma_obj *obj; 2297 2298 obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags & GFP_NATIVE_MASK); 2299 if (obj == NULL) 2300 return (NULL); 2301 2302 DMA_POOL_LOCK(pool); 2303 if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) { 2304 DMA_POOL_UNLOCK(pool); 2305 uma_zfree_arg(pool->pool_zone, obj, pool); 2306 return (NULL); 2307 } 2308 DMA_POOL_UNLOCK(pool); 2309 2310 *handle = obj->dma_addr; 2311 return (obj->vaddr); 2312 } 2313 2314 void 2315 linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr) 2316 { 2317 struct linux_dma_obj *obj; 2318 2319 DMA_POOL_LOCK(pool); 2320 obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr); 2321 if (obj == NULL) { 2322 DMA_POOL_UNLOCK(pool); 2323 return; 2324 } 2325 LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr); 2326 DMA_POOL_UNLOCK(pool); 2327 2328 uma_zfree_arg(pool->pool_zone, obj, pool); 2329 } 2330 2331 static int 2332 linux_backlight_get_status(device_t dev, struct backlight_props *props) 2333 { 2334 struct pci_dev *pdev; 2335 2336 linux_set_current(curthread); 2337 pdev = device_get_softc(dev); 2338 2339 props->brightness = pdev->dev.bd->props.brightness; 2340 props->brightness = props->brightness * 100 / pdev->dev.bd->props.max_brightness; 2341 props->nlevels = 0; 2342 2343 return (0); 2344 } 2345 2346 static int 2347 linux_backlight_get_info(device_t dev, struct backlight_info *info) 2348 { 2349 struct pci_dev *pdev; 2350 2351 linux_set_current(curthread); 2352 pdev = device_get_softc(dev); 2353 2354 info->type = BACKLIGHT_TYPE_PANEL; 2355 strlcpy(info->name, pdev->dev.bd->name, BACKLIGHTMAXNAMELENGTH); 2356 return (0); 2357 } 2358 2359 static int 2360 linux_backlight_update_status(device_t dev, struct backlight_props *props) 2361 { 2362 struct pci_dev *pdev; 2363 2364 linux_set_current(curthread); 2365 pdev = device_get_softc(dev); 2366 2367 pdev->dev.bd->props.brightness = pdev->dev.bd->props.max_brightness * 2368 props->brightness / 100; 2369 pdev->dev.bd->props.power = props->brightness == 0 ? 2370 4/* FB_BLANK_POWERDOWN */ : 0/* FB_BLANK_UNBLANK */; 2371 return (pdev->dev.bd->ops->update_status(pdev->dev.bd)); 2372 } 2373 2374 struct backlight_device * 2375 linux_backlight_device_register(const char *name, struct device *dev, 2376 void *data, const struct backlight_ops *ops, struct backlight_properties *props) 2377 { 2378 2379 dev->bd = malloc(sizeof(*dev->bd), M_DEVBUF, M_WAITOK | M_ZERO); 2380 dev->bd->ops = ops; 2381 dev->bd->props.type = props->type; 2382 dev->bd->props.max_brightness = props->max_brightness; 2383 dev->bd->props.brightness = props->brightness; 2384 dev->bd->props.power = props->power; 2385 dev->bd->data = data; 2386 dev->bd->dev = dev; 2387 dev->bd->name = strdup(name, M_DEVBUF); 2388 2389 dev->backlight_dev = backlight_register(name, dev->bsddev); 2390 2391 return (dev->bd); 2392 } 2393 2394 void 2395 linux_backlight_device_unregister(struct backlight_device *bd) 2396 { 2397 2398 backlight_destroy(bd->dev->backlight_dev); 2399 free(bd->name, M_DEVBUF); 2400 free(bd, M_DEVBUF); 2401 } 2402