1 /*- 2 * Copyright (c) 2015-2016 Mellanox Technologies, Ltd. 3 * All rights reserved. 4 * Copyright (c) 2020-2026 The FreeBSD Foundation 5 * 6 * Portions of this software were developed by Björn Zeeb 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 /* 32 * We have two ways to create a pci_dev (pdev): 33 * (1) coming from the device_attach DEVMETHOD, and 34 * (2) the other from manual creation via lkpinew_pci_dev(). 35 * 36 * Only devices from (1) end up on our LinuxKPI global pci_devices list. 37 * All others are "place fillers" -- XXX if only "place filler" was always true. 38 */ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/bus.h> 43 #include <sys/malloc.h> 44 #include <sys/kernel.h> 45 #include <sys/sysctl.h> 46 #include <sys/lock.h> 47 #include <sys/mutex.h> 48 #include <sys/fcntl.h> 49 #include <sys/file.h> 50 #include <sys/filio.h> 51 #include <sys/pciio.h> 52 #include <sys/pctrie.h> 53 #include <sys/rman.h> 54 #include <sys/rwlock.h> 55 #include <sys/stdarg.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 60 #include <machine/bus.h> 61 #include <machine/resource.h> 62 63 #include <dev/pci/pcivar.h> 64 #include <dev/pci/pci_private.h> 65 #include <dev/pci/pci_iov.h> 66 #include <dev/backlight/backlight.h> 67 68 #include <linux/kernel.h> 69 #include <linux/kobject.h> 70 #include <linux/device.h> 71 #include <linux/slab.h> 72 #include <linux/module.h> 73 #include <linux/cdev.h> 74 #include <linux/file.h> 75 #include <linux/sysfs.h> 76 #include <linux/mm.h> 77 #include <linux/io.h> 78 #include <linux/vmalloc.h> 79 #define WANT_NATIVE_PCI_GET_SLOT 80 #include <linux/pci.h> 81 #include <linux/compat.h> 82 83 #include <linux/backlight.h> 84 85 #include "backlight_if.h" 86 #include "pcib_if.h" 87 88 /* Undef the linux function macro defined in linux/pci.h */ 89 #undef pci_get_class 90 91 extern int linuxkpi_debug; 92 93 SYSCTL_DECL(_compat_linuxkpi); 94 95 static counter_u64_t lkpi_pci_nseg1_fail; 96 SYSCTL_COUNTER_U64(_compat_linuxkpi, OID_AUTO, lkpi_pci_nseg1_fail, CTLFLAG_RD, 97 &lkpi_pci_nseg1_fail, "Count of busdma mapping failures of single-segment"); 98 99 static device_probe_t linux_pci_probe; 100 static device_attach_t linux_pci_attach; 101 static device_detach_t linux_pci_detach; 102 static device_suspend_t linux_pci_suspend; 103 static device_resume_t linux_pci_resume; 104 static device_shutdown_t linux_pci_shutdown; 105 static pci_iov_init_t linux_pci_iov_init; 106 static pci_iov_uninit_t linux_pci_iov_uninit; 107 static pci_iov_add_vf_t linux_pci_iov_add_vf; 108 static int linux_backlight_get_status(device_t dev, struct backlight_props *props); 109 static int linux_backlight_update_status(device_t dev, struct backlight_props *props); 110 static int linux_backlight_get_info(device_t dev, struct backlight_info *info); 111 static void lkpi_pcim_iomap_table_release(struct device *, void *); 112 static void lkpinew_pci_dev_release(struct device *); 113 114 static device_method_t pci_methods[] = { 115 DEVMETHOD(device_probe, linux_pci_probe), 116 DEVMETHOD(device_attach, linux_pci_attach), 117 DEVMETHOD(device_detach, linux_pci_detach), 118 DEVMETHOD(device_suspend, linux_pci_suspend), 119 DEVMETHOD(device_resume, linux_pci_resume), 120 DEVMETHOD(device_shutdown, linux_pci_shutdown), 121 DEVMETHOD(pci_iov_init, linux_pci_iov_init), 122 DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit), 123 DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf), 124 125 /* Bus interface. */ 126 DEVMETHOD(bus_add_child, bus_generic_add_child), 127 128 /* backlight interface */ 129 DEVMETHOD(backlight_update_status, linux_backlight_update_status), 130 DEVMETHOD(backlight_get_status, linux_backlight_get_status), 131 DEVMETHOD(backlight_get_info, linux_backlight_get_info), 132 DEVMETHOD_END 133 }; 134 135 const char *pci_power_names[] = { 136 "UNKNOWN", "D0", "D1", "D2", "D3hot", "D3cold" 137 }; 138 139 /* We need some meta-struct to keep track of these for devres. */ 140 struct pci_devres { 141 bool enable_io; 142 /* PCIR_MAX_BAR_0 + 1 = 6 => BIT(0..5). */ 143 uint8_t region_mask; 144 struct resource *region_table[PCIR_MAX_BAR_0 + 1]; /* Not needed. */ 145 }; 146 struct pcim_iomap_devres { 147 void *mmio_table[PCIR_MAX_BAR_0 + 1]; 148 struct resource *res_table[PCIR_MAX_BAR_0 + 1]; 149 }; 150 151 struct linux_dma_priv { 152 uint64_t dma_mask; 153 bus_dma_tag_t dmat; 154 uint64_t dma_coherent_mask; 155 bus_dma_tag_t dmat_coherent; 156 struct mtx lock; 157 struct pctrie ptree; 158 }; 159 #define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock) 160 #define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock) 161 162 static void 163 lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres *dr, int bar, 164 void *res) 165 { 166 dr->mmio_table[bar] = (void *)rman_get_bushandle(res); 167 dr->res_table[bar] = res; 168 } 169 170 static bool 171 lkpi_pci_bar_id_valid(int bar) 172 { 173 if (bar < 0 || bar > PCIR_MAX_BAR_0) 174 return (false); 175 176 return (true); 177 } 178 179 static int 180 linux_pdev_dma_uninit(struct pci_dev *pdev) 181 { 182 struct linux_dma_priv *priv; 183 184 priv = pdev->dev.dma_priv; 185 if (priv == NULL) 186 return (0); 187 if (priv->dmat) 188 bus_dma_tag_destroy(priv->dmat); 189 if (priv->dmat_coherent) 190 bus_dma_tag_destroy(priv->dmat_coherent); 191 mtx_destroy(&priv->lock); 192 pdev->dev.dma_priv = NULL; 193 free(priv, M_DEVBUF); 194 return (0); 195 } 196 197 static int 198 linux_pdev_dma_init(struct pci_dev *pdev) 199 { 200 struct linux_dma_priv *priv; 201 int error; 202 203 priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO); 204 205 mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF); 206 pctrie_init(&priv->ptree); 207 208 pdev->dev.dma_priv = priv; 209 210 /* Create a default DMA tags. */ 211 error = linux_dma_tag_init(&pdev->dev, DMA_BIT_MASK(64)); 212 if (error != 0) 213 goto err; 214 /* Coherent is lower 32bit only by default in Linux. */ 215 error = linux_dma_tag_init_coherent(&pdev->dev, DMA_BIT_MASK(32)); 216 if (error != 0) 217 goto err; 218 219 return (error); 220 221 err: 222 linux_pdev_dma_uninit(pdev); 223 return (error); 224 } 225 226 int 227 linux_dma_tag_init(struct device *dev, u64 dma_mask) 228 { 229 struct linux_dma_priv *priv; 230 int error; 231 232 priv = dev->dma_priv; 233 234 if (priv->dmat) { 235 if (priv->dma_mask == dma_mask) 236 return (0); 237 238 bus_dma_tag_destroy(priv->dmat); 239 } 240 241 priv->dma_mask = dma_mask; 242 243 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 244 1, 0, /* alignment, boundary */ 245 dma_mask, /* lowaddr */ 246 BUS_SPACE_MAXADDR, /* highaddr */ 247 NULL, NULL, /* filtfunc, filtfuncarg */ 248 BUS_SPACE_MAXSIZE, /* maxsize */ 249 1, /* nsegments */ 250 BUS_SPACE_MAXSIZE, /* maxsegsz */ 251 0, /* flags */ 252 NULL, NULL, /* lockfunc, lockfuncarg */ 253 &priv->dmat); 254 return (-error); 255 } 256 257 int 258 linux_dma_tag_init_coherent(struct device *dev, u64 dma_mask) 259 { 260 struct linux_dma_priv *priv; 261 int error; 262 263 priv = dev->dma_priv; 264 265 if (priv->dmat_coherent) { 266 if (priv->dma_coherent_mask == dma_mask) 267 return (0); 268 269 bus_dma_tag_destroy(priv->dmat_coherent); 270 } 271 272 priv->dma_coherent_mask = dma_mask; 273 274 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 275 1, 0, /* alignment, boundary */ 276 dma_mask, /* lowaddr */ 277 BUS_SPACE_MAXADDR, /* highaddr */ 278 NULL, NULL, /* filtfunc, filtfuncarg */ 279 BUS_SPACE_MAXSIZE, /* maxsize */ 280 1, /* nsegments */ 281 BUS_SPACE_MAXSIZE, /* maxsegsz */ 282 0, /* flags */ 283 NULL, NULL, /* lockfunc, lockfuncarg */ 284 &priv->dmat_coherent); 285 return (-error); 286 } 287 288 static struct pci_driver * 289 linux_pci_find(device_t dev, const struct pci_device_id **idp) 290 { 291 const struct pci_device_id *id; 292 struct pci_driver *pdrv; 293 uint16_t vendor; 294 uint16_t device; 295 uint16_t subvendor; 296 uint16_t subdevice; 297 298 vendor = pci_get_vendor(dev); 299 device = pci_get_device(dev); 300 subvendor = pci_get_subvendor(dev); 301 subdevice = pci_get_subdevice(dev); 302 303 spin_lock(&pci_lock); 304 list_for_each_entry(pdrv, &pci_drivers, node) { 305 for (id = pdrv->id_table; id->vendor != 0; id++) { 306 if (vendor == id->vendor && 307 (PCI_ANY_ID == id->device || device == id->device) && 308 (PCI_ANY_ID == id->subvendor || subvendor == id->subvendor) && 309 (PCI_ANY_ID == id->subdevice || subdevice == id->subdevice)) { 310 *idp = id; 311 spin_unlock(&pci_lock); 312 return (pdrv); 313 } 314 } 315 } 316 spin_unlock(&pci_lock); 317 return (NULL); 318 } 319 320 struct pci_dev * 321 lkpi_pci_get_device(uint32_t vendor, uint32_t device, struct pci_dev *odev) 322 { 323 struct pci_dev *pdev, *found, *odev0; 324 325 odev0 = odev; 326 found = NULL; 327 spin_lock(&pci_lock); 328 list_for_each_entry(pdev, &pci_devices, links) { 329 /* Walk until we find odev. */ 330 if (odev != NULL) { 331 if (pdev == odev) 332 odev = NULL; 333 continue; 334 } 335 336 if ((pdev->vendor == vendor || vendor == PCI_ANY_ID) && 337 (pdev->device == device || device == PCI_ANY_ID)) { 338 found = pdev; 339 break; 340 } 341 } 342 pci_dev_get(found); 343 spin_unlock(&pci_lock); 344 pci_dev_put(odev0); 345 346 return (found); 347 } 348 349 static void 350 lkpi_pci_dev_release(struct device *dev) 351 { 352 struct pci_dev *pdev; 353 354 /* 355 * Before anything else, we have to free all the dynamic 356 * resource which are on the devres list. 357 * Otherwise we risk that supporting infrastructure 358 * is gone and we panic 'randomly'. 359 */ 360 lkpi_devres_release_free_list(dev); 361 362 /* 363 * Now undo linux_pci_attach_device() in reverse-ish 364 * order. 365 */ 366 pdev = to_pci_dev(dev); 367 368 /* 369 * pdrv->remove happens before pci_put_dev() in 370 * linux_pci_detach_device(), which means the driver should have 371 * cleaned up before we get here; see irqents and mmio below. 372 */ 373 374 /* Clear the hierarchy recursively to root. */ 375 if (pdev->bus->self != pdev) { 376 pci_dev_put(pdev->bus->self); 377 pdev->bus->self = NULL; 378 } 379 380 if (pdev->root != NULL) { 381 lkpinew_pci_dev_release(&pdev->root->dev); /* pci_dev_put(pdev->root); ? */ 382 pdev->root = NULL; 383 } 384 385 spin_lock(&pci_lock); 386 list_del(&pdev->links); 387 spin_unlock(&pci_lock); 388 389 linux_pdev_dma_uninit(pdev); 390 391 /* irq? */ 392 393 /* Undo lkpifill_pci_dev(). */ 394 /* devres is gone already; went at the very top. */ 395 if (!list_empty_careful(&pdev->dev.irqents)) { 396 dev_warn(&pdev->dev, "%s: driver did not clean up; " 397 "leaking IRQs\n", __func__); 398 /* 399 * XXX add private function to interrupt.h/linux_interrupt.c 400 * to walk the list and call free_irq on each if we have to. 401 */ 402 } 403 404 spin_lock_destroy(&dev->devres_lock); 405 spin_lock_destroy(&pdev->pcie_cap_lock); 406 407 if (!TAILQ_EMPTY(&pdev->mmio)) { 408 dev_warn(&pdev->dev, "%s: driver did not clean up; " 409 "leaking mmio resources\n", __func__); 410 /* XXX we have two functions to walk and release in here. */ 411 } 412 413 if (pdev->msi_desc != NULL) { 414 for (int i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--) 415 free(pdev->msi_desc[i], M_DEVBUF); 416 free(pdev->msi_desc, M_DEVBUF); 417 } 418 419 free(pdev->bus, M_DEVBUF); 420 kfree(pdev->path_name); 421 422 /* 423 * Lastly, apply an internal hack in order to signal 424 * that this was run (device reference fully dropped). 425 * See comment in linux_pci_detach_device(). 426 */ 427 pdev->dev.release = NULL; 428 } 429 430 static int 431 lkpifill_pci_dev(device_t dev, struct pci_dev *pdev) 432 { 433 struct pci_devinfo *dinfo; 434 int error; 435 436 error = kobject_init_and_add(&pdev->dev.kobj, &linux_dev_ktype, 437 &linux_root_device.kobj, device_get_nameunit(dev)); 438 if (error != 0) { 439 printf("%s:%d: kobject_init_and_add returned %d\n", 440 __func__, __LINE__, error); 441 return (error); 442 } 443 444 pdev->devfn = PCI_DEVFN(pci_get_slot(dev), pci_get_function(dev)); 445 pdev->vendor = pci_get_vendor(dev); 446 pdev->device = pci_get_device(dev); 447 pdev->subsystem_vendor = pci_get_subvendor(dev); 448 pdev->subsystem_device = pci_get_subdevice(dev); 449 pdev->class = pci_get_class(dev); 450 pdev->revision = pci_get_revid(dev); 451 pdev->path_name = kasprintf(GFP_KERNEL, "%04d:%02d:%02d.%d", 452 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), 453 pci_get_function(dev)); 454 455 pdev->bus = malloc(sizeof(*pdev->bus), M_DEVBUF, M_WAITOK | M_ZERO); 456 pdev->bus->number = pci_get_bus(dev); 457 pdev->bus->domain = pci_get_domain(dev); 458 459 /* Check if we have reached the root to satisfy pci_is_root_bus() */ 460 dinfo = device_get_ivars(dev); 461 if (dinfo->cfg.pcie.pcie_location != 0 && 462 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) { 463 pdev->bus->self = NULL; 464 } else { 465 /* 466 * This should be the upstream bridge; pci_upstream_bridge() 467 * handles that case on demand as otherwise we'll shadow the 468 * entire PCI hierarchy. 469 */ 470 pdev->bus->self = pdev; 471 } 472 pdev->dev.bsddev = dev; 473 pdev->dev.parent = &linux_root_device; 474 pdev->dev.release = lkpi_pci_dev_release; 475 476 if (pci_msi_count(dev) > 0) 477 pdev->msi_desc = malloc(pci_msi_count(dev) * 478 sizeof(*pdev->msi_desc), M_DEVBUF, M_WAITOK | M_ZERO); 479 480 TAILQ_INIT(&pdev->mmio); 481 spin_lock_init(&pdev->pcie_cap_lock); 482 spin_lock_init(&pdev->dev.devres_lock); 483 INIT_LIST_HEAD(&pdev->dev.devres_head); 484 INIT_LIST_HEAD(&pdev->dev.irqents); 485 INIT_LIST_HEAD(&pdev->links); 486 487 return (0); 488 } 489 490 static void 491 lkpinew_pci_dev_release(struct device *dev) 492 { 493 struct pci_dev *pdev; 494 int i; 495 496 pdev = to_pci_dev(dev); 497 if (pdev->root != NULL) 498 pci_dev_put(pdev->root); 499 if (pdev->bus->self != pdev && pdev->bus->self != NULL) 500 pci_dev_put(pdev->bus->self); 501 free(pdev->bus, M_DEVBUF); 502 if (pdev->msi_desc != NULL) { 503 for (i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--) 504 free(pdev->msi_desc[i], M_DEVBUF); 505 free(pdev->msi_desc, M_DEVBUF); 506 } 507 kfree(pdev->path_name); 508 free(pdev, M_DEVBUF); 509 } 510 511 struct pci_dev * 512 lkpinew_pci_dev(device_t dev) 513 { 514 struct pci_dev *pdev; 515 int error; 516 517 pdev = malloc(sizeof(*pdev), M_DEVBUF, M_WAITOK|M_ZERO); 518 error = lkpifill_pci_dev(dev, pdev); 519 if (error != 0) { 520 free(pdev, M_DEVBUF); 521 return (NULL); 522 } 523 pdev->dev.release = lkpinew_pci_dev_release; 524 525 return (pdev); 526 } 527 528 struct pci_dev * 529 lkpi_pci_get_class(unsigned int class, struct pci_dev *from) 530 { 531 device_t dev; 532 device_t devfrom = NULL; 533 struct pci_dev *pdev; 534 535 if (from != NULL) 536 devfrom = from->dev.bsddev; 537 538 dev = pci_find_class_from(class >> 16, (class >> 8) & 0xFF, devfrom); 539 if (dev == NULL) 540 return (NULL); 541 542 pdev = lkpinew_pci_dev(dev); 543 return (pdev); 544 } 545 546 struct pci_dev * 547 lkpi_pci_get_base_class(unsigned int baseclass, struct pci_dev *from) 548 { 549 device_t dev; 550 device_t devfrom = NULL; 551 struct pci_dev *pdev; 552 553 if (from != NULL) 554 devfrom = from->dev.bsddev; 555 556 dev = pci_find_base_class_from(baseclass, devfrom); 557 if (dev == NULL) 558 return (NULL); 559 560 pdev = lkpinew_pci_dev(dev); 561 return (pdev); 562 } 563 564 struct pci_dev * 565 lkpi_pci_get_domain_bus_and_slot(int domain, unsigned int bus, 566 unsigned int devfn) 567 { 568 device_t dev; 569 struct pci_dev *pdev; 570 571 dev = pci_find_dbsf(domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 572 if (dev == NULL) 573 return (NULL); 574 575 pdev = lkpinew_pci_dev(dev); 576 return (pdev); 577 } 578 579 struct pci_dev * 580 lkpi_pci_get_slot(struct pci_bus *pbus, unsigned int devfn) 581 { 582 device_t dev; 583 struct pci_dev *pdev; 584 585 dev = pci_find_bsf(pbus->number, PCI_SLOT(devfn), PCI_FUNC(devfn)); 586 if (dev == NULL) 587 return (NULL); 588 589 pdev = lkpinew_pci_dev(dev); 590 return (pdev); 591 } 592 593 static int 594 linux_pci_probe(device_t dev) 595 { 596 const struct pci_device_id *id; 597 struct pci_driver *pdrv; 598 599 if ((pdrv = linux_pci_find(dev, &id)) == NULL) 600 return (ENXIO); 601 if (device_get_driver(dev) != &pdrv->bsddriver) 602 return (ENXIO); 603 device_set_desc(dev, pdrv->name); 604 605 /* Assume BSS initialized (should never return BUS_PROBE_SPECIFIC). */ 606 if (pdrv->bsd_probe_return == 0) 607 return (BUS_PROBE_DEFAULT); 608 else 609 return (pdrv->bsd_probe_return); 610 } 611 612 static int 613 linux_pci_attach(device_t dev) 614 { 615 const struct pci_device_id *id; 616 struct pci_driver *pdrv; 617 struct pci_dev *pdev; 618 619 pdrv = linux_pci_find(dev, &id); 620 pdev = device_get_softc(dev); 621 622 MPASS(pdrv != NULL); 623 MPASS(pdev != NULL); 624 625 return (linux_pci_attach_device(dev, pdrv, id, pdev)); 626 } 627 628 static struct resource_list_entry * 629 linux_pci_reserve_bar(struct pci_dev *pdev, struct resource_list *rl, 630 int type, int rid) 631 { 632 device_t dev; 633 struct resource *res; 634 635 KASSERT(type == SYS_RES_IOPORT || type == SYS_RES_MEMORY, 636 ("trying to reserve non-BAR type %d", type)); 637 638 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? 639 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; 640 res = pci_reserve_map(device_get_parent(dev), dev, type, rid, 0, ~0, 641 1, 1, 0); 642 if (res == NULL) 643 return (NULL); 644 return (resource_list_find(rl, type, rid)); 645 } 646 647 static struct resource_list_entry * 648 linux_pci_get_rle(struct pci_dev *pdev, int type, int rid, bool reserve_bar) 649 { 650 struct pci_devinfo *dinfo; 651 struct resource_list *rl; 652 struct resource_list_entry *rle; 653 654 dinfo = device_get_ivars(pdev->dev.bsddev); 655 rl = &dinfo->resources; 656 rle = resource_list_find(rl, type, rid); 657 /* Reserve resources for this BAR if needed. */ 658 if (rle == NULL && reserve_bar) 659 rle = linux_pci_reserve_bar(pdev, rl, type, rid); 660 return (rle); 661 } 662 663 int 664 linux_pci_attach_device(device_t dev, struct pci_driver *pdrv, 665 const struct pci_device_id *id, struct pci_dev *pdev) 666 { 667 struct resource_list_entry *rle; 668 device_t parent; 669 struct pci_dev *pbus, *ppbus; 670 uintptr_t rid; 671 int error; 672 bool isdrm; 673 674 linux_set_current(curthread); 675 676 parent = device_get_parent(dev); 677 isdrm = pdrv != NULL && pdrv->isdrm; 678 679 if (isdrm) { 680 struct pci_devinfo *dinfo; 681 682 dinfo = device_get_ivars(parent); 683 device_set_ivars(dev, dinfo); 684 } 685 686 error = lkpifill_pci_dev(dev, pdev); 687 if (error != 0) 688 return (error); 689 690 if (isdrm) 691 PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid); 692 else 693 PCI_GET_ID(parent, dev, PCI_ID_RID, &rid); 694 pdev->devfn = rid; 695 pdev->pdrv = pdrv; 696 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0, false); 697 if (rle != NULL) 698 pdev->dev.irq = rle->start; 699 else 700 pdev->dev.irq = LINUX_IRQ_INVALID; 701 pdev->irq = pdev->dev.irq; 702 error = linux_pdev_dma_init(pdev); 703 if (error) 704 goto out_err; 705 706 spin_lock(&pci_lock); 707 list_add(&pdev->links, &pci_devices); 708 spin_unlock(&pci_lock); 709 710 /* 711 * Create the hierarchy now as we cannot on demand later. 712 * Take special care of DRM as there is a non-PCI device in the chain. 713 */ 714 pbus = pdev; 715 if (isdrm) { 716 pbus = lkpinew_pci_dev(parent); 717 if (pbus == NULL) { 718 error = ENXIO; 719 goto out_err; 720 } 721 } 722 pcie_find_root_port(pbus); 723 if (isdrm) 724 pdev->root = pbus->root; 725 ppbus = pci_upstream_bridge(pbus); 726 while (ppbus != NULL && ppbus != pbus) { 727 pbus = ppbus; 728 ppbus = pci_upstream_bridge(pbus); 729 } 730 731 if (pdrv != NULL) { 732 error = pdrv->probe(pdev, id); 733 if (error) 734 goto out_err; 735 } 736 return (0); 737 738 out_err: 739 put_device(&pdev->dev); 740 return (-error); 741 } 742 743 static int 744 linux_pci_detach(device_t dev) 745 { 746 struct pci_dev *pdev; 747 int error; 748 749 pdev = device_get_softc(dev); 750 MPASS(pdev != NULL); 751 752 error = linux_pci_detach_device(pdev); 753 if (error == 0) 754 device_set_desc(dev, NULL); 755 756 return (error); 757 } 758 759 int 760 linux_pci_detach_device(struct pci_dev *pdev) 761 { 762 763 linux_set_current(curthread); 764 765 /* 766 * We cannot do much here as almost everything will have 767 * to happen as the last reference to the LinuxKPI device 768 * goes away. That will call the release function, 769 * which lkpifill_pci_dev() set. That is were most 770 * of the cleanup will happen. But before that give 771 * the driver a chance to cleanup. 772 * The big problem is that the Linux KPI does not 773 * report back if it was the last kref (well kref 774 * does report back but then kobj, dev, pdev do not). 775 * So we have little way of knowing if the release 776 * happened or not. We have to play tricks for that 777 * and we can given the softc (pdev) is still valid 778 * until we return from here. 779 */ 780 781 if (pdev->pdrv != NULL) 782 pdev->pdrv->remove(pdev); 783 784 pci_dev_put(pdev); 785 786 /* 787 * We (ab)use the release function as a guard to 788 * know if we made it there and the device is gone. 789 */ 790 if (pdev->dev.release != lkpi_pci_dev_release) 791 return (0); 792 793 /* 794 * Detach failed. 795 * We need to re-acquire the ref and wait for 796 * the other refs to be gone... In theory this 797 * should never happen, so log it! 798 * XXX I wish there was a KPI to query the ref. 799 * 800 * If we do not error and wait, we will have a 801 * LinuxKPI device dangling active with pointers 802 * but the FreeBSD device_t will be 'gone'. 803 */ 804 device_printf(pdev->dev.bsddev, "%s failed due to %u other pending " 805 "references on the LinuxKPI device.\n", __func__, 806 kref_read(&pdev->dev.kobj.kref)); 807 pci_dev_get(pdev); 808 809 return (EBUSY); 810 } 811 812 static int 813 lkpi_pci_disable_dev(struct device *dev) 814 { 815 816 (void) pci_disable_io(dev->bsddev, SYS_RES_MEMORY); 817 (void) pci_disable_io(dev->bsddev, SYS_RES_IOPORT); 818 return (0); 819 } 820 821 static struct pci_devres * 822 lkpi_pci_devres_get_alloc(struct pci_dev *pdev) 823 { 824 struct pci_devres *dr; 825 826 dr = lkpi_devres_find(&pdev->dev, lkpi_pci_devres_release, NULL, NULL); 827 if (dr == NULL) { 828 dr = lkpi_devres_alloc(lkpi_pci_devres_release, sizeof(*dr), 829 GFP_KERNEL | __GFP_ZERO); 830 if (dr != NULL) 831 lkpi_devres_add(&pdev->dev, dr); 832 } 833 834 return (dr); 835 } 836 837 static struct pci_devres * 838 lkpi_pci_devres_find(struct pci_dev *pdev) 839 { 840 if (!pdev->managed) 841 return (NULL); 842 843 return (lkpi_pci_devres_get_alloc(pdev)); 844 } 845 846 void 847 lkpi_pci_devres_release(struct device *dev, void *p) 848 { 849 struct pci_devres *dr; 850 struct pci_dev *pdev; 851 int bar; 852 853 pdev = to_pci_dev(dev); 854 dr = p; 855 856 if (pdev->msix_enabled) 857 lkpi_pci_disable_msix(pdev); 858 if (pdev->msi_enabled) 859 lkpi_pci_disable_msi(pdev); 860 861 if (dr->enable_io && lkpi_pci_disable_dev(dev) == 0) 862 dr->enable_io = false; 863 864 if (dr->region_mask == 0) 865 return; 866 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 867 868 if ((dr->region_mask & (1 << bar)) == 0) 869 continue; 870 pci_release_region(pdev, bar); 871 } 872 } 873 874 int 875 linuxkpi_pcim_enable_device(struct pci_dev *pdev) 876 { 877 struct pci_devres *dr; 878 int error; 879 880 /* Here we cannot run through the pdev->managed check. */ 881 dr = lkpi_pci_devres_get_alloc(pdev); 882 if (dr == NULL) 883 return (-ENOMEM); 884 885 /* If resources were enabled before do not do it again. */ 886 if (dr->enable_io) 887 return (0); 888 889 error = pci_enable_device(pdev); 890 if (error == 0) 891 dr->enable_io = true; 892 893 /* This device is not managed. */ 894 pdev->managed = true; 895 896 return (error); 897 } 898 899 static struct pcim_iomap_devres * 900 lkpi_pcim_iomap_devres_find(struct pci_dev *pdev) 901 { 902 struct pcim_iomap_devres *dr; 903 904 dr = lkpi_devres_find(&pdev->dev, lkpi_pcim_iomap_table_release, 905 NULL, NULL); 906 if (dr == NULL) { 907 dr = lkpi_devres_alloc(lkpi_pcim_iomap_table_release, 908 sizeof(*dr), GFP_KERNEL | __GFP_ZERO); 909 if (dr != NULL) 910 lkpi_devres_add(&pdev->dev, dr); 911 } 912 913 if (dr == NULL) 914 device_printf(pdev->dev.bsddev, "%s: NULL\n", __func__); 915 916 return (dr); 917 } 918 919 void __iomem ** 920 linuxkpi_pcim_iomap_table(struct pci_dev *pdev) 921 { 922 struct pcim_iomap_devres *dr; 923 924 dr = lkpi_pcim_iomap_devres_find(pdev); 925 if (dr == NULL) 926 return (NULL); 927 928 /* 929 * If the driver has manually set a flag to be able to request the 930 * resource to use bus_read/write_<n>, return the shadow table. 931 */ 932 if (pdev->want_iomap_res) 933 return ((void **)dr->res_table); 934 935 /* This is the Linux default. */ 936 return (dr->mmio_table); 937 } 938 939 static struct resource * 940 _lkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen __unused) 941 { 942 struct pci_mmio_region *mmio, *p; 943 int type; 944 945 if (!lkpi_pci_bar_id_valid(bar)) 946 return (NULL); 947 948 type = pci_resource_type(pdev, bar); 949 if (type < 0) { 950 device_printf(pdev->dev.bsddev, "%s: bar %d type %d\n", 951 __func__, bar, type); 952 return (NULL); 953 } 954 955 /* 956 * Check for duplicate mappings. 957 * This can happen if a driver calls pci_request_region() first. 958 */ 959 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 960 if (mmio->type == type && mmio->rid == PCIR_BAR(bar)) { 961 return (mmio->res); 962 } 963 } 964 965 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO); 966 mmio->rid = PCIR_BAR(bar); 967 mmio->type = type; 968 mmio->res = bus_alloc_resource_any(pdev->dev.bsddev, mmio->type, 969 &mmio->rid, RF_ACTIVE|RF_SHAREABLE); 970 if (mmio->res == NULL) { 971 device_printf(pdev->dev.bsddev, "%s: failed to alloc " 972 "bar %d type %d rid %d\n", 973 __func__, bar, type, PCIR_BAR(bar)); 974 free(mmio, M_DEVBUF); 975 return (NULL); 976 } 977 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next); 978 979 return (mmio->res); 980 } 981 982 void * 983 linuxkpi_pci_iomap_range(struct pci_dev *pdev, int bar, 984 unsigned long off, unsigned long maxlen) 985 { 986 struct resource *res; 987 988 if (!lkpi_pci_bar_id_valid(bar)) 989 return (NULL); 990 991 res = _lkpi_pci_iomap(pdev, bar, maxlen); 992 if (res == NULL) 993 return (NULL); 994 /* This is a FreeBSD extension so we can use bus_*(). */ 995 if (pdev->want_iomap_res) 996 return (res); 997 MPASS(off < rman_get_size(res)); 998 return ((void *)(rman_get_bushandle(res) + off)); 999 } 1000 1001 void * 1002 linuxkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen) 1003 { 1004 if (!lkpi_pci_bar_id_valid(bar)) 1005 return (NULL); 1006 1007 return (linuxkpi_pci_iomap_range(pdev, bar, 0, maxlen)); 1008 } 1009 1010 void * 1011 linuxkpi_pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen) 1012 { 1013 struct pcim_iomap_devres *dr; 1014 void *res; 1015 1016 if (!lkpi_pci_bar_id_valid(bar)) 1017 return (NULL); 1018 1019 dr = lkpi_pcim_iomap_devres_find(pdev); 1020 if (dr == NULL) 1021 return (NULL); 1022 1023 if (dr->res_table[bar] != NULL) 1024 return (dr->res_table[bar]); 1025 1026 res = linuxkpi_pci_iomap(pdev, bar, maxlen); 1027 if (res == NULL) { 1028 /* 1029 * Do not free the devres in case there were 1030 * other valid mappings before already. 1031 */ 1032 return (NULL); 1033 } 1034 lkpi_set_pcim_iomap_devres(dr, bar, res); 1035 1036 return (res); 1037 } 1038 1039 void 1040 linuxkpi_pci_iounmap(struct pci_dev *pdev, void *res) 1041 { 1042 struct pci_mmio_region *mmio, *p; 1043 bus_space_handle_t bh = (bus_space_handle_t)res; 1044 1045 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 1046 if (pdev->want_iomap_res) { 1047 if (res != mmio->res) 1048 continue; 1049 } else { 1050 if (bh < rman_get_bushandle(mmio->res) || 1051 bh >= rman_get_bushandle(mmio->res) + 1052 rman_get_size(mmio->res)) 1053 continue; 1054 } 1055 bus_release_resource(pdev->dev.bsddev, 1056 mmio->type, mmio->rid, mmio->res); 1057 TAILQ_REMOVE(&pdev->mmio, mmio, next); 1058 free(mmio, M_DEVBUF); 1059 return; 1060 } 1061 } 1062 1063 int 1064 linuxkpi_pcim_iomap_regions(struct pci_dev *pdev, uint32_t mask, const char *name) 1065 { 1066 struct pcim_iomap_devres *dr; 1067 void *res; 1068 uint32_t mappings; 1069 int bar; 1070 1071 dr = lkpi_pcim_iomap_devres_find(pdev); 1072 if (dr == NULL) 1073 return (-ENOMEM); 1074 1075 /* Now iomap all the requested (by "mask") ones. */ 1076 for (bar = mappings = 0; mappings != mask; bar++) { 1077 if ((mask & (1 << bar)) == 0) 1078 continue; 1079 1080 /* Request double is not allowed. */ 1081 if (dr->mmio_table[bar] != NULL) { 1082 device_printf(pdev->dev.bsddev, "%s: bar %d %p\n", 1083 __func__, bar, dr->mmio_table[bar]); 1084 goto err; 1085 } 1086 1087 res = _lkpi_pci_iomap(pdev, bar, 0); 1088 if (res == NULL) 1089 goto err; 1090 lkpi_set_pcim_iomap_devres(dr, bar, res); 1091 1092 mappings |= (1 << bar); 1093 } 1094 1095 return (0); 1096 err: 1097 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 1098 if ((mappings & (1 << bar)) != 0) { 1099 res = dr->mmio_table[bar]; 1100 if (res == NULL) 1101 continue; 1102 pci_iounmap(pdev, res); 1103 } 1104 } 1105 1106 return (-EINVAL); 1107 } 1108 1109 static void 1110 lkpi_pcim_iomap_table_release(struct device *dev, void *p) 1111 { 1112 struct pcim_iomap_devres *dr; 1113 struct pci_dev *pdev; 1114 int bar; 1115 1116 dr = p; 1117 pdev = to_pci_dev(dev); 1118 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) { 1119 1120 if (dr->mmio_table[bar] == NULL) 1121 continue; 1122 1123 pci_iounmap(pdev, dr->mmio_table[bar]); 1124 } 1125 } 1126 1127 static int 1128 linux_pci_suspend(device_t dev) 1129 { 1130 const struct dev_pm_ops *pmops; 1131 struct pm_message pm = { }; 1132 struct pci_dev *pdev; 1133 int error; 1134 1135 error = 0; 1136 linux_set_current(curthread); 1137 pdev = device_get_softc(dev); 1138 pmops = pdev->pdrv->driver.pm; 1139 1140 if (pdev->pdrv->suspend != NULL) 1141 error = -pdev->pdrv->suspend(pdev, pm); 1142 else if (pmops != NULL && pmops->suspend != NULL) { 1143 error = -pmops->suspend(&pdev->dev); 1144 if (error == 0 && pmops->suspend_late != NULL) 1145 error = -pmops->suspend_late(&pdev->dev); 1146 if (error == 0 && pmops->suspend_noirq != NULL) 1147 error = -pmops->suspend_noirq(&pdev->dev); 1148 } 1149 return (error); 1150 } 1151 1152 static int 1153 linux_pci_resume(device_t dev) 1154 { 1155 const struct dev_pm_ops *pmops; 1156 struct pci_dev *pdev; 1157 int error; 1158 1159 error = 0; 1160 linux_set_current(curthread); 1161 pdev = device_get_softc(dev); 1162 pmops = pdev->pdrv->driver.pm; 1163 1164 if (pdev->pdrv->resume != NULL) 1165 error = -pdev->pdrv->resume(pdev); 1166 else if (pmops != NULL && pmops->resume != NULL) { 1167 if (pmops->resume_early != NULL) 1168 error = -pmops->resume_early(&pdev->dev); 1169 if (error == 0 && pmops->resume != NULL) 1170 error = -pmops->resume(&pdev->dev); 1171 } 1172 return (error); 1173 } 1174 1175 static int 1176 linux_pci_shutdown(device_t dev) 1177 { 1178 struct pci_dev *pdev; 1179 1180 linux_set_current(curthread); 1181 pdev = device_get_softc(dev); 1182 if (pdev->pdrv->shutdown != NULL) 1183 pdev->pdrv->shutdown(pdev); 1184 return (0); 1185 } 1186 1187 static int 1188 linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config) 1189 { 1190 struct pci_dev *pdev; 1191 int error; 1192 1193 linux_set_current(curthread); 1194 pdev = device_get_softc(dev); 1195 if (pdev->pdrv->bsd_iov_init != NULL) 1196 error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config); 1197 else 1198 error = EINVAL; 1199 return (error); 1200 } 1201 1202 static void 1203 linux_pci_iov_uninit(device_t dev) 1204 { 1205 struct pci_dev *pdev; 1206 1207 linux_set_current(curthread); 1208 pdev = device_get_softc(dev); 1209 if (pdev->pdrv->bsd_iov_uninit != NULL) 1210 pdev->pdrv->bsd_iov_uninit(dev); 1211 } 1212 1213 static int 1214 linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config) 1215 { 1216 struct pci_dev *pdev; 1217 int error; 1218 1219 linux_set_current(curthread); 1220 pdev = device_get_softc(dev); 1221 if (pdev->pdrv->bsd_iov_add_vf != NULL) 1222 error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config); 1223 else 1224 error = EINVAL; 1225 return (error); 1226 } 1227 1228 static int 1229 _linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc) 1230 { 1231 int error; 1232 1233 linux_set_current(curthread); 1234 spin_lock(&pci_lock); 1235 list_add(&pdrv->node, &pci_drivers); 1236 spin_unlock(&pci_lock); 1237 if (pdrv->bsddriver.name == NULL) 1238 pdrv->bsddriver.name = pdrv->name; 1239 pdrv->bsddriver.methods = pci_methods; 1240 pdrv->bsddriver.size = sizeof(struct pci_dev); 1241 1242 bus_topo_lock(); 1243 error = devclass_add_driver(dc, &pdrv->bsddriver, 1244 BUS_PASS_DEFAULT, &pdrv->bsdclass); 1245 bus_topo_unlock(); 1246 return (-error); 1247 } 1248 1249 int 1250 linux_pci_register_driver(struct pci_driver *pdrv) 1251 { 1252 devclass_t dc; 1253 1254 pdrv->isdrm = strcmp(pdrv->name, "drmn") == 0; 1255 dc = pdrv->isdrm ? devclass_create("vgapci") : devclass_find("pci"); 1256 if (dc == NULL) 1257 return (-ENXIO); 1258 return (_linux_pci_register_driver(pdrv, dc)); 1259 } 1260 1261 static struct resource_list_entry * 1262 lkpi_pci_get_bar(struct pci_dev *pdev, int bar, bool reserve) 1263 { 1264 int type; 1265 1266 type = pci_resource_type(pdev, bar); 1267 if (type < 0) 1268 return (NULL); 1269 bar = PCIR_BAR(bar); 1270 return (linux_pci_get_rle(pdev, type, bar, reserve)); 1271 } 1272 1273 struct device * 1274 lkpi_pci_find_irq_dev(unsigned int irq) 1275 { 1276 struct pci_dev *pdev; 1277 struct device *found; 1278 1279 found = NULL; 1280 spin_lock(&pci_lock); 1281 list_for_each_entry(pdev, &pci_devices, links) { 1282 if (irq == pdev->dev.irq || 1283 (irq >= pdev->dev.irq_start && irq < pdev->dev.irq_end)) { 1284 found = &pdev->dev; 1285 break; 1286 } 1287 } 1288 spin_unlock(&pci_lock); 1289 return (found); 1290 } 1291 1292 unsigned long 1293 pci_resource_start(struct pci_dev *pdev, int bar) 1294 { 1295 struct resource_list_entry *rle; 1296 rman_res_t newstart; 1297 device_t dev; 1298 int error; 1299 1300 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL) 1301 return (0); 1302 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? 1303 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; 1304 error = bus_translate_resource(dev, rle->type, rle->start, &newstart); 1305 if (error != 0) { 1306 device_printf(pdev->dev.bsddev, 1307 "translate of %#jx failed: %d\n", 1308 (uintmax_t)rle->start, error); 1309 return (0); 1310 } 1311 return (newstart); 1312 } 1313 1314 unsigned long 1315 pci_resource_len(struct pci_dev *pdev, int bar) 1316 { 1317 struct resource_list_entry *rle; 1318 1319 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL) 1320 return (0); 1321 return (rle->count); 1322 } 1323 1324 static int 1325 lkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name, 1326 bool managed) 1327 { 1328 struct resource *res; 1329 struct pci_devres *dr; 1330 struct pci_mmio_region *mmio; 1331 int rid; 1332 int type; 1333 1334 if (!lkpi_pci_bar_id_valid(bar)) 1335 return (-EINVAL); 1336 1337 type = pci_resource_type(pdev, bar); 1338 if (type < 0) 1339 return (0); 1340 1341 rid = PCIR_BAR(bar); 1342 res = bus_alloc_resource_any(pdev->dev.bsddev, type, &rid, 1343 RF_ACTIVE|RF_SHAREABLE); 1344 if (res == NULL) { 1345 device_printf(pdev->dev.bsddev, "%s: failed to alloc " 1346 "bar %d type %d rid %d\n", 1347 __func__, bar, type, PCIR_BAR(bar)); 1348 return (-EBUSY); 1349 } 1350 1351 /* 1352 * It seems there is an implicit devres tracking on these if the device 1353 * is managed (lkpi_pci_devres_find() case); otherwise the resources are 1354 * not automatically freed on FreeBSD/LinuxKPI though they should be/are 1355 * expected to be by Linux drivers. 1356 * Otherwise if we are called from a pcim-function with the managed 1357 * argument set, we need to track devres independent of pdev->managed. 1358 */ 1359 if (managed) 1360 dr = lkpi_pci_devres_get_alloc(pdev); 1361 else 1362 dr = lkpi_pci_devres_find(pdev); 1363 if (dr != NULL) { 1364 dr->region_mask |= (1 << bar); 1365 dr->region_table[bar] = res; 1366 } 1367 1368 /* Even if the device is not managed we need to track it for iomap. */ 1369 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO); 1370 mmio->rid = PCIR_BAR(bar); 1371 mmio->type = type; 1372 mmio->res = res; 1373 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next); 1374 1375 return (0); 1376 } 1377 1378 int 1379 linuxkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) 1380 { 1381 return (lkpi_pci_request_region(pdev, bar, res_name, false)); 1382 } 1383 1384 int 1385 linuxkpi_pci_request_regions(struct pci_dev *pdev, const char *res_name) 1386 { 1387 int error; 1388 int i; 1389 1390 for (i = 0; i <= PCIR_MAX_BAR_0; i++) { 1391 error = pci_request_region(pdev, i, res_name); 1392 if (error && error != -EBUSY) { 1393 pci_release_regions(pdev); 1394 return (error); 1395 } 1396 } 1397 return (0); 1398 } 1399 1400 int 1401 linuxkpi_pcim_request_all_regions(struct pci_dev *pdev, const char *res_name) 1402 { 1403 int bar, error; 1404 1405 for (bar = 0; bar <= PCIR_MAX_BAR_0; bar++) { 1406 error = lkpi_pci_request_region(pdev, bar, res_name, true); 1407 if (error != 0 && error != -EBUSY) { 1408 device_printf(pdev->dev.bsddev, "%s: bar %d res_name '%s': " 1409 "lkpi_pci_request_region returned %d\n", __func__, 1410 bar, res_name, error); 1411 pci_release_regions(pdev); 1412 return (error); 1413 } 1414 } 1415 return (0); 1416 } 1417 1418 void 1419 linuxkpi_pci_release_region(struct pci_dev *pdev, int bar) 1420 { 1421 struct resource_list_entry *rle; 1422 struct pci_devres *dr; 1423 struct pci_mmio_region *mmio, *p; 1424 1425 if ((rle = lkpi_pci_get_bar(pdev, bar, false)) == NULL) 1426 return; 1427 1428 /* 1429 * As we implicitly track the requests we also need to clear them on 1430 * release. Do clear before resource release. 1431 */ 1432 dr = lkpi_pci_devres_find(pdev); 1433 if (dr != NULL) { 1434 KASSERT(dr->region_table[bar] == rle->res, ("%s: pdev %p bar %d" 1435 " region_table res %p != rel->res %p\n", __func__, pdev, 1436 bar, dr->region_table[bar], rle->res)); 1437 dr->region_table[bar] = NULL; 1438 dr->region_mask &= ~(1 << bar); 1439 } 1440 1441 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) { 1442 if (rle->res != (void *)rman_get_bushandle(mmio->res)) 1443 continue; 1444 TAILQ_REMOVE(&pdev->mmio, mmio, next); 1445 free(mmio, M_DEVBUF); 1446 } 1447 1448 bus_release_resource(pdev->dev.bsddev, rle->type, rle->rid, rle->res); 1449 } 1450 1451 void 1452 linuxkpi_pci_release_regions(struct pci_dev *pdev) 1453 { 1454 int i; 1455 1456 for (i = 0; i <= PCIR_MAX_BAR_0; i++) 1457 pci_release_region(pdev, i); 1458 } 1459 1460 int 1461 linux_pci_register_drm_driver(struct pci_driver *pdrv) 1462 { 1463 devclass_t dc; 1464 1465 dc = devclass_create("vgapci"); 1466 if (dc == NULL) 1467 return (-ENXIO); 1468 pdrv->isdrm = true; 1469 pdrv->name = "drmn"; 1470 return (_linux_pci_register_driver(pdrv, dc)); 1471 } 1472 1473 void 1474 linux_pci_unregister_driver(struct pci_driver *pdrv) 1475 { 1476 devclass_t bus; 1477 1478 bus = devclass_find(pdrv->isdrm ? "vgapci" : "pci"); 1479 1480 spin_lock(&pci_lock); 1481 list_del(&pdrv->node); 1482 spin_unlock(&pci_lock); 1483 bus_topo_lock(); 1484 if (bus != NULL) 1485 devclass_delete_driver(bus, &pdrv->bsddriver); 1486 bus_topo_unlock(); 1487 } 1488 1489 void 1490 linux_pci_unregister_drm_driver(struct pci_driver *pdrv) 1491 { 1492 devclass_t bus; 1493 1494 bus = devclass_find("vgapci"); 1495 1496 spin_lock(&pci_lock); 1497 list_del(&pdrv->node); 1498 spin_unlock(&pci_lock); 1499 bus_topo_lock(); 1500 if (bus != NULL) 1501 devclass_delete_driver(bus, &pdrv->bsddriver); 1502 bus_topo_unlock(); 1503 } 1504 1505 int 1506 linuxkpi_pci_enable_msix(struct pci_dev *pdev, struct msix_entry *entries, 1507 int nreq) 1508 { 1509 struct resource_list_entry *rle; 1510 int error; 1511 int avail; 1512 int i; 1513 1514 avail = pci_msix_count(pdev->dev.bsddev); 1515 if (avail < nreq) { 1516 if (avail == 0) 1517 return -EINVAL; 1518 return avail; 1519 } 1520 avail = nreq; 1521 if ((error = -pci_alloc_msix(pdev->dev.bsddev, &avail)) != 0) 1522 return error; 1523 /* 1524 * Handle case where "pci_alloc_msix()" may allocate less 1525 * interrupts than available and return with no error: 1526 */ 1527 if (avail < nreq) { 1528 pci_release_msi(pdev->dev.bsddev); 1529 return avail; 1530 } 1531 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); 1532 pdev->dev.irq_start = rle->start; 1533 pdev->dev.irq_end = rle->start + avail; 1534 for (i = 0; i < nreq; i++) 1535 entries[i].vector = pdev->dev.irq_start + i; 1536 pdev->msix_enabled = true; 1537 return (0); 1538 } 1539 1540 int 1541 _lkpi_pci_enable_msi_range(struct pci_dev *pdev, int minvec, int maxvec) 1542 { 1543 struct resource_list_entry *rle; 1544 int error; 1545 int nvec; 1546 1547 if (maxvec < minvec) 1548 return (-EINVAL); 1549 1550 nvec = pci_msi_count(pdev->dev.bsddev); 1551 if (nvec < 1 || nvec < minvec) 1552 return (-ENOSPC); 1553 1554 nvec = min(nvec, maxvec); 1555 if ((error = -pci_alloc_msi(pdev->dev.bsddev, &nvec)) != 0) 1556 return error; 1557 1558 /* Native PCI might only ever ask for 32 vectors. */ 1559 if (nvec < minvec) { 1560 pci_release_msi(pdev->dev.bsddev); 1561 return (-ENOSPC); 1562 } 1563 1564 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); 1565 pdev->dev.irq_start = rle->start; 1566 pdev->dev.irq_end = rle->start + nvec; 1567 pdev->irq = rle->start; 1568 pdev->msi_enabled = true; 1569 return (0); 1570 } 1571 1572 int 1573 pci_alloc_irq_vectors(struct pci_dev *pdev, int minv, int maxv, 1574 unsigned int flags) 1575 { 1576 int error; 1577 1578 if ((flags & PCI_IRQ_AFFINITY) != 0) { 1579 pr_debug("%s: TODO PCI_IRQ_AFFINITY\n", __func__); 1580 } 1581 if (flags & PCI_IRQ_MSIX) { 1582 struct msix_entry *entries; 1583 int i; 1584 1585 entries = kcalloc(maxv, sizeof(*entries), GFP_KERNEL); 1586 if (entries == NULL) { 1587 error = -ENOMEM; 1588 goto out; 1589 } 1590 for (i = 0; i < maxv; ++i) 1591 entries[i].entry = i; 1592 error = pci_enable_msix(pdev, entries, maxv); 1593 out: 1594 kfree(entries); 1595 if (error == 0 && pdev->msix_enabled) 1596 return (pdev->dev.irq_end - pdev->dev.irq_start); 1597 } 1598 if (flags & PCI_IRQ_MSI) { 1599 if (pci_msi_count(pdev->dev.bsddev) < minv) 1600 return (-ENOSPC); 1601 error = _lkpi_pci_enable_msi_range(pdev, minv, maxv); 1602 if (error == 0 && pdev->msi_enabled) 1603 return (pdev->dev.irq_end - pdev->dev.irq_start); 1604 } 1605 if (flags & PCI_IRQ_INTX) { 1606 if (pdev->irq) 1607 return (1); 1608 } 1609 1610 return (-EINVAL); 1611 } 1612 1613 struct msi_desc * 1614 lkpi_pci_msi_desc_alloc(unsigned int irq) 1615 { 1616 struct device *dev; 1617 struct pci_dev *pdev; 1618 struct msi_desc *desc; 1619 struct pci_devinfo *dinfo; 1620 struct pcicfg_msi *msi; 1621 int vec; 1622 1623 dev = lkpi_pci_find_irq_dev(irq); 1624 if (dev == NULL) 1625 return (NULL); 1626 1627 pdev = to_pci_dev(dev); 1628 1629 if (pdev->msi_desc == NULL) 1630 return (NULL); 1631 1632 if (irq < pdev->dev.irq_start || irq >= pdev->dev.irq_end) 1633 return (NULL); 1634 1635 vec = pdev->dev.irq_start - irq; 1636 1637 if (pdev->msi_desc[vec] != NULL) 1638 return (pdev->msi_desc[vec]); 1639 1640 dinfo = device_get_ivars(dev->bsddev); 1641 msi = &dinfo->cfg.msi; 1642 1643 desc = malloc(sizeof(*desc), M_DEVBUF, M_WAITOK | M_ZERO); 1644 1645 desc->pci.msi_attrib.is_64 = 1646 (msi->msi_ctrl & PCIM_MSICTRL_64BIT) ? true : false; 1647 desc->msg.data = msi->msi_data; 1648 1649 pdev->msi_desc[vec] = desc; 1650 1651 return (desc); 1652 } 1653 1654 bool 1655 pci_device_is_present(struct pci_dev *pdev) 1656 { 1657 device_t dev; 1658 1659 dev = pdev->dev.bsddev; 1660 1661 return (bus_child_present(dev)); 1662 } 1663 1664 void * 1665 linuxkpi_pci_map_rom(struct pci_dev *pdev, size_t *size) 1666 { 1667 device_t dev; 1668 1669 dev = pdev->dev.bsddev; 1670 1671 if (pci_get_class(dev) != PCIC_DISPLAY && 1672 (pci_get_class(dev) != PCIC_OLD || 1673 pci_get_subclass(dev) != PCIS_OLD_VGA)) { 1674 pr_debug("%s: TODO\n", __func__); 1675 return (NULL); 1676 } 1677 1678 return (vga_pci_map_bios(device_get_parent(dev), size)); 1679 } 1680 1681 void 1682 linuxkpi_pci_unmap_rom(struct pci_dev *pdev, void *rom) 1683 { 1684 device_t dev; 1685 1686 dev = pdev->dev.bsddev; 1687 1688 vga_pci_unmap_bios(device_get_parent(dev), rom); 1689 } 1690 1691 CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t)); 1692 1693 struct linux_dma_obj { 1694 void *vaddr; 1695 uint64_t dma_addr; 1696 bus_dmamap_t dmamap; 1697 bus_dma_tag_t dmat; 1698 }; 1699 1700 static uma_zone_t linux_dma_trie_zone; 1701 static uma_zone_t linux_dma_obj_zone; 1702 1703 static void 1704 linux_dma_init(void *arg) 1705 { 1706 1707 linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie", 1708 pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL, 1709 UMA_ALIGN_PTR, 0); 1710 linux_dma_obj_zone = uma_zcreate("linux_dma_object", 1711 sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL, 1712 UMA_ALIGN_PTR, 0); 1713 lkpi_pci_nseg1_fail = counter_u64_alloc(M_WAITOK); 1714 } 1715 SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL); 1716 1717 static void 1718 linux_dma_uninit(void *arg) 1719 { 1720 1721 counter_u64_free(lkpi_pci_nseg1_fail); 1722 uma_zdestroy(linux_dma_obj_zone); 1723 uma_zdestroy(linux_dma_trie_zone); 1724 } 1725 SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL); 1726 1727 static void * 1728 linux_dma_trie_alloc(struct pctrie *ptree) 1729 { 1730 1731 return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT)); 1732 } 1733 1734 static void 1735 linux_dma_trie_free(struct pctrie *ptree, void *node) 1736 { 1737 1738 uma_zfree(linux_dma_trie_zone, node); 1739 } 1740 1741 PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, 1742 linux_dma_trie_free); 1743 1744 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 1745 static dma_addr_t 1746 linux_dma_map_phys_common(struct device *dev, vm_paddr_t phys, size_t len, 1747 bus_dma_tag_t dmat) 1748 { 1749 struct linux_dma_priv *priv; 1750 struct linux_dma_obj *obj; 1751 int error, nseg; 1752 bus_dma_segment_t seg; 1753 1754 priv = dev->dma_priv; 1755 1756 /* 1757 * If the resultant mapping will be entirely 1:1 with the 1758 * physical address, short-circuit the remainder of the 1759 * bus_dma API. This avoids tracking collisions in the pctrie 1760 * with the additional benefit of reducing overhead. 1761 */ 1762 if (bus_dma_id_mapped(dmat, phys, len)) 1763 return (phys); 1764 1765 obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT); 1766 if (obj == NULL) { 1767 return (0); 1768 } 1769 obj->dmat = dmat; 1770 1771 DMA_PRIV_LOCK(priv); 1772 if (bus_dmamap_create(obj->dmat, 0, &obj->dmamap) != 0) { 1773 DMA_PRIV_UNLOCK(priv); 1774 uma_zfree(linux_dma_obj_zone, obj); 1775 return (0); 1776 } 1777 1778 nseg = -1; 1779 error = _bus_dmamap_load_phys(obj->dmat, obj->dmamap, phys, len, 1780 BUS_DMA_NOWAIT, &seg, &nseg); 1781 if (error != 0) { 1782 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1783 DMA_PRIV_UNLOCK(priv); 1784 uma_zfree(linux_dma_obj_zone, obj); 1785 counter_u64_add(lkpi_pci_nseg1_fail, 1); 1786 if (linuxkpi_debug) { 1787 device_printf(dev->bsddev, "%s: _bus_dmamap_load_phys " 1788 "error %d, phys %#018jx len %zu\n", __func__, 1789 error, (uintmax_t)phys, len); 1790 dump_stack(); 1791 } 1792 return (0); 1793 } 1794 1795 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 1796 obj->dma_addr = seg.ds_addr; 1797 1798 error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); 1799 if (error != 0) { 1800 bus_dmamap_unload(obj->dmat, obj->dmamap); 1801 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1802 DMA_PRIV_UNLOCK(priv); 1803 uma_zfree(linux_dma_obj_zone, obj); 1804 return (0); 1805 } 1806 DMA_PRIV_UNLOCK(priv); 1807 return (obj->dma_addr); 1808 } 1809 #else 1810 static dma_addr_t 1811 linux_dma_map_phys_common(struct device *dev __unused, vm_paddr_t phys, 1812 size_t len __unused, bus_dma_tag_t dmat __unused) 1813 { 1814 return (phys); 1815 } 1816 #endif 1817 1818 dma_addr_t 1819 lkpi_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len, 1820 enum dma_data_direction direction, unsigned long attrs) 1821 { 1822 struct linux_dma_priv *priv; 1823 dma_addr_t dma; 1824 1825 priv = dev->dma_priv; 1826 dma = linux_dma_map_phys_common(dev, phys, len, priv->dmat); 1827 if (dma_mapping_error(dev, dma)) 1828 return (dma); 1829 1830 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 1831 dma_sync_single_for_device(dev, dma, len, direction); 1832 1833 return (dma); 1834 } 1835 1836 /* For backward compat only so we can MFC this. Remove before 15. */ 1837 dma_addr_t 1838 linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) 1839 { 1840 return (lkpi_dma_map_phys(dev, phys, len, DMA_NONE, 0)); 1841 } 1842 1843 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 1844 void 1845 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, 1846 enum dma_data_direction direction, unsigned long attrs) 1847 { 1848 struct linux_dma_priv *priv; 1849 struct linux_dma_obj *obj; 1850 1851 priv = dev->dma_priv; 1852 1853 if (pctrie_is_empty(&priv->ptree)) 1854 return; 1855 1856 DMA_PRIV_LOCK(priv); 1857 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); 1858 if (obj == NULL) { 1859 DMA_PRIV_UNLOCK(priv); 1860 return; 1861 } 1862 LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); 1863 1864 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 1865 goto skip_sync; 1866 1867 /* dma_sync_single_for_cpu() unrolled to avoid lock recursicn. */ 1868 switch (direction) { 1869 case DMA_BIDIRECTIONAL: 1870 bus_dmamap_sync(obj->dmat, obj->dmamap, 1871 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1872 break; 1873 case DMA_TO_DEVICE: 1874 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_POSTWRITE); 1875 break; 1876 case DMA_FROM_DEVICE: 1877 bus_dmamap_sync(obj->dmat, obj->dmamap, BUS_DMASYNC_POSTREAD); 1878 break; 1879 default: 1880 break; 1881 } 1882 1883 skip_sync: 1884 bus_dmamap_unload(obj->dmat, obj->dmamap); 1885 bus_dmamap_destroy(obj->dmat, obj->dmamap); 1886 DMA_PRIV_UNLOCK(priv); 1887 1888 uma_zfree(linux_dma_obj_zone, obj); 1889 } 1890 #else 1891 void 1892 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len, 1893 enum dma_data_direction direction, unsigned long attrs) 1894 { 1895 } 1896 #endif 1897 1898 /* For backward compat only so we can MFC this. Remove before 15. */ 1899 void 1900 linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) 1901 { 1902 lkpi_dma_unmap(dev, dma_addr, len, DMA_NONE, 0); 1903 } 1904 1905 void * 1906 linux_dma_alloc_coherent(struct device *dev, size_t size, 1907 dma_addr_t *dma_handle, gfp_t flag) 1908 { 1909 struct linux_dma_priv *priv; 1910 vm_paddr_t high; 1911 size_t align; 1912 void *mem; 1913 1914 if (dev == NULL || dev->dma_priv == NULL) { 1915 *dma_handle = 0; 1916 return (NULL); 1917 } 1918 priv = dev->dma_priv; 1919 if (priv->dma_coherent_mask) 1920 high = priv->dma_coherent_mask; 1921 else 1922 /* Coherent is lower 32bit only by default in Linux. */ 1923 high = BUS_SPACE_MAXADDR_32BIT; 1924 align = PAGE_SIZE << get_order(size); 1925 /* Always zero the allocation. */ 1926 flag |= M_ZERO; 1927 mem = kmem_alloc_contig(size, flag & GFP_NATIVE_MASK, 0, high, 1928 align, 0, VM_MEMATTR_DEFAULT); 1929 if (mem != NULL) { 1930 *dma_handle = linux_dma_map_phys_common(dev, vtophys(mem), size, 1931 priv->dmat_coherent); 1932 if (*dma_handle == 0) { 1933 kmem_free(mem, size); 1934 mem = NULL; 1935 } 1936 } else { 1937 *dma_handle = 0; 1938 } 1939 return (mem); 1940 } 1941 1942 struct lkpi_devres_dmam_coherent { 1943 size_t size; 1944 dma_addr_t handle; 1945 void *mem; 1946 }; 1947 1948 static void 1949 lkpi_dmam_free_coherent(struct device *dev, void *p) 1950 { 1951 struct lkpi_devres_dmam_coherent *dr; 1952 1953 dr = p; 1954 dma_free_coherent(dev, dr->size, dr->mem, dr->handle); 1955 } 1956 1957 static int 1958 lkpi_dmam_coherent_match(struct device *dev, void *dr, void *mp) 1959 { 1960 struct lkpi_devres_dmam_coherent *a, *b; 1961 1962 a = dr; 1963 b = mp; 1964 1965 if (a->mem != b->mem) 1966 return (0); 1967 if (a->size != b->size || a->handle != b->handle) 1968 dev_WARN(dev, "for mem %p: size %zu != %zu || handle %#jx != %#jx\n", 1969 a->mem, a->size, b->size, 1970 (uintmax_t)a->handle, (uintmax_t)b->handle); 1971 return (1); 1972 } 1973 1974 void 1975 linuxkpi_dmam_free_coherent(struct device *dev, size_t size, 1976 void *addr, dma_addr_t dma_handle) 1977 { 1978 struct lkpi_devres_dmam_coherent match = { 1979 .size = size, 1980 .handle = dma_handle, 1981 .mem = addr 1982 }; 1983 int error; 1984 1985 error = devres_destroy(dev, lkpi_dmam_free_coherent, 1986 lkpi_dmam_coherent_match, &match); 1987 if (error != 0) 1988 dev_WARN(dev, "devres_destroy returned %d, size %zu addr %p " 1989 "dma_handle %#jx\n", error, size, addr, (uintmax_t)dma_handle); 1990 dma_free_coherent(dev, size, addr, dma_handle); 1991 } 1992 1993 void * 1994 linuxkpi_dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, 1995 gfp_t flag) 1996 { 1997 struct lkpi_devres_dmam_coherent *dr; 1998 1999 dr = lkpi_devres_alloc(lkpi_dmam_free_coherent, 2000 sizeof(*dr), GFP_KERNEL | __GFP_ZERO); 2001 2002 if (dr == NULL) 2003 return (NULL); 2004 2005 dr->size = size; 2006 dr->mem = linux_dma_alloc_coherent(dev, size, dma_handle, flag); 2007 dr->handle = *dma_handle; 2008 if (dr->mem == NULL) { 2009 lkpi_devres_free(dr); 2010 return (NULL); 2011 } 2012 2013 lkpi_devres_add(dev, dr); 2014 return (dr->mem); 2015 } 2016 2017 void * 2018 linuxkpi_dma_alloc_noncoherent(struct device *dev, size_t size, dma_addr_t *dma_handle, 2019 enum dma_data_direction direction, gfp_t gfp) 2020 { 2021 struct linux_dma_priv *priv; 2022 size_t align; 2023 void *mem; 2024 2025 size = PAGE_ALIGN(size); 2026 align = PAGE_SIZE << get_order(size); 2027 mem = kmem_alloc_contig(size, gfp & GFP_NATIVE_MASK, 0, BUS_SPACE_MAXADDR, 2028 align, 0, VM_MEMATTR_DEFAULT); 2029 if (mem == NULL) { 2030 *dma_handle = 0; 2031 return (NULL); 2032 } 2033 2034 priv = dev->dma_priv; 2035 *dma_handle = linux_dma_map_phys_common(dev, vtophys(mem), size, priv->dmat); 2036 if (*dma_handle == 0) { 2037 kmem_free(mem, size); 2038 mem = NULL; 2039 } 2040 return (mem); 2041 } 2042 2043 void 2044 linuxkpi_dma_free_noncoherent(struct device *dev, size_t size, void *vaddr, 2045 dma_addr_t dma_handle, enum dma_data_direction direction) 2046 { 2047 lkpi_dma_unmap(dev, dma_handle, size, direction, 0); 2048 kmem_free(vaddr, size); 2049 } 2050 2051 void * 2052 linuxkpi_dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 2053 gfp_t gfp, unsigned long attrs) 2054 { 2055 return (linuxkpi_dma_alloc_noncoherent(dev, size, dma_handle, 2056 DMA_BIDIRECTIONAL, gfp)); 2057 } 2058 2059 void 2060 linuxkpi_dma_free_attrs(struct device *dev, size_t size, void *vaddr, 2061 dma_addr_t dma_handle, unsigned long attrs) 2062 { 2063 linuxkpi_dma_free_noncoherent(dev, size, vaddr, dma_handle, 2064 DMA_BIDIRECTIONAL); 2065 } 2066 2067 void 2068 linuxkpi_dma_sync(struct device *dev, dma_addr_t dma_addr, size_t size, 2069 bus_dmasync_op_t op) 2070 { 2071 struct linux_dma_priv *priv; 2072 struct linux_dma_obj *obj; 2073 2074 priv = dev->dma_priv; 2075 2076 if (pctrie_is_empty(&priv->ptree)) 2077 return; 2078 2079 DMA_PRIV_LOCK(priv); 2080 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); 2081 if (obj == NULL) { 2082 DMA_PRIV_UNLOCK(priv); 2083 return; 2084 } 2085 2086 bus_dmamap_sync(obj->dmat, obj->dmamap, op); 2087 DMA_PRIV_UNLOCK(priv); 2088 } 2089 2090 void 2091 lkpi_dma_sync_sg(struct device *dev, struct scatterlist *sgl, bus_dmasync_op_t op) 2092 { 2093 struct linux_dma_priv *priv; 2094 2095 priv = dev->dma_priv; 2096 DMA_PRIV_LOCK(priv); 2097 bus_dmamap_sync(priv->dmat, sgl->dma_map, op); 2098 DMA_PRIV_UNLOCK(priv); 2099 } 2100 2101 int 2102 linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, 2103 enum dma_data_direction direction, unsigned long attrs) 2104 { 2105 struct linux_dma_priv *priv; 2106 struct scatterlist *sg; 2107 int i, nseg; 2108 bus_dma_segment_t seg; 2109 2110 priv = dev->dma_priv; 2111 2112 DMA_PRIV_LOCK(priv); 2113 2114 /* create common DMA map in the first S/G entry */ 2115 if (bus_dmamap_create(priv->dmat, 0, &sgl->dma_map) != 0) { 2116 DMA_PRIV_UNLOCK(priv); 2117 return (0); 2118 } 2119 2120 /* load all S/G list entries */ 2121 for_each_sg(sgl, sg, nents, i) { 2122 nseg = -1; 2123 if (_bus_dmamap_load_phys(priv->dmat, sgl->dma_map, 2124 sg_phys(sg), sg->length, BUS_DMA_NOWAIT, 2125 &seg, &nseg) != 0) { 2126 bus_dmamap_unload(priv->dmat, sgl->dma_map); 2127 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 2128 DMA_PRIV_UNLOCK(priv); 2129 return (0); 2130 } 2131 KASSERT(nseg == 0, 2132 ("More than one segment (nseg=%d)", nseg + 1)); 2133 2134 sg_dma_address(sg) = seg.ds_addr; 2135 sg->dma_length = sg->length; 2136 } 2137 2138 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 2139 goto skip_sync; 2140 2141 switch (direction) { 2142 case DMA_BIDIRECTIONAL: 2143 bus_dmamap_sync(priv->dmat, sgl->dma_map, 2144 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2145 break; 2146 case DMA_TO_DEVICE: 2147 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE); 2148 break; 2149 case DMA_FROM_DEVICE: 2150 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREREAD); 2151 break; 2152 default: 2153 break; 2154 } 2155 skip_sync: 2156 2157 DMA_PRIV_UNLOCK(priv); 2158 2159 return (nents); 2160 } 2161 2162 void 2163 linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, 2164 int nents __unused, enum dma_data_direction direction, 2165 unsigned long attrs) 2166 { 2167 struct linux_dma_priv *priv; 2168 2169 priv = dev->dma_priv; 2170 2171 DMA_PRIV_LOCK(priv); 2172 2173 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0) 2174 goto skip_sync; 2175 2176 switch (direction) { 2177 case DMA_BIDIRECTIONAL: 2178 bus_dmamap_sync(priv->dmat, sgl->dma_map, 2179 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2180 break; 2181 case DMA_TO_DEVICE: 2182 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTWRITE); 2183 break; 2184 case DMA_FROM_DEVICE: 2185 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD); 2186 break; 2187 default: 2188 break; 2189 } 2190 skip_sync: 2191 2192 bus_dmamap_unload(priv->dmat, sgl->dma_map); 2193 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 2194 DMA_PRIV_UNLOCK(priv); 2195 } 2196 2197 struct dma_pool { 2198 struct device *pool_device; 2199 uma_zone_t pool_zone; 2200 struct mtx pool_lock; 2201 bus_dma_tag_t pool_dmat; 2202 size_t pool_entry_size; 2203 struct pctrie pool_ptree; 2204 }; 2205 2206 #define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock) 2207 #define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock) 2208 2209 static inline int 2210 dma_pool_obj_ctor(void *mem, int size, void *arg, int flags) 2211 { 2212 struct linux_dma_obj *obj = mem; 2213 struct dma_pool *pool = arg; 2214 int error, nseg; 2215 bus_dma_segment_t seg; 2216 2217 nseg = -1; 2218 DMA_POOL_LOCK(pool); 2219 error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap, 2220 vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT, 2221 &seg, &nseg); 2222 DMA_POOL_UNLOCK(pool); 2223 if (error != 0) { 2224 return (error); 2225 } 2226 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 2227 obj->dma_addr = seg.ds_addr; 2228 2229 return (0); 2230 } 2231 2232 static void 2233 dma_pool_obj_dtor(void *mem, int size, void *arg) 2234 { 2235 struct linux_dma_obj *obj = mem; 2236 struct dma_pool *pool = arg; 2237 2238 DMA_POOL_LOCK(pool); 2239 bus_dmamap_unload(pool->pool_dmat, obj->dmamap); 2240 DMA_POOL_UNLOCK(pool); 2241 } 2242 2243 static int 2244 dma_pool_obj_import(void *arg, void **store, int count, int domain __unused, 2245 int flags) 2246 { 2247 struct dma_pool *pool = arg; 2248 struct linux_dma_obj *obj; 2249 int error, i; 2250 2251 for (i = 0; i < count; i++) { 2252 obj = uma_zalloc(linux_dma_obj_zone, flags); 2253 if (obj == NULL) 2254 break; 2255 2256 error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr, 2257 BUS_DMA_NOWAIT, &obj->dmamap); 2258 if (error!= 0) { 2259 uma_zfree(linux_dma_obj_zone, obj); 2260 break; 2261 } 2262 2263 store[i] = obj; 2264 } 2265 2266 return (i); 2267 } 2268 2269 static void 2270 dma_pool_obj_release(void *arg, void **store, int count) 2271 { 2272 struct dma_pool *pool = arg; 2273 struct linux_dma_obj *obj; 2274 int i; 2275 2276 for (i = 0; i < count; i++) { 2277 obj = store[i]; 2278 bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap); 2279 uma_zfree(linux_dma_obj_zone, obj); 2280 } 2281 } 2282 2283 struct dma_pool * 2284 linux_dma_pool_create(char *name, struct device *dev, size_t size, 2285 size_t align, size_t boundary) 2286 { 2287 struct linux_dma_priv *priv; 2288 struct dma_pool *pool; 2289 2290 priv = dev->dma_priv; 2291 2292 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 2293 pool->pool_device = dev; 2294 pool->pool_entry_size = size; 2295 2296 if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 2297 align, boundary, /* alignment, boundary */ 2298 priv->dma_mask, /* lowaddr */ 2299 BUS_SPACE_MAXADDR, /* highaddr */ 2300 NULL, NULL, /* filtfunc, filtfuncarg */ 2301 size, /* maxsize */ 2302 1, /* nsegments */ 2303 size, /* maxsegsz */ 2304 0, /* flags */ 2305 NULL, NULL, /* lockfunc, lockfuncarg */ 2306 &pool->pool_dmat)) { 2307 kfree(pool); 2308 return (NULL); 2309 } 2310 2311 pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor, 2312 dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import, 2313 dma_pool_obj_release, pool, 0); 2314 2315 mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF); 2316 pctrie_init(&pool->pool_ptree); 2317 2318 return (pool); 2319 } 2320 2321 void 2322 linux_dma_pool_destroy(struct dma_pool *pool) 2323 { 2324 2325 uma_zdestroy(pool->pool_zone); 2326 bus_dma_tag_destroy(pool->pool_dmat); 2327 mtx_destroy(&pool->pool_lock); 2328 kfree(pool); 2329 } 2330 2331 void 2332 lkpi_dmam_pool_destroy(struct device *dev, void *p) 2333 { 2334 struct dma_pool *pool; 2335 2336 pool = *(struct dma_pool **)p; 2337 LINUX_DMA_PCTRIE_RECLAIM(&pool->pool_ptree); 2338 linux_dma_pool_destroy(pool); 2339 } 2340 2341 void * 2342 linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, 2343 dma_addr_t *handle) 2344 { 2345 struct linux_dma_obj *obj; 2346 2347 obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags & GFP_NATIVE_MASK); 2348 if (obj == NULL) 2349 return (NULL); 2350 2351 DMA_POOL_LOCK(pool); 2352 if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) { 2353 DMA_POOL_UNLOCK(pool); 2354 uma_zfree_arg(pool->pool_zone, obj, pool); 2355 return (NULL); 2356 } 2357 DMA_POOL_UNLOCK(pool); 2358 2359 *handle = obj->dma_addr; 2360 return (obj->vaddr); 2361 } 2362 2363 void 2364 linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr) 2365 { 2366 struct linux_dma_obj *obj; 2367 2368 DMA_POOL_LOCK(pool); 2369 obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr); 2370 if (obj == NULL) { 2371 DMA_POOL_UNLOCK(pool); 2372 return; 2373 } 2374 LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr); 2375 DMA_POOL_UNLOCK(pool); 2376 2377 uma_zfree_arg(pool->pool_zone, obj, pool); 2378 } 2379 2380 static int 2381 linux_backlight_get_status(device_t dev, struct backlight_props *props) 2382 { 2383 struct pci_dev *pdev; 2384 2385 linux_set_current(curthread); 2386 pdev = device_get_softc(dev); 2387 2388 props->brightness = pdev->dev.bd->props.brightness; 2389 props->brightness = props->brightness * 100 / pdev->dev.bd->props.max_brightness; 2390 props->nlevels = 0; 2391 2392 return (0); 2393 } 2394 2395 static int 2396 linux_backlight_get_info(device_t dev, struct backlight_info *info) 2397 { 2398 struct pci_dev *pdev; 2399 2400 linux_set_current(curthread); 2401 pdev = device_get_softc(dev); 2402 2403 info->type = BACKLIGHT_TYPE_PANEL; 2404 strlcpy(info->name, pdev->dev.bd->name, BACKLIGHTMAXNAMELENGTH); 2405 return (0); 2406 } 2407 2408 static int 2409 linux_backlight_update_status(device_t dev, struct backlight_props *props) 2410 { 2411 struct pci_dev *pdev; 2412 2413 linux_set_current(curthread); 2414 pdev = device_get_softc(dev); 2415 2416 pdev->dev.bd->props.brightness = pdev->dev.bd->props.max_brightness * 2417 props->brightness / 100; 2418 pdev->dev.bd->props.power = props->brightness == 0 ? 2419 4/* FB_BLANK_POWERDOWN */ : 0/* FB_BLANK_UNBLANK */; 2420 return (pdev->dev.bd->ops->update_status(pdev->dev.bd)); 2421 } 2422 2423 struct backlight_device * 2424 linux_backlight_device_register(const char *name, struct device *dev, 2425 void *data, const struct backlight_ops *ops, struct backlight_properties *props) 2426 { 2427 2428 dev->bd = malloc(sizeof(*dev->bd), M_DEVBUF, M_WAITOK | M_ZERO); 2429 dev->bd->ops = ops; 2430 dev->bd->props.type = props->type; 2431 dev->bd->props.max_brightness = props->max_brightness; 2432 dev->bd->props.brightness = props->brightness; 2433 dev->bd->props.power = props->power; 2434 dev->bd->data = data; 2435 dev->bd->dev = dev; 2436 dev->bd->name = strdup(name, M_DEVBUF); 2437 2438 dev->backlight_dev = backlight_register(name, dev->bsddev); 2439 2440 return (dev->bd); 2441 } 2442 2443 void 2444 linux_backlight_device_unregister(struct backlight_device *bd) 2445 { 2446 2447 backlight_destroy(bd->dev->backlight_dev); 2448 free(bd->name, M_DEVBUF); 2449 free(bd, M_DEVBUF); 2450 } 2451