1 /*- 2 * Copyright (c) 2015-2016 Mellanox Technologies, Ltd. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/bus.h> 33 #include <sys/malloc.h> 34 #include <sys/kernel.h> 35 #include <sys/sysctl.h> 36 #include <sys/lock.h> 37 #include <sys/mutex.h> 38 #include <sys/fcntl.h> 39 #include <sys/file.h> 40 #include <sys/filio.h> 41 #include <sys/pciio.h> 42 #include <sys/pctrie.h> 43 #include <sys/rwlock.h> 44 45 #include <vm/vm.h> 46 #include <vm/pmap.h> 47 48 #include <machine/stdarg.h> 49 50 #include <dev/pci/pcivar.h> 51 #include <dev/pci/pci_private.h> 52 #include <dev/pci/pci_iov.h> 53 #include <dev/backlight/backlight.h> 54 55 #include <linux/kobject.h> 56 #include <linux/device.h> 57 #include <linux/slab.h> 58 #include <linux/module.h> 59 #include <linux/cdev.h> 60 #include <linux/file.h> 61 #include <linux/sysfs.h> 62 #include <linux/mm.h> 63 #include <linux/io.h> 64 #include <linux/vmalloc.h> 65 #include <linux/pci.h> 66 #include <linux/compat.h> 67 68 #include <linux/backlight.h> 69 70 #include "backlight_if.h" 71 #include "pcib_if.h" 72 73 static device_probe_t linux_pci_probe; 74 static device_attach_t linux_pci_attach; 75 static device_detach_t linux_pci_detach; 76 static device_suspend_t linux_pci_suspend; 77 static device_resume_t linux_pci_resume; 78 static device_shutdown_t linux_pci_shutdown; 79 static pci_iov_init_t linux_pci_iov_init; 80 static pci_iov_uninit_t linux_pci_iov_uninit; 81 static pci_iov_add_vf_t linux_pci_iov_add_vf; 82 static int linux_backlight_get_status(device_t dev, struct backlight_props *props); 83 static int linux_backlight_update_status(device_t dev, struct backlight_props *props); 84 static int linux_backlight_get_info(device_t dev, struct backlight_info *info); 85 86 static device_method_t pci_methods[] = { 87 DEVMETHOD(device_probe, linux_pci_probe), 88 DEVMETHOD(device_attach, linux_pci_attach), 89 DEVMETHOD(device_detach, linux_pci_detach), 90 DEVMETHOD(device_suspend, linux_pci_suspend), 91 DEVMETHOD(device_resume, linux_pci_resume), 92 DEVMETHOD(device_shutdown, linux_pci_shutdown), 93 DEVMETHOD(pci_iov_init, linux_pci_iov_init), 94 DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit), 95 DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf), 96 97 /* backlight interface */ 98 DEVMETHOD(backlight_update_status, linux_backlight_update_status), 99 DEVMETHOD(backlight_get_status, linux_backlight_get_status), 100 DEVMETHOD(backlight_get_info, linux_backlight_get_info), 101 DEVMETHOD_END 102 }; 103 104 struct linux_dma_priv { 105 uint64_t dma_mask; 106 struct mtx lock; 107 bus_dma_tag_t dmat; 108 struct pctrie ptree; 109 }; 110 #define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock) 111 #define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock) 112 113 static int 114 linux_pdev_dma_init(struct pci_dev *pdev) 115 { 116 struct linux_dma_priv *priv; 117 int error; 118 119 priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO); 120 pdev->dev.dma_priv = priv; 121 122 mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF); 123 124 pctrie_init(&priv->ptree); 125 126 /* create a default DMA tag */ 127 error = linux_dma_tag_init(&pdev->dev, DMA_BIT_MASK(64)); 128 if (error) { 129 mtx_destroy(&priv->lock); 130 free(priv, M_DEVBUF); 131 pdev->dev.dma_priv = NULL; 132 } 133 return (error); 134 } 135 136 static int 137 linux_pdev_dma_uninit(struct pci_dev *pdev) 138 { 139 struct linux_dma_priv *priv; 140 141 priv = pdev->dev.dma_priv; 142 if (priv->dmat) 143 bus_dma_tag_destroy(priv->dmat); 144 mtx_destroy(&priv->lock); 145 free(priv, M_DEVBUF); 146 pdev->dev.dma_priv = NULL; 147 return (0); 148 } 149 150 int 151 linux_dma_tag_init(struct device *dev, u64 dma_mask) 152 { 153 struct linux_dma_priv *priv; 154 int error; 155 156 priv = dev->dma_priv; 157 158 if (priv->dmat) { 159 if (priv->dma_mask == dma_mask) 160 return (0); 161 162 bus_dma_tag_destroy(priv->dmat); 163 } 164 165 priv->dma_mask = dma_mask; 166 167 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 168 1, 0, /* alignment, boundary */ 169 dma_mask, /* lowaddr */ 170 BUS_SPACE_MAXADDR, /* highaddr */ 171 NULL, NULL, /* filtfunc, filtfuncarg */ 172 BUS_SPACE_MAXSIZE, /* maxsize */ 173 1, /* nsegments */ 174 BUS_SPACE_MAXSIZE, /* maxsegsz */ 175 0, /* flags */ 176 NULL, NULL, /* lockfunc, lockfuncarg */ 177 &priv->dmat); 178 return (-error); 179 } 180 181 static struct pci_driver * 182 linux_pci_find(device_t dev, const struct pci_device_id **idp) 183 { 184 const struct pci_device_id *id; 185 struct pci_driver *pdrv; 186 uint16_t vendor; 187 uint16_t device; 188 uint16_t subvendor; 189 uint16_t subdevice; 190 191 vendor = pci_get_vendor(dev); 192 device = pci_get_device(dev); 193 subvendor = pci_get_subvendor(dev); 194 subdevice = pci_get_subdevice(dev); 195 196 spin_lock(&pci_lock); 197 list_for_each_entry(pdrv, &pci_drivers, links) { 198 for (id = pdrv->id_table; id->vendor != 0; id++) { 199 if (vendor == id->vendor && 200 (PCI_ANY_ID == id->device || device == id->device) && 201 (PCI_ANY_ID == id->subvendor || subvendor == id->subvendor) && 202 (PCI_ANY_ID == id->subdevice || subdevice == id->subdevice)) { 203 *idp = id; 204 spin_unlock(&pci_lock); 205 return (pdrv); 206 } 207 } 208 } 209 spin_unlock(&pci_lock); 210 return (NULL); 211 } 212 213 static int 214 linux_pci_probe(device_t dev) 215 { 216 const struct pci_device_id *id; 217 struct pci_driver *pdrv; 218 219 if ((pdrv = linux_pci_find(dev, &id)) == NULL) 220 return (ENXIO); 221 if (device_get_driver(dev) != &pdrv->bsddriver) 222 return (ENXIO); 223 device_set_desc(dev, pdrv->name); 224 return (0); 225 } 226 227 static int 228 linux_pci_attach(device_t dev) 229 { 230 const struct pci_device_id *id; 231 struct pci_driver *pdrv; 232 struct pci_dev *pdev; 233 234 pdrv = linux_pci_find(dev, &id); 235 pdev = device_get_softc(dev); 236 237 MPASS(pdrv != NULL); 238 MPASS(pdev != NULL); 239 240 return (linux_pci_attach_device(dev, pdrv, id, pdev)); 241 } 242 243 int 244 linux_pci_attach_device(device_t dev, struct pci_driver *pdrv, 245 const struct pci_device_id *id, struct pci_dev *pdev) 246 { 247 struct resource_list_entry *rle; 248 struct pci_bus *pbus; 249 struct pci_devinfo *dinfo; 250 device_t parent; 251 uintptr_t rid; 252 int error; 253 bool isdrm; 254 255 linux_set_current(curthread); 256 257 parent = device_get_parent(dev); 258 isdrm = pdrv != NULL && pdrv->isdrm; 259 260 if (isdrm) { 261 dinfo = device_get_ivars(parent); 262 device_set_ivars(dev, dinfo); 263 } else { 264 dinfo = device_get_ivars(dev); 265 } 266 267 pdev->dev.parent = &linux_root_device; 268 pdev->dev.bsddev = dev; 269 INIT_LIST_HEAD(&pdev->dev.irqents); 270 if (isdrm) 271 PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid); 272 else 273 PCI_GET_ID(parent, dev, PCI_ID_RID, &rid); 274 pdev->devfn = rid; 275 pdev->device = dinfo->cfg.device; 276 pdev->vendor = dinfo->cfg.vendor; 277 pdev->subsystem_vendor = dinfo->cfg.subvendor; 278 pdev->subsystem_device = dinfo->cfg.subdevice; 279 pdev->class = pci_get_class(dev); 280 pdev->revision = pci_get_revid(dev); 281 pdev->pdrv = pdrv; 282 kobject_init(&pdev->dev.kobj, &linux_dev_ktype); 283 kobject_set_name(&pdev->dev.kobj, device_get_nameunit(dev)); 284 kobject_add(&pdev->dev.kobj, &linux_root_device.kobj, 285 kobject_name(&pdev->dev.kobj)); 286 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0); 287 if (rle != NULL) 288 pdev->dev.irq = rle->start; 289 else 290 pdev->dev.irq = LINUX_IRQ_INVALID; 291 pdev->irq = pdev->dev.irq; 292 error = linux_pdev_dma_init(pdev); 293 if (error) 294 goto out_dma_init; 295 296 TAILQ_INIT(&pdev->mmio); 297 pbus = malloc(sizeof(*pbus), M_DEVBUF, M_WAITOK | M_ZERO); 298 pbus->self = pdev; 299 pbus->number = pci_get_bus(dev); 300 pbus->domain = pci_get_domain(dev); 301 pdev->bus = pbus; 302 303 spin_lock(&pci_lock); 304 list_add(&pdev->links, &pci_devices); 305 spin_unlock(&pci_lock); 306 307 if (pdrv != NULL) { 308 error = pdrv->probe(pdev, id); 309 if (error) 310 goto out_probe; 311 } 312 return (0); 313 314 out_probe: 315 free(pdev->bus, M_DEVBUF); 316 linux_pdev_dma_uninit(pdev); 317 out_dma_init: 318 spin_lock(&pci_lock); 319 list_del(&pdev->links); 320 spin_unlock(&pci_lock); 321 put_device(&pdev->dev); 322 return (-error); 323 } 324 325 static int 326 linux_pci_detach(device_t dev) 327 { 328 struct pci_dev *pdev; 329 330 pdev = device_get_softc(dev); 331 332 MPASS(pdev != NULL); 333 334 device_set_desc(dev, NULL); 335 336 return (linux_pci_detach_device(pdev)); 337 } 338 339 int 340 linux_pci_detach_device(struct pci_dev *pdev) 341 { 342 343 linux_set_current(curthread); 344 345 if (pdev->pdrv != NULL) 346 pdev->pdrv->remove(pdev); 347 348 free(pdev->bus, M_DEVBUF); 349 linux_pdev_dma_uninit(pdev); 350 351 spin_lock(&pci_lock); 352 list_del(&pdev->links); 353 spin_unlock(&pci_lock); 354 put_device(&pdev->dev); 355 356 return (0); 357 } 358 359 static int 360 linux_pci_suspend(device_t dev) 361 { 362 const struct dev_pm_ops *pmops; 363 struct pm_message pm = { }; 364 struct pci_dev *pdev; 365 int error; 366 367 error = 0; 368 linux_set_current(curthread); 369 pdev = device_get_softc(dev); 370 pmops = pdev->pdrv->driver.pm; 371 372 if (pdev->pdrv->suspend != NULL) 373 error = -pdev->pdrv->suspend(pdev, pm); 374 else if (pmops != NULL && pmops->suspend != NULL) { 375 error = -pmops->suspend(&pdev->dev); 376 if (error == 0 && pmops->suspend_late != NULL) 377 error = -pmops->suspend_late(&pdev->dev); 378 } 379 return (error); 380 } 381 382 static int 383 linux_pci_resume(device_t dev) 384 { 385 const struct dev_pm_ops *pmops; 386 struct pci_dev *pdev; 387 int error; 388 389 error = 0; 390 linux_set_current(curthread); 391 pdev = device_get_softc(dev); 392 pmops = pdev->pdrv->driver.pm; 393 394 if (pdev->pdrv->resume != NULL) 395 error = -pdev->pdrv->resume(pdev); 396 else if (pmops != NULL && pmops->resume != NULL) { 397 if (pmops->resume_early != NULL) 398 error = -pmops->resume_early(&pdev->dev); 399 if (error == 0 && pmops->resume != NULL) 400 error = -pmops->resume(&pdev->dev); 401 } 402 return (error); 403 } 404 405 static int 406 linux_pci_shutdown(device_t dev) 407 { 408 struct pci_dev *pdev; 409 410 linux_set_current(curthread); 411 pdev = device_get_softc(dev); 412 if (pdev->pdrv->shutdown != NULL) 413 pdev->pdrv->shutdown(pdev); 414 return (0); 415 } 416 417 static int 418 linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config) 419 { 420 struct pci_dev *pdev; 421 int error; 422 423 linux_set_current(curthread); 424 pdev = device_get_softc(dev); 425 if (pdev->pdrv->bsd_iov_init != NULL) 426 error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config); 427 else 428 error = EINVAL; 429 return (error); 430 } 431 432 static void 433 linux_pci_iov_uninit(device_t dev) 434 { 435 struct pci_dev *pdev; 436 437 linux_set_current(curthread); 438 pdev = device_get_softc(dev); 439 if (pdev->pdrv->bsd_iov_uninit != NULL) 440 pdev->pdrv->bsd_iov_uninit(dev); 441 } 442 443 static int 444 linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config) 445 { 446 struct pci_dev *pdev; 447 int error; 448 449 linux_set_current(curthread); 450 pdev = device_get_softc(dev); 451 if (pdev->pdrv->bsd_iov_add_vf != NULL) 452 error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config); 453 else 454 error = EINVAL; 455 return (error); 456 } 457 458 static int 459 _linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc) 460 { 461 int error; 462 463 linux_set_current(curthread); 464 spin_lock(&pci_lock); 465 list_add(&pdrv->links, &pci_drivers); 466 spin_unlock(&pci_lock); 467 pdrv->bsddriver.name = pdrv->name; 468 pdrv->bsddriver.methods = pci_methods; 469 pdrv->bsddriver.size = sizeof(struct pci_dev); 470 471 mtx_lock(&Giant); 472 error = devclass_add_driver(dc, &pdrv->bsddriver, 473 BUS_PASS_DEFAULT, &pdrv->bsdclass); 474 mtx_unlock(&Giant); 475 return (-error); 476 } 477 478 int 479 linux_pci_register_driver(struct pci_driver *pdrv) 480 { 481 devclass_t dc; 482 483 dc = devclass_find("pci"); 484 if (dc == NULL) 485 return (-ENXIO); 486 pdrv->isdrm = false; 487 return (_linux_pci_register_driver(pdrv, dc)); 488 } 489 490 unsigned long 491 pci_resource_start(struct pci_dev *pdev, int bar) 492 { 493 struct resource_list_entry *rle; 494 rman_res_t newstart; 495 device_t dev; 496 497 if ((rle = linux_pci_get_bar(pdev, bar)) == NULL) 498 return (0); 499 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? 500 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; 501 if (BUS_TRANSLATE_RESOURCE(dev, rle->type, rle->start, &newstart)) { 502 device_printf(pdev->dev.bsddev, "translate of %#jx failed\n", 503 (uintmax_t)rle->start); 504 return (0); 505 } 506 return (newstart); 507 } 508 509 unsigned long 510 pci_resource_len(struct pci_dev *pdev, int bar) 511 { 512 struct resource_list_entry *rle; 513 514 if ((rle = linux_pci_get_bar(pdev, bar)) == NULL) 515 return (0); 516 return (rle->count); 517 } 518 519 int 520 linux_pci_register_drm_driver(struct pci_driver *pdrv) 521 { 522 devclass_t dc; 523 524 dc = devclass_create("vgapci"); 525 if (dc == NULL) 526 return (-ENXIO); 527 pdrv->isdrm = true; 528 pdrv->name = "drmn"; 529 return (_linux_pci_register_driver(pdrv, dc)); 530 } 531 532 void 533 linux_pci_unregister_driver(struct pci_driver *pdrv) 534 { 535 devclass_t bus; 536 537 bus = devclass_find("pci"); 538 539 spin_lock(&pci_lock); 540 list_del(&pdrv->links); 541 spin_unlock(&pci_lock); 542 mtx_lock(&Giant); 543 if (bus != NULL) 544 devclass_delete_driver(bus, &pdrv->bsddriver); 545 mtx_unlock(&Giant); 546 } 547 548 void 549 linux_pci_unregister_drm_driver(struct pci_driver *pdrv) 550 { 551 devclass_t bus; 552 553 bus = devclass_find("vgapci"); 554 555 spin_lock(&pci_lock); 556 list_del(&pdrv->links); 557 spin_unlock(&pci_lock); 558 mtx_lock(&Giant); 559 if (bus != NULL) 560 devclass_delete_driver(bus, &pdrv->bsddriver); 561 mtx_unlock(&Giant); 562 } 563 564 CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t)); 565 566 struct linux_dma_obj { 567 void *vaddr; 568 uint64_t dma_addr; 569 bus_dmamap_t dmamap; 570 }; 571 572 static uma_zone_t linux_dma_trie_zone; 573 static uma_zone_t linux_dma_obj_zone; 574 575 static void 576 linux_dma_init(void *arg) 577 { 578 579 linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie", 580 pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL, 581 UMA_ALIGN_PTR, 0); 582 linux_dma_obj_zone = uma_zcreate("linux_dma_object", 583 sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL, 584 UMA_ALIGN_PTR, 0); 585 586 } 587 SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL); 588 589 static void 590 linux_dma_uninit(void *arg) 591 { 592 593 uma_zdestroy(linux_dma_obj_zone); 594 uma_zdestroy(linux_dma_trie_zone); 595 } 596 SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL); 597 598 static void * 599 linux_dma_trie_alloc(struct pctrie *ptree) 600 { 601 602 return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT)); 603 } 604 605 static void 606 linux_dma_trie_free(struct pctrie *ptree, void *node) 607 { 608 609 uma_zfree(linux_dma_trie_zone, node); 610 } 611 612 PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, 613 linux_dma_trie_free); 614 615 void * 616 linux_dma_alloc_coherent(struct device *dev, size_t size, 617 dma_addr_t *dma_handle, gfp_t flag) 618 { 619 struct linux_dma_priv *priv; 620 vm_paddr_t high; 621 size_t align; 622 void *mem; 623 624 if (dev == NULL || dev->dma_priv == NULL) { 625 *dma_handle = 0; 626 return (NULL); 627 } 628 priv = dev->dma_priv; 629 if (priv->dma_mask) 630 high = priv->dma_mask; 631 else if (flag & GFP_DMA32) 632 high = BUS_SPACE_MAXADDR_32BIT; 633 else 634 high = BUS_SPACE_MAXADDR; 635 align = PAGE_SIZE << get_order(size); 636 mem = (void *)kmem_alloc_contig(size, flag & GFP_NATIVE_MASK, 0, high, 637 align, 0, VM_MEMATTR_DEFAULT); 638 if (mem != NULL) { 639 *dma_handle = linux_dma_map_phys(dev, vtophys(mem), size); 640 if (*dma_handle == 0) { 641 kmem_free((vm_offset_t)mem, size); 642 mem = NULL; 643 } 644 } else { 645 *dma_handle = 0; 646 } 647 return (mem); 648 } 649 650 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 651 dma_addr_t 652 linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) 653 { 654 struct linux_dma_priv *priv; 655 struct linux_dma_obj *obj; 656 int error, nseg; 657 bus_dma_segment_t seg; 658 659 priv = dev->dma_priv; 660 661 /* 662 * If the resultant mapping will be entirely 1:1 with the 663 * physical address, short-circuit the remainder of the 664 * bus_dma API. This avoids tracking collisions in the pctrie 665 * with the additional benefit of reducing overhead. 666 */ 667 if (bus_dma_id_mapped(priv->dmat, phys, len)) 668 return (phys); 669 670 obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT); 671 if (obj == NULL) { 672 return (0); 673 } 674 675 DMA_PRIV_LOCK(priv); 676 if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) { 677 DMA_PRIV_UNLOCK(priv); 678 uma_zfree(linux_dma_obj_zone, obj); 679 return (0); 680 } 681 682 nseg = -1; 683 if (_bus_dmamap_load_phys(priv->dmat, obj->dmamap, phys, len, 684 BUS_DMA_NOWAIT, &seg, &nseg) != 0) { 685 bus_dmamap_destroy(priv->dmat, obj->dmamap); 686 DMA_PRIV_UNLOCK(priv); 687 uma_zfree(linux_dma_obj_zone, obj); 688 return (0); 689 } 690 691 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 692 obj->dma_addr = seg.ds_addr; 693 694 error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); 695 if (error != 0) { 696 bus_dmamap_unload(priv->dmat, obj->dmamap); 697 bus_dmamap_destroy(priv->dmat, obj->dmamap); 698 DMA_PRIV_UNLOCK(priv); 699 uma_zfree(linux_dma_obj_zone, obj); 700 return (0); 701 } 702 DMA_PRIV_UNLOCK(priv); 703 return (obj->dma_addr); 704 } 705 #else 706 dma_addr_t 707 linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) 708 { 709 return (phys); 710 } 711 #endif 712 713 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) 714 void 715 linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) 716 { 717 struct linux_dma_priv *priv; 718 struct linux_dma_obj *obj; 719 720 priv = dev->dma_priv; 721 722 if (pctrie_is_empty(&priv->ptree)) 723 return; 724 725 DMA_PRIV_LOCK(priv); 726 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); 727 if (obj == NULL) { 728 DMA_PRIV_UNLOCK(priv); 729 return; 730 } 731 LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); 732 bus_dmamap_unload(priv->dmat, obj->dmamap); 733 bus_dmamap_destroy(priv->dmat, obj->dmamap); 734 DMA_PRIV_UNLOCK(priv); 735 736 uma_zfree(linux_dma_obj_zone, obj); 737 } 738 #else 739 void 740 linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) 741 { 742 } 743 #endif 744 745 int 746 linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, 747 enum dma_data_direction dir, struct dma_attrs *attrs) 748 { 749 struct linux_dma_priv *priv; 750 struct scatterlist *sg; 751 int i, nseg; 752 bus_dma_segment_t seg; 753 754 priv = dev->dma_priv; 755 756 DMA_PRIV_LOCK(priv); 757 758 /* create common DMA map in the first S/G entry */ 759 if (bus_dmamap_create(priv->dmat, 0, &sgl->dma_map) != 0) { 760 DMA_PRIV_UNLOCK(priv); 761 return (0); 762 } 763 764 /* load all S/G list entries */ 765 for_each_sg(sgl, sg, nents, i) { 766 nseg = -1; 767 if (_bus_dmamap_load_phys(priv->dmat, sgl->dma_map, 768 sg_phys(sg), sg->length, BUS_DMA_NOWAIT, 769 &seg, &nseg) != 0) { 770 bus_dmamap_unload(priv->dmat, sgl->dma_map); 771 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 772 DMA_PRIV_UNLOCK(priv); 773 return (0); 774 } 775 KASSERT(nseg == 0, 776 ("More than one segment (nseg=%d)", nseg + 1)); 777 778 sg_dma_address(sg) = seg.ds_addr; 779 } 780 DMA_PRIV_UNLOCK(priv); 781 782 return (nents); 783 } 784 785 void 786 linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, 787 int nents, enum dma_data_direction dir, struct dma_attrs *attrs) 788 { 789 struct linux_dma_priv *priv; 790 791 priv = dev->dma_priv; 792 793 DMA_PRIV_LOCK(priv); 794 bus_dmamap_unload(priv->dmat, sgl->dma_map); 795 bus_dmamap_destroy(priv->dmat, sgl->dma_map); 796 DMA_PRIV_UNLOCK(priv); 797 } 798 799 struct dma_pool { 800 struct device *pool_device; 801 uma_zone_t pool_zone; 802 struct mtx pool_lock; 803 bus_dma_tag_t pool_dmat; 804 size_t pool_entry_size; 805 struct pctrie pool_ptree; 806 }; 807 808 #define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock) 809 #define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock) 810 811 static inline int 812 dma_pool_obj_ctor(void *mem, int size, void *arg, int flags) 813 { 814 struct linux_dma_obj *obj = mem; 815 struct dma_pool *pool = arg; 816 int error, nseg; 817 bus_dma_segment_t seg; 818 819 nseg = -1; 820 DMA_POOL_LOCK(pool); 821 error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap, 822 vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT, 823 &seg, &nseg); 824 DMA_POOL_UNLOCK(pool); 825 if (error != 0) { 826 return (error); 827 } 828 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); 829 obj->dma_addr = seg.ds_addr; 830 831 return (0); 832 } 833 834 static void 835 dma_pool_obj_dtor(void *mem, int size, void *arg) 836 { 837 struct linux_dma_obj *obj = mem; 838 struct dma_pool *pool = arg; 839 840 DMA_POOL_LOCK(pool); 841 bus_dmamap_unload(pool->pool_dmat, obj->dmamap); 842 DMA_POOL_UNLOCK(pool); 843 } 844 845 static int 846 dma_pool_obj_import(void *arg, void **store, int count, int domain __unused, 847 int flags) 848 { 849 struct dma_pool *pool = arg; 850 struct linux_dma_priv *priv; 851 struct linux_dma_obj *obj; 852 int error, i; 853 854 priv = pool->pool_device->dma_priv; 855 for (i = 0; i < count; i++) { 856 obj = uma_zalloc(linux_dma_obj_zone, flags); 857 if (obj == NULL) 858 break; 859 860 error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr, 861 BUS_DMA_NOWAIT, &obj->dmamap); 862 if (error!= 0) { 863 uma_zfree(linux_dma_obj_zone, obj); 864 break; 865 } 866 867 store[i] = obj; 868 } 869 870 return (i); 871 } 872 873 static void 874 dma_pool_obj_release(void *arg, void **store, int count) 875 { 876 struct dma_pool *pool = arg; 877 struct linux_dma_priv *priv; 878 struct linux_dma_obj *obj; 879 int i; 880 881 priv = pool->pool_device->dma_priv; 882 for (i = 0; i < count; i++) { 883 obj = store[i]; 884 bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap); 885 uma_zfree(linux_dma_obj_zone, obj); 886 } 887 } 888 889 struct dma_pool * 890 linux_dma_pool_create(char *name, struct device *dev, size_t size, 891 size_t align, size_t boundary) 892 { 893 struct linux_dma_priv *priv; 894 struct dma_pool *pool; 895 896 priv = dev->dma_priv; 897 898 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 899 pool->pool_device = dev; 900 pool->pool_entry_size = size; 901 902 if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), 903 align, boundary, /* alignment, boundary */ 904 priv->dma_mask, /* lowaddr */ 905 BUS_SPACE_MAXADDR, /* highaddr */ 906 NULL, NULL, /* filtfunc, filtfuncarg */ 907 size, /* maxsize */ 908 1, /* nsegments */ 909 size, /* maxsegsz */ 910 0, /* flags */ 911 NULL, NULL, /* lockfunc, lockfuncarg */ 912 &pool->pool_dmat)) { 913 kfree(pool); 914 return (NULL); 915 } 916 917 pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor, 918 dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import, 919 dma_pool_obj_release, pool, 0); 920 921 mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF); 922 pctrie_init(&pool->pool_ptree); 923 924 return (pool); 925 } 926 927 void 928 linux_dma_pool_destroy(struct dma_pool *pool) 929 { 930 931 uma_zdestroy(pool->pool_zone); 932 bus_dma_tag_destroy(pool->pool_dmat); 933 mtx_destroy(&pool->pool_lock); 934 kfree(pool); 935 } 936 937 void * 938 linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, 939 dma_addr_t *handle) 940 { 941 struct linux_dma_obj *obj; 942 943 obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags & GFP_NATIVE_MASK); 944 if (obj == NULL) 945 return (NULL); 946 947 DMA_POOL_LOCK(pool); 948 if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) { 949 DMA_POOL_UNLOCK(pool); 950 uma_zfree_arg(pool->pool_zone, obj, pool); 951 return (NULL); 952 } 953 DMA_POOL_UNLOCK(pool); 954 955 *handle = obj->dma_addr; 956 return (obj->vaddr); 957 } 958 959 void 960 linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr) 961 { 962 struct linux_dma_obj *obj; 963 964 DMA_POOL_LOCK(pool); 965 obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr); 966 if (obj == NULL) { 967 DMA_POOL_UNLOCK(pool); 968 return; 969 } 970 LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr); 971 DMA_POOL_UNLOCK(pool); 972 973 uma_zfree_arg(pool->pool_zone, obj, pool); 974 } 975 976 static int 977 linux_backlight_get_status(device_t dev, struct backlight_props *props) 978 { 979 struct pci_dev *pdev; 980 981 linux_set_current(curthread); 982 pdev = device_get_softc(dev); 983 984 props->brightness = pdev->dev.bd->props.brightness; 985 props->brightness = props->brightness * 100 / pdev->dev.bd->props.max_brightness; 986 props->nlevels = 0; 987 988 return (0); 989 } 990 991 static int 992 linux_backlight_get_info(device_t dev, struct backlight_info *info) 993 { 994 struct pci_dev *pdev; 995 996 linux_set_current(curthread); 997 pdev = device_get_softc(dev); 998 999 info->type = BACKLIGHT_TYPE_PANEL; 1000 strlcpy(info->name, pdev->dev.bd->name, BACKLIGHTMAXNAMELENGTH); 1001 return (0); 1002 } 1003 1004 static int 1005 linux_backlight_update_status(device_t dev, struct backlight_props *props) 1006 { 1007 struct pci_dev *pdev; 1008 1009 linux_set_current(curthread); 1010 pdev = device_get_softc(dev); 1011 1012 pdev->dev.bd->props.brightness = pdev->dev.bd->props.max_brightness * 1013 props->brightness / 100; 1014 return (pdev->dev.bd->ops->update_status(pdev->dev.bd)); 1015 } 1016 1017 struct backlight_device * 1018 linux_backlight_device_register(const char *name, struct device *dev, 1019 void *data, const struct backlight_ops *ops, struct backlight_properties *props) 1020 { 1021 1022 dev->bd = malloc(sizeof(*dev->bd), M_DEVBUF, M_WAITOK | M_ZERO); 1023 dev->bd->ops = ops; 1024 dev->bd->props.type = props->type; 1025 dev->bd->props.max_brightness = props->max_brightness; 1026 dev->bd->props.brightness = props->brightness; 1027 dev->bd->props.power = props->power; 1028 dev->bd->data = data; 1029 dev->bd->dev = dev; 1030 dev->bd->name = strdup(name, M_DEVBUF); 1031 1032 dev->backlight_dev = backlight_register(name, dev->bsddev); 1033 1034 return (dev->bd); 1035 } 1036 1037 void 1038 linux_backlight_device_unregister(struct backlight_device *bd) 1039 { 1040 1041 backlight_destroy(bd->dev->backlight_dev); 1042 free(bd->name, M_DEVBUF); 1043 free(bd, M_DEVBUF); 1044 } 1045