1 /*-
2 * Copyright (c) 2015-2016 Mellanox Technologies, Ltd.
3 * All rights reserved.
4 * Copyright (c) 2020-2025 The FreeBSD Foundation
5 *
6 * Portions of this software were developed by Björn Zeeb
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice unmodified, this list of conditions, and the following
14 * disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/bus.h>
34 #include <sys/malloc.h>
35 #include <sys/kernel.h>
36 #include <sys/sysctl.h>
37 #include <sys/lock.h>
38 #include <sys/mutex.h>
39 #include <sys/fcntl.h>
40 #include <sys/file.h>
41 #include <sys/filio.h>
42 #include <sys/pciio.h>
43 #include <sys/pctrie.h>
44 #include <sys/rman.h>
45 #include <sys/rwlock.h>
46 #include <sys/stdarg.h>
47
48 #include <vm/vm.h>
49 #include <vm/pmap.h>
50
51 #include <machine/bus.h>
52 #include <machine/resource.h>
53
54 #include <dev/pci/pcivar.h>
55 #include <dev/pci/pci_private.h>
56 #include <dev/pci/pci_iov.h>
57 #include <dev/backlight/backlight.h>
58
59 #include <linux/kernel.h>
60 #include <linux/kobject.h>
61 #include <linux/device.h>
62 #include <linux/slab.h>
63 #include <linux/module.h>
64 #include <linux/cdev.h>
65 #include <linux/file.h>
66 #include <linux/sysfs.h>
67 #include <linux/mm.h>
68 #include <linux/io.h>
69 #include <linux/vmalloc.h>
70 #define WANT_NATIVE_PCI_GET_SLOT
71 #include <linux/pci.h>
72 #include <linux/compat.h>
73
74 #include <linux/backlight.h>
75
76 #include "backlight_if.h"
77 #include "pcib_if.h"
78
79 /* Undef the linux function macro defined in linux/pci.h */
80 #undef pci_get_class
81
82 extern int linuxkpi_debug;
83
84 SYSCTL_DECL(_compat_linuxkpi);
85
86 static counter_u64_t lkpi_pci_nseg1_fail;
87 SYSCTL_COUNTER_U64(_compat_linuxkpi, OID_AUTO, lkpi_pci_nseg1_fail, CTLFLAG_RD,
88 &lkpi_pci_nseg1_fail, "Count of busdma mapping failures of single-segment");
89
90 static device_probe_t linux_pci_probe;
91 static device_attach_t linux_pci_attach;
92 static device_detach_t linux_pci_detach;
93 static device_suspend_t linux_pci_suspend;
94 static device_resume_t linux_pci_resume;
95 static device_shutdown_t linux_pci_shutdown;
96 static pci_iov_init_t linux_pci_iov_init;
97 static pci_iov_uninit_t linux_pci_iov_uninit;
98 static pci_iov_add_vf_t linux_pci_iov_add_vf;
99 static int linux_backlight_get_status(device_t dev, struct backlight_props *props);
100 static int linux_backlight_update_status(device_t dev, struct backlight_props *props);
101 static int linux_backlight_get_info(device_t dev, struct backlight_info *info);
102 static void lkpi_pcim_iomap_table_release(struct device *, void *);
103
104 static device_method_t pci_methods[] = {
105 DEVMETHOD(device_probe, linux_pci_probe),
106 DEVMETHOD(device_attach, linux_pci_attach),
107 DEVMETHOD(device_detach, linux_pci_detach),
108 DEVMETHOD(device_suspend, linux_pci_suspend),
109 DEVMETHOD(device_resume, linux_pci_resume),
110 DEVMETHOD(device_shutdown, linux_pci_shutdown),
111 DEVMETHOD(pci_iov_init, linux_pci_iov_init),
112 DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit),
113 DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf),
114
115 /* Bus interface. */
116 DEVMETHOD(bus_add_child, bus_generic_add_child),
117
118 /* backlight interface */
119 DEVMETHOD(backlight_update_status, linux_backlight_update_status),
120 DEVMETHOD(backlight_get_status, linux_backlight_get_status),
121 DEVMETHOD(backlight_get_info, linux_backlight_get_info),
122 DEVMETHOD_END
123 };
124
125 const char *pci_power_names[] = {
126 "UNKNOWN", "D0", "D1", "D2", "D3hot", "D3cold"
127 };
128
129 /* We need some meta-struct to keep track of these for devres. */
130 struct pci_devres {
131 bool enable_io;
132 /* PCIR_MAX_BAR_0 + 1 = 6 => BIT(0..5). */
133 uint8_t region_mask;
134 struct resource *region_table[PCIR_MAX_BAR_0 + 1]; /* Not needed. */
135 };
136 struct pcim_iomap_devres {
137 void *mmio_table[PCIR_MAX_BAR_0 + 1];
138 struct resource *res_table[PCIR_MAX_BAR_0 + 1];
139 };
140
141 struct linux_dma_priv {
142 uint64_t dma_mask;
143 bus_dma_tag_t dmat;
144 uint64_t dma_coherent_mask;
145 bus_dma_tag_t dmat_coherent;
146 struct mtx lock;
147 struct pctrie ptree;
148 };
149 #define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock)
150 #define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock)
151
152 static void
lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres * dr,int bar,void * res)153 lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres *dr, int bar,
154 void *res)
155 {
156 dr->mmio_table[bar] = (void *)rman_get_bushandle(res);
157 dr->res_table[bar] = res;
158 }
159
160 static bool
lkpi_pci_bar_id_valid(int bar)161 lkpi_pci_bar_id_valid(int bar)
162 {
163 if (bar < 0 || bar > PCIR_MAX_BAR_0)
164 return (false);
165
166 return (true);
167 }
168
169 static int
linux_pdev_dma_uninit(struct pci_dev * pdev)170 linux_pdev_dma_uninit(struct pci_dev *pdev)
171 {
172 struct linux_dma_priv *priv;
173
174 priv = pdev->dev.dma_priv;
175 if (priv->dmat)
176 bus_dma_tag_destroy(priv->dmat);
177 if (priv->dmat_coherent)
178 bus_dma_tag_destroy(priv->dmat_coherent);
179 mtx_destroy(&priv->lock);
180 pdev->dev.dma_priv = NULL;
181 free(priv, M_DEVBUF);
182 return (0);
183 }
184
185 static int
linux_pdev_dma_init(struct pci_dev * pdev)186 linux_pdev_dma_init(struct pci_dev *pdev)
187 {
188 struct linux_dma_priv *priv;
189 int error;
190
191 priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO);
192
193 mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF);
194 pctrie_init(&priv->ptree);
195
196 pdev->dev.dma_priv = priv;
197
198 /* Create a default DMA tags. */
199 error = linux_dma_tag_init(&pdev->dev, DMA_BIT_MASK(64));
200 if (error != 0)
201 goto err;
202 /* Coherent is lower 32bit only by default in Linux. */
203 error = linux_dma_tag_init_coherent(&pdev->dev, DMA_BIT_MASK(32));
204 if (error != 0)
205 goto err;
206
207 return (error);
208
209 err:
210 linux_pdev_dma_uninit(pdev);
211 return (error);
212 }
213
214 int
linux_dma_tag_init(struct device * dev,u64 dma_mask)215 linux_dma_tag_init(struct device *dev, u64 dma_mask)
216 {
217 struct linux_dma_priv *priv;
218 int error;
219
220 priv = dev->dma_priv;
221
222 if (priv->dmat) {
223 if (priv->dma_mask == dma_mask)
224 return (0);
225
226 bus_dma_tag_destroy(priv->dmat);
227 }
228
229 priv->dma_mask = dma_mask;
230
231 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev),
232 1, 0, /* alignment, boundary */
233 dma_mask, /* lowaddr */
234 BUS_SPACE_MAXADDR, /* highaddr */
235 NULL, NULL, /* filtfunc, filtfuncarg */
236 BUS_SPACE_MAXSIZE, /* maxsize */
237 1, /* nsegments */
238 BUS_SPACE_MAXSIZE, /* maxsegsz */
239 0, /* flags */
240 NULL, NULL, /* lockfunc, lockfuncarg */
241 &priv->dmat);
242 return (-error);
243 }
244
245 int
linux_dma_tag_init_coherent(struct device * dev,u64 dma_mask)246 linux_dma_tag_init_coherent(struct device *dev, u64 dma_mask)
247 {
248 struct linux_dma_priv *priv;
249 int error;
250
251 priv = dev->dma_priv;
252
253 if (priv->dmat_coherent) {
254 if (priv->dma_coherent_mask == dma_mask)
255 return (0);
256
257 bus_dma_tag_destroy(priv->dmat_coherent);
258 }
259
260 priv->dma_coherent_mask = dma_mask;
261
262 error = bus_dma_tag_create(bus_get_dma_tag(dev->bsddev),
263 1, 0, /* alignment, boundary */
264 dma_mask, /* lowaddr */
265 BUS_SPACE_MAXADDR, /* highaddr */
266 NULL, NULL, /* filtfunc, filtfuncarg */
267 BUS_SPACE_MAXSIZE, /* maxsize */
268 1, /* nsegments */
269 BUS_SPACE_MAXSIZE, /* maxsegsz */
270 0, /* flags */
271 NULL, NULL, /* lockfunc, lockfuncarg */
272 &priv->dmat_coherent);
273 return (-error);
274 }
275
276 static struct pci_driver *
linux_pci_find(device_t dev,const struct pci_device_id ** idp)277 linux_pci_find(device_t dev, const struct pci_device_id **idp)
278 {
279 const struct pci_device_id *id;
280 struct pci_driver *pdrv;
281 uint16_t vendor;
282 uint16_t device;
283 uint16_t subvendor;
284 uint16_t subdevice;
285
286 vendor = pci_get_vendor(dev);
287 device = pci_get_device(dev);
288 subvendor = pci_get_subvendor(dev);
289 subdevice = pci_get_subdevice(dev);
290
291 spin_lock(&pci_lock);
292 list_for_each_entry(pdrv, &pci_drivers, node) {
293 for (id = pdrv->id_table; id->vendor != 0; id++) {
294 if (vendor == id->vendor &&
295 (PCI_ANY_ID == id->device || device == id->device) &&
296 (PCI_ANY_ID == id->subvendor || subvendor == id->subvendor) &&
297 (PCI_ANY_ID == id->subdevice || subdevice == id->subdevice)) {
298 *idp = id;
299 spin_unlock(&pci_lock);
300 return (pdrv);
301 }
302 }
303 }
304 spin_unlock(&pci_lock);
305 return (NULL);
306 }
307
308 struct pci_dev *
lkpi_pci_get_device(uint32_t vendor,uint32_t device,struct pci_dev * odev)309 lkpi_pci_get_device(uint32_t vendor, uint32_t device, struct pci_dev *odev)
310 {
311 struct pci_dev *pdev, *found;
312
313 found = NULL;
314 spin_lock(&pci_lock);
315 list_for_each_entry(pdev, &pci_devices, links) {
316 /* Walk until we find odev. */
317 if (odev != NULL) {
318 if (pdev == odev)
319 odev = NULL;
320 continue;
321 }
322
323 if ((pdev->vendor == vendor || vendor == PCI_ANY_ID) &&
324 (pdev->device == device || device == PCI_ANY_ID)) {
325 found = pdev;
326 break;
327 }
328 }
329 pci_dev_get(found);
330 spin_unlock(&pci_lock);
331
332 return (found);
333 }
334
335 static void
lkpi_pci_dev_release(struct device * dev)336 lkpi_pci_dev_release(struct device *dev)
337 {
338
339 lkpi_devres_release_free_list(dev);
340 spin_lock_destroy(&dev->devres_lock);
341 }
342
343 static int
lkpifill_pci_dev(device_t dev,struct pci_dev * pdev)344 lkpifill_pci_dev(device_t dev, struct pci_dev *pdev)
345 {
346 struct pci_devinfo *dinfo;
347 int error;
348
349 error = kobject_init_and_add(&pdev->dev.kobj, &linux_dev_ktype,
350 &linux_root_device.kobj, device_get_nameunit(dev));
351 if (error != 0) {
352 printf("%s:%d: kobject_init_and_add returned %d\n",
353 __func__, __LINE__, error);
354 return (error);
355 }
356
357 pdev->devfn = PCI_DEVFN(pci_get_slot(dev), pci_get_function(dev));
358 pdev->vendor = pci_get_vendor(dev);
359 pdev->device = pci_get_device(dev);
360 pdev->subsystem_vendor = pci_get_subvendor(dev);
361 pdev->subsystem_device = pci_get_subdevice(dev);
362 pdev->class = pci_get_class(dev);
363 pdev->revision = pci_get_revid(dev);
364 pdev->path_name = kasprintf(GFP_KERNEL, "%04d:%02d:%02d.%d",
365 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
366 pci_get_function(dev));
367
368 pdev->bus = malloc(sizeof(*pdev->bus), M_DEVBUF, M_WAITOK | M_ZERO);
369 pdev->bus->number = pci_get_bus(dev);
370 pdev->bus->domain = pci_get_domain(dev);
371
372 /* Check if we have reached the root to satisfy pci_is_root_bus() */
373 dinfo = device_get_ivars(dev);
374 if (dinfo->cfg.pcie.pcie_location != 0 &&
375 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) {
376 pdev->bus->self = NULL;
377 } else {
378 /*
379 * This should be the upstream bridge; pci_upstream_bridge()
380 * handles that case on demand as otherwise we'll shadow the
381 * entire PCI hierarchy.
382 */
383 pdev->bus->self = pdev;
384 }
385 pdev->dev.bsddev = dev;
386 pdev->dev.parent = &linux_root_device;
387 pdev->dev.release = lkpi_pci_dev_release;
388 INIT_LIST_HEAD(&pdev->dev.irqents);
389
390 if (pci_msi_count(dev) > 0)
391 pdev->msi_desc = malloc(pci_msi_count(dev) *
392 sizeof(*pdev->msi_desc), M_DEVBUF, M_WAITOK | M_ZERO);
393
394 spin_lock_init(&pdev->dev.devres_lock);
395 INIT_LIST_HEAD(&pdev->dev.devres_head);
396
397 return (0);
398 }
399
400 static void
lkpinew_pci_dev_release(struct device * dev)401 lkpinew_pci_dev_release(struct device *dev)
402 {
403 struct pci_dev *pdev;
404 int i;
405
406 pdev = to_pci_dev(dev);
407 if (pdev->root != NULL)
408 pci_dev_put(pdev->root);
409 if (pdev->bus->self != pdev && pdev->bus->self != NULL)
410 pci_dev_put(pdev->bus->self);
411 free(pdev->bus, M_DEVBUF);
412 if (pdev->msi_desc != NULL) {
413 for (i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--)
414 free(pdev->msi_desc[i], M_DEVBUF);
415 free(pdev->msi_desc, M_DEVBUF);
416 }
417 kfree(pdev->path_name);
418 free(pdev, M_DEVBUF);
419 }
420
421 struct pci_dev *
lkpinew_pci_dev(device_t dev)422 lkpinew_pci_dev(device_t dev)
423 {
424 struct pci_dev *pdev;
425 int error;
426
427 pdev = malloc(sizeof(*pdev), M_DEVBUF, M_WAITOK|M_ZERO);
428 error = lkpifill_pci_dev(dev, pdev);
429 if (error != 0) {
430 free(pdev, M_DEVBUF);
431 return (NULL);
432 }
433 pdev->dev.release = lkpinew_pci_dev_release;
434
435 return (pdev);
436 }
437
438 struct pci_dev *
lkpi_pci_get_class(unsigned int class,struct pci_dev * from)439 lkpi_pci_get_class(unsigned int class, struct pci_dev *from)
440 {
441 device_t dev;
442 device_t devfrom = NULL;
443 struct pci_dev *pdev;
444
445 if (from != NULL)
446 devfrom = from->dev.bsddev;
447
448 dev = pci_find_class_from(class >> 16, (class >> 8) & 0xFF, devfrom);
449 if (dev == NULL)
450 return (NULL);
451
452 pdev = lkpinew_pci_dev(dev);
453 return (pdev);
454 }
455
456 struct pci_dev *
lkpi_pci_get_base_class(unsigned int baseclass,struct pci_dev * from)457 lkpi_pci_get_base_class(unsigned int baseclass, struct pci_dev *from)
458 {
459 device_t dev;
460 device_t devfrom = NULL;
461 struct pci_dev *pdev;
462
463 if (from != NULL)
464 devfrom = from->dev.bsddev;
465
466 dev = pci_find_base_class_from(baseclass, devfrom);
467 if (dev == NULL)
468 return (NULL);
469
470 pdev = lkpinew_pci_dev(dev);
471 return (pdev);
472 }
473
474 struct pci_dev *
lkpi_pci_get_domain_bus_and_slot(int domain,unsigned int bus,unsigned int devfn)475 lkpi_pci_get_domain_bus_and_slot(int domain, unsigned int bus,
476 unsigned int devfn)
477 {
478 device_t dev;
479 struct pci_dev *pdev;
480
481 dev = pci_find_dbsf(domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
482 if (dev == NULL)
483 return (NULL);
484
485 pdev = lkpinew_pci_dev(dev);
486 return (pdev);
487 }
488
489 struct pci_dev *
lkpi_pci_get_slot(struct pci_bus * pbus,unsigned int devfn)490 lkpi_pci_get_slot(struct pci_bus *pbus, unsigned int devfn)
491 {
492 device_t dev;
493 struct pci_dev *pdev;
494
495 dev = pci_find_bsf(pbus->number, PCI_SLOT(devfn), PCI_FUNC(devfn));
496 if (dev == NULL)
497 return (NULL);
498
499 pdev = lkpinew_pci_dev(dev);
500 return (pdev);
501 }
502
503 static int
linux_pci_probe(device_t dev)504 linux_pci_probe(device_t dev)
505 {
506 const struct pci_device_id *id;
507 struct pci_driver *pdrv;
508
509 if ((pdrv = linux_pci_find(dev, &id)) == NULL)
510 return (ENXIO);
511 if (device_get_driver(dev) != &pdrv->bsddriver)
512 return (ENXIO);
513 device_set_desc(dev, pdrv->name);
514
515 /* Assume BSS initialized (should never return BUS_PROBE_SPECIFIC). */
516 if (pdrv->bsd_probe_return == 0)
517 return (BUS_PROBE_DEFAULT);
518 else
519 return (pdrv->bsd_probe_return);
520 }
521
522 static int
linux_pci_attach(device_t dev)523 linux_pci_attach(device_t dev)
524 {
525 const struct pci_device_id *id;
526 struct pci_driver *pdrv;
527 struct pci_dev *pdev;
528
529 pdrv = linux_pci_find(dev, &id);
530 pdev = device_get_softc(dev);
531
532 MPASS(pdrv != NULL);
533 MPASS(pdev != NULL);
534
535 return (linux_pci_attach_device(dev, pdrv, id, pdev));
536 }
537
538 static struct resource_list_entry *
linux_pci_reserve_bar(struct pci_dev * pdev,struct resource_list * rl,int type,int rid)539 linux_pci_reserve_bar(struct pci_dev *pdev, struct resource_list *rl,
540 int type, int rid)
541 {
542 device_t dev;
543 struct resource *res;
544
545 KASSERT(type == SYS_RES_IOPORT || type == SYS_RES_MEMORY,
546 ("trying to reserve non-BAR type %d", type));
547
548 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ?
549 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev;
550 res = pci_reserve_map(device_get_parent(dev), dev, type, &rid, 0, ~0,
551 1, 1, 0);
552 if (res == NULL)
553 return (NULL);
554 return (resource_list_find(rl, type, rid));
555 }
556
557 static struct resource_list_entry *
linux_pci_get_rle(struct pci_dev * pdev,int type,int rid,bool reserve_bar)558 linux_pci_get_rle(struct pci_dev *pdev, int type, int rid, bool reserve_bar)
559 {
560 struct pci_devinfo *dinfo;
561 struct resource_list *rl;
562 struct resource_list_entry *rle;
563
564 dinfo = device_get_ivars(pdev->dev.bsddev);
565 rl = &dinfo->resources;
566 rle = resource_list_find(rl, type, rid);
567 /* Reserve resources for this BAR if needed. */
568 if (rle == NULL && reserve_bar)
569 rle = linux_pci_reserve_bar(pdev, rl, type, rid);
570 return (rle);
571 }
572
573 int
linux_pci_attach_device(device_t dev,struct pci_driver * pdrv,const struct pci_device_id * id,struct pci_dev * pdev)574 linux_pci_attach_device(device_t dev, struct pci_driver *pdrv,
575 const struct pci_device_id *id, struct pci_dev *pdev)
576 {
577 struct resource_list_entry *rle;
578 device_t parent;
579 struct pci_dev *pbus, *ppbus;
580 uintptr_t rid;
581 int error;
582 bool isdrm;
583
584 linux_set_current(curthread);
585
586 parent = device_get_parent(dev);
587 isdrm = pdrv != NULL && pdrv->isdrm;
588
589 if (isdrm) {
590 struct pci_devinfo *dinfo;
591
592 dinfo = device_get_ivars(parent);
593 device_set_ivars(dev, dinfo);
594 }
595
596 error = lkpifill_pci_dev(dev, pdev);
597 if (error != 0)
598 return (error);
599
600 if (isdrm)
601 PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid);
602 else
603 PCI_GET_ID(parent, dev, PCI_ID_RID, &rid);
604 pdev->devfn = rid;
605 pdev->pdrv = pdrv;
606 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0, false);
607 if (rle != NULL)
608 pdev->dev.irq = rle->start;
609 else
610 pdev->dev.irq = LINUX_IRQ_INVALID;
611 pdev->irq = pdev->dev.irq;
612 error = linux_pdev_dma_init(pdev);
613 if (error)
614 goto out_dma_init;
615
616 TAILQ_INIT(&pdev->mmio);
617 spin_lock_init(&pdev->pcie_cap_lock);
618
619 spin_lock(&pci_lock);
620 list_add(&pdev->links, &pci_devices);
621 spin_unlock(&pci_lock);
622
623 /*
624 * Create the hierarchy now as we cannot on demand later.
625 * Take special care of DRM as there is a non-PCI device in the chain.
626 */
627 pbus = pdev;
628 if (isdrm) {
629 pbus = lkpinew_pci_dev(parent);
630 if (pbus == NULL) {
631 error = ENXIO;
632 goto out_dma_init;
633 }
634 }
635 pcie_find_root_port(pbus);
636 if (isdrm)
637 pdev->root = pbus->root;
638 ppbus = pci_upstream_bridge(pbus);
639 while (ppbus != NULL && ppbus != pbus) {
640 pbus = ppbus;
641 ppbus = pci_upstream_bridge(pbus);
642 }
643
644 if (pdrv != NULL) {
645 error = pdrv->probe(pdev, id);
646 if (error)
647 goto out_probe;
648 }
649 return (0);
650
651 /* XXX the cleanup does not match the allocation up there. */
652 out_probe:
653 free(pdev->bus, M_DEVBUF);
654 spin_lock_destroy(&pdev->pcie_cap_lock);
655 linux_pdev_dma_uninit(pdev);
656 out_dma_init:
657 spin_lock(&pci_lock);
658 list_del(&pdev->links);
659 spin_unlock(&pci_lock);
660 put_device(&pdev->dev);
661 return (-error);
662 }
663
664 static int
linux_pci_detach(device_t dev)665 linux_pci_detach(device_t dev)
666 {
667 struct pci_dev *pdev;
668
669 pdev = device_get_softc(dev);
670
671 MPASS(pdev != NULL);
672
673 device_set_desc(dev, NULL);
674
675 return (linux_pci_detach_device(pdev));
676 }
677
678 int
linux_pci_detach_device(struct pci_dev * pdev)679 linux_pci_detach_device(struct pci_dev *pdev)
680 {
681
682 linux_set_current(curthread);
683
684 if (pdev->pdrv != NULL)
685 pdev->pdrv->remove(pdev);
686
687 if (pdev->root != NULL)
688 pci_dev_put(pdev->root);
689 free(pdev->bus, M_DEVBUF);
690 linux_pdev_dma_uninit(pdev);
691
692 spin_lock(&pci_lock);
693 list_del(&pdev->links);
694 spin_unlock(&pci_lock);
695 spin_lock_destroy(&pdev->pcie_cap_lock);
696 put_device(&pdev->dev);
697
698 return (0);
699 }
700
701 static int
lkpi_pci_disable_dev(struct device * dev)702 lkpi_pci_disable_dev(struct device *dev)
703 {
704
705 (void) pci_disable_io(dev->bsddev, SYS_RES_MEMORY);
706 (void) pci_disable_io(dev->bsddev, SYS_RES_IOPORT);
707 return (0);
708 }
709
710 static struct pci_devres *
lkpi_pci_devres_get_alloc(struct pci_dev * pdev)711 lkpi_pci_devres_get_alloc(struct pci_dev *pdev)
712 {
713 struct pci_devres *dr;
714
715 dr = lkpi_devres_find(&pdev->dev, lkpi_pci_devres_release, NULL, NULL);
716 if (dr == NULL) {
717 dr = lkpi_devres_alloc(lkpi_pci_devres_release, sizeof(*dr),
718 GFP_KERNEL | __GFP_ZERO);
719 if (dr != NULL)
720 lkpi_devres_add(&pdev->dev, dr);
721 }
722
723 return (dr);
724 }
725
726 static struct pci_devres *
lkpi_pci_devres_find(struct pci_dev * pdev)727 lkpi_pci_devres_find(struct pci_dev *pdev)
728 {
729 if (!pdev->managed)
730 return (NULL);
731
732 return (lkpi_pci_devres_get_alloc(pdev));
733 }
734
735 void
lkpi_pci_devres_release(struct device * dev,void * p)736 lkpi_pci_devres_release(struct device *dev, void *p)
737 {
738 struct pci_devres *dr;
739 struct pci_dev *pdev;
740 int bar;
741
742 pdev = to_pci_dev(dev);
743 dr = p;
744
745 if (pdev->msix_enabled)
746 lkpi_pci_disable_msix(pdev);
747 if (pdev->msi_enabled)
748 lkpi_pci_disable_msi(pdev);
749
750 if (dr->enable_io && lkpi_pci_disable_dev(dev) == 0)
751 dr->enable_io = false;
752
753 if (dr->region_mask == 0)
754 return;
755 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) {
756
757 if ((dr->region_mask & (1 << bar)) == 0)
758 continue;
759 pci_release_region(pdev, bar);
760 }
761 }
762
763 int
linuxkpi_pcim_enable_device(struct pci_dev * pdev)764 linuxkpi_pcim_enable_device(struct pci_dev *pdev)
765 {
766 struct pci_devres *dr;
767 int error;
768
769 /* Here we cannot run through the pdev->managed check. */
770 dr = lkpi_pci_devres_get_alloc(pdev);
771 if (dr == NULL)
772 return (-ENOMEM);
773
774 /* If resources were enabled before do not do it again. */
775 if (dr->enable_io)
776 return (0);
777
778 error = pci_enable_device(pdev);
779 if (error == 0)
780 dr->enable_io = true;
781
782 /* This device is not managed. */
783 pdev->managed = true;
784
785 return (error);
786 }
787
788 static struct pcim_iomap_devres *
lkpi_pcim_iomap_devres_find(struct pci_dev * pdev)789 lkpi_pcim_iomap_devres_find(struct pci_dev *pdev)
790 {
791 struct pcim_iomap_devres *dr;
792
793 dr = lkpi_devres_find(&pdev->dev, lkpi_pcim_iomap_table_release,
794 NULL, NULL);
795 if (dr == NULL) {
796 dr = lkpi_devres_alloc(lkpi_pcim_iomap_table_release,
797 sizeof(*dr), GFP_KERNEL | __GFP_ZERO);
798 if (dr != NULL)
799 lkpi_devres_add(&pdev->dev, dr);
800 }
801
802 if (dr == NULL)
803 device_printf(pdev->dev.bsddev, "%s: NULL\n", __func__);
804
805 return (dr);
806 }
807
808 void __iomem **
linuxkpi_pcim_iomap_table(struct pci_dev * pdev)809 linuxkpi_pcim_iomap_table(struct pci_dev *pdev)
810 {
811 struct pcim_iomap_devres *dr;
812
813 dr = lkpi_pcim_iomap_devres_find(pdev);
814 if (dr == NULL)
815 return (NULL);
816
817 /*
818 * If the driver has manually set a flag to be able to request the
819 * resource to use bus_read/write_<n>, return the shadow table.
820 */
821 if (pdev->want_iomap_res)
822 return ((void **)dr->res_table);
823
824 /* This is the Linux default. */
825 return (dr->mmio_table);
826 }
827
828 static struct resource *
_lkpi_pci_iomap(struct pci_dev * pdev,int bar,unsigned long maxlen __unused)829 _lkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen __unused)
830 {
831 struct pci_mmio_region *mmio, *p;
832 int type;
833
834 if (!lkpi_pci_bar_id_valid(bar))
835 return (NULL);
836
837 type = pci_resource_type(pdev, bar);
838 if (type < 0) {
839 device_printf(pdev->dev.bsddev, "%s: bar %d type %d\n",
840 __func__, bar, type);
841 return (NULL);
842 }
843
844 /*
845 * Check for duplicate mappings.
846 * This can happen if a driver calls pci_request_region() first.
847 */
848 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) {
849 if (mmio->type == type && mmio->rid == PCIR_BAR(bar)) {
850 return (mmio->res);
851 }
852 }
853
854 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO);
855 mmio->rid = PCIR_BAR(bar);
856 mmio->type = type;
857 mmio->res = bus_alloc_resource_any(pdev->dev.bsddev, mmio->type,
858 &mmio->rid, RF_ACTIVE|RF_SHAREABLE);
859 if (mmio->res == NULL) {
860 device_printf(pdev->dev.bsddev, "%s: failed to alloc "
861 "bar %d type %d rid %d\n",
862 __func__, bar, type, PCIR_BAR(bar));
863 free(mmio, M_DEVBUF);
864 return (NULL);
865 }
866 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next);
867
868 return (mmio->res);
869 }
870
871 void *
linuxkpi_pci_iomap_range(struct pci_dev * pdev,int bar,unsigned long off,unsigned long maxlen)872 linuxkpi_pci_iomap_range(struct pci_dev *pdev, int bar,
873 unsigned long off, unsigned long maxlen)
874 {
875 struct resource *res;
876
877 if (!lkpi_pci_bar_id_valid(bar))
878 return (NULL);
879
880 res = _lkpi_pci_iomap(pdev, bar, maxlen);
881 if (res == NULL)
882 return (NULL);
883 /* This is a FreeBSD extension so we can use bus_*(). */
884 if (pdev->want_iomap_res)
885 return (res);
886 MPASS(off < rman_get_size(res));
887 return ((void *)(rman_get_bushandle(res) + off));
888 }
889
890 void *
linuxkpi_pci_iomap(struct pci_dev * pdev,int bar,unsigned long maxlen)891 linuxkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen)
892 {
893 if (!lkpi_pci_bar_id_valid(bar))
894 return (NULL);
895
896 return (linuxkpi_pci_iomap_range(pdev, bar, 0, maxlen));
897 }
898
899 void *
linuxkpi_pcim_iomap(struct pci_dev * pdev,int bar,unsigned long maxlen)900 linuxkpi_pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen)
901 {
902 struct pcim_iomap_devres *dr;
903 void *res;
904
905 if (!lkpi_pci_bar_id_valid(bar))
906 return (NULL);
907
908 dr = lkpi_pcim_iomap_devres_find(pdev);
909 if (dr == NULL)
910 return (NULL);
911
912 if (dr->res_table[bar] != NULL)
913 return (dr->res_table[bar]);
914
915 res = linuxkpi_pci_iomap(pdev, bar, maxlen);
916 if (res == NULL) {
917 /*
918 * Do not free the devres in case there were
919 * other valid mappings before already.
920 */
921 return (NULL);
922 }
923 lkpi_set_pcim_iomap_devres(dr, bar, res);
924
925 return (res);
926 }
927
928 void
linuxkpi_pci_iounmap(struct pci_dev * pdev,void * res)929 linuxkpi_pci_iounmap(struct pci_dev *pdev, void *res)
930 {
931 struct pci_mmio_region *mmio, *p;
932 bus_space_handle_t bh = (bus_space_handle_t)res;
933
934 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) {
935 if (pdev->want_iomap_res) {
936 if (res != mmio->res)
937 continue;
938 } else {
939 if (bh < rman_get_bushandle(mmio->res) ||
940 bh >= rman_get_bushandle(mmio->res) +
941 rman_get_size(mmio->res))
942 continue;
943 }
944 bus_release_resource(pdev->dev.bsddev,
945 mmio->type, mmio->rid, mmio->res);
946 TAILQ_REMOVE(&pdev->mmio, mmio, next);
947 free(mmio, M_DEVBUF);
948 return;
949 }
950 }
951
952 int
linuxkpi_pcim_iomap_regions(struct pci_dev * pdev,uint32_t mask,const char * name)953 linuxkpi_pcim_iomap_regions(struct pci_dev *pdev, uint32_t mask, const char *name)
954 {
955 struct pcim_iomap_devres *dr;
956 void *res;
957 uint32_t mappings;
958 int bar;
959
960 dr = lkpi_pcim_iomap_devres_find(pdev);
961 if (dr == NULL)
962 return (-ENOMEM);
963
964 /* Now iomap all the requested (by "mask") ones. */
965 for (bar = mappings = 0; mappings != mask; bar++) {
966 if ((mask & (1 << bar)) == 0)
967 continue;
968
969 /* Request double is not allowed. */
970 if (dr->mmio_table[bar] != NULL) {
971 device_printf(pdev->dev.bsddev, "%s: bar %d %p\n",
972 __func__, bar, dr->mmio_table[bar]);
973 goto err;
974 }
975
976 res = _lkpi_pci_iomap(pdev, bar, 0);
977 if (res == NULL)
978 goto err;
979 lkpi_set_pcim_iomap_devres(dr, bar, res);
980
981 mappings |= (1 << bar);
982 }
983
984 return (0);
985 err:
986 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) {
987 if ((mappings & (1 << bar)) != 0) {
988 res = dr->mmio_table[bar];
989 if (res == NULL)
990 continue;
991 pci_iounmap(pdev, res);
992 }
993 }
994
995 return (-EINVAL);
996 }
997
998 static void
lkpi_pcim_iomap_table_release(struct device * dev,void * p)999 lkpi_pcim_iomap_table_release(struct device *dev, void *p)
1000 {
1001 struct pcim_iomap_devres *dr;
1002 struct pci_dev *pdev;
1003 int bar;
1004
1005 dr = p;
1006 pdev = to_pci_dev(dev);
1007 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) {
1008
1009 if (dr->mmio_table[bar] == NULL)
1010 continue;
1011
1012 pci_iounmap(pdev, dr->mmio_table[bar]);
1013 }
1014 }
1015
1016 static int
linux_pci_suspend(device_t dev)1017 linux_pci_suspend(device_t dev)
1018 {
1019 const struct dev_pm_ops *pmops;
1020 struct pm_message pm = { };
1021 struct pci_dev *pdev;
1022 int error;
1023
1024 error = 0;
1025 linux_set_current(curthread);
1026 pdev = device_get_softc(dev);
1027 pmops = pdev->pdrv->driver.pm;
1028
1029 if (pdev->pdrv->suspend != NULL)
1030 error = -pdev->pdrv->suspend(pdev, pm);
1031 else if (pmops != NULL && pmops->suspend != NULL) {
1032 error = -pmops->suspend(&pdev->dev);
1033 if (error == 0 && pmops->suspend_late != NULL)
1034 error = -pmops->suspend_late(&pdev->dev);
1035 if (error == 0 && pmops->suspend_noirq != NULL)
1036 error = -pmops->suspend_noirq(&pdev->dev);
1037 }
1038 return (error);
1039 }
1040
1041 static int
linux_pci_resume(device_t dev)1042 linux_pci_resume(device_t dev)
1043 {
1044 const struct dev_pm_ops *pmops;
1045 struct pci_dev *pdev;
1046 int error;
1047
1048 error = 0;
1049 linux_set_current(curthread);
1050 pdev = device_get_softc(dev);
1051 pmops = pdev->pdrv->driver.pm;
1052
1053 if (pdev->pdrv->resume != NULL)
1054 error = -pdev->pdrv->resume(pdev);
1055 else if (pmops != NULL && pmops->resume != NULL) {
1056 if (pmops->resume_early != NULL)
1057 error = -pmops->resume_early(&pdev->dev);
1058 if (error == 0 && pmops->resume != NULL)
1059 error = -pmops->resume(&pdev->dev);
1060 }
1061 return (error);
1062 }
1063
1064 static int
linux_pci_shutdown(device_t dev)1065 linux_pci_shutdown(device_t dev)
1066 {
1067 struct pci_dev *pdev;
1068
1069 linux_set_current(curthread);
1070 pdev = device_get_softc(dev);
1071 if (pdev->pdrv->shutdown != NULL)
1072 pdev->pdrv->shutdown(pdev);
1073 return (0);
1074 }
1075
1076 static int
linux_pci_iov_init(device_t dev,uint16_t num_vfs,const nvlist_t * pf_config)1077 linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config)
1078 {
1079 struct pci_dev *pdev;
1080 int error;
1081
1082 linux_set_current(curthread);
1083 pdev = device_get_softc(dev);
1084 if (pdev->pdrv->bsd_iov_init != NULL)
1085 error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config);
1086 else
1087 error = EINVAL;
1088 return (error);
1089 }
1090
1091 static void
linux_pci_iov_uninit(device_t dev)1092 linux_pci_iov_uninit(device_t dev)
1093 {
1094 struct pci_dev *pdev;
1095
1096 linux_set_current(curthread);
1097 pdev = device_get_softc(dev);
1098 if (pdev->pdrv->bsd_iov_uninit != NULL)
1099 pdev->pdrv->bsd_iov_uninit(dev);
1100 }
1101
1102 static int
linux_pci_iov_add_vf(device_t dev,uint16_t vfnum,const nvlist_t * vf_config)1103 linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config)
1104 {
1105 struct pci_dev *pdev;
1106 int error;
1107
1108 linux_set_current(curthread);
1109 pdev = device_get_softc(dev);
1110 if (pdev->pdrv->bsd_iov_add_vf != NULL)
1111 error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config);
1112 else
1113 error = EINVAL;
1114 return (error);
1115 }
1116
1117 static int
_linux_pci_register_driver(struct pci_driver * pdrv,devclass_t dc)1118 _linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc)
1119 {
1120 int error;
1121
1122 linux_set_current(curthread);
1123 spin_lock(&pci_lock);
1124 list_add(&pdrv->node, &pci_drivers);
1125 spin_unlock(&pci_lock);
1126 if (pdrv->bsddriver.name == NULL)
1127 pdrv->bsddriver.name = pdrv->name;
1128 pdrv->bsddriver.methods = pci_methods;
1129 pdrv->bsddriver.size = sizeof(struct pci_dev);
1130
1131 bus_topo_lock();
1132 error = devclass_add_driver(dc, &pdrv->bsddriver,
1133 BUS_PASS_DEFAULT, &pdrv->bsdclass);
1134 bus_topo_unlock();
1135 return (-error);
1136 }
1137
1138 int
linux_pci_register_driver(struct pci_driver * pdrv)1139 linux_pci_register_driver(struct pci_driver *pdrv)
1140 {
1141 devclass_t dc;
1142
1143 pdrv->isdrm = strcmp(pdrv->name, "drmn") == 0;
1144 dc = pdrv->isdrm ? devclass_create("vgapci") : devclass_find("pci");
1145 if (dc == NULL)
1146 return (-ENXIO);
1147 return (_linux_pci_register_driver(pdrv, dc));
1148 }
1149
1150 static struct resource_list_entry *
lkpi_pci_get_bar(struct pci_dev * pdev,int bar,bool reserve)1151 lkpi_pci_get_bar(struct pci_dev *pdev, int bar, bool reserve)
1152 {
1153 int type;
1154
1155 type = pci_resource_type(pdev, bar);
1156 if (type < 0)
1157 return (NULL);
1158 bar = PCIR_BAR(bar);
1159 return (linux_pci_get_rle(pdev, type, bar, reserve));
1160 }
1161
1162 struct device *
lkpi_pci_find_irq_dev(unsigned int irq)1163 lkpi_pci_find_irq_dev(unsigned int irq)
1164 {
1165 struct pci_dev *pdev;
1166 struct device *found;
1167
1168 found = NULL;
1169 spin_lock(&pci_lock);
1170 list_for_each_entry(pdev, &pci_devices, links) {
1171 if (irq == pdev->dev.irq ||
1172 (irq >= pdev->dev.irq_start && irq < pdev->dev.irq_end)) {
1173 found = &pdev->dev;
1174 break;
1175 }
1176 }
1177 spin_unlock(&pci_lock);
1178 return (found);
1179 }
1180
1181 unsigned long
pci_resource_start(struct pci_dev * pdev,int bar)1182 pci_resource_start(struct pci_dev *pdev, int bar)
1183 {
1184 struct resource_list_entry *rle;
1185 rman_res_t newstart;
1186 device_t dev;
1187 int error;
1188
1189 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL)
1190 return (0);
1191 dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ?
1192 device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev;
1193 error = bus_translate_resource(dev, rle->type, rle->start, &newstart);
1194 if (error != 0) {
1195 device_printf(pdev->dev.bsddev,
1196 "translate of %#jx failed: %d\n",
1197 (uintmax_t)rle->start, error);
1198 return (0);
1199 }
1200 return (newstart);
1201 }
1202
1203 unsigned long
pci_resource_len(struct pci_dev * pdev,int bar)1204 pci_resource_len(struct pci_dev *pdev, int bar)
1205 {
1206 struct resource_list_entry *rle;
1207
1208 if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL)
1209 return (0);
1210 return (rle->count);
1211 }
1212
1213 static int
lkpi_pci_request_region(struct pci_dev * pdev,int bar,const char * res_name,bool managed)1214 lkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name,
1215 bool managed)
1216 {
1217 struct resource *res;
1218 struct pci_devres *dr;
1219 struct pci_mmio_region *mmio;
1220 int rid;
1221 int type;
1222
1223 if (!lkpi_pci_bar_id_valid(bar))
1224 return (-EINVAL);
1225
1226 type = pci_resource_type(pdev, bar);
1227 if (type < 0)
1228 return (0);
1229
1230 rid = PCIR_BAR(bar);
1231 res = bus_alloc_resource_any(pdev->dev.bsddev, type, &rid,
1232 RF_ACTIVE|RF_SHAREABLE);
1233 if (res == NULL) {
1234 device_printf(pdev->dev.bsddev, "%s: failed to alloc "
1235 "bar %d type %d rid %d\n",
1236 __func__, bar, type, PCIR_BAR(bar));
1237 return (-EBUSY);
1238 }
1239
1240 /*
1241 * It seems there is an implicit devres tracking on these if the device
1242 * is managed (lkpi_pci_devres_find() case); otherwise the resources are
1243 * not automatically freed on FreeBSD/LinuxKPI though they should be/are
1244 * expected to be by Linux drivers.
1245 * Otherwise if we are called from a pcim-function with the managed
1246 * argument set, we need to track devres independent of pdev->managed.
1247 */
1248 if (managed)
1249 dr = lkpi_pci_devres_get_alloc(pdev);
1250 else
1251 dr = lkpi_pci_devres_find(pdev);
1252 if (dr != NULL) {
1253 dr->region_mask |= (1 << bar);
1254 dr->region_table[bar] = res;
1255 }
1256
1257 /* Even if the device is not managed we need to track it for iomap. */
1258 mmio = malloc(sizeof(*mmio), M_DEVBUF, M_WAITOK | M_ZERO);
1259 mmio->rid = PCIR_BAR(bar);
1260 mmio->type = type;
1261 mmio->res = res;
1262 TAILQ_INSERT_TAIL(&pdev->mmio, mmio, next);
1263
1264 return (0);
1265 }
1266
1267 int
linuxkpi_pci_request_region(struct pci_dev * pdev,int bar,const char * res_name)1268 linuxkpi_pci_request_region(struct pci_dev *pdev, int bar, const char *res_name)
1269 {
1270 return (lkpi_pci_request_region(pdev, bar, res_name, false));
1271 }
1272
1273 int
linuxkpi_pci_request_regions(struct pci_dev * pdev,const char * res_name)1274 linuxkpi_pci_request_regions(struct pci_dev *pdev, const char *res_name)
1275 {
1276 int error;
1277 int i;
1278
1279 for (i = 0; i <= PCIR_MAX_BAR_0; i++) {
1280 error = pci_request_region(pdev, i, res_name);
1281 if (error && error != -EBUSY) {
1282 pci_release_regions(pdev);
1283 return (error);
1284 }
1285 }
1286 return (0);
1287 }
1288
1289 int
linuxkpi_pcim_request_all_regions(struct pci_dev * pdev,const char * res_name)1290 linuxkpi_pcim_request_all_regions(struct pci_dev *pdev, const char *res_name)
1291 {
1292 int bar, error;
1293
1294 for (bar = 0; bar <= PCIR_MAX_BAR_0; bar++) {
1295 error = lkpi_pci_request_region(pdev, bar, res_name, true);
1296 if (error != 0 && error != -EBUSY) {
1297 device_printf(pdev->dev.bsddev, "%s: bar %d res_name '%s': "
1298 "lkpi_pci_request_region returned %d\n", __func__,
1299 bar, res_name, error);
1300 pci_release_regions(pdev);
1301 return (error);
1302 }
1303 }
1304 return (0);
1305 }
1306
1307 void
linuxkpi_pci_release_region(struct pci_dev * pdev,int bar)1308 linuxkpi_pci_release_region(struct pci_dev *pdev, int bar)
1309 {
1310 struct resource_list_entry *rle;
1311 struct pci_devres *dr;
1312 struct pci_mmio_region *mmio, *p;
1313
1314 if ((rle = lkpi_pci_get_bar(pdev, bar, false)) == NULL)
1315 return;
1316
1317 /*
1318 * As we implicitly track the requests we also need to clear them on
1319 * release. Do clear before resource release.
1320 */
1321 dr = lkpi_pci_devres_find(pdev);
1322 if (dr != NULL) {
1323 KASSERT(dr->region_table[bar] == rle->res, ("%s: pdev %p bar %d"
1324 " region_table res %p != rel->res %p\n", __func__, pdev,
1325 bar, dr->region_table[bar], rle->res));
1326 dr->region_table[bar] = NULL;
1327 dr->region_mask &= ~(1 << bar);
1328 }
1329
1330 TAILQ_FOREACH_SAFE(mmio, &pdev->mmio, next, p) {
1331 if (rle->res != (void *)rman_get_bushandle(mmio->res))
1332 continue;
1333 TAILQ_REMOVE(&pdev->mmio, mmio, next);
1334 free(mmio, M_DEVBUF);
1335 }
1336
1337 bus_release_resource(pdev->dev.bsddev, rle->type, rle->rid, rle->res);
1338 }
1339
1340 void
linuxkpi_pci_release_regions(struct pci_dev * pdev)1341 linuxkpi_pci_release_regions(struct pci_dev *pdev)
1342 {
1343 int i;
1344
1345 for (i = 0; i <= PCIR_MAX_BAR_0; i++)
1346 pci_release_region(pdev, i);
1347 }
1348
1349 int
linux_pci_register_drm_driver(struct pci_driver * pdrv)1350 linux_pci_register_drm_driver(struct pci_driver *pdrv)
1351 {
1352 devclass_t dc;
1353
1354 dc = devclass_create("vgapci");
1355 if (dc == NULL)
1356 return (-ENXIO);
1357 pdrv->isdrm = true;
1358 pdrv->name = "drmn";
1359 return (_linux_pci_register_driver(pdrv, dc));
1360 }
1361
1362 void
linux_pci_unregister_driver(struct pci_driver * pdrv)1363 linux_pci_unregister_driver(struct pci_driver *pdrv)
1364 {
1365 devclass_t bus;
1366
1367 bus = devclass_find(pdrv->isdrm ? "vgapci" : "pci");
1368
1369 spin_lock(&pci_lock);
1370 list_del(&pdrv->node);
1371 spin_unlock(&pci_lock);
1372 bus_topo_lock();
1373 if (bus != NULL)
1374 devclass_delete_driver(bus, &pdrv->bsddriver);
1375 bus_topo_unlock();
1376 }
1377
1378 void
linux_pci_unregister_drm_driver(struct pci_driver * pdrv)1379 linux_pci_unregister_drm_driver(struct pci_driver *pdrv)
1380 {
1381 devclass_t bus;
1382
1383 bus = devclass_find("vgapci");
1384
1385 spin_lock(&pci_lock);
1386 list_del(&pdrv->node);
1387 spin_unlock(&pci_lock);
1388 bus_topo_lock();
1389 if (bus != NULL)
1390 devclass_delete_driver(bus, &pdrv->bsddriver);
1391 bus_topo_unlock();
1392 }
1393
1394 int
linuxkpi_pci_enable_msix(struct pci_dev * pdev,struct msix_entry * entries,int nreq)1395 linuxkpi_pci_enable_msix(struct pci_dev *pdev, struct msix_entry *entries,
1396 int nreq)
1397 {
1398 struct resource_list_entry *rle;
1399 int error;
1400 int avail;
1401 int i;
1402
1403 avail = pci_msix_count(pdev->dev.bsddev);
1404 if (avail < nreq) {
1405 if (avail == 0)
1406 return -EINVAL;
1407 return avail;
1408 }
1409 avail = nreq;
1410 if ((error = -pci_alloc_msix(pdev->dev.bsddev, &avail)) != 0)
1411 return error;
1412 /*
1413 * Handle case where "pci_alloc_msix()" may allocate less
1414 * interrupts than available and return with no error:
1415 */
1416 if (avail < nreq) {
1417 pci_release_msi(pdev->dev.bsddev);
1418 return avail;
1419 }
1420 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false);
1421 pdev->dev.irq_start = rle->start;
1422 pdev->dev.irq_end = rle->start + avail;
1423 for (i = 0; i < nreq; i++)
1424 entries[i].vector = pdev->dev.irq_start + i;
1425 pdev->msix_enabled = true;
1426 return (0);
1427 }
1428
1429 int
_lkpi_pci_enable_msi_range(struct pci_dev * pdev,int minvec,int maxvec)1430 _lkpi_pci_enable_msi_range(struct pci_dev *pdev, int minvec, int maxvec)
1431 {
1432 struct resource_list_entry *rle;
1433 int error;
1434 int nvec;
1435
1436 if (maxvec < minvec)
1437 return (-EINVAL);
1438
1439 nvec = pci_msi_count(pdev->dev.bsddev);
1440 if (nvec < 1 || nvec < minvec)
1441 return (-ENOSPC);
1442
1443 nvec = min(nvec, maxvec);
1444 if ((error = -pci_alloc_msi(pdev->dev.bsddev, &nvec)) != 0)
1445 return error;
1446
1447 /* Native PCI might only ever ask for 32 vectors. */
1448 if (nvec < minvec) {
1449 pci_release_msi(pdev->dev.bsddev);
1450 return (-ENOSPC);
1451 }
1452
1453 rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false);
1454 pdev->dev.irq_start = rle->start;
1455 pdev->dev.irq_end = rle->start + nvec;
1456 pdev->irq = rle->start;
1457 pdev->msi_enabled = true;
1458 return (0);
1459 }
1460
1461 int
pci_alloc_irq_vectors(struct pci_dev * pdev,int minv,int maxv,unsigned int flags)1462 pci_alloc_irq_vectors(struct pci_dev *pdev, int minv, int maxv,
1463 unsigned int flags)
1464 {
1465 int error;
1466
1467 if (flags & PCI_IRQ_MSIX) {
1468 struct msix_entry *entries;
1469 int i;
1470
1471 entries = kcalloc(maxv, sizeof(*entries), GFP_KERNEL);
1472 if (entries == NULL) {
1473 error = -ENOMEM;
1474 goto out;
1475 }
1476 for (i = 0; i < maxv; ++i)
1477 entries[i].entry = i;
1478 error = pci_enable_msix(pdev, entries, maxv);
1479 out:
1480 kfree(entries);
1481 if (error == 0 && pdev->msix_enabled)
1482 return (pdev->dev.irq_end - pdev->dev.irq_start);
1483 }
1484 if (flags & PCI_IRQ_MSI) {
1485 if (pci_msi_count(pdev->dev.bsddev) < minv)
1486 return (-ENOSPC);
1487 error = _lkpi_pci_enable_msi_range(pdev, minv, maxv);
1488 if (error == 0 && pdev->msi_enabled)
1489 return (pdev->dev.irq_end - pdev->dev.irq_start);
1490 }
1491 if (flags & PCI_IRQ_INTX) {
1492 if (pdev->irq)
1493 return (1);
1494 }
1495
1496 return (-EINVAL);
1497 }
1498
1499 struct msi_desc *
lkpi_pci_msi_desc_alloc(int irq)1500 lkpi_pci_msi_desc_alloc(int irq)
1501 {
1502 struct device *dev;
1503 struct pci_dev *pdev;
1504 struct msi_desc *desc;
1505 struct pci_devinfo *dinfo;
1506 struct pcicfg_msi *msi;
1507 int vec;
1508
1509 dev = lkpi_pci_find_irq_dev(irq);
1510 if (dev == NULL)
1511 return (NULL);
1512
1513 pdev = to_pci_dev(dev);
1514
1515 if (pdev->msi_desc == NULL)
1516 return (NULL);
1517
1518 if (irq < pdev->dev.irq_start || irq >= pdev->dev.irq_end)
1519 return (NULL);
1520
1521 vec = pdev->dev.irq_start - irq;
1522
1523 if (pdev->msi_desc[vec] != NULL)
1524 return (pdev->msi_desc[vec]);
1525
1526 dinfo = device_get_ivars(dev->bsddev);
1527 msi = &dinfo->cfg.msi;
1528
1529 desc = malloc(sizeof(*desc), M_DEVBUF, M_WAITOK | M_ZERO);
1530
1531 desc->pci.msi_attrib.is_64 =
1532 (msi->msi_ctrl & PCIM_MSICTRL_64BIT) ? true : false;
1533 desc->msg.data = msi->msi_data;
1534
1535 pdev->msi_desc[vec] = desc;
1536
1537 return (desc);
1538 }
1539
1540 bool
pci_device_is_present(struct pci_dev * pdev)1541 pci_device_is_present(struct pci_dev *pdev)
1542 {
1543 device_t dev;
1544
1545 dev = pdev->dev.bsddev;
1546
1547 return (bus_child_present(dev));
1548 }
1549
1550 CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t));
1551
1552 struct linux_dma_obj {
1553 void *vaddr;
1554 uint64_t dma_addr;
1555 bus_dmamap_t dmamap;
1556 bus_dma_tag_t dmat;
1557 };
1558
1559 static uma_zone_t linux_dma_trie_zone;
1560 static uma_zone_t linux_dma_obj_zone;
1561
1562 static void
linux_dma_init(void * arg)1563 linux_dma_init(void *arg)
1564 {
1565
1566 linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie",
1567 pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL,
1568 UMA_ALIGN_PTR, 0);
1569 linux_dma_obj_zone = uma_zcreate("linux_dma_object",
1570 sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL,
1571 UMA_ALIGN_PTR, 0);
1572 lkpi_pci_nseg1_fail = counter_u64_alloc(M_WAITOK);
1573 }
1574 SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL);
1575
1576 static void
linux_dma_uninit(void * arg)1577 linux_dma_uninit(void *arg)
1578 {
1579
1580 counter_u64_free(lkpi_pci_nseg1_fail);
1581 uma_zdestroy(linux_dma_obj_zone);
1582 uma_zdestroy(linux_dma_trie_zone);
1583 }
1584 SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL);
1585
1586 static void *
linux_dma_trie_alloc(struct pctrie * ptree)1587 linux_dma_trie_alloc(struct pctrie *ptree)
1588 {
1589
1590 return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT));
1591 }
1592
1593 static void
linux_dma_trie_free(struct pctrie * ptree,void * node)1594 linux_dma_trie_free(struct pctrie *ptree, void *node)
1595 {
1596
1597 uma_zfree(linux_dma_trie_zone, node);
1598 }
1599
1600 PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc,
1601 linux_dma_trie_free);
1602
1603 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
1604 static dma_addr_t
linux_dma_map_phys_common(struct device * dev,vm_paddr_t phys,size_t len,bus_dma_tag_t dmat)1605 linux_dma_map_phys_common(struct device *dev, vm_paddr_t phys, size_t len,
1606 bus_dma_tag_t dmat)
1607 {
1608 struct linux_dma_priv *priv;
1609 struct linux_dma_obj *obj;
1610 int error, nseg;
1611 bus_dma_segment_t seg;
1612
1613 priv = dev->dma_priv;
1614
1615 /*
1616 * If the resultant mapping will be entirely 1:1 with the
1617 * physical address, short-circuit the remainder of the
1618 * bus_dma API. This avoids tracking collisions in the pctrie
1619 * with the additional benefit of reducing overhead.
1620 */
1621 if (bus_dma_id_mapped(dmat, phys, len))
1622 return (phys);
1623
1624 obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT);
1625 if (obj == NULL) {
1626 return (0);
1627 }
1628 obj->dmat = dmat;
1629
1630 DMA_PRIV_LOCK(priv);
1631 if (bus_dmamap_create(obj->dmat, 0, &obj->dmamap) != 0) {
1632 DMA_PRIV_UNLOCK(priv);
1633 uma_zfree(linux_dma_obj_zone, obj);
1634 return (0);
1635 }
1636
1637 nseg = -1;
1638 error = _bus_dmamap_load_phys(obj->dmat, obj->dmamap, phys, len,
1639 BUS_DMA_NOWAIT, &seg, &nseg);
1640 if (error != 0) {
1641 bus_dmamap_destroy(obj->dmat, obj->dmamap);
1642 DMA_PRIV_UNLOCK(priv);
1643 uma_zfree(linux_dma_obj_zone, obj);
1644 counter_u64_add(lkpi_pci_nseg1_fail, 1);
1645 if (linuxkpi_debug) {
1646 device_printf(dev->bsddev, "%s: _bus_dmamap_load_phys "
1647 "error %d, phys %#018jx len %zu\n", __func__,
1648 error, (uintmax_t)phys, len);
1649 dump_stack();
1650 }
1651 return (0);
1652 }
1653
1654 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));
1655 obj->dma_addr = seg.ds_addr;
1656
1657 error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj);
1658 if (error != 0) {
1659 bus_dmamap_unload(obj->dmat, obj->dmamap);
1660 bus_dmamap_destroy(obj->dmat, obj->dmamap);
1661 DMA_PRIV_UNLOCK(priv);
1662 uma_zfree(linux_dma_obj_zone, obj);
1663 return (0);
1664 }
1665 DMA_PRIV_UNLOCK(priv);
1666 return (obj->dma_addr);
1667 }
1668 #else
1669 static dma_addr_t
linux_dma_map_phys_common(struct device * dev __unused,vm_paddr_t phys,size_t len __unused,bus_dma_tag_t dmat __unused)1670 linux_dma_map_phys_common(struct device *dev __unused, vm_paddr_t phys,
1671 size_t len __unused, bus_dma_tag_t dmat __unused)
1672 {
1673 return (phys);
1674 }
1675 #endif
1676
1677 dma_addr_t
lkpi_dma_map_phys(struct device * dev,vm_paddr_t phys,size_t len,enum dma_data_direction direction,unsigned long attrs)1678 lkpi_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len,
1679 enum dma_data_direction direction, unsigned long attrs)
1680 {
1681 struct linux_dma_priv *priv;
1682 dma_addr_t dma;
1683
1684 priv = dev->dma_priv;
1685 dma = linux_dma_map_phys_common(dev, phys, len, priv->dmat);
1686 if (dma_mapping_error(dev, dma))
1687 return (dma);
1688
1689 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1690 dma_sync_single_for_device(dev, dma, len, direction);
1691
1692 return (dma);
1693 }
1694
1695 /* For backward compat only so we can MFC this. Remove before 15. */
1696 dma_addr_t
linux_dma_map_phys(struct device * dev,vm_paddr_t phys,size_t len)1697 linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len)
1698 {
1699 return (lkpi_dma_map_phys(dev, phys, len, DMA_NONE, 0));
1700 }
1701
1702 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
1703 void
lkpi_dma_unmap(struct device * dev,dma_addr_t dma_addr,size_t len,enum dma_data_direction direction,unsigned long attrs)1704 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len,
1705 enum dma_data_direction direction, unsigned long attrs)
1706 {
1707 struct linux_dma_priv *priv;
1708 struct linux_dma_obj *obj;
1709
1710 priv = dev->dma_priv;
1711
1712 if (pctrie_is_empty(&priv->ptree))
1713 return;
1714
1715 DMA_PRIV_LOCK(priv);
1716 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr);
1717 if (obj == NULL) {
1718 DMA_PRIV_UNLOCK(priv);
1719 return;
1720 }
1721 LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr);
1722
1723 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
1724 dma_sync_single_for_cpu(dev, dma_addr, len, direction);
1725
1726 bus_dmamap_unload(obj->dmat, obj->dmamap);
1727 bus_dmamap_destroy(obj->dmat, obj->dmamap);
1728 DMA_PRIV_UNLOCK(priv);
1729
1730 uma_zfree(linux_dma_obj_zone, obj);
1731 }
1732 #else
1733 void
lkpi_dma_unmap(struct device * dev,dma_addr_t dma_addr,size_t len,enum dma_data_direction direction,unsigned long attrs)1734 lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len,
1735 enum dma_data_direction direction, unsigned long attrs)
1736 {
1737 }
1738 #endif
1739
1740 /* For backward compat only so we can MFC this. Remove before 15. */
1741 void
linux_dma_unmap(struct device * dev,dma_addr_t dma_addr,size_t len)1742 linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len)
1743 {
1744 lkpi_dma_unmap(dev, dma_addr, len, DMA_NONE, 0);
1745 }
1746
1747 void *
linux_dma_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t flag)1748 linux_dma_alloc_coherent(struct device *dev, size_t size,
1749 dma_addr_t *dma_handle, gfp_t flag)
1750 {
1751 struct linux_dma_priv *priv;
1752 vm_paddr_t high;
1753 size_t align;
1754 void *mem;
1755
1756 if (dev == NULL || dev->dma_priv == NULL) {
1757 *dma_handle = 0;
1758 return (NULL);
1759 }
1760 priv = dev->dma_priv;
1761 if (priv->dma_coherent_mask)
1762 high = priv->dma_coherent_mask;
1763 else
1764 /* Coherent is lower 32bit only by default in Linux. */
1765 high = BUS_SPACE_MAXADDR_32BIT;
1766 align = PAGE_SIZE << get_order(size);
1767 /* Always zero the allocation. */
1768 flag |= M_ZERO;
1769 mem = kmem_alloc_contig(size, flag & GFP_NATIVE_MASK, 0, high,
1770 align, 0, VM_MEMATTR_DEFAULT);
1771 if (mem != NULL) {
1772 *dma_handle = linux_dma_map_phys_common(dev, vtophys(mem), size,
1773 priv->dmat_coherent);
1774 if (*dma_handle == 0) {
1775 kmem_free(mem, size);
1776 mem = NULL;
1777 }
1778 } else {
1779 *dma_handle = 0;
1780 }
1781 return (mem);
1782 }
1783
1784 struct lkpi_devres_dmam_coherent {
1785 size_t size;
1786 dma_addr_t *handle;
1787 void *mem;
1788 };
1789
1790 static void
lkpi_dmam_free_coherent(struct device * dev,void * p)1791 lkpi_dmam_free_coherent(struct device *dev, void *p)
1792 {
1793 struct lkpi_devres_dmam_coherent *dr;
1794
1795 dr = p;
1796 dma_free_coherent(dev, dr->size, dr->mem, *dr->handle);
1797 }
1798
1799 void *
linuxkpi_dmam_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t flag)1800 linuxkpi_dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
1801 gfp_t flag)
1802 {
1803 struct lkpi_devres_dmam_coherent *dr;
1804
1805 dr = lkpi_devres_alloc(lkpi_dmam_free_coherent,
1806 sizeof(*dr), GFP_KERNEL | __GFP_ZERO);
1807
1808 if (dr == NULL)
1809 return (NULL);
1810
1811 dr->size = size;
1812 dr->mem = linux_dma_alloc_coherent(dev, size, dma_handle, flag);
1813 dr->handle = dma_handle;
1814 if (dr->mem == NULL) {
1815 lkpi_devres_free(dr);
1816 return (NULL);
1817 }
1818
1819 lkpi_devres_add(dev, dr);
1820 return (dr->mem);
1821 }
1822
1823 void
linuxkpi_dma_sync(struct device * dev,dma_addr_t dma_addr,size_t size,bus_dmasync_op_t op)1824 linuxkpi_dma_sync(struct device *dev, dma_addr_t dma_addr, size_t size,
1825 bus_dmasync_op_t op)
1826 {
1827 struct linux_dma_priv *priv;
1828 struct linux_dma_obj *obj;
1829
1830 priv = dev->dma_priv;
1831
1832 if (pctrie_is_empty(&priv->ptree))
1833 return;
1834
1835 DMA_PRIV_LOCK(priv);
1836 obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr);
1837 if (obj == NULL) {
1838 DMA_PRIV_UNLOCK(priv);
1839 return;
1840 }
1841
1842 bus_dmamap_sync(obj->dmat, obj->dmamap, op);
1843 DMA_PRIV_UNLOCK(priv);
1844 }
1845
1846 int
linux_dma_map_sg_attrs(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction direction,unsigned long attrs)1847 linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents,
1848 enum dma_data_direction direction, unsigned long attrs)
1849 {
1850 struct linux_dma_priv *priv;
1851 struct scatterlist *sg;
1852 int i, nseg;
1853 bus_dma_segment_t seg;
1854
1855 priv = dev->dma_priv;
1856
1857 DMA_PRIV_LOCK(priv);
1858
1859 /* create common DMA map in the first S/G entry */
1860 if (bus_dmamap_create(priv->dmat, 0, &sgl->dma_map) != 0) {
1861 DMA_PRIV_UNLOCK(priv);
1862 return (0);
1863 }
1864
1865 /* load all S/G list entries */
1866 for_each_sg(sgl, sg, nents, i) {
1867 nseg = -1;
1868 if (_bus_dmamap_load_phys(priv->dmat, sgl->dma_map,
1869 sg_phys(sg), sg->length, BUS_DMA_NOWAIT,
1870 &seg, &nseg) != 0) {
1871 bus_dmamap_unload(priv->dmat, sgl->dma_map);
1872 bus_dmamap_destroy(priv->dmat, sgl->dma_map);
1873 DMA_PRIV_UNLOCK(priv);
1874 return (0);
1875 }
1876 KASSERT(nseg == 0,
1877 ("More than one segment (nseg=%d)", nseg + 1));
1878
1879 sg_dma_address(sg) = seg.ds_addr;
1880 }
1881
1882 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0)
1883 goto skip_sync;
1884
1885 switch (direction) {
1886 case DMA_BIDIRECTIONAL:
1887 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE);
1888 break;
1889 case DMA_TO_DEVICE:
1890 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREREAD);
1891 break;
1892 case DMA_FROM_DEVICE:
1893 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREWRITE);
1894 break;
1895 default:
1896 break;
1897 }
1898 skip_sync:
1899
1900 DMA_PRIV_UNLOCK(priv);
1901
1902 return (nents);
1903 }
1904
1905 void
linux_dma_unmap_sg_attrs(struct device * dev,struct scatterlist * sgl,int nents __unused,enum dma_data_direction direction,unsigned long attrs)1906 linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
1907 int nents __unused, enum dma_data_direction direction,
1908 unsigned long attrs)
1909 {
1910 struct linux_dma_priv *priv;
1911
1912 priv = dev->dma_priv;
1913
1914 DMA_PRIV_LOCK(priv);
1915
1916 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) != 0)
1917 goto skip_sync;
1918
1919 switch (direction) {
1920 case DMA_BIDIRECTIONAL:
1921 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD);
1922 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_PREREAD);
1923 break;
1924 case DMA_TO_DEVICE:
1925 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTWRITE);
1926 break;
1927 case DMA_FROM_DEVICE:
1928 bus_dmamap_sync(priv->dmat, sgl->dma_map, BUS_DMASYNC_POSTREAD);
1929 break;
1930 default:
1931 break;
1932 }
1933 skip_sync:
1934
1935 bus_dmamap_unload(priv->dmat, sgl->dma_map);
1936 bus_dmamap_destroy(priv->dmat, sgl->dma_map);
1937 DMA_PRIV_UNLOCK(priv);
1938 }
1939
1940 struct dma_pool {
1941 struct device *pool_device;
1942 uma_zone_t pool_zone;
1943 struct mtx pool_lock;
1944 bus_dma_tag_t pool_dmat;
1945 size_t pool_entry_size;
1946 struct pctrie pool_ptree;
1947 };
1948
1949 #define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock)
1950 #define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock)
1951
1952 static inline int
dma_pool_obj_ctor(void * mem,int size,void * arg,int flags)1953 dma_pool_obj_ctor(void *mem, int size, void *arg, int flags)
1954 {
1955 struct linux_dma_obj *obj = mem;
1956 struct dma_pool *pool = arg;
1957 int error, nseg;
1958 bus_dma_segment_t seg;
1959
1960 nseg = -1;
1961 DMA_POOL_LOCK(pool);
1962 error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap,
1963 vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT,
1964 &seg, &nseg);
1965 DMA_POOL_UNLOCK(pool);
1966 if (error != 0) {
1967 return (error);
1968 }
1969 KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));
1970 obj->dma_addr = seg.ds_addr;
1971
1972 return (0);
1973 }
1974
1975 static void
dma_pool_obj_dtor(void * mem,int size,void * arg)1976 dma_pool_obj_dtor(void *mem, int size, void *arg)
1977 {
1978 struct linux_dma_obj *obj = mem;
1979 struct dma_pool *pool = arg;
1980
1981 DMA_POOL_LOCK(pool);
1982 bus_dmamap_unload(pool->pool_dmat, obj->dmamap);
1983 DMA_POOL_UNLOCK(pool);
1984 }
1985
1986 static int
dma_pool_obj_import(void * arg,void ** store,int count,int domain __unused,int flags)1987 dma_pool_obj_import(void *arg, void **store, int count, int domain __unused,
1988 int flags)
1989 {
1990 struct dma_pool *pool = arg;
1991 struct linux_dma_obj *obj;
1992 int error, i;
1993
1994 for (i = 0; i < count; i++) {
1995 obj = uma_zalloc(linux_dma_obj_zone, flags);
1996 if (obj == NULL)
1997 break;
1998
1999 error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr,
2000 BUS_DMA_NOWAIT, &obj->dmamap);
2001 if (error!= 0) {
2002 uma_zfree(linux_dma_obj_zone, obj);
2003 break;
2004 }
2005
2006 store[i] = obj;
2007 }
2008
2009 return (i);
2010 }
2011
2012 static void
dma_pool_obj_release(void * arg,void ** store,int count)2013 dma_pool_obj_release(void *arg, void **store, int count)
2014 {
2015 struct dma_pool *pool = arg;
2016 struct linux_dma_obj *obj;
2017 int i;
2018
2019 for (i = 0; i < count; i++) {
2020 obj = store[i];
2021 bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap);
2022 uma_zfree(linux_dma_obj_zone, obj);
2023 }
2024 }
2025
2026 struct dma_pool *
linux_dma_pool_create(char * name,struct device * dev,size_t size,size_t align,size_t boundary)2027 linux_dma_pool_create(char *name, struct device *dev, size_t size,
2028 size_t align, size_t boundary)
2029 {
2030 struct linux_dma_priv *priv;
2031 struct dma_pool *pool;
2032
2033 priv = dev->dma_priv;
2034
2035 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
2036 pool->pool_device = dev;
2037 pool->pool_entry_size = size;
2038
2039 if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev),
2040 align, boundary, /* alignment, boundary */
2041 priv->dma_mask, /* lowaddr */
2042 BUS_SPACE_MAXADDR, /* highaddr */
2043 NULL, NULL, /* filtfunc, filtfuncarg */
2044 size, /* maxsize */
2045 1, /* nsegments */
2046 size, /* maxsegsz */
2047 0, /* flags */
2048 NULL, NULL, /* lockfunc, lockfuncarg */
2049 &pool->pool_dmat)) {
2050 kfree(pool);
2051 return (NULL);
2052 }
2053
2054 pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor,
2055 dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import,
2056 dma_pool_obj_release, pool, 0);
2057
2058 mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF);
2059 pctrie_init(&pool->pool_ptree);
2060
2061 return (pool);
2062 }
2063
2064 void
linux_dma_pool_destroy(struct dma_pool * pool)2065 linux_dma_pool_destroy(struct dma_pool *pool)
2066 {
2067
2068 uma_zdestroy(pool->pool_zone);
2069 bus_dma_tag_destroy(pool->pool_dmat);
2070 mtx_destroy(&pool->pool_lock);
2071 kfree(pool);
2072 }
2073
2074 void
lkpi_dmam_pool_destroy(struct device * dev,void * p)2075 lkpi_dmam_pool_destroy(struct device *dev, void *p)
2076 {
2077 struct dma_pool *pool;
2078
2079 pool = *(struct dma_pool **)p;
2080 LINUX_DMA_PCTRIE_RECLAIM(&pool->pool_ptree);
2081 linux_dma_pool_destroy(pool);
2082 }
2083
2084 void *
linux_dma_pool_alloc(struct dma_pool * pool,gfp_t mem_flags,dma_addr_t * handle)2085 linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
2086 dma_addr_t *handle)
2087 {
2088 struct linux_dma_obj *obj;
2089
2090 obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags & GFP_NATIVE_MASK);
2091 if (obj == NULL)
2092 return (NULL);
2093
2094 DMA_POOL_LOCK(pool);
2095 if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) {
2096 DMA_POOL_UNLOCK(pool);
2097 uma_zfree_arg(pool->pool_zone, obj, pool);
2098 return (NULL);
2099 }
2100 DMA_POOL_UNLOCK(pool);
2101
2102 *handle = obj->dma_addr;
2103 return (obj->vaddr);
2104 }
2105
2106 void
linux_dma_pool_free(struct dma_pool * pool,void * vaddr,dma_addr_t dma_addr)2107 linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr)
2108 {
2109 struct linux_dma_obj *obj;
2110
2111 DMA_POOL_LOCK(pool);
2112 obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr);
2113 if (obj == NULL) {
2114 DMA_POOL_UNLOCK(pool);
2115 return;
2116 }
2117 LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr);
2118 DMA_POOL_UNLOCK(pool);
2119
2120 uma_zfree_arg(pool->pool_zone, obj, pool);
2121 }
2122
2123 static int
linux_backlight_get_status(device_t dev,struct backlight_props * props)2124 linux_backlight_get_status(device_t dev, struct backlight_props *props)
2125 {
2126 struct pci_dev *pdev;
2127
2128 linux_set_current(curthread);
2129 pdev = device_get_softc(dev);
2130
2131 props->brightness = pdev->dev.bd->props.brightness;
2132 props->brightness = props->brightness * 100 / pdev->dev.bd->props.max_brightness;
2133 props->nlevels = 0;
2134
2135 return (0);
2136 }
2137
2138 static int
linux_backlight_get_info(device_t dev,struct backlight_info * info)2139 linux_backlight_get_info(device_t dev, struct backlight_info *info)
2140 {
2141 struct pci_dev *pdev;
2142
2143 linux_set_current(curthread);
2144 pdev = device_get_softc(dev);
2145
2146 info->type = BACKLIGHT_TYPE_PANEL;
2147 strlcpy(info->name, pdev->dev.bd->name, BACKLIGHTMAXNAMELENGTH);
2148 return (0);
2149 }
2150
2151 static int
linux_backlight_update_status(device_t dev,struct backlight_props * props)2152 linux_backlight_update_status(device_t dev, struct backlight_props *props)
2153 {
2154 struct pci_dev *pdev;
2155
2156 linux_set_current(curthread);
2157 pdev = device_get_softc(dev);
2158
2159 pdev->dev.bd->props.brightness = pdev->dev.bd->props.max_brightness *
2160 props->brightness / 100;
2161 pdev->dev.bd->props.power = props->brightness == 0 ?
2162 4/* FB_BLANK_POWERDOWN */ : 0/* FB_BLANK_UNBLANK */;
2163 return (pdev->dev.bd->ops->update_status(pdev->dev.bd));
2164 }
2165
2166 struct backlight_device *
linux_backlight_device_register(const char * name,struct device * dev,void * data,const struct backlight_ops * ops,struct backlight_properties * props)2167 linux_backlight_device_register(const char *name, struct device *dev,
2168 void *data, const struct backlight_ops *ops, struct backlight_properties *props)
2169 {
2170
2171 dev->bd = malloc(sizeof(*dev->bd), M_DEVBUF, M_WAITOK | M_ZERO);
2172 dev->bd->ops = ops;
2173 dev->bd->props.type = props->type;
2174 dev->bd->props.max_brightness = props->max_brightness;
2175 dev->bd->props.brightness = props->brightness;
2176 dev->bd->props.power = props->power;
2177 dev->bd->data = data;
2178 dev->bd->dev = dev;
2179 dev->bd->name = strdup(name, M_DEVBUF);
2180
2181 dev->backlight_dev = backlight_register(name, dev->bsddev);
2182
2183 return (dev->bd);
2184 }
2185
2186 void
linux_backlight_device_unregister(struct backlight_device * bd)2187 linux_backlight_device_unregister(struct backlight_device *bd)
2188 {
2189
2190 backlight_destroy(bd->dev->backlight_dev);
2191 free(bd->name, M_DEVBUF);
2192 free(bd, M_DEVBUF);
2193 }
2194