1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1997, 1998 Justin T. Gibbs.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions, and the following disclaimer,
12 * without modification, immediately at the beginning of the file.
13 * 2. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 /*
30 * From amd64/busdma_machdep.c, r204214
31 */
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/domainset.h>
36 #include <sys/malloc.h>
37 #include <sys/bus.h>
38 #include <sys/interrupt.h>
39 #include <sys/kernel.h>
40 #include <sys/ktr.h>
41 #include <sys/lock.h>
42 #include <sys/proc.h>
43 #include <sys/memdesc.h>
44 #include <sys/mutex.h>
45 #include <sys/sysctl.h>
46 #include <sys/uio.h>
47
48 #include <vm/vm.h>
49 #include <vm/vm_extern.h>
50 #include <vm/vm_kern.h>
51 #include <vm/vm_page.h>
52 #include <vm/vm_map.h>
53 #include <vm/vm_phys.h>
54
55 #include <machine/atomic.h>
56 #include <machine/bus.h>
57 #include <machine/cpufunc.h>
58 #include <machine/md_var.h>
59
60 #include "iommu_if.h"
61
62 #define MAX_BPAGES MIN(8192, physmem/40)
63
64 enum {
65 BF_COULD_BOUNCE = 0x01,
66 BF_MIN_ALLOC_COMP = 0x02,
67 BF_KMEM_ALLOC = 0x04,
68 BF_COHERENT = 0x10,
69 };
70
71 struct bounce_page;
72 struct bounce_zone;
73
74 struct bus_dma_tag {
75 struct bus_dma_tag_common common;
76 int map_count;
77 int bounce_flags;
78 struct bounce_zone *bounce_zone;
79 device_t iommu;
80 void *iommu_cookie;
81 };
82
83 struct bus_dmamap {
84 STAILQ_HEAD(, bounce_page) bpages;
85 int pagesneeded;
86 int pagesreserved;
87 bus_dma_tag_t dmat;
88 struct memdesc mem;
89 bus_dma_segment_t *segments;
90 int nsegs;
91 bus_dmamap_callback_t *callback;
92 void *callback_arg;
93 __sbintime_t queued_time;
94 STAILQ_ENTRY(bus_dmamap) links;
95 int contigalloc;
96 };
97
98 static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata");
99
100 #define dmat_alignment(dmat) ((dmat)->common.alignment)
101 #define dmat_bounce_flags(dmat) ((dmat)->bounce_flags)
102 #define dmat_boundary(dmat) ((dmat)->common.boundary)
103 #define dmat_domain(dmat) ((dmat)->common.domain)
104 #define dmat_flags(dmat) ((dmat)->common.flags)
105 #define dmat_highaddr(dmat) ((dmat)->common.highaddr)
106 #define dmat_lowaddr(dmat) ((dmat)->common.lowaddr)
107 #define dmat_lockfunc(dmat) ((dmat)->common.lockfunc)
108 #define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg)
109 #define dmat_maxsegsz(dmat) ((dmat)->common.maxsegsz)
110 #define dmat_nsegments(dmat) ((dmat)->common.nsegments)
111
112 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
113 "Busdma parameters");
114
115 #include "../../kern/subr_busdma_bounce.c"
116
117 /*
118 * Returns true if the address falls within the tag's exclusion window, or
119 * fails to meet its alignment requirements.
120 */
121 static __inline bool
must_bounce(bus_dma_tag_t dmat,bus_addr_t paddr)122 must_bounce(bus_dma_tag_t dmat, bus_addr_t paddr)
123 {
124
125 if (dmat->iommu == NULL && paddr > dmat->common.lowaddr &&
126 paddr <= dmat->common.highaddr)
127 return (true);
128 if (!vm_addr_align_ok(paddr, dmat->common.alignment))
129 return (true);
130
131 return (false);
132 }
133
134 static int
bounce_bus_dma_zone_setup(bus_dma_tag_t newtag)135 bounce_bus_dma_zone_setup(bus_dma_tag_t newtag)
136 {
137 struct bounce_zone *bz;
138 const u_long maxsize = newtag->common.maxsize;
139 int error = 0;
140
141 if ((error = alloc_bounce_zone(newtag)) != 0) {
142 return (error);
143 }
144 bz = newtag->bounce_zone;
145
146 if (ptoa(bz->total_bpages) < maxsize) {
147 int pages;
148
149 pages = atop(maxsize) - bz->total_bpages;
150
151 /* Add pages to our bounce pool */
152 if (alloc_bounce_pages(newtag, pages) < pages)
153 error = ENOMEM;
154 }
155 /* Performed initial allocation */
156 newtag->bounce_flags |= BF_MIN_ALLOC_COMP;
157
158 return (error);
159 }
160
161 /*
162 * Allocate a device specific dma_tag.
163 */
164 static int
bounce_bus_dma_tag_create(bus_dma_tag_t parent,bus_size_t alignment,bus_addr_t boundary,bus_addr_t lowaddr,bus_addr_t highaddr,bus_size_t maxsize,int nsegments,bus_size_t maxsegsz,int flags,bus_dma_lock_t * lockfunc,void * lockfuncarg,bus_dma_tag_t * dmat)165 bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
166 bus_addr_t boundary, bus_addr_t lowaddr,
167 bus_addr_t highaddr, bus_size_t maxsize, int nsegments,
168 bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
169 void *lockfuncarg, bus_dma_tag_t *dmat)
170 {
171 bus_dma_tag_t newtag;
172 int error = 0;
173
174 /* Basic sanity checking */
175 if (boundary != 0 && boundary < maxsegsz)
176 maxsegsz = boundary;
177
178 if (maxsegsz == 0) {
179 return (EINVAL);
180 }
181
182 /* Return a NULL tag on failure */
183 *dmat = NULL;
184
185 error = common_bus_dma_tag_create(parent != NULL ? &parent->common :
186 NULL, alignment, boundary, lowaddr, highaddr, maxsize, nsegments,
187 maxsegsz, flags, lockfunc, lockfuncarg,
188 sizeof (struct bus_dma_tag), (void **)&newtag);
189 if (error != 0)
190 return (error);
191
192 newtag->map_count = 0;
193 newtag->common.impl = &bus_dma_bounce_impl;
194
195 /* Take into account any restrictions imposed by our parent tag */
196 if (parent != NULL) {
197 newtag->iommu = parent->iommu;
198 newtag->iommu_cookie = parent->iommu_cookie;
199 }
200
201 if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) && newtag->iommu == NULL)
202 newtag->bounce_flags |= BF_COULD_BOUNCE;
203
204 if (newtag->common.alignment > 1)
205 newtag->bounce_flags |= BF_COULD_BOUNCE;
206
207 if (((newtag->bounce_flags & BF_COULD_BOUNCE) != 0) &&
208 (flags & BUS_DMA_ALLOCNOW) != 0) {
209 /* Must bounce */
210 error = bounce_bus_dma_zone_setup(newtag);
211 }
212
213 if (error != 0) {
214 free(newtag, M_DEVBUF);
215 } else {
216 *dmat = newtag;
217 }
218 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
219 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), error);
220 return (error);
221 }
222
223 static int
bounce_bus_dma_tag_set_domain(bus_dma_tag_t dmat,int domain)224 bounce_bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain)
225 {
226
227 /* TODO */
228 return (0);
229 }
230
231 static int
bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat)232 bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat)
233 {
234 int error = 0;
235
236 if (dmat != NULL) {
237 if (dmat->map_count != 0) {
238 error = EBUSY;
239 goto out;
240 }
241
242 free(dmat, M_DEVBUF);
243 }
244 out:
245 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat, error);
246 return (error);
247 }
248
249 static bus_dmamap_t
alloc_dmamap(bus_dma_tag_t dmat,int flags)250 alloc_dmamap(bus_dma_tag_t dmat, int flags)
251 {
252 u_long mapsize;
253 bus_dmamap_t map;
254
255 mapsize = sizeof(*map);
256 /* TODO: sync_list */
257 map = malloc_domainset(mapsize, M_DEVBUF,
258 DOMAINSET_PREF(dmat->common.domain), flags | M_ZERO);
259 if (map == NULL)
260 return (NULL);
261
262 /* Initialize the new map */
263 STAILQ_INIT(&map->bpages);
264
265 return (map);
266 }
267
268 /*
269 * Allocate a handle for mapping from kva/uva/physical
270 * address space into bus device space.
271 */
272 static int
bounce_bus_dmamap_create(bus_dma_tag_t dmat,int flags,bus_dmamap_t * mapp)273 bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
274 {
275 int error;
276
277 error = 0;
278
279 *mapp = alloc_dmamap(dmat, M_NOWAIT);
280 if (*mapp == NULL) {
281 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
282 __func__, dmat, ENOMEM);
283 return (ENOMEM);
284 }
285
286 /*
287 * Bouncing might be required if the driver asks for an active
288 * exclusion region, a data alignment that is stricter than 1, and/or
289 * an active address boundary.
290 */
291 if (dmat->bounce_flags & BF_COULD_BOUNCE) {
292 /* Must bounce */
293 struct bounce_zone *bz;
294 int maxpages;
295
296 if (dmat->bounce_zone == NULL) {
297 if ((error = alloc_bounce_zone(dmat)) != 0)
298 return (error);
299 }
300 bz = dmat->bounce_zone;
301
302 /*
303 * Attempt to add pages to our pool on a per-instance
304 * basis up to a sane limit.
305 */
306 if (dmat->common.alignment > 1)
307 maxpages = MAX_BPAGES;
308 else
309 maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->common.lowaddr));
310 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0
311 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
312 int pages;
313
314 pages = MAX(atop(dmat->common.maxsize), 1);
315 pages = MIN(maxpages - bz->total_bpages, pages);
316 pages = MAX(pages, 1);
317 if (alloc_bounce_pages(dmat, pages) < pages)
318 error = ENOMEM;
319
320 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0) {
321 if (error == 0)
322 dmat->bounce_flags |= BF_MIN_ALLOC_COMP;
323 } else {
324 error = 0;
325 }
326 }
327 bz->map_count++;
328 }
329
330 (*mapp)->nsegs = 0;
331 (*mapp)->segments = (bus_dma_segment_t *)malloc(
332 sizeof(bus_dma_segment_t) * dmat->common.nsegments, M_DEVBUF,
333 M_NOWAIT);
334 if ((*mapp)->segments == NULL) {
335 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
336 __func__, dmat, ENOMEM);
337 error = ENOMEM;
338 }
339
340 if (error == 0) {
341 dmat->map_count++;
342 } else {
343 free((*mapp)->segments, M_DEVBUF);
344 free(*mapp, M_DEVBUF);
345 *mapp = NULL;
346 }
347 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
348 __func__, dmat, dmat->common.flags, error);
349 return (error);
350 }
351
352 /*
353 * Destroy a handle for mapping from kva/uva/physical
354 * address space into bus device space.
355 */
356 static int
bounce_bus_dmamap_destroy(bus_dma_tag_t dmat,bus_dmamap_t map)357 bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
358 {
359 if (dmat->bounce_flags & BF_COULD_BOUNCE) {
360 if (STAILQ_FIRST(&map->bpages) != NULL) {
361 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
362 __func__, dmat, EBUSY);
363 return (EBUSY);
364 }
365 if (dmat->bounce_zone)
366 dmat->bounce_zone->map_count--;
367 }
368 free(map->segments, M_DEVBUF);
369 free(map, M_DEVBUF);
370 dmat->map_count--;
371 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
372 return (0);
373 }
374
375 /*
376 * Allocate a piece of memory that can be efficiently mapped into
377 * bus device space based on the constraints lited in the dma tag.
378 * A dmamap to for use with dmamap_load is also allocated.
379 */
380 static int
bounce_bus_dmamem_alloc(bus_dma_tag_t dmat,void ** vaddr,int flags,bus_dmamap_t * mapp)381 bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
382 bus_dmamap_t *mapp)
383 {
384 vm_memattr_t attr;
385 int mflags;
386
387 if (flags & BUS_DMA_NOWAIT)
388 mflags = M_NOWAIT;
389 else
390 mflags = M_WAITOK;
391
392 bounce_bus_dmamap_create(dmat, mflags, mapp);
393 if (*mapp == NULL) {
394 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d\n",
395 __func__, dmat, dmat->common.flags, ENOMEM);
396 return (ENOMEM);
397 }
398
399 if (flags & BUS_DMA_ZERO)
400 mflags |= M_ZERO;
401 if (flags & BUS_DMA_NOCACHE)
402 attr = VM_MEMATTR_UNCACHEABLE;
403 else
404 attr = VM_MEMATTR_DEFAULT;
405
406 /*
407 * XXX:
408 * (dmat->common.alignment <= dmat->common.maxsize) is just a quick hack; the exact
409 * alignment guarantees of malloc need to be nailed down, and the
410 * code below should be rewritten to take that into account.
411 *
412 * In the meantime, we'll warn the user if malloc gets it wrong.
413 */
414 if ((dmat->common.maxsize <= PAGE_SIZE) &&
415 (dmat->common.alignment <= dmat->common.maxsize) &&
416 dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
417 attr == VM_MEMATTR_DEFAULT) {
418 *vaddr = malloc(dmat->common.maxsize, M_DEVBUF, mflags);
419 } else {
420 /*
421 * XXX Use Contigmalloc until it is merged into this facility
422 * and handles multi-seg allocations. Nobody is doing
423 * multi-seg allocations yet though.
424 * XXX Certain AGP hardware does.
425 */
426 *vaddr = kmem_alloc_contig(dmat->common.maxsize, mflags, 0ul,
427 dmat->common.lowaddr, dmat->common.alignment ? dmat->common.alignment : 1ul,
428 dmat->common.boundary, attr);
429 (*mapp)->contigalloc = 1;
430 }
431 if (*vaddr == NULL) {
432 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
433 __func__, dmat, dmat->common.flags, ENOMEM);
434 bounce_bus_dmamap_destroy(dmat, *mapp);
435 *mapp = NULL;
436 return (ENOMEM);
437 } else if (!vm_addr_align_ok(vtophys(*vaddr), dmat->common.alignment)) {
438 printf("bus_dmamem_alloc failed to align memory properly.\n");
439 }
440 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
441 __func__, dmat, dmat->common.flags, 0);
442 return (0);
443 }
444
445 /*
446 * Free a piece of memory and it's allociated dmamap, that was allocated
447 * via bus_dmamem_alloc.
448 */
449 static void
bounce_bus_dmamem_free(bus_dma_tag_t dmat,void * vaddr,bus_dmamap_t map)450 bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
451 {
452
453 if (!map->contigalloc)
454 free(vaddr, M_DEVBUF);
455 else
456 kmem_free(vaddr, dmat->common.maxsize);
457 bounce_bus_dmamap_destroy(dmat, map);
458 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat,
459 dmat->common.flags);
460 }
461
462 static void
_bus_dmamap_count_phys(bus_dma_tag_t dmat,bus_dmamap_t map,vm_paddr_t buf,bus_size_t buflen,int flags)463 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
464 bus_size_t buflen, int flags)
465 {
466 bus_addr_t curaddr;
467 bus_size_t sgsize;
468
469 if (map->pagesneeded == 0) {
470 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
471 "alignment= %d", dmat->common.lowaddr, ptoa((vm_paddr_t)Maxmem),
472 dmat->common.boundary, dmat->common.alignment);
473 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded);
474 /*
475 * Count the number of bounce pages
476 * needed in order to complete this transfer
477 */
478 curaddr = buf;
479 while (buflen != 0) {
480 sgsize = buflen;
481 if (must_bounce(dmat, curaddr)) {
482 sgsize = MIN(sgsize,
483 PAGE_SIZE - (curaddr & PAGE_MASK));
484 map->pagesneeded++;
485 }
486 curaddr += sgsize;
487 buflen -= sgsize;
488 }
489 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
490 }
491 }
492
493 static void
_bus_dmamap_count_pages(bus_dma_tag_t dmat,bus_dmamap_t map,pmap_t pmap,void * buf,bus_size_t buflen,int flags)494 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
495 void *buf, bus_size_t buflen, int flags)
496 {
497 vm_offset_t vaddr;
498 vm_offset_t vendaddr;
499 bus_addr_t paddr;
500
501 if (map->pagesneeded == 0) {
502 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
503 "alignment= %d", dmat->common.lowaddr, ptoa((vm_paddr_t)Maxmem),
504 dmat->common.boundary, dmat->common.alignment);
505 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded);
506 /*
507 * Count the number of bounce pages
508 * needed in order to complete this transfer
509 */
510 vaddr = (vm_offset_t)buf;
511 vendaddr = (vm_offset_t)buf + buflen;
512
513 while (vaddr < vendaddr) {
514 bus_size_t sg_len;
515
516 sg_len = MIN(vendaddr - vaddr,
517 PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
518 if (pmap == kernel_pmap)
519 paddr = pmap_kextract(vaddr);
520 else
521 paddr = pmap_extract(pmap, vaddr);
522 if (must_bounce(dmat, paddr)) {
523 sg_len = roundup2(sg_len, dmat->common.alignment);
524 map->pagesneeded++;
525 }
526 vaddr += sg_len;
527 }
528 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
529 }
530 }
531
532 /*
533 * Utility function to load a physical buffer. segp contains
534 * the starting segment on entrace, and the ending segment on exit.
535 */
536 static int
bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat,bus_dmamap_t map,vm_paddr_t buf,bus_size_t buflen,int flags,bus_dma_segment_t * segs,int * segp)537 bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat,
538 bus_dmamap_t map,
539 vm_paddr_t buf, bus_size_t buflen,
540 int flags,
541 bus_dma_segment_t *segs,
542 int *segp)
543 {
544 bus_addr_t curaddr;
545 bus_size_t sgsize;
546 int error;
547
548 if (segs == NULL)
549 segs = map->segments;
550
551 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) {
552 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
553 if (map->pagesneeded != 0) {
554 error = _bus_dmamap_reserve_pages(dmat, map, flags);
555 if (error)
556 return (error);
557 }
558 }
559
560 while (buflen > 0) {
561 curaddr = buf;
562 sgsize = buflen;
563 if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
564 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
565 curaddr = add_bounce_page(dmat, map, 0, curaddr,
566 sgsize);
567 }
568 if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
569 segp))
570 break;
571 buf += sgsize;
572 buflen -= sgsize;
573 }
574
575 /*
576 * Did we fit?
577 */
578 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
579 }
580
581 static int
bounce_bus_dmamap_load_ma(bus_dma_tag_t dmat,bus_dmamap_t map,struct vm_page ** ma,bus_size_t tlen,int ma_offs,int flags,bus_dma_segment_t * segs,int * segp)582 bounce_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map,
583 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
584 bus_dma_segment_t *segs, int *segp)
585 {
586
587 return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags,
588 segs, segp));
589 }
590
591 /*
592 * Utility function to load a linear buffer. segp contains
593 * the starting segment on entrance, and the ending segment on exit.
594 */
595 static int
bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat,bus_dmamap_t map,void * buf,bus_size_t buflen,pmap_t pmap,int flags,bus_dma_segment_t * segs,int * segp)596 bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
597 bus_dmamap_t map,
598 void *buf, bus_size_t buflen,
599 pmap_t pmap,
600 int flags,
601 bus_dma_segment_t *segs,
602 int *segp)
603 {
604 bus_size_t sgsize;
605 bus_addr_t curaddr;
606 char *kvaddr, *vaddr;
607 int error;
608
609 if (segs == NULL)
610 segs = map->segments;
611
612 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) {
613 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
614 if (map->pagesneeded != 0) {
615 error = _bus_dmamap_reserve_pages(dmat, map, flags);
616 if (error)
617 return (error);
618 }
619 }
620
621 vaddr = buf;
622
623 while (buflen > 0) {
624 /*
625 * Get the physical address for this segment.
626 */
627 if (pmap == kernel_pmap) {
628 curaddr = pmap_kextract((vm_offset_t)vaddr);
629 kvaddr = vaddr;
630 } else {
631 curaddr = pmap_extract(pmap, (vm_offset_t)vaddr);
632 kvaddr = NULL;
633 }
634
635 /*
636 * Compute the segment size, and adjust counts.
637 */
638 sgsize = MIN(buflen, PAGE_SIZE - (curaddr & PAGE_MASK));
639 if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
640 sgsize = roundup2(sgsize, dmat->common.alignment);
641 sgsize = MIN(sgsize, buflen);
642 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
643 sgsize);
644 }
645
646 if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
647 segp))
648 break;
649 vaddr += sgsize;
650 buflen -= MIN(sgsize, buflen); /* avoid underflow */
651 }
652
653 /*
654 * Did we fit?
655 */
656 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
657 }
658
659 static void
bounce_bus_dmamap_waitok(bus_dma_tag_t dmat,bus_dmamap_t map,struct memdesc * mem,bus_dmamap_callback_t * callback,void * callback_arg)660 bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
661 struct memdesc *mem, bus_dmamap_callback_t *callback,
662 void *callback_arg)
663 {
664
665 if (dmat->bounce_flags & BF_COULD_BOUNCE) {
666 map->dmat = dmat;
667 map->mem = *mem;
668 map->callback = callback;
669 map->callback_arg = callback_arg;
670 }
671 }
672
673 static bus_dma_segment_t *
bounce_bus_dmamap_complete(bus_dma_tag_t dmat,bus_dmamap_t map,bus_dma_segment_t * segs,int nsegs,int error)674 bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
675 bus_dma_segment_t *segs, int nsegs, int error)
676 {
677
678 map->nsegs = nsegs;
679 if (segs != NULL)
680 memcpy(map->segments, segs, map->nsegs*sizeof(segs[0]));
681 if (dmat->iommu != NULL)
682 IOMMU_MAP(dmat->iommu, map->segments, &map->nsegs,
683 dmat->common.lowaddr, dmat->common.highaddr, dmat->common.alignment,
684 dmat->common.boundary, dmat->iommu_cookie);
685
686 if (segs != NULL)
687 memcpy(segs, map->segments, map->nsegs*sizeof(segs[0]));
688 else
689 segs = map->segments;
690
691 return (segs);
692 }
693
694 /*
695 * Release the mapping held by map.
696 */
697 static void
bounce_bus_dmamap_unload(bus_dma_tag_t dmat,bus_dmamap_t map)698 bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
699 {
700 if (dmat->iommu) {
701 IOMMU_UNMAP(dmat->iommu, map->segments, map->nsegs, dmat->iommu_cookie);
702 map->nsegs = 0;
703 }
704
705 free_bounce_pages(dmat, map);
706 }
707
708 static void
bounce_bus_dmamap_sync(bus_dma_tag_t dmat,bus_dmamap_t map,bus_dmasync_op_t op)709 bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
710 {
711 struct bounce_page *bpage;
712 char *datavaddr, *tempvaddr;
713
714 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
715 /*
716 * Handle data bouncing. We might also
717 * want to add support for invalidating
718 * the caches on broken hardware
719 */
720 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
721 "performing bounce", __func__, dmat, dmat->common.flags, op);
722
723 if (op & BUS_DMASYNC_PREWRITE) {
724 while (bpage != NULL) {
725 tempvaddr = NULL;
726 datavaddr = bpage->datavaddr;
727 if (datavaddr == NULL) {
728 tempvaddr = pmap_quick_enter_page(
729 bpage->datapage);
730 datavaddr = tempvaddr +
731 bpage->dataoffs;
732 }
733
734 bcopy(datavaddr, bpage->vaddr, bpage->datacount);
735
736 if (tempvaddr != NULL)
737 pmap_quick_remove_page(tempvaddr);
738 bpage = STAILQ_NEXT(bpage, links);
739 }
740 dmat->bounce_zone->total_bounced++;
741 }
742
743 if (op & BUS_DMASYNC_POSTREAD) {
744 while (bpage != NULL) {
745 tempvaddr = NULL;
746 datavaddr = bpage->datavaddr;
747 if (datavaddr == NULL) {
748 tempvaddr = pmap_quick_enter_page(
749 bpage->datapage);
750 datavaddr = tempvaddr +
751 bpage->dataoffs;
752 }
753
754 bcopy(bpage->vaddr, datavaddr, bpage->datacount);
755
756 if (tempvaddr != NULL)
757 pmap_quick_remove_page(tempvaddr);
758 bpage = STAILQ_NEXT(bpage, links);
759 }
760 dmat->bounce_zone->total_bounced++;
761 }
762 }
763
764 powerpc_sync();
765 }
766
767 static int
bounce_bus_dma_tag_set_iommu(bus_dma_tag_t tag,device_t iommu,void * cookie)768 bounce_bus_dma_tag_set_iommu(bus_dma_tag_t tag, device_t iommu, void *cookie)
769 {
770 tag->iommu = iommu;
771 tag->iommu_cookie = cookie;
772
773 return (0);
774 }
775
776 struct bus_dma_impl bus_dma_bounce_impl = {
777 .tag_create = bounce_bus_dma_tag_create,
778 .tag_destroy = bounce_bus_dma_tag_destroy,
779 .tag_set_domain = bounce_bus_dma_tag_set_domain,
780 .map_create = bounce_bus_dmamap_create,
781 .map_destroy = bounce_bus_dmamap_destroy,
782 .mem_alloc = bounce_bus_dmamem_alloc,
783 .mem_free = bounce_bus_dmamem_free,
784 .load_ma = bounce_bus_dmamap_load_ma,
785 .load_phys = bounce_bus_dmamap_load_phys,
786 .load_buffer = bounce_bus_dmamap_load_buffer,
787 .map_waitok = bounce_bus_dmamap_waitok,
788 .map_complete = bounce_bus_dmamap_complete,
789 .map_unload = bounce_bus_dmamap_unload,
790 .map_sync = bounce_bus_dmamap_sync,
791 .set_iommu = bounce_bus_dma_tag_set_iommu,
792 };
793