xref: /freebsd/sys/dev/iommu/busdma_iommu.c (revision 29dc9349149657d6f00f1f5fc3ade589525e0d38)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2013 The FreeBSD Foundation
5  *
6  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7  * under sponsorship from the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/domainset.h>
34 #include <sys/malloc.h>
35 #include <sys/bus.h>
36 #include <sys/conf.h>
37 #include <sys/interrupt.h>
38 #include <sys/kernel.h>
39 #include <sys/ktr.h>
40 #include <sys/lock.h>
41 #include <sys/proc.h>
42 #include <sys/memdesc.h>
43 #include <sys/msan.h>
44 #include <sys/mutex.h>
45 #include <sys/sysctl.h>
46 #include <sys/rman.h>
47 #include <sys/taskqueue.h>
48 #include <sys/tree.h>
49 #include <sys/uio.h>
50 #include <sys/vmem.h>
51 #include <dev/pci/pcireg.h>
52 #include <dev/pci/pcivar.h>
53 #include <vm/vm.h>
54 #include <vm/vm_extern.h>
55 #include <vm/vm_kern.h>
56 #include <vm/vm_object.h>
57 #include <vm/vm_page.h>
58 #include <vm/vm_map.h>
59 #include <dev/iommu/iommu.h>
60 #include <machine/atomic.h>
61 #include <machine/bus.h>
62 #include <machine/md_var.h>
63 #include <machine/iommu.h>
64 #include <dev/iommu/busdma_iommu.h>
65 
66 /*
67  * busdma_iommu.c, the implementation of the busdma(9) interface using
68  * IOMMU units from Intel VT-d.
69  */
70 
71 static bool
72 iommu_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
73 {
74 	char str[128], *env;
75 	int default_bounce;
76 	bool ret;
77 	static const char bounce_str[] = "bounce";
78 	static const char iommu_str[] = "iommu";
79 	static const char dmar_str[] = "dmar"; /* compatibility */
80 
81 	default_bounce = 0;
82 	env = kern_getenv("hw.busdma.default");
83 	if (env != NULL) {
84 		if (strcmp(env, bounce_str) == 0)
85 			default_bounce = 1;
86 		else if (strcmp(env, iommu_str) == 0 ||
87 		    strcmp(env, dmar_str) == 0)
88 			default_bounce = 0;
89 		freeenv(env);
90 	}
91 
92 	snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d",
93 	    domain, bus, slot, func);
94 	env = kern_getenv(str);
95 	if (env == NULL)
96 		return (default_bounce != 0);
97 	if (strcmp(env, bounce_str) == 0)
98 		ret = true;
99 	else if (strcmp(env, iommu_str) == 0 ||
100 	    strcmp(env, dmar_str) == 0)
101 		ret = false;
102 	else
103 		ret = default_bounce != 0;
104 	freeenv(env);
105 	return (ret);
106 }
107 
108 /*
109  * Given original device, find the requester ID that will be seen by
110  * the IOMMU unit and used for page table lookup.  PCI bridges may take
111  * ownership of transactions from downstream devices, so it may not be
112  * the same as the BSF of the target device.  In those cases, all
113  * devices downstream of the bridge must share a single mapping
114  * domain, and must collectively be assigned to use either IOMMU or
115  * bounce mapping.
116  */
117 device_t
118 iommu_get_requester(device_t dev, uint16_t *rid)
119 {
120 	devclass_t pci_class;
121 	device_t l, pci, pcib, pcip, pcibp, requester;
122 	int cap_offset;
123 	uint16_t pcie_flags;
124 	bool bridge_is_pcie;
125 
126 	pci_class = devclass_find("pci");
127 	l = requester = dev;
128 
129 	*rid = pci_get_rid(dev);
130 
131 	/*
132 	 * Walk the bridge hierarchy from the target device to the
133 	 * host port to find the translating bridge nearest the IOMMU
134 	 * unit.
135 	 */
136 	for (;;) {
137 		pci = device_get_parent(l);
138 		KASSERT(pci != NULL, ("iommu_get_requester(%s): NULL parent "
139 		    "for %s", device_get_name(dev), device_get_name(l)));
140 		KASSERT(device_get_devclass(pci) == pci_class,
141 		    ("iommu_get_requester(%s): non-pci parent %s for %s",
142 		    device_get_name(dev), device_get_name(pci),
143 		    device_get_name(l)));
144 
145 		pcib = device_get_parent(pci);
146 		KASSERT(pcib != NULL, ("iommu_get_requester(%s): NULL bridge "
147 		    "for %s", device_get_name(dev), device_get_name(pci)));
148 
149 		/*
150 		 * The parent of our "bridge" isn't another PCI bus,
151 		 * so pcib isn't a PCI->PCI bridge but rather a host
152 		 * port, and the requester ID won't be translated
153 		 * further.
154 		 */
155 		pcip = device_get_parent(pcib);
156 		if (device_get_devclass(pcip) != pci_class)
157 			break;
158 		pcibp = device_get_parent(pcip);
159 
160 		if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) {
161 			/*
162 			 * Do not stop the loop even if the target
163 			 * device is PCIe, because it is possible (but
164 			 * unlikely) to have a PCI->PCIe bridge
165 			 * somewhere in the hierarchy.
166 			 */
167 			l = pcib;
168 		} else {
169 			/*
170 			 * Device is not PCIe, it cannot be seen as a
171 			 * requester by IOMMU unit.  Check whether the
172 			 * bridge is PCIe.
173 			 */
174 			bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,
175 			    &cap_offset) == 0;
176 			requester = pcib;
177 
178 			/*
179 			 * Check for a buggy PCIe/PCI bridge that
180 			 * doesn't report the express capability.  If
181 			 * the bridge above it is express but isn't a
182 			 * PCI bridge, then we know pcib is actually a
183 			 * PCIe/PCI bridge.
184 			 */
185 			if (!bridge_is_pcie && pci_find_cap(pcibp,
186 			    PCIY_EXPRESS, &cap_offset) == 0) {
187 				pcie_flags = pci_read_config(pcibp,
188 				    cap_offset + PCIER_FLAGS, 2);
189 				if ((pcie_flags & PCIEM_FLAGS_TYPE) !=
190 				    PCIEM_TYPE_PCI_BRIDGE)
191 					bridge_is_pcie = true;
192 			}
193 
194 			if (bridge_is_pcie) {
195 				/*
196 				 * The current device is not PCIe, but
197 				 * the bridge above it is.  This is a
198 				 * PCIe->PCI bridge.  Assume that the
199 				 * requester ID will be the secondary
200 				 * bus number with slot and function
201 				 * set to zero.
202 				 *
203 				 * XXX: Doesn't handle the case where
204 				 * the bridge is PCIe->PCI-X, and the
205 				 * bridge will only take ownership of
206 				 * requests in some cases.  We should
207 				 * provide context entries with the
208 				 * same page tables for taken and
209 				 * non-taken transactions.
210 				 */
211 				*rid = PCI_RID(pci_get_bus(l), 0, 0);
212 				l = pcibp;
213 			} else {
214 				/*
215 				 * Neither the device nor the bridge
216 				 * above it are PCIe.  This is a
217 				 * conventional PCI->PCI bridge, which
218 				 * will use the bridge's BSF as the
219 				 * requester ID.
220 				 */
221 				*rid = pci_get_rid(pcib);
222 				l = pcib;
223 			}
224 		}
225 	}
226 	return (requester);
227 }
228 
229 struct iommu_ctx *
230 iommu_instantiate_ctx(struct iommu_unit *unit, device_t dev, bool rmrr)
231 {
232 	device_t requester;
233 	struct iommu_ctx *ctx;
234 	bool disabled;
235 	uint16_t rid;
236 
237 	requester = iommu_get_requester(dev, &rid);
238 
239 	/*
240 	 * If the user requested the IOMMU disabled for the device, we
241 	 * cannot disable the IOMMU unit, due to possibility of other
242 	 * devices on the same IOMMU unit still requiring translation.
243 	 * Instead provide the identity mapping for the device
244 	 * context.
245 	 */
246 	disabled = iommu_bus_dma_is_dev_disabled(pci_get_domain(requester),
247 	    pci_get_bus(requester), pci_get_slot(requester),
248 	    pci_get_function(requester));
249 	ctx = iommu_get_ctx(unit, requester, rid, disabled, rmrr);
250 	if (ctx == NULL)
251 		return (NULL);
252 	if (disabled) {
253 		/*
254 		 * Keep the first reference on context, release the
255 		 * later refs.
256 		 */
257 		IOMMU_LOCK(unit);
258 		if ((ctx->flags & IOMMU_CTX_DISABLED) == 0) {
259 			ctx->flags |= IOMMU_CTX_DISABLED;
260 			IOMMU_UNLOCK(unit);
261 		} else {
262 			iommu_free_ctx_locked(unit, ctx);
263 		}
264 		ctx = NULL;
265 	}
266 	return (ctx);
267 }
268 
269 struct iommu_ctx *
270 iommu_get_dev_ctx(device_t dev)
271 {
272 	struct iommu_unit *unit;
273 
274 	unit = iommu_find(dev, bootverbose);
275 	/* Not in scope of any IOMMU ? */
276 	if (unit == NULL)
277 		return (NULL);
278 	if (!unit->dma_enabled)
279 		return (NULL);
280 
281 #if defined(__amd64__) || defined(__i386__)
282 	dmar_quirks_pre_use(unit);
283 	dmar_instantiate_rmrr_ctxs(unit);
284 #endif
285 
286 	return (iommu_instantiate_ctx(unit, dev, false));
287 }
288 
289 bus_dma_tag_t
290 iommu_get_dma_tag(device_t dev, device_t child)
291 {
292 	struct iommu_ctx *ctx;
293 	bus_dma_tag_t res;
294 
295 	ctx = iommu_get_dev_ctx(child);
296 	if (ctx == NULL)
297 		return (NULL);
298 
299 	res = (bus_dma_tag_t)ctx->tag;
300 	return (res);
301 }
302 
303 bool
304 bus_dma_iommu_set_buswide(device_t dev)
305 {
306 	struct iommu_unit *unit;
307 	device_t parent;
308 	u_int busno, slot, func;
309 
310 	parent = device_get_parent(dev);
311 	if (device_get_devclass(parent) != devclass_find("pci"))
312 		return (false);
313 	unit = iommu_find(dev, bootverbose);
314 	if (unit == NULL)
315 		return (false);
316 	busno = pci_get_bus(dev);
317 	slot = pci_get_slot(dev);
318 	func = pci_get_function(dev);
319 	if (slot != 0 || func != 0) {
320 		if (bootverbose) {
321 			device_printf(dev,
322 			    "iommu%d pci%d:%d:%d requested buswide busdma\n",
323 			    unit->unit, busno, slot, func);
324 		}
325 		return (false);
326 	}
327 	iommu_set_buswide_ctx(unit, busno);
328 	return (true);
329 }
330 
331 void
332 iommu_set_buswide_ctx(struct iommu_unit *unit, u_int busno)
333 {
334 
335 	MPASS(busno <= PCI_BUSMAX);
336 	IOMMU_LOCK(unit);
337 	unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] |=
338 	    1 << (busno % (NBBY * sizeof(uint32_t)));
339 	IOMMU_UNLOCK(unit);
340 }
341 
342 bool
343 iommu_is_buswide_ctx(struct iommu_unit *unit, u_int busno)
344 {
345 
346 	MPASS(busno <= PCI_BUSMAX);
347 	return ((unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] &
348 	    (1U << (busno % (NBBY * sizeof(uint32_t))))) != 0);
349 }
350 
351 static MALLOC_DEFINE(M_IOMMU_DMAMAP, "iommu_dmamap", "IOMMU DMA Map");
352 
353 static void iommu_bus_schedule_dmamap(struct iommu_unit *unit,
354     struct bus_dmamap_iommu *map);
355 
356 static int
357 iommu_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
358     bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
359     bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags,
360     bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat)
361 {
362 	struct bus_dma_tag_iommu *newtag, *oldtag;
363 	int error;
364 
365 	*dmat = NULL;
366 	error = common_bus_dma_tag_create(parent != NULL ?
367 	    &((struct bus_dma_tag_iommu *)parent)->common : NULL, alignment,
368 	    boundary, lowaddr, highaddr, maxsize, nsegments, maxsegsz, flags,
369 	    lockfunc, lockfuncarg, sizeof(struct bus_dma_tag_iommu),
370 	    (void **)&newtag);
371 	if (error != 0)
372 		goto out;
373 
374 	oldtag = (struct bus_dma_tag_iommu *)parent;
375 	newtag->common.impl = &bus_dma_iommu_impl;
376 	newtag->ctx = oldtag->ctx;
377 	newtag->owner = oldtag->owner;
378 
379 	*dmat = (bus_dma_tag_t)newtag;
380 out:
381 	CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
382 	    __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
383 	    error);
384 	return (error);
385 }
386 
387 static int
388 iommu_bus_dma_tag_set_domain(bus_dma_tag_t dmat)
389 {
390 
391 	return (0);
392 }
393 
394 static int
395 iommu_bus_dma_tag_destroy(bus_dma_tag_t dmat1)
396 {
397 	struct bus_dma_tag_iommu *dmat;
398 	int error;
399 
400 	error = 0;
401 	dmat = (struct bus_dma_tag_iommu *)dmat1;
402 
403 	if (dmat != NULL) {
404 		if (dmat->map_count != 0) {
405 			error = EBUSY;
406 			goto out;
407 		}
408 		if (dmat == dmat->ctx->tag)
409 			iommu_free_ctx(dmat->ctx);
410 		free(dmat->segments, M_IOMMU_DMAMAP);
411 		free(dmat, M_DEVBUF);
412 	}
413 out:
414 	CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat, error);
415 	return (error);
416 }
417 
418 static bool
419 iommu_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen)
420 {
421 
422 	return (false);
423 }
424 
425 static int
426 iommu_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
427 {
428 	struct bus_dma_tag_iommu *tag;
429 	struct bus_dmamap_iommu *map;
430 
431 	tag = (struct bus_dma_tag_iommu *)dmat;
432 	map = malloc_domainset(sizeof(*map), M_IOMMU_DMAMAP,
433 	    DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO);
434 	if (map == NULL) {
435 		*mapp = NULL;
436 		return (ENOMEM);
437 	}
438 	if (tag->segments == NULL) {
439 		tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) *
440 		    tag->common.nsegments, M_IOMMU_DMAMAP,
441 		    DOMAINSET_PREF(tag->common.domain), M_NOWAIT);
442 		if (tag->segments == NULL) {
443 			free(map, M_IOMMU_DMAMAP);
444 			*mapp = NULL;
445 			return (ENOMEM);
446 		}
447 	}
448 	IOMMU_DMAMAP_INIT(map);
449 	TAILQ_INIT(&map->map_entries);
450 	map->tag = tag;
451 	map->locked = true;
452 	map->cansleep = false;
453 	tag->map_count++;
454 	*mapp = (bus_dmamap_t)map;
455 
456 	return (0);
457 }
458 
459 static int
460 iommu_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1)
461 {
462 	struct bus_dma_tag_iommu *tag;
463 	struct bus_dmamap_iommu *map;
464 
465 	tag = (struct bus_dma_tag_iommu *)dmat;
466 	map = (struct bus_dmamap_iommu *)map1;
467 	if (map != NULL) {
468 		IOMMU_DMAMAP_LOCK(map);
469 		if (!TAILQ_EMPTY(&map->map_entries)) {
470 			IOMMU_DMAMAP_UNLOCK(map);
471 			return (EBUSY);
472 		}
473 		IOMMU_DMAMAP_DESTROY(map);
474 		free(map, M_IOMMU_DMAMAP);
475 	}
476 	tag->map_count--;
477 	return (0);
478 }
479 
480 
481 static int
482 iommu_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
483     bus_dmamap_t *mapp)
484 {
485 	struct bus_dma_tag_iommu *tag;
486 	struct bus_dmamap_iommu *map;
487 	int error, mflags;
488 	vm_memattr_t attr;
489 
490 	error = iommu_bus_dmamap_create(dmat, flags, mapp);
491 	if (error != 0)
492 		return (error);
493 
494 	mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK;
495 	mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0;
496 	attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE :
497 	    VM_MEMATTR_DEFAULT;
498 
499 	tag = (struct bus_dma_tag_iommu *)dmat;
500 	map = (struct bus_dmamap_iommu *)*mapp;
501 
502 	if (tag->common.maxsize < PAGE_SIZE &&
503 	    tag->common.alignment <= tag->common.maxsize &&
504 	    attr == VM_MEMATTR_DEFAULT) {
505 		*vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF,
506 		    DOMAINSET_PREF(tag->common.domain), mflags);
507 		map->flags |= BUS_DMAMAP_IOMMU_MALLOC;
508 	} else {
509 		*vaddr = kmem_alloc_attr_domainset(
510 		    DOMAINSET_PREF(tag->common.domain), tag->common.maxsize,
511 		    mflags, 0ul, BUS_SPACE_MAXADDR, attr);
512 		map->flags |= BUS_DMAMAP_IOMMU_KMEM_ALLOC;
513 	}
514 	if (*vaddr == NULL) {
515 		iommu_bus_dmamap_destroy(dmat, *mapp);
516 		*mapp = NULL;
517 		return (ENOMEM);
518 	}
519 	return (0);
520 }
521 
522 static void
523 iommu_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1)
524 {
525 	struct bus_dma_tag_iommu *tag;
526 	struct bus_dmamap_iommu *map;
527 
528 	tag = (struct bus_dma_tag_iommu *)dmat;
529 	map = (struct bus_dmamap_iommu *)map1;
530 
531 	if ((map->flags & BUS_DMAMAP_IOMMU_MALLOC) != 0) {
532 		free(vaddr, M_DEVBUF);
533 		map->flags &= ~BUS_DMAMAP_IOMMU_MALLOC;
534 	} else {
535 		KASSERT((map->flags & BUS_DMAMAP_IOMMU_KMEM_ALLOC) != 0,
536 		    ("iommu_bus_dmamem_free for non alloced map %p", map));
537 		kmem_free(vaddr, tag->common.maxsize);
538 		map->flags &= ~BUS_DMAMAP_IOMMU_KMEM_ALLOC;
539 	}
540 
541 	iommu_bus_dmamap_destroy(dmat, map1);
542 }
543 
544 static int
545 iommu_bus_dmamap_load_something1(struct bus_dma_tag_iommu *tag,
546     struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen,
547     int flags, bus_dma_segment_t *segs, int *segp,
548     struct iommu_map_entries_tailq *entries)
549 {
550 	struct iommu_ctx *ctx;
551 	struct iommu_domain *domain;
552 	struct iommu_map_entry *entry;
553 	bus_size_t buflen1;
554 	int error, e_flags, idx, gas_flags, seg;
555 
556 	KASSERT(offset < IOMMU_PAGE_SIZE, ("offset %d", offset));
557 	if (segs == NULL)
558 		segs = tag->segments;
559 	ctx = tag->ctx;
560 	domain = ctx->domain;
561 	e_flags = IOMMU_MAP_ENTRY_READ |
562 	    ((flags & BUS_DMA_NOWRITE) == 0 ? IOMMU_MAP_ENTRY_WRITE : 0);
563 	seg = *segp;
564 	error = 0;
565 	idx = 0;
566 	while (buflen > 0) {
567 		seg++;
568 		if (seg >= tag->common.nsegments) {
569 			error = EFBIG;
570 			break;
571 		}
572 		buflen1 = buflen > tag->common.maxsegsz ?
573 		    tag->common.maxsegsz : buflen;
574 
575 		/*
576 		 * (Too) optimistically allow split if there are more
577 		 * then one segments left.
578 		 */
579 		gas_flags = map->cansleep ? IOMMU_MF_CANWAIT : 0;
580 		if (seg + 1 < tag->common.nsegments)
581 			gas_flags |= IOMMU_MF_CANSPLIT;
582 
583 		error = iommu_gas_map(domain, &tag->common, buflen1,
584 		    offset, e_flags, gas_flags, ma + idx, &entry);
585 		if (error != 0)
586 			break;
587 		/* Update buflen1 in case buffer split. */
588 		if (buflen1 > entry->end - entry->start - offset)
589 			buflen1 = entry->end - entry->start - offset;
590 
591 		KASSERT(vm_addr_align_ok(entry->start + offset,
592 		    tag->common.alignment),
593 		    ("alignment failed: ctx %p start 0x%jx offset %x "
594 		    "align 0x%jx", ctx, (uintmax_t)entry->start, offset,
595 		    (uintmax_t)tag->common.alignment));
596 		KASSERT(entry->end <= tag->common.lowaddr ||
597 		    entry->start >= tag->common.highaddr,
598 		    ("entry placement failed: ctx %p start 0x%jx end 0x%jx "
599 		    "lowaddr 0x%jx highaddr 0x%jx", ctx,
600 		    (uintmax_t)entry->start, (uintmax_t)entry->end,
601 		    (uintmax_t)tag->common.lowaddr,
602 		    (uintmax_t)tag->common.highaddr));
603 		KASSERT(vm_addr_bound_ok(entry->start + offset, buflen1,
604 		    tag->common.boundary),
605 		    ("boundary failed: ctx %p start 0x%jx end 0x%jx "
606 		    "boundary 0x%jx", ctx, (uintmax_t)entry->start,
607 		    (uintmax_t)entry->end, (uintmax_t)tag->common.boundary));
608 		KASSERT(buflen1 <= tag->common.maxsegsz,
609 		    ("segment too large: ctx %p start 0x%jx end 0x%jx "
610 		    "buflen1 0x%jx maxsegsz 0x%jx", ctx,
611 		    (uintmax_t)entry->start, (uintmax_t)entry->end,
612 		    (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz));
613 
614 		KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
615 		    ("entry %p missing IOMMU_MAP_ENTRY_MAP", entry));
616 		TAILQ_INSERT_TAIL(entries, entry, dmamap_link);
617 
618 		segs[seg].ds_addr = entry->start + offset;
619 		segs[seg].ds_len = buflen1;
620 
621 		idx += OFF_TO_IDX(offset + buflen1);
622 		offset += buflen1;
623 		offset &= IOMMU_PAGE_MASK;
624 		buflen -= buflen1;
625 	}
626 	if (error == 0)
627 		*segp = seg;
628 	return (error);
629 }
630 
631 static int
632 iommu_bus_dmamap_load_something(struct bus_dma_tag_iommu *tag,
633     struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen,
634     int flags, bus_dma_segment_t *segs, int *segp)
635 {
636 	struct iommu_ctx *ctx;
637 	struct iommu_domain *domain;
638 	struct iommu_map_entries_tailq entries;
639 	int error;
640 
641 	ctx = tag->ctx;
642 	domain = ctx->domain;
643 	atomic_add_long(&ctx->loads, 1);
644 
645 	TAILQ_INIT(&entries);
646 	error = iommu_bus_dmamap_load_something1(tag, map, ma, offset,
647 	    buflen, flags, segs, segp, &entries);
648 	if (error == 0) {
649 		IOMMU_DMAMAP_LOCK(map);
650 		TAILQ_CONCAT(&map->map_entries, &entries, dmamap_link);
651 		IOMMU_DMAMAP_UNLOCK(map);
652 	} else if (!TAILQ_EMPTY(&entries)) {
653 		/*
654 		 * The busdma interface does not allow us to report
655 		 * partial buffer load, so unfortunately we have to
656 		 * revert all work done.
657 		 */
658 		IOMMU_DOMAIN_LOCK(domain);
659 		TAILQ_CONCAT(&domain->unload_entries, &entries, dmamap_link);
660 		IOMMU_DOMAIN_UNLOCK(domain);
661 		taskqueue_enqueue(domain->iommu->delayed_taskqueue,
662 		    &domain->unload_task);
663 	}
664 
665 	if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 &&
666 	    !map->cansleep)
667 		error = EINPROGRESS;
668 	if (error == EINPROGRESS)
669 		iommu_bus_schedule_dmamap(domain->iommu, map);
670 	return (error);
671 }
672 
673 static int
674 iommu_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1,
675     struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
676     bus_dma_segment_t *segs, int *segp)
677 {
678 	struct bus_dma_tag_iommu *tag;
679 	struct bus_dmamap_iommu *map;
680 
681 	tag = (struct bus_dma_tag_iommu *)dmat;
682 	map = (struct bus_dmamap_iommu *)map1;
683 	return (iommu_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen,
684 	    flags, segs, segp));
685 }
686 
687 static int
688 iommu_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1,
689     vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
690     int *segp)
691 {
692 	struct bus_dma_tag_iommu *tag;
693 	struct bus_dmamap_iommu *map;
694 	vm_page_t *ma, fma;
695 	vm_paddr_t pstart, pend, paddr;
696 	int error, i, ma_cnt, mflags, offset;
697 
698 	tag = (struct bus_dma_tag_iommu *)dmat;
699 	map = (struct bus_dmamap_iommu *)map1;
700 	pstart = trunc_page(buf);
701 	pend = round_page(buf + buflen);
702 	offset = buf & PAGE_MASK;
703 	ma_cnt = OFF_TO_IDX(pend - pstart);
704 	mflags = map->cansleep ? M_WAITOK : M_NOWAIT;
705 	ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags);
706 	if (ma == NULL)
707 		return (ENOMEM);
708 	fma = NULL;
709 	for (i = 0; i < ma_cnt; i++) {
710 		paddr = pstart + ptoa(i);
711 		ma[i] = PHYS_TO_VM_PAGE(paddr);
712 		if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) {
713 			/*
714 			 * If PHYS_TO_VM_PAGE() returned NULL or the
715 			 * vm_page was not initialized we'll use a
716 			 * fake page.
717 			 */
718 			if (fma == NULL) {
719 				fma = malloc(sizeof(struct vm_page) * ma_cnt,
720 				    M_DEVBUF, M_ZERO | mflags);
721 				if (fma == NULL) {
722 					free(ma, M_DEVBUF);
723 					return (ENOMEM);
724 				}
725 			}
726 			vm_page_initfake(&fma[i], pstart + ptoa(i),
727 			    VM_MEMATTR_DEFAULT);
728 			ma[i] = &fma[i];
729 		}
730 	}
731 	error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen,
732 	    flags, segs, segp);
733 	free(fma, M_DEVBUF);
734 	free(ma, M_DEVBUF);
735 	return (error);
736 }
737 
738 static int
739 iommu_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf,
740     bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
741     int *segp)
742 {
743 	struct bus_dma_tag_iommu *tag;
744 	struct bus_dmamap_iommu *map;
745 	vm_page_t *ma, fma;
746 	vm_paddr_t pstart, pend, paddr;
747 	int error, i, ma_cnt, mflags, offset;
748 
749 	tag = (struct bus_dma_tag_iommu *)dmat;
750 	map = (struct bus_dmamap_iommu *)map1;
751 	pstart = trunc_page((vm_offset_t)buf);
752 	pend = round_page((vm_offset_t)buf + buflen);
753 	offset = (vm_offset_t)buf & PAGE_MASK;
754 	ma_cnt = OFF_TO_IDX(pend - pstart);
755 	mflags = map->cansleep ? M_WAITOK : M_NOWAIT;
756 	ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags);
757 	if (ma == NULL)
758 		return (ENOMEM);
759 	fma = NULL;
760 	for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
761 		if (pmap == kernel_pmap)
762 			paddr = pmap_kextract(pstart);
763 		else
764 			paddr = pmap_extract(pmap, pstart);
765 		ma[i] = PHYS_TO_VM_PAGE(paddr);
766 		if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) {
767 			/*
768 			 * If PHYS_TO_VM_PAGE() returned NULL or the
769 			 * vm_page was not initialized we'll use a
770 			 * fake page.
771 			 */
772 			if (fma == NULL) {
773 				fma = malloc(sizeof(struct vm_page) * ma_cnt,
774 				    M_DEVBUF, M_ZERO | mflags);
775 				if (fma == NULL) {
776 					free(ma, M_DEVBUF);
777 					return (ENOMEM);
778 				}
779 			}
780 			vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT);
781 			ma[i] = &fma[i];
782 		}
783 	}
784 	error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen,
785 	    flags, segs, segp);
786 	free(ma, M_DEVBUF);
787 	free(fma, M_DEVBUF);
788 	return (error);
789 }
790 
791 static void
792 iommu_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1,
793     struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
794 {
795 	struct bus_dmamap_iommu *map;
796 
797 	if (map1 == NULL)
798 		return;
799 	map = (struct bus_dmamap_iommu *)map1;
800 	map->mem = *mem;
801 	map->tag = (struct bus_dma_tag_iommu *)dmat;
802 	map->callback = callback;
803 	map->callback_arg = callback_arg;
804 }
805 
806 static bus_dma_segment_t *
807 iommu_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
808     bus_dma_segment_t *segs, int nsegs, int error)
809 {
810 	struct bus_dma_tag_iommu *tag;
811 	struct bus_dmamap_iommu *map;
812 
813 	tag = (struct bus_dma_tag_iommu *)dmat;
814 	map = (struct bus_dmamap_iommu *)map1;
815 
816 	if (!map->locked) {
817 		KASSERT(map->cansleep,
818 		    ("map not locked and not sleepable context %p", map));
819 
820 		/*
821 		 * We are called from the delayed context.  Relock the
822 		 * driver.
823 		 */
824 		(tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK);
825 		map->locked = true;
826 	}
827 
828 	if (segs == NULL)
829 		segs = tag->segments;
830 	return (segs);
831 }
832 
833 /*
834  * The limitations of busdma KPI forces the iommu to perform the actual
835  * unload, consisting of the unmapping of the map entries page tables,
836  * from the delayed context on i386, since page table page mapping
837  * might require a sleep to be successfull.  The unfortunate
838  * consequence is that the DMA requests can be served some time after
839  * the bus_dmamap_unload() call returned.
840  *
841  * On amd64, we assume that sf allocation cannot fail.
842  */
843 static void
844 iommu_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1)
845 {
846 	struct bus_dma_tag_iommu *tag;
847 	struct bus_dmamap_iommu *map;
848 	struct iommu_ctx *ctx;
849 	struct iommu_domain *domain;
850 	struct iommu_map_entries_tailq entries;
851 
852 	tag = (struct bus_dma_tag_iommu *)dmat;
853 	map = (struct bus_dmamap_iommu *)map1;
854 	ctx = tag->ctx;
855 	domain = ctx->domain;
856 	atomic_add_long(&ctx->unloads, 1);
857 
858 	TAILQ_INIT(&entries);
859 	IOMMU_DMAMAP_LOCK(map);
860 	TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link);
861 	IOMMU_DMAMAP_UNLOCK(map);
862 #if defined(IOMMU_DOMAIN_UNLOAD_SLEEP)
863 	IOMMU_DOMAIN_LOCK(domain);
864 	TAILQ_CONCAT(&domain->unload_entries, &entries, dmamap_link);
865 	IOMMU_DOMAIN_UNLOCK(domain);
866 	taskqueue_enqueue(domain->iommu->delayed_taskqueue,
867 	    &domain->unload_task);
868 #else
869 	THREAD_NO_SLEEPING();
870 	iommu_domain_unload(domain, &entries, false);
871 	THREAD_SLEEPING_OK();
872 	KASSERT(TAILQ_EMPTY(&entries), ("lazy iommu_ctx_unload %p", ctx));
873 #endif
874 }
875 
876 static void
877 iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map1,
878     bus_dmasync_op_t op)
879 {
880 	struct bus_dmamap_iommu *map __unused;
881 
882 	map = (struct bus_dmamap_iommu *)map1;
883 	kmsan_bus_dmamap_sync(&map->kmsan_mem, op);
884 }
885 
886 #ifdef KMSAN
887 static void
888 iommu_bus_dmamap_load_kmsan(bus_dmamap_t map1, struct memdesc *mem)
889 {
890 	struct bus_dmamap_iommu *map;
891 
892 	map = (struct bus_dmamap_iommu *)map1;
893 	if (map == NULL)
894 		return;
895 	memcpy(&map->kmsan_mem, mem, sizeof(struct memdesc));
896 }
897 #endif
898 
899 struct bus_dma_impl bus_dma_iommu_impl = {
900 	.tag_create = iommu_bus_dma_tag_create,
901 	.tag_destroy = iommu_bus_dma_tag_destroy,
902 	.tag_set_domain = iommu_bus_dma_tag_set_domain,
903 	.id_mapped = iommu_bus_dma_id_mapped,
904 	.map_create = iommu_bus_dmamap_create,
905 	.map_destroy = iommu_bus_dmamap_destroy,
906 	.mem_alloc = iommu_bus_dmamem_alloc,
907 	.mem_free = iommu_bus_dmamem_free,
908 	.load_phys = iommu_bus_dmamap_load_phys,
909 	.load_buffer = iommu_bus_dmamap_load_buffer,
910 	.load_ma = iommu_bus_dmamap_load_ma,
911 	.map_waitok = iommu_bus_dmamap_waitok,
912 	.map_complete = iommu_bus_dmamap_complete,
913 	.map_unload = iommu_bus_dmamap_unload,
914 	.map_sync = iommu_bus_dmamap_sync,
915 #ifdef KMSAN
916 	.load_kmsan = iommu_bus_dmamap_load_kmsan,
917 #endif
918 };
919 
920 static void
921 iommu_bus_task_dmamap(void *arg, int pending)
922 {
923 	struct bus_dma_tag_iommu *tag;
924 	struct bus_dmamap_iommu *map;
925 	struct iommu_unit *unit;
926 
927 	unit = arg;
928 	IOMMU_LOCK(unit);
929 	while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) {
930 		TAILQ_REMOVE(&unit->delayed_maps, map, delay_link);
931 		IOMMU_UNLOCK(unit);
932 		tag = map->tag;
933 		map->cansleep = true;
934 		map->locked = false;
935 		bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map,
936 		    &map->mem, map->callback, map->callback_arg,
937 		    BUS_DMA_WAITOK);
938 		map->cansleep = false;
939 		if (map->locked) {
940 			(tag->common.lockfunc)(tag->common.lockfuncarg,
941 			    BUS_DMA_UNLOCK);
942 		} else
943 			map->locked = true;
944 		map->cansleep = false;
945 		IOMMU_LOCK(unit);
946 	}
947 	IOMMU_UNLOCK(unit);
948 }
949 
950 static void
951 iommu_bus_schedule_dmamap(struct iommu_unit *unit, struct bus_dmamap_iommu *map)
952 {
953 
954 	map->locked = false;
955 	IOMMU_LOCK(unit);
956 	TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link);
957 	IOMMU_UNLOCK(unit);
958 	taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task);
959 }
960 
961 int
962 iommu_init_busdma(struct iommu_unit *unit)
963 {
964 	int error;
965 
966 	unit->dma_enabled = 0;
967 	error = TUNABLE_INT_FETCH("hw.iommu.dma", &unit->dma_enabled);
968 	if (error == 0) /* compatibility */
969 		TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled);
970 	TAILQ_INIT(&unit->delayed_maps);
971 	TASK_INIT(&unit->dmamap_load_task, 0, iommu_bus_task_dmamap, unit);
972 	unit->delayed_taskqueue = taskqueue_create("iommu", M_WAITOK,
973 	    taskqueue_thread_enqueue, &unit->delayed_taskqueue);
974 	taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK,
975 	    "iommu%d busdma taskq", unit->unit);
976 	return (0);
977 }
978 
979 void
980 iommu_fini_busdma(struct iommu_unit *unit)
981 {
982 
983 	if (unit->delayed_taskqueue == NULL)
984 		return;
985 
986 	taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task);
987 	taskqueue_free(unit->delayed_taskqueue);
988 	unit->delayed_taskqueue = NULL;
989 }
990 
991 int
992 bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map1,
993     vm_paddr_t start, vm_size_t length, int flags)
994 {
995 	struct bus_dma_tag_common *tc;
996 	struct bus_dma_tag_iommu *tag;
997 	struct bus_dmamap_iommu *map;
998 	struct iommu_ctx *ctx;
999 	struct iommu_domain *domain;
1000 	struct iommu_map_entry *entry;
1001 	vm_page_t *ma;
1002 	vm_size_t i;
1003 	int error;
1004 	bool waitok;
1005 
1006 	MPASS((start & PAGE_MASK) == 0);
1007 	MPASS((length & PAGE_MASK) == 0);
1008 	MPASS(length > 0);
1009 	MPASS(start + length >= start);
1010 	MPASS((flags & ~(BUS_DMA_NOWAIT | BUS_DMA_NOWRITE)) == 0);
1011 
1012 	tc = (struct bus_dma_tag_common *)dmat;
1013 	if (tc->impl != &bus_dma_iommu_impl)
1014 		return (0);
1015 
1016 	tag = (struct bus_dma_tag_iommu *)dmat;
1017 	ctx = tag->ctx;
1018 	domain = ctx->domain;
1019 	map = (struct bus_dmamap_iommu *)map1;
1020 	waitok = (flags & BUS_DMA_NOWAIT) != 0;
1021 
1022 	entry = iommu_gas_alloc_entry(domain, waitok ? 0 : IOMMU_PGF_WAITOK);
1023 	if (entry == NULL)
1024 		return (ENOMEM);
1025 	entry->start = start;
1026 	entry->end = start + length;
1027 	ma = malloc(sizeof(vm_page_t) * atop(length), M_TEMP, waitok ?
1028 	    M_WAITOK : M_NOWAIT);
1029 	if (ma == NULL) {
1030 		iommu_gas_free_entry(entry);
1031 		return (ENOMEM);
1032 	}
1033 	for (i = 0; i < atop(length); i++) {
1034 		ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
1035 		    VM_MEMATTR_DEFAULT);
1036 	}
1037 	error = iommu_gas_map_region(domain, entry, IOMMU_MAP_ENTRY_READ |
1038 	    ((flags & BUS_DMA_NOWRITE) ? 0 : IOMMU_MAP_ENTRY_WRITE) |
1039 	    IOMMU_MAP_ENTRY_MAP, waitok ? IOMMU_MF_CANWAIT : 0, ma);
1040 	if (error == 0) {
1041 		IOMMU_DMAMAP_LOCK(map);
1042 		TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
1043 		IOMMU_DMAMAP_UNLOCK(map);
1044 	} else {
1045 		iommu_gas_free_entry(entry);
1046 	}
1047 	for (i = 0; i < atop(length); i++)
1048 		vm_page_putfake(ma[i]);
1049 	free(ma, M_TEMP);
1050 	return (error);
1051 }
1052 
1053 static void
1054 iommu_domain_unload_task(void *arg, int pending)
1055 {
1056 	struct iommu_domain *domain;
1057 	struct iommu_map_entries_tailq entries;
1058 
1059 	domain = arg;
1060 	TAILQ_INIT(&entries);
1061 
1062 	for (;;) {
1063 		IOMMU_DOMAIN_LOCK(domain);
1064 		TAILQ_SWAP(&domain->unload_entries, &entries,
1065 		    iommu_map_entry, dmamap_link);
1066 		IOMMU_DOMAIN_UNLOCK(domain);
1067 		if (TAILQ_EMPTY(&entries))
1068 			break;
1069 		iommu_domain_unload(domain, &entries, true);
1070 	}
1071 }
1072 
1073 void
1074 iommu_domain_init(struct iommu_unit *unit, struct iommu_domain *domain,
1075     const struct iommu_domain_map_ops *ops)
1076 {
1077 
1078 	domain->ops = ops;
1079 	domain->iommu = unit;
1080 
1081 	TASK_INIT(&domain->unload_task, 0, iommu_domain_unload_task, domain);
1082 	RB_INIT(&domain->rb_root);
1083 	TAILQ_INIT(&domain->unload_entries);
1084 	mtx_init(&domain->lock, "iodom", NULL, MTX_DEF);
1085 }
1086 
1087 void
1088 iommu_domain_fini(struct iommu_domain *domain)
1089 {
1090 
1091 	mtx_destroy(&domain->lock);
1092 }
1093