1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2013 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/domainset.h>
34 #include <sys/malloc.h>
35 #include <sys/bus.h>
36 #include <sys/conf.h>
37 #include <sys/interrupt.h>
38 #include <sys/kernel.h>
39 #include <sys/ktr.h>
40 #include <sys/lock.h>
41 #include <sys/proc.h>
42 #include <sys/memdesc.h>
43 #include <sys/msan.h>
44 #include <sys/mutex.h>
45 #include <sys/sysctl.h>
46 #include <sys/rman.h>
47 #include <sys/taskqueue.h>
48 #include <sys/tree.h>
49 #include <sys/uio.h>
50 #include <sys/vmem.h>
51 #include <dev/pci/pcireg.h>
52 #include <dev/pci/pcivar.h>
53 #include <vm/vm.h>
54 #include <vm/vm_extern.h>
55 #include <vm/vm_kern.h>
56 #include <vm/vm_object.h>
57 #include <vm/vm_page.h>
58 #include <vm/vm_map.h>
59 #include <dev/iommu/iommu.h>
60 #include <machine/atomic.h>
61 #include <machine/bus.h>
62 #include <machine/md_var.h>
63 #include <machine/iommu.h>
64 #include <dev/iommu/busdma_iommu.h>
65
66 /*
67 * busdma_iommu.c, the implementation of the busdma(9) interface using
68 * IOMMU units from Intel VT-d.
69 */
70
71 static bool
iommu_bus_dma_is_dev_disabled(int domain,int bus,int slot,int func)72 iommu_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
73 {
74 char str[128], *env;
75 int default_bounce;
76 bool ret;
77 static const char bounce_str[] = "bounce";
78 static const char iommu_str[] = "iommu";
79 static const char dmar_str[] = "dmar"; /* compatibility */
80
81 default_bounce = 0;
82 env = kern_getenv("hw.busdma.default");
83 if (env != NULL) {
84 if (strcmp(env, bounce_str) == 0)
85 default_bounce = 1;
86 else if (strcmp(env, iommu_str) == 0 ||
87 strcmp(env, dmar_str) == 0)
88 default_bounce = 0;
89 freeenv(env);
90 }
91
92 snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d",
93 domain, bus, slot, func);
94 env = kern_getenv(str);
95 if (env == NULL)
96 return (default_bounce != 0);
97 if (strcmp(env, bounce_str) == 0)
98 ret = true;
99 else if (strcmp(env, iommu_str) == 0 ||
100 strcmp(env, dmar_str) == 0)
101 ret = false;
102 else
103 ret = default_bounce != 0;
104 freeenv(env);
105 return (ret);
106 }
107
108 /*
109 * Given original device, find the requester ID that will be seen by
110 * the IOMMU unit and used for page table lookup. PCI bridges may take
111 * ownership of transactions from downstream devices, so it may not be
112 * the same as the BSF of the target device. In those cases, all
113 * devices downstream of the bridge must share a single mapping
114 * domain, and must collectively be assigned to use either IOMMU or
115 * bounce mapping.
116 */
117 int
iommu_get_requester(device_t dev,device_t * requesterp,uint16_t * rid)118 iommu_get_requester(device_t dev, device_t *requesterp, uint16_t *rid)
119 {
120 device_t l, pci, pcib, pcip, pcibp, requester;
121 int cap_offset;
122 uint16_t pcie_flags;
123 bool bridge_is_pcie;
124
125 l = requester = dev;
126
127 if (!is_pci_device(dev)) {
128 *rid = 0; /* XXXKIB: Could be ACPI HID */
129 *requesterp = NULL;
130 return (ENOTTY);
131 }
132
133 *rid = pci_get_rid(dev);
134
135 /*
136 * Walk the bridge hierarchy from the target device to the
137 * host port to find the translating bridge nearest the IOMMU
138 * unit.
139 */
140 for (;;) {
141 if (!is_pci_device(l)) {
142 if (bootverbose) {
143 printf(
144 "iommu_get_requester(%s): non-pci ancestor %s\n",
145 device_get_name(dev), device_get_name(l));
146 }
147 *rid = 0;
148 *requesterp = NULL;
149 return (ENXIO);
150 }
151
152 pci = device_get_parent(l);
153 pcib = device_get_parent(pci);
154 if (pcib == NULL) {
155 if (bootverbose) {
156 printf(
157 "iommu_get_requester(%s): NULL bridge for %s\n",
158 device_get_name(dev), device_get_name(pci));
159 }
160 *rid = 0;
161 *requesterp = NULL;
162 return (ENXIO);
163 }
164
165 /*
166 * The parent of our "bridge" isn't another PCI bus,
167 * so pcib isn't a PCI->PCI bridge but rather a host
168 * port, and the requester ID won't be translated
169 * further.
170 */
171 if (!is_pci_device(pcib))
172 break;
173
174 if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) {
175 /*
176 * Do not stop the loop even if the target
177 * device is PCIe, because it is possible (but
178 * unlikely) to have a PCI->PCIe bridge
179 * somewhere in the hierarchy.
180 */
181 l = pcib;
182 } else {
183 /*
184 * Device is not PCIe, it cannot be seen as a
185 * requester by IOMMU unit. Check whether the
186 * bridge is PCIe.
187 */
188 bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,
189 &cap_offset) == 0;
190 requester = pcib;
191
192 /*
193 * Check for a buggy PCIe/PCI bridge that
194 * doesn't report the express capability. If
195 * the bridge above it is express but isn't a
196 * PCI bridge, then we know pcib is actually a
197 * PCIe/PCI bridge.
198 */
199 pcip = device_get_parent(pcib);
200 pcibp = device_get_parent(pcip);
201 if (!bridge_is_pcie && pci_find_cap(pcibp,
202 PCIY_EXPRESS, &cap_offset) == 0) {
203 pcie_flags = pci_read_config(pcibp,
204 cap_offset + PCIER_FLAGS, 2);
205 if ((pcie_flags & PCIEM_FLAGS_TYPE) !=
206 PCIEM_TYPE_PCI_BRIDGE)
207 bridge_is_pcie = true;
208 }
209
210 if (bridge_is_pcie) {
211 /*
212 * The current device is not PCIe, but
213 * the bridge above it is. This is a
214 * PCIe->PCI bridge. Assume that the
215 * requester ID will be the secondary
216 * bus number with slot and function
217 * set to zero.
218 *
219 * XXX: Doesn't handle the case where
220 * the bridge is PCIe->PCI-X, and the
221 * bridge will only take ownership of
222 * requests in some cases. We should
223 * provide context entries with the
224 * same page tables for taken and
225 * non-taken transactions.
226 */
227 *rid = PCI_RID(pci_get_bus(l), 0, 0);
228 l = pcibp;
229 } else {
230 /*
231 * Neither the device nor the bridge
232 * above it are PCIe. This is a
233 * conventional PCI->PCI bridge, which
234 * will use the bridge's BSF as the
235 * requester ID.
236 */
237 *rid = pci_get_rid(pcib);
238 l = pcib;
239 }
240 }
241 }
242 *requesterp = requester;
243 return (0);
244 }
245
246 struct iommu_ctx *
iommu_instantiate_ctx(struct iommu_unit * unit,device_t dev,bool rmrr)247 iommu_instantiate_ctx(struct iommu_unit *unit, device_t dev, bool rmrr)
248 {
249 device_t requester;
250 struct iommu_ctx *ctx;
251 int error;
252 bool disabled;
253 uint16_t rid;
254
255 error = iommu_get_requester(dev, &requester, &rid);
256 if (error != 0)
257 return (NULL);
258
259 /*
260 * If the user requested the IOMMU disabled for the device, we
261 * cannot disable the IOMMU unit, due to possibility of other
262 * devices on the same IOMMU unit still requiring translation.
263 * Instead provide the identity mapping for the device
264 * context.
265 */
266 disabled = iommu_bus_dma_is_dev_disabled(pci_get_domain(requester),
267 pci_get_bus(requester), pci_get_slot(requester),
268 pci_get_function(requester));
269 ctx = iommu_get_ctx(unit, requester, rid, disabled, rmrr);
270 if (ctx == NULL)
271 return (NULL);
272 if (disabled) {
273 /*
274 * Keep the first reference on context, release the
275 * later refs.
276 */
277 IOMMU_LOCK(unit);
278 if ((ctx->flags & IOMMU_CTX_DISABLED) == 0) {
279 ctx->flags |= IOMMU_CTX_DISABLED;
280 IOMMU_UNLOCK(unit);
281 } else {
282 iommu_free_ctx_locked(unit, ctx);
283 }
284 }
285 return (ctx);
286 }
287
288 struct iommu_ctx *
iommu_get_dev_ctx(device_t dev)289 iommu_get_dev_ctx(device_t dev)
290 {
291 struct iommu_ctx *ctx;
292 struct iommu_unit *unit;
293
294 unit = iommu_find(dev, bootverbose);
295 /* Not in scope of any IOMMU ? */
296 if (unit == NULL)
297 return (NULL);
298 if (!unit->dma_enabled)
299 return (NULL);
300
301 iommu_unit_pre_instantiate_ctx(unit);
302 ctx = iommu_instantiate_ctx(unit, dev, false);
303 if (ctx != NULL && (ctx->flags & IOMMU_CTX_DISABLED) != 0)
304 ctx = NULL;
305 return (ctx);
306 }
307
308 bus_dma_tag_t
iommu_get_dma_tag(device_t dev,device_t child)309 iommu_get_dma_tag(device_t dev, device_t child)
310 {
311 struct iommu_ctx *ctx;
312 bus_dma_tag_t res;
313
314 ctx = iommu_get_dev_ctx(child);
315 if (ctx == NULL)
316 return (NULL);
317
318 res = (bus_dma_tag_t)ctx->tag;
319 return (res);
320 }
321
322 bool
bus_dma_iommu_set_buswide(device_t dev)323 bus_dma_iommu_set_buswide(device_t dev)
324 {
325 struct iommu_unit *unit;
326 u_int busno, slot, func;
327
328 if (!is_pci_device(dev))
329 return (false);
330 unit = iommu_find(dev, bootverbose);
331 if (unit == NULL)
332 return (false);
333 busno = pci_get_bus(dev);
334 slot = pci_get_slot(dev);
335 func = pci_get_function(dev);
336 if (slot != 0 || func != 0) {
337 if (bootverbose) {
338 device_printf(dev,
339 "iommu%d pci%d:%d:%d requested buswide busdma\n",
340 unit->unit, busno, slot, func);
341 }
342 return (false);
343 }
344 iommu_set_buswide_ctx(unit, busno);
345 return (true);
346 }
347
348 void
iommu_set_buswide_ctx(struct iommu_unit * unit,u_int busno)349 iommu_set_buswide_ctx(struct iommu_unit *unit, u_int busno)
350 {
351
352 MPASS(busno <= PCI_BUSMAX);
353 IOMMU_LOCK(unit);
354 unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] |=
355 1 << (busno % (NBBY * sizeof(uint32_t)));
356 IOMMU_UNLOCK(unit);
357 }
358
359 bool
iommu_is_buswide_ctx(struct iommu_unit * unit,u_int busno)360 iommu_is_buswide_ctx(struct iommu_unit *unit, u_int busno)
361 {
362
363 MPASS(busno <= PCI_BUSMAX);
364 return ((unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] &
365 (1U << (busno % (NBBY * sizeof(uint32_t))))) != 0);
366 }
367
368 static MALLOC_DEFINE(M_IOMMU_DMAMAP, "iommu_dmamap", "IOMMU DMA Map");
369
370 static void iommu_bus_schedule_dmamap(struct iommu_unit *unit,
371 struct bus_dmamap_iommu *map);
372
373 static int
iommu_bus_dma_tag_create(bus_dma_tag_t parent,bus_size_t alignment,bus_addr_t boundary,bus_addr_t lowaddr,bus_addr_t highaddr,bus_size_t maxsize,int nsegments,bus_size_t maxsegsz,int flags,bus_dma_lock_t * lockfunc,void * lockfuncarg,bus_dma_tag_t * dmat)374 iommu_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
375 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
376 bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags,
377 bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat)
378 {
379 struct bus_dma_tag_iommu *newtag, *oldtag;
380 int error;
381
382 *dmat = NULL;
383 error = common_bus_dma_tag_create(parent != NULL ?
384 &((struct bus_dma_tag_iommu *)parent)->common : NULL, alignment,
385 boundary, lowaddr, highaddr, maxsize, nsegments, maxsegsz, flags,
386 lockfunc, lockfuncarg, sizeof(struct bus_dma_tag_iommu),
387 (void **)&newtag);
388 if (error != 0)
389 goto out;
390
391 oldtag = (struct bus_dma_tag_iommu *)parent;
392 newtag->common.impl = &bus_dma_iommu_impl;
393 newtag->ctx = oldtag->ctx;
394 newtag->owner = oldtag->owner;
395
396 *dmat = (bus_dma_tag_t)newtag;
397 out:
398 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
399 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
400 error);
401 return (error);
402 }
403
404 static int
iommu_bus_dma_tag_set_domain(bus_dma_tag_t dmat)405 iommu_bus_dma_tag_set_domain(bus_dma_tag_t dmat)
406 {
407
408 return (0);
409 }
410
411 static int
iommu_bus_dma_tag_destroy(bus_dma_tag_t dmat1)412 iommu_bus_dma_tag_destroy(bus_dma_tag_t dmat1)
413 {
414 struct bus_dma_tag_iommu *dmat;
415 struct iommu_unit *iommu;
416 struct iommu_ctx *ctx;
417 int error;
418
419 error = 0;
420 dmat = (struct bus_dma_tag_iommu *)dmat1;
421
422 if (dmat != NULL) {
423 if (dmat->map_count != 0) {
424 error = EBUSY;
425 goto out;
426 }
427 ctx = dmat->ctx;
428 if (dmat == ctx->tag) {
429 iommu = ctx->domain->iommu;
430 IOMMU_LOCK(iommu);
431 iommu_free_ctx_locked(iommu, dmat->ctx);
432 }
433 free(dmat->segments, M_IOMMU_DMAMAP);
434 free(dmat, M_DEVBUF);
435 }
436 out:
437 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat, error);
438 return (error);
439 }
440
441 static bool
iommu_bus_dma_id_mapped(bus_dma_tag_t dmat,vm_paddr_t buf,bus_size_t buflen)442 iommu_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen)
443 {
444
445 return (false);
446 }
447
448 static int
iommu_bus_dmamap_create(bus_dma_tag_t dmat,int flags,bus_dmamap_t * mapp)449 iommu_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
450 {
451 struct bus_dma_tag_iommu *tag;
452 struct bus_dmamap_iommu *map;
453
454 tag = (struct bus_dma_tag_iommu *)dmat;
455 map = malloc_domainset(sizeof(*map), M_IOMMU_DMAMAP,
456 DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO);
457 if (map == NULL) {
458 *mapp = NULL;
459 return (ENOMEM);
460 }
461 if (tag->segments == NULL) {
462 tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) *
463 tag->common.nsegments, M_IOMMU_DMAMAP,
464 DOMAINSET_PREF(tag->common.domain), M_NOWAIT);
465 if (tag->segments == NULL) {
466 free(map, M_IOMMU_DMAMAP);
467 *mapp = NULL;
468 return (ENOMEM);
469 }
470 }
471 IOMMU_DMAMAP_INIT(map);
472 TAILQ_INIT(&map->map_entries);
473 map->tag = tag;
474 map->locked = true;
475 map->cansleep = false;
476 tag->map_count++;
477 *mapp = (bus_dmamap_t)map;
478
479 return (0);
480 }
481
482 static int
iommu_bus_dmamap_destroy(bus_dma_tag_t dmat,bus_dmamap_t map1)483 iommu_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1)
484 {
485 struct bus_dma_tag_iommu *tag;
486 struct bus_dmamap_iommu *map;
487
488 tag = (struct bus_dma_tag_iommu *)dmat;
489 map = (struct bus_dmamap_iommu *)map1;
490 if (map != NULL) {
491 IOMMU_DMAMAP_LOCK(map);
492 if (!TAILQ_EMPTY(&map->map_entries)) {
493 IOMMU_DMAMAP_UNLOCK(map);
494 return (EBUSY);
495 }
496 IOMMU_DMAMAP_DESTROY(map);
497 free(map, M_IOMMU_DMAMAP);
498 }
499 tag->map_count--;
500 return (0);
501 }
502
503
504 static int
iommu_bus_dmamem_alloc(bus_dma_tag_t dmat,void ** vaddr,int flags,bus_dmamap_t * mapp)505 iommu_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
506 bus_dmamap_t *mapp)
507 {
508 struct bus_dma_tag_iommu *tag;
509 struct bus_dmamap_iommu *map;
510 int error, mflags;
511 vm_memattr_t attr;
512
513 error = iommu_bus_dmamap_create(dmat, flags, mapp);
514 if (error != 0)
515 return (error);
516
517 mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK;
518 mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0;
519 attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE :
520 VM_MEMATTR_DEFAULT;
521
522 tag = (struct bus_dma_tag_iommu *)dmat;
523 map = (struct bus_dmamap_iommu *)*mapp;
524
525 if (tag->common.maxsize < PAGE_SIZE &&
526 tag->common.alignment <= tag->common.maxsize &&
527 attr == VM_MEMATTR_DEFAULT) {
528 *vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF,
529 DOMAINSET_PREF(tag->common.domain), mflags);
530 map->flags |= BUS_DMAMAP_IOMMU_MALLOC;
531 } else {
532 *vaddr = kmem_alloc_attr_domainset(
533 DOMAINSET_PREF(tag->common.domain), tag->common.maxsize,
534 mflags, 0ul, BUS_SPACE_MAXADDR, attr);
535 map->flags |= BUS_DMAMAP_IOMMU_KMEM_ALLOC;
536 }
537 if (*vaddr == NULL) {
538 iommu_bus_dmamap_destroy(dmat, *mapp);
539 *mapp = NULL;
540 return (ENOMEM);
541 }
542 return (0);
543 }
544
545 static void
iommu_bus_dmamem_free(bus_dma_tag_t dmat,void * vaddr,bus_dmamap_t map1)546 iommu_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1)
547 {
548 struct bus_dma_tag_iommu *tag;
549 struct bus_dmamap_iommu *map;
550
551 tag = (struct bus_dma_tag_iommu *)dmat;
552 map = (struct bus_dmamap_iommu *)map1;
553
554 if ((map->flags & BUS_DMAMAP_IOMMU_MALLOC) != 0) {
555 free(vaddr, M_DEVBUF);
556 map->flags &= ~BUS_DMAMAP_IOMMU_MALLOC;
557 } else {
558 KASSERT((map->flags & BUS_DMAMAP_IOMMU_KMEM_ALLOC) != 0,
559 ("iommu_bus_dmamem_free for non alloced map %p", map));
560 kmem_free(vaddr, tag->common.maxsize);
561 map->flags &= ~BUS_DMAMAP_IOMMU_KMEM_ALLOC;
562 }
563
564 iommu_bus_dmamap_destroy(dmat, map1);
565 }
566
567 static int
iommu_bus_dmamap_load_something1(struct bus_dma_tag_iommu * tag,struct bus_dmamap_iommu * map,vm_page_t * ma,int offset,bus_size_t buflen,int flags,bus_dma_segment_t * segs,int * segp,struct iommu_map_entries_tailq * entries)568 iommu_bus_dmamap_load_something1(struct bus_dma_tag_iommu *tag,
569 struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen,
570 int flags, bus_dma_segment_t *segs, int *segp,
571 struct iommu_map_entries_tailq *entries)
572 {
573 struct iommu_ctx *ctx;
574 struct iommu_domain *domain;
575 struct iommu_map_entry *entry;
576 bus_size_t buflen1;
577 int error, e_flags, idx, gas_flags, seg;
578
579 KASSERT(offset < IOMMU_PAGE_SIZE, ("offset %d", offset));
580 if (segs == NULL)
581 segs = tag->segments;
582 ctx = tag->ctx;
583 domain = ctx->domain;
584 e_flags = IOMMU_MAP_ENTRY_READ |
585 ((flags & BUS_DMA_NOWRITE) == 0 ? IOMMU_MAP_ENTRY_WRITE : 0);
586 seg = *segp;
587 error = 0;
588 idx = 0;
589 while (buflen > 0) {
590 seg++;
591 if (seg >= tag->common.nsegments) {
592 error = EFBIG;
593 break;
594 }
595 buflen1 = buflen > tag->common.maxsegsz ?
596 tag->common.maxsegsz : buflen;
597
598 /*
599 * (Too) optimistically allow split if there are more
600 * then one segments left.
601 */
602 gas_flags = map->cansleep ? IOMMU_MF_CANWAIT : 0;
603 if (seg + 1 < tag->common.nsegments)
604 gas_flags |= IOMMU_MF_CANSPLIT;
605
606 error = iommu_gas_map(domain, &tag->common, buflen1,
607 offset, e_flags, gas_flags, ma + idx, &entry);
608 if (error != 0)
609 break;
610 /* Update buflen1 in case buffer split. */
611 if (buflen1 > entry->end - entry->start - offset)
612 buflen1 = entry->end - entry->start - offset;
613
614 KASSERT(vm_addr_align_ok(entry->start + offset,
615 tag->common.alignment),
616 ("alignment failed: ctx %p start 0x%jx offset %x "
617 "align 0x%jx", ctx, (uintmax_t)entry->start, offset,
618 (uintmax_t)tag->common.alignment));
619 KASSERT(entry->end <= tag->common.lowaddr ||
620 entry->start >= tag->common.highaddr,
621 ("entry placement failed: ctx %p start 0x%jx end 0x%jx "
622 "lowaddr 0x%jx highaddr 0x%jx", ctx,
623 (uintmax_t)entry->start, (uintmax_t)entry->end,
624 (uintmax_t)tag->common.lowaddr,
625 (uintmax_t)tag->common.highaddr));
626 KASSERT(vm_addr_bound_ok(entry->start + offset, buflen1,
627 tag->common.boundary),
628 ("boundary failed: ctx %p start 0x%jx end 0x%jx "
629 "boundary 0x%jx", ctx, (uintmax_t)entry->start,
630 (uintmax_t)entry->end, (uintmax_t)tag->common.boundary));
631 KASSERT(buflen1 <= tag->common.maxsegsz,
632 ("segment too large: ctx %p start 0x%jx end 0x%jx "
633 "buflen1 0x%jx maxsegsz 0x%jx", ctx,
634 (uintmax_t)entry->start, (uintmax_t)entry->end,
635 (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz));
636
637 KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
638 ("entry %p missing IOMMU_MAP_ENTRY_MAP", entry));
639 TAILQ_INSERT_TAIL(entries, entry, dmamap_link);
640
641 segs[seg].ds_addr = entry->start + offset;
642 segs[seg].ds_len = buflen1;
643
644 idx += OFF_TO_IDX(offset + buflen1);
645 offset += buflen1;
646 offset &= IOMMU_PAGE_MASK;
647 buflen -= buflen1;
648 }
649 if (error == 0)
650 *segp = seg;
651 return (error);
652 }
653
654 static int
iommu_bus_dmamap_load_something(struct bus_dma_tag_iommu * tag,struct bus_dmamap_iommu * map,vm_page_t * ma,int offset,bus_size_t buflen,int flags,bus_dma_segment_t * segs,int * segp)655 iommu_bus_dmamap_load_something(struct bus_dma_tag_iommu *tag,
656 struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen,
657 int flags, bus_dma_segment_t *segs, int *segp)
658 {
659 struct iommu_ctx *ctx;
660 struct iommu_domain *domain;
661 struct iommu_map_entries_tailq entries;
662 int error;
663
664 ctx = tag->ctx;
665 domain = ctx->domain;
666 atomic_add_long(&ctx->loads, 1);
667
668 TAILQ_INIT(&entries);
669 error = iommu_bus_dmamap_load_something1(tag, map, ma, offset,
670 buflen, flags, segs, segp, &entries);
671 if (error == 0) {
672 IOMMU_DMAMAP_LOCK(map);
673 TAILQ_CONCAT(&map->map_entries, &entries, dmamap_link);
674 IOMMU_DMAMAP_UNLOCK(map);
675 } else if (!TAILQ_EMPTY(&entries)) {
676 /*
677 * The busdma interface does not allow us to report
678 * partial buffer load, so unfortunately we have to
679 * revert all work done.
680 */
681 IOMMU_DOMAIN_LOCK(domain);
682 TAILQ_CONCAT(&domain->unload_entries, &entries, dmamap_link);
683 IOMMU_DOMAIN_UNLOCK(domain);
684 taskqueue_enqueue(domain->iommu->delayed_taskqueue,
685 &domain->unload_task);
686 }
687
688 if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 &&
689 !map->cansleep)
690 error = EINPROGRESS;
691 if (error == EINPROGRESS)
692 iommu_bus_schedule_dmamap(domain->iommu, map);
693 return (error);
694 }
695
696 static int
iommu_bus_dmamap_load_ma(bus_dma_tag_t dmat,bus_dmamap_t map1,struct vm_page ** ma,bus_size_t tlen,int ma_offs,int flags,bus_dma_segment_t * segs,int * segp)697 iommu_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1,
698 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
699 bus_dma_segment_t *segs, int *segp)
700 {
701 struct bus_dma_tag_iommu *tag;
702 struct bus_dmamap_iommu *map;
703
704 tag = (struct bus_dma_tag_iommu *)dmat;
705 map = (struct bus_dmamap_iommu *)map1;
706 return (iommu_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen,
707 flags, segs, segp));
708 }
709
710 static int
iommu_bus_dmamap_load_phys(bus_dma_tag_t dmat,bus_dmamap_t map1,vm_paddr_t buf,bus_size_t buflen,int flags,bus_dma_segment_t * segs,int * segp)711 iommu_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1,
712 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
713 int *segp)
714 {
715 struct bus_dma_tag_iommu *tag;
716 struct bus_dmamap_iommu *map;
717 vm_page_t *ma, fma;
718 vm_paddr_t pstart, pend, paddr;
719 int error, i, ma_cnt, mflags, offset;
720
721 tag = (struct bus_dma_tag_iommu *)dmat;
722 map = (struct bus_dmamap_iommu *)map1;
723 pstart = trunc_page(buf);
724 pend = round_page(buf + buflen);
725 offset = buf & PAGE_MASK;
726 ma_cnt = OFF_TO_IDX(pend - pstart);
727 mflags = map->cansleep ? M_WAITOK : M_NOWAIT;
728 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags);
729 if (ma == NULL)
730 return (ENOMEM);
731 fma = NULL;
732 for (i = 0; i < ma_cnt; i++) {
733 paddr = pstart + ptoa(i);
734 ma[i] = PHYS_TO_VM_PAGE(paddr);
735 if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) {
736 /*
737 * If PHYS_TO_VM_PAGE() returned NULL or the
738 * vm_page was not initialized we'll use a
739 * fake page.
740 */
741 if (fma == NULL) {
742 fma = malloc(sizeof(struct vm_page) * ma_cnt,
743 M_DEVBUF, M_ZERO | mflags);
744 if (fma == NULL) {
745 free(ma, M_DEVBUF);
746 return (ENOMEM);
747 }
748 }
749 vm_page_initfake(&fma[i], pstart + ptoa(i),
750 VM_MEMATTR_DEFAULT);
751 ma[i] = &fma[i];
752 }
753 }
754 error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen,
755 flags, segs, segp);
756 free(fma, M_DEVBUF);
757 free(ma, M_DEVBUF);
758 return (error);
759 }
760
761 static int
iommu_bus_dmamap_load_buffer(bus_dma_tag_t dmat,bus_dmamap_t map1,void * buf,bus_size_t buflen,pmap_t pmap,int flags,bus_dma_segment_t * segs,int * segp)762 iommu_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf,
763 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
764 int *segp)
765 {
766 struct bus_dma_tag_iommu *tag;
767 struct bus_dmamap_iommu *map;
768 vm_page_t *ma, fma;
769 vm_paddr_t pstart, pend, paddr;
770 int error, i, ma_cnt, mflags, offset;
771
772 tag = (struct bus_dma_tag_iommu *)dmat;
773 map = (struct bus_dmamap_iommu *)map1;
774 pstart = trunc_page((vm_offset_t)buf);
775 pend = round_page((vm_offset_t)buf + buflen);
776 offset = (vm_offset_t)buf & PAGE_MASK;
777 ma_cnt = OFF_TO_IDX(pend - pstart);
778 mflags = map->cansleep ? M_WAITOK : M_NOWAIT;
779 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags);
780 if (ma == NULL)
781 return (ENOMEM);
782 fma = NULL;
783 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
784 if (pmap == kernel_pmap)
785 paddr = pmap_kextract(pstart);
786 else
787 paddr = pmap_extract(pmap, pstart);
788 ma[i] = PHYS_TO_VM_PAGE(paddr);
789 if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) {
790 /*
791 * If PHYS_TO_VM_PAGE() returned NULL or the
792 * vm_page was not initialized we'll use a
793 * fake page.
794 */
795 if (fma == NULL) {
796 fma = malloc(sizeof(struct vm_page) * ma_cnt,
797 M_DEVBUF, M_ZERO | mflags);
798 if (fma == NULL) {
799 free(ma, M_DEVBUF);
800 return (ENOMEM);
801 }
802 }
803 vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT);
804 ma[i] = &fma[i];
805 }
806 }
807 error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen,
808 flags, segs, segp);
809 free(ma, M_DEVBUF);
810 free(fma, M_DEVBUF);
811 return (error);
812 }
813
814 static void
iommu_bus_dmamap_waitok(bus_dma_tag_t dmat,bus_dmamap_t map1,struct memdesc * mem,bus_dmamap_callback_t * callback,void * callback_arg)815 iommu_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1,
816 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
817 {
818 struct bus_dmamap_iommu *map;
819
820 if (map1 == NULL)
821 return;
822 map = (struct bus_dmamap_iommu *)map1;
823 map->mem = *mem;
824 map->tag = (struct bus_dma_tag_iommu *)dmat;
825 map->callback = callback;
826 map->callback_arg = callback_arg;
827 }
828
829 static bus_dma_segment_t *
iommu_bus_dmamap_complete(bus_dma_tag_t dmat,bus_dmamap_t map1,bus_dma_segment_t * segs,int nsegs,int error)830 iommu_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
831 bus_dma_segment_t *segs, int nsegs, int error)
832 {
833 struct bus_dma_tag_iommu *tag;
834 struct bus_dmamap_iommu *map;
835
836 tag = (struct bus_dma_tag_iommu *)dmat;
837 map = (struct bus_dmamap_iommu *)map1;
838
839 if (!map->locked) {
840 KASSERT(map->cansleep,
841 ("map not locked and not sleepable context %p", map));
842
843 /*
844 * We are called from the delayed context. Relock the
845 * driver.
846 */
847 (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK);
848 map->locked = true;
849 }
850
851 if (segs == NULL)
852 segs = tag->segments;
853 return (segs);
854 }
855
856 /*
857 * The limitations of busdma KPI forces the iommu to perform the actual
858 * unload, consisting of the unmapping of the map entries page tables,
859 * from the delayed context on i386, since page table page mapping
860 * might require a sleep to be successfull. The unfortunate
861 * consequence is that the DMA requests can be served some time after
862 * the bus_dmamap_unload() call returned.
863 *
864 * On amd64, we assume that sf allocation cannot fail.
865 */
866 static void
iommu_bus_dmamap_unload(bus_dma_tag_t dmat,bus_dmamap_t map1)867 iommu_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1)
868 {
869 struct bus_dma_tag_iommu *tag;
870 struct bus_dmamap_iommu *map;
871 struct iommu_ctx *ctx;
872 struct iommu_domain *domain;
873 struct iommu_map_entries_tailq entries;
874
875 tag = (struct bus_dma_tag_iommu *)dmat;
876 map = (struct bus_dmamap_iommu *)map1;
877 ctx = tag->ctx;
878 domain = ctx->domain;
879 atomic_add_long(&ctx->unloads, 1);
880
881 TAILQ_INIT(&entries);
882 IOMMU_DMAMAP_LOCK(map);
883 TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link);
884 IOMMU_DMAMAP_UNLOCK(map);
885 #if defined(IOMMU_DOMAIN_UNLOAD_SLEEP)
886 IOMMU_DOMAIN_LOCK(domain);
887 TAILQ_CONCAT(&domain->unload_entries, &entries, dmamap_link);
888 IOMMU_DOMAIN_UNLOCK(domain);
889 taskqueue_enqueue(domain->iommu->delayed_taskqueue,
890 &domain->unload_task);
891 #else
892 THREAD_NO_SLEEPING();
893 iommu_domain_unload(domain, &entries, false);
894 THREAD_SLEEPING_OK();
895 KASSERT(TAILQ_EMPTY(&entries), ("lazy iommu_ctx_unload %p", ctx));
896 #endif
897 }
898
899 static void
iommu_bus_dmamap_sync(bus_dma_tag_t dmat,bus_dmamap_t map1,bus_dmasync_op_t op)900 iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map1,
901 bus_dmasync_op_t op)
902 {
903 struct bus_dmamap_iommu *map __unused;
904
905 map = (struct bus_dmamap_iommu *)map1;
906 kmsan_bus_dmamap_sync(&map->kmsan_mem, op);
907 }
908
909 #ifdef KMSAN
910 static void
iommu_bus_dmamap_load_kmsan(bus_dmamap_t map1,struct memdesc * mem)911 iommu_bus_dmamap_load_kmsan(bus_dmamap_t map1, struct memdesc *mem)
912 {
913 struct bus_dmamap_iommu *map;
914
915 map = (struct bus_dmamap_iommu *)map1;
916 if (map == NULL)
917 return;
918 memcpy(&map->kmsan_mem, mem, sizeof(struct memdesc));
919 }
920 #endif
921
922 struct bus_dma_impl bus_dma_iommu_impl = {
923 .tag_create = iommu_bus_dma_tag_create,
924 .tag_destroy = iommu_bus_dma_tag_destroy,
925 .tag_set_domain = iommu_bus_dma_tag_set_domain,
926 .id_mapped = iommu_bus_dma_id_mapped,
927 .map_create = iommu_bus_dmamap_create,
928 .map_destroy = iommu_bus_dmamap_destroy,
929 .mem_alloc = iommu_bus_dmamem_alloc,
930 .mem_free = iommu_bus_dmamem_free,
931 .load_phys = iommu_bus_dmamap_load_phys,
932 .load_buffer = iommu_bus_dmamap_load_buffer,
933 .load_ma = iommu_bus_dmamap_load_ma,
934 .map_waitok = iommu_bus_dmamap_waitok,
935 .map_complete = iommu_bus_dmamap_complete,
936 .map_unload = iommu_bus_dmamap_unload,
937 .map_sync = iommu_bus_dmamap_sync,
938 #ifdef KMSAN
939 .load_kmsan = iommu_bus_dmamap_load_kmsan,
940 #endif
941 };
942
943 static void
iommu_bus_task_dmamap(void * arg,int pending)944 iommu_bus_task_dmamap(void *arg, int pending)
945 {
946 struct bus_dma_tag_iommu *tag;
947 struct bus_dmamap_iommu *map;
948 struct iommu_unit *unit;
949
950 unit = arg;
951 IOMMU_LOCK(unit);
952 while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) {
953 TAILQ_REMOVE(&unit->delayed_maps, map, delay_link);
954 IOMMU_UNLOCK(unit);
955 tag = map->tag;
956 map->cansleep = true;
957 map->locked = false;
958 bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map,
959 &map->mem, map->callback, map->callback_arg,
960 BUS_DMA_WAITOK);
961 map->cansleep = false;
962 if (map->locked) {
963 (tag->common.lockfunc)(tag->common.lockfuncarg,
964 BUS_DMA_UNLOCK);
965 } else
966 map->locked = true;
967 map->cansleep = false;
968 IOMMU_LOCK(unit);
969 }
970 IOMMU_UNLOCK(unit);
971 }
972
973 static void
iommu_bus_schedule_dmamap(struct iommu_unit * unit,struct bus_dmamap_iommu * map)974 iommu_bus_schedule_dmamap(struct iommu_unit *unit, struct bus_dmamap_iommu *map)
975 {
976
977 map->locked = false;
978 IOMMU_LOCK(unit);
979 TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link);
980 IOMMU_UNLOCK(unit);
981 taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task);
982 }
983
984 int
iommu_init_busdma(struct iommu_unit * unit)985 iommu_init_busdma(struct iommu_unit *unit)
986 {
987 int error;
988
989 unit->dma_enabled = 0;
990 error = TUNABLE_INT_FETCH("hw.iommu.dma", &unit->dma_enabled);
991 if (error == 0) /* compatibility */
992 TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled);
993 SYSCTL_ADD_INT(&unit->sysctl_ctx,
994 SYSCTL_CHILDREN(device_get_sysctl_tree(unit->dev)),
995 OID_AUTO, "dma", CTLFLAG_RD, &unit->dma_enabled, 0,
996 "DMA ops enabled");
997 TAILQ_INIT(&unit->delayed_maps);
998 TASK_INIT(&unit->dmamap_load_task, 0, iommu_bus_task_dmamap, unit);
999 unit->delayed_taskqueue = taskqueue_create("iommu", M_WAITOK,
1000 taskqueue_thread_enqueue, &unit->delayed_taskqueue);
1001 taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK,
1002 "iommu%d busdma taskq", unit->unit);
1003 return (0);
1004 }
1005
1006 void
iommu_fini_busdma(struct iommu_unit * unit)1007 iommu_fini_busdma(struct iommu_unit *unit)
1008 {
1009
1010 if (unit->delayed_taskqueue == NULL)
1011 return;
1012
1013 taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task);
1014 taskqueue_free(unit->delayed_taskqueue);
1015 unit->delayed_taskqueue = NULL;
1016 }
1017
1018 int
bus_dma_iommu_load_ident(bus_dma_tag_t dmat,bus_dmamap_t map1,vm_paddr_t start,vm_size_t length,int flags)1019 bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map1,
1020 vm_paddr_t start, vm_size_t length, int flags)
1021 {
1022 struct bus_dma_tag_common *tc;
1023 struct bus_dma_tag_iommu *tag;
1024 struct bus_dmamap_iommu *map;
1025 struct iommu_ctx *ctx;
1026 struct iommu_domain *domain;
1027 struct iommu_map_entry *entry;
1028 vm_page_t *ma;
1029 vm_size_t i;
1030 int error;
1031 bool waitok;
1032
1033 MPASS((start & PAGE_MASK) == 0);
1034 MPASS((length & PAGE_MASK) == 0);
1035 MPASS(length > 0);
1036 MPASS(start + length >= start);
1037 MPASS((flags & ~(BUS_DMA_NOWAIT | BUS_DMA_NOWRITE)) == 0);
1038
1039 tc = (struct bus_dma_tag_common *)dmat;
1040 if (tc->impl != &bus_dma_iommu_impl)
1041 return (0);
1042
1043 tag = (struct bus_dma_tag_iommu *)dmat;
1044 ctx = tag->ctx;
1045 domain = ctx->domain;
1046 map = (struct bus_dmamap_iommu *)map1;
1047 waitok = (flags & BUS_DMA_NOWAIT) != 0;
1048
1049 entry = iommu_gas_alloc_entry(domain, waitok ? 0 : IOMMU_PGF_WAITOK);
1050 if (entry == NULL)
1051 return (ENOMEM);
1052 entry->start = start;
1053 entry->end = start + length;
1054 ma = malloc(sizeof(vm_page_t) * atop(length), M_TEMP, waitok ?
1055 M_WAITOK : M_NOWAIT);
1056 if (ma == NULL) {
1057 iommu_gas_free_entry(entry);
1058 return (ENOMEM);
1059 }
1060 for (i = 0; i < atop(length); i++) {
1061 ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
1062 VM_MEMATTR_DEFAULT);
1063 }
1064 error = iommu_gas_map_region(domain, entry, IOMMU_MAP_ENTRY_READ |
1065 ((flags & BUS_DMA_NOWRITE) ? 0 : IOMMU_MAP_ENTRY_WRITE) |
1066 IOMMU_MAP_ENTRY_MAP, waitok ? IOMMU_MF_CANWAIT : 0, ma);
1067 if (error == 0) {
1068 IOMMU_DMAMAP_LOCK(map);
1069 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
1070 IOMMU_DMAMAP_UNLOCK(map);
1071 } else {
1072 iommu_gas_free_entry(entry);
1073 }
1074 for (i = 0; i < atop(length); i++)
1075 vm_page_putfake(ma[i]);
1076 free(ma, M_TEMP);
1077 return (error);
1078 }
1079
1080 static void
iommu_domain_unload_task(void * arg,int pending)1081 iommu_domain_unload_task(void *arg, int pending)
1082 {
1083 struct iommu_domain *domain;
1084 struct iommu_map_entries_tailq entries;
1085
1086 domain = arg;
1087 TAILQ_INIT(&entries);
1088
1089 for (;;) {
1090 IOMMU_DOMAIN_LOCK(domain);
1091 TAILQ_SWAP(&domain->unload_entries, &entries,
1092 iommu_map_entry, dmamap_link);
1093 IOMMU_DOMAIN_UNLOCK(domain);
1094 if (TAILQ_EMPTY(&entries))
1095 break;
1096 iommu_domain_unload(domain, &entries, true);
1097 }
1098 }
1099
1100 void
iommu_domain_init(struct iommu_unit * unit,struct iommu_domain * domain,const struct iommu_domain_map_ops * ops)1101 iommu_domain_init(struct iommu_unit *unit, struct iommu_domain *domain,
1102 const struct iommu_domain_map_ops *ops)
1103 {
1104
1105 domain->ops = ops;
1106 domain->iommu = unit;
1107
1108 TASK_INIT(&domain->unload_task, 0, iommu_domain_unload_task, domain);
1109 RB_INIT(&domain->rb_root);
1110 TAILQ_INIT(&domain->unload_entries);
1111 mtx_init(&domain->lock, "iodom", NULL, MTX_DEF);
1112 }
1113
1114 void
iommu_domain_fini(struct iommu_domain * domain)1115 iommu_domain_fini(struct iommu_domain *domain)
1116 {
1117
1118 mtx_destroy(&domain->lock);
1119 }
1120