xref: /freebsd/sys/x86/iommu/intel_ctx.c (revision 9e5787d2284e187abb5b654d924394a65772e004)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2013 The FreeBSD Foundation
5  * All rights reserved.
6  *
7  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8  * under sponsorship from the FreeBSD Foundation.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  */
31 
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34 
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/malloc.h>
38 #include <sys/bus.h>
39 #include <sys/interrupt.h>
40 #include <sys/kernel.h>
41 #include <sys/ktr.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/memdesc.h>
45 #include <sys/mutex.h>
46 #include <sys/proc.h>
47 #include <sys/rwlock.h>
48 #include <sys/rman.h>
49 #include <sys/sysctl.h>
50 #include <sys/taskqueue.h>
51 #include <sys/tree.h>
52 #include <sys/uio.h>
53 #include <sys/vmem.h>
54 #include <vm/vm.h>
55 #include <vm/vm_extern.h>
56 #include <vm/vm_kern.h>
57 #include <vm/vm_object.h>
58 #include <vm/vm_page.h>
59 #include <vm/vm_pager.h>
60 #include <vm/vm_map.h>
61 #include <contrib/dev/acpica/include/acpi.h>
62 #include <contrib/dev/acpica/include/accommon.h>
63 #include <dev/pci/pcireg.h>
64 #include <dev/pci/pcivar.h>
65 #include <machine/atomic.h>
66 #include <machine/bus.h>
67 #include <machine/md_var.h>
68 #include <machine/specialreg.h>
69 #include <x86/include/busdma_impl.h>
70 #include <dev/iommu/busdma_iommu.h>
71 #include <x86/iommu/intel_reg.h>
72 #include <x86/iommu/intel_dmar.h>
73 
74 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context");
75 static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain");
76 
77 static void dmar_unref_domain_locked(struct dmar_unit *dmar,
78     struct dmar_domain *domain);
79 static void dmar_domain_destroy(struct dmar_domain *domain);
80 
81 static void
82 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus)
83 {
84 	struct sf_buf *sf;
85 	dmar_root_entry_t *re;
86 	vm_page_t ctxm;
87 
88 	/*
89 	 * Allocated context page must be linked.
90 	 */
91 	ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_NOALLOC);
92 	if (ctxm != NULL)
93 		return;
94 
95 	/*
96 	 * Page not present, allocate and link.  Note that other
97 	 * thread might execute this sequence in parallel.  This
98 	 * should be safe, because the context entries written by both
99 	 * threads are equal.
100 	 */
101 	TD_PREP_PINNED_ASSERT;
102 	ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_ZERO |
103 	    IOMMU_PGF_WAITOK);
104 	re = dmar_map_pgtbl(dmar->ctx_obj, 0, IOMMU_PGF_NOALLOC, &sf);
105 	re += bus;
106 	dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK &
107 	    VM_PAGE_TO_PHYS(ctxm)));
108 	dmar_flush_root_to_ram(dmar, re);
109 	dmar_unmap_pgtbl(sf);
110 	TD_PINNED_ASSERT;
111 }
112 
113 static dmar_ctx_entry_t *
114 dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp)
115 {
116 	struct dmar_unit *dmar;
117 	dmar_ctx_entry_t *ctxp;
118 
119 	dmar = CTX2DMAR(ctx);
120 
121 	ctxp = dmar_map_pgtbl(dmar->ctx_obj, 1 +
122 	    PCI_RID2BUS(ctx->rid), IOMMU_PGF_NOALLOC | IOMMU_PGF_WAITOK, sfp);
123 	ctxp += ctx->rid & 0xff;
124 	return (ctxp);
125 }
126 
127 static void
128 device_tag_init(struct dmar_ctx *ctx, device_t dev)
129 {
130 	struct dmar_domain *domain;
131 	bus_addr_t maxaddr;
132 
133 	domain = CTX2DOM(ctx);
134 	maxaddr = MIN(domain->iodom.end, BUS_SPACE_MAXADDR);
135 	ctx->context.tag->common.ref_count = 1; /* Prevent free */
136 	ctx->context.tag->common.impl = &bus_dma_iommu_impl;
137 	ctx->context.tag->common.boundary = 0;
138 	ctx->context.tag->common.lowaddr = maxaddr;
139 	ctx->context.tag->common.highaddr = maxaddr;
140 	ctx->context.tag->common.maxsize = maxaddr;
141 	ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED;
142 	ctx->context.tag->common.maxsegsz = maxaddr;
143 	ctx->context.tag->ctx = CTX2IOCTX(ctx);
144 	ctx->context.tag->owner = dev;
145 }
146 
147 static void
148 ctx_id_entry_init_one(dmar_ctx_entry_t *ctxp, struct dmar_domain *domain,
149     vm_page_t ctx_root)
150 {
151 	/*
152 	 * For update due to move, the store is not atomic.  It is
153 	 * possible that DMAR read upper doubleword, while low
154 	 * doubleword is not yet updated.  The domain id is stored in
155 	 * the upper doubleword, while the table pointer in the lower.
156 	 *
157 	 * There is no good solution, for the same reason it is wrong
158 	 * to clear P bit in the ctx entry for update.
159 	 */
160 	dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) |
161 	    domain->awlvl);
162 	if (ctx_root == NULL) {
163 		dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P);
164 	} else {
165 		dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR |
166 		    (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) |
167 		    DMAR_CTX1_P);
168 	}
169 }
170 
171 static void
172 ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move,
173     int busno)
174 {
175 	struct dmar_unit *unit;
176 	struct dmar_domain *domain;
177 	vm_page_t ctx_root;
178 	int i;
179 
180 	domain = CTX2DOM(ctx);
181 	unit = DOM2DMAR(domain);
182 	KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0),
183 	    ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",
184 	    unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner),
185 	    pci_get_function(ctx->context.tag->owner),
186 	    ctxp->ctx1, ctxp->ctx2));
187 
188 	if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 &&
189 	    (unit->hw_ecap & DMAR_ECAP_PT) != 0) {
190 		KASSERT(domain->pgtbl_obj == NULL,
191 		    ("ctx %p non-null pgtbl_obj", ctx));
192 		ctx_root = NULL;
193 	} else {
194 		ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0,
195 		    IOMMU_PGF_NOALLOC);
196 	}
197 
198 	if (iommu_is_buswide_ctx(DMAR2IOMMU(unit), busno)) {
199 		MPASS(!move);
200 		for (i = 0; i <= PCI_BUSMAX; i++) {
201 			ctx_id_entry_init_one(&ctxp[i], domain, ctx_root);
202 		}
203 	} else {
204 		ctx_id_entry_init_one(ctxp, domain, ctx_root);
205 	}
206 	dmar_flush_ctx_to_ram(unit, ctxp);
207 }
208 
209 static int
210 dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force)
211 {
212 	int error;
213 
214 	/*
215 	 * If dmar declares Caching Mode as Set, follow 11.5 "Caching
216 	 * Mode Consideration" and do the (global) invalidation of the
217 	 * negative TLB entries.
218 	 */
219 	if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force)
220 		return (0);
221 	if (dmar->qi_enabled) {
222 		dmar_qi_invalidate_ctx_glob_locked(dmar);
223 		if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force)
224 			dmar_qi_invalidate_iotlb_glob_locked(dmar);
225 		return (0);
226 	}
227 	error = dmar_inv_ctx_glob(dmar);
228 	if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force))
229 		error = dmar_inv_iotlb_glob(dmar);
230 	return (error);
231 }
232 
233 static int
234 domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus,
235     int slot, int func, int dev_domain, int dev_busno,
236     const void *dev_path, int dev_path_len)
237 {
238 	struct iommu_map_entries_tailq rmrr_entries;
239 	struct iommu_map_entry *entry, *entry1;
240 	vm_page_t *ma;
241 	iommu_gaddr_t start, end;
242 	vm_pindex_t size, i;
243 	int error, error1;
244 
245 	error = 0;
246 	TAILQ_INIT(&rmrr_entries);
247 	dmar_dev_parse_rmrr(domain, dev_domain, dev_busno, dev_path,
248 	    dev_path_len, &rmrr_entries);
249 	TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) {
250 		/*
251 		 * VT-d specification requires that the start of an
252 		 * RMRR entry is 4k-aligned.  Buggy BIOSes put
253 		 * anything into the start and end fields.  Truncate
254 		 * and round as neccesary.
255 		 *
256 		 * We also allow the overlapping RMRR entries, see
257 		 * iommu_gas_alloc_region().
258 		 */
259 		start = entry->start;
260 		end = entry->end;
261 		if (bootverbose)
262 			printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n",
263 			    domain->iodom.iommu->unit, bus, slot, func,
264 			    (uintmax_t)start, (uintmax_t)end);
265 		entry->start = trunc_page(start);
266 		entry->end = round_page(end);
267 		if (entry->start == entry->end) {
268 			/* Workaround for some AMI (?) BIOSes */
269 			if (bootverbose) {
270 				if (dev != NULL)
271 					device_printf(dev, "");
272 				printf("pci%d:%d:%d ", bus, slot, func);
273 				printf("BIOS bug: dmar%d RMRR "
274 				    "region (%jx, %jx) corrected\n",
275 				    domain->iodom.iommu->unit, start, end);
276 			}
277 			entry->end += DMAR_PAGE_SIZE * 0x20;
278 		}
279 		size = OFF_TO_IDX(entry->end - entry->start);
280 		ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK);
281 		for (i = 0; i < size; i++) {
282 			ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
283 			    VM_MEMATTR_DEFAULT);
284 		}
285 		error1 = iommu_gas_map_region(DOM2IODOM(domain), entry,
286 		    IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE,
287 		    IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma);
288 		/*
289 		 * Non-failed RMRR entries are owned by context rb
290 		 * tree.  Get rid of the failed entry, but do not stop
291 		 * the loop.  Rest of the parsed RMRR entries are
292 		 * loaded and removed on the context destruction.
293 		 */
294 		if (error1 == 0 && entry->end != entry->start) {
295 			IOMMU_LOCK(domain->iodom.iommu);
296 			domain->refs++; /* XXXKIB prevent free */
297 			domain->iodom.flags |= IOMMU_DOMAIN_RMRR;
298 			IOMMU_UNLOCK(domain->iodom.iommu);
299 		} else {
300 			if (error1 != 0) {
301 				if (dev != NULL)
302 					device_printf(dev, "");
303 				printf("pci%d:%d:%d ", bus, slot, func);
304 				printf(
305 			    "dmar%d failed to map RMRR region (%jx, %jx) %d\n",
306 				    domain->iodom.iommu->unit, start, end,
307 				    error1);
308 				error = error1;
309 			}
310 			TAILQ_REMOVE(&rmrr_entries, entry, unroll_link);
311 			iommu_gas_free_entry(DOM2IODOM(domain), entry);
312 		}
313 		for (i = 0; i < size; i++)
314 			vm_page_putfake(ma[i]);
315 		free(ma, M_TEMP);
316 	}
317 	return (error);
318 }
319 
320 static struct dmar_domain *
321 dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped)
322 {
323 	struct iommu_domain *iodom;
324 	struct iommu_unit *unit;
325 	struct dmar_domain *domain;
326 	int error, id, mgaw;
327 
328 	id = alloc_unr(dmar->domids);
329 	if (id == -1)
330 		return (NULL);
331 	domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO);
332 	iodom = DOM2IODOM(domain);
333 	unit = DMAR2IOMMU(dmar);
334 	domain->domain = id;
335 	LIST_INIT(&domain->contexts);
336 	iommu_domain_init(unit, iodom, &dmar_domain_map_ops);
337 
338 	domain->dmar = dmar;
339 
340 	/*
341 	 * For now, use the maximal usable physical address of the
342 	 * installed memory to calculate the mgaw on id_mapped domain.
343 	 * It is useful for the identity mapping, and less so for the
344 	 * virtualized bus address space.
345 	 */
346 	domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR;
347 	mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped);
348 	error = domain_set_agaw(domain, mgaw);
349 	if (error != 0)
350 		goto fail;
351 	if (!id_mapped)
352 		/* Use all supported address space for remapping. */
353 		domain->iodom.end = 1ULL << (domain->agaw - 1);
354 
355 	iommu_gas_init_domain(DOM2IODOM(domain));
356 
357 	if (id_mapped) {
358 		if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) {
359 			domain->pgtbl_obj = domain_get_idmap_pgtbl(domain,
360 			    domain->iodom.end);
361 		}
362 		domain->iodom.flags |= IOMMU_DOMAIN_IDMAP;
363 	} else {
364 		error = domain_alloc_pgtbl(domain);
365 		if (error != 0)
366 			goto fail;
367 		/* Disable local apic region access */
368 		error = iommu_gas_reserve_region(iodom, 0xfee00000,
369 		    0xfeefffff + 1);
370 		if (error != 0)
371 			goto fail;
372 	}
373 	return (domain);
374 
375 fail:
376 	dmar_domain_destroy(domain);
377 	return (NULL);
378 }
379 
380 static struct dmar_ctx *
381 dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid)
382 {
383 	struct dmar_ctx *ctx;
384 
385 	ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO);
386 	ctx->context.domain = DOM2IODOM(domain);
387 	ctx->context.tag = malloc(sizeof(struct bus_dma_tag_iommu),
388 	    M_DMAR_CTX, M_WAITOK | M_ZERO);
389 	ctx->rid = rid;
390 	ctx->refs = 1;
391 	return (ctx);
392 }
393 
394 static void
395 dmar_ctx_link(struct dmar_ctx *ctx)
396 {
397 	struct dmar_domain *domain;
398 
399 	domain = CTX2DOM(ctx);
400 	IOMMU_ASSERT_LOCKED(domain->iodom.iommu);
401 	KASSERT(domain->refs >= domain->ctx_cnt,
402 	    ("dom %p ref underflow %d %d", domain, domain->refs,
403 	    domain->ctx_cnt));
404 	domain->refs++;
405 	domain->ctx_cnt++;
406 	LIST_INSERT_HEAD(&domain->contexts, ctx, link);
407 }
408 
409 static void
410 dmar_ctx_unlink(struct dmar_ctx *ctx)
411 {
412 	struct dmar_domain *domain;
413 
414 	domain = CTX2DOM(ctx);
415 	IOMMU_ASSERT_LOCKED(domain->iodom.iommu);
416 	KASSERT(domain->refs > 0,
417 	    ("domain %p ctx dtr refs %d", domain, domain->refs));
418 	KASSERT(domain->ctx_cnt >= domain->refs,
419 	    ("domain %p ctx dtr refs %d ctx_cnt %d", domain,
420 	    domain->refs, domain->ctx_cnt));
421 	domain->refs--;
422 	domain->ctx_cnt--;
423 	LIST_REMOVE(ctx, link);
424 }
425 
426 static void
427 dmar_domain_destroy(struct dmar_domain *domain)
428 {
429 	struct iommu_domain *iodom;
430 	struct dmar_unit *dmar;
431 
432 	iodom = DOM2IODOM(domain);
433 
434 	KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries),
435 	    ("unfinished unloads %p", domain));
436 	KASSERT(LIST_EMPTY(&domain->contexts),
437 	    ("destroying dom %p with contexts", domain));
438 	KASSERT(domain->ctx_cnt == 0,
439 	    ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt));
440 	KASSERT(domain->refs == 0,
441 	    ("destroying dom %p with refs %d", domain, domain->refs));
442 	if ((domain->iodom.flags & IOMMU_DOMAIN_GAS_INITED) != 0) {
443 		DMAR_DOMAIN_LOCK(domain);
444 		iommu_gas_fini_domain(iodom);
445 		DMAR_DOMAIN_UNLOCK(domain);
446 	}
447 	if ((domain->iodom.flags & IOMMU_DOMAIN_PGTBL_INITED) != 0) {
448 		if (domain->pgtbl_obj != NULL)
449 			DMAR_DOMAIN_PGLOCK(domain);
450 		domain_free_pgtbl(domain);
451 	}
452 	iommu_domain_fini(iodom);
453 	dmar = DOM2DMAR(domain);
454 	free_unr(dmar->domids, domain->domain);
455 	free(domain, M_DMAR_DOMAIN);
456 }
457 
458 static struct dmar_ctx *
459 dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid,
460     int dev_domain, int dev_busno, const void *dev_path, int dev_path_len,
461     bool id_mapped, bool rmrr_init)
462 {
463 	struct dmar_domain *domain, *domain1;
464 	struct dmar_ctx *ctx, *ctx1;
465 	struct iommu_unit *unit;
466 	dmar_ctx_entry_t *ctxp;
467 	struct sf_buf *sf;
468 	int bus, slot, func, error;
469 	bool enable;
470 
471 	if (dev != NULL) {
472 		bus = pci_get_bus(dev);
473 		slot = pci_get_slot(dev);
474 		func = pci_get_function(dev);
475 	} else {
476 		bus = PCI_RID2BUS(rid);
477 		slot = PCI_RID2SLOT(rid);
478 		func = PCI_RID2FUNC(rid);
479 	}
480 	enable = false;
481 	TD_PREP_PINNED_ASSERT;
482 	unit = DMAR2IOMMU(dmar);
483 	DMAR_LOCK(dmar);
484 	KASSERT(!iommu_is_buswide_ctx(unit, bus) || (slot == 0 && func == 0),
485 	    ("iommu%d pci%d:%d:%d get_ctx for buswide", dmar->iommu.unit, bus,
486 	    slot, func));
487 	ctx = dmar_find_ctx_locked(dmar, rid);
488 	error = 0;
489 	if (ctx == NULL) {
490 		/*
491 		 * Perform the allocations which require sleep or have
492 		 * higher chance to succeed if the sleep is allowed.
493 		 */
494 		DMAR_UNLOCK(dmar);
495 		dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid));
496 		domain1 = dmar_domain_alloc(dmar, id_mapped);
497 		if (domain1 == NULL) {
498 			TD_PINNED_ASSERT;
499 			return (NULL);
500 		}
501 		if (!id_mapped) {
502 			error = domain_init_rmrr(domain1, dev, bus,
503 			    slot, func, dev_domain, dev_busno, dev_path,
504 			    dev_path_len);
505 			if (error != 0) {
506 				dmar_domain_destroy(domain1);
507 				TD_PINNED_ASSERT;
508 				return (NULL);
509 			}
510 		}
511 		ctx1 = dmar_ctx_alloc(domain1, rid);
512 		ctxp = dmar_map_ctx_entry(ctx1, &sf);
513 		DMAR_LOCK(dmar);
514 
515 		/*
516 		 * Recheck the contexts, other thread might have
517 		 * already allocated needed one.
518 		 */
519 		ctx = dmar_find_ctx_locked(dmar, rid);
520 		if (ctx == NULL) {
521 			domain = domain1;
522 			ctx = ctx1;
523 			dmar_ctx_link(ctx);
524 			ctx->context.tag->owner = dev;
525 			device_tag_init(ctx, dev);
526 
527 			/*
528 			 * This is the first activated context for the
529 			 * DMAR unit.  Enable the translation after
530 			 * everything is set up.
531 			 */
532 			if (LIST_EMPTY(&dmar->domains))
533 				enable = true;
534 			LIST_INSERT_HEAD(&dmar->domains, domain, link);
535 			ctx_id_entry_init(ctx, ctxp, false, bus);
536 			if (dev != NULL) {
537 				device_printf(dev,
538 			    "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d "
539 				    "agaw %d %s-mapped\n",
540 				    dmar->iommu.unit, dmar->segment, bus, slot,
541 				    func, rid, domain->domain, domain->mgaw,
542 				    domain->agaw, id_mapped ? "id" : "re");
543 			}
544 			dmar_unmap_pgtbl(sf);
545 		} else {
546 			dmar_unmap_pgtbl(sf);
547 			dmar_domain_destroy(domain1);
548 			/* Nothing needs to be done to destroy ctx1. */
549 			free(ctx1, M_DMAR_CTX);
550 			domain = CTX2DOM(ctx);
551 			ctx->refs++; /* tag referenced us */
552 		}
553 	} else {
554 		domain = CTX2DOM(ctx);
555 		if (ctx->context.tag->owner == NULL)
556 			ctx->context.tag->owner = dev;
557 		ctx->refs++; /* tag referenced us */
558 	}
559 
560 	error = dmar_flush_for_ctx_entry(dmar, enable);
561 	if (error != 0) {
562 		dmar_free_ctx_locked(dmar, ctx);
563 		TD_PINNED_ASSERT;
564 		return (NULL);
565 	}
566 
567 	/*
568 	 * The dmar lock was potentially dropped between check for the
569 	 * empty context list and now.  Recheck the state of GCMD_TE
570 	 * to avoid unneeded command.
571 	 */
572 	if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) {
573 		error = dmar_enable_translation(dmar);
574 		if (error == 0) {
575 			if (bootverbose) {
576 				printf("dmar%d: enabled translation\n",
577 				    dmar->iommu.unit);
578 			}
579 		} else {
580 			printf("dmar%d: enabling translation failed, "
581 			    "error %d\n", dmar->iommu.unit, error);
582 			dmar_free_ctx_locked(dmar, ctx);
583 			TD_PINNED_ASSERT;
584 			return (NULL);
585 		}
586 	}
587 	DMAR_UNLOCK(dmar);
588 	TD_PINNED_ASSERT;
589 	return (ctx);
590 }
591 
592 struct dmar_ctx *
593 dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid,
594     bool id_mapped, bool rmrr_init)
595 {
596 	int dev_domain, dev_path_len, dev_busno;
597 
598 	dev_domain = pci_get_domain(dev);
599 	dev_path_len = dmar_dev_depth(dev);
600 	ACPI_DMAR_PCI_PATH dev_path[dev_path_len];
601 	dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len);
602 	return (dmar_get_ctx_for_dev1(dmar, dev, rid, dev_domain, dev_busno,
603 	    dev_path, dev_path_len, id_mapped, rmrr_init));
604 }
605 
606 struct dmar_ctx *
607 dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid,
608     int dev_domain, int dev_busno,
609     const void *dev_path, int dev_path_len,
610     bool id_mapped, bool rmrr_init)
611 {
612 
613 	return (dmar_get_ctx_for_dev1(dmar, NULL, rid, dev_domain, dev_busno,
614 	    dev_path, dev_path_len, id_mapped, rmrr_init));
615 }
616 
617 int
618 dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx)
619 {
620 	struct dmar_unit *dmar;
621 	struct dmar_domain *old_domain;
622 	dmar_ctx_entry_t *ctxp;
623 	struct sf_buf *sf;
624 	int error;
625 
626 	dmar = domain->dmar;
627 	old_domain = CTX2DOM(ctx);
628 	if (domain == old_domain)
629 		return (0);
630 	KASSERT(old_domain->iodom.iommu == domain->iodom.iommu,
631 	    ("domain %p %u moving between dmars %u %u", domain,
632 	    domain->domain, old_domain->iodom.iommu->unit,
633 	    domain->iodom.iommu->unit));
634 	TD_PREP_PINNED_ASSERT;
635 
636 	ctxp = dmar_map_ctx_entry(ctx, &sf);
637 	DMAR_LOCK(dmar);
638 	dmar_ctx_unlink(ctx);
639 	ctx->context.domain = &domain->iodom;
640 	dmar_ctx_link(ctx);
641 	ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100);
642 	dmar_unmap_pgtbl(sf);
643 	error = dmar_flush_for_ctx_entry(dmar, true);
644 	/* If flush failed, rolling back would not work as well. */
645 	printf("dmar%d rid %x domain %d->%d %s-mapped\n",
646 	    dmar->iommu.unit, ctx->rid, old_domain->domain, domain->domain,
647 	    (domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 ? "id" : "re");
648 	dmar_unref_domain_locked(dmar, old_domain);
649 	TD_PINNED_ASSERT;
650 	return (error);
651 }
652 
653 static void
654 dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain)
655 {
656 
657 	DMAR_ASSERT_LOCKED(dmar);
658 	KASSERT(domain->refs >= 1,
659 	    ("dmar %d domain %p refs %u", dmar->iommu.unit, domain,
660 	    domain->refs));
661 	KASSERT(domain->refs > domain->ctx_cnt,
662 	    ("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain,
663 	    domain->refs, domain->ctx_cnt));
664 
665 	if (domain->refs > 1) {
666 		domain->refs--;
667 		DMAR_UNLOCK(dmar);
668 		return;
669 	}
670 
671 	KASSERT((domain->iodom.flags & IOMMU_DOMAIN_RMRR) == 0,
672 	    ("lost ref on RMRR domain %p", domain));
673 
674 	LIST_REMOVE(domain, link);
675 	DMAR_UNLOCK(dmar);
676 
677 	taskqueue_drain(dmar->iommu.delayed_taskqueue,
678 	    &domain->iodom.unload_task);
679 	dmar_domain_destroy(domain);
680 }
681 
682 void
683 dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx)
684 {
685 	struct sf_buf *sf;
686 	dmar_ctx_entry_t *ctxp;
687 	struct dmar_domain *domain;
688 
689 	DMAR_ASSERT_LOCKED(dmar);
690 	KASSERT(ctx->refs >= 1,
691 	    ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
692 
693 	/*
694 	 * If our reference is not last, only the dereference should
695 	 * be performed.
696 	 */
697 	if (ctx->refs > 1) {
698 		ctx->refs--;
699 		DMAR_UNLOCK(dmar);
700 		return;
701 	}
702 
703 	KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0,
704 	    ("lost ref on disabled ctx %p", ctx));
705 
706 	/*
707 	 * Otherwise, the context entry must be cleared before the
708 	 * page table is destroyed.  The mapping of the context
709 	 * entries page could require sleep, unlock the dmar.
710 	 */
711 	DMAR_UNLOCK(dmar);
712 	TD_PREP_PINNED_ASSERT;
713 	ctxp = dmar_map_ctx_entry(ctx, &sf);
714 	DMAR_LOCK(dmar);
715 	KASSERT(ctx->refs >= 1,
716 	    ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
717 
718 	/*
719 	 * Other thread might have referenced the context, in which
720 	 * case again only the dereference should be performed.
721 	 */
722 	if (ctx->refs > 1) {
723 		ctx->refs--;
724 		DMAR_UNLOCK(dmar);
725 		dmar_unmap_pgtbl(sf);
726 		TD_PINNED_ASSERT;
727 		return;
728 	}
729 
730 	KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0,
731 	    ("lost ref on disabled ctx %p", ctx));
732 
733 	/*
734 	 * Clear the context pointer and flush the caches.
735 	 * XXXKIB: cannot do this if any RMRR entries are still present.
736 	 */
737 	dmar_pte_clear(&ctxp->ctx1);
738 	ctxp->ctx2 = 0;
739 	dmar_flush_ctx_to_ram(dmar, ctxp);
740 	dmar_inv_ctx_glob(dmar);
741 	if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) {
742 		if (dmar->qi_enabled)
743 			dmar_qi_invalidate_iotlb_glob_locked(dmar);
744 		else
745 			dmar_inv_iotlb_glob(dmar);
746 	}
747 	dmar_unmap_pgtbl(sf);
748 	domain = CTX2DOM(ctx);
749 	dmar_ctx_unlink(ctx);
750 	free(ctx->context.tag, M_DMAR_CTX);
751 	free(ctx, M_DMAR_CTX);
752 	dmar_unref_domain_locked(dmar, domain);
753 	TD_PINNED_ASSERT;
754 }
755 
756 void
757 dmar_free_ctx(struct dmar_ctx *ctx)
758 {
759 	struct dmar_unit *dmar;
760 
761 	dmar = CTX2DMAR(ctx);
762 	DMAR_LOCK(dmar);
763 	dmar_free_ctx_locked(dmar, ctx);
764 }
765 
766 /*
767  * Returns with the domain locked.
768  */
769 struct dmar_ctx *
770 dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid)
771 {
772 	struct dmar_domain *domain;
773 	struct dmar_ctx *ctx;
774 
775 	DMAR_ASSERT_LOCKED(dmar);
776 
777 	LIST_FOREACH(domain, &dmar->domains, link) {
778 		LIST_FOREACH(ctx, &domain->contexts, link) {
779 			if (ctx->rid == rid)
780 				return (ctx);
781 		}
782 	}
783 	return (NULL);
784 }
785 
786 void
787 dmar_domain_free_entry(struct iommu_map_entry *entry, bool free)
788 {
789 	struct iommu_domain *domain;
790 
791 	domain = entry->domain;
792 	IOMMU_DOMAIN_LOCK(domain);
793 	if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0)
794 		iommu_gas_free_region(domain, entry);
795 	else
796 		iommu_gas_free_space(domain, entry);
797 	IOMMU_DOMAIN_UNLOCK(domain);
798 	if (free)
799 		iommu_gas_free_entry(domain, entry);
800 	else
801 		entry->flags = 0;
802 }
803 
804 void
805 dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free)
806 {
807 	struct dmar_domain *domain;
808 	struct dmar_unit *unit;
809 
810 	domain = IODOM2DOM(entry->domain);
811 	unit = DOM2DMAR(domain);
812 	if (unit->qi_enabled) {
813 		DMAR_LOCK(unit);
814 		dmar_qi_invalidate_locked(IODOM2DOM(entry->domain),
815 		    entry->start, entry->end - entry->start, &entry->gseq,
816 		    true);
817 		if (!free)
818 			entry->flags |= IOMMU_MAP_ENTRY_QI_NF;
819 		TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
820 		DMAR_UNLOCK(unit);
821 	} else {
822 		domain_flush_iotlb_sync(IODOM2DOM(entry->domain),
823 		    entry->start, entry->end - entry->start);
824 		dmar_domain_free_entry(entry, free);
825 	}
826 }
827 
828 static bool
829 dmar_domain_unload_emit_wait(struct dmar_domain *domain,
830     struct iommu_map_entry *entry)
831 {
832 
833 	if (TAILQ_NEXT(entry, dmamap_link) == NULL)
834 		return (true);
835 	return (domain->batch_no++ % dmar_batch_coalesce == 0);
836 }
837 
838 void
839 dmar_domain_unload(struct dmar_domain *domain,
840     struct iommu_map_entries_tailq *entries, bool cansleep)
841 {
842 	struct dmar_unit *unit;
843 	struct iommu_domain *iodom;
844 	struct iommu_map_entry *entry, *entry1;
845 	int error;
846 
847 	iodom = DOM2IODOM(domain);
848 	unit = DOM2DMAR(domain);
849 
850 	TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
851 		KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
852 		    ("not mapped entry %p %p", domain, entry));
853 		error = iodom->ops->unmap(iodom, entry->start, entry->end -
854 		    entry->start, cansleep ? IOMMU_PGF_WAITOK : 0);
855 		KASSERT(error == 0, ("unmap %p error %d", domain, error));
856 		if (!unit->qi_enabled) {
857 			domain_flush_iotlb_sync(domain, entry->start,
858 			    entry->end - entry->start);
859 			TAILQ_REMOVE(entries, entry, dmamap_link);
860 			dmar_domain_free_entry(entry, true);
861 		}
862 	}
863 	if (TAILQ_EMPTY(entries))
864 		return;
865 
866 	KASSERT(unit->qi_enabled, ("loaded entry left"));
867 	DMAR_LOCK(unit);
868 	TAILQ_FOREACH(entry, entries, dmamap_link) {
869 		dmar_qi_invalidate_locked(domain, entry->start, entry->end -
870 		    entry->start, &entry->gseq,
871 		    dmar_domain_unload_emit_wait(domain, entry));
872 	}
873 	TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link);
874 	DMAR_UNLOCK(unit);
875 }
876 
877 struct iommu_ctx *
878 iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid,
879     bool id_mapped, bool rmrr_init)
880 {
881 	struct dmar_unit *dmar;
882 	struct dmar_ctx *ret;
883 
884 	dmar = IOMMU2DMAR(iommu);
885 
886 	ret = dmar_get_ctx_for_dev(dmar, dev, rid, id_mapped, rmrr_init);
887 
888 	return (CTX2IOCTX(ret));
889 }
890 
891 void
892 iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context)
893 {
894 	struct dmar_unit *dmar;
895 	struct dmar_ctx *ctx;
896 
897 	dmar = IOMMU2DMAR(iommu);
898 	ctx = IOCTX2CTX(context);
899 
900 	dmar_free_ctx_locked(dmar, ctx);
901 }
902 
903 void
904 iommu_free_ctx(struct iommu_ctx *context)
905 {
906 	struct dmar_ctx *ctx;
907 
908 	ctx = IOCTX2CTX(context);
909 
910 	dmar_free_ctx(ctx);
911 }
912 
913 void
914 iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free)
915 {
916 
917 	dmar_domain_unload_entry(entry, free);
918 }
919 
920 void
921 iommu_domain_unload(struct iommu_domain *iodom,
922     struct iommu_map_entries_tailq *entries, bool cansleep)
923 {
924 	struct dmar_domain *domain;
925 
926 	domain = IODOM2DOM(iodom);
927 
928 	dmar_domain_unload(domain, entries, cansleep);
929 }
930