xref: /freebsd/sys/x86/iommu/intel_ctx.c (revision de7b456e596ff18032d2cbfdf244c66f36770da4)
1 /*-
2  * Copyright (c) 2013 The FreeBSD Foundation
3  * All rights reserved.
4  *
5  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6  * under sponsorship from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/malloc.h>
36 #include <sys/bus.h>
37 #include <sys/interrupt.h>
38 #include <sys/kernel.h>
39 #include <sys/ktr.h>
40 #include <sys/limits.h>
41 #include <sys/lock.h>
42 #include <sys/memdesc.h>
43 #include <sys/mutex.h>
44 #include <sys/proc.h>
45 #include <sys/rwlock.h>
46 #include <sys/rman.h>
47 #include <sys/sysctl.h>
48 #include <sys/taskqueue.h>
49 #include <sys/tree.h>
50 #include <sys/uio.h>
51 #include <vm/vm.h>
52 #include <vm/vm_extern.h>
53 #include <vm/vm_kern.h>
54 #include <vm/vm_object.h>
55 #include <vm/vm_page.h>
56 #include <vm/vm_pager.h>
57 #include <vm/vm_map.h>
58 #include <machine/atomic.h>
59 #include <machine/bus.h>
60 #include <machine/md_var.h>
61 #include <machine/specialreg.h>
62 #include <x86/include/busdma_impl.h>
63 #include <x86/iommu/intel_reg.h>
64 #include <x86/iommu/busdma_dmar.h>
65 #include <x86/iommu/intel_dmar.h>
66 #include <dev/pci/pcivar.h>
67 
68 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context");
69 
70 static void dmar_ctx_unload_task(void *arg, int pending);
71 
72 static void
73 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus)
74 {
75 	struct sf_buf *sf;
76 	dmar_root_entry_t *re;
77 	vm_page_t ctxm;
78 
79 	/*
80 	 * Allocated context page must be linked.
81 	 */
82 	ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_NOALLOC);
83 	if (ctxm != NULL)
84 		return;
85 
86 	/*
87 	 * Page not present, allocate and link.  Note that other
88 	 * thread might execute this sequence in parallel.  This
89 	 * should be safe, because the context entries written by both
90 	 * threads are equal.
91 	 */
92 	TD_PREP_PINNED_ASSERT;
93 	ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_ZERO |
94 	    DMAR_PGF_WAITOK);
95 	re = dmar_map_pgtbl(dmar->ctx_obj, 0, DMAR_PGF_NOALLOC, &sf);
96 	re += bus;
97 	dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK &
98 	    VM_PAGE_TO_PHYS(ctxm)));
99 	dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar));
100 	TD_PINNED_ASSERT;
101 }
102 
103 static dmar_ctx_entry_t *
104 dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp)
105 {
106 	dmar_ctx_entry_t *ctxp;
107 
108 	ctxp = dmar_map_pgtbl(ctx->dmar->ctx_obj, 1 + ctx->bus,
109 	    DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp);
110 	ctxp += ((ctx->slot & 0x1f) << 3) + (ctx->func & 0x7);
111 	return (ctxp);
112 }
113 
114 static void
115 ctx_tag_init(struct dmar_ctx *ctx)
116 {
117 	bus_addr_t maxaddr;
118 
119 	maxaddr = MIN(ctx->end, BUS_SPACE_MAXADDR);
120 	ctx->ctx_tag.common.ref_count = 1; /* Prevent free */
121 	ctx->ctx_tag.common.impl = &bus_dma_dmar_impl;
122 	ctx->ctx_tag.common.boundary = PCI_DMA_BOUNDARY;
123 	ctx->ctx_tag.common.lowaddr = maxaddr;
124 	ctx->ctx_tag.common.highaddr = maxaddr;
125 	ctx->ctx_tag.common.maxsize = maxaddr;
126 	ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED;
127 	ctx->ctx_tag.common.maxsegsz = maxaddr;
128 	ctx->ctx_tag.ctx = ctx;
129 	/* XXXKIB initialize tag further */
130 }
131 
132 static void
133 ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp)
134 {
135 	struct dmar_unit *unit;
136 	vm_page_t ctx_root;
137 
138 	unit = ctx->dmar;
139 	KASSERT(ctxp->ctx1 == 0 && ctxp->ctx2 == 0,
140 	    ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",
141 	    unit->unit, ctx->bus, ctx->slot, ctx->func, ctxp->ctx1,
142 	    ctxp->ctx2));
143 	ctxp->ctx2 = DMAR_CTX2_DID(ctx->domain);
144 	ctxp->ctx2 |= ctx->awlvl;
145 	if ((ctx->flags & DMAR_CTX_IDMAP) != 0 &&
146 	    (unit->hw_ecap & DMAR_ECAP_PT) != 0) {
147 		KASSERT(ctx->pgtbl_obj == NULL,
148 		    ("ctx %p non-null pgtbl_obj", ctx));
149 		dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P);
150 	} else {
151 		ctx_root = dmar_pgalloc(ctx->pgtbl_obj, 0, DMAR_PGF_NOALLOC);
152 		dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_UNTR |
153 		    (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) |
154 		    DMAR_CTX1_P);
155 	}
156 }
157 
158 static int
159 ctx_init_rmrr(struct dmar_ctx *ctx, device_t dev)
160 {
161 	struct dmar_map_entries_tailq rmrr_entries;
162 	struct dmar_map_entry *entry, *entry1;
163 	vm_page_t *ma;
164 	dmar_gaddr_t start, end;
165 	vm_pindex_t size, i;
166 	int error, error1;
167 
168 	error = 0;
169 	TAILQ_INIT(&rmrr_entries);
170 	dmar_ctx_parse_rmrr(ctx, dev, &rmrr_entries);
171 	TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) {
172 		/*
173 		 * VT-d specification requires that the start of an
174 		 * RMRR entry is 4k-aligned.  Buggy BIOSes put
175 		 * anything into the start and end fields.  Truncate
176 		 * and round as neccesary.
177 		 *
178 		 * We also allow the overlapping RMRR entries, see
179 		 * dmar_gas_alloc_region().
180 		 */
181 		start = entry->start;
182 		end = entry->end;
183 		entry->start = trunc_page(start);
184 		entry->end = round_page(end);
185 		size = OFF_TO_IDX(entry->end - entry->start);
186 		ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK);
187 		for (i = 0; i < size; i++) {
188 			ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
189 			    VM_MEMATTR_DEFAULT);
190 		}
191 		error1 = dmar_gas_map_region(ctx, entry, DMAR_MAP_ENTRY_READ |
192 		    DMAR_MAP_ENTRY_WRITE, DMAR_GM_CANWAIT, ma);
193 		/*
194 		 * Non-failed RMRR entries are owned by context rb
195 		 * tree.  Get rid of the failed entry, but do not stop
196 		 * the loop.  Rest of the parsed RMRR entries are
197 		 * loaded and removed on the context destruction.
198 		 */
199 		if (error1 == 0 && entry->end != entry->start) {
200 			DMAR_LOCK(ctx->dmar);
201 			ctx->flags |= DMAR_CTX_RMRR;
202 			DMAR_UNLOCK(ctx->dmar);
203 		} else {
204 			if (error1 != 0) {
205 				device_printf(dev,
206 			    "dmar%d failed to map RMRR region (%jx, %jx) %d\n",
207 				    ctx->dmar->unit, start, end, error1);
208 				error = error1;
209 			}
210 			TAILQ_REMOVE(&rmrr_entries, entry, unroll_link);
211 			dmar_gas_free_entry(ctx, entry);
212 		}
213 		for (i = 0; i < size; i++)
214 			vm_page_putfake(ma[i]);
215 		free(ma, M_TEMP);
216 	}
217 	return (error);
218 }
219 
220 static struct dmar_ctx *
221 dmar_get_ctx_alloc(struct dmar_unit *dmar, int bus, int slot, int func)
222 {
223 	struct dmar_ctx *ctx;
224 
225 	ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO);
226 	RB_INIT(&ctx->rb_root);
227 	TAILQ_INIT(&ctx->unload_entries);
228 	TASK_INIT(&ctx->unload_task, 0, dmar_ctx_unload_task, ctx);
229 	mtx_init(&ctx->lock, "dmarctx", NULL, MTX_DEF);
230 	ctx->dmar = dmar;
231 	ctx->bus = bus;
232 	ctx->slot = slot;
233 	ctx->func = func;
234 	return (ctx);
235 }
236 
237 static void
238 dmar_ctx_dtr(struct dmar_ctx *ctx, bool gas_inited, bool pgtbl_inited)
239 {
240 
241 	if (gas_inited) {
242 		DMAR_CTX_LOCK(ctx);
243 		dmar_gas_fini_ctx(ctx);
244 		DMAR_CTX_UNLOCK(ctx);
245 	}
246 	if (pgtbl_inited) {
247 		if (ctx->pgtbl_obj != NULL)
248 			DMAR_CTX_PGLOCK(ctx);
249 		ctx_free_pgtbl(ctx);
250 	}
251 	mtx_destroy(&ctx->lock);
252 	free(ctx, M_DMAR_CTX);
253 }
254 
255 struct dmar_ctx *
256 dmar_get_ctx(struct dmar_unit *dmar, device_t dev, bool id_mapped, bool rmrr_init)
257 {
258 	struct dmar_ctx *ctx, *ctx1;
259 	dmar_ctx_entry_t *ctxp;
260 	struct sf_buf *sf;
261 	int bus, slot, func, error, mgaw;
262 	bool enable;
263 
264 	bus = pci_get_bus(dev);
265 	slot = pci_get_slot(dev);
266 	func = pci_get_function(dev);
267 	enable = false;
268 	TD_PREP_PINNED_ASSERT;
269 	DMAR_LOCK(dmar);
270 	ctx = dmar_find_ctx_locked(dmar, bus, slot, func);
271 	error = 0;
272 	if (ctx == NULL) {
273 		/*
274 		 * Perform the allocations which require sleep or have
275 		 * higher chance to succeed if the sleep is allowed.
276 		 */
277 		DMAR_UNLOCK(dmar);
278 		dmar_ensure_ctx_page(dmar, bus);
279 		ctx1 = dmar_get_ctx_alloc(dmar, bus, slot, func);
280 
281 		if (id_mapped) {
282 			/*
283 			 * For now, use the maximal usable physical
284 			 * address of the installed memory to
285 			 * calculate the mgaw.  It is useful for the
286 			 * identity mapping, and less so for the
287 			 * virtualized bus address space.
288 			 */
289 			ctx1->end = ptoa(Maxmem);
290 			mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, false);
291 			error = ctx_set_agaw(ctx1, mgaw);
292 			if (error != 0) {
293 				dmar_ctx_dtr(ctx1, false, false);
294 				TD_PINNED_ASSERT;
295 				return (NULL);
296 			}
297 		} else {
298 			ctx1->end = BUS_SPACE_MAXADDR;
299 			mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, true);
300 			error = ctx_set_agaw(ctx1, mgaw);
301 			if (error != 0) {
302 				dmar_ctx_dtr(ctx1, false, false);
303 				TD_PINNED_ASSERT;
304 				return (NULL);
305 			}
306 			/* Use all supported address space for remapping. */
307 			ctx1->end = 1ULL << (ctx1->agaw - 1);
308 		}
309 
310 
311 		dmar_gas_init_ctx(ctx1);
312 		if (id_mapped) {
313 			if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) {
314 				ctx1->pgtbl_obj = ctx_get_idmap_pgtbl(ctx1,
315 				    ctx1->end);
316 			}
317 			ctx1->flags |= DMAR_CTX_IDMAP;
318 		} else {
319 			error = ctx_alloc_pgtbl(ctx1);
320 			if (error != 0) {
321 				dmar_ctx_dtr(ctx1, true, false);
322 				TD_PINNED_ASSERT;
323 				return (NULL);
324 			}
325 			/* Disable local apic region access */
326 			error = dmar_gas_reserve_region(ctx1, 0xfee00000,
327 			    0xfeefffff + 1);
328 			if (error != 0) {
329 				dmar_ctx_dtr(ctx1, true, true);
330 				TD_PINNED_ASSERT;
331 				return (NULL);
332 			}
333 			error = ctx_init_rmrr(ctx1, dev);
334 			if (error != 0) {
335 				dmar_ctx_dtr(ctx1, true, true);
336 				TD_PINNED_ASSERT;
337 				return (NULL);
338 			}
339 		}
340 		ctxp = dmar_map_ctx_entry(ctx1, &sf);
341 		DMAR_LOCK(dmar);
342 
343 		/*
344 		 * Recheck the contexts, other thread might have
345 		 * already allocated needed one.
346 		 */
347 		ctx = dmar_find_ctx_locked(dmar, bus, slot, func);
348 		if (ctx == NULL) {
349 			ctx = ctx1;
350 			ctx->domain = alloc_unrl(dmar->domids);
351 			if (ctx->domain == -1) {
352 				DMAR_UNLOCK(dmar);
353 				dmar_unmap_pgtbl(sf, true);
354 				dmar_ctx_dtr(ctx, true, true);
355 				TD_PINNED_ASSERT;
356 				return (NULL);
357 			}
358 			ctx_tag_init(ctx);
359 
360 			/*
361 			 * This is the first activated context for the
362 			 * DMAR unit.  Enable the translation after
363 			 * everything is set up.
364 			 */
365 			if (LIST_EMPTY(&dmar->contexts))
366 				enable = true;
367 			LIST_INSERT_HEAD(&dmar->contexts, ctx, link);
368 			ctx_id_entry_init(ctx, ctxp);
369 			device_printf(dev,
370 			    "dmar%d pci%d:%d:%d:%d domain %d mgaw %d agaw %d\n",
371 			    dmar->unit, dmar->segment, bus, slot,
372 			    func, ctx->domain, ctx->mgaw, ctx->agaw);
373 		} else {
374 			dmar_ctx_dtr(ctx1, true, true);
375 		}
376 		dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar));
377 	}
378 	ctx->refs++;
379 	if ((ctx->flags & DMAR_CTX_RMRR) != 0)
380 		ctx->refs++; /* XXXKIB */
381 
382 	/*
383 	 * If dmar declares Caching Mode as Set, follow 11.5 "Caching
384 	 * Mode Consideration" and do the (global) invalidation of the
385 	 * negative TLB entries.
386 	 */
387 	if ((dmar->hw_cap & DMAR_CAP_CM) != 0 || enable) {
388 		if (dmar->qi_enabled) {
389 			dmar_qi_invalidate_ctx_glob_locked(dmar);
390 			if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0)
391 				dmar_qi_invalidate_iotlb_glob_locked(dmar);
392 		} else {
393 			error = dmar_inv_ctx_glob(dmar);
394 			if (error == 0 &&
395 			    (dmar->hw_ecap & DMAR_ECAP_DI) != 0)
396 				error = dmar_inv_iotlb_glob(dmar);
397 			if (error != 0) {
398 				dmar_free_ctx_locked(dmar, ctx);
399 				TD_PINNED_ASSERT;
400 				return (NULL);
401 			}
402 		}
403 	}
404 
405 	/*
406 	 * The dmar lock was potentially dropped between check for the
407 	 * empty context list and now.  Recheck the state of GCMD_TE
408 	 * to avoid unneeded command.
409 	 */
410 	if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) {
411 		error = dmar_enable_translation(dmar);
412 		if (error != 0) {
413 			dmar_free_ctx_locked(dmar, ctx);
414 			TD_PINNED_ASSERT;
415 			return (NULL);
416 		}
417 	}
418 	DMAR_UNLOCK(dmar);
419 	TD_PINNED_ASSERT;
420 	return (ctx);
421 }
422 
423 void
424 dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx)
425 {
426 	struct sf_buf *sf;
427 	dmar_ctx_entry_t *ctxp;
428 
429 	DMAR_ASSERT_LOCKED(dmar);
430 	KASSERT(ctx->refs >= 1,
431 	    ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
432 
433 	/*
434 	 * If our reference is not last, only the dereference should
435 	 * be performed.
436 	 */
437 	if (ctx->refs > 1) {
438 		ctx->refs--;
439 		DMAR_UNLOCK(dmar);
440 		return;
441 	}
442 
443 	KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0,
444 	    ("lost ref on RMRR ctx %p", ctx));
445 	KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0,
446 	    ("lost ref on disabled ctx %p", ctx));
447 
448 	/*
449 	 * Otherwise, the context entry must be cleared before the
450 	 * page table is destroyed.  The mapping of the context
451 	 * entries page could require sleep, unlock the dmar.
452 	 */
453 	DMAR_UNLOCK(dmar);
454 	TD_PREP_PINNED_ASSERT;
455 	ctxp = dmar_map_ctx_entry(ctx, &sf);
456 	DMAR_LOCK(dmar);
457 	KASSERT(ctx->refs >= 1,
458 	    ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
459 
460 	/*
461 	 * Other thread might have referenced the context, in which
462 	 * case again only the dereference should be performed.
463 	 */
464 	if (ctx->refs > 1) {
465 		ctx->refs--;
466 		DMAR_UNLOCK(dmar);
467 		dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar));
468 		TD_PINNED_ASSERT;
469 		return;
470 	}
471 
472 	KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0,
473 	    ("lost ref on RMRR ctx %p", ctx));
474 	KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0,
475 	    ("lost ref on disabled ctx %p", ctx));
476 
477 	/*
478 	 * Clear the context pointer and flush the caches.
479 	 * XXXKIB: cannot do this if any RMRR entries are still present.
480 	 */
481 	dmar_pte_clear(&ctxp->ctx1);
482 	ctxp->ctx2 = 0;
483 	dmar_inv_ctx_glob(dmar);
484 	if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) {
485 		if (dmar->qi_enabled)
486 			dmar_qi_invalidate_iotlb_glob_locked(dmar);
487 		else
488 			dmar_inv_iotlb_glob(dmar);
489 	}
490 	LIST_REMOVE(ctx, link);
491 	DMAR_UNLOCK(dmar);
492 
493 	/*
494 	 * The rest of the destruction is invisible for other users of
495 	 * the dmar unit.
496 	 */
497 	taskqueue_drain(dmar->delayed_taskqueue, &ctx->unload_task);
498 	KASSERT(TAILQ_EMPTY(&ctx->unload_entries),
499 	    ("unfinished unloads %p", ctx));
500 	dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar));
501 	free_unr(dmar->domids, ctx->domain);
502 	dmar_ctx_dtr(ctx, true, true);
503 	TD_PINNED_ASSERT;
504 }
505 
506 void
507 dmar_free_ctx(struct dmar_ctx *ctx)
508 {
509 	struct dmar_unit *dmar;
510 
511 	dmar = ctx->dmar;
512 	DMAR_LOCK(dmar);
513 	dmar_free_ctx_locked(dmar, ctx);
514 }
515 
516 struct dmar_ctx *
517 dmar_find_ctx_locked(struct dmar_unit *dmar, int bus, int slot, int func)
518 {
519 	struct dmar_ctx *ctx;
520 
521 	DMAR_ASSERT_LOCKED(dmar);
522 
523 	LIST_FOREACH(ctx, &dmar->contexts, link) {
524 		if (ctx->bus == bus && ctx->slot == slot && ctx->func == func)
525 			return (ctx);
526 	}
527 	return (NULL);
528 }
529 
530 void
531 dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free)
532 {
533 	struct dmar_ctx *ctx;
534 
535 	ctx = entry->ctx;
536 	DMAR_CTX_LOCK(ctx);
537 	if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0)
538 		dmar_gas_free_region(ctx, entry);
539 	else
540 		dmar_gas_free_space(ctx, entry);
541 	DMAR_CTX_UNLOCK(ctx);
542 	if (free)
543 		dmar_gas_free_entry(ctx, entry);
544 	else
545 		entry->flags = 0;
546 }
547 
548 void
549 dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free)
550 {
551 	struct dmar_unit *unit;
552 
553 	unit = entry->ctx->dmar;
554 	if (unit->qi_enabled) {
555 		DMAR_LOCK(unit);
556 		dmar_qi_invalidate_locked(entry->ctx, entry->start,
557 		    entry->end - entry->start, &entry->gseq);
558 		if (!free)
559 			entry->flags |= DMAR_MAP_ENTRY_QI_NF;
560 		TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
561 		DMAR_UNLOCK(unit);
562 	} else {
563 		ctx_flush_iotlb_sync(entry->ctx, entry->start, entry->end -
564 		    entry->start);
565 		dmar_ctx_free_entry(entry, free);
566 	}
567 }
568 
569 void
570 dmar_ctx_unload(struct dmar_ctx *ctx, struct dmar_map_entries_tailq *entries,
571     bool cansleep)
572 {
573 	struct dmar_unit *unit;
574 	struct dmar_map_entry *entry, *entry1;
575 	struct dmar_qi_genseq gseq;
576 	int error;
577 
578 	unit = ctx->dmar;
579 
580 	TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
581 		KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0,
582 		    ("not mapped entry %p %p", ctx, entry));
583 		error = ctx_unmap_buf(ctx, entry->start, entry->end -
584 		    entry->start, cansleep ? DMAR_PGF_WAITOK : 0);
585 		KASSERT(error == 0, ("unmap %p error %d", ctx, error));
586 		if (!unit->qi_enabled) {
587 			ctx_flush_iotlb_sync(ctx, entry->start,
588 			    entry->end - entry->start);
589 			TAILQ_REMOVE(entries, entry, dmamap_link);
590 			dmar_ctx_free_entry(entry, true);
591 		}
592 	}
593 	if (TAILQ_EMPTY(entries))
594 		return;
595 
596 	KASSERT(unit->qi_enabled, ("loaded entry left"));
597 	DMAR_LOCK(unit);
598 	TAILQ_FOREACH(entry, entries, dmamap_link) {
599 		entry->gseq.gen = 0;
600 		entry->gseq.seq = 0;
601 		dmar_qi_invalidate_locked(ctx, entry->start, entry->end -
602 		    entry->start, TAILQ_NEXT(entry, dmamap_link) == NULL ?
603 		    &gseq : NULL);
604 	}
605 	TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
606 		entry->gseq = gseq;
607 		TAILQ_REMOVE(entries, entry, dmamap_link);
608 		TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
609 	}
610 	DMAR_UNLOCK(unit);
611 }
612 
613 static void
614 dmar_ctx_unload_task(void *arg, int pending)
615 {
616 	struct dmar_ctx *ctx;
617 	struct dmar_map_entries_tailq entries;
618 
619 	ctx = arg;
620 	TAILQ_INIT(&entries);
621 
622 	for (;;) {
623 		DMAR_CTX_LOCK(ctx);
624 		TAILQ_SWAP(&ctx->unload_entries, &entries, dmar_map_entry,
625 		    dmamap_link);
626 		DMAR_CTX_UNLOCK(ctx);
627 		if (TAILQ_EMPTY(&entries))
628 			break;
629 		dmar_ctx_unload(ctx, &entries, true);
630 	}
631 }
632