xref: /freebsd/sys/x86/iommu/intel_ctx.c (revision b29d186cb97eb86c207d689775a83b43e6d7e96e)
186be9f0dSKonstantin Belousov /*-
286be9f0dSKonstantin Belousov  * Copyright (c) 2013 The FreeBSD Foundation
386be9f0dSKonstantin Belousov  * All rights reserved.
486be9f0dSKonstantin Belousov  *
586be9f0dSKonstantin Belousov  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
686be9f0dSKonstantin Belousov  * under sponsorship from the FreeBSD Foundation.
786be9f0dSKonstantin Belousov  *
886be9f0dSKonstantin Belousov  * Redistribution and use in source and binary forms, with or without
986be9f0dSKonstantin Belousov  * modification, are permitted provided that the following conditions
1086be9f0dSKonstantin Belousov  * are met:
1186be9f0dSKonstantin Belousov  * 1. Redistributions of source code must retain the above copyright
1286be9f0dSKonstantin Belousov  *    notice, this list of conditions and the following disclaimer.
1386be9f0dSKonstantin Belousov  * 2. Redistributions in binary form must reproduce the above copyright
1486be9f0dSKonstantin Belousov  *    notice, this list of conditions and the following disclaimer in the
1586be9f0dSKonstantin Belousov  *    documentation and/or other materials provided with the distribution.
1686be9f0dSKonstantin Belousov  *
1786be9f0dSKonstantin Belousov  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
1886be9f0dSKonstantin Belousov  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1986be9f0dSKonstantin Belousov  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2086be9f0dSKonstantin Belousov  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
2186be9f0dSKonstantin Belousov  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
2286be9f0dSKonstantin Belousov  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
2386be9f0dSKonstantin Belousov  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2486be9f0dSKonstantin Belousov  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2586be9f0dSKonstantin Belousov  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2686be9f0dSKonstantin Belousov  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2786be9f0dSKonstantin Belousov  * SUCH DAMAGE.
2886be9f0dSKonstantin Belousov  */
2986be9f0dSKonstantin Belousov 
3086be9f0dSKonstantin Belousov #include <sys/cdefs.h>
3186be9f0dSKonstantin Belousov __FBSDID("$FreeBSD$");
3286be9f0dSKonstantin Belousov 
3386be9f0dSKonstantin Belousov #include <sys/param.h>
3486be9f0dSKonstantin Belousov #include <sys/systm.h>
3586be9f0dSKonstantin Belousov #include <sys/malloc.h>
3686be9f0dSKonstantin Belousov #include <sys/bus.h>
3786be9f0dSKonstantin Belousov #include <sys/interrupt.h>
3886be9f0dSKonstantin Belousov #include <sys/kernel.h>
3986be9f0dSKonstantin Belousov #include <sys/ktr.h>
4086be9f0dSKonstantin Belousov #include <sys/limits.h>
4186be9f0dSKonstantin Belousov #include <sys/lock.h>
4286be9f0dSKonstantin Belousov #include <sys/memdesc.h>
4386be9f0dSKonstantin Belousov #include <sys/mutex.h>
4486be9f0dSKonstantin Belousov #include <sys/proc.h>
4586be9f0dSKonstantin Belousov #include <sys/rwlock.h>
4686be9f0dSKonstantin Belousov #include <sys/rman.h>
4786be9f0dSKonstantin Belousov #include <sys/sysctl.h>
4886be9f0dSKonstantin Belousov #include <sys/taskqueue.h>
4986be9f0dSKonstantin Belousov #include <sys/tree.h>
5086be9f0dSKonstantin Belousov #include <sys/uio.h>
5186be9f0dSKonstantin Belousov #include <vm/vm.h>
5286be9f0dSKonstantin Belousov #include <vm/vm_extern.h>
5386be9f0dSKonstantin Belousov #include <vm/vm_kern.h>
5486be9f0dSKonstantin Belousov #include <vm/vm_object.h>
5586be9f0dSKonstantin Belousov #include <vm/vm_page.h>
5686be9f0dSKonstantin Belousov #include <vm/vm_pager.h>
5786be9f0dSKonstantin Belousov #include <vm/vm_map.h>
5886be9f0dSKonstantin Belousov #include <machine/atomic.h>
5986be9f0dSKonstantin Belousov #include <machine/bus.h>
6086be9f0dSKonstantin Belousov #include <machine/md_var.h>
6186be9f0dSKonstantin Belousov #include <machine/specialreg.h>
6286be9f0dSKonstantin Belousov #include <x86/include/busdma_impl.h>
6386be9f0dSKonstantin Belousov #include <x86/iommu/intel_reg.h>
6486be9f0dSKonstantin Belousov #include <x86/iommu/busdma_dmar.h>
6586be9f0dSKonstantin Belousov #include <x86/iommu/intel_dmar.h>
6667499354SRyan Stone #include <dev/pci/pcireg.h>
6786be9f0dSKonstantin Belousov #include <dev/pci/pcivar.h>
6886be9f0dSKonstantin Belousov 
6986be9f0dSKonstantin Belousov static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context");
7086be9f0dSKonstantin Belousov 
7186be9f0dSKonstantin Belousov static void dmar_ctx_unload_task(void *arg, int pending);
7286be9f0dSKonstantin Belousov 
7386be9f0dSKonstantin Belousov static void
7486be9f0dSKonstantin Belousov dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus)
7586be9f0dSKonstantin Belousov {
7686be9f0dSKonstantin Belousov 	struct sf_buf *sf;
7786be9f0dSKonstantin Belousov 	dmar_root_entry_t *re;
7886be9f0dSKonstantin Belousov 	vm_page_t ctxm;
7986be9f0dSKonstantin Belousov 
8086be9f0dSKonstantin Belousov 	/*
8186be9f0dSKonstantin Belousov 	 * Allocated context page must be linked.
8286be9f0dSKonstantin Belousov 	 */
8386be9f0dSKonstantin Belousov 	ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_NOALLOC);
8486be9f0dSKonstantin Belousov 	if (ctxm != NULL)
8586be9f0dSKonstantin Belousov 		return;
8686be9f0dSKonstantin Belousov 
8786be9f0dSKonstantin Belousov 	/*
8886be9f0dSKonstantin Belousov 	 * Page not present, allocate and link.  Note that other
8986be9f0dSKonstantin Belousov 	 * thread might execute this sequence in parallel.  This
9086be9f0dSKonstantin Belousov 	 * should be safe, because the context entries written by both
9186be9f0dSKonstantin Belousov 	 * threads are equal.
9286be9f0dSKonstantin Belousov 	 */
9386be9f0dSKonstantin Belousov 	TD_PREP_PINNED_ASSERT;
9486be9f0dSKonstantin Belousov 	ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_ZERO |
9586be9f0dSKonstantin Belousov 	    DMAR_PGF_WAITOK);
9686be9f0dSKonstantin Belousov 	re = dmar_map_pgtbl(dmar->ctx_obj, 0, DMAR_PGF_NOALLOC, &sf);
9786be9f0dSKonstantin Belousov 	re += bus;
9886be9f0dSKonstantin Belousov 	dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK &
9986be9f0dSKonstantin Belousov 	    VM_PAGE_TO_PHYS(ctxm)));
10086be9f0dSKonstantin Belousov 	dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar));
10186be9f0dSKonstantin Belousov 	TD_PINNED_ASSERT;
10286be9f0dSKonstantin Belousov }
10386be9f0dSKonstantin Belousov 
10486be9f0dSKonstantin Belousov static dmar_ctx_entry_t *
10586be9f0dSKonstantin Belousov dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp)
10686be9f0dSKonstantin Belousov {
10786be9f0dSKonstantin Belousov 	dmar_ctx_entry_t *ctxp;
10886be9f0dSKonstantin Belousov 
10967499354SRyan Stone 	ctxp = dmar_map_pgtbl(ctx->dmar->ctx_obj, 1 + PCI_RID2BUS(ctx->rid),
11086be9f0dSKonstantin Belousov 	    DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp);
11167499354SRyan Stone 	ctxp += ctx->rid & 0xff;
11286be9f0dSKonstantin Belousov 	return (ctxp);
11386be9f0dSKonstantin Belousov }
11486be9f0dSKonstantin Belousov 
11586be9f0dSKonstantin Belousov static void
11667499354SRyan Stone ctx_tag_init(struct dmar_ctx *ctx, device_t dev)
11786be9f0dSKonstantin Belousov {
11886be9f0dSKonstantin Belousov 	bus_addr_t maxaddr;
11986be9f0dSKonstantin Belousov 
12086be9f0dSKonstantin Belousov 	maxaddr = MIN(ctx->end, BUS_SPACE_MAXADDR);
12186be9f0dSKonstantin Belousov 	ctx->ctx_tag.common.ref_count = 1; /* Prevent free */
12286be9f0dSKonstantin Belousov 	ctx->ctx_tag.common.impl = &bus_dma_dmar_impl;
12386be9f0dSKonstantin Belousov 	ctx->ctx_tag.common.boundary = PCI_DMA_BOUNDARY;
12486be9f0dSKonstantin Belousov 	ctx->ctx_tag.common.lowaddr = maxaddr;
12586be9f0dSKonstantin Belousov 	ctx->ctx_tag.common.highaddr = maxaddr;
12686be9f0dSKonstantin Belousov 	ctx->ctx_tag.common.maxsize = maxaddr;
12786be9f0dSKonstantin Belousov 	ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED;
12886be9f0dSKonstantin Belousov 	ctx->ctx_tag.common.maxsegsz = maxaddr;
12986be9f0dSKonstantin Belousov 	ctx->ctx_tag.ctx = ctx;
13067499354SRyan Stone 	ctx->ctx_tag.owner = dev;
13186be9f0dSKonstantin Belousov 	/* XXXKIB initialize tag further */
13286be9f0dSKonstantin Belousov }
13386be9f0dSKonstantin Belousov 
13486be9f0dSKonstantin Belousov static void
13586be9f0dSKonstantin Belousov ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp)
13686be9f0dSKonstantin Belousov {
13786be9f0dSKonstantin Belousov 	struct dmar_unit *unit;
13886be9f0dSKonstantin Belousov 	vm_page_t ctx_root;
13986be9f0dSKonstantin Belousov 
14086be9f0dSKonstantin Belousov 	unit = ctx->dmar;
14186be9f0dSKonstantin Belousov 	KASSERT(ctxp->ctx1 == 0 && ctxp->ctx2 == 0,
14286be9f0dSKonstantin Belousov 	    ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",
14367499354SRyan Stone 	    unit->unit, pci_get_bus(ctx->ctx_tag.owner),
14467499354SRyan Stone 	    pci_get_slot(ctx->ctx_tag.owner),
14567499354SRyan Stone 	    pci_get_function(ctx->ctx_tag.owner),
14667499354SRyan Stone 	    ctxp->ctx1,
14786be9f0dSKonstantin Belousov 	    ctxp->ctx2));
14886be9f0dSKonstantin Belousov 	ctxp->ctx2 = DMAR_CTX2_DID(ctx->domain);
14986be9f0dSKonstantin Belousov 	ctxp->ctx2 |= ctx->awlvl;
15086be9f0dSKonstantin Belousov 	if ((ctx->flags & DMAR_CTX_IDMAP) != 0 &&
15186be9f0dSKonstantin Belousov 	    (unit->hw_ecap & DMAR_ECAP_PT) != 0) {
15286be9f0dSKonstantin Belousov 		KASSERT(ctx->pgtbl_obj == NULL,
15386be9f0dSKonstantin Belousov 		    ("ctx %p non-null pgtbl_obj", ctx));
15486be9f0dSKonstantin Belousov 		dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P);
15586be9f0dSKonstantin Belousov 	} else {
15686be9f0dSKonstantin Belousov 		ctx_root = dmar_pgalloc(ctx->pgtbl_obj, 0, DMAR_PGF_NOALLOC);
15786be9f0dSKonstantin Belousov 		dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_UNTR |
15886be9f0dSKonstantin Belousov 		    (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) |
15986be9f0dSKonstantin Belousov 		    DMAR_CTX1_P);
16086be9f0dSKonstantin Belousov 	}
16186be9f0dSKonstantin Belousov }
16286be9f0dSKonstantin Belousov 
16386be9f0dSKonstantin Belousov static int
16486be9f0dSKonstantin Belousov ctx_init_rmrr(struct dmar_ctx *ctx, device_t dev)
16586be9f0dSKonstantin Belousov {
16686be9f0dSKonstantin Belousov 	struct dmar_map_entries_tailq rmrr_entries;
16786be9f0dSKonstantin Belousov 	struct dmar_map_entry *entry, *entry1;
16886be9f0dSKonstantin Belousov 	vm_page_t *ma;
16986be9f0dSKonstantin Belousov 	dmar_gaddr_t start, end;
17086be9f0dSKonstantin Belousov 	vm_pindex_t size, i;
17186be9f0dSKonstantin Belousov 	int error, error1;
17286be9f0dSKonstantin Belousov 
17386be9f0dSKonstantin Belousov 	error = 0;
17486be9f0dSKonstantin Belousov 	TAILQ_INIT(&rmrr_entries);
17586be9f0dSKonstantin Belousov 	dmar_ctx_parse_rmrr(ctx, dev, &rmrr_entries);
17686be9f0dSKonstantin Belousov 	TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) {
17786be9f0dSKonstantin Belousov 		/*
17886be9f0dSKonstantin Belousov 		 * VT-d specification requires that the start of an
17986be9f0dSKonstantin Belousov 		 * RMRR entry is 4k-aligned.  Buggy BIOSes put
18086be9f0dSKonstantin Belousov 		 * anything into the start and end fields.  Truncate
18186be9f0dSKonstantin Belousov 		 * and round as neccesary.
18286be9f0dSKonstantin Belousov 		 *
18386be9f0dSKonstantin Belousov 		 * We also allow the overlapping RMRR entries, see
18486be9f0dSKonstantin Belousov 		 * dmar_gas_alloc_region().
18586be9f0dSKonstantin Belousov 		 */
18686be9f0dSKonstantin Belousov 		start = entry->start;
18786be9f0dSKonstantin Belousov 		end = entry->end;
18886be9f0dSKonstantin Belousov 		entry->start = trunc_page(start);
18986be9f0dSKonstantin Belousov 		entry->end = round_page(end);
190e02b05b3SKonstantin Belousov 		if (entry->start == entry->end) {
191e02b05b3SKonstantin Belousov 			/* Workaround for some AMI (?) BIOSes */
192e02b05b3SKonstantin Belousov 			if (bootverbose) {
193e02b05b3SKonstantin Belousov 				device_printf(dev, "BIOS bug: dmar%d RMRR "
194e02b05b3SKonstantin Belousov 				    "region (%jx, %jx) corrected\n",
195e02b05b3SKonstantin Belousov 				    ctx->dmar->unit, start, end);
196e02b05b3SKonstantin Belousov 			}
197e02b05b3SKonstantin Belousov 			entry->end += DMAR_PAGE_SIZE * 0x20;
198e02b05b3SKonstantin Belousov 		}
19986be9f0dSKonstantin Belousov 		size = OFF_TO_IDX(entry->end - entry->start);
20086be9f0dSKonstantin Belousov 		ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK);
20186be9f0dSKonstantin Belousov 		for (i = 0; i < size; i++) {
20286be9f0dSKonstantin Belousov 			ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
20386be9f0dSKonstantin Belousov 			    VM_MEMATTR_DEFAULT);
20486be9f0dSKonstantin Belousov 		}
20586be9f0dSKonstantin Belousov 		error1 = dmar_gas_map_region(ctx, entry, DMAR_MAP_ENTRY_READ |
20686be9f0dSKonstantin Belousov 		    DMAR_MAP_ENTRY_WRITE, DMAR_GM_CANWAIT, ma);
20786be9f0dSKonstantin Belousov 		/*
20886be9f0dSKonstantin Belousov 		 * Non-failed RMRR entries are owned by context rb
20986be9f0dSKonstantin Belousov 		 * tree.  Get rid of the failed entry, but do not stop
21086be9f0dSKonstantin Belousov 		 * the loop.  Rest of the parsed RMRR entries are
21186be9f0dSKonstantin Belousov 		 * loaded and removed on the context destruction.
21286be9f0dSKonstantin Belousov 		 */
21386be9f0dSKonstantin Belousov 		if (error1 == 0 && entry->end != entry->start) {
21486be9f0dSKonstantin Belousov 			DMAR_LOCK(ctx->dmar);
21586be9f0dSKonstantin Belousov 			ctx->flags |= DMAR_CTX_RMRR;
21686be9f0dSKonstantin Belousov 			DMAR_UNLOCK(ctx->dmar);
21786be9f0dSKonstantin Belousov 		} else {
21886be9f0dSKonstantin Belousov 			if (error1 != 0) {
21986be9f0dSKonstantin Belousov 				device_printf(dev,
22086be9f0dSKonstantin Belousov 			    "dmar%d failed to map RMRR region (%jx, %jx) %d\n",
22186be9f0dSKonstantin Belousov 				    ctx->dmar->unit, start, end, error1);
22286be9f0dSKonstantin Belousov 				error = error1;
22386be9f0dSKonstantin Belousov 			}
22486be9f0dSKonstantin Belousov 			TAILQ_REMOVE(&rmrr_entries, entry, unroll_link);
22586be9f0dSKonstantin Belousov 			dmar_gas_free_entry(ctx, entry);
22686be9f0dSKonstantin Belousov 		}
22786be9f0dSKonstantin Belousov 		for (i = 0; i < size; i++)
22886be9f0dSKonstantin Belousov 			vm_page_putfake(ma[i]);
22986be9f0dSKonstantin Belousov 		free(ma, M_TEMP);
23086be9f0dSKonstantin Belousov 	}
23186be9f0dSKonstantin Belousov 	return (error);
23286be9f0dSKonstantin Belousov }
23386be9f0dSKonstantin Belousov 
23486be9f0dSKonstantin Belousov static struct dmar_ctx *
23567499354SRyan Stone dmar_get_ctx_alloc(struct dmar_unit *dmar, uint16_t rid)
23686be9f0dSKonstantin Belousov {
23786be9f0dSKonstantin Belousov 	struct dmar_ctx *ctx;
23886be9f0dSKonstantin Belousov 
23986be9f0dSKonstantin Belousov 	ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO);
24086be9f0dSKonstantin Belousov 	RB_INIT(&ctx->rb_root);
24186be9f0dSKonstantin Belousov 	TAILQ_INIT(&ctx->unload_entries);
24286be9f0dSKonstantin Belousov 	TASK_INIT(&ctx->unload_task, 0, dmar_ctx_unload_task, ctx);
24386be9f0dSKonstantin Belousov 	mtx_init(&ctx->lock, "dmarctx", NULL, MTX_DEF);
24486be9f0dSKonstantin Belousov 	ctx->dmar = dmar;
24567499354SRyan Stone 	ctx->rid = rid;
24686be9f0dSKonstantin Belousov 	return (ctx);
24786be9f0dSKonstantin Belousov }
24886be9f0dSKonstantin Belousov 
24986be9f0dSKonstantin Belousov static void
25086be9f0dSKonstantin Belousov dmar_ctx_dtr(struct dmar_ctx *ctx, bool gas_inited, bool pgtbl_inited)
25186be9f0dSKonstantin Belousov {
25286be9f0dSKonstantin Belousov 
25386be9f0dSKonstantin Belousov 	if (gas_inited) {
25486be9f0dSKonstantin Belousov 		DMAR_CTX_LOCK(ctx);
25586be9f0dSKonstantin Belousov 		dmar_gas_fini_ctx(ctx);
25686be9f0dSKonstantin Belousov 		DMAR_CTX_UNLOCK(ctx);
25786be9f0dSKonstantin Belousov 	}
25886be9f0dSKonstantin Belousov 	if (pgtbl_inited) {
25986be9f0dSKonstantin Belousov 		if (ctx->pgtbl_obj != NULL)
26086be9f0dSKonstantin Belousov 			DMAR_CTX_PGLOCK(ctx);
26186be9f0dSKonstantin Belousov 		ctx_free_pgtbl(ctx);
26286be9f0dSKonstantin Belousov 	}
26386be9f0dSKonstantin Belousov 	mtx_destroy(&ctx->lock);
26486be9f0dSKonstantin Belousov 	free(ctx, M_DMAR_CTX);
26586be9f0dSKonstantin Belousov }
26686be9f0dSKonstantin Belousov 
26786be9f0dSKonstantin Belousov struct dmar_ctx *
26867499354SRyan Stone dmar_get_ctx(struct dmar_unit *dmar, device_t dev, uint16_t rid, bool id_mapped,
26967499354SRyan Stone     bool rmrr_init)
27086be9f0dSKonstantin Belousov {
27186be9f0dSKonstantin Belousov 	struct dmar_ctx *ctx, *ctx1;
27286be9f0dSKonstantin Belousov 	dmar_ctx_entry_t *ctxp;
27386be9f0dSKonstantin Belousov 	struct sf_buf *sf;
27467499354SRyan Stone 	int bus, slot, func, error, mgaw;
27586be9f0dSKonstantin Belousov 	bool enable;
27686be9f0dSKonstantin Belousov 
27767499354SRyan Stone 	bus = pci_get_bus(dev);
27867499354SRyan Stone 	slot = pci_get_slot(dev);
27967499354SRyan Stone 	func = pci_get_function(dev);
28086be9f0dSKonstantin Belousov 	enable = false;
28186be9f0dSKonstantin Belousov 	TD_PREP_PINNED_ASSERT;
28286be9f0dSKonstantin Belousov 	DMAR_LOCK(dmar);
28367499354SRyan Stone 	ctx = dmar_find_ctx_locked(dmar, rid);
28486be9f0dSKonstantin Belousov 	error = 0;
28586be9f0dSKonstantin Belousov 	if (ctx == NULL) {
28686be9f0dSKonstantin Belousov 		/*
28786be9f0dSKonstantin Belousov 		 * Perform the allocations which require sleep or have
28886be9f0dSKonstantin Belousov 		 * higher chance to succeed if the sleep is allowed.
28986be9f0dSKonstantin Belousov 		 */
29086be9f0dSKonstantin Belousov 		DMAR_UNLOCK(dmar);
291*b29d186cSKonstantin Belousov 		dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid));
29267499354SRyan Stone 		ctx1 = dmar_get_ctx_alloc(dmar, rid);
29386be9f0dSKonstantin Belousov 
29486be9f0dSKonstantin Belousov 		if (id_mapped) {
29586be9f0dSKonstantin Belousov 			/*
29686be9f0dSKonstantin Belousov 			 * For now, use the maximal usable physical
29786be9f0dSKonstantin Belousov 			 * address of the installed memory to
29886be9f0dSKonstantin Belousov 			 * calculate the mgaw.  It is useful for the
29986be9f0dSKonstantin Belousov 			 * identity mapping, and less so for the
30086be9f0dSKonstantin Belousov 			 * virtualized bus address space.
30186be9f0dSKonstantin Belousov 			 */
30286be9f0dSKonstantin Belousov 			ctx1->end = ptoa(Maxmem);
30386be9f0dSKonstantin Belousov 			mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, false);
30486be9f0dSKonstantin Belousov 			error = ctx_set_agaw(ctx1, mgaw);
30586be9f0dSKonstantin Belousov 			if (error != 0) {
30686be9f0dSKonstantin Belousov 				dmar_ctx_dtr(ctx1, false, false);
30786be9f0dSKonstantin Belousov 				TD_PINNED_ASSERT;
30886be9f0dSKonstantin Belousov 				return (NULL);
30986be9f0dSKonstantin Belousov 			}
31086be9f0dSKonstantin Belousov 		} else {
31186be9f0dSKonstantin Belousov 			ctx1->end = BUS_SPACE_MAXADDR;
31286be9f0dSKonstantin Belousov 			mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, true);
31386be9f0dSKonstantin Belousov 			error = ctx_set_agaw(ctx1, mgaw);
31486be9f0dSKonstantin Belousov 			if (error != 0) {
31586be9f0dSKonstantin Belousov 				dmar_ctx_dtr(ctx1, false, false);
31686be9f0dSKonstantin Belousov 				TD_PINNED_ASSERT;
31786be9f0dSKonstantin Belousov 				return (NULL);
31886be9f0dSKonstantin Belousov 			}
31986be9f0dSKonstantin Belousov 			/* Use all supported address space for remapping. */
32086be9f0dSKonstantin Belousov 			ctx1->end = 1ULL << (ctx1->agaw - 1);
32186be9f0dSKonstantin Belousov 		}
32286be9f0dSKonstantin Belousov 
32386be9f0dSKonstantin Belousov 
32486be9f0dSKonstantin Belousov 		dmar_gas_init_ctx(ctx1);
32586be9f0dSKonstantin Belousov 		if (id_mapped) {
32686be9f0dSKonstantin Belousov 			if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) {
32786be9f0dSKonstantin Belousov 				ctx1->pgtbl_obj = ctx_get_idmap_pgtbl(ctx1,
32886be9f0dSKonstantin Belousov 				    ctx1->end);
32986be9f0dSKonstantin Belousov 			}
33086be9f0dSKonstantin Belousov 			ctx1->flags |= DMAR_CTX_IDMAP;
33186be9f0dSKonstantin Belousov 		} else {
33286be9f0dSKonstantin Belousov 			error = ctx_alloc_pgtbl(ctx1);
33386be9f0dSKonstantin Belousov 			if (error != 0) {
33486be9f0dSKonstantin Belousov 				dmar_ctx_dtr(ctx1, true, false);
33586be9f0dSKonstantin Belousov 				TD_PINNED_ASSERT;
33686be9f0dSKonstantin Belousov 				return (NULL);
33786be9f0dSKonstantin Belousov 			}
33886be9f0dSKonstantin Belousov 			/* Disable local apic region access */
33986be9f0dSKonstantin Belousov 			error = dmar_gas_reserve_region(ctx1, 0xfee00000,
34086be9f0dSKonstantin Belousov 			    0xfeefffff + 1);
34186be9f0dSKonstantin Belousov 			if (error != 0) {
34286be9f0dSKonstantin Belousov 				dmar_ctx_dtr(ctx1, true, true);
34386be9f0dSKonstantin Belousov 				TD_PINNED_ASSERT;
34486be9f0dSKonstantin Belousov 				return (NULL);
34586be9f0dSKonstantin Belousov 			}
34686be9f0dSKonstantin Belousov 			error = ctx_init_rmrr(ctx1, dev);
34786be9f0dSKonstantin Belousov 			if (error != 0) {
34886be9f0dSKonstantin Belousov 				dmar_ctx_dtr(ctx1, true, true);
34986be9f0dSKonstantin Belousov 				TD_PINNED_ASSERT;
35086be9f0dSKonstantin Belousov 				return (NULL);
35186be9f0dSKonstantin Belousov 			}
35286be9f0dSKonstantin Belousov 		}
35386be9f0dSKonstantin Belousov 		ctxp = dmar_map_ctx_entry(ctx1, &sf);
35486be9f0dSKonstantin Belousov 		DMAR_LOCK(dmar);
35586be9f0dSKonstantin Belousov 
35686be9f0dSKonstantin Belousov 		/*
35786be9f0dSKonstantin Belousov 		 * Recheck the contexts, other thread might have
35886be9f0dSKonstantin Belousov 		 * already allocated needed one.
35986be9f0dSKonstantin Belousov 		 */
36067499354SRyan Stone 		ctx = dmar_find_ctx_locked(dmar, rid);
36186be9f0dSKonstantin Belousov 		if (ctx == NULL) {
36286be9f0dSKonstantin Belousov 			ctx = ctx1;
3639d0bc6d8SKonstantin Belousov 			ctx->ctx_tag.owner = dev;
36486be9f0dSKonstantin Belousov 			ctx->domain = alloc_unrl(dmar->domids);
36586be9f0dSKonstantin Belousov 			if (ctx->domain == -1) {
36686be9f0dSKonstantin Belousov 				DMAR_UNLOCK(dmar);
36786be9f0dSKonstantin Belousov 				dmar_unmap_pgtbl(sf, true);
36886be9f0dSKonstantin Belousov 				dmar_ctx_dtr(ctx, true, true);
36986be9f0dSKonstantin Belousov 				TD_PINNED_ASSERT;
37086be9f0dSKonstantin Belousov 				return (NULL);
37186be9f0dSKonstantin Belousov 			}
37267499354SRyan Stone 			ctx_tag_init(ctx, dev);
37386be9f0dSKonstantin Belousov 
37486be9f0dSKonstantin Belousov 			/*
37586be9f0dSKonstantin Belousov 			 * This is the first activated context for the
37686be9f0dSKonstantin Belousov 			 * DMAR unit.  Enable the translation after
37786be9f0dSKonstantin Belousov 			 * everything is set up.
37886be9f0dSKonstantin Belousov 			 */
37986be9f0dSKonstantin Belousov 			if (LIST_EMPTY(&dmar->contexts))
38086be9f0dSKonstantin Belousov 				enable = true;
38186be9f0dSKonstantin Belousov 			LIST_INSERT_HEAD(&dmar->contexts, ctx, link);
38286be9f0dSKonstantin Belousov 			ctx_id_entry_init(ctx, ctxp);
38386be9f0dSKonstantin Belousov 			device_printf(dev,
3849d0bc6d8SKonstantin Belousov 			    "dmar%d pci%d:%d:%d:%d domain %d mgaw %d "
3859d0bc6d8SKonstantin Belousov 			    "agaw %d %s-mapped\n",
38686be9f0dSKonstantin Belousov 			    dmar->unit, dmar->segment, bus, slot,
3879d0bc6d8SKonstantin Belousov 			    func, ctx->domain, ctx->mgaw, ctx->agaw,
3889d0bc6d8SKonstantin Belousov 			    id_mapped ? "id" : "re");
38986be9f0dSKonstantin Belousov 		} else {
39086be9f0dSKonstantin Belousov 			dmar_ctx_dtr(ctx1, true, true);
39186be9f0dSKonstantin Belousov 		}
39286be9f0dSKonstantin Belousov 		dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar));
39386be9f0dSKonstantin Belousov 	}
39486be9f0dSKonstantin Belousov 	ctx->refs++;
39586be9f0dSKonstantin Belousov 	if ((ctx->flags & DMAR_CTX_RMRR) != 0)
39686be9f0dSKonstantin Belousov 		ctx->refs++; /* XXXKIB */
39786be9f0dSKonstantin Belousov 
39886be9f0dSKonstantin Belousov 	/*
39986be9f0dSKonstantin Belousov 	 * If dmar declares Caching Mode as Set, follow 11.5 "Caching
40086be9f0dSKonstantin Belousov 	 * Mode Consideration" and do the (global) invalidation of the
40186be9f0dSKonstantin Belousov 	 * negative TLB entries.
40286be9f0dSKonstantin Belousov 	 */
40386be9f0dSKonstantin Belousov 	if ((dmar->hw_cap & DMAR_CAP_CM) != 0 || enable) {
40468eeb96aSKonstantin Belousov 		if (dmar->qi_enabled) {
40568eeb96aSKonstantin Belousov 			dmar_qi_invalidate_ctx_glob_locked(dmar);
40668eeb96aSKonstantin Belousov 			if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0)
40768eeb96aSKonstantin Belousov 				dmar_qi_invalidate_iotlb_glob_locked(dmar);
40868eeb96aSKonstantin Belousov 		} else {
40986be9f0dSKonstantin Belousov 			error = dmar_inv_ctx_glob(dmar);
41086be9f0dSKonstantin Belousov 			if (error == 0 &&
41186be9f0dSKonstantin Belousov 			    (dmar->hw_ecap & DMAR_ECAP_DI) != 0)
41286be9f0dSKonstantin Belousov 				error = dmar_inv_iotlb_glob(dmar);
41386be9f0dSKonstantin Belousov 			if (error != 0) {
41486be9f0dSKonstantin Belousov 				dmar_free_ctx_locked(dmar, ctx);
41586be9f0dSKonstantin Belousov 				TD_PINNED_ASSERT;
41686be9f0dSKonstantin Belousov 				return (NULL);
41786be9f0dSKonstantin Belousov 			}
41886be9f0dSKonstantin Belousov 		}
41968eeb96aSKonstantin Belousov 	}
42068eeb96aSKonstantin Belousov 
42168eeb96aSKonstantin Belousov 	/*
42268eeb96aSKonstantin Belousov 	 * The dmar lock was potentially dropped between check for the
42368eeb96aSKonstantin Belousov 	 * empty context list and now.  Recheck the state of GCMD_TE
42468eeb96aSKonstantin Belousov 	 * to avoid unneeded command.
42568eeb96aSKonstantin Belousov 	 */
42668eeb96aSKonstantin Belousov 	if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) {
42786be9f0dSKonstantin Belousov 		error = dmar_enable_translation(dmar);
42886be9f0dSKonstantin Belousov 		if (error != 0) {
42986be9f0dSKonstantin Belousov 			dmar_free_ctx_locked(dmar, ctx);
43086be9f0dSKonstantin Belousov 			TD_PINNED_ASSERT;
43186be9f0dSKonstantin Belousov 			return (NULL);
43286be9f0dSKonstantin Belousov 		}
43386be9f0dSKonstantin Belousov 	}
43486be9f0dSKonstantin Belousov 	DMAR_UNLOCK(dmar);
43586be9f0dSKonstantin Belousov 	TD_PINNED_ASSERT;
43686be9f0dSKonstantin Belousov 	return (ctx);
43786be9f0dSKonstantin Belousov }
43886be9f0dSKonstantin Belousov 
43986be9f0dSKonstantin Belousov void
44086be9f0dSKonstantin Belousov dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx)
44186be9f0dSKonstantin Belousov {
44286be9f0dSKonstantin Belousov 	struct sf_buf *sf;
44386be9f0dSKonstantin Belousov 	dmar_ctx_entry_t *ctxp;
44486be9f0dSKonstantin Belousov 
44586be9f0dSKonstantin Belousov 	DMAR_ASSERT_LOCKED(dmar);
44686be9f0dSKonstantin Belousov 	KASSERT(ctx->refs >= 1,
44786be9f0dSKonstantin Belousov 	    ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
44886be9f0dSKonstantin Belousov 
44986be9f0dSKonstantin Belousov 	/*
45086be9f0dSKonstantin Belousov 	 * If our reference is not last, only the dereference should
45186be9f0dSKonstantin Belousov 	 * be performed.
45286be9f0dSKonstantin Belousov 	 */
45386be9f0dSKonstantin Belousov 	if (ctx->refs > 1) {
45486be9f0dSKonstantin Belousov 		ctx->refs--;
45586be9f0dSKonstantin Belousov 		DMAR_UNLOCK(dmar);
45686be9f0dSKonstantin Belousov 		return;
45786be9f0dSKonstantin Belousov 	}
45886be9f0dSKonstantin Belousov 
45986be9f0dSKonstantin Belousov 	KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0,
46086be9f0dSKonstantin Belousov 	    ("lost ref on RMRR ctx %p", ctx));
46186be9f0dSKonstantin Belousov 	KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0,
46286be9f0dSKonstantin Belousov 	    ("lost ref on disabled ctx %p", ctx));
46386be9f0dSKonstantin Belousov 
46486be9f0dSKonstantin Belousov 	/*
46586be9f0dSKonstantin Belousov 	 * Otherwise, the context entry must be cleared before the
46686be9f0dSKonstantin Belousov 	 * page table is destroyed.  The mapping of the context
46786be9f0dSKonstantin Belousov 	 * entries page could require sleep, unlock the dmar.
46886be9f0dSKonstantin Belousov 	 */
46986be9f0dSKonstantin Belousov 	DMAR_UNLOCK(dmar);
47086be9f0dSKonstantin Belousov 	TD_PREP_PINNED_ASSERT;
47186be9f0dSKonstantin Belousov 	ctxp = dmar_map_ctx_entry(ctx, &sf);
47286be9f0dSKonstantin Belousov 	DMAR_LOCK(dmar);
47386be9f0dSKonstantin Belousov 	KASSERT(ctx->refs >= 1,
47486be9f0dSKonstantin Belousov 	    ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));
47586be9f0dSKonstantin Belousov 
47686be9f0dSKonstantin Belousov 	/*
47786be9f0dSKonstantin Belousov 	 * Other thread might have referenced the context, in which
47886be9f0dSKonstantin Belousov 	 * case again only the dereference should be performed.
47986be9f0dSKonstantin Belousov 	 */
48086be9f0dSKonstantin Belousov 	if (ctx->refs > 1) {
48186be9f0dSKonstantin Belousov 		ctx->refs--;
48286be9f0dSKonstantin Belousov 		DMAR_UNLOCK(dmar);
48386be9f0dSKonstantin Belousov 		dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar));
48486be9f0dSKonstantin Belousov 		TD_PINNED_ASSERT;
48586be9f0dSKonstantin Belousov 		return;
48686be9f0dSKonstantin Belousov 	}
48786be9f0dSKonstantin Belousov 
48886be9f0dSKonstantin Belousov 	KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0,
48986be9f0dSKonstantin Belousov 	    ("lost ref on RMRR ctx %p", ctx));
49086be9f0dSKonstantin Belousov 	KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0,
49186be9f0dSKonstantin Belousov 	    ("lost ref on disabled ctx %p", ctx));
49286be9f0dSKonstantin Belousov 
49386be9f0dSKonstantin Belousov 	/*
49486be9f0dSKonstantin Belousov 	 * Clear the context pointer and flush the caches.
49586be9f0dSKonstantin Belousov 	 * XXXKIB: cannot do this if any RMRR entries are still present.
49686be9f0dSKonstantin Belousov 	 */
49786be9f0dSKonstantin Belousov 	dmar_pte_clear(&ctxp->ctx1);
49886be9f0dSKonstantin Belousov 	ctxp->ctx2 = 0;
49986be9f0dSKonstantin Belousov 	dmar_inv_ctx_glob(dmar);
50068eeb96aSKonstantin Belousov 	if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) {
50168eeb96aSKonstantin Belousov 		if (dmar->qi_enabled)
50268eeb96aSKonstantin Belousov 			dmar_qi_invalidate_iotlb_glob_locked(dmar);
50368eeb96aSKonstantin Belousov 		else
50486be9f0dSKonstantin Belousov 			dmar_inv_iotlb_glob(dmar);
50568eeb96aSKonstantin Belousov 	}
50686be9f0dSKonstantin Belousov 	LIST_REMOVE(ctx, link);
50786be9f0dSKonstantin Belousov 	DMAR_UNLOCK(dmar);
50886be9f0dSKonstantin Belousov 
50986be9f0dSKonstantin Belousov 	/*
51086be9f0dSKonstantin Belousov 	 * The rest of the destruction is invisible for other users of
51186be9f0dSKonstantin Belousov 	 * the dmar unit.
51286be9f0dSKonstantin Belousov 	 */
51386be9f0dSKonstantin Belousov 	taskqueue_drain(dmar->delayed_taskqueue, &ctx->unload_task);
51486be9f0dSKonstantin Belousov 	KASSERT(TAILQ_EMPTY(&ctx->unload_entries),
51586be9f0dSKonstantin Belousov 	    ("unfinished unloads %p", ctx));
51686be9f0dSKonstantin Belousov 	dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar));
51786be9f0dSKonstantin Belousov 	free_unr(dmar->domids, ctx->domain);
51886be9f0dSKonstantin Belousov 	dmar_ctx_dtr(ctx, true, true);
51986be9f0dSKonstantin Belousov 	TD_PINNED_ASSERT;
52086be9f0dSKonstantin Belousov }
52186be9f0dSKonstantin Belousov 
52286be9f0dSKonstantin Belousov void
52386be9f0dSKonstantin Belousov dmar_free_ctx(struct dmar_ctx *ctx)
52486be9f0dSKonstantin Belousov {
52586be9f0dSKonstantin Belousov 	struct dmar_unit *dmar;
52686be9f0dSKonstantin Belousov 
52786be9f0dSKonstantin Belousov 	dmar = ctx->dmar;
52886be9f0dSKonstantin Belousov 	DMAR_LOCK(dmar);
52986be9f0dSKonstantin Belousov 	dmar_free_ctx_locked(dmar, ctx);
53086be9f0dSKonstantin Belousov }
53186be9f0dSKonstantin Belousov 
53286be9f0dSKonstantin Belousov struct dmar_ctx *
53367499354SRyan Stone dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid)
53486be9f0dSKonstantin Belousov {
53586be9f0dSKonstantin Belousov 	struct dmar_ctx *ctx;
53686be9f0dSKonstantin Belousov 
53786be9f0dSKonstantin Belousov 	DMAR_ASSERT_LOCKED(dmar);
53886be9f0dSKonstantin Belousov 
53986be9f0dSKonstantin Belousov 	LIST_FOREACH(ctx, &dmar->contexts, link) {
54067499354SRyan Stone 		if (ctx->rid == rid)
54186be9f0dSKonstantin Belousov 			return (ctx);
54286be9f0dSKonstantin Belousov 	}
54386be9f0dSKonstantin Belousov 	return (NULL);
54486be9f0dSKonstantin Belousov }
54586be9f0dSKonstantin Belousov 
54686be9f0dSKonstantin Belousov void
54768eeb96aSKonstantin Belousov dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free)
54868eeb96aSKonstantin Belousov {
54968eeb96aSKonstantin Belousov 	struct dmar_ctx *ctx;
55068eeb96aSKonstantin Belousov 
55168eeb96aSKonstantin Belousov 	ctx = entry->ctx;
55268eeb96aSKonstantin Belousov 	DMAR_CTX_LOCK(ctx);
55368eeb96aSKonstantin Belousov 	if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0)
55468eeb96aSKonstantin Belousov 		dmar_gas_free_region(ctx, entry);
55568eeb96aSKonstantin Belousov 	else
55668eeb96aSKonstantin Belousov 		dmar_gas_free_space(ctx, entry);
55768eeb96aSKonstantin Belousov 	DMAR_CTX_UNLOCK(ctx);
55868eeb96aSKonstantin Belousov 	if (free)
55968eeb96aSKonstantin Belousov 		dmar_gas_free_entry(ctx, entry);
56068eeb96aSKonstantin Belousov 	else
56168eeb96aSKonstantin Belousov 		entry->flags = 0;
56268eeb96aSKonstantin Belousov }
56368eeb96aSKonstantin Belousov 
56468eeb96aSKonstantin Belousov void
56568eeb96aSKonstantin Belousov dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free)
56668eeb96aSKonstantin Belousov {
56768eeb96aSKonstantin Belousov 	struct dmar_unit *unit;
56868eeb96aSKonstantin Belousov 
56968eeb96aSKonstantin Belousov 	unit = entry->ctx->dmar;
57068eeb96aSKonstantin Belousov 	if (unit->qi_enabled) {
57168eeb96aSKonstantin Belousov 		DMAR_LOCK(unit);
57268eeb96aSKonstantin Belousov 		dmar_qi_invalidate_locked(entry->ctx, entry->start,
57368eeb96aSKonstantin Belousov 		    entry->end - entry->start, &entry->gseq);
57468eeb96aSKonstantin Belousov 		if (!free)
57568eeb96aSKonstantin Belousov 			entry->flags |= DMAR_MAP_ENTRY_QI_NF;
57668eeb96aSKonstantin Belousov 		TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
57768eeb96aSKonstantin Belousov 		DMAR_UNLOCK(unit);
57868eeb96aSKonstantin Belousov 	} else {
57968eeb96aSKonstantin Belousov 		ctx_flush_iotlb_sync(entry->ctx, entry->start, entry->end -
58068eeb96aSKonstantin Belousov 		    entry->start);
58168eeb96aSKonstantin Belousov 		dmar_ctx_free_entry(entry, free);
58268eeb96aSKonstantin Belousov 	}
58368eeb96aSKonstantin Belousov }
58468eeb96aSKonstantin Belousov 
58568eeb96aSKonstantin Belousov void
58686be9f0dSKonstantin Belousov dmar_ctx_unload(struct dmar_ctx *ctx, struct dmar_map_entries_tailq *entries,
58786be9f0dSKonstantin Belousov     bool cansleep)
58886be9f0dSKonstantin Belousov {
58968eeb96aSKonstantin Belousov 	struct dmar_unit *unit;
59068eeb96aSKonstantin Belousov 	struct dmar_map_entry *entry, *entry1;
59168eeb96aSKonstantin Belousov 	struct dmar_qi_genseq gseq;
59286be9f0dSKonstantin Belousov 	int error;
59386be9f0dSKonstantin Belousov 
59468eeb96aSKonstantin Belousov 	unit = ctx->dmar;
59568eeb96aSKonstantin Belousov 
59668eeb96aSKonstantin Belousov 	TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
59786be9f0dSKonstantin Belousov 		KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0,
59886be9f0dSKonstantin Belousov 		    ("not mapped entry %p %p", ctx, entry));
59986be9f0dSKonstantin Belousov 		error = ctx_unmap_buf(ctx, entry->start, entry->end -
60086be9f0dSKonstantin Belousov 		    entry->start, cansleep ? DMAR_PGF_WAITOK : 0);
60186be9f0dSKonstantin Belousov 		KASSERT(error == 0, ("unmap %p error %d", ctx, error));
60268eeb96aSKonstantin Belousov 		if (!unit->qi_enabled) {
60368eeb96aSKonstantin Belousov 			ctx_flush_iotlb_sync(ctx, entry->start,
60468eeb96aSKonstantin Belousov 			    entry->end - entry->start);
60568eeb96aSKonstantin Belousov 			TAILQ_REMOVE(entries, entry, dmamap_link);
60668eeb96aSKonstantin Belousov 			dmar_ctx_free_entry(entry, true);
60786be9f0dSKonstantin Belousov 		}
60886be9f0dSKonstantin Belousov 	}
60968eeb96aSKonstantin Belousov 	if (TAILQ_EMPTY(entries))
61068eeb96aSKonstantin Belousov 		return;
61168eeb96aSKonstantin Belousov 
61268eeb96aSKonstantin Belousov 	KASSERT(unit->qi_enabled, ("loaded entry left"));
61368eeb96aSKonstantin Belousov 	DMAR_LOCK(unit);
61468eeb96aSKonstantin Belousov 	TAILQ_FOREACH(entry, entries, dmamap_link) {
61568eeb96aSKonstantin Belousov 		entry->gseq.gen = 0;
61668eeb96aSKonstantin Belousov 		entry->gseq.seq = 0;
61768eeb96aSKonstantin Belousov 		dmar_qi_invalidate_locked(ctx, entry->start, entry->end -
61868eeb96aSKonstantin Belousov 		    entry->start, TAILQ_NEXT(entry, dmamap_link) == NULL ?
61968eeb96aSKonstantin Belousov 		    &gseq : NULL);
62068eeb96aSKonstantin Belousov 	}
62168eeb96aSKonstantin Belousov 	TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
62268eeb96aSKonstantin Belousov 		entry->gseq = gseq;
62368eeb96aSKonstantin Belousov 		TAILQ_REMOVE(entries, entry, dmamap_link);
62468eeb96aSKonstantin Belousov 		TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
62568eeb96aSKonstantin Belousov 	}
62668eeb96aSKonstantin Belousov 	DMAR_UNLOCK(unit);
62768eeb96aSKonstantin Belousov }
62886be9f0dSKonstantin Belousov 
62986be9f0dSKonstantin Belousov static void
63086be9f0dSKonstantin Belousov dmar_ctx_unload_task(void *arg, int pending)
63186be9f0dSKonstantin Belousov {
63286be9f0dSKonstantin Belousov 	struct dmar_ctx *ctx;
63386be9f0dSKonstantin Belousov 	struct dmar_map_entries_tailq entries;
63486be9f0dSKonstantin Belousov 
63586be9f0dSKonstantin Belousov 	ctx = arg;
63686be9f0dSKonstantin Belousov 	TAILQ_INIT(&entries);
63786be9f0dSKonstantin Belousov 
63886be9f0dSKonstantin Belousov 	for (;;) {
63986be9f0dSKonstantin Belousov 		DMAR_CTX_LOCK(ctx);
64086be9f0dSKonstantin Belousov 		TAILQ_SWAP(&ctx->unload_entries, &entries, dmar_map_entry,
64186be9f0dSKonstantin Belousov 		    dmamap_link);
64286be9f0dSKonstantin Belousov 		DMAR_CTX_UNLOCK(ctx);
64386be9f0dSKonstantin Belousov 		if (TAILQ_EMPTY(&entries))
64486be9f0dSKonstantin Belousov 			break;
64586be9f0dSKonstantin Belousov 		dmar_ctx_unload(ctx, &entries, true);
64686be9f0dSKonstantin Belousov 	}
64786be9f0dSKonstantin Belousov }
648