186be9f0dSKonstantin Belousov /*- 286be9f0dSKonstantin Belousov * Copyright (c) 2013 The FreeBSD Foundation 386be9f0dSKonstantin Belousov * All rights reserved. 486be9f0dSKonstantin Belousov * 586be9f0dSKonstantin Belousov * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 686be9f0dSKonstantin Belousov * under sponsorship from the FreeBSD Foundation. 786be9f0dSKonstantin Belousov * 886be9f0dSKonstantin Belousov * Redistribution and use in source and binary forms, with or without 986be9f0dSKonstantin Belousov * modification, are permitted provided that the following conditions 1086be9f0dSKonstantin Belousov * are met: 1186be9f0dSKonstantin Belousov * 1. Redistributions of source code must retain the above copyright 1286be9f0dSKonstantin Belousov * notice, this list of conditions and the following disclaimer. 1386be9f0dSKonstantin Belousov * 2. Redistributions in binary form must reproduce the above copyright 1486be9f0dSKonstantin Belousov * notice, this list of conditions and the following disclaimer in the 1586be9f0dSKonstantin Belousov * documentation and/or other materials provided with the distribution. 1686be9f0dSKonstantin Belousov * 1786be9f0dSKonstantin Belousov * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1886be9f0dSKonstantin Belousov * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1986be9f0dSKonstantin Belousov * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2086be9f0dSKonstantin Belousov * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2186be9f0dSKonstantin Belousov * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2286be9f0dSKonstantin Belousov * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2386be9f0dSKonstantin Belousov * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2486be9f0dSKonstantin Belousov * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2586be9f0dSKonstantin Belousov * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2686be9f0dSKonstantin Belousov * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2786be9f0dSKonstantin Belousov * SUCH DAMAGE. 2886be9f0dSKonstantin Belousov */ 2986be9f0dSKonstantin Belousov 3086be9f0dSKonstantin Belousov #include <sys/cdefs.h> 3186be9f0dSKonstantin Belousov __FBSDID("$FreeBSD$"); 3286be9f0dSKonstantin Belousov 3386be9f0dSKonstantin Belousov #include <sys/param.h> 3486be9f0dSKonstantin Belousov #include <sys/systm.h> 3586be9f0dSKonstantin Belousov #include <sys/malloc.h> 3686be9f0dSKonstantin Belousov #include <sys/bus.h> 3786be9f0dSKonstantin Belousov #include <sys/interrupt.h> 3886be9f0dSKonstantin Belousov #include <sys/kernel.h> 3986be9f0dSKonstantin Belousov #include <sys/ktr.h> 4086be9f0dSKonstantin Belousov #include <sys/limits.h> 4186be9f0dSKonstantin Belousov #include <sys/lock.h> 4286be9f0dSKonstantin Belousov #include <sys/memdesc.h> 4386be9f0dSKonstantin Belousov #include <sys/mutex.h> 4486be9f0dSKonstantin Belousov #include <sys/proc.h> 4586be9f0dSKonstantin Belousov #include <sys/rwlock.h> 4686be9f0dSKonstantin Belousov #include <sys/rman.h> 4786be9f0dSKonstantin Belousov #include <sys/sysctl.h> 4886be9f0dSKonstantin Belousov #include <sys/taskqueue.h> 4986be9f0dSKonstantin Belousov #include <sys/tree.h> 5086be9f0dSKonstantin Belousov #include <sys/uio.h> 5186be9f0dSKonstantin Belousov #include <vm/vm.h> 5286be9f0dSKonstantin Belousov #include <vm/vm_extern.h> 5386be9f0dSKonstantin Belousov #include <vm/vm_kern.h> 5486be9f0dSKonstantin Belousov #include <vm/vm_object.h> 5586be9f0dSKonstantin Belousov #include <vm/vm_page.h> 5686be9f0dSKonstantin Belousov #include <vm/vm_pager.h> 5786be9f0dSKonstantin Belousov #include <vm/vm_map.h> 5886be9f0dSKonstantin Belousov #include <machine/atomic.h> 5986be9f0dSKonstantin Belousov #include <machine/bus.h> 6086be9f0dSKonstantin Belousov #include <machine/md_var.h> 6186be9f0dSKonstantin Belousov #include <machine/specialreg.h> 6286be9f0dSKonstantin Belousov #include <x86/include/busdma_impl.h> 6386be9f0dSKonstantin Belousov #include <x86/iommu/intel_reg.h> 6486be9f0dSKonstantin Belousov #include <x86/iommu/busdma_dmar.h> 6586be9f0dSKonstantin Belousov #include <x86/iommu/intel_dmar.h> 6686be9f0dSKonstantin Belousov #include <dev/pci/pcivar.h> 6786be9f0dSKonstantin Belousov 6886be9f0dSKonstantin Belousov static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); 6986be9f0dSKonstantin Belousov 7086be9f0dSKonstantin Belousov static void dmar_ctx_unload_task(void *arg, int pending); 7186be9f0dSKonstantin Belousov 7286be9f0dSKonstantin Belousov static void 7386be9f0dSKonstantin Belousov dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) 7486be9f0dSKonstantin Belousov { 7586be9f0dSKonstantin Belousov struct sf_buf *sf; 7686be9f0dSKonstantin Belousov dmar_root_entry_t *re; 7786be9f0dSKonstantin Belousov vm_page_t ctxm; 7886be9f0dSKonstantin Belousov 7986be9f0dSKonstantin Belousov /* 8086be9f0dSKonstantin Belousov * Allocated context page must be linked. 8186be9f0dSKonstantin Belousov */ 8286be9f0dSKonstantin Belousov ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_NOALLOC); 8386be9f0dSKonstantin Belousov if (ctxm != NULL) 8486be9f0dSKonstantin Belousov return; 8586be9f0dSKonstantin Belousov 8686be9f0dSKonstantin Belousov /* 8786be9f0dSKonstantin Belousov * Page not present, allocate and link. Note that other 8886be9f0dSKonstantin Belousov * thread might execute this sequence in parallel. This 8986be9f0dSKonstantin Belousov * should be safe, because the context entries written by both 9086be9f0dSKonstantin Belousov * threads are equal. 9186be9f0dSKonstantin Belousov */ 9286be9f0dSKonstantin Belousov TD_PREP_PINNED_ASSERT; 9386be9f0dSKonstantin Belousov ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_ZERO | 9486be9f0dSKonstantin Belousov DMAR_PGF_WAITOK); 9586be9f0dSKonstantin Belousov re = dmar_map_pgtbl(dmar->ctx_obj, 0, DMAR_PGF_NOALLOC, &sf); 9686be9f0dSKonstantin Belousov re += bus; 9786be9f0dSKonstantin Belousov dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & 9886be9f0dSKonstantin Belousov VM_PAGE_TO_PHYS(ctxm))); 9986be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 10086be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 10186be9f0dSKonstantin Belousov } 10286be9f0dSKonstantin Belousov 10386be9f0dSKonstantin Belousov static dmar_ctx_entry_t * 10486be9f0dSKonstantin Belousov dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) 10586be9f0dSKonstantin Belousov { 10686be9f0dSKonstantin Belousov dmar_ctx_entry_t *ctxp; 10786be9f0dSKonstantin Belousov 10886be9f0dSKonstantin Belousov ctxp = dmar_map_pgtbl(ctx->dmar->ctx_obj, 1 + ctx->bus, 10986be9f0dSKonstantin Belousov DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp); 11086be9f0dSKonstantin Belousov ctxp += ((ctx->slot & 0x1f) << 3) + (ctx->func & 0x7); 11186be9f0dSKonstantin Belousov return (ctxp); 11286be9f0dSKonstantin Belousov } 11386be9f0dSKonstantin Belousov 11486be9f0dSKonstantin Belousov static void 11586be9f0dSKonstantin Belousov ctx_tag_init(struct dmar_ctx *ctx) 11686be9f0dSKonstantin Belousov { 11786be9f0dSKonstantin Belousov bus_addr_t maxaddr; 11886be9f0dSKonstantin Belousov 11986be9f0dSKonstantin Belousov maxaddr = MIN(ctx->end, BUS_SPACE_MAXADDR); 12086be9f0dSKonstantin Belousov ctx->ctx_tag.common.ref_count = 1; /* Prevent free */ 12186be9f0dSKonstantin Belousov ctx->ctx_tag.common.impl = &bus_dma_dmar_impl; 12286be9f0dSKonstantin Belousov ctx->ctx_tag.common.boundary = PCI_DMA_BOUNDARY; 12386be9f0dSKonstantin Belousov ctx->ctx_tag.common.lowaddr = maxaddr; 12486be9f0dSKonstantin Belousov ctx->ctx_tag.common.highaddr = maxaddr; 12586be9f0dSKonstantin Belousov ctx->ctx_tag.common.maxsize = maxaddr; 12686be9f0dSKonstantin Belousov ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED; 12786be9f0dSKonstantin Belousov ctx->ctx_tag.common.maxsegsz = maxaddr; 12886be9f0dSKonstantin Belousov ctx->ctx_tag.ctx = ctx; 12986be9f0dSKonstantin Belousov /* XXXKIB initialize tag further */ 13086be9f0dSKonstantin Belousov } 13186be9f0dSKonstantin Belousov 13286be9f0dSKonstantin Belousov static void 13386be9f0dSKonstantin Belousov ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp) 13486be9f0dSKonstantin Belousov { 13586be9f0dSKonstantin Belousov struct dmar_unit *unit; 13686be9f0dSKonstantin Belousov vm_page_t ctx_root; 13786be9f0dSKonstantin Belousov 13886be9f0dSKonstantin Belousov unit = ctx->dmar; 13986be9f0dSKonstantin Belousov KASSERT(ctxp->ctx1 == 0 && ctxp->ctx2 == 0, 14086be9f0dSKonstantin Belousov ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", 14186be9f0dSKonstantin Belousov unit->unit, ctx->bus, ctx->slot, ctx->func, ctxp->ctx1, 14286be9f0dSKonstantin Belousov ctxp->ctx2)); 14386be9f0dSKonstantin Belousov ctxp->ctx2 = DMAR_CTX2_DID(ctx->domain); 14486be9f0dSKonstantin Belousov ctxp->ctx2 |= ctx->awlvl; 14586be9f0dSKonstantin Belousov if ((ctx->flags & DMAR_CTX_IDMAP) != 0 && 14686be9f0dSKonstantin Belousov (unit->hw_ecap & DMAR_ECAP_PT) != 0) { 14786be9f0dSKonstantin Belousov KASSERT(ctx->pgtbl_obj == NULL, 14886be9f0dSKonstantin Belousov ("ctx %p non-null pgtbl_obj", ctx)); 14986be9f0dSKonstantin Belousov dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); 15086be9f0dSKonstantin Belousov } else { 15186be9f0dSKonstantin Belousov ctx_root = dmar_pgalloc(ctx->pgtbl_obj, 0, DMAR_PGF_NOALLOC); 15286be9f0dSKonstantin Belousov dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_UNTR | 15386be9f0dSKonstantin Belousov (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | 15486be9f0dSKonstantin Belousov DMAR_CTX1_P); 15586be9f0dSKonstantin Belousov } 15686be9f0dSKonstantin Belousov } 15786be9f0dSKonstantin Belousov 15886be9f0dSKonstantin Belousov static int 15986be9f0dSKonstantin Belousov ctx_init_rmrr(struct dmar_ctx *ctx, device_t dev) 16086be9f0dSKonstantin Belousov { 16186be9f0dSKonstantin Belousov struct dmar_map_entries_tailq rmrr_entries; 16286be9f0dSKonstantin Belousov struct dmar_map_entry *entry, *entry1; 16386be9f0dSKonstantin Belousov vm_page_t *ma; 16486be9f0dSKonstantin Belousov dmar_gaddr_t start, end; 16586be9f0dSKonstantin Belousov vm_pindex_t size, i; 16686be9f0dSKonstantin Belousov int error, error1; 16786be9f0dSKonstantin Belousov 16886be9f0dSKonstantin Belousov error = 0; 16986be9f0dSKonstantin Belousov TAILQ_INIT(&rmrr_entries); 17086be9f0dSKonstantin Belousov dmar_ctx_parse_rmrr(ctx, dev, &rmrr_entries); 17186be9f0dSKonstantin Belousov TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) { 17286be9f0dSKonstantin Belousov /* 17386be9f0dSKonstantin Belousov * VT-d specification requires that the start of an 17486be9f0dSKonstantin Belousov * RMRR entry is 4k-aligned. Buggy BIOSes put 17586be9f0dSKonstantin Belousov * anything into the start and end fields. Truncate 17686be9f0dSKonstantin Belousov * and round as neccesary. 17786be9f0dSKonstantin Belousov * 17886be9f0dSKonstantin Belousov * We also allow the overlapping RMRR entries, see 17986be9f0dSKonstantin Belousov * dmar_gas_alloc_region(). 18086be9f0dSKonstantin Belousov */ 18186be9f0dSKonstantin Belousov start = entry->start; 18286be9f0dSKonstantin Belousov end = entry->end; 18386be9f0dSKonstantin Belousov entry->start = trunc_page(start); 18486be9f0dSKonstantin Belousov entry->end = round_page(end); 185*e02b05b3SKonstantin Belousov if (entry->start == entry->end) { 186*e02b05b3SKonstantin Belousov /* Workaround for some AMI (?) BIOSes */ 187*e02b05b3SKonstantin Belousov if (bootverbose) { 188*e02b05b3SKonstantin Belousov device_printf(dev, "BIOS bug: dmar%d RMRR " 189*e02b05b3SKonstantin Belousov "region (%jx, %jx) corrected\n", 190*e02b05b3SKonstantin Belousov ctx->dmar->unit, start, end); 191*e02b05b3SKonstantin Belousov } 192*e02b05b3SKonstantin Belousov entry->end += DMAR_PAGE_SIZE * 0x20; 193*e02b05b3SKonstantin Belousov } 19486be9f0dSKonstantin Belousov size = OFF_TO_IDX(entry->end - entry->start); 19586be9f0dSKonstantin Belousov ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); 19686be9f0dSKonstantin Belousov for (i = 0; i < size; i++) { 19786be9f0dSKonstantin Belousov ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, 19886be9f0dSKonstantin Belousov VM_MEMATTR_DEFAULT); 19986be9f0dSKonstantin Belousov } 20086be9f0dSKonstantin Belousov error1 = dmar_gas_map_region(ctx, entry, DMAR_MAP_ENTRY_READ | 20186be9f0dSKonstantin Belousov DMAR_MAP_ENTRY_WRITE, DMAR_GM_CANWAIT, ma); 20286be9f0dSKonstantin Belousov /* 20386be9f0dSKonstantin Belousov * Non-failed RMRR entries are owned by context rb 20486be9f0dSKonstantin Belousov * tree. Get rid of the failed entry, but do not stop 20586be9f0dSKonstantin Belousov * the loop. Rest of the parsed RMRR entries are 20686be9f0dSKonstantin Belousov * loaded and removed on the context destruction. 20786be9f0dSKonstantin Belousov */ 20886be9f0dSKonstantin Belousov if (error1 == 0 && entry->end != entry->start) { 20986be9f0dSKonstantin Belousov DMAR_LOCK(ctx->dmar); 21086be9f0dSKonstantin Belousov ctx->flags |= DMAR_CTX_RMRR; 21186be9f0dSKonstantin Belousov DMAR_UNLOCK(ctx->dmar); 21286be9f0dSKonstantin Belousov } else { 21386be9f0dSKonstantin Belousov if (error1 != 0) { 21486be9f0dSKonstantin Belousov device_printf(dev, 21586be9f0dSKonstantin Belousov "dmar%d failed to map RMRR region (%jx, %jx) %d\n", 21686be9f0dSKonstantin Belousov ctx->dmar->unit, start, end, error1); 21786be9f0dSKonstantin Belousov error = error1; 21886be9f0dSKonstantin Belousov } 21986be9f0dSKonstantin Belousov TAILQ_REMOVE(&rmrr_entries, entry, unroll_link); 22086be9f0dSKonstantin Belousov dmar_gas_free_entry(ctx, entry); 22186be9f0dSKonstantin Belousov } 22286be9f0dSKonstantin Belousov for (i = 0; i < size; i++) 22386be9f0dSKonstantin Belousov vm_page_putfake(ma[i]); 22486be9f0dSKonstantin Belousov free(ma, M_TEMP); 22586be9f0dSKonstantin Belousov } 22686be9f0dSKonstantin Belousov return (error); 22786be9f0dSKonstantin Belousov } 22886be9f0dSKonstantin Belousov 22986be9f0dSKonstantin Belousov static struct dmar_ctx * 23086be9f0dSKonstantin Belousov dmar_get_ctx_alloc(struct dmar_unit *dmar, int bus, int slot, int func) 23186be9f0dSKonstantin Belousov { 23286be9f0dSKonstantin Belousov struct dmar_ctx *ctx; 23386be9f0dSKonstantin Belousov 23486be9f0dSKonstantin Belousov ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); 23586be9f0dSKonstantin Belousov RB_INIT(&ctx->rb_root); 23686be9f0dSKonstantin Belousov TAILQ_INIT(&ctx->unload_entries); 23786be9f0dSKonstantin Belousov TASK_INIT(&ctx->unload_task, 0, dmar_ctx_unload_task, ctx); 23886be9f0dSKonstantin Belousov mtx_init(&ctx->lock, "dmarctx", NULL, MTX_DEF); 23986be9f0dSKonstantin Belousov ctx->dmar = dmar; 24086be9f0dSKonstantin Belousov ctx->bus = bus; 24186be9f0dSKonstantin Belousov ctx->slot = slot; 24286be9f0dSKonstantin Belousov ctx->func = func; 24386be9f0dSKonstantin Belousov return (ctx); 24486be9f0dSKonstantin Belousov } 24586be9f0dSKonstantin Belousov 24686be9f0dSKonstantin Belousov static void 24786be9f0dSKonstantin Belousov dmar_ctx_dtr(struct dmar_ctx *ctx, bool gas_inited, bool pgtbl_inited) 24886be9f0dSKonstantin Belousov { 24986be9f0dSKonstantin Belousov 25086be9f0dSKonstantin Belousov if (gas_inited) { 25186be9f0dSKonstantin Belousov DMAR_CTX_LOCK(ctx); 25286be9f0dSKonstantin Belousov dmar_gas_fini_ctx(ctx); 25386be9f0dSKonstantin Belousov DMAR_CTX_UNLOCK(ctx); 25486be9f0dSKonstantin Belousov } 25586be9f0dSKonstantin Belousov if (pgtbl_inited) { 25686be9f0dSKonstantin Belousov if (ctx->pgtbl_obj != NULL) 25786be9f0dSKonstantin Belousov DMAR_CTX_PGLOCK(ctx); 25886be9f0dSKonstantin Belousov ctx_free_pgtbl(ctx); 25986be9f0dSKonstantin Belousov } 26086be9f0dSKonstantin Belousov mtx_destroy(&ctx->lock); 26186be9f0dSKonstantin Belousov free(ctx, M_DMAR_CTX); 26286be9f0dSKonstantin Belousov } 26386be9f0dSKonstantin Belousov 26486be9f0dSKonstantin Belousov struct dmar_ctx * 26586be9f0dSKonstantin Belousov dmar_get_ctx(struct dmar_unit *dmar, device_t dev, bool id_mapped, bool rmrr_init) 26686be9f0dSKonstantin Belousov { 26786be9f0dSKonstantin Belousov struct dmar_ctx *ctx, *ctx1; 26886be9f0dSKonstantin Belousov dmar_ctx_entry_t *ctxp; 26986be9f0dSKonstantin Belousov struct sf_buf *sf; 27086be9f0dSKonstantin Belousov int bus, slot, func, error, mgaw; 27186be9f0dSKonstantin Belousov bool enable; 27286be9f0dSKonstantin Belousov 27386be9f0dSKonstantin Belousov bus = pci_get_bus(dev); 27486be9f0dSKonstantin Belousov slot = pci_get_slot(dev); 27586be9f0dSKonstantin Belousov func = pci_get_function(dev); 27686be9f0dSKonstantin Belousov enable = false; 27786be9f0dSKonstantin Belousov TD_PREP_PINNED_ASSERT; 27886be9f0dSKonstantin Belousov DMAR_LOCK(dmar); 27986be9f0dSKonstantin Belousov ctx = dmar_find_ctx_locked(dmar, bus, slot, func); 28086be9f0dSKonstantin Belousov error = 0; 28186be9f0dSKonstantin Belousov if (ctx == NULL) { 28286be9f0dSKonstantin Belousov /* 28386be9f0dSKonstantin Belousov * Perform the allocations which require sleep or have 28486be9f0dSKonstantin Belousov * higher chance to succeed if the sleep is allowed. 28586be9f0dSKonstantin Belousov */ 28686be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 28786be9f0dSKonstantin Belousov dmar_ensure_ctx_page(dmar, bus); 28886be9f0dSKonstantin Belousov ctx1 = dmar_get_ctx_alloc(dmar, bus, slot, func); 28986be9f0dSKonstantin Belousov 29086be9f0dSKonstantin Belousov if (id_mapped) { 29186be9f0dSKonstantin Belousov /* 29286be9f0dSKonstantin Belousov * For now, use the maximal usable physical 29386be9f0dSKonstantin Belousov * address of the installed memory to 29486be9f0dSKonstantin Belousov * calculate the mgaw. It is useful for the 29586be9f0dSKonstantin Belousov * identity mapping, and less so for the 29686be9f0dSKonstantin Belousov * virtualized bus address space. 29786be9f0dSKonstantin Belousov */ 29886be9f0dSKonstantin Belousov ctx1->end = ptoa(Maxmem); 29986be9f0dSKonstantin Belousov mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, false); 30086be9f0dSKonstantin Belousov error = ctx_set_agaw(ctx1, mgaw); 30186be9f0dSKonstantin Belousov if (error != 0) { 30286be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, false, false); 30386be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 30486be9f0dSKonstantin Belousov return (NULL); 30586be9f0dSKonstantin Belousov } 30686be9f0dSKonstantin Belousov } else { 30786be9f0dSKonstantin Belousov ctx1->end = BUS_SPACE_MAXADDR; 30886be9f0dSKonstantin Belousov mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, true); 30986be9f0dSKonstantin Belousov error = ctx_set_agaw(ctx1, mgaw); 31086be9f0dSKonstantin Belousov if (error != 0) { 31186be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, false, false); 31286be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 31386be9f0dSKonstantin Belousov return (NULL); 31486be9f0dSKonstantin Belousov } 31586be9f0dSKonstantin Belousov /* Use all supported address space for remapping. */ 31686be9f0dSKonstantin Belousov ctx1->end = 1ULL << (ctx1->agaw - 1); 31786be9f0dSKonstantin Belousov } 31886be9f0dSKonstantin Belousov 31986be9f0dSKonstantin Belousov 32086be9f0dSKonstantin Belousov dmar_gas_init_ctx(ctx1); 32186be9f0dSKonstantin Belousov if (id_mapped) { 32286be9f0dSKonstantin Belousov if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { 32386be9f0dSKonstantin Belousov ctx1->pgtbl_obj = ctx_get_idmap_pgtbl(ctx1, 32486be9f0dSKonstantin Belousov ctx1->end); 32586be9f0dSKonstantin Belousov } 32686be9f0dSKonstantin Belousov ctx1->flags |= DMAR_CTX_IDMAP; 32786be9f0dSKonstantin Belousov } else { 32886be9f0dSKonstantin Belousov error = ctx_alloc_pgtbl(ctx1); 32986be9f0dSKonstantin Belousov if (error != 0) { 33086be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, true, false); 33186be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 33286be9f0dSKonstantin Belousov return (NULL); 33386be9f0dSKonstantin Belousov } 33486be9f0dSKonstantin Belousov /* Disable local apic region access */ 33586be9f0dSKonstantin Belousov error = dmar_gas_reserve_region(ctx1, 0xfee00000, 33686be9f0dSKonstantin Belousov 0xfeefffff + 1); 33786be9f0dSKonstantin Belousov if (error != 0) { 33886be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, true, true); 33986be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 34086be9f0dSKonstantin Belousov return (NULL); 34186be9f0dSKonstantin Belousov } 34286be9f0dSKonstantin Belousov error = ctx_init_rmrr(ctx1, dev); 34386be9f0dSKonstantin Belousov if (error != 0) { 34486be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, true, true); 34586be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 34686be9f0dSKonstantin Belousov return (NULL); 34786be9f0dSKonstantin Belousov } 34886be9f0dSKonstantin Belousov } 34986be9f0dSKonstantin Belousov ctxp = dmar_map_ctx_entry(ctx1, &sf); 35086be9f0dSKonstantin Belousov DMAR_LOCK(dmar); 35186be9f0dSKonstantin Belousov 35286be9f0dSKonstantin Belousov /* 35386be9f0dSKonstantin Belousov * Recheck the contexts, other thread might have 35486be9f0dSKonstantin Belousov * already allocated needed one. 35586be9f0dSKonstantin Belousov */ 35686be9f0dSKonstantin Belousov ctx = dmar_find_ctx_locked(dmar, bus, slot, func); 35786be9f0dSKonstantin Belousov if (ctx == NULL) { 35886be9f0dSKonstantin Belousov ctx = ctx1; 35986be9f0dSKonstantin Belousov ctx->domain = alloc_unrl(dmar->domids); 36086be9f0dSKonstantin Belousov if (ctx->domain == -1) { 36186be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 36286be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, true); 36386be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx, true, true); 36486be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 36586be9f0dSKonstantin Belousov return (NULL); 36686be9f0dSKonstantin Belousov } 36786be9f0dSKonstantin Belousov ctx_tag_init(ctx); 36886be9f0dSKonstantin Belousov 36986be9f0dSKonstantin Belousov /* 37086be9f0dSKonstantin Belousov * This is the first activated context for the 37186be9f0dSKonstantin Belousov * DMAR unit. Enable the translation after 37286be9f0dSKonstantin Belousov * everything is set up. 37386be9f0dSKonstantin Belousov */ 37486be9f0dSKonstantin Belousov if (LIST_EMPTY(&dmar->contexts)) 37586be9f0dSKonstantin Belousov enable = true; 37686be9f0dSKonstantin Belousov LIST_INSERT_HEAD(&dmar->contexts, ctx, link); 37786be9f0dSKonstantin Belousov ctx_id_entry_init(ctx, ctxp); 37886be9f0dSKonstantin Belousov device_printf(dev, 37986be9f0dSKonstantin Belousov "dmar%d pci%d:%d:%d:%d domain %d mgaw %d agaw %d\n", 38086be9f0dSKonstantin Belousov dmar->unit, dmar->segment, bus, slot, 38186be9f0dSKonstantin Belousov func, ctx->domain, ctx->mgaw, ctx->agaw); 38286be9f0dSKonstantin Belousov } else { 38386be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, true, true); 38486be9f0dSKonstantin Belousov } 38586be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 38686be9f0dSKonstantin Belousov } 38786be9f0dSKonstantin Belousov ctx->refs++; 38886be9f0dSKonstantin Belousov if ((ctx->flags & DMAR_CTX_RMRR) != 0) 38986be9f0dSKonstantin Belousov ctx->refs++; /* XXXKIB */ 39086be9f0dSKonstantin Belousov 39186be9f0dSKonstantin Belousov /* 39286be9f0dSKonstantin Belousov * If dmar declares Caching Mode as Set, follow 11.5 "Caching 39386be9f0dSKonstantin Belousov * Mode Consideration" and do the (global) invalidation of the 39486be9f0dSKonstantin Belousov * negative TLB entries. 39586be9f0dSKonstantin Belousov */ 39686be9f0dSKonstantin Belousov if ((dmar->hw_cap & DMAR_CAP_CM) != 0 || enable) { 39768eeb96aSKonstantin Belousov if (dmar->qi_enabled) { 39868eeb96aSKonstantin Belousov dmar_qi_invalidate_ctx_glob_locked(dmar); 39968eeb96aSKonstantin Belousov if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) 40068eeb96aSKonstantin Belousov dmar_qi_invalidate_iotlb_glob_locked(dmar); 40168eeb96aSKonstantin Belousov } else { 40286be9f0dSKonstantin Belousov error = dmar_inv_ctx_glob(dmar); 40386be9f0dSKonstantin Belousov if (error == 0 && 40486be9f0dSKonstantin Belousov (dmar->hw_ecap & DMAR_ECAP_DI) != 0) 40586be9f0dSKonstantin Belousov error = dmar_inv_iotlb_glob(dmar); 40686be9f0dSKonstantin Belousov if (error != 0) { 40786be9f0dSKonstantin Belousov dmar_free_ctx_locked(dmar, ctx); 40886be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 40986be9f0dSKonstantin Belousov return (NULL); 41086be9f0dSKonstantin Belousov } 41186be9f0dSKonstantin Belousov } 41268eeb96aSKonstantin Belousov } 41368eeb96aSKonstantin Belousov 41468eeb96aSKonstantin Belousov /* 41568eeb96aSKonstantin Belousov * The dmar lock was potentially dropped between check for the 41668eeb96aSKonstantin Belousov * empty context list and now. Recheck the state of GCMD_TE 41768eeb96aSKonstantin Belousov * to avoid unneeded command. 41868eeb96aSKonstantin Belousov */ 41968eeb96aSKonstantin Belousov if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) { 42086be9f0dSKonstantin Belousov error = dmar_enable_translation(dmar); 42186be9f0dSKonstantin Belousov if (error != 0) { 42286be9f0dSKonstantin Belousov dmar_free_ctx_locked(dmar, ctx); 42386be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 42486be9f0dSKonstantin Belousov return (NULL); 42586be9f0dSKonstantin Belousov } 42686be9f0dSKonstantin Belousov } 42786be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 42886be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 42986be9f0dSKonstantin Belousov return (ctx); 43086be9f0dSKonstantin Belousov } 43186be9f0dSKonstantin Belousov 43286be9f0dSKonstantin Belousov void 43386be9f0dSKonstantin Belousov dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) 43486be9f0dSKonstantin Belousov { 43586be9f0dSKonstantin Belousov struct sf_buf *sf; 43686be9f0dSKonstantin Belousov dmar_ctx_entry_t *ctxp; 43786be9f0dSKonstantin Belousov 43886be9f0dSKonstantin Belousov DMAR_ASSERT_LOCKED(dmar); 43986be9f0dSKonstantin Belousov KASSERT(ctx->refs >= 1, 44086be9f0dSKonstantin Belousov ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 44186be9f0dSKonstantin Belousov 44286be9f0dSKonstantin Belousov /* 44386be9f0dSKonstantin Belousov * If our reference is not last, only the dereference should 44486be9f0dSKonstantin Belousov * be performed. 44586be9f0dSKonstantin Belousov */ 44686be9f0dSKonstantin Belousov if (ctx->refs > 1) { 44786be9f0dSKonstantin Belousov ctx->refs--; 44886be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 44986be9f0dSKonstantin Belousov return; 45086be9f0dSKonstantin Belousov } 45186be9f0dSKonstantin Belousov 45286be9f0dSKonstantin Belousov KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0, 45386be9f0dSKonstantin Belousov ("lost ref on RMRR ctx %p", ctx)); 45486be9f0dSKonstantin Belousov KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 45586be9f0dSKonstantin Belousov ("lost ref on disabled ctx %p", ctx)); 45686be9f0dSKonstantin Belousov 45786be9f0dSKonstantin Belousov /* 45886be9f0dSKonstantin Belousov * Otherwise, the context entry must be cleared before the 45986be9f0dSKonstantin Belousov * page table is destroyed. The mapping of the context 46086be9f0dSKonstantin Belousov * entries page could require sleep, unlock the dmar. 46186be9f0dSKonstantin Belousov */ 46286be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 46386be9f0dSKonstantin Belousov TD_PREP_PINNED_ASSERT; 46486be9f0dSKonstantin Belousov ctxp = dmar_map_ctx_entry(ctx, &sf); 46586be9f0dSKonstantin Belousov DMAR_LOCK(dmar); 46686be9f0dSKonstantin Belousov KASSERT(ctx->refs >= 1, 46786be9f0dSKonstantin Belousov ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 46886be9f0dSKonstantin Belousov 46986be9f0dSKonstantin Belousov /* 47086be9f0dSKonstantin Belousov * Other thread might have referenced the context, in which 47186be9f0dSKonstantin Belousov * case again only the dereference should be performed. 47286be9f0dSKonstantin Belousov */ 47386be9f0dSKonstantin Belousov if (ctx->refs > 1) { 47486be9f0dSKonstantin Belousov ctx->refs--; 47586be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 47686be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 47786be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 47886be9f0dSKonstantin Belousov return; 47986be9f0dSKonstantin Belousov } 48086be9f0dSKonstantin Belousov 48186be9f0dSKonstantin Belousov KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0, 48286be9f0dSKonstantin Belousov ("lost ref on RMRR ctx %p", ctx)); 48386be9f0dSKonstantin Belousov KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 48486be9f0dSKonstantin Belousov ("lost ref on disabled ctx %p", ctx)); 48586be9f0dSKonstantin Belousov 48686be9f0dSKonstantin Belousov /* 48786be9f0dSKonstantin Belousov * Clear the context pointer and flush the caches. 48886be9f0dSKonstantin Belousov * XXXKIB: cannot do this if any RMRR entries are still present. 48986be9f0dSKonstantin Belousov */ 49086be9f0dSKonstantin Belousov dmar_pte_clear(&ctxp->ctx1); 49186be9f0dSKonstantin Belousov ctxp->ctx2 = 0; 49286be9f0dSKonstantin Belousov dmar_inv_ctx_glob(dmar); 49368eeb96aSKonstantin Belousov if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { 49468eeb96aSKonstantin Belousov if (dmar->qi_enabled) 49568eeb96aSKonstantin Belousov dmar_qi_invalidate_iotlb_glob_locked(dmar); 49668eeb96aSKonstantin Belousov else 49786be9f0dSKonstantin Belousov dmar_inv_iotlb_glob(dmar); 49868eeb96aSKonstantin Belousov } 49986be9f0dSKonstantin Belousov LIST_REMOVE(ctx, link); 50086be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 50186be9f0dSKonstantin Belousov 50286be9f0dSKonstantin Belousov /* 50386be9f0dSKonstantin Belousov * The rest of the destruction is invisible for other users of 50486be9f0dSKonstantin Belousov * the dmar unit. 50586be9f0dSKonstantin Belousov */ 50686be9f0dSKonstantin Belousov taskqueue_drain(dmar->delayed_taskqueue, &ctx->unload_task); 50786be9f0dSKonstantin Belousov KASSERT(TAILQ_EMPTY(&ctx->unload_entries), 50886be9f0dSKonstantin Belousov ("unfinished unloads %p", ctx)); 50986be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 51086be9f0dSKonstantin Belousov free_unr(dmar->domids, ctx->domain); 51186be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx, true, true); 51286be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 51386be9f0dSKonstantin Belousov } 51486be9f0dSKonstantin Belousov 51586be9f0dSKonstantin Belousov void 51686be9f0dSKonstantin Belousov dmar_free_ctx(struct dmar_ctx *ctx) 51786be9f0dSKonstantin Belousov { 51886be9f0dSKonstantin Belousov struct dmar_unit *dmar; 51986be9f0dSKonstantin Belousov 52086be9f0dSKonstantin Belousov dmar = ctx->dmar; 52186be9f0dSKonstantin Belousov DMAR_LOCK(dmar); 52286be9f0dSKonstantin Belousov dmar_free_ctx_locked(dmar, ctx); 52386be9f0dSKonstantin Belousov } 52486be9f0dSKonstantin Belousov 52586be9f0dSKonstantin Belousov struct dmar_ctx * 52686be9f0dSKonstantin Belousov dmar_find_ctx_locked(struct dmar_unit *dmar, int bus, int slot, int func) 52786be9f0dSKonstantin Belousov { 52886be9f0dSKonstantin Belousov struct dmar_ctx *ctx; 52986be9f0dSKonstantin Belousov 53086be9f0dSKonstantin Belousov DMAR_ASSERT_LOCKED(dmar); 53186be9f0dSKonstantin Belousov 53286be9f0dSKonstantin Belousov LIST_FOREACH(ctx, &dmar->contexts, link) { 53386be9f0dSKonstantin Belousov if (ctx->bus == bus && ctx->slot == slot && ctx->func == func) 53486be9f0dSKonstantin Belousov return (ctx); 53586be9f0dSKonstantin Belousov } 53686be9f0dSKonstantin Belousov return (NULL); 53786be9f0dSKonstantin Belousov } 53886be9f0dSKonstantin Belousov 53986be9f0dSKonstantin Belousov void 54068eeb96aSKonstantin Belousov dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free) 54168eeb96aSKonstantin Belousov { 54268eeb96aSKonstantin Belousov struct dmar_ctx *ctx; 54368eeb96aSKonstantin Belousov 54468eeb96aSKonstantin Belousov ctx = entry->ctx; 54568eeb96aSKonstantin Belousov DMAR_CTX_LOCK(ctx); 54668eeb96aSKonstantin Belousov if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0) 54768eeb96aSKonstantin Belousov dmar_gas_free_region(ctx, entry); 54868eeb96aSKonstantin Belousov else 54968eeb96aSKonstantin Belousov dmar_gas_free_space(ctx, entry); 55068eeb96aSKonstantin Belousov DMAR_CTX_UNLOCK(ctx); 55168eeb96aSKonstantin Belousov if (free) 55268eeb96aSKonstantin Belousov dmar_gas_free_entry(ctx, entry); 55368eeb96aSKonstantin Belousov else 55468eeb96aSKonstantin Belousov entry->flags = 0; 55568eeb96aSKonstantin Belousov } 55668eeb96aSKonstantin Belousov 55768eeb96aSKonstantin Belousov void 55868eeb96aSKonstantin Belousov dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free) 55968eeb96aSKonstantin Belousov { 56068eeb96aSKonstantin Belousov struct dmar_unit *unit; 56168eeb96aSKonstantin Belousov 56268eeb96aSKonstantin Belousov unit = entry->ctx->dmar; 56368eeb96aSKonstantin Belousov if (unit->qi_enabled) { 56468eeb96aSKonstantin Belousov DMAR_LOCK(unit); 56568eeb96aSKonstantin Belousov dmar_qi_invalidate_locked(entry->ctx, entry->start, 56668eeb96aSKonstantin Belousov entry->end - entry->start, &entry->gseq); 56768eeb96aSKonstantin Belousov if (!free) 56868eeb96aSKonstantin Belousov entry->flags |= DMAR_MAP_ENTRY_QI_NF; 56968eeb96aSKonstantin Belousov TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 57068eeb96aSKonstantin Belousov DMAR_UNLOCK(unit); 57168eeb96aSKonstantin Belousov } else { 57268eeb96aSKonstantin Belousov ctx_flush_iotlb_sync(entry->ctx, entry->start, entry->end - 57368eeb96aSKonstantin Belousov entry->start); 57468eeb96aSKonstantin Belousov dmar_ctx_free_entry(entry, free); 57568eeb96aSKonstantin Belousov } 57668eeb96aSKonstantin Belousov } 57768eeb96aSKonstantin Belousov 57868eeb96aSKonstantin Belousov void 57986be9f0dSKonstantin Belousov dmar_ctx_unload(struct dmar_ctx *ctx, struct dmar_map_entries_tailq *entries, 58086be9f0dSKonstantin Belousov bool cansleep) 58186be9f0dSKonstantin Belousov { 58268eeb96aSKonstantin Belousov struct dmar_unit *unit; 58368eeb96aSKonstantin Belousov struct dmar_map_entry *entry, *entry1; 58468eeb96aSKonstantin Belousov struct dmar_qi_genseq gseq; 58586be9f0dSKonstantin Belousov int error; 58686be9f0dSKonstantin Belousov 58768eeb96aSKonstantin Belousov unit = ctx->dmar; 58868eeb96aSKonstantin Belousov 58968eeb96aSKonstantin Belousov TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 59086be9f0dSKonstantin Belousov KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0, 59186be9f0dSKonstantin Belousov ("not mapped entry %p %p", ctx, entry)); 59286be9f0dSKonstantin Belousov error = ctx_unmap_buf(ctx, entry->start, entry->end - 59386be9f0dSKonstantin Belousov entry->start, cansleep ? DMAR_PGF_WAITOK : 0); 59486be9f0dSKonstantin Belousov KASSERT(error == 0, ("unmap %p error %d", ctx, error)); 59568eeb96aSKonstantin Belousov if (!unit->qi_enabled) { 59668eeb96aSKonstantin Belousov ctx_flush_iotlb_sync(ctx, entry->start, 59768eeb96aSKonstantin Belousov entry->end - entry->start); 59868eeb96aSKonstantin Belousov TAILQ_REMOVE(entries, entry, dmamap_link); 59968eeb96aSKonstantin Belousov dmar_ctx_free_entry(entry, true); 60086be9f0dSKonstantin Belousov } 60186be9f0dSKonstantin Belousov } 60268eeb96aSKonstantin Belousov if (TAILQ_EMPTY(entries)) 60368eeb96aSKonstantin Belousov return; 60468eeb96aSKonstantin Belousov 60568eeb96aSKonstantin Belousov KASSERT(unit->qi_enabled, ("loaded entry left")); 60668eeb96aSKonstantin Belousov DMAR_LOCK(unit); 60768eeb96aSKonstantin Belousov TAILQ_FOREACH(entry, entries, dmamap_link) { 60868eeb96aSKonstantin Belousov entry->gseq.gen = 0; 60968eeb96aSKonstantin Belousov entry->gseq.seq = 0; 61068eeb96aSKonstantin Belousov dmar_qi_invalidate_locked(ctx, entry->start, entry->end - 61168eeb96aSKonstantin Belousov entry->start, TAILQ_NEXT(entry, dmamap_link) == NULL ? 61268eeb96aSKonstantin Belousov &gseq : NULL); 61368eeb96aSKonstantin Belousov } 61468eeb96aSKonstantin Belousov TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 61568eeb96aSKonstantin Belousov entry->gseq = gseq; 61668eeb96aSKonstantin Belousov TAILQ_REMOVE(entries, entry, dmamap_link); 61768eeb96aSKonstantin Belousov TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 61868eeb96aSKonstantin Belousov } 61968eeb96aSKonstantin Belousov DMAR_UNLOCK(unit); 62068eeb96aSKonstantin Belousov } 62186be9f0dSKonstantin Belousov 62286be9f0dSKonstantin Belousov static void 62386be9f0dSKonstantin Belousov dmar_ctx_unload_task(void *arg, int pending) 62486be9f0dSKonstantin Belousov { 62586be9f0dSKonstantin Belousov struct dmar_ctx *ctx; 62686be9f0dSKonstantin Belousov struct dmar_map_entries_tailq entries; 62786be9f0dSKonstantin Belousov 62886be9f0dSKonstantin Belousov ctx = arg; 62986be9f0dSKonstantin Belousov TAILQ_INIT(&entries); 63086be9f0dSKonstantin Belousov 63186be9f0dSKonstantin Belousov for (;;) { 63286be9f0dSKonstantin Belousov DMAR_CTX_LOCK(ctx); 63386be9f0dSKonstantin Belousov TAILQ_SWAP(&ctx->unload_entries, &entries, dmar_map_entry, 63486be9f0dSKonstantin Belousov dmamap_link); 63586be9f0dSKonstantin Belousov DMAR_CTX_UNLOCK(ctx); 63686be9f0dSKonstantin Belousov if (TAILQ_EMPTY(&entries)) 63786be9f0dSKonstantin Belousov break; 63886be9f0dSKonstantin Belousov dmar_ctx_unload(ctx, &entries, true); 63986be9f0dSKonstantin Belousov } 64086be9f0dSKonstantin Belousov } 641