186be9f0dSKonstantin Belousov /*- 286be9f0dSKonstantin Belousov * Copyright (c) 2013 The FreeBSD Foundation 386be9f0dSKonstantin Belousov * All rights reserved. 486be9f0dSKonstantin Belousov * 586be9f0dSKonstantin Belousov * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 686be9f0dSKonstantin Belousov * under sponsorship from the FreeBSD Foundation. 786be9f0dSKonstantin Belousov * 886be9f0dSKonstantin Belousov * Redistribution and use in source and binary forms, with or without 986be9f0dSKonstantin Belousov * modification, are permitted provided that the following conditions 1086be9f0dSKonstantin Belousov * are met: 1186be9f0dSKonstantin Belousov * 1. Redistributions of source code must retain the above copyright 1286be9f0dSKonstantin Belousov * notice, this list of conditions and the following disclaimer. 1386be9f0dSKonstantin Belousov * 2. Redistributions in binary form must reproduce the above copyright 1486be9f0dSKonstantin Belousov * notice, this list of conditions and the following disclaimer in the 1586be9f0dSKonstantin Belousov * documentation and/or other materials provided with the distribution. 1686be9f0dSKonstantin Belousov * 1786be9f0dSKonstantin Belousov * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1886be9f0dSKonstantin Belousov * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1986be9f0dSKonstantin Belousov * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2086be9f0dSKonstantin Belousov * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2186be9f0dSKonstantin Belousov * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2286be9f0dSKonstantin Belousov * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2386be9f0dSKonstantin Belousov * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2486be9f0dSKonstantin Belousov * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2586be9f0dSKonstantin Belousov * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2686be9f0dSKonstantin Belousov * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2786be9f0dSKonstantin Belousov * SUCH DAMAGE. 2886be9f0dSKonstantin Belousov */ 2986be9f0dSKonstantin Belousov 3086be9f0dSKonstantin Belousov #include <sys/cdefs.h> 3186be9f0dSKonstantin Belousov __FBSDID("$FreeBSD$"); 3286be9f0dSKonstantin Belousov 3386be9f0dSKonstantin Belousov #include <sys/param.h> 3486be9f0dSKonstantin Belousov #include <sys/systm.h> 3586be9f0dSKonstantin Belousov #include <sys/malloc.h> 3686be9f0dSKonstantin Belousov #include <sys/bus.h> 3786be9f0dSKonstantin Belousov #include <sys/interrupt.h> 3886be9f0dSKonstantin Belousov #include <sys/kernel.h> 3986be9f0dSKonstantin Belousov #include <sys/ktr.h> 4086be9f0dSKonstantin Belousov #include <sys/limits.h> 4186be9f0dSKonstantin Belousov #include <sys/lock.h> 4286be9f0dSKonstantin Belousov #include <sys/memdesc.h> 4386be9f0dSKonstantin Belousov #include <sys/mutex.h> 4486be9f0dSKonstantin Belousov #include <sys/proc.h> 4586be9f0dSKonstantin Belousov #include <sys/rwlock.h> 4686be9f0dSKonstantin Belousov #include <sys/rman.h> 4786be9f0dSKonstantin Belousov #include <sys/sysctl.h> 4886be9f0dSKonstantin Belousov #include <sys/taskqueue.h> 4986be9f0dSKonstantin Belousov #include <sys/tree.h> 5086be9f0dSKonstantin Belousov #include <sys/uio.h> 5186be9f0dSKonstantin Belousov #include <vm/vm.h> 5286be9f0dSKonstantin Belousov #include <vm/vm_extern.h> 5386be9f0dSKonstantin Belousov #include <vm/vm_kern.h> 5486be9f0dSKonstantin Belousov #include <vm/vm_object.h> 5586be9f0dSKonstantin Belousov #include <vm/vm_page.h> 5686be9f0dSKonstantin Belousov #include <vm/vm_pager.h> 5786be9f0dSKonstantin Belousov #include <vm/vm_map.h> 5886be9f0dSKonstantin Belousov #include <machine/atomic.h> 5986be9f0dSKonstantin Belousov #include <machine/bus.h> 6086be9f0dSKonstantin Belousov #include <machine/md_var.h> 6186be9f0dSKonstantin Belousov #include <machine/specialreg.h> 6286be9f0dSKonstantin Belousov #include <x86/include/busdma_impl.h> 6386be9f0dSKonstantin Belousov #include <x86/iommu/intel_reg.h> 6486be9f0dSKonstantin Belousov #include <x86/iommu/busdma_dmar.h> 6586be9f0dSKonstantin Belousov #include <x86/iommu/intel_dmar.h> 6667499354SRyan Stone #include <dev/pci/pcireg.h> 6786be9f0dSKonstantin Belousov #include <dev/pci/pcivar.h> 6886be9f0dSKonstantin Belousov 6986be9f0dSKonstantin Belousov static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); 7086be9f0dSKonstantin Belousov 7186be9f0dSKonstantin Belousov static void dmar_ctx_unload_task(void *arg, int pending); 7286be9f0dSKonstantin Belousov 7386be9f0dSKonstantin Belousov static void 7486be9f0dSKonstantin Belousov dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) 7586be9f0dSKonstantin Belousov { 7686be9f0dSKonstantin Belousov struct sf_buf *sf; 7786be9f0dSKonstantin Belousov dmar_root_entry_t *re; 7886be9f0dSKonstantin Belousov vm_page_t ctxm; 7986be9f0dSKonstantin Belousov 8086be9f0dSKonstantin Belousov /* 8186be9f0dSKonstantin Belousov * Allocated context page must be linked. 8286be9f0dSKonstantin Belousov */ 8386be9f0dSKonstantin Belousov ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_NOALLOC); 8486be9f0dSKonstantin Belousov if (ctxm != NULL) 8586be9f0dSKonstantin Belousov return; 8686be9f0dSKonstantin Belousov 8786be9f0dSKonstantin Belousov /* 8886be9f0dSKonstantin Belousov * Page not present, allocate and link. Note that other 8986be9f0dSKonstantin Belousov * thread might execute this sequence in parallel. This 9086be9f0dSKonstantin Belousov * should be safe, because the context entries written by both 9186be9f0dSKonstantin Belousov * threads are equal. 9286be9f0dSKonstantin Belousov */ 9386be9f0dSKonstantin Belousov TD_PREP_PINNED_ASSERT; 9486be9f0dSKonstantin Belousov ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_ZERO | 9586be9f0dSKonstantin Belousov DMAR_PGF_WAITOK); 9686be9f0dSKonstantin Belousov re = dmar_map_pgtbl(dmar->ctx_obj, 0, DMAR_PGF_NOALLOC, &sf); 9786be9f0dSKonstantin Belousov re += bus; 9886be9f0dSKonstantin Belousov dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & 9986be9f0dSKonstantin Belousov VM_PAGE_TO_PHYS(ctxm))); 10086be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 10186be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 10286be9f0dSKonstantin Belousov } 10386be9f0dSKonstantin Belousov 10486be9f0dSKonstantin Belousov static dmar_ctx_entry_t * 10586be9f0dSKonstantin Belousov dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) 10686be9f0dSKonstantin Belousov { 10786be9f0dSKonstantin Belousov dmar_ctx_entry_t *ctxp; 10886be9f0dSKonstantin Belousov 10967499354SRyan Stone ctxp = dmar_map_pgtbl(ctx->dmar->ctx_obj, 1 + PCI_RID2BUS(ctx->rid), 11086be9f0dSKonstantin Belousov DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp); 11167499354SRyan Stone ctxp += ctx->rid & 0xff; 11286be9f0dSKonstantin Belousov return (ctxp); 11386be9f0dSKonstantin Belousov } 11486be9f0dSKonstantin Belousov 11586be9f0dSKonstantin Belousov static void 11667499354SRyan Stone ctx_tag_init(struct dmar_ctx *ctx, device_t dev) 11786be9f0dSKonstantin Belousov { 11886be9f0dSKonstantin Belousov bus_addr_t maxaddr; 11986be9f0dSKonstantin Belousov 12086be9f0dSKonstantin Belousov maxaddr = MIN(ctx->end, BUS_SPACE_MAXADDR); 12186be9f0dSKonstantin Belousov ctx->ctx_tag.common.ref_count = 1; /* Prevent free */ 12286be9f0dSKonstantin Belousov ctx->ctx_tag.common.impl = &bus_dma_dmar_impl; 12386be9f0dSKonstantin Belousov ctx->ctx_tag.common.boundary = PCI_DMA_BOUNDARY; 12486be9f0dSKonstantin Belousov ctx->ctx_tag.common.lowaddr = maxaddr; 12586be9f0dSKonstantin Belousov ctx->ctx_tag.common.highaddr = maxaddr; 12686be9f0dSKonstantin Belousov ctx->ctx_tag.common.maxsize = maxaddr; 12786be9f0dSKonstantin Belousov ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED; 12886be9f0dSKonstantin Belousov ctx->ctx_tag.common.maxsegsz = maxaddr; 12986be9f0dSKonstantin Belousov ctx->ctx_tag.ctx = ctx; 13067499354SRyan Stone ctx->ctx_tag.owner = dev; 13186be9f0dSKonstantin Belousov /* XXXKIB initialize tag further */ 13286be9f0dSKonstantin Belousov } 13386be9f0dSKonstantin Belousov 13486be9f0dSKonstantin Belousov static void 13586be9f0dSKonstantin Belousov ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp) 13686be9f0dSKonstantin Belousov { 13786be9f0dSKonstantin Belousov struct dmar_unit *unit; 13886be9f0dSKonstantin Belousov vm_page_t ctx_root; 13986be9f0dSKonstantin Belousov 14086be9f0dSKonstantin Belousov unit = ctx->dmar; 14186be9f0dSKonstantin Belousov KASSERT(ctxp->ctx1 == 0 && ctxp->ctx2 == 0, 14286be9f0dSKonstantin Belousov ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", 14367499354SRyan Stone unit->unit, pci_get_bus(ctx->ctx_tag.owner), 14467499354SRyan Stone pci_get_slot(ctx->ctx_tag.owner), 14567499354SRyan Stone pci_get_function(ctx->ctx_tag.owner), 14667499354SRyan Stone ctxp->ctx1, 14786be9f0dSKonstantin Belousov ctxp->ctx2)); 14886be9f0dSKonstantin Belousov ctxp->ctx2 = DMAR_CTX2_DID(ctx->domain); 14986be9f0dSKonstantin Belousov ctxp->ctx2 |= ctx->awlvl; 15086be9f0dSKonstantin Belousov if ((ctx->flags & DMAR_CTX_IDMAP) != 0 && 15186be9f0dSKonstantin Belousov (unit->hw_ecap & DMAR_ECAP_PT) != 0) { 15286be9f0dSKonstantin Belousov KASSERT(ctx->pgtbl_obj == NULL, 15386be9f0dSKonstantin Belousov ("ctx %p non-null pgtbl_obj", ctx)); 15486be9f0dSKonstantin Belousov dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); 15586be9f0dSKonstantin Belousov } else { 15686be9f0dSKonstantin Belousov ctx_root = dmar_pgalloc(ctx->pgtbl_obj, 0, DMAR_PGF_NOALLOC); 15786be9f0dSKonstantin Belousov dmar_pte_store(&ctxp->ctx1, DMAR_CTX1_T_UNTR | 15886be9f0dSKonstantin Belousov (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | 15986be9f0dSKonstantin Belousov DMAR_CTX1_P); 16086be9f0dSKonstantin Belousov } 16186be9f0dSKonstantin Belousov } 16286be9f0dSKonstantin Belousov 16386be9f0dSKonstantin Belousov static int 16486be9f0dSKonstantin Belousov ctx_init_rmrr(struct dmar_ctx *ctx, device_t dev) 16586be9f0dSKonstantin Belousov { 16686be9f0dSKonstantin Belousov struct dmar_map_entries_tailq rmrr_entries; 16786be9f0dSKonstantin Belousov struct dmar_map_entry *entry, *entry1; 16886be9f0dSKonstantin Belousov vm_page_t *ma; 16986be9f0dSKonstantin Belousov dmar_gaddr_t start, end; 17086be9f0dSKonstantin Belousov vm_pindex_t size, i; 17186be9f0dSKonstantin Belousov int error, error1; 17286be9f0dSKonstantin Belousov 17386be9f0dSKonstantin Belousov error = 0; 17486be9f0dSKonstantin Belousov TAILQ_INIT(&rmrr_entries); 17586be9f0dSKonstantin Belousov dmar_ctx_parse_rmrr(ctx, dev, &rmrr_entries); 17686be9f0dSKonstantin Belousov TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) { 17786be9f0dSKonstantin Belousov /* 17886be9f0dSKonstantin Belousov * VT-d specification requires that the start of an 17986be9f0dSKonstantin Belousov * RMRR entry is 4k-aligned. Buggy BIOSes put 18086be9f0dSKonstantin Belousov * anything into the start and end fields. Truncate 18186be9f0dSKonstantin Belousov * and round as neccesary. 18286be9f0dSKonstantin Belousov * 18386be9f0dSKonstantin Belousov * We also allow the overlapping RMRR entries, see 18486be9f0dSKonstantin Belousov * dmar_gas_alloc_region(). 18586be9f0dSKonstantin Belousov */ 18686be9f0dSKonstantin Belousov start = entry->start; 18786be9f0dSKonstantin Belousov end = entry->end; 18886be9f0dSKonstantin Belousov entry->start = trunc_page(start); 18986be9f0dSKonstantin Belousov entry->end = round_page(end); 190e02b05b3SKonstantin Belousov if (entry->start == entry->end) { 191e02b05b3SKonstantin Belousov /* Workaround for some AMI (?) BIOSes */ 192e02b05b3SKonstantin Belousov if (bootverbose) { 193e02b05b3SKonstantin Belousov device_printf(dev, "BIOS bug: dmar%d RMRR " 194e02b05b3SKonstantin Belousov "region (%jx, %jx) corrected\n", 195e02b05b3SKonstantin Belousov ctx->dmar->unit, start, end); 196e02b05b3SKonstantin Belousov } 197e02b05b3SKonstantin Belousov entry->end += DMAR_PAGE_SIZE * 0x20; 198e02b05b3SKonstantin Belousov } 19986be9f0dSKonstantin Belousov size = OFF_TO_IDX(entry->end - entry->start); 20086be9f0dSKonstantin Belousov ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); 20186be9f0dSKonstantin Belousov for (i = 0; i < size; i++) { 20286be9f0dSKonstantin Belousov ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, 20386be9f0dSKonstantin Belousov VM_MEMATTR_DEFAULT); 20486be9f0dSKonstantin Belousov } 20586be9f0dSKonstantin Belousov error1 = dmar_gas_map_region(ctx, entry, DMAR_MAP_ENTRY_READ | 20686be9f0dSKonstantin Belousov DMAR_MAP_ENTRY_WRITE, DMAR_GM_CANWAIT, ma); 20786be9f0dSKonstantin Belousov /* 20886be9f0dSKonstantin Belousov * Non-failed RMRR entries are owned by context rb 20986be9f0dSKonstantin Belousov * tree. Get rid of the failed entry, but do not stop 21086be9f0dSKonstantin Belousov * the loop. Rest of the parsed RMRR entries are 21186be9f0dSKonstantin Belousov * loaded and removed on the context destruction. 21286be9f0dSKonstantin Belousov */ 21386be9f0dSKonstantin Belousov if (error1 == 0 && entry->end != entry->start) { 21486be9f0dSKonstantin Belousov DMAR_LOCK(ctx->dmar); 21586be9f0dSKonstantin Belousov ctx->flags |= DMAR_CTX_RMRR; 21686be9f0dSKonstantin Belousov DMAR_UNLOCK(ctx->dmar); 21786be9f0dSKonstantin Belousov } else { 21886be9f0dSKonstantin Belousov if (error1 != 0) { 21986be9f0dSKonstantin Belousov device_printf(dev, 22086be9f0dSKonstantin Belousov "dmar%d failed to map RMRR region (%jx, %jx) %d\n", 22186be9f0dSKonstantin Belousov ctx->dmar->unit, start, end, error1); 22286be9f0dSKonstantin Belousov error = error1; 22386be9f0dSKonstantin Belousov } 22486be9f0dSKonstantin Belousov TAILQ_REMOVE(&rmrr_entries, entry, unroll_link); 22586be9f0dSKonstantin Belousov dmar_gas_free_entry(ctx, entry); 22686be9f0dSKonstantin Belousov } 22786be9f0dSKonstantin Belousov for (i = 0; i < size; i++) 22886be9f0dSKonstantin Belousov vm_page_putfake(ma[i]); 22986be9f0dSKonstantin Belousov free(ma, M_TEMP); 23086be9f0dSKonstantin Belousov } 23186be9f0dSKonstantin Belousov return (error); 23286be9f0dSKonstantin Belousov } 23386be9f0dSKonstantin Belousov 23486be9f0dSKonstantin Belousov static struct dmar_ctx * 23567499354SRyan Stone dmar_get_ctx_alloc(struct dmar_unit *dmar, uint16_t rid) 23686be9f0dSKonstantin Belousov { 23786be9f0dSKonstantin Belousov struct dmar_ctx *ctx; 23886be9f0dSKonstantin Belousov 23986be9f0dSKonstantin Belousov ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); 24086be9f0dSKonstantin Belousov RB_INIT(&ctx->rb_root); 24186be9f0dSKonstantin Belousov TAILQ_INIT(&ctx->unload_entries); 24286be9f0dSKonstantin Belousov TASK_INIT(&ctx->unload_task, 0, dmar_ctx_unload_task, ctx); 24386be9f0dSKonstantin Belousov mtx_init(&ctx->lock, "dmarctx", NULL, MTX_DEF); 24486be9f0dSKonstantin Belousov ctx->dmar = dmar; 24567499354SRyan Stone ctx->rid = rid; 24686be9f0dSKonstantin Belousov return (ctx); 24786be9f0dSKonstantin Belousov } 24886be9f0dSKonstantin Belousov 24986be9f0dSKonstantin Belousov static void 25086be9f0dSKonstantin Belousov dmar_ctx_dtr(struct dmar_ctx *ctx, bool gas_inited, bool pgtbl_inited) 25186be9f0dSKonstantin Belousov { 25286be9f0dSKonstantin Belousov 25386be9f0dSKonstantin Belousov if (gas_inited) { 25486be9f0dSKonstantin Belousov DMAR_CTX_LOCK(ctx); 25586be9f0dSKonstantin Belousov dmar_gas_fini_ctx(ctx); 25686be9f0dSKonstantin Belousov DMAR_CTX_UNLOCK(ctx); 25786be9f0dSKonstantin Belousov } 25886be9f0dSKonstantin Belousov if (pgtbl_inited) { 25986be9f0dSKonstantin Belousov if (ctx->pgtbl_obj != NULL) 26086be9f0dSKonstantin Belousov DMAR_CTX_PGLOCK(ctx); 26186be9f0dSKonstantin Belousov ctx_free_pgtbl(ctx); 26286be9f0dSKonstantin Belousov } 26386be9f0dSKonstantin Belousov mtx_destroy(&ctx->lock); 26486be9f0dSKonstantin Belousov free(ctx, M_DMAR_CTX); 26586be9f0dSKonstantin Belousov } 26686be9f0dSKonstantin Belousov 26786be9f0dSKonstantin Belousov struct dmar_ctx * 26867499354SRyan Stone dmar_get_ctx(struct dmar_unit *dmar, device_t dev, uint16_t rid, bool id_mapped, 26967499354SRyan Stone bool rmrr_init) 27086be9f0dSKonstantin Belousov { 27186be9f0dSKonstantin Belousov struct dmar_ctx *ctx, *ctx1; 27286be9f0dSKonstantin Belousov dmar_ctx_entry_t *ctxp; 27386be9f0dSKonstantin Belousov struct sf_buf *sf; 27467499354SRyan Stone int bus, slot, func, error, mgaw; 27586be9f0dSKonstantin Belousov bool enable; 27686be9f0dSKonstantin Belousov 27767499354SRyan Stone bus = pci_get_bus(dev); 27867499354SRyan Stone slot = pci_get_slot(dev); 27967499354SRyan Stone func = pci_get_function(dev); 28086be9f0dSKonstantin Belousov enable = false; 28186be9f0dSKonstantin Belousov TD_PREP_PINNED_ASSERT; 28286be9f0dSKonstantin Belousov DMAR_LOCK(dmar); 28367499354SRyan Stone ctx = dmar_find_ctx_locked(dmar, rid); 28486be9f0dSKonstantin Belousov error = 0; 28586be9f0dSKonstantin Belousov if (ctx == NULL) { 28686be9f0dSKonstantin Belousov /* 28786be9f0dSKonstantin Belousov * Perform the allocations which require sleep or have 28886be9f0dSKonstantin Belousov * higher chance to succeed if the sleep is allowed. 28986be9f0dSKonstantin Belousov */ 29086be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 291*b29d186cSKonstantin Belousov dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid)); 29267499354SRyan Stone ctx1 = dmar_get_ctx_alloc(dmar, rid); 29386be9f0dSKonstantin Belousov 29486be9f0dSKonstantin Belousov if (id_mapped) { 29586be9f0dSKonstantin Belousov /* 29686be9f0dSKonstantin Belousov * For now, use the maximal usable physical 29786be9f0dSKonstantin Belousov * address of the installed memory to 29886be9f0dSKonstantin Belousov * calculate the mgaw. It is useful for the 29986be9f0dSKonstantin Belousov * identity mapping, and less so for the 30086be9f0dSKonstantin Belousov * virtualized bus address space. 30186be9f0dSKonstantin Belousov */ 30286be9f0dSKonstantin Belousov ctx1->end = ptoa(Maxmem); 30386be9f0dSKonstantin Belousov mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, false); 30486be9f0dSKonstantin Belousov error = ctx_set_agaw(ctx1, mgaw); 30586be9f0dSKonstantin Belousov if (error != 0) { 30686be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, false, false); 30786be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 30886be9f0dSKonstantin Belousov return (NULL); 30986be9f0dSKonstantin Belousov } 31086be9f0dSKonstantin Belousov } else { 31186be9f0dSKonstantin Belousov ctx1->end = BUS_SPACE_MAXADDR; 31286be9f0dSKonstantin Belousov mgaw = dmar_maxaddr2mgaw(dmar, ctx1->end, true); 31386be9f0dSKonstantin Belousov error = ctx_set_agaw(ctx1, mgaw); 31486be9f0dSKonstantin Belousov if (error != 0) { 31586be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, false, false); 31686be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 31786be9f0dSKonstantin Belousov return (NULL); 31886be9f0dSKonstantin Belousov } 31986be9f0dSKonstantin Belousov /* Use all supported address space for remapping. */ 32086be9f0dSKonstantin Belousov ctx1->end = 1ULL << (ctx1->agaw - 1); 32186be9f0dSKonstantin Belousov } 32286be9f0dSKonstantin Belousov 32386be9f0dSKonstantin Belousov 32486be9f0dSKonstantin Belousov dmar_gas_init_ctx(ctx1); 32586be9f0dSKonstantin Belousov if (id_mapped) { 32686be9f0dSKonstantin Belousov if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { 32786be9f0dSKonstantin Belousov ctx1->pgtbl_obj = ctx_get_idmap_pgtbl(ctx1, 32886be9f0dSKonstantin Belousov ctx1->end); 32986be9f0dSKonstantin Belousov } 33086be9f0dSKonstantin Belousov ctx1->flags |= DMAR_CTX_IDMAP; 33186be9f0dSKonstantin Belousov } else { 33286be9f0dSKonstantin Belousov error = ctx_alloc_pgtbl(ctx1); 33386be9f0dSKonstantin Belousov if (error != 0) { 33486be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, true, false); 33586be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 33686be9f0dSKonstantin Belousov return (NULL); 33786be9f0dSKonstantin Belousov } 33886be9f0dSKonstantin Belousov /* Disable local apic region access */ 33986be9f0dSKonstantin Belousov error = dmar_gas_reserve_region(ctx1, 0xfee00000, 34086be9f0dSKonstantin Belousov 0xfeefffff + 1); 34186be9f0dSKonstantin Belousov if (error != 0) { 34286be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, true, true); 34386be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 34486be9f0dSKonstantin Belousov return (NULL); 34586be9f0dSKonstantin Belousov } 34686be9f0dSKonstantin Belousov error = ctx_init_rmrr(ctx1, dev); 34786be9f0dSKonstantin Belousov if (error != 0) { 34886be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, true, true); 34986be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 35086be9f0dSKonstantin Belousov return (NULL); 35186be9f0dSKonstantin Belousov } 35286be9f0dSKonstantin Belousov } 35386be9f0dSKonstantin Belousov ctxp = dmar_map_ctx_entry(ctx1, &sf); 35486be9f0dSKonstantin Belousov DMAR_LOCK(dmar); 35586be9f0dSKonstantin Belousov 35686be9f0dSKonstantin Belousov /* 35786be9f0dSKonstantin Belousov * Recheck the contexts, other thread might have 35886be9f0dSKonstantin Belousov * already allocated needed one. 35986be9f0dSKonstantin Belousov */ 36067499354SRyan Stone ctx = dmar_find_ctx_locked(dmar, rid); 36186be9f0dSKonstantin Belousov if (ctx == NULL) { 36286be9f0dSKonstantin Belousov ctx = ctx1; 3639d0bc6d8SKonstantin Belousov ctx->ctx_tag.owner = dev; 36486be9f0dSKonstantin Belousov ctx->domain = alloc_unrl(dmar->domids); 36586be9f0dSKonstantin Belousov if (ctx->domain == -1) { 36686be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 36786be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, true); 36886be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx, true, true); 36986be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 37086be9f0dSKonstantin Belousov return (NULL); 37186be9f0dSKonstantin Belousov } 37267499354SRyan Stone ctx_tag_init(ctx, dev); 37386be9f0dSKonstantin Belousov 37486be9f0dSKonstantin Belousov /* 37586be9f0dSKonstantin Belousov * This is the first activated context for the 37686be9f0dSKonstantin Belousov * DMAR unit. Enable the translation after 37786be9f0dSKonstantin Belousov * everything is set up. 37886be9f0dSKonstantin Belousov */ 37986be9f0dSKonstantin Belousov if (LIST_EMPTY(&dmar->contexts)) 38086be9f0dSKonstantin Belousov enable = true; 38186be9f0dSKonstantin Belousov LIST_INSERT_HEAD(&dmar->contexts, ctx, link); 38286be9f0dSKonstantin Belousov ctx_id_entry_init(ctx, ctxp); 38386be9f0dSKonstantin Belousov device_printf(dev, 3849d0bc6d8SKonstantin Belousov "dmar%d pci%d:%d:%d:%d domain %d mgaw %d " 3859d0bc6d8SKonstantin Belousov "agaw %d %s-mapped\n", 38686be9f0dSKonstantin Belousov dmar->unit, dmar->segment, bus, slot, 3879d0bc6d8SKonstantin Belousov func, ctx->domain, ctx->mgaw, ctx->agaw, 3889d0bc6d8SKonstantin Belousov id_mapped ? "id" : "re"); 38986be9f0dSKonstantin Belousov } else { 39086be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx1, true, true); 39186be9f0dSKonstantin Belousov } 39286be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 39386be9f0dSKonstantin Belousov } 39486be9f0dSKonstantin Belousov ctx->refs++; 39586be9f0dSKonstantin Belousov if ((ctx->flags & DMAR_CTX_RMRR) != 0) 39686be9f0dSKonstantin Belousov ctx->refs++; /* XXXKIB */ 39786be9f0dSKonstantin Belousov 39886be9f0dSKonstantin Belousov /* 39986be9f0dSKonstantin Belousov * If dmar declares Caching Mode as Set, follow 11.5 "Caching 40086be9f0dSKonstantin Belousov * Mode Consideration" and do the (global) invalidation of the 40186be9f0dSKonstantin Belousov * negative TLB entries. 40286be9f0dSKonstantin Belousov */ 40386be9f0dSKonstantin Belousov if ((dmar->hw_cap & DMAR_CAP_CM) != 0 || enable) { 40468eeb96aSKonstantin Belousov if (dmar->qi_enabled) { 40568eeb96aSKonstantin Belousov dmar_qi_invalidate_ctx_glob_locked(dmar); 40668eeb96aSKonstantin Belousov if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) 40768eeb96aSKonstantin Belousov dmar_qi_invalidate_iotlb_glob_locked(dmar); 40868eeb96aSKonstantin Belousov } else { 40986be9f0dSKonstantin Belousov error = dmar_inv_ctx_glob(dmar); 41086be9f0dSKonstantin Belousov if (error == 0 && 41186be9f0dSKonstantin Belousov (dmar->hw_ecap & DMAR_ECAP_DI) != 0) 41286be9f0dSKonstantin Belousov error = dmar_inv_iotlb_glob(dmar); 41386be9f0dSKonstantin Belousov if (error != 0) { 41486be9f0dSKonstantin Belousov dmar_free_ctx_locked(dmar, ctx); 41586be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 41686be9f0dSKonstantin Belousov return (NULL); 41786be9f0dSKonstantin Belousov } 41886be9f0dSKonstantin Belousov } 41968eeb96aSKonstantin Belousov } 42068eeb96aSKonstantin Belousov 42168eeb96aSKonstantin Belousov /* 42268eeb96aSKonstantin Belousov * The dmar lock was potentially dropped between check for the 42368eeb96aSKonstantin Belousov * empty context list and now. Recheck the state of GCMD_TE 42468eeb96aSKonstantin Belousov * to avoid unneeded command. 42568eeb96aSKonstantin Belousov */ 42668eeb96aSKonstantin Belousov if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) { 42786be9f0dSKonstantin Belousov error = dmar_enable_translation(dmar); 42886be9f0dSKonstantin Belousov if (error != 0) { 42986be9f0dSKonstantin Belousov dmar_free_ctx_locked(dmar, ctx); 43086be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 43186be9f0dSKonstantin Belousov return (NULL); 43286be9f0dSKonstantin Belousov } 43386be9f0dSKonstantin Belousov } 43486be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 43586be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 43686be9f0dSKonstantin Belousov return (ctx); 43786be9f0dSKonstantin Belousov } 43886be9f0dSKonstantin Belousov 43986be9f0dSKonstantin Belousov void 44086be9f0dSKonstantin Belousov dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) 44186be9f0dSKonstantin Belousov { 44286be9f0dSKonstantin Belousov struct sf_buf *sf; 44386be9f0dSKonstantin Belousov dmar_ctx_entry_t *ctxp; 44486be9f0dSKonstantin Belousov 44586be9f0dSKonstantin Belousov DMAR_ASSERT_LOCKED(dmar); 44686be9f0dSKonstantin Belousov KASSERT(ctx->refs >= 1, 44786be9f0dSKonstantin Belousov ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 44886be9f0dSKonstantin Belousov 44986be9f0dSKonstantin Belousov /* 45086be9f0dSKonstantin Belousov * If our reference is not last, only the dereference should 45186be9f0dSKonstantin Belousov * be performed. 45286be9f0dSKonstantin Belousov */ 45386be9f0dSKonstantin Belousov if (ctx->refs > 1) { 45486be9f0dSKonstantin Belousov ctx->refs--; 45586be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 45686be9f0dSKonstantin Belousov return; 45786be9f0dSKonstantin Belousov } 45886be9f0dSKonstantin Belousov 45986be9f0dSKonstantin Belousov KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0, 46086be9f0dSKonstantin Belousov ("lost ref on RMRR ctx %p", ctx)); 46186be9f0dSKonstantin Belousov KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 46286be9f0dSKonstantin Belousov ("lost ref on disabled ctx %p", ctx)); 46386be9f0dSKonstantin Belousov 46486be9f0dSKonstantin Belousov /* 46586be9f0dSKonstantin Belousov * Otherwise, the context entry must be cleared before the 46686be9f0dSKonstantin Belousov * page table is destroyed. The mapping of the context 46786be9f0dSKonstantin Belousov * entries page could require sleep, unlock the dmar. 46886be9f0dSKonstantin Belousov */ 46986be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 47086be9f0dSKonstantin Belousov TD_PREP_PINNED_ASSERT; 47186be9f0dSKonstantin Belousov ctxp = dmar_map_ctx_entry(ctx, &sf); 47286be9f0dSKonstantin Belousov DMAR_LOCK(dmar); 47386be9f0dSKonstantin Belousov KASSERT(ctx->refs >= 1, 47486be9f0dSKonstantin Belousov ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 47586be9f0dSKonstantin Belousov 47686be9f0dSKonstantin Belousov /* 47786be9f0dSKonstantin Belousov * Other thread might have referenced the context, in which 47886be9f0dSKonstantin Belousov * case again only the dereference should be performed. 47986be9f0dSKonstantin Belousov */ 48086be9f0dSKonstantin Belousov if (ctx->refs > 1) { 48186be9f0dSKonstantin Belousov ctx->refs--; 48286be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 48386be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 48486be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 48586be9f0dSKonstantin Belousov return; 48686be9f0dSKonstantin Belousov } 48786be9f0dSKonstantin Belousov 48886be9f0dSKonstantin Belousov KASSERT((ctx->flags & DMAR_CTX_RMRR) == 0, 48986be9f0dSKonstantin Belousov ("lost ref on RMRR ctx %p", ctx)); 49086be9f0dSKonstantin Belousov KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 49186be9f0dSKonstantin Belousov ("lost ref on disabled ctx %p", ctx)); 49286be9f0dSKonstantin Belousov 49386be9f0dSKonstantin Belousov /* 49486be9f0dSKonstantin Belousov * Clear the context pointer and flush the caches. 49586be9f0dSKonstantin Belousov * XXXKIB: cannot do this if any RMRR entries are still present. 49686be9f0dSKonstantin Belousov */ 49786be9f0dSKonstantin Belousov dmar_pte_clear(&ctxp->ctx1); 49886be9f0dSKonstantin Belousov ctxp->ctx2 = 0; 49986be9f0dSKonstantin Belousov dmar_inv_ctx_glob(dmar); 50068eeb96aSKonstantin Belousov if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { 50168eeb96aSKonstantin Belousov if (dmar->qi_enabled) 50268eeb96aSKonstantin Belousov dmar_qi_invalidate_iotlb_glob_locked(dmar); 50368eeb96aSKonstantin Belousov else 50486be9f0dSKonstantin Belousov dmar_inv_iotlb_glob(dmar); 50568eeb96aSKonstantin Belousov } 50686be9f0dSKonstantin Belousov LIST_REMOVE(ctx, link); 50786be9f0dSKonstantin Belousov DMAR_UNLOCK(dmar); 50886be9f0dSKonstantin Belousov 50986be9f0dSKonstantin Belousov /* 51086be9f0dSKonstantin Belousov * The rest of the destruction is invisible for other users of 51186be9f0dSKonstantin Belousov * the dmar unit. 51286be9f0dSKonstantin Belousov */ 51386be9f0dSKonstantin Belousov taskqueue_drain(dmar->delayed_taskqueue, &ctx->unload_task); 51486be9f0dSKonstantin Belousov KASSERT(TAILQ_EMPTY(&ctx->unload_entries), 51586be9f0dSKonstantin Belousov ("unfinished unloads %p", ctx)); 51686be9f0dSKonstantin Belousov dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); 51786be9f0dSKonstantin Belousov free_unr(dmar->domids, ctx->domain); 51886be9f0dSKonstantin Belousov dmar_ctx_dtr(ctx, true, true); 51986be9f0dSKonstantin Belousov TD_PINNED_ASSERT; 52086be9f0dSKonstantin Belousov } 52186be9f0dSKonstantin Belousov 52286be9f0dSKonstantin Belousov void 52386be9f0dSKonstantin Belousov dmar_free_ctx(struct dmar_ctx *ctx) 52486be9f0dSKonstantin Belousov { 52586be9f0dSKonstantin Belousov struct dmar_unit *dmar; 52686be9f0dSKonstantin Belousov 52786be9f0dSKonstantin Belousov dmar = ctx->dmar; 52886be9f0dSKonstantin Belousov DMAR_LOCK(dmar); 52986be9f0dSKonstantin Belousov dmar_free_ctx_locked(dmar, ctx); 53086be9f0dSKonstantin Belousov } 53186be9f0dSKonstantin Belousov 53286be9f0dSKonstantin Belousov struct dmar_ctx * 53367499354SRyan Stone dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid) 53486be9f0dSKonstantin Belousov { 53586be9f0dSKonstantin Belousov struct dmar_ctx *ctx; 53686be9f0dSKonstantin Belousov 53786be9f0dSKonstantin Belousov DMAR_ASSERT_LOCKED(dmar); 53886be9f0dSKonstantin Belousov 53986be9f0dSKonstantin Belousov LIST_FOREACH(ctx, &dmar->contexts, link) { 54067499354SRyan Stone if (ctx->rid == rid) 54186be9f0dSKonstantin Belousov return (ctx); 54286be9f0dSKonstantin Belousov } 54386be9f0dSKonstantin Belousov return (NULL); 54486be9f0dSKonstantin Belousov } 54586be9f0dSKonstantin Belousov 54686be9f0dSKonstantin Belousov void 54768eeb96aSKonstantin Belousov dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free) 54868eeb96aSKonstantin Belousov { 54968eeb96aSKonstantin Belousov struct dmar_ctx *ctx; 55068eeb96aSKonstantin Belousov 55168eeb96aSKonstantin Belousov ctx = entry->ctx; 55268eeb96aSKonstantin Belousov DMAR_CTX_LOCK(ctx); 55368eeb96aSKonstantin Belousov if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0) 55468eeb96aSKonstantin Belousov dmar_gas_free_region(ctx, entry); 55568eeb96aSKonstantin Belousov else 55668eeb96aSKonstantin Belousov dmar_gas_free_space(ctx, entry); 55768eeb96aSKonstantin Belousov DMAR_CTX_UNLOCK(ctx); 55868eeb96aSKonstantin Belousov if (free) 55968eeb96aSKonstantin Belousov dmar_gas_free_entry(ctx, entry); 56068eeb96aSKonstantin Belousov else 56168eeb96aSKonstantin Belousov entry->flags = 0; 56268eeb96aSKonstantin Belousov } 56368eeb96aSKonstantin Belousov 56468eeb96aSKonstantin Belousov void 56568eeb96aSKonstantin Belousov dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free) 56668eeb96aSKonstantin Belousov { 56768eeb96aSKonstantin Belousov struct dmar_unit *unit; 56868eeb96aSKonstantin Belousov 56968eeb96aSKonstantin Belousov unit = entry->ctx->dmar; 57068eeb96aSKonstantin Belousov if (unit->qi_enabled) { 57168eeb96aSKonstantin Belousov DMAR_LOCK(unit); 57268eeb96aSKonstantin Belousov dmar_qi_invalidate_locked(entry->ctx, entry->start, 57368eeb96aSKonstantin Belousov entry->end - entry->start, &entry->gseq); 57468eeb96aSKonstantin Belousov if (!free) 57568eeb96aSKonstantin Belousov entry->flags |= DMAR_MAP_ENTRY_QI_NF; 57668eeb96aSKonstantin Belousov TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 57768eeb96aSKonstantin Belousov DMAR_UNLOCK(unit); 57868eeb96aSKonstantin Belousov } else { 57968eeb96aSKonstantin Belousov ctx_flush_iotlb_sync(entry->ctx, entry->start, entry->end - 58068eeb96aSKonstantin Belousov entry->start); 58168eeb96aSKonstantin Belousov dmar_ctx_free_entry(entry, free); 58268eeb96aSKonstantin Belousov } 58368eeb96aSKonstantin Belousov } 58468eeb96aSKonstantin Belousov 58568eeb96aSKonstantin Belousov void 58686be9f0dSKonstantin Belousov dmar_ctx_unload(struct dmar_ctx *ctx, struct dmar_map_entries_tailq *entries, 58786be9f0dSKonstantin Belousov bool cansleep) 58886be9f0dSKonstantin Belousov { 58968eeb96aSKonstantin Belousov struct dmar_unit *unit; 59068eeb96aSKonstantin Belousov struct dmar_map_entry *entry, *entry1; 59168eeb96aSKonstantin Belousov struct dmar_qi_genseq gseq; 59286be9f0dSKonstantin Belousov int error; 59386be9f0dSKonstantin Belousov 59468eeb96aSKonstantin Belousov unit = ctx->dmar; 59568eeb96aSKonstantin Belousov 59668eeb96aSKonstantin Belousov TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 59786be9f0dSKonstantin Belousov KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0, 59886be9f0dSKonstantin Belousov ("not mapped entry %p %p", ctx, entry)); 59986be9f0dSKonstantin Belousov error = ctx_unmap_buf(ctx, entry->start, entry->end - 60086be9f0dSKonstantin Belousov entry->start, cansleep ? DMAR_PGF_WAITOK : 0); 60186be9f0dSKonstantin Belousov KASSERT(error == 0, ("unmap %p error %d", ctx, error)); 60268eeb96aSKonstantin Belousov if (!unit->qi_enabled) { 60368eeb96aSKonstantin Belousov ctx_flush_iotlb_sync(ctx, entry->start, 60468eeb96aSKonstantin Belousov entry->end - entry->start); 60568eeb96aSKonstantin Belousov TAILQ_REMOVE(entries, entry, dmamap_link); 60668eeb96aSKonstantin Belousov dmar_ctx_free_entry(entry, true); 60786be9f0dSKonstantin Belousov } 60886be9f0dSKonstantin Belousov } 60968eeb96aSKonstantin Belousov if (TAILQ_EMPTY(entries)) 61068eeb96aSKonstantin Belousov return; 61168eeb96aSKonstantin Belousov 61268eeb96aSKonstantin Belousov KASSERT(unit->qi_enabled, ("loaded entry left")); 61368eeb96aSKonstantin Belousov DMAR_LOCK(unit); 61468eeb96aSKonstantin Belousov TAILQ_FOREACH(entry, entries, dmamap_link) { 61568eeb96aSKonstantin Belousov entry->gseq.gen = 0; 61668eeb96aSKonstantin Belousov entry->gseq.seq = 0; 61768eeb96aSKonstantin Belousov dmar_qi_invalidate_locked(ctx, entry->start, entry->end - 61868eeb96aSKonstantin Belousov entry->start, TAILQ_NEXT(entry, dmamap_link) == NULL ? 61968eeb96aSKonstantin Belousov &gseq : NULL); 62068eeb96aSKonstantin Belousov } 62168eeb96aSKonstantin Belousov TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 62268eeb96aSKonstantin Belousov entry->gseq = gseq; 62368eeb96aSKonstantin Belousov TAILQ_REMOVE(entries, entry, dmamap_link); 62468eeb96aSKonstantin Belousov TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 62568eeb96aSKonstantin Belousov } 62668eeb96aSKonstantin Belousov DMAR_UNLOCK(unit); 62768eeb96aSKonstantin Belousov } 62886be9f0dSKonstantin Belousov 62986be9f0dSKonstantin Belousov static void 63086be9f0dSKonstantin Belousov dmar_ctx_unload_task(void *arg, int pending) 63186be9f0dSKonstantin Belousov { 63286be9f0dSKonstantin Belousov struct dmar_ctx *ctx; 63386be9f0dSKonstantin Belousov struct dmar_map_entries_tailq entries; 63486be9f0dSKonstantin Belousov 63586be9f0dSKonstantin Belousov ctx = arg; 63686be9f0dSKonstantin Belousov TAILQ_INIT(&entries); 63786be9f0dSKonstantin Belousov 63886be9f0dSKonstantin Belousov for (;;) { 63986be9f0dSKonstantin Belousov DMAR_CTX_LOCK(ctx); 64086be9f0dSKonstantin Belousov TAILQ_SWAP(&ctx->unload_entries, &entries, dmar_map_entry, 64186be9f0dSKonstantin Belousov dmamap_link); 64286be9f0dSKonstantin Belousov DMAR_CTX_UNLOCK(ctx); 64386be9f0dSKonstantin Belousov if (TAILQ_EMPTY(&entries)) 64486be9f0dSKonstantin Belousov break; 64586be9f0dSKonstantin Belousov dmar_ctx_unload(ctx, &entries, true); 64686be9f0dSKonstantin Belousov } 64786be9f0dSKonstantin Belousov } 648