/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Portions Copyright (c) 2010, Oracle and/or its affiliates. * All rights reserved. */ /* * Copyright (c) 2009, Intel Corporation. * All rights reserved. */ /* * Copyright 2012 Garrett D'Amore . All rights reserved. */ /* * DVMA code * This file contains Intel IOMMU code that deals with DVMA * i.e. DMA remapping. */ #include #include #include #include #include #include #include #include #include #include #include #undef TEST /* * Macros based on PCI spec */ #define IMMU_PCI_REV2CLASS(r) ((r) >> 8) /* classcode from revid */ #define IMMU_PCI_CLASS2BASE(c) ((c) >> 16) /* baseclass from classcode */ #define IMMU_PCI_CLASS2SUB(c) (((c) >> 8) & 0xff); /* classcode */ #define IMMU_CONTIG_PADDR(d, p) \ ((d).dck_paddr && ((d).dck_paddr + IMMU_PAGESIZE) == (p)) typedef struct dvma_arg { immu_t *dva_immu; dev_info_t *dva_rdip; dev_info_t *dva_ddip; domain_t *dva_domain; int dva_level; immu_flags_t dva_flags; list_t *dva_list; int dva_error; } dvma_arg_t; static domain_t *domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip, immu_flags_t immu_flags); static immu_devi_t *create_immu_devi(dev_info_t *rdip, int bus, int dev, int func, immu_flags_t immu_flags); static void destroy_immu_devi(immu_devi_t *immu_devi); static boolean_t dvma_map(domain_t *domain, uint64_t sdvma, uint64_t nvpages, immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip, immu_flags_t immu_flags); /* Extern globals */ extern struct memlist *phys_install; /* * iommulib interface functions. */ static int immu_probe(iommulib_handle_t unitp, dev_info_t *dip); static int immu_allochdl(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep); static int immu_freehdl(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle); static int immu_bindhdl(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, struct ddi_dma_req *dma_req, ddi_dma_cookie_t *cookiep, uint_t *ccountp); static int immu_unbindhdl(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle); static int immu_sync(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off, size_t len, uint_t cachefl); static int immu_win(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp); static int immu_mapobject(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao); static int immu_unmapobject(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao); /* static Globals */ /* * Used to setup DMA objects (memory regions) * for DMA reads by IOMMU units */ static ddi_dma_attr_t immu_dma_attr = { DMA_ATTR_V0, 0U, 0xffffffffffffffffULL, 0xffffffffU, MMU_PAGESIZE, /* MMU page aligned */ 0x1, 0x1, 0xffffffffU, 0xffffffffffffffffULL, 1, 4, 0 }; static ddi_device_acc_attr_t immu_acc_attr = { DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, DDI_STRICTORDER_ACC }; struct iommulib_ops immulib_ops = { IOMMU_OPS_VERSION, INTEL_IOMMU, "Intel IOMMU", NULL, immu_probe, immu_allochdl, immu_freehdl, immu_bindhdl, immu_unbindhdl, immu_sync, immu_win, immu_mapobject, immu_unmapobject, }; /* * Fake physical address range used to set up initial prealloc mappings. * This memory is never actually accessed. It is mapped read-only, * and is overwritten as soon as the first DMA bind operation is * performed. Since 0 is a special case, just start at the 2nd * physical page. */ static immu_dcookie_t immu_precookie = { MMU_PAGESIZE, IMMU_NPREPTES }; /* globals private to this file */ static kmutex_t immu_domain_lock; static list_t immu_unity_domain_list; static list_t immu_xlate_domain_list; /* structure used to store idx into each level of the page tables */ typedef struct xlate { int xlt_level; uint_t xlt_idx; pgtable_t *xlt_pgtable; } xlate_t; /* 0 is reserved by Vt-d spec. Solaris reserves 1 */ #define IMMU_UNITY_DID 1 static mod_hash_t *bdf_domain_hash; int immu_use_alh; int immu_use_tm; static domain_t * bdf_domain_lookup(immu_devi_t *immu_devi) { domain_t *domain; int16_t seg = immu_devi->imd_seg; int16_t bus = immu_devi->imd_bus; int16_t devfunc = immu_devi->imd_devfunc; uintptr_t bdf = (seg << 16 | bus << 8 | devfunc); if (seg < 0 || bus < 0 || devfunc < 0) { return (NULL); } domain = NULL; if (mod_hash_find(bdf_domain_hash, (void *)bdf, (void *)&domain) == 0) { ASSERT(domain); ASSERT(domain->dom_did > 0); return (domain); } else { return (NULL); } } static void bdf_domain_insert(immu_devi_t *immu_devi, domain_t *domain) { int16_t seg = immu_devi->imd_seg; int16_t bus = immu_devi->imd_bus; int16_t devfunc = immu_devi->imd_devfunc; uintptr_t bdf = (seg << 16 | bus << 8 | devfunc); if (seg < 0 || bus < 0 || devfunc < 0) { return; } (void) mod_hash_insert(bdf_domain_hash, (void *)bdf, (void *)domain); } static int match_lpc(dev_info_t *pdip, void *arg) { immu_devi_t *immu_devi; dvma_arg_t *dvap = (dvma_arg_t *)arg; if (list_is_empty(dvap->dva_list)) { return (DDI_WALK_TERMINATE); } immu_devi = list_head(dvap->dva_list); for (; immu_devi; immu_devi = list_next(dvap->dva_list, immu_devi)) { if (immu_devi->imd_dip == pdip) { dvap->dva_ddip = pdip; dvap->dva_error = DDI_SUCCESS; return (DDI_WALK_TERMINATE); } } return (DDI_WALK_CONTINUE); } static void immu_devi_set_spclist(dev_info_t *dip, immu_t *immu) { list_t *spclist = NULL; immu_devi_t *immu_devi; immu_devi = IMMU_DEVI(dip); if (immu_devi->imd_display == B_TRUE) { spclist = &(immu->immu_dvma_gfx_list); } else if (immu_devi->imd_lpc == B_TRUE) { spclist = &(immu->immu_dvma_lpc_list); } if (spclist) { mutex_enter(&(immu->immu_lock)); list_insert_head(spclist, immu_devi); mutex_exit(&(immu->immu_lock)); } } /* * Set the immu_devi struct in the immu_devi field of a devinfo node */ int immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags) { int bus, dev, func; immu_devi_t *new_imd; immu_devi_t *immu_devi; immu_devi = immu_devi_get(dip); if (immu_devi != NULL) { return (DDI_SUCCESS); } bus = dev = func = -1; /* * Assume a new immu_devi struct is needed */ if (!DEVI_IS_PCI(dip) || acpica_get_bdf(dip, &bus, &dev, &func) != 0) { /* * No BDF. Set bus = -1 to indicate this. * We still need to create a immu_devi struct * though */ bus = -1; dev = 0; func = 0; } new_imd = create_immu_devi(dip, bus, dev, func, immu_flags); if (new_imd == NULL) { ddi_err(DER_WARN, dip, "Failed to create immu_devi " "structure"); return (DDI_FAILURE); } /* * Check if some other thread allocated a immu_devi while we * didn't own the lock. */ mutex_enter(&(DEVI(dip)->devi_lock)); if (IMMU_DEVI(dip) == NULL) { IMMU_DEVI_SET(dip, new_imd); } else { destroy_immu_devi(new_imd); } mutex_exit(&(DEVI(dip)->devi_lock)); return (DDI_SUCCESS); } static dev_info_t * get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags) { dvma_arg_t dvarg = {0}; dvarg.dva_list = &(immu->immu_dvma_lpc_list); dvarg.dva_rdip = rdip; dvarg.dva_error = DDI_FAILURE; if (immu_walk_ancestor(rdip, NULL, match_lpc, &dvarg, NULL, immu_flags) != DDI_SUCCESS) { ddi_err(DER_MODE, rdip, "Could not walk ancestors to " "find lpc_devinfo for ISA device"); return (NULL); } if (dvarg.dva_error != DDI_SUCCESS || dvarg.dva_ddip == NULL) { ddi_err(DER_MODE, rdip, "Could not find lpc_devinfo for " "ISA device"); return (NULL); } return (dvarg.dva_ddip); } static dev_info_t * get_gfx_devinfo(dev_info_t *rdip) { immu_t *immu; immu_devi_t *immu_devi; list_t *list_gfx; /* * The GFX device may not be on the same iommu unit as "agpgart" * so search globally */ immu_devi = NULL; immu = list_head(&immu_list); for (; immu; immu = list_next(&immu_list, immu)) { list_gfx = &(immu->immu_dvma_gfx_list); if (!list_is_empty(list_gfx)) { immu_devi = list_head(list_gfx); break; } } if (immu_devi == NULL) { ddi_err(DER_WARN, rdip, "iommu: No GFX device. " "Cannot redirect agpgart"); return (NULL); } ddi_err(DER_LOG, rdip, "iommu: GFX redirect to %s", ddi_node_name(immu_devi->imd_dip)); return (immu_devi->imd_dip); } static immu_flags_t dma_to_immu_flags(struct ddi_dma_req *dmareq) { immu_flags_t flags = 0; if (dmareq->dmar_fp == DDI_DMA_SLEEP) { flags |= IMMU_FLAGS_SLEEP; } else { flags |= IMMU_FLAGS_NOSLEEP; } #ifdef BUGGY_DRIVERS flags |= (IMMU_FLAGS_READ | IMMU_FLAGS_WRITE); #else /* * Read and write flags need to be reversed. * DMA_READ means read from device and write * to memory. So DMA read means DVMA write. */ if (dmareq->dmar_flags & DDI_DMA_READ) flags |= IMMU_FLAGS_WRITE; if (dmareq->dmar_flags & DDI_DMA_WRITE) flags |= IMMU_FLAGS_READ; /* * Some buggy drivers specify neither READ or WRITE * For such drivers set both read and write permissions */ if ((dmareq->dmar_flags & (DDI_DMA_READ | DDI_DMA_WRITE)) == 0) { flags |= (IMMU_FLAGS_READ | IMMU_FLAGS_WRITE); } #endif return (flags); } /*ARGSUSED*/ int pgtable_ctor(void *buf, void *arg, int kmflag) { size_t actual_size = 0; pgtable_t *pgtable; int (*dmafp)(caddr_t); caddr_t vaddr; void *next; uint_t flags; immu_t *immu = arg; pgtable = (pgtable_t *)buf; dmafp = (kmflag & KM_NOSLEEP) ? DDI_DMA_DONTWAIT : DDI_DMA_SLEEP; next = kmem_zalloc(IMMU_PAGESIZE, kmflag); if (next == NULL) { return (-1); } if (ddi_dma_alloc_handle(root_devinfo, &immu_dma_attr, dmafp, NULL, &pgtable->hwpg_dmahdl) != DDI_SUCCESS) { kmem_free(next, IMMU_PAGESIZE); return (-1); } flags = DDI_DMA_CONSISTENT; if (!immu->immu_dvma_coherent) flags |= IOMEM_DATA_UC_WR_COMBINE; if (ddi_dma_mem_alloc(pgtable->hwpg_dmahdl, IMMU_PAGESIZE, &immu_acc_attr, flags, dmafp, NULL, &vaddr, &actual_size, &pgtable->hwpg_memhdl) != DDI_SUCCESS) { ddi_dma_free_handle(&pgtable->hwpg_dmahdl); kmem_free(next, IMMU_PAGESIZE); return (-1); } /* * Memory allocation failure. Maybe a temporary condition * so return error rather than panic, so we can try again */ if (actual_size < IMMU_PAGESIZE) { ddi_dma_mem_free(&pgtable->hwpg_memhdl); ddi_dma_free_handle(&pgtable->hwpg_dmahdl); kmem_free(next, IMMU_PAGESIZE); return (-1); } pgtable->hwpg_paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, vaddr)); pgtable->hwpg_vaddr = vaddr; pgtable->swpg_next_array = next; rw_init(&(pgtable->swpg_rwlock), NULL, RW_DEFAULT, NULL); return (0); } /*ARGSUSED*/ void pgtable_dtor(void *buf, void *arg) { pgtable_t *pgtable; pgtable = (pgtable_t *)buf; /* destroy will panic if lock is held. */ rw_destroy(&(pgtable->swpg_rwlock)); ddi_dma_mem_free(&pgtable->hwpg_memhdl); ddi_dma_free_handle(&pgtable->hwpg_dmahdl); kmem_free(pgtable->swpg_next_array, IMMU_PAGESIZE); } /* * pgtable_alloc() * alloc a IOMMU pgtable structure. * This same struct is used for root and context tables as well. * This routine allocs the f/ollowing: * - a pgtable_t struct * - a HW page which holds PTEs/entries which is accesssed by HW * so we set up DMA for this page * - a SW page which is only for our bookeeping * (for example to hold pointers to the next level pgtable). * So a simple kmem_alloc suffices */ static pgtable_t * pgtable_alloc(immu_t *immu, immu_flags_t immu_flags) { pgtable_t *pgtable; int kmflags; kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; pgtable = kmem_cache_alloc(immu->immu_pgtable_cache, kmflags); if (pgtable == NULL) { return (NULL); } return (pgtable); } static void pgtable_zero(pgtable_t *pgtable) { bzero(pgtable->hwpg_vaddr, IMMU_PAGESIZE); bzero(pgtable->swpg_next_array, IMMU_PAGESIZE); } static void pgtable_free(immu_t *immu, pgtable_t *pgtable) { kmem_cache_free(immu->immu_pgtable_cache, pgtable); } /* * Function to identify a display device from the PCI class code */ static boolean_t device_is_display(uint_t classcode) { static uint_t disp_classes[] = { 0x000100, 0x030000, 0x030001 }; int i, nclasses = sizeof (disp_classes) / sizeof (uint_t); for (i = 0; i < nclasses; i++) { if (classcode == disp_classes[i]) return (B_TRUE); } return (B_FALSE); } /* * Function that determines if device is PCIEX and/or PCIEX bridge */ static boolean_t device_is_pciex( uchar_t bus, uchar_t dev, uchar_t func, boolean_t *is_pcib) { ushort_t cap; ushort_t capsp; ushort_t cap_count = PCI_CAP_MAX_PTR; ushort_t status; boolean_t is_pciex = B_FALSE; *is_pcib = B_FALSE; status = pci_getw_func(bus, dev, func, PCI_CONF_STAT); if (!(status & PCI_STAT_CAP)) return (B_FALSE); capsp = pci_getb_func(bus, dev, func, PCI_CONF_CAP_PTR); while (cap_count-- && capsp >= PCI_CAP_PTR_OFF) { capsp &= PCI_CAP_PTR_MASK; cap = pci_getb_func(bus, dev, func, capsp); if (cap == PCI_CAP_ID_PCI_E) { status = pci_getw_func(bus, dev, func, capsp + 2); /* * See section 7.8.2 of PCI-Express Base Spec v1.0a * for Device/Port Type. * PCIE_PCIECAP_DEV_TYPE_PCIE2PCI implies that the * device is a PCIE2PCI bridge */ *is_pcib = ((status & PCIE_PCIECAP_DEV_TYPE_MASK) == PCIE_PCIECAP_DEV_TYPE_PCIE2PCI) ? B_TRUE : B_FALSE; is_pciex = B_TRUE; } capsp = (*pci_getb_func)(bus, dev, func, capsp + PCI_CAP_NEXT_PTR); } return (is_pciex); } static boolean_t device_use_premap(uint_t classcode) { if (IMMU_PCI_CLASS2BASE(classcode) == PCI_CLASS_NET) return (B_TRUE); return (B_FALSE); } /* * immu_dvma_get_immu() * get the immu unit structure for a dev_info node */ immu_t * immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags) { immu_devi_t *immu_devi; immu_t *immu; /* * check if immu unit was already found earlier. * If yes, then it will be stashed in immu_devi struct. */ immu_devi = immu_devi_get(dip); if (immu_devi == NULL) { if (immu_devi_set(dip, immu_flags) != DDI_SUCCESS) { /* * May fail because of low memory. Return error rather * than panic as we want driver to rey again later */ ddi_err(DER_PANIC, dip, "immu_dvma_get_immu: " "No immu_devi structure"); /*NOTREACHED*/ } immu_devi = immu_devi_get(dip); } mutex_enter(&(DEVI(dip)->devi_lock)); if (immu_devi->imd_immu) { immu = immu_devi->imd_immu; mutex_exit(&(DEVI(dip)->devi_lock)); return (immu); } mutex_exit(&(DEVI(dip)->devi_lock)); immu = immu_dmar_get_immu(dip); if (immu == NULL) { ddi_err(DER_PANIC, dip, "immu_dvma_get_immu: " "Cannot find immu_t for device"); /*NOTREACHED*/ } /* * Check if some other thread found immu * while lock was not held */ immu_devi = immu_devi_get(dip); /* immu_devi should be present as we found it earlier */ if (immu_devi == NULL) { ddi_err(DER_PANIC, dip, "immu_dvma_get_immu: No immu_devi structure"); /*NOTREACHED*/ } mutex_enter(&(DEVI(dip)->devi_lock)); if (immu_devi->imd_immu == NULL) { /* nobody else set it, so we should do it */ immu_devi->imd_immu = immu; immu_devi_set_spclist(dip, immu); } else { /* * if some other thread got immu before * us, it should get the same results */ if (immu_devi->imd_immu != immu) { ddi_err(DER_PANIC, dip, "Multiple " "immu units found for device. Expected (%p), " "actual (%p)", (void *)immu, (void *)immu_devi->imd_immu); mutex_exit(&(DEVI(dip)->devi_lock)); /*NOTREACHED*/ } } mutex_exit(&(DEVI(dip)->devi_lock)); return (immu); } /* ############################# IMMU_DEVI code ############################ */ /* * Allocate a immu_devi structure and initialize it */ static immu_devi_t * create_immu_devi(dev_info_t *rdip, int bus, int dev, int func, immu_flags_t immu_flags) { uchar_t baseclass, subclass; uint_t classcode, revclass; immu_devi_t *immu_devi; boolean_t pciex = B_FALSE; int kmflags; boolean_t is_pcib = B_FALSE; /* bus == -1 indicate non-PCI device (no BDF) */ ASSERT(bus == -1 || bus >= 0); ASSERT(dev >= 0); ASSERT(func >= 0); kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; immu_devi = kmem_zalloc(sizeof (immu_devi_t), kmflags); if (immu_devi == NULL) { ddi_err(DER_WARN, rdip, "Failed to allocate memory for " "Intel IOMMU immu_devi structure"); return (NULL); } immu_devi->imd_dip = rdip; immu_devi->imd_seg = 0; /* Currently seg can only be 0 */ immu_devi->imd_bus = bus; immu_devi->imd_pcib_type = IMMU_PCIB_BAD; if (bus == -1) { immu_devi->imd_pcib_type = IMMU_PCIB_NOBDF; return (immu_devi); } immu_devi->imd_devfunc = IMMU_PCI_DEVFUNC(dev, func); immu_devi->imd_sec = 0; immu_devi->imd_sub = 0; revclass = pci_getl_func(bus, dev, func, PCI_CONF_REVID); classcode = IMMU_PCI_REV2CLASS(revclass); baseclass = IMMU_PCI_CLASS2BASE(classcode); subclass = IMMU_PCI_CLASS2SUB(classcode); if (baseclass == PCI_CLASS_BRIDGE && subclass == PCI_BRIDGE_PCI) { immu_devi->imd_sec = pci_getb_func(bus, dev, func, PCI_BCNF_SECBUS); immu_devi->imd_sub = pci_getb_func(bus, dev, func, PCI_BCNF_SUBBUS); pciex = device_is_pciex(bus, dev, func, &is_pcib); if (pciex == B_TRUE && is_pcib == B_TRUE) { immu_devi->imd_pcib_type = IMMU_PCIB_PCIE_PCI; } else if (pciex == B_TRUE) { immu_devi->imd_pcib_type = IMMU_PCIB_PCIE_PCIE; } else { immu_devi->imd_pcib_type = IMMU_PCIB_PCI_PCI; } } else { immu_devi->imd_pcib_type = IMMU_PCIB_ENDPOINT; } /* check for certain special devices */ immu_devi->imd_display = device_is_display(classcode); immu_devi->imd_lpc = ((baseclass == PCI_CLASS_BRIDGE) && (subclass == PCI_BRIDGE_ISA)) ? B_TRUE : B_FALSE; immu_devi->imd_use_premap = device_use_premap(classcode); immu_devi->imd_domain = NULL; immu_devi->imd_dvma_flags = immu_global_dvma_flags; return (immu_devi); } static void destroy_immu_devi(immu_devi_t *immu_devi) { kmem_free(immu_devi, sizeof (immu_devi_t)); } static domain_t * immu_devi_domain(dev_info_t *rdip, dev_info_t **ddipp) { immu_devi_t *immu_devi; domain_t *domain; dev_info_t *ddip; *ddipp = NULL; immu_devi = immu_devi_get(rdip); if (immu_devi == NULL) { return (NULL); } mutex_enter(&(DEVI(rdip)->devi_lock)); domain = immu_devi->imd_domain; ddip = immu_devi->imd_ddip; mutex_exit(&(DEVI(rdip)->devi_lock)); if (domain) *ddipp = ddip; return (domain); } /* ############################# END IMMU_DEVI code ######################## */ /* ############################# DOMAIN code ############################### */ /* * This routine always succeeds */ static int did_alloc(immu_t *immu, dev_info_t *rdip, dev_info_t *ddip, immu_flags_t immu_flags) { int did; did = (uintptr_t)vmem_alloc(immu->immu_did_arena, 1, (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP); if (did == 0) { ddi_err(DER_WARN, rdip, "device domain-id alloc error" " domain-device: %s%d. immu unit is %s. Using " "unity domain with domain-id (%d)", ddi_driver_name(ddip), ddi_get_instance(ddip), immu->immu_name, immu->immu_unity_domain->dom_did); did = immu->immu_unity_domain->dom_did; } return (did); } static int get_branch_domain(dev_info_t *pdip, void *arg) { immu_devi_t *immu_devi; domain_t *domain; dev_info_t *ddip; immu_t *immu; dvma_arg_t *dvp = (dvma_arg_t *)arg; /* * The field dvp->dva_rdip is a work-in-progress * and gets updated as we walk up the ancestor * tree. The final ddip is set only when we reach * the top of the tree. So the dvp->dva_ddip field cannot * be relied on until we reach the top of the field. */ /* immu_devi may not be set. */ immu_devi = immu_devi_get(pdip); if (immu_devi == NULL) { if (immu_devi_set(pdip, dvp->dva_flags) != DDI_SUCCESS) { dvp->dva_error = DDI_FAILURE; return (DDI_WALK_TERMINATE); } } immu_devi = immu_devi_get(pdip); immu = immu_devi->imd_immu; if (immu == NULL) immu = immu_dvma_get_immu(pdip, dvp->dva_flags); /* * If we encounter a PCIE_PCIE bridge *ANCESTOR* we need to * terminate the walk (since the device under the PCIE bridge * is a PCIE device and has an independent entry in the * root/context table) */ if (dvp->dva_rdip != pdip && immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCIE) { return (DDI_WALK_TERMINATE); } /* * In order to be a domain-dim, it must be a PCI device i.e. * must have valid BDF. This also eliminates the root complex. */ if (immu_devi->imd_pcib_type != IMMU_PCIB_BAD && immu_devi->imd_pcib_type != IMMU_PCIB_NOBDF) { ASSERT(immu_devi->imd_bus >= 0); ASSERT(immu_devi->imd_devfunc >= 0); dvp->dva_ddip = pdip; } if (immu_devi->imd_display == B_TRUE || (dvp->dva_flags & IMMU_FLAGS_UNITY)) { dvp->dva_domain = immu->immu_unity_domain; /* continue walking to find ddip */ return (DDI_WALK_CONTINUE); } mutex_enter(&(DEVI(pdip)->devi_lock)); domain = immu_devi->imd_domain; ddip = immu_devi->imd_ddip; mutex_exit(&(DEVI(pdip)->devi_lock)); if (domain && ddip) { /* if domain is set, it must be the same */ if (dvp->dva_domain) { ASSERT(domain == dvp->dva_domain); } dvp->dva_domain = domain; dvp->dva_ddip = ddip; return (DDI_WALK_TERMINATE); } /* Domain may already be set, continue walking so that ddip gets set */ if (dvp->dva_domain) { return (DDI_WALK_CONTINUE); } /* domain is not set in either immu_devi or dvp */ domain = bdf_domain_lookup(immu_devi); if (domain == NULL) { return (DDI_WALK_CONTINUE); } /* ok, the BDF hash had a domain for this BDF. */ /* Grab lock again to check if something else set immu_devi fields */ mutex_enter(&(DEVI(pdip)->devi_lock)); if (immu_devi->imd_domain != NULL) { dvp->dva_domain = domain; } else { dvp->dva_domain = domain; } mutex_exit(&(DEVI(pdip)->devi_lock)); /* * walk upwards until the topmost PCI bridge is found */ return (DDI_WALK_CONTINUE); } static void map_unity_domain(domain_t *domain) { struct memlist *mp; uint64_t start; uint64_t npages; immu_dcookie_t dcookies[1] = {0}; int dcount = 0; /* * UNITY arenas are a mirror of the physical memory * installed on the system. */ #ifdef BUGGY_DRIVERS /* * Dont skip page0. Some broken HW/FW access it. */ dcookies[0].dck_paddr = 0; dcookies[0].dck_npages = 1; dcount = 1; (void) dvma_map(domain, 0, 1, dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1); #endif memlist_read_lock(); mp = phys_install; if (mp->ml_address == 0) { /* since we already mapped page1 above */ start = IMMU_PAGESIZE; } else { start = mp->ml_address; } npages = mp->ml_size/IMMU_PAGESIZE + 1; dcookies[0].dck_paddr = start; dcookies[0].dck_npages = npages; dcount = 1; (void) dvma_map(domain, start, npages, dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE); ddi_err(DER_LOG, domain->dom_dip, "iommu: mapping PHYS span [0x%" PRIx64 " - 0x%" PRIx64 "]", start, start + mp->ml_size); mp = mp->ml_next; while (mp) { ddi_err(DER_LOG, domain->dom_dip, "iommu: mapping PHYS span [0x%" PRIx64 " - 0x%" PRIx64 "]", mp->ml_address, mp->ml_address + mp->ml_size); start = mp->ml_address; npages = mp->ml_size/IMMU_PAGESIZE + 1; dcookies[0].dck_paddr = start; dcookies[0].dck_npages = npages; dcount = 1; (void) dvma_map(domain, start, npages, dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE); mp = mp->ml_next; } mp = bios_rsvd; while (mp) { ddi_err(DER_LOG, domain->dom_dip, "iommu: mapping PHYS span [0x%" PRIx64 " - 0x%" PRIx64 "]", mp->ml_address, mp->ml_address + mp->ml_size); start = mp->ml_address; npages = mp->ml_size/IMMU_PAGESIZE + 1; dcookies[0].dck_paddr = start; dcookies[0].dck_npages = npages; dcount = 1; (void) dvma_map(domain, start, npages, dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE); mp = mp->ml_next; } memlist_read_unlock(); } /* * create_xlate_arena() * Create the dvma arena for a domain with translation * mapping */ static void create_xlate_arena(immu_t *immu, domain_t *domain, dev_info_t *rdip, immu_flags_t immu_flags) { char *arena_name; struct memlist *mp; int vmem_flags; uint64_t start; uint_t mgaw; uint64_t size; uint64_t maxaddr; void *vmem_ret; arena_name = domain->dom_dvma_arena_name; /* Note, don't do sizeof (arena_name) - it is just a pointer */ (void) snprintf(arena_name, sizeof (domain->dom_dvma_arena_name), "%s-domain-%d-xlate-DVMA-arena", immu->immu_name, domain->dom_did); vmem_flags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP; /* Restrict mgaddr (max guest addr) to MGAW */ mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap); /* * To ensure we avoid ioapic and PCI MMIO ranges we just * use the physical memory address range of the system as the * range */ maxaddr = ((uint64_t)1 << mgaw); memlist_read_lock(); mp = phys_install; if (mp->ml_address == 0) start = MMU_PAGESIZE; else start = mp->ml_address; if (start + mp->ml_size > maxaddr) size = maxaddr - start; else size = mp->ml_size; ddi_err(DER_VERB, rdip, "iommu: %s: Creating dvma vmem arena [0x%" PRIx64 " - 0x%" PRIx64 "]", arena_name, start, start + size); /* * We always allocate in quanta of IMMU_PAGESIZE */ domain->dom_dvma_arena = vmem_create(arena_name, (void *)(uintptr_t)start, /* start addr */ size, /* size */ IMMU_PAGESIZE, /* quantum */ NULL, /* afunc */ NULL, /* ffunc */ NULL, /* source */ 0, /* qcache_max */ vmem_flags); if (domain->dom_dvma_arena == NULL) { ddi_err(DER_PANIC, rdip, "Failed to allocate DVMA arena(%s) " "for domain ID (%d)", arena_name, domain->dom_did); /*NOTREACHED*/ } mp = mp->ml_next; while (mp) { if (mp->ml_address == 0) start = MMU_PAGESIZE; else start = mp->ml_address; if (start + mp->ml_size > maxaddr) size = maxaddr - start; else size = mp->ml_size; ddi_err(DER_VERB, rdip, "iommu: %s: Adding dvma vmem span [0x%" PRIx64 " - 0x%" PRIx64 "]", arena_name, start, start + size); vmem_ret = vmem_add(domain->dom_dvma_arena, (void *)(uintptr_t)start, size, vmem_flags); if (vmem_ret == NULL) { ddi_err(DER_PANIC, rdip, "Failed to allocate DVMA arena(%s) " "for domain ID (%d)", arena_name, domain->dom_did); /*NOTREACHED*/ } mp = mp->ml_next; } memlist_read_unlock(); } /* ################################### DOMAIN CODE ######################### */ /* * Set the domain and domain-dip for a dip */ static void set_domain( dev_info_t *dip, dev_info_t *ddip, domain_t *domain) { immu_devi_t *immu_devi; domain_t *fdomain; dev_info_t *fddip; immu_devi = immu_devi_get(dip); mutex_enter(&(DEVI(dip)->devi_lock)); fddip = immu_devi->imd_ddip; fdomain = immu_devi->imd_domain; if (fddip) { ASSERT(fddip == ddip); } else { immu_devi->imd_ddip = ddip; } if (fdomain) { ASSERT(fdomain == domain); } else { immu_devi->imd_domain = domain; } mutex_exit(&(DEVI(dip)->devi_lock)); } /* * device_domain() * Get domain for a device. The domain may be global in which case it * is shared between all IOMMU units. Due to potential AGAW differences * between IOMMU units, such global domains *have to be* UNITY mapping * domains. Alternatively, the domain may be local to a IOMMU unit. * Local domains may be shared or immu_devi, although the * scope of sharing * is restricted to devices controlled by the IOMMU unit to * which the domain * belongs. If shared, they (currently) have to be UNITY domains. If * immu_devi a domain may be either UNITY or translation (XLATE) domain. */ static domain_t * device_domain(dev_info_t *rdip, dev_info_t **ddipp, immu_flags_t immu_flags) { dev_info_t *ddip; /* topmost dip in domain i.e. domain owner */ immu_t *immu; domain_t *domain; dvma_arg_t dvarg = {0}; int level; *ddipp = NULL; /* * Check if the domain is already set. This is usually true * if this is not the first DVMA transaction. */ ddip = NULL; domain = immu_devi_domain(rdip, &ddip); if (domain) { *ddipp = ddip; return (domain); } immu = immu_dvma_get_immu(rdip, immu_flags); if (immu == NULL) { /* * possible that there is no IOMMU unit for this device * - BIOS bugs are one example. */ ddi_err(DER_WARN, rdip, "No iommu unit found for device"); return (NULL); } immu_flags |= immu_devi_get(rdip)->imd_dvma_flags; dvarg.dva_rdip = rdip; dvarg.dva_ddip = NULL; dvarg.dva_domain = NULL; dvarg.dva_flags = immu_flags; level = 0; if (immu_walk_ancestor(rdip, NULL, get_branch_domain, &dvarg, &level, immu_flags) != DDI_SUCCESS) { /* * maybe low memory. return error, * so driver tries again later */ return (NULL); } /* should have walked at least 1 dip (i.e. edip) */ ASSERT(level > 0); ddip = dvarg.dva_ddip; /* must be present */ domain = dvarg.dva_domain; /* may be NULL */ /* * We may find the domain during our ancestor walk on any one of our * ancestor dips, If the domain is found then the domain-dip * (i.e. ddip) will also be found in the same immu_devi struct. * The domain-dip is the highest ancestor dip which shares the * same domain with edip. * The domain may or may not be found, but the domain dip must * be found. */ if (ddip == NULL) { ddi_err(DER_MODE, rdip, "Cannot find domain dip for device."); return (NULL); } /* * Did we find a domain ? */ if (domain) { goto found; } /* nope, so allocate */ domain = domain_create(immu, ddip, rdip, immu_flags); if (domain == NULL) { return (NULL); } /*FALLTHROUGH*/ found: /* * We know *domain *is* the right domain, so panic if * another domain is set for either the request-dip or * effective dip. */ set_domain(ddip, ddip, domain); set_domain(rdip, ddip, domain); *ddipp = ddip; return (domain); } static void create_unity_domain(immu_t *immu) { domain_t *domain; /* domain created during boot and always use sleep flag */ domain = kmem_zalloc(sizeof (domain_t), KM_SLEEP); rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL); domain->dom_did = IMMU_UNITY_DID; domain->dom_maptype = IMMU_MAPTYPE_UNITY; domain->dom_immu = immu; immu->immu_unity_domain = domain; /* * Setup the domain's initial page table * should never fail. */ domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP); pgtable_zero(domain->dom_pgtable_root); /* * Only map all physical memory in to the unity domain * if passthrough is not supported. If it is supported, * passthrough is set in the context entry instead. */ if (!IMMU_ECAP_GET_PT(immu->immu_regs_excap)) map_unity_domain(domain); /* * put it on the system-wide UNITY domain list */ mutex_enter(&(immu_domain_lock)); list_insert_tail(&immu_unity_domain_list, domain); mutex_exit(&(immu_domain_lock)); } /* * ddip is the domain-dip - the topmost dip in a domain * rdip is the requesting-dip - the device which is * requesting DVMA setup * if domain is a non-shared domain rdip == ddip */ static domain_t * domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip, immu_flags_t immu_flags) { int kmflags; domain_t *domain; char mod_hash_name[128]; immu_devi_t *immu_devi; int did; immu_dcookie_t dcookies[1] = {0}; int dcount = 0; immu_devi = immu_devi_get(rdip); /* * First allocate a domainid. * This routine will never fail, since if we run out * of domains the unity domain will be allocated. */ did = did_alloc(immu, rdip, ddip, immu_flags); if (did == IMMU_UNITY_DID) { /* domain overflow */ ASSERT(immu->immu_unity_domain); return (immu->immu_unity_domain); } kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; domain = kmem_zalloc(sizeof (domain_t), kmflags); if (domain == NULL) { ddi_err(DER_PANIC, rdip, "Failed to alloc DVMA domain " "structure for device. IOMMU unit: %s", immu->immu_name); /*NOTREACHED*/ } rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL); (void) snprintf(mod_hash_name, sizeof (mod_hash_name), "immu%s-domain%d-pava-hash", immu->immu_name, did); domain->dom_did = did; domain->dom_immu = immu; domain->dom_maptype = IMMU_MAPTYPE_XLATE; domain->dom_dip = ddip; /* * Create xlate DVMA arena for this domain. */ create_xlate_arena(immu, domain, rdip, immu_flags); /* * Setup the domain's initial page table */ domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags); if (domain->dom_pgtable_root == NULL) { ddi_err(DER_PANIC, rdip, "Failed to alloc root " "pgtable for domain (%d). IOMMU unit: %s", domain->dom_did, immu->immu_name); /*NOTREACHED*/ } pgtable_zero(domain->dom_pgtable_root); /* * Since this is a immu unit-specific domain, put it on * the per-immu domain list. */ mutex_enter(&(immu->immu_lock)); list_insert_head(&immu->immu_domain_list, domain); mutex_exit(&(immu->immu_lock)); /* * Also put it on the system-wide xlate domain list */ mutex_enter(&(immu_domain_lock)); list_insert_head(&immu_xlate_domain_list, domain); mutex_exit(&(immu_domain_lock)); bdf_domain_insert(immu_devi, domain); #ifdef BUGGY_DRIVERS /* * Map page0. Some broken HW/FW access it. */ dcookies[0].dck_paddr = 0; dcookies[0].dck_npages = 1; dcount = 1; (void) dvma_map(domain, 0, 1, dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1); #endif return (domain); } /* * Create domainid arena. * Domainid 0 is reserved by Vt-d spec and cannot be used by * system software. * Domainid 1 is reserved by solaris and used for *all* of the following: * as the "uninitialized" domain - For devices not yet controlled * by Solaris * as the "unity" domain - For devices that will always belong * to the unity domain * as the "overflow" domain - Used for any new device after we * run out of domains * All of the above domains map into a single domain with * domainid 1 and UNITY DVMA mapping * Each IMMU unity has its own unity/uninit/overflow domain */ static void did_init(immu_t *immu) { (void) snprintf(immu->immu_did_arena_name, sizeof (immu->immu_did_arena_name), "%s_domainid_arena", immu->immu_name); ddi_err(DER_VERB, immu->immu_dip, "creating domainid arena %s", immu->immu_did_arena_name); immu->immu_did_arena = vmem_create( immu->immu_did_arena_name, (void *)(uintptr_t)(IMMU_UNITY_DID + 1), /* start addr */ immu->immu_max_domains - IMMU_UNITY_DID, 1, /* quantum */ NULL, /* afunc */ NULL, /* ffunc */ NULL, /* source */ 0, /* qcache_max */ VM_SLEEP); /* Even with SLEEP flag, vmem_create() can fail */ if (immu->immu_did_arena == NULL) { ddi_err(DER_PANIC, NULL, "%s: Failed to create Intel " "IOMMU domainid allocator: %s", immu->immu_name, immu->immu_did_arena_name); } } /* ######################### CONTEXT CODE ################################# */ static void context_set(immu_t *immu, domain_t *domain, pgtable_t *root_table, int bus, int devfunc) { pgtable_t *context; pgtable_t *pgtable_root; hw_rce_t *hw_rent; hw_rce_t *hw_cent; hw_rce_t *ctxp; int sid; krw_t rwtype; boolean_t fill_root; boolean_t fill_ctx; pgtable_root = domain->dom_pgtable_root; ctxp = (hw_rce_t *)(root_table->swpg_next_array); context = *(pgtable_t **)(ctxp + bus); hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr) + bus; fill_root = B_FALSE; fill_ctx = B_FALSE; /* Check the most common case first with reader lock */ rw_enter(&(immu->immu_ctx_rwlock), RW_READER); rwtype = RW_READER; again: if (ROOT_GET_P(hw_rent)) { hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc; if (CONT_GET_AVAIL(hw_cent) == IMMU_CONT_INITED) { rw_exit(&(immu->immu_ctx_rwlock)); return; } else { fill_ctx = B_TRUE; } } else { fill_root = B_TRUE; fill_ctx = B_TRUE; } if (rwtype == RW_READER && rw_tryupgrade(&(immu->immu_ctx_rwlock)) == 0) { rw_exit(&(immu->immu_ctx_rwlock)); rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER); rwtype = RW_WRITER; goto again; } rwtype = RW_WRITER; if (fill_root == B_TRUE) { ROOT_SET_CONT(hw_rent, context->hwpg_paddr); ROOT_SET_P(hw_rent); immu_regs_cpu_flush(immu, (caddr_t)hw_rent, sizeof (hw_rce_t)); } if (fill_ctx == B_TRUE) { hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc; /* need to disable context entry before reprogramming it */ bzero(hw_cent, sizeof (hw_rce_t)); /* flush caches */ immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t)); sid = ((bus << 8) | devfunc); immu_flush_context_fsi(immu, 0, sid, domain->dom_did, &immu->immu_ctx_inv_wait); CONT_SET_AVAIL(hw_cent, IMMU_CONT_INITED); CONT_SET_DID(hw_cent, domain->dom_did); CONT_SET_AW(hw_cent, immu->immu_dvma_agaw); CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr); if (domain->dom_did == IMMU_UNITY_DID && IMMU_ECAP_GET_PT(immu->immu_regs_excap)) CONT_SET_TTYPE(hw_cent, TTYPE_PASSTHRU); else /*LINTED*/ CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY); CONT_SET_P(hw_cent); if (IMMU_ECAP_GET_CH(immu->immu_regs_excap)) { CONT_SET_EH(hw_cent); if (immu_use_alh) CONT_SET_ALH(hw_cent); } immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t)); } rw_exit(&(immu->immu_ctx_rwlock)); } static pgtable_t * context_create(immu_t *immu) { int bus; int devfunc; pgtable_t *root_table; pgtable_t *context; pgtable_t *pgtable_root; hw_rce_t *ctxp; hw_rce_t *hw_rent; hw_rce_t *hw_cent; /* Allocate a zeroed root table (4K 256b entries) */ root_table = pgtable_alloc(immu, IMMU_FLAGS_SLEEP); pgtable_zero(root_table); /* * Setup context tables for all possible root table entries. * Start out with unity domains for all entries. */ ctxp = (hw_rce_t *)(root_table->swpg_next_array); hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr); for (bus = 0; bus < IMMU_ROOT_NUM; bus++, ctxp++, hw_rent++) { context = pgtable_alloc(immu, IMMU_FLAGS_SLEEP); pgtable_zero(context); ROOT_SET_P(hw_rent); ROOT_SET_CONT(hw_rent, context->hwpg_paddr); hw_cent = (hw_rce_t *)(context->hwpg_vaddr); for (devfunc = 0; devfunc < IMMU_CONT_NUM; devfunc++, hw_cent++) { pgtable_root = immu->immu_unity_domain->dom_pgtable_root; CONT_SET_DID(hw_cent, immu->immu_unity_domain->dom_did); CONT_SET_AW(hw_cent, immu->immu_dvma_agaw); CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr); if (IMMU_ECAP_GET_PT(immu->immu_regs_excap)) CONT_SET_TTYPE(hw_cent, TTYPE_PASSTHRU); else /*LINTED*/ CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY); CONT_SET_AVAIL(hw_cent, IMMU_CONT_UNINITED); CONT_SET_P(hw_cent); } immu_regs_cpu_flush(immu, context->hwpg_vaddr, IMMU_PAGESIZE); *((pgtable_t **)ctxp) = context; } return (root_table); } /* * Called during rootnex attach, so no locks needed */ static void context_init(immu_t *immu) { rw_init(&(immu->immu_ctx_rwlock), NULL, RW_DEFAULT, NULL); immu_init_inv_wait(&immu->immu_ctx_inv_wait, "ctxglobal", B_TRUE); immu_regs_wbf_flush(immu); immu->immu_ctx_root = context_create(immu); immu_regs_set_root_table(immu); rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER); immu_flush_context_gbl(immu, &immu->immu_ctx_inv_wait); immu_flush_iotlb_gbl(immu, &immu->immu_ctx_inv_wait); rw_exit(&(immu->immu_ctx_rwlock)); } /* * Find top pcib */ static int find_top_pcib(dev_info_t *dip, void *arg) { immu_devi_t *immu_devi; dev_info_t **pcibdipp = (dev_info_t **)arg; immu_devi = immu_devi_get(dip); if (immu_devi->imd_pcib_type == IMMU_PCIB_PCI_PCI) { *pcibdipp = dip; } return (DDI_WALK_CONTINUE); } static int immu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip, dev_info_t *rdip, immu_flags_t immu_flags) { immu_devi_t *r_immu_devi; immu_devi_t *d_immu_devi; int r_bus; int d_bus; int r_devfunc; int d_devfunc; immu_pcib_t d_pcib_type; dev_info_t *pcibdip; if (ddip == NULL || rdip == NULL || ddip == root_devinfo || rdip == root_devinfo) { ddi_err(DER_MODE, rdip, "immu_contexts_update: domain-dip or " "request-dip are NULL or are root devinfo"); return (DDI_FAILURE); } /* * We need to set the context fields * based on what type of device rdip and ddip are. * To do that we need the immu_devi field. * Set the immu_devi field (if not already set) */ if (immu_devi_set(ddip, immu_flags) == DDI_FAILURE) { ddi_err(DER_MODE, rdip, "immu_context_update: failed to set immu_devi for ddip"); return (DDI_FAILURE); } if (immu_devi_set(rdip, immu_flags) == DDI_FAILURE) { ddi_err(DER_MODE, rdip, "immu_context_update: failed to set immu_devi for rdip"); return (DDI_FAILURE); } d_immu_devi = immu_devi_get(ddip); r_immu_devi = immu_devi_get(rdip); d_bus = d_immu_devi->imd_bus; d_devfunc = d_immu_devi->imd_devfunc; d_pcib_type = d_immu_devi->imd_pcib_type; r_bus = r_immu_devi->imd_bus; r_devfunc = r_immu_devi->imd_devfunc; if (rdip == ddip) { /* rdip is a PCIE device. set context for it only */ context_set(immu, domain, immu->immu_ctx_root, r_bus, r_devfunc); #ifdef BUGGY_DRIVERS } else if (r_immu_devi == d_immu_devi) { #ifdef TEST ddi_err(DER_WARN, rdip, "Driver bug: Devices 0x%lx and " "0x%lx are identical", rdip, ddip); #endif /* rdip is a PCIE device. set context for it only */ context_set(immu, domain, immu->immu_ctx_root, r_bus, r_devfunc); #endif } else if (d_pcib_type == IMMU_PCIB_PCIE_PCI) { /* * ddip is a PCIE_PCI bridge. Set context for ddip's * secondary bus. If rdip is on ddip's secondary * bus, set context for rdip. Else, set context * for rdip's PCI bridge on ddip's secondary bus. */ context_set(immu, domain, immu->immu_ctx_root, d_immu_devi->imd_sec, 0); if (d_immu_devi->imd_sec == r_bus) { context_set(immu, domain, immu->immu_ctx_root, r_bus, r_devfunc); } else { pcibdip = NULL; if (immu_walk_ancestor(rdip, ddip, find_top_pcib, &pcibdip, NULL, immu_flags) == DDI_SUCCESS && pcibdip != NULL) { r_immu_devi = immu_devi_get(pcibdip); r_bus = r_immu_devi->imd_bus; r_devfunc = r_immu_devi->imd_devfunc; context_set(immu, domain, immu->immu_ctx_root, r_bus, r_devfunc); } else { ddi_err(DER_PANIC, rdip, "Failed to find PCI " " bridge for PCI device"); /*NOTREACHED*/ } } } else if (d_pcib_type == IMMU_PCIB_PCI_PCI) { context_set(immu, domain, immu->immu_ctx_root, d_bus, d_devfunc); } else if (d_pcib_type == IMMU_PCIB_ENDPOINT) { /* * ddip is a PCIE device which has a non-PCI device under it * i.e. it is a PCI-nonPCI bridge. Example: pciicde-ata */ context_set(immu, domain, immu->immu_ctx_root, d_bus, d_devfunc); } else { ddi_err(DER_PANIC, rdip, "unknown device type. Cannot " "set iommu context."); /*NOTREACHED*/ } /* XXX do we need a membar_producer() here */ return (DDI_SUCCESS); } /* ##################### END CONTEXT CODE ################################## */ /* ##################### MAPPING CODE ################################## */ #ifdef DEBUG static boolean_t PDTE_check(immu_t *immu, hw_pdte_t pdte, pgtable_t *next, paddr_t paddr, dev_info_t *rdip, immu_flags_t immu_flags) { /* The PDTE must be set i.e. present bit is set */ if (!PDTE_P(pdte)) { ddi_err(DER_MODE, rdip, "No present flag"); return (B_FALSE); } /* * Just assert to check most significant system software field * (PDTE_SW4) as it is same as present bit and we * checked that above */ ASSERT(PDTE_SW4(pdte)); /* * TM field should be clear if not reserved. * non-leaf is always reserved */ if (next == NULL && immu->immu_TM_reserved == B_FALSE) { if (PDTE_TM(pdte)) { ddi_err(DER_MODE, rdip, "TM flag set"); return (B_FALSE); } } /* * The SW3 field is not used and must be clear */ if (PDTE_SW3(pdte)) { ddi_err(DER_MODE, rdip, "SW3 set"); return (B_FALSE); } /* * PFN (for PTE) or next level pgtable-paddr (for PDE) must be set */ if (next == NULL) { ASSERT(paddr % IMMU_PAGESIZE == 0); if (PDTE_PADDR(pdte) != paddr) { ddi_err(DER_MODE, rdip, "PTE paddr mismatch: %lx != %lx", PDTE_PADDR(pdte), paddr); return (B_FALSE); } } else { if (PDTE_PADDR(pdte) != next->hwpg_paddr) { ddi_err(DER_MODE, rdip, "PDE paddr mismatch: %lx != %lx", PDTE_PADDR(pdte), next->hwpg_paddr); return (B_FALSE); } } /* * SNP field should be clear if not reserved. * non-leaf is always reserved */ if (next == NULL && immu->immu_SNP_reserved == B_FALSE) { if (PDTE_SNP(pdte)) { ddi_err(DER_MODE, rdip, "SNP set"); return (B_FALSE); } } /* second field available for system software should be clear */ if (PDTE_SW2(pdte)) { ddi_err(DER_MODE, rdip, "SW2 set"); return (B_FALSE); } /* Super pages field should be clear */ if (PDTE_SP(pdte)) { ddi_err(DER_MODE, rdip, "SP set"); return (B_FALSE); } /* * least significant field available for * system software should be clear */ if (PDTE_SW1(pdte)) { ddi_err(DER_MODE, rdip, "SW1 set"); return (B_FALSE); } if ((immu_flags & IMMU_FLAGS_READ) && !PDTE_READ(pdte)) { ddi_err(DER_MODE, rdip, "READ not set"); return (B_FALSE); } if ((immu_flags & IMMU_FLAGS_WRITE) && !PDTE_WRITE(pdte)) { ddi_err(DER_MODE, rdip, "WRITE not set"); return (B_FALSE); } return (B_TRUE); } #endif /*ARGSUSED*/ static void PTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate, uint64_t *dvma_ptr, uint64_t *npages_ptr, dev_info_t *rdip) { uint64_t npages; uint64_t dvma; pgtable_t *pgtable; hw_pdte_t *hwp; hw_pdte_t *shwp; int idx; pgtable = xlate->xlt_pgtable; idx = xlate->xlt_idx; dvma = *dvma_ptr; npages = *npages_ptr; /* * since a caller gets a unique dvma for a physical address, * no other concurrent thread will be writing to the same * PTE even if it has the same paddr. So no locks needed. */ shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx; hwp = shwp; for (; npages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) { PDTE_CLEAR_P(*hwp); dvma += IMMU_PAGESIZE; npages--; } *dvma_ptr = dvma; *npages_ptr = npages; xlate->xlt_idx = idx; } static void xlate_setup(uint64_t dvma, xlate_t *xlate, int nlevels) { int level; uint64_t offbits; /* * Skip the first 12 bits which is the offset into * 4K PFN (phys page frame based on IMMU_PAGESIZE) */ offbits = dvma >> IMMU_PAGESHIFT; /* skip to level 1 i.e. leaf PTE */ for (level = 1, xlate++; level <= nlevels; level++, xlate++) { xlate->xlt_level = level; xlate->xlt_idx = (offbits & IMMU_PGTABLE_LEVEL_MASK); ASSERT(xlate->xlt_idx <= IMMU_PGTABLE_MAXIDX); xlate->xlt_pgtable = NULL; offbits >>= IMMU_PGTABLE_LEVEL_STRIDE; } } /* * Read the pgtables */ static boolean_t PDE_lookup(domain_t *domain, xlate_t *xlate, int nlevels) { pgtable_t *pgtable; pgtable_t *next; uint_t idx; /* start with highest level pgtable i.e. root */ xlate += nlevels; if (xlate->xlt_pgtable == NULL) { xlate->xlt_pgtable = domain->dom_pgtable_root; } for (; xlate->xlt_level > 1; xlate--) { idx = xlate->xlt_idx; pgtable = xlate->xlt_pgtable; if ((xlate - 1)->xlt_pgtable) { continue; } /* Lock the pgtable in read mode */ rw_enter(&(pgtable->swpg_rwlock), RW_READER); /* * since we are unmapping, the pgtable should * already point to a leafier pgtable. */ next = *(pgtable->swpg_next_array + idx); (xlate - 1)->xlt_pgtable = next; rw_exit(&(pgtable->swpg_rwlock)); if (next == NULL) return (B_FALSE); } return (B_TRUE); } static void immu_fault_walk(void *arg, void *base, size_t len) { uint64_t dvma, start; dvma = *(uint64_t *)arg; start = (uint64_t)(uintptr_t)base; if (dvma >= start && dvma < (start + len)) { ddi_err(DER_WARN, NULL, "faulting DVMA address is in vmem arena " "(%" PRIx64 "-%" PRIx64 ")", start, start + len); *(uint64_t *)arg = ~0ULL; } } void immu_print_fault_info(uint_t sid, uint64_t dvma) { int nlevels; xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0}; xlate_t *xlatep; hw_pdte_t pte; domain_t *domain; immu_t *immu; uint64_t dvma_arg; if (mod_hash_find(bdf_domain_hash, (void *)(uintptr_t)sid, (void *)&domain) != 0) { ddi_err(DER_WARN, NULL, "no domain for faulting SID %08x", sid); return; } immu = domain->dom_immu; dvma_arg = dvma; vmem_walk(domain->dom_dvma_arena, VMEM_ALLOC, immu_fault_walk, (void *)&dvma_arg); if (dvma_arg != ~0ULL) ddi_err(DER_WARN, domain->dom_dip, "faulting DVMA address is not in vmem arena"); nlevels = immu->immu_dvma_nlevels; xlate_setup(dvma, xlate, nlevels); if (!PDE_lookup(domain, xlate, nlevels)) { ddi_err(DER_WARN, domain->dom_dip, "pte not found in domid %d for faulting addr %" PRIx64, domain->dom_did, dvma); return; } xlatep = &xlate[1]; pte = *((hw_pdte_t *) (xlatep->xlt_pgtable->hwpg_vaddr) + xlatep->xlt_idx); ddi_err(DER_WARN, domain->dom_dip, "domid %d pte: %" PRIx64 "(paddr %" PRIx64 ")", domain->dom_did, (unsigned long long)pte, (unsigned long long)PDTE_PADDR(pte)); } /*ARGSUSED*/ static void PTE_set_one(immu_t *immu, hw_pdte_t *hwp, paddr_t paddr, dev_info_t *rdip, immu_flags_t immu_flags) { hw_pdte_t pte; #ifndef DEBUG pte = immu->immu_ptemask; PDTE_SET_PADDR(pte, paddr); #else pte = *hwp; if (PDTE_P(pte)) { if (PDTE_PADDR(pte) != paddr) { ddi_err(DER_MODE, rdip, "PTE paddr %lx != paddr %lx", PDTE_PADDR(pte), paddr); } #ifdef BUGGY_DRIVERS return; #else goto out; #endif } /* clear TM field if not reserved */ if (immu->immu_TM_reserved == B_FALSE) { PDTE_CLEAR_TM(pte); } /* Clear 3rd field for system software - not used */ PDTE_CLEAR_SW3(pte); /* Set paddr */ ASSERT(paddr % IMMU_PAGESIZE == 0); PDTE_CLEAR_PADDR(pte); PDTE_SET_PADDR(pte, paddr); /* clear SNP field if not reserved. */ if (immu->immu_SNP_reserved == B_FALSE) { PDTE_CLEAR_SNP(pte); } /* Clear SW2 field available for software */ PDTE_CLEAR_SW2(pte); /* SP is don't care for PTEs. Clear it for cleanliness */ PDTE_CLEAR_SP(pte); /* Clear SW1 field available for software */ PDTE_CLEAR_SW1(pte); /* * Now that we are done writing the PTE * set the "present" flag. Note this present * flag is a bit in the PDE/PTE that the * spec says is available for system software. * This is an implementation detail of Solaris * bare-metal Intel IOMMU. * The present field in a PDE/PTE is not defined * by the Vt-d spec */ PDTE_SET_P(pte); pte |= immu->immu_ptemask; out: #endif /* DEBUG */ #ifdef BUGGY_DRIVERS PDTE_SET_READ(pte); PDTE_SET_WRITE(pte); #else if (immu_flags & IMMU_FLAGS_READ) PDTE_SET_READ(pte); if (immu_flags & IMMU_FLAGS_WRITE) PDTE_SET_WRITE(pte); #endif /* BUGGY_DRIVERS */ *hwp = pte; } /*ARGSUSED*/ static void PTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, uint64_t *dvma_ptr, uint64_t *nvpages_ptr, immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip, immu_flags_t immu_flags) { paddr_t paddr; uint64_t nvpages; uint64_t nppages; uint64_t dvma; pgtable_t *pgtable; hw_pdte_t *hwp; hw_pdte_t *shwp; int idx, nset; int j; pgtable = xlate->xlt_pgtable; idx = xlate->xlt_idx; dvma = *dvma_ptr; nvpages = *nvpages_ptr; /* * since a caller gets a unique dvma for a physical address, * no other concurrent thread will be writing to the same * PTE even if it has the same paddr. So no locks needed. */ shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx; hwp = shwp; for (j = dcount - 1; j >= 0; j--) { if (nvpages <= dcookies[j].dck_npages) break; nvpages -= dcookies[j].dck_npages; } nppages = nvpages; paddr = dcookies[j].dck_paddr + (dcookies[j].dck_npages - nppages) * IMMU_PAGESIZE; nvpages = *nvpages_ptr; nset = 0; for (; nvpages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) { PTE_set_one(immu, hwp, paddr, rdip, immu_flags); nset++; ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags) == B_TRUE); nppages--; nvpages--; paddr += IMMU_PAGESIZE; dvma += IMMU_PAGESIZE; if (nppages == 0) { j++; } if (j == dcount) break; if (nppages == 0) { nppages = dcookies[j].dck_npages; paddr = dcookies[j].dck_paddr; } } if (nvpages) { *dvma_ptr = dvma; *nvpages_ptr = nvpages; } else { *dvma_ptr = 0; *nvpages_ptr = 0; } xlate->xlt_idx = idx; } /*ARGSUSED*/ static void PDE_set_one(immu_t *immu, hw_pdte_t *hwp, pgtable_t *next, dev_info_t *rdip, immu_flags_t immu_flags) { hw_pdte_t pde; pde = *hwp; /* if PDE is already set, make sure it is correct */ if (PDTE_P(pde)) { ASSERT(PDTE_PADDR(pde) == next->hwpg_paddr); #ifdef BUGGY_DRIVERS return; #else goto out; #endif } /* Dont touch SW4, it is the present bit */ /* don't touch TM field it is reserved for PDEs */ /* 3rd field available for system software is not used */ PDTE_CLEAR_SW3(pde); /* Set next level pgtable-paddr for PDE */ PDTE_CLEAR_PADDR(pde); PDTE_SET_PADDR(pde, next->hwpg_paddr); /* don't touch SNP field it is reserved for PDEs */ /* Clear second field available for system software */ PDTE_CLEAR_SW2(pde); /* No super pages for PDEs */ PDTE_CLEAR_SP(pde); /* Clear SW1 for software */ PDTE_CLEAR_SW1(pde); /* * Now that we are done writing the PDE * set the "present" flag. Note this present * flag is a bit in the PDE/PTE that the * spec says is available for system software. * This is an implementation detail of Solaris * base-metal Intel IOMMU. * The present field in a PDE/PTE is not defined * by the Vt-d spec */ out: #ifdef BUGGY_DRIVERS PDTE_SET_READ(pde); PDTE_SET_WRITE(pde); #else if (immu_flags & IMMU_FLAGS_READ) PDTE_SET_READ(pde); if (immu_flags & IMMU_FLAGS_WRITE) PDTE_SET_WRITE(pde); #endif PDTE_SET_P(pde); *hwp = pde; } /* * Used to set PDEs */ static boolean_t PDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels, dev_info_t *rdip, immu_flags_t immu_flags) { pgtable_t *pgtable; pgtable_t *new; pgtable_t *next; hw_pdte_t *hwp; int level; uint_t idx; krw_t rwtype; boolean_t set = B_FALSE; /* start with highest level pgtable i.e. root */ xlate += nlevels; new = NULL; xlate->xlt_pgtable = domain->dom_pgtable_root; for (level = nlevels; level > 1; level--, xlate--) { idx = xlate->xlt_idx; pgtable = xlate->xlt_pgtable; /* Lock the pgtable in READ mode first */ rw_enter(&(pgtable->swpg_rwlock), RW_READER); rwtype = RW_READER; again: hwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx; next = (pgtable->swpg_next_array)[idx]; /* * check if leafier level already has a pgtable * if yes, verify */ if (next == NULL) { if (new == NULL) { IMMU_DPROBE2(immu__pdp__alloc, dev_info_t *, rdip, int, level); new = pgtable_alloc(immu, immu_flags); if (new == NULL) { ddi_err(DER_PANIC, rdip, "pgtable alloc err"); } pgtable_zero(new); } /* Change to a write lock */ if (rwtype == RW_READER && rw_tryupgrade(&(pgtable->swpg_rwlock)) == 0) { rw_exit(&(pgtable->swpg_rwlock)); rw_enter(&(pgtable->swpg_rwlock), RW_WRITER); rwtype = RW_WRITER; goto again; } rwtype = RW_WRITER; next = new; (pgtable->swpg_next_array)[idx] = next; new = NULL; PDE_set_one(immu, hwp, next, rdip, immu_flags); set = B_TRUE; rw_downgrade(&(pgtable->swpg_rwlock)); rwtype = RW_READER; } #ifndef BUGGY_DRIVERS else { hw_pdte_t pde = *hwp; /* * If buggy driver we already set permission * READ+WRITE so nothing to do for that case * XXX Check that read writer perms change before * actually setting perms. Also need to hold lock */ if (immu_flags & IMMU_FLAGS_READ) PDTE_SET_READ(pde); if (immu_flags & IMMU_FLAGS_WRITE) PDTE_SET_WRITE(pde); *hwp = pde; } #endif ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags) == B_TRUE); (xlate - 1)->xlt_pgtable = next; rw_exit(&(pgtable->swpg_rwlock)); } if (new) { pgtable_free(immu, new); } return (set); } /* * dvma_map() * map a contiguous range of DVMA pages * * immu: IOMMU unit for which we are generating DVMA cookies * domain: domain * sdvma: Starting dvma * spaddr: Starting paddr * npages: Number of pages * rdip: requesting device * immu_flags: flags */ static boolean_t dvma_map(domain_t *domain, uint64_t sdvma, uint64_t snvpages, immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip, immu_flags_t immu_flags) { uint64_t dvma; uint64_t n; immu_t *immu = domain->dom_immu; int nlevels = immu->immu_dvma_nlevels; xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0}; boolean_t pde_set = B_FALSE; n = snvpages; dvma = sdvma; while (n > 0) { xlate_setup(dvma, xlate, nlevels); /* Lookup or allocate PGDIRs and PGTABLEs if necessary */ if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags) == B_TRUE) { pde_set = B_TRUE; } /* set all matching ptes that fit into this leaf pgtable */ PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies, dcount, rdip, immu_flags); } return (pde_set); } /* * dvma_unmap() * unmap a range of DVMAs * * immu: IOMMU unit state * domain: domain for requesting device * ddip: domain-dip * dvma: starting DVMA * npages: Number of IMMU pages to be unmapped * rdip: requesting device */ static void dvma_unmap(domain_t *domain, uint64_t sdvma, uint64_t snpages, dev_info_t *rdip) { immu_t *immu = domain->dom_immu; int nlevels = immu->immu_dvma_nlevels; xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0}; uint64_t n; uint64_t dvma; dvma = sdvma; n = snpages; while (n > 0) { /* setup the xlate array */ xlate_setup(dvma, xlate, nlevels); /* just lookup existing pgtables. Should never fail */ if (!PDE_lookup(domain, xlate, nlevels)) ddi_err(DER_PANIC, rdip, "PTE not found for addr %" PRIx64, (unsigned long long)dvma); /* clear all matching ptes that fit into this leaf pgtable */ PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip); } /* No need to flush IOTLB after unmap */ } static uint64_t dvma_alloc(domain_t *domain, ddi_dma_attr_t *dma_attr, uint_t npages, int kmf) { uint64_t dvma; size_t xsize, align; uint64_t minaddr, maxaddr; /* parameters */ xsize = npages * IMMU_PAGESIZE; align = MAX((size_t)(dma_attr->dma_attr_align), IMMU_PAGESIZE); minaddr = dma_attr->dma_attr_addr_lo; maxaddr = dma_attr->dma_attr_addr_hi + 1; /* handle the rollover cases */ if (maxaddr < dma_attr->dma_attr_addr_hi) { maxaddr = dma_attr->dma_attr_addr_hi; } /* * allocate from vmem arena. */ dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena, xsize, align, 0, 0, (void *)(uintptr_t)minaddr, (void *)(uintptr_t)maxaddr, kmf); return (dvma); } static void dvma_prealloc(dev_info_t *rdip, immu_hdl_priv_t *ihp, ddi_dma_attr_t *dma_attr) { int nlevels; xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0}, *xlp; uint64_t dvma, n; size_t xsize, align; uint64_t minaddr, maxaddr, dmamax; int on, npte, pindex; hw_pdte_t *shwp; immu_t *immu; domain_t *domain; /* parameters */ domain = IMMU_DEVI(rdip)->imd_domain; immu = domain->dom_immu; nlevels = immu->immu_dvma_nlevels; xsize = IMMU_NPREPTES * IMMU_PAGESIZE; align = MAX((size_t)(dma_attr->dma_attr_align), IMMU_PAGESIZE); minaddr = dma_attr->dma_attr_addr_lo; if (dma_attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) dmamax = dma_attr->dma_attr_seg; else dmamax = dma_attr->dma_attr_addr_hi; maxaddr = dmamax + 1; if (maxaddr < dmamax) maxaddr = dmamax; dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena, xsize, align, 0, dma_attr->dma_attr_seg + 1, (void *)(uintptr_t)minaddr, (void *)(uintptr_t)maxaddr, VM_NOSLEEP); ihp->ihp_predvma = dvma; ihp->ihp_npremapped = 0; if (dvma == 0) return; n = IMMU_NPREPTES; pindex = 0; /* * Set up a mapping at address 0, just so that all PDPs get allocated * now. Although this initial mapping should never be used, * explicitly set it to read-only, just to be safe. */ while (n > 0) { xlate_setup(dvma, xlate, nlevels); (void) PDE_set_all(immu, domain, xlate, nlevels, rdip, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE); xlp = &xlate[1]; shwp = (hw_pdte_t *)(xlp->xlt_pgtable->hwpg_vaddr) + xlp->xlt_idx; on = n; PTE_set_all(immu, domain, xlp, &dvma, &n, &immu_precookie, 1, rdip, IMMU_FLAGS_READ); npte = on - n; while (npte > 0) { ihp->ihp_preptes[pindex++] = shwp; #ifdef BUGGY_DRIVERS PDTE_CLEAR_WRITE(*shwp); #endif shwp++; npte--; } } } static void dvma_prefree(dev_info_t *rdip, immu_hdl_priv_t *ihp) { domain_t *domain; domain = IMMU_DEVI(rdip)->imd_domain; if (ihp->ihp_predvma != 0) { dvma_unmap(domain, ihp->ihp_predvma, IMMU_NPREPTES, rdip); vmem_free(domain->dom_dvma_arena, (void *)(uintptr_t)ihp->ihp_predvma, IMMU_NPREPTES * IMMU_PAGESIZE); } } static void dvma_free(domain_t *domain, uint64_t dvma, uint64_t npages) { uint64_t size = npages * IMMU_PAGESIZE; if (domain->dom_maptype != IMMU_MAPTYPE_XLATE) return; vmem_free(domain->dom_dvma_arena, (void *)(uintptr_t)dvma, size); } static int immu_map_dvmaseg(dev_info_t *rdip, ddi_dma_handle_t handle, immu_hdl_priv_t *ihp, struct ddi_dma_req *dmareq, ddi_dma_obj_t *dma_out) { domain_t *domain; immu_t *immu; immu_flags_t immu_flags; ddi_dma_atyp_t buftype; ddi_dma_obj_t *dmar_object; ddi_dma_attr_t *attrp; uint64_t offset, paddr, dvma, sdvma, rwmask; size_t npages, npgalloc; uint_t psize, size, pcnt, dmax; page_t **pparray; caddr_t vaddr; page_t *page; struct as *vas; immu_dcookie_t *dcookies; int pde_set; domain = IMMU_DEVI(rdip)->imd_domain; immu = domain->dom_immu; immu_flags = dma_to_immu_flags(dmareq); attrp = &((ddi_dma_impl_t *)handle)->dmai_attr; dmar_object = &dmareq->dmar_object; pparray = dmar_object->dmao_obj.virt_obj.v_priv; vaddr = dmar_object->dmao_obj.virt_obj.v_addr; buftype = dmar_object->dmao_type; size = dmar_object->dmao_size; IMMU_DPROBE3(immu__map__dvma, dev_info_t *, rdip, ddi_dma_atyp_t, buftype, uint_t, size); dcookies = &ihp->ihp_dcookies[0]; pcnt = dmax = 0; /* retrieve paddr, psize, offset from dmareq */ if (buftype == DMA_OTYP_PAGES) { page = dmar_object->dmao_obj.pp_obj.pp_pp; offset = dmar_object->dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; paddr = pfn_to_pa(page->p_pagenum) + offset; psize = MIN((MMU_PAGESIZE - offset), size); page = page->p_next; vas = dmar_object->dmao_obj.virt_obj.v_as; } else { if (vas == NULL) { vas = &kas; } offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; if (pparray != NULL) { paddr = pfn_to_pa(pparray[pcnt]->p_pagenum) + offset; psize = MIN((MMU_PAGESIZE - offset), size); pcnt++; } else { paddr = pfn_to_pa(hat_getpfnum(vas->a_hat, vaddr)) + offset; psize = MIN(size, (MMU_PAGESIZE - offset)); vaddr += psize; } } npgalloc = IMMU_BTOPR(size + offset); if (npgalloc <= IMMU_NPREPTES && ihp->ihp_predvma != 0) { #ifdef BUGGY_DRIVERS rwmask = PDTE_MASK_R | PDTE_MASK_W | immu->immu_ptemask; #else rwmask = immu->immu_ptemask; if (immu_flags & IMMU_FLAGS_READ) rwmask |= PDTE_MASK_R; if (immu_flags & IMMU_FLAGS_WRITE) rwmask |= PDTE_MASK_W; #endif #ifdef DEBUG rwmask |= PDTE_MASK_P; #endif sdvma = ihp->ihp_predvma; ihp->ihp_npremapped = npgalloc; *ihp->ihp_preptes[0] = PDTE_PADDR(paddr & ~MMU_PAGEOFFSET) | rwmask; } else { ihp->ihp_npremapped = 0; sdvma = dvma_alloc(domain, attrp, npgalloc, dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP); if (sdvma == 0) return (DDI_DMA_NORESOURCES); dcookies[0].dck_paddr = (paddr & ~MMU_PAGEOFFSET); dcookies[0].dck_npages = 1; } IMMU_DPROBE3(immu__dvma__alloc, dev_info_t *, rdip, uint64_t, npgalloc, uint64_t, sdvma); dvma = sdvma; pde_set = 0; npages = 1; size -= psize; while (size > 0) { /* get the size for this page (i.e. partial or full page) */ psize = MIN(size, MMU_PAGESIZE); if (buftype == DMA_OTYP_PAGES) { /* get the paddr from the page_t */ paddr = pfn_to_pa(page->p_pagenum); page = page->p_next; } else if (pparray != NULL) { /* index into the array of page_t's to get the paddr */ paddr = pfn_to_pa(pparray[pcnt]->p_pagenum); pcnt++; } else { /* call into the VM to get the paddr */ paddr = pfn_to_pa(hat_getpfnum(vas->a_hat, vaddr)); vaddr += psize; } npages++; if (ihp->ihp_npremapped > 0) { *ihp->ihp_preptes[npages - 1] = PDTE_PADDR(paddr) | rwmask; } else if (IMMU_CONTIG_PADDR(dcookies[dmax], paddr)) { dcookies[dmax].dck_npages++; } else { /* No, we need a new dcookie */ if (dmax == (IMMU_NDCK - 1)) { /* * Ran out of dcookies. Map them now. */ if (dvma_map(domain, dvma, npages, dcookies, dmax + 1, rdip, immu_flags)) pde_set++; IMMU_DPROBE4(immu__dvmamap__early, dev_info_t *, rdip, uint64_t, dvma, uint_t, npages, uint_t, dmax+1); dvma += (npages << IMMU_PAGESHIFT); npages = 0; dmax = 0; } else dmax++; dcookies[dmax].dck_paddr = paddr; dcookies[dmax].dck_npages = 1; } size -= psize; } /* * Finish up, mapping all, or all of the remaining, * physical memory ranges. */ if (ihp->ihp_npremapped == 0 && npages > 0) { IMMU_DPROBE4(immu__dvmamap__late, dev_info_t *, rdip, \ uint64_t, dvma, uint_t, npages, uint_t, dmax+1); if (dvma_map(domain, dvma, npages, dcookies, dmax + 1, rdip, immu_flags)) pde_set++; } /* Invalidate the IOTLB */ immu_flush_iotlb_psi(immu, domain->dom_did, sdvma, npgalloc, pde_set > 0 ? TLB_IVA_WHOLE : TLB_IVA_LEAF, &ihp->ihp_inv_wait); ihp->ihp_ndvseg = 1; ihp->ihp_dvseg[0].dvs_start = sdvma; ihp->ihp_dvseg[0].dvs_len = dmar_object->dmao_size; dma_out->dmao_size = dmar_object->dmao_size; dma_out->dmao_obj.dvma_obj.dv_off = offset & IMMU_PAGEOFFSET; dma_out->dmao_obj.dvma_obj.dv_nseg = 1; dma_out->dmao_obj.dvma_obj.dv_seg = &ihp->ihp_dvseg[0]; dma_out->dmao_type = DMA_OTYP_DVADDR; return (DDI_DMA_MAPPED); } static int immu_unmap_dvmaseg(dev_info_t *rdip, ddi_dma_obj_t *dmao) { uint64_t dvma, npages; domain_t *domain; struct dvmaseg *dvs; domain = IMMU_DEVI(rdip)->imd_domain; dvs = dmao->dmao_obj.dvma_obj.dv_seg; dvma = dvs[0].dvs_start; npages = IMMU_BTOPR(dvs[0].dvs_len + dmao->dmao_obj.dvma_obj.dv_off); #ifdef DEBUG /* Unmap only in DEBUG mode */ dvma_unmap(domain, dvma, npages, rdip); #endif dvma_free(domain, dvma, npages); IMMU_DPROBE3(immu__dvma__free, dev_info_t *, rdip, uint_t, npages, uint64_t, dvma); #ifdef DEBUG /* * In the DEBUG case, the unmap was actually done, * but an IOTLB flush was not done. So, an explicit * write back flush is needed. */ immu_regs_wbf_flush(domain->dom_immu); #endif return (DDI_SUCCESS); } /* ############################# Functions exported ######################## */ /* * setup the DVMA subsystem * this code runs only for the first IOMMU unit */ void immu_dvma_setup(list_t *listp) { immu_t *immu; uint_t kval; size_t nchains; /* locks */ mutex_init(&immu_domain_lock, NULL, MUTEX_DEFAULT, NULL); /* Create lists */ list_create(&immu_unity_domain_list, sizeof (domain_t), offsetof(domain_t, dom_maptype_node)); list_create(&immu_xlate_domain_list, sizeof (domain_t), offsetof(domain_t, dom_maptype_node)); /* Setup BDF domain hash */ nchains = 0xff; kval = mod_hash_iddata_gen(nchains); bdf_domain_hash = mod_hash_create_extended("BDF-DOMAIN_HASH", nchains, mod_hash_null_keydtor, mod_hash_null_valdtor, mod_hash_byid, (void *)(uintptr_t)kval, mod_hash_idkey_cmp, KM_NOSLEEP); immu = list_head(listp); for (; immu; immu = list_next(listp, immu)) { create_unity_domain(immu); did_init(immu); context_init(immu); immu->immu_dvma_setup = B_TRUE; } } /* * Startup up one DVMA unit */ void immu_dvma_startup(immu_t *immu) { if (immu_gfxdvma_enable == B_FALSE && immu->immu_dvma_gfx_only == B_TRUE) { return; } /* * DVMA will start once IOMMU is "running" */ immu->immu_dvma_running = B_TRUE; } /* * immu_dvma_physmem_update() * called when the installed memory on a * system increases, to expand domain DVMA * for domains with UNITY mapping */ void immu_dvma_physmem_update(uint64_t addr, uint64_t size) { uint64_t start; uint64_t npages; int dcount; immu_dcookie_t dcookies[1] = {0}; domain_t *domain; /* * Just walk the system-wide list of domains with * UNITY mapping. Both the list of *all* domains * and *UNITY* domains is protected by the same * single lock */ mutex_enter(&immu_domain_lock); domain = list_head(&immu_unity_domain_list); for (; domain; domain = list_next(&immu_unity_domain_list, domain)) { /* * Nothing to do if the IOMMU supports passthrough. */ if (IMMU_ECAP_GET_PT(domain->dom_immu->immu_regs_excap)) continue; /* There is no vmem_arena for unity domains. Just map it */ ddi_err(DER_LOG, domain->dom_dip, "iommu: unity-domain: Adding map " "[0x%" PRIx64 " - 0x%" PRIx64 "]", addr, addr + size); start = IMMU_ROUNDOWN(addr); npages = (IMMU_ROUNDUP(size) / IMMU_PAGESIZE) + 1; dcookies[0].dck_paddr = start; dcookies[0].dck_npages = npages; dcount = 1; (void) dvma_map(domain, start, npages, dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE); } mutex_exit(&immu_domain_lock); } int immu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags) { dev_info_t *ddip, *odip; immu_t *immu; domain_t *domain; odip = rdip; immu = immu_dvma_get_immu(rdip, immu_flags); if (immu == NULL) { /* * possible that there is no IOMMU unit for this device * - BIOS bugs are one example. */ ddi_err(DER_WARN, rdip, "No iommu unit found for device"); return (DDI_DMA_NORESOURCES); } /* * redirect isa devices attached under lpc to lpc dip */ if (strcmp(ddi_node_name(ddi_get_parent(rdip)), "isa") == 0) { rdip = get_lpc_devinfo(immu, rdip, immu_flags); if (rdip == NULL) { ddi_err(DER_PANIC, rdip, "iommu redirect failed"); /*NOTREACHED*/ } } /* Reset immu, as redirection can change IMMU */ immu = NULL; /* * for gart, redirect to the real graphic devinfo */ if (strcmp(ddi_node_name(rdip), "agpgart") == 0) { rdip = get_gfx_devinfo(rdip); if (rdip == NULL) { ddi_err(DER_PANIC, rdip, "iommu redirect failed"); /*NOTREACHED*/ } } /* * Setup DVMA domain for the device. This does * work only the first time we do DVMA for a * device. */ ddip = NULL; domain = device_domain(rdip, &ddip, immu_flags); if (domain == NULL) { ddi_err(DER_MODE, rdip, "Intel IOMMU setup failed for device"); return (DDI_DMA_NORESOURCES); } immu = domain->dom_immu; /* * If a domain is found, we must also have a domain dip * which is the topmost ancestor dip of rdip that shares * the same domain with rdip. */ if (domain->dom_did == 0 || ddip == NULL) { ddi_err(DER_MODE, rdip, "domain did 0(%d) or ddip NULL(%p)", domain->dom_did, ddip); return (DDI_DMA_NORESOURCES); } if (odip != rdip) set_domain(odip, ddip, domain); /* * Update the root and context entries */ if (immu_context_update(immu, domain, ddip, rdip, immu_flags) != DDI_SUCCESS) { ddi_err(DER_MODE, rdip, "DVMA map: context update failed"); return (DDI_DMA_NORESOURCES); } return (DDI_SUCCESS); } int immu_map_memrange(dev_info_t *rdip, memrng_t *mrng) { immu_dcookie_t dcookies[1] = {0}; boolean_t pde_set; immu_t *immu; domain_t *domain; immu_inv_wait_t iw; dcookies[0].dck_paddr = mrng->mrng_start; dcookies[0].dck_npages = mrng->mrng_npages; domain = IMMU_DEVI(rdip)->imd_domain; immu = domain->dom_immu; pde_set = dvma_map(domain, mrng->mrng_start, mrng->mrng_npages, dcookies, 1, rdip, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE); immu_init_inv_wait(&iw, "memrange", B_TRUE); immu_flush_iotlb_psi(immu, domain->dom_did, mrng->mrng_start, mrng->mrng_npages, pde_set == B_TRUE ? TLB_IVA_WHOLE : TLB_IVA_LEAF, &iw); return (DDI_SUCCESS); } immu_devi_t * immu_devi_get(dev_info_t *rdip) { immu_devi_t *immu_devi; volatile uintptr_t *vptr = (uintptr_t *)&(DEVI(rdip)->devi_iommu); /* Just want atomic reads. No need for lock */ immu_devi = (immu_devi_t *)(uintptr_t)atomic_or_64_nv((uint64_t *)vptr, 0); return (immu_devi); } /*ARGSUSED*/ int immu_hdl_priv_ctor(void *buf, void *arg, int kmf) { immu_hdl_priv_t *ihp; ihp = buf; immu_init_inv_wait(&ihp->ihp_inv_wait, "dmahandle", B_FALSE); return (0); } /* * iommulib interface functions */ static int immu_probe(iommulib_handle_t handle, dev_info_t *dip) { immu_devi_t *immu_devi; int ret; if (!immu_enable) return (DDI_FAILURE); /* * Make sure the device has all the IOMMU structures * initialized. If this device goes through an IOMMU * unit (e.g. this probe function returns success), * this will be called at most N times, with N being * the number of IOMMUs in the system. * * After that, when iommulib_nex_open succeeds, * we can always assume that this device has all * the structures initialized. IOMMU_USED(dip) will * be true. There is no need to find the controlling * IOMMU/domain again. */ ret = immu_dvma_device_setup(dip, IMMU_FLAGS_NOSLEEP); if (ret != DDI_SUCCESS) return (ret); immu_devi = IMMU_DEVI(dip); /* * For unity domains, there is no need to call in to * the IOMMU code. */ if (immu_devi->imd_domain->dom_did == IMMU_UNITY_DID) return (DDI_FAILURE); if (immu_devi->imd_immu->immu_dip == iommulib_iommu_getdip(handle)) return (DDI_SUCCESS); return (DDI_FAILURE); } /*ARGSUSED*/ static int immu_allochdl(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep) { int ret; immu_hdl_priv_t *ihp; immu_t *immu; ret = iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp, arg, dma_handlep); if (ret == DDI_SUCCESS) { immu = IMMU_DEVI(rdip)->imd_immu; ihp = kmem_cache_alloc(immu->immu_hdl_cache, waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP); if (ihp == NULL) { (void) iommulib_iommu_dma_freehdl(dip, rdip, *dma_handlep); return (DDI_DMA_NORESOURCES); } if (IMMU_DEVI(rdip)->imd_use_premap) dvma_prealloc(rdip, ihp, attr); else { ihp->ihp_npremapped = 0; ihp->ihp_predvma = 0; } ret = iommulib_iommu_dmahdl_setprivate(dip, rdip, *dma_handlep, ihp); } return (ret); } /*ARGSUSED*/ static int immu_freehdl(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle) { immu_hdl_priv_t *ihp; ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle); if (ihp != NULL) { if (IMMU_DEVI(rdip)->imd_use_premap) dvma_prefree(rdip, ihp); kmem_cache_free(IMMU_DEVI(rdip)->imd_immu->immu_hdl_cache, ihp); } return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle)); } /*ARGSUSED*/ static int immu_bindhdl(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, struct ddi_dma_req *dma_req, ddi_dma_cookie_t *cookiep, uint_t *ccountp) { int ret; immu_hdl_priv_t *ihp; ret = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle, dma_req, cookiep, ccountp); if (ret == DDI_DMA_MAPPED) { ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle); immu_flush_wait(IMMU_DEVI(rdip)->imd_immu, &ihp->ihp_inv_wait); } return (ret); } /*ARGSUSED*/ static int immu_unbindhdl(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle) { return (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle)); } /*ARGSUSED*/ static int immu_sync(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off, size_t len, uint_t cachefl) { return (iommulib_iommu_dma_sync(dip, rdip, dma_handle, off, len, cachefl)); } /*ARGSUSED*/ static int immu_win(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp) { return (iommulib_iommu_dma_win(dip, rdip, dma_handle, win, offp, lenp, cookiep, ccountp)); } /*ARGSUSED*/ static int immu_mapobject(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao) { immu_hdl_priv_t *ihp; ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle); return (immu_map_dvmaseg(rdip, dma_handle, ihp, dmareq, dmao)); } /*ARGSUSED*/ static int immu_unmapobject(iommulib_handle_t handle, dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao) { immu_hdl_priv_t *ihp; ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle); if (ihp->ihp_npremapped > 0) return (DDI_SUCCESS); return (immu_unmap_dvmaseg(rdip, dmao)); }