/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved. */ /* * PCI iommu initialization and configuration */ #include <sys/types.h> #include <sys/kmem.h> #include <sys/async.h> #include <sys/sysmacros.h> #include <sys/sunddi.h> #include <sys/ddi_impldefs.h> #include <sys/vmem.h> #include <sys/machsystm.h> /* lddphys() */ #include <sys/iommutsb.h> #include <sys/pci/pci_obj.h> /*LINTLIBRARY*/ static void iommu_tlb_flushall(iommu_t *iommu_p); static void iommu_preserve_tsb(iommu_t *iommu_p); void iommu_create(pci_t *pci_p) { dev_info_t *dip = pci_p->pci_dip; iommu_t *iommu_p; uintptr_t a; size_t cache_size; uint32_t tsb_entries; char map_name[32]; extern uint64_t va_to_pa(void *); pci_dvma_range_prop_t pci_dvma_range; /* * Allocate iommu state structure and link it to the * pci state structure. */ iommu_p = (iommu_t *)kmem_zalloc(sizeof (iommu_t), KM_SLEEP); pci_p->pci_iommu_p = iommu_p; iommu_p->iommu_pci_p = pci_p; iommu_p->iommu_inst = ddi_get_instance(dip); /* * chip specific dvma_end, tsb_size & context support */ iommu_p->iommu_dvma_end = pci_iommu_dvma_end; a = pci_iommu_setup(iommu_p); /* * Determine the virtual address of iommu registers. */ iommu_p->iommu_ctrl_reg = (uint64_t *)(a + COMMON_IOMMU_CTRL_REG_OFFSET); iommu_p->iommu_tsb_base_addr_reg = (uint64_t *)(a + COMMON_IOMMU_TSB_BASE_ADDR_REG_OFFSET); iommu_p->iommu_flush_page_reg = (uint64_t *)(a + COMMON_IOMMU_FLUSH_PAGE_REG_OFFSET); /* * Configure the rest of the iommu parameters according to: * tsb_size and dvma_end */ iommu_p->iommu_tsb_vaddr = /* retrieve TSB VA reserved by system */ iommu_tsb_cookie_to_va(pci_p->pci_tsb_cookie); iommu_p->iommu_tsb_entries = tsb_entries = IOMMU_TSBSIZE_TO_TSBENTRIES(iommu_p->iommu_tsb_size); iommu_p->iommu_tsb_paddr = va_to_pa((caddr_t)iommu_p->iommu_tsb_vaddr); iommu_p->iommu_dvma_cache_locks = kmem_zalloc(pci_dvma_page_cache_entries, KM_SLEEP); iommu_p->iommu_dvma_base = iommu_p->iommu_dvma_end + 1 - (tsb_entries * IOMMU_PAGE_SIZE); iommu_p->dvma_base_pg = IOMMU_BTOP(iommu_p->iommu_dvma_base); iommu_p->iommu_dvma_reserve = tsb_entries >> 1; iommu_p->dvma_end_pg = IOMMU_BTOP(iommu_p->iommu_dvma_end); iommu_p->iommu_dma_bypass_base = COMMON_IOMMU_BYPASS_BASE; iommu_p->iommu_dma_bypass_end = pci_iommu_bypass_end_configure(); /* * export "virtual-dma" software property to support * child devices needing to know DVMA range */ pci_dvma_range.dvma_base = (uint32_t)iommu_p->iommu_dvma_base; pci_dvma_range.dvma_len = (uint32_t) iommu_p->iommu_dvma_end - iommu_p->iommu_dvma_base + 1; (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP, "virtual-dma", (caddr_t)&pci_dvma_range, sizeof (pci_dvma_range)); DEBUG2(DBG_ATTACH, dip, "iommu_create: ctrl=%p, tsb=%p\n", iommu_p->iommu_ctrl_reg, iommu_p->iommu_tsb_base_addr_reg); DEBUG2(DBG_ATTACH, dip, "iommu_create: page_flush=%p, ctx_flush=%p\n", iommu_p->iommu_flush_page_reg, iommu_p->iommu_flush_ctx_reg); DEBUG2(DBG_ATTACH, dip, "iommu_create: tsb vaddr=%p tsb_paddr=%p\n", iommu_p->iommu_tsb_vaddr, iommu_p->iommu_tsb_paddr); DEBUG1(DBG_ATTACH, dip, "iommu_create: allocated size=%x\n", iommu_tsb_cookie_to_size(pci_p->pci_tsb_cookie)); DEBUG2(DBG_ATTACH, dip, "iommu_create: fast tsb tte addr: %x + %x\n", iommu_p->iommu_tsb_vaddr, pci_dvma_page_cache_entries * pci_dvma_page_cache_clustsz); DEBUG3(DBG_ATTACH, dip, "iommu_create: tsb size=%x, tsb entries=%x, dvma base=%x\n", iommu_p->iommu_tsb_size, iommu_p->iommu_tsb_entries, iommu_p->iommu_dvma_base); DEBUG2(DBG_ATTACH, dip, "iommu_create: dvma_cache_locks=%x cache_entries=%x\n", iommu_p->iommu_dvma_cache_locks, pci_dvma_page_cache_entries); /* * zero out the area to be used for iommu tsb */ bzero(iommu_p->iommu_tsb_vaddr, tsb_entries << 3); /* * Create a virtual memory map for dvma address space. * Reserve 'size' bytes of low dvma space for fast track cache. */ (void) snprintf(map_name, sizeof (map_name), "%s%d_dvma", ddi_driver_name(dip), ddi_get_instance(dip)); cache_size = IOMMU_PTOB(pci_dvma_page_cache_entries * pci_dvma_page_cache_clustsz); iommu_p->iommu_dvma_fast_end = iommu_p->iommu_dvma_base + cache_size - 1; iommu_p->iommu_dvma_map = vmem_create(map_name, (void *)(iommu_p->iommu_dvma_fast_end + 1), IOMMU_PTOB(tsb_entries) - cache_size, IOMMU_PAGE_SIZE, NULL, NULL, NULL, IOMMU_PAGE_SIZE, VM_SLEEP); mutex_init(&iommu_p->dvma_debug_lock, NULL, MUTEX_DRIVER, NULL); /* * On detach, the TSB Base Address Register gets set to zero, * so if its zero here, there is no need to preserve TTEs. */ if (pci_preserve_iommu_tsb && *iommu_p->iommu_tsb_base_addr_reg) iommu_preserve_tsb(iommu_p); iommu_configure(iommu_p); } void iommu_destroy(pci_t *pci_p) { #ifdef DEBUG dev_info_t *dip = pci_p->pci_dip; #endif iommu_t *iommu_p = pci_p->pci_iommu_p; volatile uint64_t ctl_val = *iommu_p->iommu_ctrl_reg; DEBUG0(DBG_DETACH, dip, "iommu_destroy:\n"); /* * Disable the IOMMU by setting the TSB Base Address to zero * and the TSB Table size to the smallest possible. */ ctl_val = ctl_val & ~(7 << COMMON_IOMMU_CTRL_TSB_SZ_SHIFT); *iommu_p->iommu_ctrl_reg = ctl_val; *iommu_p->iommu_tsb_base_addr_reg = 0; /* * Return the boot time allocated tsb. */ iommu_tsb_free(pci_p->pci_tsb_cookie); /* * Teardown any implementation-specific structures set up in * pci_iommu_setup. */ pci_iommu_teardown(iommu_p); if (DVMA_DBG_ON(iommu_p)) pci_dvma_debug_fini(iommu_p); mutex_destroy(&iommu_p->dvma_debug_lock); /* * Free the dvma resource map. */ vmem_destroy(iommu_p->iommu_dvma_map); kmem_free(iommu_p->iommu_dvma_cache_locks, pci_dvma_page_cache_entries); /* * Free the iommu state structure. */ kmem_free(iommu_p, sizeof (iommu_t)); pci_p->pci_iommu_p = NULL; } /* * re-program iommu on the fly while preserving on-going dma * transactions on the PCI bus. */ void iommu_configure(iommu_t *iommu_p) { pci_t *pci_p = iommu_p->iommu_pci_p; uint64_t cfgpa = pci_get_cfg_pabase(pci_p); dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip; dev_info_t *cdip = NULL; volatile uint64_t ctl_val = (uint64_t) ((iommu_p->iommu_tsb_size << COMMON_IOMMU_CTRL_TSB_SZ_SHIFT) | (0 /* 8k page */ << COMMON_IOMMU_CTRL_TBW_SZ_SHIFT) | COMMON_IOMMU_CTRL_ENABLE | COMMON_IOMMU_CTRL_DIAG_ENABLE | (pci_lock_tlb ? COMMON_IOMMU_CTRL_LCK_ENABLE : 0)); DEBUG2(DBG_ATTACH, dip, "iommu_configure: iommu_ctl=%08x.%08x\n", HI32(ctl_val), LO32(ctl_val)); if (!pci_preserve_iommu_tsb || !(*iommu_p->iommu_tsb_base_addr_reg)) { *iommu_p->iommu_ctrl_reg = COMMON_IOMMU_CTRL_DIAG_ENABLE; iommu_tlb_flushall(iommu_p); goto config; } cdip = ddi_get_child(dip); for (; cdip; cdip = ddi_get_next_sibling(cdip)) { uint32_t *reg_p; int reg_len; if (ddi_getlongprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS, "reg", (caddr_t)®_p, ®_len) != DDI_PROP_SUCCESS) continue; cfgpa += (*reg_p) & (PCI_CONF_ADDR_MASK ^ PCI_REG_REG_M); kmem_free(reg_p, reg_len); break; } config: pci_iommu_config(iommu_p, ctl_val, cdip ? cfgpa : 0); } void iommu_map_pages(iommu_t *iommu_p, ddi_dma_impl_t *mp, dvma_addr_t dvma_pg, size_t npages, size_t pfn_index) { int i; dvma_addr_t pg_index = dvma_pg - iommu_p->dvma_base_pg; uint64_t *tte_addr = iommu_p->iommu_tsb_vaddr + pg_index; size_t pfn_last = pfn_index + npages; uint64_t tte = PCI_GET_MP_TTE(mp->dmai_tte); #ifdef DEBUG dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip; #endif ASSERT(pfn_last <= mp->dmai_ndvmapages); DEBUG5(DBG_MAP_WIN, dip, "iommu_map_pages:%x+%x=%x npages=0x%x pfn_index=0x%x\n", (uint_t)iommu_p->dvma_base_pg, (uint_t)pg_index, dvma_pg, (uint_t)npages, (uint_t)pfn_index); for (i = pfn_index; i < pfn_last; i++, pg_index++, tte_addr++) { iopfn_t pfn = PCI_GET_MP_PFN(mp, i); volatile uint64_t cur_tte = IOMMU_PTOB(pfn) | tte; DEBUG3(DBG_MAP_WIN, dip, "iommu_map_pages: mp=%p pg[%x]=%x\n", mp, i, (uint_t)pfn); DEBUG3(DBG_MAP_WIN, dip, "iommu_map_pages: pg_index=%x tte=%08x.%08x\n", pg_index, HI32(cur_tte), LO32(cur_tte)); ASSERT(TTE_IS_INVALID(*tte_addr)); *tte_addr = cur_tte; #ifdef DEBUG if (pfn == 0 && pci_warn_pp0) cmn_err(CE_WARN, "%s%d <%p> doing DMA to pp0\n", ddi_driver_name(mp->dmai_rdip), ddi_get_instance(mp->dmai_rdip), mp); #endif } ASSERT(tte_addr == iommu_p->iommu_tsb_vaddr + pg_index); #ifdef DEBUG if (HAS_REDZONE(mp)) { DEBUG1(DBG_MAP_WIN, dip, "iommu_map_pages: redzone pg=%x\n", pg_index); ASSERT(TTE_IS_INVALID(iommu_p->iommu_tsb_vaddr[pg_index])); } #endif if (DVMA_DBG_ON(iommu_p)) pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping, mp->dmai_size, mp); } /* * iommu_map_window - map a dvma window into the iommu * * used by: pci_dma_win(), pci_dma_ctlops() - DDI_DMA_MOVWIN * * return value: none */ /*ARGSUSED*/ void iommu_map_window(iommu_t *iommu_p, ddi_dma_impl_t *mp, window_t win_no) { uint32_t obj_pg0_off = mp->dmai_roffset; uint32_t win_pg0_off = win_no ? 0 : obj_pg0_off; size_t win_size = mp->dmai_winsize; size_t pfn_index = win_size * win_no; /* temp value */ size_t obj_off = win_no ? pfn_index - obj_pg0_off : 0; /* xferred sz */ dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping); size_t res_size = mp->dmai_object.dmao_size - obj_off + win_pg0_off; ASSERT(!(win_size & IOMMU_PAGE_OFFSET)); if (win_no >= mp->dmai_nwin) return; if (res_size < win_size) /* last window */ win_size = res_size; /* mp->dmai_winsize unchanged */ mp->dmai_mapping = IOMMU_PTOB(dvma_pg) | win_pg0_off; mp->dmai_size = win_size - win_pg0_off; /* cur win xferrable size */ mp->dmai_offset = obj_off; /* win offset into object */ pfn_index = IOMMU_BTOP(pfn_index); /* index into pfnlist */ iommu_map_pages(iommu_p, mp, dvma_pg, IOMMU_BTOPR(win_size), pfn_index); } void iommu_unmap_pages(iommu_t *iommu_p, dvma_addr_t dvma_pg, uint_t npages) { dvma_addr_t pg_index = IOMMU_PAGE_INDEX(iommu_p, dvma_pg); for (; npages; npages--, dvma_pg++, pg_index++) { DEBUG1(DBG_UNMAP_WIN|DBG_CONT, 0, " %x", dvma_pg); IOMMU_UNLOAD_TTE(iommu_p, pg_index); if (!tm_mtlb_gc) IOMMU_PAGE_FLUSH(iommu_p, dvma_pg); } } void iommu_remap_pages(iommu_t *iommu_p, ddi_dma_impl_t *mp, dvma_addr_t dvma_pg, size_t npages, size_t pfn_index) { iommu_unmap_pages(iommu_p, dvma_pg, npages); iommu_map_pages(iommu_p, mp, dvma_pg, npages, pfn_index); } /* * iommu_unmap_window * * This routine is called to break down the iommu mappings to a dvma window. * Non partial mappings are viewed as single window mapping. * * used by: pci_dma_unbindhdl(), pci_dma_window(), * and pci_dma_ctlops() - DDI_DMA_FREE, DDI_DMA_MOVWIN, DDI_DMA_NEXTWIN * * return value: none */ /*ARGSUSED*/ void iommu_unmap_window(iommu_t *iommu_p, ddi_dma_impl_t *mp) { dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping); dvma_addr_t pg_index = IOMMU_PAGE_INDEX(iommu_p, dvma_pg); uint_t npages = IOMMU_BTOP(mp->dmai_winsize); #ifdef DEBUG dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip; #endif /* * Invalidate each page of the mapping in the tsb and flush * it from the tlb. */ DEBUG2(DBG_UNMAP_WIN, dip, "mp=%p %x pfns:", mp, npages); if (mp->dmai_flags & DMAI_FLAGS_CONTEXT) { dvma_context_t ctx = MP2CTX(mp); for (; npages; npages--, pg_index++) { DEBUG1(DBG_UNMAP_WIN|DBG_CONT, dip, " %x", pg_index); IOMMU_UNLOAD_TTE(iommu_p, pg_index); } DEBUG1(DBG_UNMAP_WIN|DBG_CONT, dip, " (context %x)", ctx); *iommu_p->iommu_flush_ctx_reg = ctx; } else iommu_unmap_pages(iommu_p, dvma_pg, npages); DEBUG0(DBG_UNMAP_WIN|DBG_CONT, dip, "\n"); if (DVMA_DBG_ON(iommu_p)) pci_dvma_free_debug(iommu_p, (char *)mp->dmai_mapping, mp->dmai_size, mp); } int pci_alloc_tsb(pci_t *pci_p) { uint16_t tsbc; if ((tsbc = iommu_tsb_alloc(pci_p->pci_id)) == IOMMU_TSB_COOKIE_NONE) { cmn_err(CE_WARN, "%s%d: Unable to allocate IOMMU TSB.", ddi_driver_name(pci_p->pci_dip), ddi_get_instance(pci_p->pci_dip)); return (DDI_FAILURE); } pci_p->pci_tsb_cookie = tsbc; return (DDI_SUCCESS); } void pci_free_tsb(pci_t *pci_p) { iommu_tsb_free(pci_p->pci_tsb_cookie); } #if 0 /* * The following data structure is used to map a tsb size * to a tsb size configuration parameter in the iommu * control register. * This is a hardware table. It is here for reference only. */ static int pci_iommu_tsb_sizes[] = { 0x2000, /* 0 - 8 mb */ 0x4000, /* 1 - 16 mb */ 0x8000, /* 2 - 32 mb */ 0x10000, /* 3 - 64 mb */ 0x20000, /* 4 - 128 mb */ 0x40000, /* 5 - 256 mb */ 0x80000, /* 6 - 512 mb */ 0x100000 /* 7 - 1 gb */ }; #endif uint_t iommu_tsb_size_encode(uint_t tsb_bytes) { uint_t i; for (i = 7; i && (tsb_bytes < (0x2000 << i)); i--) /* empty */; return (i); } /* * invalidate IOMMU TLB entries through diagnostic registers. */ static void iommu_tlb_flushall(iommu_t *iommu_p) { int i; uint64_t base = (uint64_t)(iommu_p->iommu_ctrl_reg) - COMMON_IOMMU_CTRL_REG_OFFSET; volatile uint64_t *tlb_tag = (volatile uint64_t *) (base + COMMON_IOMMU_TLB_TAG_DIAG_ACC_OFFSET); volatile uint64_t *tlb_data = (volatile uint64_t *) (base + COMMON_IOMMU_TLB_DATA_DIAG_ACC_OFFSET); for (i = 0; i < IOMMU_TLB_ENTRIES; i++) tlb_tag[i] = tlb_data[i] = 0ull; } static void iommu_preserve_tsb(iommu_t *iommu_p) { #ifdef DEBUG dev_info_t *dip = iommu_p->iommu_pci_p->pci_dip; #endif uint_t i, obp_tsb_entries, obp_tsb_size, base_pg_index; uint64_t ctl = *iommu_p->iommu_ctrl_reg; uint64_t obp_tsb_pa = *iommu_p->iommu_tsb_base_addr_reg; uint64_t *base_tte_addr; DEBUG3(DBG_ATTACH, dip, "iommu_tsb_base_addr_reg=0x%08x (0x%08x.0x%08x)\n", iommu_p->iommu_tsb_base_addr_reg, (uint32_t)(*iommu_p->iommu_tsb_base_addr_reg >> 32), (uint32_t)(*iommu_p->iommu_tsb_base_addr_reg & 0xffffffff)); obp_tsb_size = IOMMU_CTL_TO_TSBSIZE(ctl); obp_tsb_entries = IOMMU_TSBSIZE_TO_TSBENTRIES(obp_tsb_size); base_pg_index = iommu_p->dvma_end_pg - obp_tsb_entries + 1; base_tte_addr = iommu_p->iommu_tsb_vaddr + (iommu_p->iommu_tsb_entries - obp_tsb_entries); /* * old darwin prom does not set tsb size correctly, bail out. */ if ((obp_tsb_size == IOMMU_DARWIN_BOGUS_TSBSIZE) && (CHIP_TYPE(iommu_p->iommu_pci_p) == PCI_CHIP_SABRE)) return; DEBUG3(DBG_ATTACH, dip, "iommu_preserve_tsb: kernel info\n" "iommu_tsb_vaddr=%08x copy to base_tte_addr=%08x " "base_pg_index=%x\n", iommu_p->iommu_tsb_vaddr, base_tte_addr, base_pg_index); DEBUG3(DBG_ATTACH | DBG_CONT, dip, "iommu_preserve_tsb: obp info " "obp_tsb_entries=0x%x obp_tsb_pa=%08x.%08x\n", obp_tsb_entries, (uint32_t)(obp_tsb_pa >> 32), (uint32_t)obp_tsb_pa); for (i = 0; i < obp_tsb_entries; i++) { uint64_t tte = lddphys(obp_tsb_pa + i * 8); caddr_t va; if (TTE_IS_INVALID(tte)) { DEBUG0(DBG_ATTACH | DBG_CONT, dip, "."); continue; } base_tte_addr[i] = tte; DEBUG3(DBG_ATTACH | DBG_CONT, dip, "\npreserve_tsb: (%x)=%08x.%08x\n", base_tte_addr + i, (uint_t)(tte >> 32), (uint_t)(tte & 0xffffffff)); /* * permanantly reserve this page from dvma address space * resource map */ va = (caddr_t)(IOMMU_PTOB(base_pg_index + i)); (void) vmem_xalloc(iommu_p->iommu_dvma_map, IOMMU_PAGE_SIZE, IOMMU_PAGE_SIZE, 0, 0, va, va + IOMMU_PAGE_SIZE, VM_NOSLEEP | VM_BESTFIT | VM_PANIC); } }