/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Useful debugging Stuff */ #include #include /* Bitfield debugging definitions for this file */ #define IOMMU_GETDVMAPAGES_DEBUG 0x1 #define IOMMU_DMAMAP_DEBUG 0x2 #define IOMMU_DMAMCTL_DEBUG 0x4 #define IOMMU_DMAMCTL_SYNC_DEBUG 0x8 #define IOMMU_DMAMCTL_HTOC_DEBUG 0x10 #define IOMMU_DMAMCTL_KVADDR_DEBUG 0x20 #define IOMMU_DMAMCTL_NEXTWIN_DEBUG 0x40 #define IOMMU_DMAMCTL_NEXTSEG_DEBUG 0x80 #define IOMMU_DMAMCTL_MOVWIN_DEBUG 0x100 #define IOMMU_DMAMCTL_REPWIN_DEBUG 0x200 #define IOMMU_DMAMCTL_GETERR_DEBUG 0x400 #define IOMMU_DMAMCTL_COFF_DEBUG 0x800 #define IOMMU_DMAMCTL_DMA_FREE_DEBUG 0x1000 #define IOMMU_REGISTERS_DEBUG 0x2000 #define IOMMU_DMA_SETUP_DEBUG 0x4000 #define IOMMU_DMA_UNBINDHDL_DEBUG 0x8000 #define IOMMU_DMA_BINDHDL_DEBUG 0x10000 #define IOMMU_DMA_WIN_DEBUG 0x20000 #define IOMMU_DMA_ALLOCHDL_DEBUG 0x40000 #define IOMMU_DMA_LIM_SETUP_DEBUG 0x80000 #define IOMMU_FASTDMA_RESERVE 0x100000 #define IOMMU_FASTDMA_LOAD 0x200000 #define IOMMU_INTER_INTRA_XFER 0x400000 #define IOMMU_TTE 0x800000 #define IOMMU_TLB 0x1000000 #define IOMMU_FASTDMA_SYNC 0x2000000 /* Turn on if you need to keep track of outstanding IOMMU usage */ /* #define IO_MEMUSAGE */ /* Turn on to debug IOMMU unmapping code */ /* #define IO_MEMDEBUG */ static struct dvma_ops iommu_dvma_ops = { DVMAO_REV, iommu_dvma_kaddr_load, iommu_dvma_unload, iommu_dvma_sync }; extern void *sbusp; /* sbus soft state hook */ #define DVMA_MAX_CACHE 65536 /* * This is the number of pages that a mapping request needs before we force * the TLB flush code to use diagnostic registers. This value was determined * through a series of test runs measuring dma mapping settup performance. */ int tlb_flush_using_diag = 16; int sysio_iommu_tsb_sizes[] = { IOMMU_TSB_SIZE_8M, IOMMU_TSB_SIZE_16M, IOMMU_TSB_SIZE_32M, IOMMU_TSB_SIZE_64M, IOMMU_TSB_SIZE_128M, IOMMU_TSB_SIZE_256M, IOMMU_TSB_SIZE_512M, IOMMU_TSB_SIZE_1G }; static int iommu_map_window(ddi_dma_impl_t *, off_t, size_t); int iommu_init(struct sbus_soft_state *softsp, caddr_t address) { int i; char name[40]; #ifdef DEBUG debug_info = 1; #endif /* * Simply add each registers offset to the base address * to calculate the already mapped virtual address of * the device register... * * define a macro for the pointer arithmetic; all registers * are 64 bits wide and are defined as uint64_t's. */ #define REG_ADDR(b, o) (uint64_t *)((caddr_t)(b) + (o)) softsp->iommu_ctrl_reg = REG_ADDR(address, OFF_IOMMU_CTRL_REG); softsp->tsb_base_addr = REG_ADDR(address, OFF_TSB_BASE_ADDR); softsp->iommu_flush_reg = REG_ADDR(address, OFF_IOMMU_FLUSH_REG); softsp->iommu_tlb_tag = REG_ADDR(address, OFF_IOMMU_TLB_TAG); softsp->iommu_tlb_data = REG_ADDR(address, OFF_IOMMU_TLB_DATA); #undef REG_ADDR mutex_init(&softsp->dma_pool_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&softsp->intr_poll_list_lock, NULL, MUTEX_DEFAULT, NULL); /* Set up the DVMA resource sizes */ if ((softsp->iommu_tsb_cookie = iommu_tsb_alloc(softsp->upa_id)) == IOMMU_TSB_COOKIE_NONE) { cmn_err(CE_WARN, "%s%d: Unable to retrieve IOMMU array.", ddi_driver_name(softsp->dip), ddi_get_instance(softsp->dip)); return (DDI_FAILURE); } softsp->soft_tsb_base_addr = iommu_tsb_cookie_to_va(softsp->iommu_tsb_cookie); softsp->iommu_dvma_size = iommu_tsb_cookie_to_size(softsp->iommu_tsb_cookie) << IOMMU_TSB_TO_RNG; softsp->iommu_dvma_base = (ioaddr_t) (0 - (ioaddr_t)softsp->iommu_dvma_size); (void) snprintf(name, sizeof (name), "%s%d_dvma", ddi_driver_name(softsp->dip), ddi_get_instance(softsp->dip)); /* * Initialize the DVMA vmem arena. */ softsp->dvma_arena = vmem_create(name, (void *)softsp->iommu_dvma_base, softsp->iommu_dvma_size, PAGESIZE, NULL, NULL, NULL, DVMA_MAX_CACHE, VM_SLEEP); /* Set the limit for dvma_reserve() to 1/2 of the total dvma space */ softsp->dma_reserve = iommu_btop(softsp->iommu_dvma_size >> 1); #if defined(DEBUG) && defined(IO_MEMUSAGE) mutex_init(&softsp->iomemlock, NULL, MUTEX_DEFAULT, NULL); softsp->iomem = (struct io_mem_list *)0; #endif /* DEBUG && IO_MEMUSAGE */ /* * Get the base address of the TSB table and store it in the hardware */ /* * We plan on the PROM flushing all TLB entries. If this is not the * case, this is where we should flush the hardware TLB. */ /* Set the IOMMU registers */ (void) iommu_resume_init(softsp); /* check the convenient copy of TSB base, and flush write buffers */ if (*softsp->tsb_base_addr != va_to_pa((caddr_t)softsp->soft_tsb_base_addr)) { iommu_tsb_free(softsp->iommu_tsb_cookie); return (DDI_FAILURE); } softsp->sbus_io_lo_pfn = UINT32_MAX; softsp->sbus_io_hi_pfn = 0; for (i = 0; i < sysio_pd_getnrng(softsp->dip); i++) { struct rangespec *rangep; uint64_t addr; pfn_t hipfn, lopfn; rangep = sysio_pd_getrng(softsp->dip, i); addr = (uint64_t)((uint64_t)rangep->rng_bustype << 32); addr |= (uint64_t)rangep->rng_offset; lopfn = (pfn_t)(addr >> MMU_PAGESHIFT); addr += (uint64_t)(rangep->rng_size - 1); hipfn = (pfn_t)(addr >> MMU_PAGESHIFT); softsp->sbus_io_lo_pfn = (lopfn < softsp->sbus_io_lo_pfn) ? lopfn : softsp->sbus_io_lo_pfn; softsp->sbus_io_hi_pfn = (hipfn > softsp->sbus_io_hi_pfn) ? hipfn : softsp->sbus_io_hi_pfn; } DPRINTF(IOMMU_REGISTERS_DEBUG, ("IOMMU Control reg: %p IOMMU TSB " "base reg: %p IOMMU flush reg: %p TSB base addr %p\n", softsp->iommu_ctrl_reg, softsp->tsb_base_addr, softsp->iommu_flush_reg, softsp->soft_tsb_base_addr)); return (DDI_SUCCESS); } /* * function to uninitialize the iommu and release the tsb back to * the spare pool. See startup.c for tsb spare management. */ int iommu_uninit(struct sbus_soft_state *softsp) { vmem_destroy(softsp->dvma_arena); /* flip off the IOMMU enable switch */ *softsp->iommu_ctrl_reg &= (TSB_SIZE << TSB_SIZE_SHIFT | IOMMU_DISABLE); iommu_tsb_free(softsp->iommu_tsb_cookie); return (DDI_SUCCESS); } /* * Initialize iommu hardware registers when the system is being resumed. * (Subset of iommu_init()) */ int iommu_resume_init(struct sbus_soft_state *softsp) { int i; uint_t tsb_size; uint_t tsb_bytes; /* * Reset the base address of the TSB table in the hardware */ *softsp->tsb_base_addr = va_to_pa((caddr_t)softsp->soft_tsb_base_addr); /* * Figure out the correct size of the IOMMU TSB entries. If we * end up with a size smaller than that needed for 8M of IOMMU * space, default the size to 8M. XXX We could probably panic here */ i = sizeof (sysio_iommu_tsb_sizes) / sizeof (sysio_iommu_tsb_sizes[0]) - 1; tsb_bytes = iommu_tsb_cookie_to_size(softsp->iommu_tsb_cookie); while (i > 0) { if (tsb_bytes >= sysio_iommu_tsb_sizes[i]) break; i--; } tsb_size = i; /* OK, lets flip the "on" switch of the IOMMU */ *softsp->iommu_ctrl_reg = (uint64_t)(tsb_size << TSB_SIZE_SHIFT | IOMMU_ENABLE | IOMMU_DIAG_ENABLE); return (DDI_SUCCESS); } void iommu_tlb_flush(struct sbus_soft_state *softsp, ioaddr_t addr, pgcnt_t npages) { volatile uint64_t tmpreg; volatile uint64_t *vaddr_reg, *valid_bit_reg; ioaddr_t hiaddr, ioaddr; int i, do_flush = 0; if (npages == 1) { *softsp->iommu_flush_reg = (uint64_t)addr; tmpreg = *softsp->sbus_ctrl_reg; return; } hiaddr = addr + (ioaddr_t)(npages * IOMMU_PAGESIZE); for (i = 0, vaddr_reg = softsp->iommu_tlb_tag, valid_bit_reg = softsp->iommu_tlb_data; i < IOMMU_TLB_ENTRIES; i++, vaddr_reg++, valid_bit_reg++) { tmpreg = *vaddr_reg; ioaddr = (ioaddr_t)((tmpreg & IOMMU_TLBTAG_VA_MASK) << IOMMU_TLBTAG_VA_SHIFT); DPRINTF(IOMMU_TLB, ("Vaddr reg 0x%x, " "TLB vaddr reg %llx, IO addr 0x%x " "Base addr 0x%x, Hi addr 0x%x\n", vaddr_reg, tmpreg, ioaddr, addr, hiaddr)); if (ioaddr >= addr && ioaddr <= hiaddr) { tmpreg = *valid_bit_reg; DPRINTF(IOMMU_TLB, ("Valid reg addr 0x%x, " "TLB valid reg %llx\n", valid_bit_reg, tmpreg)); if (tmpreg & IOMMU_TLB_VALID) { *softsp->iommu_flush_reg = (uint64_t)ioaddr; do_flush = 1; } } } if (do_flush) tmpreg = *softsp->sbus_ctrl_reg; } /* * Shorthand defines */ #define ALO dma_lim->dlim_addr_lo #define AHI dma_lim->dlim_addr_hi #define OBJSIZE dmareq->dmar_object.dmao_size #define IOTTE_NDX(vaddr, base) (base + \ (int)(iommu_btop((vaddr & ~IOMMU_PAGEMASK) - \ softsp->iommu_dvma_base))) /* * If DDI_DMA_PARTIAL flag is set and the request is for * less than MIN_DVMA_WIN_SIZE, it's not worth the hassle so * we turn off the DDI_DMA_PARTIAL flag */ #define MIN_DVMA_WIN_SIZE (128) /* ARGSUSED */ void iommu_remove_mappings(ddi_dma_impl_t *mp) { #if defined(DEBUG) && defined(IO_MEMDEBUG) pgcnt_t npages; ioaddr_t ioaddr; volatile uint64_t *iotte_ptr; ioaddr_t ioaddr = mp->dmai_mapping & ~IOMMU_PAGEOFFSET; pgcnt_t npages = mp->dmai_ndvmapages; struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp; struct sbus_soft_state *softsp = mppriv->softsp; #if defined(IO_MEMUSAGE) struct io_mem_list **prevp, *walk; #endif /* DEBUG && IO_MEMUSAGE */ ASSERT(softsp != NULL); /* * Run thru the mapped entries and free 'em */ ioaddr = mp->dmai_mapping & ~IOMMU_PAGEOFFSET; npages = mp->dmai_ndvmapages; #if defined(IO_MEMUSAGE) mutex_enter(&softsp->iomemlock); prevp = &softsp->iomem; walk = softsp->iomem; while (walk) { if (walk->ioaddr == ioaddr) { *prevp = walk->next; break; } prevp = &walk->next; walk = walk->next; } mutex_exit(&softsp->iomemlock); kmem_free(walk->pfn, sizeof (pfn_t) * (npages + 1)); kmem_free(walk, sizeof (struct io_mem_list)); #endif /* IO_MEMUSAGE */ iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr); while (npages) { DPRINTF(IOMMU_DMAMCTL_DEBUG, ("dma_mctl: freeing ioaddr %x iotte %p\n", ioaddr, iotte_ptr)); *iotte_ptr = (uint64_t)0; /* unload tte */ iommu_tlb_flush(softsp, ioaddr, 1); npages--; ioaddr += IOMMU_PAGESIZE; iotte_ptr++; } #endif /* DEBUG && IO_MEMDEBUG */ } int iommu_create_vaddr_mappings(ddi_dma_impl_t *mp, uintptr_t addr) { pfn_t pfn; struct as *as = NULL; pgcnt_t npages; ioaddr_t ioaddr; uint_t offset; volatile uint64_t *iotte_ptr; uint64_t tmp_iotte_flag; int rval = DDI_DMA_MAPPED; struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp; struct sbus_soft_state *softsp = mppriv->softsp; int diag_tlb_flush; #if defined(DEBUG) && defined(IO_MEMUSAGE) struct io_mem_list *iomemp; pfn_t *pfnp; #endif /* DEBUG && IO_MEMUSAGE */ ASSERT(softsp != NULL); /* Set Valid and Cache for mem xfer */ tmp_iotte_flag = IOTTE_VALID | IOTTE_CACHE | IOTTE_WRITE | IOTTE_STREAM; offset = (uint_t)(mp->dmai_mapping & IOMMU_PAGEOFFSET); npages = iommu_btopr(mp->dmai_size + offset); ioaddr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET); iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr); diag_tlb_flush = npages > tlb_flush_using_diag ? 1 : 0; as = mp->dmai_object.dmao_obj.virt_obj.v_as; if (as == NULL) as = &kas; /* * Set the per object bits of the TTE here. We optimize this for * the memory case so that the while loop overhead is minimal. */ /* Turn on NOSYNC if we need consistent mem */ if (mp->dmai_rflags & DDI_DMA_CONSISTENT) { mp->dmai_rflags |= DMP_NOSYNC; tmp_iotte_flag ^= IOTTE_STREAM; /* Set streaming mode if not consistent mem */ } else if (softsp->stream_buf_off) { tmp_iotte_flag ^= IOTTE_STREAM; } #if defined(DEBUG) && defined(IO_MEMUSAGE) iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP); iomemp->rdip = mp->dmai_rdip; iomemp->ioaddr = ioaddr; iomemp->addr = addr; iomemp->npages = npages; pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1), KM_SLEEP); #endif /* DEBUG && IO_MEMUSAGE */ /* * Grab the mappings from the dmmu and stick 'em into the * iommu. */ ASSERT(npages != 0); /* If we're going to flush the TLB using diag mode, do it now. */ if (diag_tlb_flush) iommu_tlb_flush(softsp, ioaddr, npages); do { uint64_t iotte_flag = tmp_iotte_flag; /* * Fetch the pfn for the DMA object */ ASSERT(as); pfn = hat_getpfnum(as->a_hat, (caddr_t)addr); ASSERT(pfn != PFN_INVALID); if (!pf_is_memory(pfn)) { /* DVMA'ing to IO space */ /* Turn off cache bit if set */ if (iotte_flag & IOTTE_CACHE) iotte_flag ^= IOTTE_CACHE; /* Turn off stream bit if set */ if (iotte_flag & IOTTE_STREAM) iotte_flag ^= IOTTE_STREAM; if (IS_INTRA_SBUS(softsp, pfn)) { /* Intra sbus transfer */ /* Turn on intra flag */ iotte_flag |= IOTTE_INTRA; DPRINTF(IOMMU_INTER_INTRA_XFER, ( "Intra xfer pfnum %x TTE %llx\n", pfn, iotte_flag)); } else { if (pf_is_dmacapable(pfn) == 1) { /*EMPTY*/ DPRINTF(IOMMU_INTER_INTRA_XFER, ("Inter xfer pfnum %lx " "tte hi %llx\n", pfn, iotte_flag)); } else { rval = DDI_DMA_NOMAPPING; #if defined(DEBUG) && defined(IO_MEMDEBUG) goto bad; #endif /* DEBUG && IO_MEMDEBUG */ } } } addr += IOMMU_PAGESIZE; DPRINTF(IOMMU_TTE, ("vaddr mapping: tte index %x pfn %lx " "tte flag %llx addr %p ioaddr %x\n", iotte_ptr, pfn, iotte_flag, addr, ioaddr)); /* Flush the IOMMU TLB before loading a new mapping */ if (!diag_tlb_flush) iommu_tlb_flush(softsp, ioaddr, 1); /* Set the hardware IO TTE */ *iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag; ioaddr += IOMMU_PAGESIZE; npages--; iotte_ptr++; #if defined(DEBUG) && defined(IO_MEMUSAGE) *pfnp = pfn; pfnp++; #endif /* DEBUG && IO_MEMUSAGE */ } while (npages != 0); #if defined(DEBUG) && defined(IO_MEMUSAGE) mutex_enter(&softsp->iomemlock); iomemp->next = softsp->iomem; softsp->iomem = iomemp; mutex_exit(&softsp->iomemlock); #endif /* DEBUG && IO_MEMUSAGE */ return (rval); #if defined(DEBUG) && defined(IO_MEMDEBUG) bad: /* If we fail a mapping, free up any mapping resources used */ iommu_remove_mappings(mp); return (rval); #endif /* DEBUG && IO_MEMDEBUG */ } int iommu_create_pp_mappings(ddi_dma_impl_t *mp, page_t *pp, page_t **pplist) { pfn_t pfn; pgcnt_t npages; ioaddr_t ioaddr; uint_t offset; volatile uint64_t *iotte_ptr; uint64_t tmp_iotte_flag; struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp; struct sbus_soft_state *softsp = mppriv->softsp; int diag_tlb_flush; #if defined(DEBUG) && defined(IO_MEMUSAGE) struct io_mem_list *iomemp; pfn_t *pfnp; #endif /* DEBUG && IO_MEMUSAGE */ int rval = DDI_DMA_MAPPED; /* Set Valid and Cache for mem xfer */ tmp_iotte_flag = IOTTE_VALID | IOTTE_CACHE | IOTTE_WRITE | IOTTE_STREAM; ASSERT(softsp != NULL); offset = (uint_t)(mp->dmai_mapping & IOMMU_PAGEOFFSET); npages = iommu_btopr(mp->dmai_size + offset); ioaddr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET); iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr); diag_tlb_flush = npages > tlb_flush_using_diag ? 1 : 0; /* * Set the per object bits of the TTE here. We optimize this for * the memory case so that the while loop overhead is minimal. */ if (mp->dmai_rflags & DDI_DMA_CONSISTENT) { /* Turn on NOSYNC if we need consistent mem */ mp->dmai_rflags |= DMP_NOSYNC; tmp_iotte_flag ^= IOTTE_STREAM; } else if (softsp->stream_buf_off) { /* Set streaming mode if not consistent mem */ tmp_iotte_flag ^= IOTTE_STREAM; } #if defined(DEBUG) && defined(IO_MEMUSAGE) iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP); iomemp->rdip = mp->dmai_rdip; iomemp->ioaddr = ioaddr; iomemp->npages = npages; pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1), KM_SLEEP); #endif /* DEBUG && IO_MEMUSAGE */ /* * Grab the mappings from the dmmu and stick 'em into the * iommu. */ ASSERT(npages != 0); /* If we're going to flush the TLB using diag mode, do it now. */ if (diag_tlb_flush) iommu_tlb_flush(softsp, ioaddr, npages); do { uint64_t iotte_flag; iotte_flag = tmp_iotte_flag; if (pp != NULL) { pfn = pp->p_pagenum; pp = pp->p_next; } else { pfn = (*pplist)->p_pagenum; pplist++; } DPRINTF(IOMMU_TTE, ("pp mapping TTE index %x pfn %lx " "tte flag %llx ioaddr %x\n", iotte_ptr, pfn, iotte_flag, ioaddr)); /* Flush the IOMMU TLB before loading a new mapping */ if (!diag_tlb_flush) iommu_tlb_flush(softsp, ioaddr, 1); /* Set the hardware IO TTE */ *iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag; ioaddr += IOMMU_PAGESIZE; npages--; iotte_ptr++; #if defined(DEBUG) && defined(IO_MEMUSAGE) *pfnp = pfn; pfnp++; #endif /* DEBUG && IO_MEMUSAGE */ } while (npages != 0); #if defined(DEBUG) && defined(IO_MEMUSAGE) mutex_enter(&softsp->iomemlock); iomemp->next = softsp->iomem; softsp->iomem = iomemp; mutex_exit(&softsp->iomemlock); #endif /* DEBUG && IO_MEMUSAGE */ return (rval); } int iommu_dma_lim_setup(dev_info_t *dip, dev_info_t *rdip, struct sbus_soft_state *softsp, uint_t *burstsizep, uint_t burstsize64, uint_t *minxferp, uint_t dma_flags) { struct regspec *rp; /* Take care of 64 bit limits. */ if (!(dma_flags & DDI_DMA_SBUS_64BIT)) { /* * return burst size for 32-bit mode */ *burstsizep &= softsp->sbus_burst_sizes; return (DDI_FAILURE); } /* * check if SBus supports 64 bit and if caller * is child of SBus. No support through bridges */ if (!softsp->sbus64_burst_sizes || (ddi_get_parent(rdip) != dip)) { /* * SBus doesn't support it or bridge. Do 32-bit * xfers */ *burstsizep &= softsp->sbus_burst_sizes; return (DDI_FAILURE); } rp = ddi_rnumber_to_regspec(rdip, 0); if (rp == NULL) { *burstsizep &= softsp->sbus_burst_sizes; return (DDI_FAILURE); } /* Check for old-style 64 bit burstsizes */ if (burstsize64 & SYSIO64_BURST_MASK) { /* Scale back burstsizes if Necessary */ *burstsizep &= (softsp->sbus64_burst_sizes | softsp->sbus_burst_sizes); } else { /* Get the 64 bit burstsizes. */ *burstsizep = burstsize64; /* Scale back burstsizes if Necessary */ *burstsizep &= (softsp->sbus64_burst_sizes >> SYSIO64_BURST_SHIFT); } /* * Set the largest value of the smallest burstsize that the * device or the bus can manage. */ *minxferp = MAX(*minxferp, (1 << (ddi_ffs(softsp->sbus64_burst_sizes) - 1))); return (DDI_SUCCESS); } int iommu_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *dma_attr, int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) { ioaddr_t addrlow, addrhigh, segalign; ddi_dma_impl_t *mp; struct dma_impl_priv *mppriv; struct sbus_soft_state *softsp = (struct sbus_soft_state *) ddi_get_soft_state(sbusp, ddi_get_instance(dip)); /* * Setup dma burstsizes and min-xfer counts. */ (void) iommu_dma_lim_setup(dip, rdip, softsp, &dma_attr->dma_attr_burstsizes, dma_attr->dma_attr_burstsizes, &dma_attr->dma_attr_minxfer, dma_attr->dma_attr_flags); if (dma_attr->dma_attr_burstsizes == 0) return (DDI_DMA_BADATTR); addrlow = (ioaddr_t)dma_attr->dma_attr_addr_lo; addrhigh = (ioaddr_t)dma_attr->dma_attr_addr_hi; segalign = (ioaddr_t)dma_attr->dma_attr_seg; /* * Check sanity for hi and lo address limits */ if ((addrhigh <= addrlow) || (addrhigh < (ioaddr_t)softsp->iommu_dvma_base)) { return (DDI_DMA_BADATTR); } if (dma_attr->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) return (DDI_DMA_BADATTR); mppriv = kmem_zalloc(sizeof (*mppriv), (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP); if (mppriv == NULL) { if (waitfp != DDI_DMA_DONTWAIT) { ddi_set_callback(waitfp, arg, &softsp->dvma_call_list_id); } return (DDI_DMA_NORESOURCES); } mp = (ddi_dma_impl_t *)mppriv; DPRINTF(IOMMU_DMA_ALLOCHDL_DEBUG, ("dma_allochdl: (%s) handle %x " "hi %x lo %x min %x burst %x\n", ddi_get_name(dip), mp, addrhigh, addrlow, dma_attr->dma_attr_minxfer, dma_attr->dma_attr_burstsizes)); mp->dmai_rdip = rdip; mp->dmai_minxfer = (uint_t)dma_attr->dma_attr_minxfer; mp->dmai_burstsizes = (uint_t)dma_attr->dma_attr_burstsizes; mp->dmai_attr = *dma_attr; /* See if the DMA engine has any limit restrictions. */ if (segalign == (ioaddr_t)UINT32_MAX && addrhigh == (ioaddr_t)UINT32_MAX && (dma_attr->dma_attr_align <= IOMMU_PAGESIZE) && addrlow == 0) { mp->dmai_rflags |= DMP_NOLIMIT; } mppriv->softsp = softsp; mppriv->phys_sync_flag = va_to_pa((caddr_t)&mppriv->sync_flag); *handlep = (ddi_dma_handle_t)mp; return (DDI_SUCCESS); } /*ARGSUSED*/ int iommu_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) { struct dma_impl_priv *mppriv = (struct dma_impl_priv *)handle; struct sbus_soft_state *softsp = mppriv->softsp; ASSERT(softsp != NULL); kmem_free(mppriv, sizeof (*mppriv)); if (softsp->dvma_call_list_id != 0) { ddi_run_callback(&softsp->dvma_call_list_id); } return (DDI_SUCCESS); } static int check_dma_attr(struct ddi_dma_req *dmareq, ddi_dma_attr_t *dma_attr, uint32_t *size) { ioaddr_t addrlow; ioaddr_t addrhigh; uint32_t segalign; uint32_t smask; smask = *size - 1; segalign = dma_attr->dma_attr_seg; if (smask > segalign) { if ((dmareq->dmar_flags & DDI_DMA_PARTIAL) == 0) return (DDI_DMA_TOOBIG); *size = segalign + 1; } addrlow = (ioaddr_t)dma_attr->dma_attr_addr_lo; addrhigh = (ioaddr_t)dma_attr->dma_attr_addr_hi; if (addrlow + smask > addrhigh || addrlow + smask < addrlow) { if (!((addrlow + dmareq->dmar_object.dmao_size == 0) && (addrhigh == (ioaddr_t)-1))) { if ((dmareq->dmar_flags & DDI_DMA_PARTIAL) == 0) return (DDI_DMA_TOOBIG); *size = MIN(addrhigh - addrlow + 1, *size); } } return (DDI_DMA_MAPOK); } int iommu_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cp, uint_t *ccountp) { page_t *pp; uint32_t size; ioaddr_t ioaddr; uint_t offset; uintptr_t addr = 0; pgcnt_t npages; int rval; ddi_dma_attr_t *dma_attr; struct sbus_soft_state *softsp; struct page **pplist = NULL; ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp; #ifdef lint dip = dip; rdip = rdip; #endif if (mp->dmai_inuse) return (DDI_DMA_INUSE); dma_attr = &mp->dmai_attr; size = (uint32_t)dmareq->dmar_object.dmao_size; if (!(mp->dmai_rflags & DMP_NOLIMIT)) { rval = check_dma_attr(dmareq, dma_attr, &size); if (rval != DDI_DMA_MAPOK) return (rval); } mp->dmai_inuse = 1; mp->dmai_offset = 0; mp->dmai_rflags = (dmareq->dmar_flags & DMP_DDIFLAGS) | (mp->dmai_rflags & DMP_NOLIMIT); switch (dmareq->dmar_object.dmao_type) { case DMA_OTYP_VADDR: case DMA_OTYP_BUFVADDR: addr = (uintptr_t)dmareq->dmar_object.dmao_obj.virt_obj.v_addr; offset = addr & IOMMU_PAGEOFFSET; pplist = dmareq->dmar_object.dmao_obj.virt_obj.v_priv; npages = iommu_btopr(OBJSIZE + offset); DPRINTF(IOMMU_DMAMAP_DEBUG, ("dma_map vaddr: %x pages " "req addr %lx off %x OBJSIZE %x\n", npages, addr, offset, OBJSIZE)); /* We don't need the addr anymore if we have a shadow list */ if (pplist != NULL) addr = NULL; pp = NULL; break; case DMA_OTYP_PAGES: pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; offset = dmareq->dmar_object.dmao_obj.pp_obj.pp_offset; npages = iommu_btopr(OBJSIZE + offset); break; case DMA_OTYP_PADDR: default: /* * Not a supported type for this implementation */ rval = DDI_DMA_NOMAPPING; goto bad; } /* Get our soft state once we know we're mapping an object. */ softsp = mppriv->softsp; ASSERT(softsp != NULL); if (mp->dmai_rflags & DDI_DMA_PARTIAL) { if (size != OBJSIZE) { /* * If the request is for partial mapping arrangement, * the device has to be able to address at least the * size of the window we are establishing. */ if (size < iommu_ptob(MIN_DVMA_WIN_SIZE)) { rval = DDI_DMA_NOMAPPING; goto bad; } npages = iommu_btopr(size + offset); } /* * If the size requested is less than a moderate amt, * skip the partial mapping stuff- it's not worth the * effort. */ if (npages > MIN_DVMA_WIN_SIZE) { npages = MIN_DVMA_WIN_SIZE + iommu_btopr(offset); size = iommu_ptob(MIN_DVMA_WIN_SIZE); DPRINTF(IOMMU_DMA_SETUP_DEBUG, ("dma_setup: SZ %x pg " "%x sz %lx\n", OBJSIZE, npages, size)); if (pplist != NULL) { mp->dmai_minfo = (void *)pplist; mp->dmai_rflags |= DMP_SHADOW; } } else { mp->dmai_rflags ^= DDI_DMA_PARTIAL; } } else { if (npages >= iommu_btop(softsp->iommu_dvma_size) - MIN_DVMA_WIN_SIZE) { rval = DDI_DMA_TOOBIG; goto bad; } } /* * save dmareq-object, size and npages into mp */ mp->dmai_object = dmareq->dmar_object; mp->dmai_size = size; mp->dmai_ndvmapages = npages; if (mp->dmai_rflags & DMP_NOLIMIT) { ioaddr = (ioaddr_t)vmem_alloc(softsp->dvma_arena, iommu_ptob(npages), dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP); if (ioaddr == 0) { rval = DDI_DMA_NORESOURCES; goto bad; } /* * If we have a 1 page request and we're working with a page * list, we're going to speed load an IOMMU entry. */ if (npages == 1 && !addr) { uint64_t iotte_flag = IOTTE_VALID | IOTTE_CACHE | IOTTE_WRITE | IOTTE_STREAM; volatile uint64_t *iotte_ptr; pfn_t pfn; #if defined(DEBUG) && defined(IO_MEMUSAGE) struct io_mem_list *iomemp; pfn_t *pfnp; #endif /* DEBUG && IO_MEMUSAGE */ iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr); if (mp->dmai_rflags & DDI_DMA_CONSISTENT) { mp->dmai_rflags |= DMP_NOSYNC; iotte_flag ^= IOTTE_STREAM; } else if (softsp->stream_buf_off) iotte_flag ^= IOTTE_STREAM; mp->dmai_rflags ^= DDI_DMA_PARTIAL; if (pp != NULL) pfn = pp->p_pagenum; else pfn = (*pplist)->p_pagenum; iommu_tlb_flush(softsp, ioaddr, 1); *iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag; mp->dmai_mapping = (ioaddr_t)(ioaddr + offset); mp->dmai_nwin = 0; if (cp != NULL) { cp->dmac_notused = 0; cp->dmac_address = (ioaddr_t)mp->dmai_mapping; cp->dmac_size = mp->dmai_size; cp->dmac_type = 0; *ccountp = 1; } DPRINTF(IOMMU_TTE, ("speed loading: TTE index %x " "pfn %lx tte flag %llx addr %lx ioaddr %x\n", iotte_ptr, pfn, iotte_flag, addr, ioaddr)); #if defined(DEBUG) && defined(IO_MEMUSAGE) iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP); iomemp->rdip = mp->dmai_rdip; iomemp->ioaddr = ioaddr; iomemp->addr = addr; iomemp->npages = npages; pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1), KM_SLEEP); *pfnp = pfn; mutex_enter(&softsp->iomemlock); iomemp->next = softsp->iomem; softsp->iomem = iomemp; mutex_exit(&softsp->iomemlock); #endif /* DEBUG && IO_MEMUSAGE */ return (DDI_DMA_MAPPED); } } else { ioaddr = (ioaddr_t)vmem_xalloc(softsp->dvma_arena, iommu_ptob(npages), MAX((uint_t)dma_attr->dma_attr_align, IOMMU_PAGESIZE), 0, (uint_t)dma_attr->dma_attr_seg + 1, (void *)(ioaddr_t)dma_attr->dma_attr_addr_lo, (void *)((ioaddr_t)dma_attr->dma_attr_addr_hi + 1), dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP); } if (ioaddr == 0) { if (dmareq->dmar_fp == DDI_DMA_SLEEP) rval = DDI_DMA_NOMAPPING; else rval = DDI_DMA_NORESOURCES; goto bad; } mp->dmai_mapping = ioaddr + offset; ASSERT(mp->dmai_mapping >= softsp->iommu_dvma_base); /* * At this point we have a range of virtual address allocated * with which we now have to map to the requested object. */ if (addr) { rval = iommu_create_vaddr_mappings(mp, addr & ~IOMMU_PAGEOFFSET); if (rval == DDI_DMA_NOMAPPING) goto bad_nomap; } else { rval = iommu_create_pp_mappings(mp, pp, pplist); if (rval == DDI_DMA_NOMAPPING) goto bad_nomap; } if (cp) { cp->dmac_notused = 0; cp->dmac_address = (ioaddr_t)mp->dmai_mapping; cp->dmac_size = mp->dmai_size; cp->dmac_type = 0; *ccountp = 1; } if (mp->dmai_rflags & DDI_DMA_PARTIAL) { size = iommu_ptob(mp->dmai_ndvmapages - iommu_btopr(offset)); mp->dmai_nwin = (dmareq->dmar_object.dmao_size + (size - 1)) / size; return (DDI_DMA_PARTIAL_MAP); } else { mp->dmai_nwin = 0; return (DDI_DMA_MAPPED); } bad_nomap: /* * Could not create mmu mappings. */ if (mp->dmai_rflags & DMP_NOLIMIT) { vmem_free(softsp->dvma_arena, (void *)ioaddr, iommu_ptob(npages)); } else { vmem_xfree(softsp->dvma_arena, (void *)ioaddr, iommu_ptob(npages)); } bad: if (rval == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, &softsp->dvma_call_list_id); } mp->dmai_inuse = 0; return (rval); } /* ARGSUSED */ int iommu_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) { ioaddr_t addr; uint_t npages; size_t size; ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp; struct sbus_soft_state *softsp = mppriv->softsp; ASSERT(softsp != NULL); addr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET); npages = mp->dmai_ndvmapages; size = iommu_ptob(npages); DPRINTF(IOMMU_DMA_UNBINDHDL_DEBUG, ("iommu_dma_unbindhdl: " "unbinding addr %x for %x pages\n", addr, mp->dmai_ndvmapages)); /* sync the entire object */ if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) { /* flush stream write buffers */ sync_stream_buf(softsp, addr, npages, (int *)&mppriv->sync_flag, mppriv->phys_sync_flag); } #if defined(DEBUG) && defined(IO_MEMDEBUG) /* * 'Free' the dma mappings. */ iommu_remove_mappings(mp); #endif /* DEBUG && IO_MEMDEBUG */ ASSERT(npages > (uint_t)0); if (mp->dmai_rflags & DMP_NOLIMIT) vmem_free(softsp->dvma_arena, (void *)addr, size); else vmem_xfree(softsp->dvma_arena, (void *)addr, size); mp->dmai_ndvmapages = 0; mp->dmai_inuse = 0; mp->dmai_minfo = NULL; if (softsp->dvma_call_list_id != 0) ddi_run_callback(&softsp->dvma_call_list_id); return (DDI_SUCCESS); } /*ARGSUSED*/ int iommu_dma_flush(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags) { ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp; if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) { sync_stream_buf(mppriv->softsp, mp->dmai_mapping, mp->dmai_ndvmapages, (int *)&mppriv->sync_flag, mppriv->phys_sync_flag); } return (DDI_SUCCESS); } /*ARGSUSED*/ int iommu_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp) { ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; off_t offset; uint_t winsize; uint_t newoff; int rval; offset = mp->dmai_mapping & IOMMU_PAGEOFFSET; winsize = iommu_ptob(mp->dmai_ndvmapages - iommu_btopr(offset)); DPRINTF(IOMMU_DMA_WIN_DEBUG, ("getwin win %d winsize %x\n", win, winsize)); /* * win is in the range [0 .. dmai_nwin-1] */ if (win >= mp->dmai_nwin) return (DDI_FAILURE); newoff = win * winsize; if (newoff > mp->dmai_object.dmao_size - mp->dmai_minxfer) return (DDI_FAILURE); ASSERT(cookiep); cookiep->dmac_notused = 0; cookiep->dmac_type = 0; cookiep->dmac_address = (ioaddr_t)mp->dmai_mapping; cookiep->dmac_size = mp->dmai_size; *ccountp = 1; *offp = (off_t)newoff; *lenp = (uint_t)winsize; if (newoff == mp->dmai_offset) { /* * Nothing to do... */ return (DDI_SUCCESS); } if ((rval = iommu_map_window(mp, newoff, winsize)) != DDI_SUCCESS) return (rval); /* * Set this again in case iommu_map_window() has changed it */ cookiep->dmac_size = mp->dmai_size; return (DDI_SUCCESS); } static int iommu_map_window(ddi_dma_impl_t *mp, off_t newoff, size_t winsize) { uintptr_t addr = 0; page_t *pp; uint_t flags; struct page **pplist = NULL; #if defined(DEBUG) && defined(IO_MEMDEBUG) /* Free mappings for current window */ iommu_remove_mappings(mp); #endif /* DEBUG && IO_MEMDEBUG */ mp->dmai_offset = newoff; mp->dmai_size = mp->dmai_object.dmao_size - newoff; mp->dmai_size = MIN(mp->dmai_size, winsize); if (mp->dmai_object.dmao_type == DMA_OTYP_VADDR || mp->dmai_object.dmao_type == DMA_OTYP_BUFVADDR) { if (mp->dmai_rflags & DMP_SHADOW) { pplist = (struct page **)mp->dmai_minfo; ASSERT(pplist != NULL); pplist = pplist + (newoff >> MMU_PAGESHIFT); } else { addr = (uintptr_t) mp->dmai_object.dmao_obj.virt_obj.v_addr; addr = (addr + newoff) & ~IOMMU_PAGEOFFSET; } pp = NULL; } else { pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp; flags = 0; while (flags < newoff) { pp = pp->p_next; flags += MMU_PAGESIZE; } } /* Set up mappings for next window */ if (addr) { if (iommu_create_vaddr_mappings(mp, addr) < 0) return (DDI_FAILURE); } else { if (iommu_create_pp_mappings(mp, pp, pplist) < 0) return (DDI_FAILURE); } /* * also invalidate read stream buffer */ if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) { struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp; sync_stream_buf(mppriv->softsp, mp->dmai_mapping, mp->dmai_ndvmapages, (int *)&mppriv->sync_flag, mppriv->phys_sync_flag); } return (DDI_SUCCESS); } int iommu_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) { ddi_dma_lim_t *dma_lim = dmareq->dmar_limits; ddi_dma_impl_t *mp; ddi_dma_attr_t *dma_attr; struct dma_impl_priv *mppriv; ioaddr_t addrlow, addrhigh; ioaddr_t segalign; int rval; struct sbus_soft_state *softsp = (struct sbus_soft_state *)ddi_get_soft_state(sbusp, ddi_get_instance(dip)); addrlow = dma_lim->dlim_addr_lo; addrhigh = dma_lim->dlim_addr_hi; if ((addrhigh <= addrlow) || (addrhigh < (ioaddr_t)softsp->iommu_dvma_base)) { return (DDI_DMA_NOMAPPING); } /* * Setup DMA burstsizes and min-xfer counts. */ (void) iommu_dma_lim_setup(dip, rdip, softsp, &dma_lim->dlim_burstsizes, (uint_t)dma_lim->dlim_burstsizes, &dma_lim->dlim_minxfer, dmareq->dmar_flags); if (dma_lim->dlim_burstsizes == 0) return (DDI_DMA_NOMAPPING); /* * If not an advisory call, get a DMA handle */ if (!handlep) { return (DDI_DMA_MAPOK); } mppriv = kmem_zalloc(sizeof (*mppriv), (dmareq->dmar_fp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP); if (mppriv == NULL) { if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) { ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, &softsp->dvma_call_list_id); } return (DDI_DMA_NORESOURCES); } mp = (ddi_dma_impl_t *)mppriv; mp->dmai_rdip = rdip; mp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; mp->dmai_minxfer = dma_lim->dlim_minxfer; mp->dmai_burstsizes = dma_lim->dlim_burstsizes; mp->dmai_offset = 0; mp->dmai_ndvmapages = 0; mp->dmai_minfo = 0; mp->dmai_inuse = 0; segalign = dma_lim->dlim_cntr_max; /* See if the DMA engine has any limit restrictions. */ if (segalign == UINT32_MAX && addrhigh == UINT32_MAX && addrlow == 0) { mp->dmai_rflags |= DMP_NOLIMIT; } mppriv->softsp = softsp; mppriv->phys_sync_flag = va_to_pa((caddr_t)&mppriv->sync_flag); dma_attr = &mp->dmai_attr; dma_attr->dma_attr_align = 1; dma_attr->dma_attr_addr_lo = addrlow; dma_attr->dma_attr_addr_hi = addrhigh; dma_attr->dma_attr_seg = segalign; dma_attr->dma_attr_burstsizes = dma_lim->dlim_burstsizes; rval = iommu_dma_bindhdl(dip, rdip, (ddi_dma_handle_t)mp, dmareq, NULL, NULL); if (rval && (rval != DDI_DMA_PARTIAL_MAP)) { kmem_free(mppriv, sizeof (*mppriv)); } else { *handlep = (ddi_dma_handle_t)mp; } return (rval); } /*ARGSUSED*/ int iommu_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags) { ioaddr_t addr; uint_t offset; pgcnt_t npages; size_t size; ddi_dma_cookie_t *cp; ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; DPRINTF(IOMMU_DMAMCTL_DEBUG, ("dma_mctl: handle %p ", mp)); switch (request) { case DDI_DMA_FREE: { struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp; struct sbus_soft_state *softsp = mppriv->softsp; ASSERT(softsp != NULL); /* * 'Free' the dma mappings. */ addr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET); npages = mp->dmai_ndvmapages; size = iommu_ptob(npages); DPRINTF(IOMMU_DMAMCTL_DMA_FREE_DEBUG, ("iommu_dma_mctl dmafree:" "freeing vaddr %x for %x pages.\n", addr, mp->dmai_ndvmapages)); /* sync the entire object */ if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) { /* flush stream write buffers */ sync_stream_buf(softsp, addr, npages, (int *)&mppriv->sync_flag, mppriv->phys_sync_flag); } #if defined(DEBUG) && defined(IO_MEMDEBUG) iommu_remove_mappings(mp); #endif /* DEBUG && IO_MEMDEBUG */ ASSERT(npages > (uint_t)0); if (mp->dmai_rflags & DMP_NOLIMIT) vmem_free(softsp->dvma_arena, (void *)addr, size); else vmem_xfree(softsp->dvma_arena, (void *)addr, size); kmem_free(mppriv, sizeof (*mppriv)); if (softsp->dvma_call_list_id != 0) ddi_run_callback(&softsp->dvma_call_list_id); break; } case DDI_DMA_SET_SBUS64: { struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp; return (iommu_dma_lim_setup(dip, rdip, mppriv->softsp, &mp->dmai_burstsizes, (uint_t)*lenp, &mp->dmai_minxfer, DDI_DMA_SBUS_64BIT)); } case DDI_DMA_HTOC: DPRINTF(IOMMU_DMAMCTL_HTOC_DEBUG, ("htoc off %lx mapping %lx " "size %lx\n", *offp, mp->dmai_mapping, mp->dmai_size)); if ((uint_t)(*offp) >= mp->dmai_size) return (DDI_FAILURE); cp = (ddi_dma_cookie_t *)objp; cp->dmac_notused = 0; cp->dmac_address = (mp->dmai_mapping + (uint_t)(*offp)); cp->dmac_size = mp->dmai_mapping + mp->dmai_size - cp->dmac_address; cp->dmac_type = 0; break; case DDI_DMA_KVADDR: /* * If a physical address mapping has percolated this high, * that is an error (maybe?). */ if (mp->dmai_rflags & DMP_PHYSADDR) { DPRINTF(IOMMU_DMAMCTL_KVADDR_DEBUG, ("kvaddr of phys " "mapping\n")); return (DDI_FAILURE); } return (DDI_FAILURE); case DDI_DMA_NEXTWIN: { ddi_dma_win_t *owin, *nwin; uint_t winsize, newoff; int rval; DPRINTF(IOMMU_DMAMCTL_NEXTWIN_DEBUG, ("nextwin\n")); mp = (ddi_dma_impl_t *)handle; owin = (ddi_dma_win_t *)offp; nwin = (ddi_dma_win_t *)objp; if (mp->dmai_rflags & DDI_DMA_PARTIAL) { if (*owin == NULL) { DPRINTF(IOMMU_DMAMCTL_NEXTWIN_DEBUG, ("nextwin: win == NULL\n")); mp->dmai_offset = 0; *nwin = (ddi_dma_win_t)mp; return (DDI_SUCCESS); } offset = (uint_t)(mp->dmai_mapping & IOMMU_PAGEOFFSET); winsize = iommu_ptob(mp->dmai_ndvmapages - iommu_btopr(offset)); newoff = (uint_t)(mp->dmai_offset + winsize); if (newoff > mp->dmai_object.dmao_size - mp->dmai_minxfer) return (DDI_DMA_DONE); if ((rval = iommu_map_window(mp, newoff, winsize)) != DDI_SUCCESS) return (rval); } else { DPRINTF(IOMMU_DMAMCTL_NEXTWIN_DEBUG, ("nextwin: no " "partial mapping\n")); if (*owin != NULL) return (DDI_DMA_DONE); mp->dmai_offset = 0; *nwin = (ddi_dma_win_t)mp; } break; } case DDI_DMA_NEXTSEG: { ddi_dma_seg_t *oseg, *nseg; DPRINTF(IOMMU_DMAMCTL_NEXTSEG_DEBUG, ("nextseg:\n")); oseg = (ddi_dma_seg_t *)lenp; if (*oseg != NULL) return (DDI_DMA_DONE); nseg = (ddi_dma_seg_t *)objp; *nseg = *((ddi_dma_seg_t *)offp); break; } case DDI_DMA_SEGTOC: { ddi_dma_seg_impl_t *seg; seg = (ddi_dma_seg_impl_t *)handle; cp = (ddi_dma_cookie_t *)objp; cp->dmac_notused = 0; cp->dmac_address = (ioaddr_t)seg->dmai_mapping; cp->dmac_size = *lenp = seg->dmai_size; cp->dmac_type = 0; *offp = seg->dmai_offset; break; } case DDI_DMA_MOVWIN: { uint_t winsize; uint_t newoff; int rval; offset = (uint_t)(mp->dmai_mapping & IOMMU_PAGEOFFSET); winsize = iommu_ptob(mp->dmai_ndvmapages - iommu_btopr(offset)); DPRINTF(IOMMU_DMAMCTL_MOVWIN_DEBUG, ("movwin off %lx len %x " "winsize %x\n", *offp, *lenp, winsize)); if ((mp->dmai_rflags & DDI_DMA_PARTIAL) == 0) return (DDI_FAILURE); if (*lenp != (uint_t)-1 && *lenp != winsize) { DPRINTF(IOMMU_DMAMCTL_MOVWIN_DEBUG, ("bad length\n")); return (DDI_FAILURE); } newoff = (uint_t)*offp; if (newoff & (winsize - 1)) { DPRINTF(IOMMU_DMAMCTL_MOVWIN_DEBUG, ("bad off\n")); return (DDI_FAILURE); } if (newoff == mp->dmai_offset) { /* * Nothing to do... */ break; } /* * Check out new address... */ if (newoff > mp->dmai_object.dmao_size - mp->dmai_minxfer) { DPRINTF(IOMMU_DMAMCTL_MOVWIN_DEBUG, ("newoff out of " "range\n")); return (DDI_FAILURE); } rval = iommu_map_window(mp, newoff, winsize); if (rval != DDI_SUCCESS) return (rval); if ((cp = (ddi_dma_cookie_t *)objp) != 0) { cp->dmac_notused = 0; cp->dmac_address = (ioaddr_t)mp->dmai_mapping; cp->dmac_size = mp->dmai_size; cp->dmac_type = 0; } *offp = (off_t)newoff; *lenp = (uint_t)winsize; break; } case DDI_DMA_REPWIN: if ((mp->dmai_rflags & DDI_DMA_PARTIAL) == 0) { DPRINTF(IOMMU_DMAMCTL_REPWIN_DEBUG, ("repwin fail\n")); return (DDI_FAILURE); } *offp = (off_t)mp->dmai_offset; addr = mp->dmai_ndvmapages - iommu_btopr(mp->dmai_mapping & IOMMU_PAGEOFFSET); *lenp = (uint_t)iommu_ptob(addr); DPRINTF(IOMMU_DMAMCTL_REPWIN_DEBUG, ("repwin off %x len %x\n", mp->dmai_offset, mp->dmai_size)); break; case DDI_DMA_GETERR: DPRINTF(IOMMU_DMAMCTL_GETERR_DEBUG, ("iommu_dma_mctl: geterr\n")); break; case DDI_DMA_COFF: cp = (ddi_dma_cookie_t *)offp; addr = cp->dmac_address; if (addr < mp->dmai_mapping || addr >= mp->dmai_mapping + mp->dmai_size) return (DDI_FAILURE); *objp = (caddr_t)(addr - mp->dmai_mapping); DPRINTF(IOMMU_DMAMCTL_COFF_DEBUG, ("coff off %lx mapping %x " "size %x\n", (ulong_t)*objp, mp->dmai_mapping, mp->dmai_size)); break; case DDI_DMA_RESERVE: { struct ddi_dma_req *dmareq = (struct ddi_dma_req *)offp; ddi_dma_lim_t *dma_lim; ddi_dma_handle_t *handlep; uint_t np; ioaddr_t ioaddr; int i; struct fast_dvma *iommu_fast_dvma; struct sbus_soft_state *softsp = (struct sbus_soft_state *)ddi_get_soft_state(sbusp, ddi_get_instance(dip)); /* Some simple sanity checks */ dma_lim = dmareq->dmar_limits; if (dma_lim->dlim_burstsizes == 0) { DPRINTF(IOMMU_FASTDMA_RESERVE, ("Reserve: bad burstsizes\n")); return (DDI_DMA_BADLIMITS); } if ((AHI <= ALO) || (AHI < softsp->iommu_dvma_base)) { DPRINTF(IOMMU_FASTDMA_RESERVE, ("Reserve: bad limits\n")); return (DDI_DMA_BADLIMITS); } np = dmareq->dmar_object.dmao_size; mutex_enter(&softsp->dma_pool_lock); if (np > softsp->dma_reserve) { mutex_exit(&softsp->dma_pool_lock); DPRINTF(IOMMU_FASTDMA_RESERVE, ("Reserve: dma_reserve is exhausted\n")); return (DDI_DMA_NORESOURCES); } softsp->dma_reserve -= np; mutex_exit(&softsp->dma_pool_lock); mp = kmem_zalloc(sizeof (*mp), KM_SLEEP); mp->dmai_rflags = DMP_BYPASSNEXUS; mp->dmai_rdip = rdip; mp->dmai_minxfer = dma_lim->dlim_minxfer; mp->dmai_burstsizes = dma_lim->dlim_burstsizes; ioaddr = (ioaddr_t)vmem_xalloc(softsp->dvma_arena, iommu_ptob(np), IOMMU_PAGESIZE, 0, dma_lim->dlim_cntr_max + 1, (void *)ALO, (void *)(AHI + 1), dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP); if (ioaddr == 0) { mutex_enter(&softsp->dma_pool_lock); softsp->dma_reserve += np; mutex_exit(&softsp->dma_pool_lock); kmem_free(mp, sizeof (*mp)); DPRINTF(IOMMU_FASTDMA_RESERVE, ("Reserve: No dvma resources available\n")); return (DDI_DMA_NOMAPPING); } /* create a per request structure */ iommu_fast_dvma = kmem_alloc(sizeof (struct fast_dvma), KM_SLEEP); /* * We need to remember the size of the transfer so that * we can figure the virtual pages to sync when the transfer * is complete. */ iommu_fast_dvma->pagecnt = kmem_zalloc(np * sizeof (uint_t), KM_SLEEP); /* Allocate a streaming cache sync flag for each index */ iommu_fast_dvma->sync_flag = kmem_zalloc(np * sizeof (int), KM_SLEEP); /* Allocate a physical sync flag for each index */ iommu_fast_dvma->phys_sync_flag = kmem_zalloc(np * sizeof (uint64_t), KM_SLEEP); for (i = 0; i < np; i++) iommu_fast_dvma->phys_sync_flag[i] = va_to_pa((caddr_t) &iommu_fast_dvma->sync_flag[i]); mp->dmai_mapping = ioaddr; mp->dmai_ndvmapages = np; iommu_fast_dvma->ops = &iommu_dvma_ops; iommu_fast_dvma->softsp = (caddr_t)softsp; mp->dmai_nexus_private = (caddr_t)iommu_fast_dvma; handlep = (ddi_dma_handle_t *)objp; *handlep = (ddi_dma_handle_t)mp; DPRINTF(IOMMU_FASTDMA_RESERVE, ("Reserve: mapping object %p base addr %lx size %x\n", mp, mp->dmai_mapping, mp->dmai_ndvmapages)); break; } case DDI_DMA_RELEASE: { ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; uint_t np = npages = mp->dmai_ndvmapages; ioaddr_t ioaddr = mp->dmai_mapping; volatile uint64_t *iotte_ptr; struct fast_dvma *iommu_fast_dvma = (struct fast_dvma *) mp->dmai_nexus_private; struct sbus_soft_state *softsp = (struct sbus_soft_state *) iommu_fast_dvma->softsp; ASSERT(softsp != NULL); /* Unload stale mappings and flush stale tlb's */ iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr); while (npages > (uint_t)0) { *iotte_ptr = (uint64_t)0; /* unload tte */ iommu_tlb_flush(softsp, ioaddr, 1); npages--; iotte_ptr++; ioaddr += IOMMU_PAGESIZE; } ioaddr = (ioaddr_t)mp->dmai_mapping; mutex_enter(&softsp->dma_pool_lock); softsp->dma_reserve += np; mutex_exit(&softsp->dma_pool_lock); if (mp->dmai_rflags & DMP_NOLIMIT) vmem_free(softsp->dvma_arena, (void *)ioaddr, iommu_ptob(np)); else vmem_xfree(softsp->dvma_arena, (void *)ioaddr, iommu_ptob(np)); kmem_free(mp, sizeof (*mp)); kmem_free(iommu_fast_dvma->pagecnt, np * sizeof (uint_t)); kmem_free(iommu_fast_dvma->sync_flag, np * sizeof (int)); kmem_free(iommu_fast_dvma->phys_sync_flag, np * sizeof (uint64_t)); kmem_free(iommu_fast_dvma, sizeof (struct fast_dvma)); DPRINTF(IOMMU_FASTDMA_RESERVE, ("Release: Base addr %x size %x\n", ioaddr, np)); /* * Now that we've freed some resource, * if there is anybody waiting for it * try and get them going. */ if (softsp->dvma_call_list_id != 0) ddi_run_callback(&softsp->dvma_call_list_id); break; } default: DPRINTF(IOMMU_DMAMCTL_DEBUG, ("iommu_dma_mctl: unknown option " "0%x\n", request)); return (DDI_FAILURE); } return (DDI_SUCCESS); } /*ARGSUSED*/ void iommu_dvma_kaddr_load(ddi_dma_handle_t h, caddr_t a, uint_t len, uint_t index, ddi_dma_cookie_t *cp) { uintptr_t addr; ioaddr_t ioaddr; uint_t offset; pfn_t pfn; int npages; volatile uint64_t *iotte_ptr; uint64_t iotte_flag = 0; struct as *as = NULL; extern struct as kas; ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h; struct fast_dvma *iommu_fast_dvma = (struct fast_dvma *)mp->dmai_nexus_private; struct sbus_soft_state *softsp = (struct sbus_soft_state *) iommu_fast_dvma->softsp; #if defined(DEBUG) && defined(IO_MEMUSAGE) struct io_mem_list *iomemp; pfn_t *pfnp; #endif /* DEBUG && IO_MEMUSAGE */ ASSERT(softsp != NULL); addr = (uintptr_t)a; ioaddr = (ioaddr_t)(mp->dmai_mapping + iommu_ptob(index)); offset = (uint_t)(addr & IOMMU_PAGEOFFSET); iommu_fast_dvma->pagecnt[index] = iommu_btopr(len + offset); as = &kas; addr &= ~IOMMU_PAGEOFFSET; npages = iommu_btopr(len + offset); #if defined(DEBUG) && defined(IO_MEMUSAGE) iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP); iomemp->rdip = mp->dmai_rdip; iomemp->ioaddr = ioaddr; iomemp->addr = addr; iomemp->npages = npages; pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1), KM_SLEEP); #endif /* DEBUG && IO_MEMUSAGE */ cp->dmac_address = ioaddr | offset; cp->dmac_size = len; iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr); /* read/write and streaming io on */ iotte_flag = IOTTE_VALID | IOTTE_WRITE | IOTTE_CACHE; if (mp->dmai_rflags & DDI_DMA_CONSISTENT) mp->dmai_rflags |= DMP_NOSYNC; else if (!softsp->stream_buf_off) iotte_flag |= IOTTE_STREAM; DPRINTF(IOMMU_FASTDMA_LOAD, ("kaddr_load: ioaddr %x " "size %x offset %x index %x kaddr %p\n", ioaddr, len, offset, index, addr)); ASSERT(npages > 0); do { pfn = hat_getpfnum(as->a_hat, (caddr_t)addr); if (pfn == PFN_INVALID) { DPRINTF(IOMMU_FASTDMA_LOAD, ("kaddr_load: invalid pfn " "from hat_getpfnum()\n")); } iommu_tlb_flush(softsp, ioaddr, 1); /* load tte */ *iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag; npages--; iotte_ptr++; addr += IOMMU_PAGESIZE; ioaddr += IOMMU_PAGESIZE; #if defined(DEBUG) && defined(IO_MEMUSAGE) *pfnp = pfn; pfnp++; #endif /* DEBUG && IO_MEMUSAGE */ } while (npages > 0); #if defined(DEBUG) && defined(IO_MEMUSAGE) mutex_enter(&softsp->iomemlock); iomemp->next = softsp->iomem; softsp->iomem = iomemp; mutex_exit(&softsp->iomemlock); #endif /* DEBUG && IO_MEMUSAGE */ } /*ARGSUSED*/ void iommu_dvma_unload(ddi_dma_handle_t h, uint_t index, uint_t view) { ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h; ioaddr_t ioaddr; pgcnt_t npages; struct fast_dvma *iommu_fast_dvma = (struct fast_dvma *)mp->dmai_nexus_private; struct sbus_soft_state *softsp = (struct sbus_soft_state *) iommu_fast_dvma->softsp; #if defined(DEBUG) && defined(IO_MEMUSAGE) struct io_mem_list **prevp, *walk; #endif /* DEBUG && IO_MEMUSAGE */ ASSERT(softsp != NULL); ioaddr = (ioaddr_t)(mp->dmai_mapping + iommu_ptob(index)); npages = iommu_fast_dvma->pagecnt[index]; #if defined(DEBUG) && defined(IO_MEMUSAGE) mutex_enter(&softsp->iomemlock); prevp = &softsp->iomem; walk = softsp->iomem; while (walk != NULL) { if (walk->ioaddr == ioaddr) { *prevp = walk->next; break; } prevp = &walk->next; walk = walk->next; } mutex_exit(&softsp->iomemlock); kmem_free(walk->pfn, sizeof (pfn_t) * (npages + 1)); kmem_free(walk, sizeof (struct io_mem_list)); #endif /* DEBUG && IO_MEMUSAGE */ DPRINTF(IOMMU_FASTDMA_SYNC, ("kaddr_unload: handle %p sync flag " "addr %p sync flag pfn %x index %x page count %x\n", mp, &iommu_fast_dvma->sync_flag[index], iommu_fast_dvma->phys_sync_flag[index], index, npages)); if ((mp->dmai_rflags & DMP_NOSYNC) != DMP_NOSYNC) { sync_stream_buf(softsp, ioaddr, npages, (int *)&iommu_fast_dvma->sync_flag[index], iommu_fast_dvma->phys_sync_flag[index]); } } /*ARGSUSED*/ void iommu_dvma_sync(ddi_dma_handle_t h, uint_t index, uint_t view) { ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h; ioaddr_t ioaddr; uint_t npages; struct fast_dvma *iommu_fast_dvma = (struct fast_dvma *)mp->dmai_nexus_private; struct sbus_soft_state *softsp = (struct sbus_soft_state *) iommu_fast_dvma->softsp; if ((mp->dmai_rflags & DMP_NOSYNC) == DMP_NOSYNC) return; ASSERT(softsp != NULL); ioaddr = (ioaddr_t)(mp->dmai_mapping + iommu_ptob(index)); npages = iommu_fast_dvma->pagecnt[index]; DPRINTF(IOMMU_FASTDMA_SYNC, ("kaddr_sync: handle %p, " "sync flag addr %p, sync flag pfn %x\n", mp, &iommu_fast_dvma->sync_flag[index], iommu_fast_dvma->phys_sync_flag[index])); sync_stream_buf(softsp, ioaddr, npages, (int *)&iommu_fast_dvma->sync_flag[index], iommu_fast_dvma->phys_sync_flag[index]); }