1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * iommu.c: IOMMU specific routines for memory management. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) 61da177e4SLinus Torvalds * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com) 71da177e4SLinus Torvalds * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) 81da177e4SLinus Torvalds * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 111da177e4SLinus Torvalds #include <linux/kernel.h> 121da177e4SLinus Torvalds #include <linux/init.h> 131da177e4SLinus Torvalds #include <linux/mm.h> 141da177e4SLinus Torvalds #include <linux/slab.h> 151da177e4SLinus Torvalds #include <linux/highmem.h> /* pte_offset_map => kmap_atomic */ 16ce65d36fSChristoph Hellwig #include <linux/dma-mapping.h> 179dc69230SDavid S. Miller #include <linux/of.h> 189dc69230SDavid S. Miller #include <linux/of_device.h> 191da177e4SLinus Torvalds 201da177e4SLinus Torvalds #include <asm/pgalloc.h> 211da177e4SLinus Torvalds #include <asm/pgtable.h> 221da177e4SLinus Torvalds #include <asm/io.h> 231da177e4SLinus Torvalds #include <asm/mxcc.h> 241da177e4SLinus Torvalds #include <asm/mbus.h> 251da177e4SLinus Torvalds #include <asm/cacheflush.h> 261da177e4SLinus Torvalds #include <asm/tlbflush.h> 271da177e4SLinus Torvalds #include <asm/bitext.h> 281da177e4SLinus Torvalds #include <asm/iommu.h> 291da177e4SLinus Torvalds #include <asm/dma.h> 301da177e4SLinus Torvalds 31e8c29c83SSam Ravnborg #include "mm_32.h" 32e8c29c83SSam Ravnborg 331da177e4SLinus Torvalds /* 341da177e4SLinus Torvalds * This can be sized dynamically, but we will do this 351da177e4SLinus Torvalds * only when we have a guidance about actual I/O pressures. 361da177e4SLinus Torvalds */ 371da177e4SLinus Torvalds #define IOMMU_RNGE IOMMU_RNGE_256MB 381da177e4SLinus Torvalds #define IOMMU_START 0xF0000000 391da177e4SLinus Torvalds #define IOMMU_WINSIZE (256*1024*1024U) 409a0ac1b6SAkinobu Mita #define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */ 411da177e4SLinus Torvalds #define IOMMU_ORDER 6 /* 4096 * (1<<6) */ 421da177e4SLinus Torvalds 431da177e4SLinus Torvalds static int viking_flush; 441da177e4SLinus Torvalds /* viking.S */ 451da177e4SLinus Torvalds extern void viking_flush_page(unsigned long page); 461da177e4SLinus Torvalds extern void viking_mxcc_flush_page(unsigned long page); 471da177e4SLinus Torvalds 481da177e4SLinus Torvalds /* 491da177e4SLinus Torvalds * Values precomputed according to CPU type. 501da177e4SLinus Torvalds */ 511da177e4SLinus Torvalds static unsigned int ioperm_noc; /* Consistent mapping iopte flags */ 521da177e4SLinus Torvalds static pgprot_t dvma_prot; /* Consistent mapping pte flags */ 531da177e4SLinus Torvalds 541da177e4SLinus Torvalds #define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID) 551da177e4SLinus Torvalds #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ) 561da177e4SLinus Torvalds 57*255a69a9SChristoph Hellwig static const struct dma_map_ops sbus_iommu_dma_gflush_ops; 58*255a69a9SChristoph Hellwig static const struct dma_map_ops sbus_iommu_dma_pflush_ops; 59*255a69a9SChristoph Hellwig 60cd4cd730SGrant Likely static void __init sbus_iommu_init(struct platform_device *op) 611da177e4SLinus Torvalds { 621da177e4SLinus Torvalds struct iommu_struct *iommu; 63e0039348SDavid S. Miller unsigned int impl, vers; 641da177e4SLinus Torvalds unsigned long *bitmap; 65f977ea49SSam Ravnborg unsigned long control; 66f977ea49SSam Ravnborg unsigned long base; 67e0039348SDavid S. Miller unsigned long tmp; 68e0039348SDavid S. Miller 6971cd03b0SJulia Lawall iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL); 701da177e4SLinus Torvalds if (!iommu) { 711da177e4SLinus Torvalds prom_printf("Unable to allocate iommu structure\n"); 721da177e4SLinus Torvalds prom_halt(); 731da177e4SLinus Torvalds } 74e0039348SDavid S. Miller 75046e26a8SDavid S. Miller iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3, 76e0039348SDavid S. Miller "iommu_regs"); 771da177e4SLinus Torvalds if (!iommu->regs) { 781da177e4SLinus Torvalds prom_printf("Cannot map IOMMU registers\n"); 791da177e4SLinus Torvalds prom_halt(); 801da177e4SLinus Torvalds } 81f977ea49SSam Ravnborg 82f977ea49SSam Ravnborg control = sbus_readl(&iommu->regs->control); 83f977ea49SSam Ravnborg impl = (control & IOMMU_CTRL_IMPL) >> 28; 84f977ea49SSam Ravnborg vers = (control & IOMMU_CTRL_VERS) >> 24; 85f977ea49SSam Ravnborg control &= ~(IOMMU_CTRL_RNGE); 86f977ea49SSam Ravnborg control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); 87f977ea49SSam Ravnborg sbus_writel(control, &iommu->regs->control); 88f977ea49SSam Ravnborg 891da177e4SLinus Torvalds iommu_invalidate(iommu->regs); 901da177e4SLinus Torvalds iommu->start = IOMMU_START; 911da177e4SLinus Torvalds iommu->end = 0xffffffff; 921da177e4SLinus Torvalds 931da177e4SLinus Torvalds /* Allocate IOMMU page table */ 941da177e4SLinus Torvalds /* Stupid alignment constraints give me a headache. 951da177e4SLinus Torvalds We need 256K or 512K or 1M or 2M area aligned to 961da177e4SLinus Torvalds its size and current gfp will fortunately give 971da177e4SLinus Torvalds it to us. */ 981da177e4SLinus Torvalds tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); 991da177e4SLinus Torvalds if (!tmp) { 1005da444aaSAkinobu Mita prom_printf("Unable to allocate iommu table [0x%lx]\n", 1011da177e4SLinus Torvalds IOMMU_NPTES * sizeof(iopte_t)); 1021da177e4SLinus Torvalds prom_halt(); 1031da177e4SLinus Torvalds } 1041da177e4SLinus Torvalds iommu->page_table = (iopte_t *)tmp; 1051da177e4SLinus Torvalds 1061da177e4SLinus Torvalds /* Initialize new table. */ 1071da177e4SLinus Torvalds memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t)); 1081da177e4SLinus Torvalds flush_cache_all(); 1091da177e4SLinus Torvalds flush_tlb_all(); 110f977ea49SSam Ravnborg 111f977ea49SSam Ravnborg base = __pa((unsigned long)iommu->page_table) >> 4; 112f977ea49SSam Ravnborg sbus_writel(base, &iommu->regs->base); 1131da177e4SLinus Torvalds iommu_invalidate(iommu->regs); 1141da177e4SLinus Torvalds 1151da177e4SLinus Torvalds bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL); 1161da177e4SLinus Torvalds if (!bitmap) { 1171da177e4SLinus Torvalds prom_printf("Unable to allocate iommu bitmap [%d]\n", 1181da177e4SLinus Torvalds (int)(IOMMU_NPTES>>3)); 1191da177e4SLinus Torvalds prom_halt(); 1201da177e4SLinus Torvalds } 1211da177e4SLinus Torvalds bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES); 1221da177e4SLinus Torvalds /* To be coherent on HyperSparc, the page color of DVMA 1231da177e4SLinus Torvalds * and physical addresses must match. 1241da177e4SLinus Torvalds */ 1251da177e4SLinus Torvalds if (srmmu_modtype == HyperSparc) 1261da177e4SLinus Torvalds iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT; 1271da177e4SLinus Torvalds else 1281da177e4SLinus Torvalds iommu->usemap.num_colors = 1; 1291da177e4SLinus Torvalds 130046e26a8SDavid S. Miller printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n", 1311da177e4SLinus Torvalds impl, vers, iommu->page_table, 1321da177e4SLinus Torvalds (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES); 1331da177e4SLinus Torvalds 134e0039348SDavid S. Miller op->dev.archdata.iommu = iommu; 135*255a69a9SChristoph Hellwig 136*255a69a9SChristoph Hellwig if (flush_page_for_dma_global) 137*255a69a9SChristoph Hellwig op->dev.dma_ops = &sbus_iommu_dma_gflush_ops; 138*255a69a9SChristoph Hellwig else 139*255a69a9SChristoph Hellwig op->dev.dma_ops = &sbus_iommu_dma_pflush_ops; 1401da177e4SLinus Torvalds } 1411da177e4SLinus Torvalds 142046e26a8SDavid S. Miller static int __init iommu_init(void) 143046e26a8SDavid S. Miller { 144046e26a8SDavid S. Miller struct device_node *dp; 145046e26a8SDavid S. Miller 146046e26a8SDavid S. Miller for_each_node_by_name(dp, "iommu") { 147cd4cd730SGrant Likely struct platform_device *op = of_find_device_by_node(dp); 148046e26a8SDavid S. Miller 149046e26a8SDavid S. Miller sbus_iommu_init(op); 150046e26a8SDavid S. Miller of_propagate_archdata(op); 151046e26a8SDavid S. Miller } 152046e26a8SDavid S. Miller 153046e26a8SDavid S. Miller return 0; 154046e26a8SDavid S. Miller } 155046e26a8SDavid S. Miller 156046e26a8SDavid S. Miller subsys_initcall(iommu_init); 157046e26a8SDavid S. Miller 1581da177e4SLinus Torvalds /* Flush the iotlb entries to ram. */ 1591da177e4SLinus Torvalds /* This could be better if we didn't have to flush whole pages. */ 1601da177e4SLinus Torvalds static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) 1611da177e4SLinus Torvalds { 1621da177e4SLinus Torvalds unsigned long start; 1631da177e4SLinus Torvalds unsigned long end; 1641da177e4SLinus Torvalds 1653185d4d2SBob Breuer start = (unsigned long)iopte; 1661da177e4SLinus Torvalds end = PAGE_ALIGN(start + niopte*sizeof(iopte_t)); 1673185d4d2SBob Breuer start &= PAGE_MASK; 1681da177e4SLinus Torvalds if (viking_mxcc_present) { 1691da177e4SLinus Torvalds while(start < end) { 1701da177e4SLinus Torvalds viking_mxcc_flush_page(start); 1711da177e4SLinus Torvalds start += PAGE_SIZE; 1721da177e4SLinus Torvalds } 1731da177e4SLinus Torvalds } else if (viking_flush) { 1741da177e4SLinus Torvalds while(start < end) { 1751da177e4SLinus Torvalds viking_flush_page(start); 1761da177e4SLinus Torvalds start += PAGE_SIZE; 1771da177e4SLinus Torvalds } 1781da177e4SLinus Torvalds } else { 1791da177e4SLinus Torvalds while(start < end) { 1801da177e4SLinus Torvalds __flush_page_to_ram(start); 1811da177e4SLinus Torvalds start += PAGE_SIZE; 1821da177e4SLinus Torvalds } 1831da177e4SLinus Torvalds } 1841da177e4SLinus Torvalds } 1851da177e4SLinus Torvalds 186376b1371SChristoph Hellwig static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page, 187376b1371SChristoph Hellwig unsigned long offset, size_t len, bool per_page_flush) 1881da177e4SLinus Torvalds { 189260489faSDavid S. Miller struct iommu_struct *iommu = dev->archdata.iommu; 190376b1371SChristoph Hellwig phys_addr_t paddr = page_to_phys(page) + offset; 191376b1371SChristoph Hellwig unsigned long off = paddr & ~PAGE_MASK; 192376b1371SChristoph Hellwig unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 193b8205942SChristoph Hellwig unsigned long pfn = __phys_to_pfn(paddr); 194376b1371SChristoph Hellwig unsigned int busa, busa0; 195376b1371SChristoph Hellwig iopte_t *iopte, *iopte0; 196376b1371SChristoph Hellwig int ioptex, i; 197376b1371SChristoph Hellwig 198376b1371SChristoph Hellwig /* XXX So what is maxphys for us and how do drivers know it? */ 199376b1371SChristoph Hellwig if (!len || len > 256 * 1024) 200376b1371SChristoph Hellwig return DMA_MAPPING_ERROR; 201376b1371SChristoph Hellwig 202376b1371SChristoph Hellwig /* 203376b1371SChristoph Hellwig * We expect unmapped highmem pages to be not in the cache. 204376b1371SChristoph Hellwig * XXX Is this a good assumption? 205376b1371SChristoph Hellwig * XXX What if someone else unmaps it here and races us? 206376b1371SChristoph Hellwig */ 207376b1371SChristoph Hellwig if (per_page_flush && !PageHighMem(page)) { 208376b1371SChristoph Hellwig unsigned long vaddr, p; 209376b1371SChristoph Hellwig 210376b1371SChristoph Hellwig vaddr = (unsigned long)page_address(page) + offset; 211376b1371SChristoph Hellwig for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE) 212376b1371SChristoph Hellwig flush_page_for_dma(p); 213376b1371SChristoph Hellwig } 2141da177e4SLinus Torvalds 2151da177e4SLinus Torvalds /* page color = pfn of page */ 216b8205942SChristoph Hellwig ioptex = bit_map_string_get(&iommu->usemap, npages, pfn); 2171da177e4SLinus Torvalds if (ioptex < 0) 2181da177e4SLinus Torvalds panic("iommu out"); 2191da177e4SLinus Torvalds busa0 = iommu->start + (ioptex << PAGE_SHIFT); 2201da177e4SLinus Torvalds iopte0 = &iommu->page_table[ioptex]; 2211da177e4SLinus Torvalds 2221da177e4SLinus Torvalds busa = busa0; 2231da177e4SLinus Torvalds iopte = iopte0; 2241da177e4SLinus Torvalds for (i = 0; i < npages; i++) { 225b8205942SChristoph Hellwig iopte_val(*iopte) = MKIOPTE(pfn, IOPERM); 2261da177e4SLinus Torvalds iommu_invalidate_page(iommu->regs, busa); 2271da177e4SLinus Torvalds busa += PAGE_SIZE; 2281da177e4SLinus Torvalds iopte++; 229b8205942SChristoph Hellwig pfn++; 2301da177e4SLinus Torvalds } 2311da177e4SLinus Torvalds 2321da177e4SLinus Torvalds iommu_flush_iotlb(iopte0, npages); 233376b1371SChristoph Hellwig return busa0 + off; 2341da177e4SLinus Torvalds } 2351da177e4SLinus Torvalds 236ce65d36fSChristoph Hellwig static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev, 237ce65d36fSChristoph Hellwig struct page *page, unsigned long offset, size_t len, 238ce65d36fSChristoph Hellwig enum dma_data_direction dir, unsigned long attrs) 2391da177e4SLinus Torvalds { 2401da177e4SLinus Torvalds flush_page_for_dma(0); 2418668b38cSChristoph Hellwig return __sbus_iommu_map_page(dev, page, offset, len, false); 2421da177e4SLinus Torvalds } 2431da177e4SLinus Torvalds 244ce65d36fSChristoph Hellwig static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev, 245ce65d36fSChristoph Hellwig struct page *page, unsigned long offset, size_t len, 246ce65d36fSChristoph Hellwig enum dma_data_direction dir, unsigned long attrs) 2471da177e4SLinus Torvalds { 2488668b38cSChristoph Hellwig return __sbus_iommu_map_page(dev, page, offset, len, true); 249ce65d36fSChristoph Hellwig } 250ce65d36fSChristoph Hellwig 251ff5cbec0SChristoph Hellwig static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl, 252ff5cbec0SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs, 253ff5cbec0SChristoph Hellwig bool per_page_flush) 2541da177e4SLinus Torvalds { 2556c503d0dSChristoph Hellwig struct scatterlist *sg; 256edb1f072SChristoph Hellwig int j; 2571da177e4SLinus Torvalds 2586c503d0dSChristoph Hellwig for_each_sg(sgl, sg, nents, j) { 259edb1f072SChristoph Hellwig sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg), 260edb1f072SChristoph Hellwig sg->offset, sg->length, per_page_flush); 261edb1f072SChristoph Hellwig if (sg->dma_address == DMA_MAPPING_ERROR) 262edb1f072SChristoph Hellwig return 0; 263aa83a26aSRobert Reif sg->dma_length = sg->length; 2641da177e4SLinus Torvalds } 265ce65d36fSChristoph Hellwig 2666c503d0dSChristoph Hellwig return nents; 2671da177e4SLinus Torvalds } 2681da177e4SLinus Torvalds 269ff5cbec0SChristoph Hellwig static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl, 270ff5cbec0SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs) 271ff5cbec0SChristoph Hellwig { 272ff5cbec0SChristoph Hellwig flush_page_for_dma(0); 273ff5cbec0SChristoph Hellwig return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false); 274ff5cbec0SChristoph Hellwig } 275ff5cbec0SChristoph Hellwig 276ff5cbec0SChristoph Hellwig static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl, 277ff5cbec0SChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs) 278ff5cbec0SChristoph Hellwig { 279ff5cbec0SChristoph Hellwig return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true); 280ff5cbec0SChristoph Hellwig } 281ff5cbec0SChristoph Hellwig 282f25b23bcSChristoph Hellwig static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr, 283f25b23bcSChristoph Hellwig size_t len, enum dma_data_direction dir, unsigned long attrs) 2841da177e4SLinus Torvalds { 285260489faSDavid S. Miller struct iommu_struct *iommu = dev->archdata.iommu; 286f25b23bcSChristoph Hellwig unsigned int busa = dma_addr & PAGE_MASK; 287f25b23bcSChristoph Hellwig unsigned long off = dma_addr & ~PAGE_MASK; 288f25b23bcSChristoph Hellwig unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; 289f25b23bcSChristoph Hellwig unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT; 290f25b23bcSChristoph Hellwig unsigned int i; 2911da177e4SLinus Torvalds 2921ae61388SEric Sesterhenn BUG_ON(busa < iommu->start); 2931da177e4SLinus Torvalds for (i = 0; i < npages; i++) { 2941da177e4SLinus Torvalds iopte_val(iommu->page_table[ioptex + i]) = 0; 2951da177e4SLinus Torvalds iommu_invalidate_page(iommu->regs, busa); 2961da177e4SLinus Torvalds busa += PAGE_SIZE; 2971da177e4SLinus Torvalds } 2981da177e4SLinus Torvalds bit_map_clear(&iommu->usemap, ioptex, npages); 2991da177e4SLinus Torvalds } 3001da177e4SLinus Torvalds 3016c503d0dSChristoph Hellwig static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl, 3026c503d0dSChristoph Hellwig int nents, enum dma_data_direction dir, unsigned long attrs) 3031da177e4SLinus Torvalds { 3046c503d0dSChristoph Hellwig struct scatterlist *sg; 305a7fce1f7SChristoph Hellwig int i; 3061da177e4SLinus Torvalds 3076c503d0dSChristoph Hellwig for_each_sg(sgl, sg, nents, i) { 308a7fce1f7SChristoph Hellwig sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir, 309a7fce1f7SChristoph Hellwig attrs); 310aa83a26aSRobert Reif sg->dma_address = 0x21212121; 3111da177e4SLinus Torvalds } 3121da177e4SLinus Torvalds } 3131da177e4SLinus Torvalds 3141da177e4SLinus Torvalds #ifdef CONFIG_SBUS 315ce65d36fSChristoph Hellwig static void *sbus_iommu_alloc(struct device *dev, size_t len, 316ce65d36fSChristoph Hellwig dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 3171da177e4SLinus Torvalds { 3184b1c5df2SDavid S. Miller struct iommu_struct *iommu = dev->archdata.iommu; 319ce65d36fSChristoph Hellwig unsigned long va, addr, page, end, ret; 3201da177e4SLinus Torvalds iopte_t *iopte = iommu->page_table; 3211da177e4SLinus Torvalds iopte_t *first; 3221da177e4SLinus Torvalds int ioptex; 3231da177e4SLinus Torvalds 324ce65d36fSChristoph Hellwig /* XXX So what is maxphys for us and how do drivers know it? */ 325ce65d36fSChristoph Hellwig if (!len || len > 256 * 1024) 326ce65d36fSChristoph Hellwig return NULL; 327ce65d36fSChristoph Hellwig 328ce65d36fSChristoph Hellwig len = PAGE_ALIGN(len); 329518a2f19SChristoph Hellwig va = __get_free_pages(gfp | __GFP_ZERO, get_order(len)); 330ce65d36fSChristoph Hellwig if (va == 0) 331ce65d36fSChristoph Hellwig return NULL; 332ce65d36fSChristoph Hellwig 333ce65d36fSChristoph Hellwig addr = ret = sparc_dma_alloc_resource(dev, len); 334ce65d36fSChristoph Hellwig if (!addr) 335ce65d36fSChristoph Hellwig goto out_free_pages; 336ce65d36fSChristoph Hellwig 3371ae61388SEric Sesterhenn BUG_ON((va & ~PAGE_MASK) != 0); 3381ae61388SEric Sesterhenn BUG_ON((addr & ~PAGE_MASK) != 0); 3391ae61388SEric Sesterhenn BUG_ON((len & ~PAGE_MASK) != 0); 3401da177e4SLinus Torvalds 3411da177e4SLinus Torvalds /* page color = physical address */ 3421da177e4SLinus Torvalds ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT, 3431da177e4SLinus Torvalds addr >> PAGE_SHIFT); 3441da177e4SLinus Torvalds if (ioptex < 0) 3451da177e4SLinus Torvalds panic("iommu out"); 3461da177e4SLinus Torvalds 3471da177e4SLinus Torvalds iopte += ioptex; 3481da177e4SLinus Torvalds first = iopte; 3491da177e4SLinus Torvalds end = addr + len; 3501da177e4SLinus Torvalds while(addr < end) { 3511da177e4SLinus Torvalds page = va; 3521da177e4SLinus Torvalds { 3531da177e4SLinus Torvalds pgd_t *pgdp; 3547235db26SMike Rapoport p4d_t *p4dp; 3557235db26SMike Rapoport pud_t *pudp; 3561da177e4SLinus Torvalds pmd_t *pmdp; 3571da177e4SLinus Torvalds pte_t *ptep; 3581da177e4SLinus Torvalds 3591da177e4SLinus Torvalds if (viking_mxcc_present) 3601da177e4SLinus Torvalds viking_mxcc_flush_page(page); 3611da177e4SLinus Torvalds else if (viking_flush) 3621da177e4SLinus Torvalds viking_flush_page(page); 3631da177e4SLinus Torvalds else 3641da177e4SLinus Torvalds __flush_page_to_ram(page); 3651da177e4SLinus Torvalds 3661da177e4SLinus Torvalds pgdp = pgd_offset(&init_mm, addr); 3677235db26SMike Rapoport p4dp = p4d_offset(pgdp, addr); 3687235db26SMike Rapoport pudp = pud_offset(p4dp, addr); 3697235db26SMike Rapoport pmdp = pmd_offset(pudp, addr); 3701da177e4SLinus Torvalds ptep = pte_offset_map(pmdp, addr); 3711da177e4SLinus Torvalds 3721da177e4SLinus Torvalds set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); 3731da177e4SLinus Torvalds } 3741da177e4SLinus Torvalds iopte_val(*iopte++) = 3751da177e4SLinus Torvalds MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc); 3761da177e4SLinus Torvalds addr += PAGE_SIZE; 3771da177e4SLinus Torvalds va += PAGE_SIZE; 3781da177e4SLinus Torvalds } 3791da177e4SLinus Torvalds /* P3: why do we need this? 3801da177e4SLinus Torvalds * 3811da177e4SLinus Torvalds * DAVEM: Because there are several aspects, none of which 3821da177e4SLinus Torvalds * are handled by a single interface. Some cpus are 3831da177e4SLinus Torvalds * completely not I/O DMA coherent, and some have 3841da177e4SLinus Torvalds * virtually indexed caches. The driver DMA flushing 3851da177e4SLinus Torvalds * methods handle the former case, but here during 3861da177e4SLinus Torvalds * IOMMU page table modifications, and usage of non-cacheable 3871da177e4SLinus Torvalds * cpu mappings of pages potentially in the cpu caches, we have 3881da177e4SLinus Torvalds * to handle the latter case as well. 3891da177e4SLinus Torvalds */ 3901da177e4SLinus Torvalds flush_cache_all(); 3911da177e4SLinus Torvalds iommu_flush_iotlb(first, len >> PAGE_SHIFT); 3921da177e4SLinus Torvalds flush_tlb_all(); 3931da177e4SLinus Torvalds iommu_invalidate(iommu->regs); 3941da177e4SLinus Torvalds 395ce65d36fSChristoph Hellwig *dma_handle = iommu->start + (ioptex << PAGE_SHIFT); 396ce65d36fSChristoph Hellwig return (void *)ret; 397ce65d36fSChristoph Hellwig 398ce65d36fSChristoph Hellwig out_free_pages: 399ce65d36fSChristoph Hellwig free_pages(va, get_order(len)); 400ce65d36fSChristoph Hellwig return NULL; 4011da177e4SLinus Torvalds } 4021da177e4SLinus Torvalds 403ce65d36fSChristoph Hellwig static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr, 404ce65d36fSChristoph Hellwig dma_addr_t busa, unsigned long attrs) 4051da177e4SLinus Torvalds { 4064b1c5df2SDavid S. Miller struct iommu_struct *iommu = dev->archdata.iommu; 4071da177e4SLinus Torvalds iopte_t *iopte = iommu->page_table; 408ce65d36fSChristoph Hellwig struct page *page = virt_to_page(cpu_addr); 4091da177e4SLinus Torvalds int ioptex = (busa - iommu->start) >> PAGE_SHIFT; 410ce65d36fSChristoph Hellwig unsigned long end; 411ce65d36fSChristoph Hellwig 412ce65d36fSChristoph Hellwig if (!sparc_dma_free_resource(cpu_addr, len)) 413ce65d36fSChristoph Hellwig return; 4141da177e4SLinus Torvalds 4151ae61388SEric Sesterhenn BUG_ON((busa & ~PAGE_MASK) != 0); 4161ae61388SEric Sesterhenn BUG_ON((len & ~PAGE_MASK) != 0); 4171da177e4SLinus Torvalds 4181da177e4SLinus Torvalds iopte += ioptex; 4191da177e4SLinus Torvalds end = busa + len; 4201da177e4SLinus Torvalds while (busa < end) { 4211da177e4SLinus Torvalds iopte_val(*iopte++) = 0; 4221da177e4SLinus Torvalds busa += PAGE_SIZE; 4231da177e4SLinus Torvalds } 4241da177e4SLinus Torvalds flush_tlb_all(); 4251da177e4SLinus Torvalds iommu_invalidate(iommu->regs); 4261da177e4SLinus Torvalds bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT); 427ce65d36fSChristoph Hellwig 428ce65d36fSChristoph Hellwig __free_pages(page, get_order(len)); 4291da177e4SLinus Torvalds } 4301da177e4SLinus Torvalds #endif 4311da177e4SLinus Torvalds 432ce65d36fSChristoph Hellwig static const struct dma_map_ops sbus_iommu_dma_gflush_ops = { 433d894d964SDavid S. Miller #ifdef CONFIG_SBUS 434ce65d36fSChristoph Hellwig .alloc = sbus_iommu_alloc, 435ce65d36fSChristoph Hellwig .free = sbus_iommu_free, 436d894d964SDavid S. Miller #endif 437ce65d36fSChristoph Hellwig .map_page = sbus_iommu_map_page_gflush, 438ce65d36fSChristoph Hellwig .unmap_page = sbus_iommu_unmap_page, 439ce65d36fSChristoph Hellwig .map_sg = sbus_iommu_map_sg_gflush, 440ce65d36fSChristoph Hellwig .unmap_sg = sbus_iommu_unmap_sg, 441d894d964SDavid S. Miller }; 442d894d964SDavid S. Miller 443ce65d36fSChristoph Hellwig static const struct dma_map_ops sbus_iommu_dma_pflush_ops = { 444d894d964SDavid S. Miller #ifdef CONFIG_SBUS 445ce65d36fSChristoph Hellwig .alloc = sbus_iommu_alloc, 446ce65d36fSChristoph Hellwig .free = sbus_iommu_free, 447d894d964SDavid S. Miller #endif 448ce65d36fSChristoph Hellwig .map_page = sbus_iommu_map_page_pflush, 449ce65d36fSChristoph Hellwig .unmap_page = sbus_iommu_unmap_page, 450ce65d36fSChristoph Hellwig .map_sg = sbus_iommu_map_sg_pflush, 451ce65d36fSChristoph Hellwig .unmap_sg = sbus_iommu_unmap_sg, 452d894d964SDavid S. Miller }; 453d894d964SDavid S. Miller 4541da177e4SLinus Torvalds void __init ld_mmu_iommu(void) 4551da177e4SLinus Torvalds { 4561da177e4SLinus Torvalds if (viking_mxcc_present || srmmu_modtype == HyperSparc) { 4571da177e4SLinus Torvalds dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); 4581da177e4SLinus Torvalds ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID; 4591da177e4SLinus Torvalds } else { 4601da177e4SLinus Torvalds dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV); 4611da177e4SLinus Torvalds ioperm_noc = IOPTE_WRITE | IOPTE_VALID; 4621da177e4SLinus Torvalds } 4631da177e4SLinus Torvalds } 464