11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * iommu.c: IOMMU specific routines for memory management. 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) 51da177e4SLinus Torvalds * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com) 61da177e4SLinus Torvalds * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) 71da177e4SLinus Torvalds * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 81da177e4SLinus Torvalds */ 91da177e4SLinus Torvalds 101da177e4SLinus Torvalds #include <linux/kernel.h> 111da177e4SLinus Torvalds #include <linux/init.h> 121da177e4SLinus Torvalds #include <linux/mm.h> 131da177e4SLinus Torvalds #include <linux/slab.h> 141da177e4SLinus Torvalds #include <linux/highmem.h> /* pte_offset_map => kmap_atomic */ 151da177e4SLinus Torvalds 161da177e4SLinus Torvalds #include <asm/scatterlist.h> 171da177e4SLinus Torvalds #include <asm/pgalloc.h> 181da177e4SLinus Torvalds #include <asm/pgtable.h> 191da177e4SLinus Torvalds #include <asm/sbus.h> 201da177e4SLinus Torvalds #include <asm/io.h> 211da177e4SLinus Torvalds #include <asm/mxcc.h> 221da177e4SLinus Torvalds #include <asm/mbus.h> 231da177e4SLinus Torvalds #include <asm/cacheflush.h> 241da177e4SLinus Torvalds #include <asm/tlbflush.h> 251da177e4SLinus Torvalds #include <asm/bitext.h> 261da177e4SLinus Torvalds #include <asm/iommu.h> 271da177e4SLinus Torvalds #include <asm/dma.h> 281da177e4SLinus Torvalds 291da177e4SLinus Torvalds /* 301da177e4SLinus Torvalds * This can be sized dynamically, but we will do this 311da177e4SLinus Torvalds * only when we have a guidance about actual I/O pressures. 321da177e4SLinus Torvalds */ 331da177e4SLinus Torvalds #define IOMMU_RNGE IOMMU_RNGE_256MB 341da177e4SLinus Torvalds #define IOMMU_START 0xF0000000 351da177e4SLinus Torvalds #define IOMMU_WINSIZE (256*1024*1024U) 361da177e4SLinus Torvalds #define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 265KB */ 371da177e4SLinus Torvalds #define IOMMU_ORDER 6 /* 4096 * (1<<6) */ 381da177e4SLinus Torvalds 391da177e4SLinus Torvalds /* srmmu.c */ 401da177e4SLinus Torvalds extern int viking_mxcc_present; 411da177e4SLinus Torvalds BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long) 421da177e4SLinus Torvalds #define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page) 431da177e4SLinus Torvalds extern int flush_page_for_dma_global; 441da177e4SLinus Torvalds static int viking_flush; 451da177e4SLinus Torvalds /* viking.S */ 461da177e4SLinus Torvalds extern void viking_flush_page(unsigned long page); 471da177e4SLinus Torvalds extern void viking_mxcc_flush_page(unsigned long page); 481da177e4SLinus Torvalds 491da177e4SLinus Torvalds /* 501da177e4SLinus Torvalds * Values precomputed according to CPU type. 511da177e4SLinus Torvalds */ 521da177e4SLinus Torvalds static unsigned int ioperm_noc; /* Consistent mapping iopte flags */ 531da177e4SLinus Torvalds static pgprot_t dvma_prot; /* Consistent mapping pte flags */ 541da177e4SLinus Torvalds 551da177e4SLinus Torvalds #define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID) 561da177e4SLinus Torvalds #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ) 571da177e4SLinus Torvalds 581da177e4SLinus Torvalds void __init 591da177e4SLinus Torvalds iommu_init(int iommund, struct sbus_bus *sbus) 601da177e4SLinus Torvalds { 611da177e4SLinus Torvalds unsigned int impl, vers; 621da177e4SLinus Torvalds unsigned long tmp; 631da177e4SLinus Torvalds struct iommu_struct *iommu; 641da177e4SLinus Torvalds struct linux_prom_registers iommu_promregs[PROMREG_MAX]; 651da177e4SLinus Torvalds struct resource r; 661da177e4SLinus Torvalds unsigned long *bitmap; 671da177e4SLinus Torvalds 681da177e4SLinus Torvalds iommu = kmalloc(sizeof(struct iommu_struct), GFP_ATOMIC); 691da177e4SLinus Torvalds if (!iommu) { 701da177e4SLinus Torvalds prom_printf("Unable to allocate iommu structure\n"); 711da177e4SLinus Torvalds prom_halt(); 721da177e4SLinus Torvalds } 731da177e4SLinus Torvalds iommu->regs = NULL; 741da177e4SLinus Torvalds if (prom_getproperty(iommund, "reg", (void *) iommu_promregs, 751da177e4SLinus Torvalds sizeof(iommu_promregs)) != -1) { 761da177e4SLinus Torvalds memset(&r, 0, sizeof(r)); 771da177e4SLinus Torvalds r.flags = iommu_promregs[0].which_io; 781da177e4SLinus Torvalds r.start = iommu_promregs[0].phys_addr; 791da177e4SLinus Torvalds iommu->regs = (struct iommu_regs *) 801da177e4SLinus Torvalds sbus_ioremap(&r, 0, PAGE_SIZE * 3, "iommu_regs"); 811da177e4SLinus Torvalds } 821da177e4SLinus Torvalds if (!iommu->regs) { 831da177e4SLinus Torvalds prom_printf("Cannot map IOMMU registers\n"); 841da177e4SLinus Torvalds prom_halt(); 851da177e4SLinus Torvalds } 861da177e4SLinus Torvalds impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28; 871da177e4SLinus Torvalds vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24; 881da177e4SLinus Torvalds tmp = iommu->regs->control; 891da177e4SLinus Torvalds tmp &= ~(IOMMU_CTRL_RNGE); 901da177e4SLinus Torvalds tmp |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); 911da177e4SLinus Torvalds iommu->regs->control = tmp; 921da177e4SLinus Torvalds iommu_invalidate(iommu->regs); 931da177e4SLinus Torvalds iommu->start = IOMMU_START; 941da177e4SLinus Torvalds iommu->end = 0xffffffff; 951da177e4SLinus Torvalds 961da177e4SLinus Torvalds /* Allocate IOMMU page table */ 971da177e4SLinus Torvalds /* Stupid alignment constraints give me a headache. 981da177e4SLinus Torvalds We need 256K or 512K or 1M or 2M area aligned to 991da177e4SLinus Torvalds its size and current gfp will fortunately give 1001da177e4SLinus Torvalds it to us. */ 1011da177e4SLinus Torvalds tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); 1021da177e4SLinus Torvalds if (!tmp) { 1031da177e4SLinus Torvalds prom_printf("Unable to allocate iommu table [0x%08x]\n", 1041da177e4SLinus Torvalds IOMMU_NPTES*sizeof(iopte_t)); 1051da177e4SLinus Torvalds prom_halt(); 1061da177e4SLinus Torvalds } 1071da177e4SLinus Torvalds iommu->page_table = (iopte_t *)tmp; 1081da177e4SLinus Torvalds 1091da177e4SLinus Torvalds /* Initialize new table. */ 1101da177e4SLinus Torvalds memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t)); 1111da177e4SLinus Torvalds flush_cache_all(); 1121da177e4SLinus Torvalds flush_tlb_all(); 1131da177e4SLinus Torvalds iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4; 1141da177e4SLinus Torvalds iommu_invalidate(iommu->regs); 1151da177e4SLinus Torvalds 1161da177e4SLinus Torvalds bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL); 1171da177e4SLinus Torvalds if (!bitmap) { 1181da177e4SLinus Torvalds prom_printf("Unable to allocate iommu bitmap [%d]\n", 1191da177e4SLinus Torvalds (int)(IOMMU_NPTES>>3)); 1201da177e4SLinus Torvalds prom_halt(); 1211da177e4SLinus Torvalds } 1221da177e4SLinus Torvalds bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES); 1231da177e4SLinus Torvalds /* To be coherent on HyperSparc, the page color of DVMA 1241da177e4SLinus Torvalds * and physical addresses must match. 1251da177e4SLinus Torvalds */ 1261da177e4SLinus Torvalds if (srmmu_modtype == HyperSparc) 1271da177e4SLinus Torvalds iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT; 1281da177e4SLinus Torvalds else 1291da177e4SLinus Torvalds iommu->usemap.num_colors = 1; 1301da177e4SLinus Torvalds 1311da177e4SLinus Torvalds printk("IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n", 1321da177e4SLinus Torvalds impl, vers, iommu->page_table, 1331da177e4SLinus Torvalds (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES); 1341da177e4SLinus Torvalds 135*3ac4c949SRobert Reif sbus->ofdev.dev.archdata.iommu = iommu; 1361da177e4SLinus Torvalds } 1371da177e4SLinus Torvalds 1381da177e4SLinus Torvalds /* This begs to be btfixup-ed by srmmu. */ 1391da177e4SLinus Torvalds /* Flush the iotlb entries to ram. */ 1401da177e4SLinus Torvalds /* This could be better if we didn't have to flush whole pages. */ 1411da177e4SLinus Torvalds static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) 1421da177e4SLinus Torvalds { 1431da177e4SLinus Torvalds unsigned long start; 1441da177e4SLinus Torvalds unsigned long end; 1451da177e4SLinus Torvalds 1463185d4d2SBob Breuer start = (unsigned long)iopte; 1471da177e4SLinus Torvalds end = PAGE_ALIGN(start + niopte*sizeof(iopte_t)); 1483185d4d2SBob Breuer start &= PAGE_MASK; 1491da177e4SLinus Torvalds if (viking_mxcc_present) { 1501da177e4SLinus Torvalds while(start < end) { 1511da177e4SLinus Torvalds viking_mxcc_flush_page(start); 1521da177e4SLinus Torvalds start += PAGE_SIZE; 1531da177e4SLinus Torvalds } 1541da177e4SLinus Torvalds } else if (viking_flush) { 1551da177e4SLinus Torvalds while(start < end) { 1561da177e4SLinus Torvalds viking_flush_page(start); 1571da177e4SLinus Torvalds start += PAGE_SIZE; 1581da177e4SLinus Torvalds } 1591da177e4SLinus Torvalds } else { 1601da177e4SLinus Torvalds while(start < end) { 1611da177e4SLinus Torvalds __flush_page_to_ram(start); 1621da177e4SLinus Torvalds start += PAGE_SIZE; 1631da177e4SLinus Torvalds } 1641da177e4SLinus Torvalds } 1651da177e4SLinus Torvalds } 1661da177e4SLinus Torvalds 1671da177e4SLinus Torvalds static u32 iommu_get_one(struct page *page, int npages, struct sbus_bus *sbus) 1681da177e4SLinus Torvalds { 169*3ac4c949SRobert Reif struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu; 1701da177e4SLinus Torvalds int ioptex; 1711da177e4SLinus Torvalds iopte_t *iopte, *iopte0; 1721da177e4SLinus Torvalds unsigned int busa, busa0; 1731da177e4SLinus Torvalds int i; 1741da177e4SLinus Torvalds 1751da177e4SLinus Torvalds /* page color = pfn of page */ 1761da177e4SLinus Torvalds ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page)); 1771da177e4SLinus Torvalds if (ioptex < 0) 1781da177e4SLinus Torvalds panic("iommu out"); 1791da177e4SLinus Torvalds busa0 = iommu->start + (ioptex << PAGE_SHIFT); 1801da177e4SLinus Torvalds iopte0 = &iommu->page_table[ioptex]; 1811da177e4SLinus Torvalds 1821da177e4SLinus Torvalds busa = busa0; 1831da177e4SLinus Torvalds iopte = iopte0; 1841da177e4SLinus Torvalds for (i = 0; i < npages; i++) { 1851da177e4SLinus Torvalds iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM); 1861da177e4SLinus Torvalds iommu_invalidate_page(iommu->regs, busa); 1871da177e4SLinus Torvalds busa += PAGE_SIZE; 1881da177e4SLinus Torvalds iopte++; 1891da177e4SLinus Torvalds page++; 1901da177e4SLinus Torvalds } 1911da177e4SLinus Torvalds 1921da177e4SLinus Torvalds iommu_flush_iotlb(iopte0, npages); 1931da177e4SLinus Torvalds 1941da177e4SLinus Torvalds return busa0; 1951da177e4SLinus Torvalds } 1961da177e4SLinus Torvalds 1971da177e4SLinus Torvalds static u32 iommu_get_scsi_one(char *vaddr, unsigned int len, 1981da177e4SLinus Torvalds struct sbus_bus *sbus) 1991da177e4SLinus Torvalds { 2001da177e4SLinus Torvalds unsigned long off; 2011da177e4SLinus Torvalds int npages; 2021da177e4SLinus Torvalds struct page *page; 2031da177e4SLinus Torvalds u32 busa; 2041da177e4SLinus Torvalds 2051da177e4SLinus Torvalds off = (unsigned long)vaddr & ~PAGE_MASK; 2061da177e4SLinus Torvalds npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; 2071da177e4SLinus Torvalds page = virt_to_page((unsigned long)vaddr & PAGE_MASK); 2081da177e4SLinus Torvalds busa = iommu_get_one(page, npages, sbus); 2091da177e4SLinus Torvalds return busa + off; 2101da177e4SLinus Torvalds } 2111da177e4SLinus Torvalds 2121da177e4SLinus Torvalds static __u32 iommu_get_scsi_one_noflush(char *vaddr, unsigned long len, struct sbus_bus *sbus) 2131da177e4SLinus Torvalds { 2141da177e4SLinus Torvalds return iommu_get_scsi_one(vaddr, len, sbus); 2151da177e4SLinus Torvalds } 2161da177e4SLinus Torvalds 2171da177e4SLinus Torvalds static __u32 iommu_get_scsi_one_gflush(char *vaddr, unsigned long len, struct sbus_bus *sbus) 2181da177e4SLinus Torvalds { 2191da177e4SLinus Torvalds flush_page_for_dma(0); 2201da177e4SLinus Torvalds return iommu_get_scsi_one(vaddr, len, sbus); 2211da177e4SLinus Torvalds } 2221da177e4SLinus Torvalds 2231da177e4SLinus Torvalds static __u32 iommu_get_scsi_one_pflush(char *vaddr, unsigned long len, struct sbus_bus *sbus) 2241da177e4SLinus Torvalds { 2251da177e4SLinus Torvalds unsigned long page = ((unsigned long) vaddr) & PAGE_MASK; 2261da177e4SLinus Torvalds 2271da177e4SLinus Torvalds while(page < ((unsigned long)(vaddr + len))) { 2281da177e4SLinus Torvalds flush_page_for_dma(page); 2291da177e4SLinus Torvalds page += PAGE_SIZE; 2301da177e4SLinus Torvalds } 2311da177e4SLinus Torvalds return iommu_get_scsi_one(vaddr, len, sbus); 2321da177e4SLinus Torvalds } 2331da177e4SLinus Torvalds 2341da177e4SLinus Torvalds static void iommu_get_scsi_sgl_noflush(struct scatterlist *sg, int sz, struct sbus_bus *sbus) 2351da177e4SLinus Torvalds { 2361da177e4SLinus Torvalds int n; 2371da177e4SLinus Torvalds 2381da177e4SLinus Torvalds while (sz != 0) { 2391da177e4SLinus Torvalds --sz; 2401da177e4SLinus Torvalds n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; 2411da177e4SLinus Torvalds sg->dvma_address = iommu_get_one(sg->page, n, sbus) + sg->offset; 2421da177e4SLinus Torvalds sg->dvma_length = (__u32) sg->length; 2431da177e4SLinus Torvalds sg++; 2441da177e4SLinus Torvalds } 2451da177e4SLinus Torvalds } 2461da177e4SLinus Torvalds 2471da177e4SLinus Torvalds static void iommu_get_scsi_sgl_gflush(struct scatterlist *sg, int sz, struct sbus_bus *sbus) 2481da177e4SLinus Torvalds { 2491da177e4SLinus Torvalds int n; 2501da177e4SLinus Torvalds 2511da177e4SLinus Torvalds flush_page_for_dma(0); 2521da177e4SLinus Torvalds while (sz != 0) { 2531da177e4SLinus Torvalds --sz; 2541da177e4SLinus Torvalds n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; 2551da177e4SLinus Torvalds sg->dvma_address = iommu_get_one(sg->page, n, sbus) + sg->offset; 2561da177e4SLinus Torvalds sg->dvma_length = (__u32) sg->length; 2571da177e4SLinus Torvalds sg++; 2581da177e4SLinus Torvalds } 2591da177e4SLinus Torvalds } 2601da177e4SLinus Torvalds 2611da177e4SLinus Torvalds static void iommu_get_scsi_sgl_pflush(struct scatterlist *sg, int sz, struct sbus_bus *sbus) 2621da177e4SLinus Torvalds { 2631da177e4SLinus Torvalds unsigned long page, oldpage = 0; 2641da177e4SLinus Torvalds int n, i; 2651da177e4SLinus Torvalds 2661da177e4SLinus Torvalds while(sz != 0) { 2671da177e4SLinus Torvalds --sz; 2681da177e4SLinus Torvalds 2691da177e4SLinus Torvalds n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; 2701da177e4SLinus Torvalds 2711da177e4SLinus Torvalds /* 2721da177e4SLinus Torvalds * We expect unmapped highmem pages to be not in the cache. 2731da177e4SLinus Torvalds * XXX Is this a good assumption? 2741da177e4SLinus Torvalds * XXX What if someone else unmaps it here and races us? 2751da177e4SLinus Torvalds */ 2761da177e4SLinus Torvalds if ((page = (unsigned long) page_address(sg->page)) != 0) { 2771da177e4SLinus Torvalds for (i = 0; i < n; i++) { 2781da177e4SLinus Torvalds if (page != oldpage) { /* Already flushed? */ 2791da177e4SLinus Torvalds flush_page_for_dma(page); 2801da177e4SLinus Torvalds oldpage = page; 2811da177e4SLinus Torvalds } 2821da177e4SLinus Torvalds page += PAGE_SIZE; 2831da177e4SLinus Torvalds } 2841da177e4SLinus Torvalds } 2851da177e4SLinus Torvalds 2861da177e4SLinus Torvalds sg->dvma_address = iommu_get_one(sg->page, n, sbus) + sg->offset; 2871da177e4SLinus Torvalds sg->dvma_length = (__u32) sg->length; 2881da177e4SLinus Torvalds sg++; 2891da177e4SLinus Torvalds } 2901da177e4SLinus Torvalds } 2911da177e4SLinus Torvalds 2921da177e4SLinus Torvalds static void iommu_release_one(u32 busa, int npages, struct sbus_bus *sbus) 2931da177e4SLinus Torvalds { 294*3ac4c949SRobert Reif struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu; 2951da177e4SLinus Torvalds int ioptex; 2961da177e4SLinus Torvalds int i; 2971da177e4SLinus Torvalds 2981ae61388SEric Sesterhenn BUG_ON(busa < iommu->start); 2991da177e4SLinus Torvalds ioptex = (busa - iommu->start) >> PAGE_SHIFT; 3001da177e4SLinus Torvalds for (i = 0; i < npages; i++) { 3011da177e4SLinus Torvalds iopte_val(iommu->page_table[ioptex + i]) = 0; 3021da177e4SLinus Torvalds iommu_invalidate_page(iommu->regs, busa); 3031da177e4SLinus Torvalds busa += PAGE_SIZE; 3041da177e4SLinus Torvalds } 3051da177e4SLinus Torvalds bit_map_clear(&iommu->usemap, ioptex, npages); 3061da177e4SLinus Torvalds } 3071da177e4SLinus Torvalds 3081da177e4SLinus Torvalds static void iommu_release_scsi_one(__u32 vaddr, unsigned long len, struct sbus_bus *sbus) 3091da177e4SLinus Torvalds { 3101da177e4SLinus Torvalds unsigned long off; 3111da177e4SLinus Torvalds int npages; 3121da177e4SLinus Torvalds 3131da177e4SLinus Torvalds off = vaddr & ~PAGE_MASK; 3141da177e4SLinus Torvalds npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; 3151da177e4SLinus Torvalds iommu_release_one(vaddr & PAGE_MASK, npages, sbus); 3161da177e4SLinus Torvalds } 3171da177e4SLinus Torvalds 3181da177e4SLinus Torvalds static void iommu_release_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_bus *sbus) 3191da177e4SLinus Torvalds { 3201da177e4SLinus Torvalds int n; 3211da177e4SLinus Torvalds 3221da177e4SLinus Torvalds while(sz != 0) { 3231da177e4SLinus Torvalds --sz; 3241da177e4SLinus Torvalds 3251da177e4SLinus Torvalds n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; 3261da177e4SLinus Torvalds iommu_release_one(sg->dvma_address & PAGE_MASK, n, sbus); 3271da177e4SLinus Torvalds sg->dvma_address = 0x21212121; 3281da177e4SLinus Torvalds sg++; 3291da177e4SLinus Torvalds } 3301da177e4SLinus Torvalds } 3311da177e4SLinus Torvalds 3321da177e4SLinus Torvalds #ifdef CONFIG_SBUS 3331da177e4SLinus Torvalds static int iommu_map_dma_area(dma_addr_t *pba, unsigned long va, 3341da177e4SLinus Torvalds unsigned long addr, int len) 3351da177e4SLinus Torvalds { 3361da177e4SLinus Torvalds unsigned long page, end; 337*3ac4c949SRobert Reif struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu; 3381da177e4SLinus Torvalds iopte_t *iopte = iommu->page_table; 3391da177e4SLinus Torvalds iopte_t *first; 3401da177e4SLinus Torvalds int ioptex; 3411da177e4SLinus Torvalds 3421ae61388SEric Sesterhenn BUG_ON((va & ~PAGE_MASK) != 0); 3431ae61388SEric Sesterhenn BUG_ON((addr & ~PAGE_MASK) != 0); 3441ae61388SEric Sesterhenn BUG_ON((len & ~PAGE_MASK) != 0); 3451da177e4SLinus Torvalds 3461da177e4SLinus Torvalds /* page color = physical address */ 3471da177e4SLinus Torvalds ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT, 3481da177e4SLinus Torvalds addr >> PAGE_SHIFT); 3491da177e4SLinus Torvalds if (ioptex < 0) 3501da177e4SLinus Torvalds panic("iommu out"); 3511da177e4SLinus Torvalds 3521da177e4SLinus Torvalds iopte += ioptex; 3531da177e4SLinus Torvalds first = iopte; 3541da177e4SLinus Torvalds end = addr + len; 3551da177e4SLinus Torvalds while(addr < end) { 3561da177e4SLinus Torvalds page = va; 3571da177e4SLinus Torvalds { 3581da177e4SLinus Torvalds pgd_t *pgdp; 3591da177e4SLinus Torvalds pmd_t *pmdp; 3601da177e4SLinus Torvalds pte_t *ptep; 3611da177e4SLinus Torvalds 3621da177e4SLinus Torvalds if (viking_mxcc_present) 3631da177e4SLinus Torvalds viking_mxcc_flush_page(page); 3641da177e4SLinus Torvalds else if (viking_flush) 3651da177e4SLinus Torvalds viking_flush_page(page); 3661da177e4SLinus Torvalds else 3671da177e4SLinus Torvalds __flush_page_to_ram(page); 3681da177e4SLinus Torvalds 3691da177e4SLinus Torvalds pgdp = pgd_offset(&init_mm, addr); 3701da177e4SLinus Torvalds pmdp = pmd_offset(pgdp, addr); 3711da177e4SLinus Torvalds ptep = pte_offset_map(pmdp, addr); 3721da177e4SLinus Torvalds 3731da177e4SLinus Torvalds set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); 3741da177e4SLinus Torvalds } 3751da177e4SLinus Torvalds iopte_val(*iopte++) = 3761da177e4SLinus Torvalds MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc); 3771da177e4SLinus Torvalds addr += PAGE_SIZE; 3781da177e4SLinus Torvalds va += PAGE_SIZE; 3791da177e4SLinus Torvalds } 3801da177e4SLinus Torvalds /* P3: why do we need this? 3811da177e4SLinus Torvalds * 3821da177e4SLinus Torvalds * DAVEM: Because there are several aspects, none of which 3831da177e4SLinus Torvalds * are handled by a single interface. Some cpus are 3841da177e4SLinus Torvalds * completely not I/O DMA coherent, and some have 3851da177e4SLinus Torvalds * virtually indexed caches. The driver DMA flushing 3861da177e4SLinus Torvalds * methods handle the former case, but here during 3871da177e4SLinus Torvalds * IOMMU page table modifications, and usage of non-cacheable 3881da177e4SLinus Torvalds * cpu mappings of pages potentially in the cpu caches, we have 3891da177e4SLinus Torvalds * to handle the latter case as well. 3901da177e4SLinus Torvalds */ 3911da177e4SLinus Torvalds flush_cache_all(); 3921da177e4SLinus Torvalds iommu_flush_iotlb(first, len >> PAGE_SHIFT); 3931da177e4SLinus Torvalds flush_tlb_all(); 3941da177e4SLinus Torvalds iommu_invalidate(iommu->regs); 3951da177e4SLinus Torvalds 3961da177e4SLinus Torvalds *pba = iommu->start + (ioptex << PAGE_SHIFT); 3971da177e4SLinus Torvalds return 0; 3981da177e4SLinus Torvalds } 3991da177e4SLinus Torvalds 4001da177e4SLinus Torvalds static void iommu_unmap_dma_area(unsigned long busa, int len) 4011da177e4SLinus Torvalds { 402*3ac4c949SRobert Reif struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu; 4031da177e4SLinus Torvalds iopte_t *iopte = iommu->page_table; 4041da177e4SLinus Torvalds unsigned long end; 4051da177e4SLinus Torvalds int ioptex = (busa - iommu->start) >> PAGE_SHIFT; 4061da177e4SLinus Torvalds 4071ae61388SEric Sesterhenn BUG_ON((busa & ~PAGE_MASK) != 0); 4081ae61388SEric Sesterhenn BUG_ON((len & ~PAGE_MASK) != 0); 4091da177e4SLinus Torvalds 4101da177e4SLinus Torvalds iopte += ioptex; 4111da177e4SLinus Torvalds end = busa + len; 4121da177e4SLinus Torvalds while (busa < end) { 4131da177e4SLinus Torvalds iopte_val(*iopte++) = 0; 4141da177e4SLinus Torvalds busa += PAGE_SIZE; 4151da177e4SLinus Torvalds } 4161da177e4SLinus Torvalds flush_tlb_all(); 4171da177e4SLinus Torvalds iommu_invalidate(iommu->regs); 4181da177e4SLinus Torvalds bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT); 4191da177e4SLinus Torvalds } 4201da177e4SLinus Torvalds 4211da177e4SLinus Torvalds static struct page *iommu_translate_dvma(unsigned long busa) 4221da177e4SLinus Torvalds { 423*3ac4c949SRobert Reif struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu; 4241da177e4SLinus Torvalds iopte_t *iopte = iommu->page_table; 4251da177e4SLinus Torvalds 4261da177e4SLinus Torvalds iopte += ((busa - iommu->start) >> PAGE_SHIFT); 4271da177e4SLinus Torvalds return pfn_to_page((iopte_val(*iopte) & IOPTE_PAGE) >> (PAGE_SHIFT-4)); 4281da177e4SLinus Torvalds } 4291da177e4SLinus Torvalds #endif 4301da177e4SLinus Torvalds 4311da177e4SLinus Torvalds static char *iommu_lockarea(char *vaddr, unsigned long len) 4321da177e4SLinus Torvalds { 4331da177e4SLinus Torvalds return vaddr; 4341da177e4SLinus Torvalds } 4351da177e4SLinus Torvalds 4361da177e4SLinus Torvalds static void iommu_unlockarea(char *vaddr, unsigned long len) 4371da177e4SLinus Torvalds { 4381da177e4SLinus Torvalds } 4391da177e4SLinus Torvalds 4401da177e4SLinus Torvalds void __init ld_mmu_iommu(void) 4411da177e4SLinus Torvalds { 4421da177e4SLinus Torvalds viking_flush = (BTFIXUPVAL_CALL(flush_page_for_dma) == (unsigned long)viking_flush_page); 4431da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_lockarea, iommu_lockarea, BTFIXUPCALL_RETO0); 4441da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_unlockarea, iommu_unlockarea, BTFIXUPCALL_NOP); 4451da177e4SLinus Torvalds 4461da177e4SLinus Torvalds if (!BTFIXUPVAL_CALL(flush_page_for_dma)) { 4471da177e4SLinus Torvalds /* IO coherent chip */ 4481da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_noflush, BTFIXUPCALL_RETO0); 4491da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_noflush, BTFIXUPCALL_NORM); 4501da177e4SLinus Torvalds } else if (flush_page_for_dma_global) { 4511da177e4SLinus Torvalds /* flush_page_for_dma flushes everything, no matter of what page is it */ 4521da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_gflush, BTFIXUPCALL_NORM); 4531da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_gflush, BTFIXUPCALL_NORM); 4541da177e4SLinus Torvalds } else { 4551da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_pflush, BTFIXUPCALL_NORM); 4561da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_pflush, BTFIXUPCALL_NORM); 4571da177e4SLinus Torvalds } 4581da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_release_scsi_one, iommu_release_scsi_one, BTFIXUPCALL_NORM); 4591da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_release_scsi_sgl, iommu_release_scsi_sgl, BTFIXUPCALL_NORM); 4601da177e4SLinus Torvalds 4611da177e4SLinus Torvalds #ifdef CONFIG_SBUS 4621da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_map_dma_area, iommu_map_dma_area, BTFIXUPCALL_NORM); 4631da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_unmap_dma_area, iommu_unmap_dma_area, BTFIXUPCALL_NORM); 4641da177e4SLinus Torvalds BTFIXUPSET_CALL(mmu_translate_dvma, iommu_translate_dvma, BTFIXUPCALL_NORM); 4651da177e4SLinus Torvalds #endif 4661da177e4SLinus Torvalds 4671da177e4SLinus Torvalds if (viking_mxcc_present || srmmu_modtype == HyperSparc) { 4681da177e4SLinus Torvalds dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); 4691da177e4SLinus Torvalds ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID; 4701da177e4SLinus Torvalds } else { 4711da177e4SLinus Torvalds dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV); 4721da177e4SLinus Torvalds ioperm_noc = IOPTE_WRITE | IOPTE_VALID; 4731da177e4SLinus Torvalds } 4741da177e4SLinus Torvalds } 475