11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * This file is subject to the terms and conditions of the GNU General Public 31da177e4SLinus Torvalds * License. See the file "COPYING" in the main directory of this archive 41da177e4SLinus Torvalds * for more details. 51da177e4SLinus Torvalds * 6641e97f3SRalf Baechle * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) 77575a49fSRalf Baechle * Copyright (C) 2007 MIPS Technologies, Inc. 81da177e4SLinus Torvalds */ 924e9d0b9SRalf Baechle #include <linux/fs.h> 1024e9d0b9SRalf Baechle #include <linux/fcntl.h> 111da177e4SLinus Torvalds #include <linux/kernel.h> 12641e97f3SRalf Baechle #include <linux/linkage.h> 13d9ba5778SPaul Gortmaker #include <linux/export.h> 141da177e4SLinus Torvalds #include <linux/sched.h> 15dbda6ac0SRalf Baechle #include <linux/syscalls.h> 161da177e4SLinus Torvalds #include <linux/mm.h> 1778b6d91eSIra Weiny #include <linux/highmem.h> 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <asm/cacheflush.h> 201da177e4SLinus Torvalds #include <asm/processor.h> 211da177e4SLinus Torvalds #include <asm/cpu.h> 221da177e4SLinus Torvalds #include <asm/cpu-features.h> 2369939424SPaul Burton #include <asm/setup.h> 241da177e4SLinus Torvalds 251da177e4SLinus Torvalds /* Cache operations. */ 261da177e4SLinus Torvalds void (*flush_cache_all)(void); 271da177e4SLinus Torvalds void (*__flush_cache_all)(void); 284fa9de5aSJames Hogan EXPORT_SYMBOL_GPL(__flush_cache_all); 291da177e4SLinus Torvalds void (*flush_cache_mm)(struct mm_struct *mm); 301da177e4SLinus Torvalds void (*flush_cache_range)(struct vm_area_struct *vma, unsigned long start, 311da177e4SLinus Torvalds unsigned long end); 3253de0d47SRalf Baechle void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page, 3353de0d47SRalf Baechle unsigned long pfn); 34d4264f18SAtsushi Nemoto void (*flush_icache_range)(unsigned long start, unsigned long end); 358229f1a0SKees Cook EXPORT_SYMBOL_GPL(flush_icache_range); 36e0cee3eeSThomas Bogendoerfer void (*local_flush_icache_range)(unsigned long start, unsigned long end); 3790f91356SJames Hogan EXPORT_SYMBOL_GPL(local_flush_icache_range); 3801882b4dSJames Hogan void (*__flush_icache_user_range)(unsigned long start, unsigned long end); 3901882b4dSJames Hogan void (*__local_flush_icache_user_range)(unsigned long start, unsigned long end); 4001882b4dSJames Hogan EXPORT_SYMBOL_GPL(__local_flush_icache_user_range); 411da177e4SLinus Torvalds 429c5a3d72SRalf Baechle void (*__flush_cache_vmap)(void); 439c5a3d72SRalf Baechle void (*__flush_cache_vunmap)(void); 449c5a3d72SRalf Baechle 45d9cdc901SRalf Baechle void (*__flush_kernel_vmap_range)(unsigned long vaddr, int size); 46d9cdc901SRalf Baechle EXPORT_SYMBOL_GPL(__flush_kernel_vmap_range); 47d9cdc901SRalf Baechle 481da177e4SLinus Torvalds /* MIPS specific cache operations */ 497e3bfc7cSRalf Baechle void (*local_flush_data_cache_page)(void * addr); 501da177e4SLinus Torvalds void (*flush_data_cache_page)(unsigned long addr); 511da177e4SLinus Torvalds void (*flush_icache_all)(void); 521da177e4SLinus Torvalds 539202f325SRalf Baechle EXPORT_SYMBOL_GPL(local_flush_data_cache_page); 549ff77c46SRalf Baechle EXPORT_SYMBOL(flush_data_cache_page); 55f2e3656dSSanjay Lal EXPORT_SYMBOL(flush_icache_all); 569ff77c46SRalf Baechle 57972dc3b7SChristoph Hellwig #ifdef CONFIG_DMA_NONCOHERENT 581da177e4SLinus Torvalds 591da177e4SLinus Torvalds /* DMA cache operations. */ 601da177e4SLinus Torvalds void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size); 611da177e4SLinus Torvalds void (*_dma_cache_wback)(unsigned long start, unsigned long size); 621da177e4SLinus Torvalds void (*_dma_cache_inv)(unsigned long start, unsigned long size); 631da177e4SLinus Torvalds 64972dc3b7SChristoph Hellwig #endif /* CONFIG_DMA_NONCOHERENT */ 651da177e4SLinus Torvalds 661da177e4SLinus Torvalds /* 671da177e4SLinus Torvalds * We could optimize the case where the cache argument is not BCACHE but 681da177e4SLinus Torvalds * that seems very atypical use ... 691da177e4SLinus Torvalds */ 70dbda6ac0SRalf Baechle SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes, 71dbda6ac0SRalf Baechle unsigned int, cache) 721da177e4SLinus Torvalds { 73750ccf68SAtsushi Nemoto if (bytes == 0) 74750ccf68SAtsushi Nemoto return 0; 7596d4f267SLinus Torvalds if (!access_ok((void __user *) addr, bytes)) 761da177e4SLinus Torvalds return -EFAULT; 771da177e4SLinus Torvalds 788e3a9f4cSJames Hogan __flush_icache_user_range(addr, addr + bytes); 791da177e4SLinus Torvalds 801da177e4SLinus Torvalds return 0; 811da177e4SLinus Torvalds } 821da177e4SLinus Torvalds 831da177e4SLinus Torvalds void __flush_dcache_page(struct page *page) 841da177e4SLinus Torvalds { 85cb9f753aSHuang Ying struct address_space *mapping = page_mapping_file(page); 861da177e4SLinus Torvalds unsigned long addr; 871da177e4SLinus Torvalds 881da177e4SLinus Torvalds if (mapping && !mapping_mapped(mapping)) { 891da177e4SLinus Torvalds SetPageDcacheDirty(page); 901da177e4SLinus Torvalds return; 911da177e4SLinus Torvalds } 921da177e4SLinus Torvalds 931da177e4SLinus Torvalds /* 941da177e4SLinus Torvalds * We could delay the flush for the !page_mapping case too. But that 951da177e4SLinus Torvalds * case is for exec env/arg pages and those are %99 certainly going to 961da177e4SLinus Torvalds * get faulted into the tlb (and thus flushed) anyways. 971da177e4SLinus Torvalds */ 98234859e4SPaul Burton if (PageHighMem(page)) 99234859e4SPaul Burton addr = (unsigned long)kmap_atomic(page); 100234859e4SPaul Burton else 1011da177e4SLinus Torvalds addr = (unsigned long)page_address(page); 102234859e4SPaul Burton 1031da177e4SLinus Torvalds flush_data_cache_page(addr); 104234859e4SPaul Burton 105234859e4SPaul Burton if (PageHighMem(page)) 106abca2500SIra Weiny kunmap_atomic((void *)addr); 1071da177e4SLinus Torvalds } 1081da177e4SLinus Torvalds 1091da177e4SLinus Torvalds EXPORT_SYMBOL(__flush_dcache_page); 1101da177e4SLinus Torvalds 1117575a49fSRalf Baechle void __flush_anon_page(struct page *page, unsigned long vmaddr) 1127575a49fSRalf Baechle { 1139a74b3ebSRalf Baechle unsigned long addr = (unsigned long) page_address(page); 1149a74b3ebSRalf Baechle 1159a74b3ebSRalf Baechle if (pages_do_alias(addr, vmaddr)) { 116e1534ae9SKirill A. Shutemov if (page_mapcount(page) && !Page_dcache_dirty(page)) { 1177575a49fSRalf Baechle void *kaddr; 1187575a49fSRalf Baechle 1197575a49fSRalf Baechle kaddr = kmap_coherent(page, vmaddr); 1207575a49fSRalf Baechle flush_data_cache_page((unsigned long)kaddr); 121eacb9d61SRalf Baechle kunmap_coherent(); 1229a74b3ebSRalf Baechle } else 1239a74b3ebSRalf Baechle flush_data_cache_page(addr); 1247575a49fSRalf Baechle } 1257575a49fSRalf Baechle } 1267575a49fSRalf Baechle 1277575a49fSRalf Baechle EXPORT_SYMBOL(__flush_anon_page); 1287575a49fSRalf Baechle 12937d22a0dSPaul Burton void __update_cache(unsigned long address, pte_t pte) 1301da177e4SLinus Torvalds { 1311da177e4SLinus Torvalds struct page *page; 1325b9593f3SLars Persson unsigned long pfn, addr; 13337d22a0dSPaul Burton int exec = !pte_no_exec(pte) && !cpu_has_ic_fills_f_dc; 1341da177e4SLinus Torvalds 1355b9593f3SLars Persson pfn = pte_pfn(pte); 136585fa724SRalf Baechle if (unlikely(!pfn_valid(pfn))) 137585fa724SRalf Baechle return; 138585fa724SRalf Baechle page = pfn_to_page(pfn); 13937d22a0dSPaul Burton if (Page_dcache_dirty(page)) { 140f4281bbaSPaul Burton if (PageHighMem(page)) 141f4281bbaSPaul Burton addr = (unsigned long)kmap_atomic(page); 142f4281bbaSPaul Burton else 1435b9593f3SLars Persson addr = (unsigned long)page_address(page); 144f4281bbaSPaul Burton 1455b9593f3SLars Persson if (exec || pages_do_alias(addr, address & PAGE_MASK)) 1465b9593f3SLars Persson flush_data_cache_page(addr); 147f4281bbaSPaul Burton 148f4281bbaSPaul Burton if (PageHighMem(page)) 149abca2500SIra Weiny kunmap_atomic((void *)addr); 150f4281bbaSPaul Burton 1511da177e4SLinus Torvalds ClearPageDcacheDirty(page); 1521da177e4SLinus Torvalds } 1531da177e4SLinus Torvalds } 1541da177e4SLinus Torvalds 15535133692SChris Dearman unsigned long _page_cachable_default; 1567b3e543dSAnton Altaparmakov EXPORT_SYMBOL(_page_cachable_default); 15735133692SChris Dearman 158*ed2adb74SThomas Bogendoerfer #define PM(p) __pgprot(_page_cachable_default | (p)) 159*ed2adb74SThomas Bogendoerfer 16035133692SChris Dearman static inline void setup_protection_map(void) 16135133692SChris Dearman { 16205857c64SSteven J. Hill if (cpu_has_rixi) { 163*ed2adb74SThomas Bogendoerfer protection_map[0] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); 164*ed2adb74SThomas Bogendoerfer protection_map[1] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC); 165*ed2adb74SThomas Bogendoerfer protection_map[2] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); 166*ed2adb74SThomas Bogendoerfer protection_map[3] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC); 167*ed2adb74SThomas Bogendoerfer protection_map[4] = PM(_PAGE_PRESENT); 168*ed2adb74SThomas Bogendoerfer protection_map[5] = PM(_PAGE_PRESENT); 169*ed2adb74SThomas Bogendoerfer protection_map[6] = PM(_PAGE_PRESENT); 170*ed2adb74SThomas Bogendoerfer protection_map[7] = PM(_PAGE_PRESENT); 1716dd9344cSDavid Daney 172*ed2adb74SThomas Bogendoerfer protection_map[8] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); 173*ed2adb74SThomas Bogendoerfer protection_map[9] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC); 174*ed2adb74SThomas Bogendoerfer protection_map[10] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE | _PAGE_NO_READ); 175*ed2adb74SThomas Bogendoerfer protection_map[11] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE); 176*ed2adb74SThomas Bogendoerfer protection_map[12] = PM(_PAGE_PRESENT); 177*ed2adb74SThomas Bogendoerfer protection_map[13] = PM(_PAGE_PRESENT); 178*ed2adb74SThomas Bogendoerfer protection_map[14] = PM(_PAGE_PRESENT | _PAGE_WRITE); 179*ed2adb74SThomas Bogendoerfer protection_map[15] = PM(_PAGE_PRESENT | _PAGE_WRITE); 1806dd9344cSDavid Daney 1816dd9344cSDavid Daney } else { 18235133692SChris Dearman protection_map[0] = PAGE_NONE; 18335133692SChris Dearman protection_map[1] = PAGE_READONLY; 18435133692SChris Dearman protection_map[2] = PAGE_COPY; 18535133692SChris Dearman protection_map[3] = PAGE_COPY; 18635133692SChris Dearman protection_map[4] = PAGE_READONLY; 18735133692SChris Dearman protection_map[5] = PAGE_READONLY; 18835133692SChris Dearman protection_map[6] = PAGE_COPY; 18935133692SChris Dearman protection_map[7] = PAGE_COPY; 19035133692SChris Dearman protection_map[8] = PAGE_NONE; 19135133692SChris Dearman protection_map[9] = PAGE_READONLY; 19235133692SChris Dearman protection_map[10] = PAGE_SHARED; 19335133692SChris Dearman protection_map[11] = PAGE_SHARED; 19435133692SChris Dearman protection_map[12] = PAGE_READONLY; 19535133692SChris Dearman protection_map[13] = PAGE_READONLY; 19635133692SChris Dearman protection_map[14] = PAGE_SHARED; 19735133692SChris Dearman protection_map[15] = PAGE_SHARED; 19835133692SChris Dearman } 1996dd9344cSDavid Daney } 2001da177e4SLinus Torvalds 201*ed2adb74SThomas Bogendoerfer #undef PM 202*ed2adb74SThomas Bogendoerfer 203078a55fcSPaul Gortmaker void cpu_cache_init(void) 2041da177e4SLinus Torvalds { 20502cf2119SRalf Baechle if (cpu_has_3k_cache) { 20602cf2119SRalf Baechle extern void __weak r3k_cache_init(void); 2071da177e4SLinus Torvalds 20802cf2119SRalf Baechle r3k_cache_init(); 2091da177e4SLinus Torvalds } 21002cf2119SRalf Baechle if (cpu_has_6k_cache) { 21102cf2119SRalf Baechle extern void __weak r6k_cache_init(void); 21202cf2119SRalf Baechle 21302cf2119SRalf Baechle r6k_cache_init(); 21402cf2119SRalf Baechle } 21502cf2119SRalf Baechle if (cpu_has_4k_cache) { 21602cf2119SRalf Baechle extern void __weak r4k_cache_init(void); 21702cf2119SRalf Baechle 21802cf2119SRalf Baechle r4k_cache_init(); 21902cf2119SRalf Baechle } 22002cf2119SRalf Baechle if (cpu_has_8k_cache) { 22102cf2119SRalf Baechle extern void __weak r8k_cache_init(void); 22202cf2119SRalf Baechle 22302cf2119SRalf Baechle r8k_cache_init(); 22402cf2119SRalf Baechle } 22502cf2119SRalf Baechle if (cpu_has_tx39_cache) { 22602cf2119SRalf Baechle extern void __weak tx39_cache_init(void); 22702cf2119SRalf Baechle 22802cf2119SRalf Baechle tx39_cache_init(); 22902cf2119SRalf Baechle } 23002cf2119SRalf Baechle 23147d979ecSDavid Daney if (cpu_has_octeon_cache) { 23247d979ecSDavid Daney extern void __weak octeon_cache_init(void); 23347d979ecSDavid Daney 23447d979ecSDavid Daney octeon_cache_init(); 23547d979ecSDavid Daney } 23647d979ecSDavid Daney 23735133692SChris Dearman setup_protection_map(); 2381da177e4SLinus Torvalds } 23924e9d0b9SRalf Baechle 24024e9d0b9SRalf Baechle int __weak __uncached_access(struct file *file, unsigned long addr) 24124e9d0b9SRalf Baechle { 2426b2f3d1fSChristoph Hellwig if (file->f_flags & O_DSYNC) 24324e9d0b9SRalf Baechle return 1; 24424e9d0b9SRalf Baechle 24524e9d0b9SRalf Baechle return addr >= __pa(high_memory); 24624e9d0b9SRalf Baechle } 247