1f8f98a93SRussell King /* 2f8f98a93SRussell King * linux/arch/arm/lib/copypage-xscale.S 3f8f98a93SRussell King * 4f8f98a93SRussell King * Copyright (C) 1995-2005 Russell King 5f8f98a93SRussell King * 6f8f98a93SRussell King * This program is free software; you can redistribute it and/or modify 7f8f98a93SRussell King * it under the terms of the GNU General Public License version 2 as 8f8f98a93SRussell King * published by the Free Software Foundation. 9f8f98a93SRussell King * 10f8f98a93SRussell King * This handles the mini data cache, as found on SA11x0 and XScale 11f8f98a93SRussell King * processors. When we copy a user page page, we map it in such a way 12f8f98a93SRussell King * that accesses to this page will not touch the main data cache, but 13f8f98a93SRussell King * will be cached in the mini data cache. This prevents us thrashing 14f8f98a93SRussell King * the main data cache on page faults. 15f8f98a93SRussell King */ 16f8f98a93SRussell King #include <linux/init.h> 17f8f98a93SRussell King #include <linux/mm.h> 18063b0a42SRussell King #include <linux/highmem.h> 19f8f98a93SRussell King 20f8f98a93SRussell King #include <asm/pgtable.h> 21f8f98a93SRussell King #include <asm/tlbflush.h> 221c9d3df5SRichard Purdie #include <asm/cacheflush.h> 23f8f98a93SRussell King 241b2e2b73SRussell King #include "mm.h" 251b2e2b73SRussell King 26f8f98a93SRussell King #define minicache_pgprot __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | \ 27bb30f36fSRussell King L_PTE_MT_MINICACHE) 28f8f98a93SRussell King 29bd31b859SThomas Gleixner static DEFINE_RAW_SPINLOCK(minicache_lock); 30f8f98a93SRussell King 31f8f98a93SRussell King /* 32063b0a42SRussell King * XScale mini-dcache optimised copy_user_highpage 33f8f98a93SRussell King * 34f8f98a93SRussell King * We flush the destination cache lines just before we write the data into the 35f8f98a93SRussell King * corresponding address. Since the Dcache is read-allocate, this removes the 36f8f98a93SRussell King * Dcache aliasing issue. The writes will be forwarded to the write buffer, 37f8f98a93SRussell King * and merged as appropriate. 38f8f98a93SRussell King */ 39*b99afae1SNicolas Pitre static void mc_copy_user_page(void *from, void *to) 40f8f98a93SRussell King { 41*b99afae1SNicolas Pitre int tmp; 42*b99afae1SNicolas Pitre 43f8f98a93SRussell King /* 44f8f98a93SRussell King * Strangely enough, best performance is achieved 45f8f98a93SRussell King * when prefetching destination as well. (NP) 46f8f98a93SRussell King */ 47*b99afae1SNicolas Pitre asm volatile ("\ 48*b99afae1SNicolas Pitre pld [%0, #0] \n\ 49*b99afae1SNicolas Pitre pld [%0, #32] \n\ 50*b99afae1SNicolas Pitre pld [%1, #0] \n\ 51*b99afae1SNicolas Pitre pld [%1, #32] \n\ 52*b99afae1SNicolas Pitre 1: pld [%0, #64] \n\ 53*b99afae1SNicolas Pitre pld [%0, #96] \n\ 54*b99afae1SNicolas Pitre pld [%1, #64] \n\ 55*b99afae1SNicolas Pitre pld [%1, #96] \n\ 56*b99afae1SNicolas Pitre 2: ldrd r2, [%0], #8 \n\ 57*b99afae1SNicolas Pitre ldrd r4, [%0], #8 \n\ 58*b99afae1SNicolas Pitre mov ip, %1 \n\ 59*b99afae1SNicolas Pitre strd r2, [%1], #8 \n\ 60*b99afae1SNicolas Pitre ldrd r2, [%0], #8 \n\ 61*b99afae1SNicolas Pitre strd r4, [%1], #8 \n\ 62*b99afae1SNicolas Pitre ldrd r4, [%0], #8 \n\ 63*b99afae1SNicolas Pitre strd r2, [%1], #8 \n\ 64*b99afae1SNicolas Pitre strd r4, [%1], #8 \n\ 65f8f98a93SRussell King mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\ 66*b99afae1SNicolas Pitre ldrd r2, [%0], #8 \n\ 67f8f98a93SRussell King mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\ 68*b99afae1SNicolas Pitre ldrd r4, [%0], #8 \n\ 69*b99afae1SNicolas Pitre mov ip, %1 \n\ 70*b99afae1SNicolas Pitre strd r2, [%1], #8 \n\ 71*b99afae1SNicolas Pitre ldrd r2, [%0], #8 \n\ 72*b99afae1SNicolas Pitre strd r4, [%1], #8 \n\ 73*b99afae1SNicolas Pitre ldrd r4, [%0], #8 \n\ 74*b99afae1SNicolas Pitre strd r2, [%1], #8 \n\ 75*b99afae1SNicolas Pitre strd r4, [%1], #8 \n\ 76f8f98a93SRussell King mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\ 77*b99afae1SNicolas Pitre subs %2, %2, #1 \n\ 78f8f98a93SRussell King mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\ 79f8f98a93SRussell King bgt 1b \n\ 80*b99afae1SNicolas Pitre beq 2b " 81*b99afae1SNicolas Pitre : "+&r" (from), "+&r" (to), "=&r" (tmp) 82*b99afae1SNicolas Pitre : "2" (PAGE_SIZE / 64 - 1) 83*b99afae1SNicolas Pitre : "r2", "r3", "r4", "r5", "ip"); 84f8f98a93SRussell King } 85f8f98a93SRussell King 86063b0a42SRussell King void xscale_mc_copy_user_highpage(struct page *to, struct page *from, 87f00a75c0SRussell King unsigned long vaddr, struct vm_area_struct *vma) 88f8f98a93SRussell King { 895472e862SCong Wang void *kto = kmap_atomic(to); 901c9d3df5SRichard Purdie 91c0177800SCatalin Marinas if (!test_and_set_bit(PG_dcache_clean, &from->flags)) 92cb9f753aSHuang Ying __flush_dcache_page(page_mapping_file(from), from); 931c9d3df5SRichard Purdie 94bd31b859SThomas Gleixner raw_spin_lock(&minicache_lock); 95f8f98a93SRussell King 9667ece144SRussell King set_top_pte(COPYPAGE_MINICACHE, mk_pte(from, minicache_pgprot)); 97f8f98a93SRussell King 98f8f98a93SRussell King mc_copy_user_page((void *)COPYPAGE_MINICACHE, kto); 99f8f98a93SRussell King 100bd31b859SThomas Gleixner raw_spin_unlock(&minicache_lock); 101063b0a42SRussell King 1025472e862SCong Wang kunmap_atomic(kto); 103f8f98a93SRussell King } 104f8f98a93SRussell King 105f8f98a93SRussell King /* 106f8f98a93SRussell King * XScale optimised clear_user_page 107f8f98a93SRussell King */ 108303c6443SRussell King void 109303c6443SRussell King xscale_mc_clear_user_highpage(struct page *page, unsigned long vaddr) 110f8f98a93SRussell King { 1115472e862SCong Wang void *ptr, *kaddr = kmap_atomic(page); 112f8f98a93SRussell King asm volatile( 11343ae286bSNicolas Pitre "mov r1, %2 \n\ 114f8f98a93SRussell King mov r2, #0 \n\ 115f8f98a93SRussell King mov r3, #0 \n\ 116303c6443SRussell King 1: mov ip, %0 \n\ 117303c6443SRussell King strd r2, [%0], #8 \n\ 118303c6443SRussell King strd r2, [%0], #8 \n\ 119303c6443SRussell King strd r2, [%0], #8 \n\ 120303c6443SRussell King strd r2, [%0], #8 \n\ 121f8f98a93SRussell King mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\ 122f8f98a93SRussell King subs r1, r1, #1 \n\ 123f8f98a93SRussell King mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\ 124303c6443SRussell King bne 1b" 12543ae286bSNicolas Pitre : "=r" (ptr) 12643ae286bSNicolas Pitre : "0" (kaddr), "I" (PAGE_SIZE / 32) 127303c6443SRussell King : "r1", "r2", "r3", "ip"); 1285472e862SCong Wang kunmap_atomic(kaddr); 129f8f98a93SRussell King } 130f8f98a93SRussell King 131f8f98a93SRussell King struct cpu_user_fns xscale_mc_user_fns __initdata = { 132303c6443SRussell King .cpu_clear_user_highpage = xscale_mc_clear_user_highpage, 133063b0a42SRussell King .cpu_copy_user_highpage = xscale_mc_copy_user_highpage, 134f8f98a93SRussell King }; 135