1f8f98a93SRussell King /* 2f8f98a93SRussell King * linux/arch/arm/lib/copypage-xscale.S 3f8f98a93SRussell King * 4f8f98a93SRussell King * Copyright (C) 1995-2005 Russell King 5f8f98a93SRussell King * 6f8f98a93SRussell King * This program is free software; you can redistribute it and/or modify 7f8f98a93SRussell King * it under the terms of the GNU General Public License version 2 as 8f8f98a93SRussell King * published by the Free Software Foundation. 9f8f98a93SRussell King * 10f8f98a93SRussell King * This handles the mini data cache, as found on SA11x0 and XScale 11f8f98a93SRussell King * processors. When we copy a user page page, we map it in such a way 12f8f98a93SRussell King * that accesses to this page will not touch the main data cache, but 13f8f98a93SRussell King * will be cached in the mini data cache. This prevents us thrashing 14f8f98a93SRussell King * the main data cache on page faults. 15f8f98a93SRussell King */ 16f8f98a93SRussell King #include <linux/init.h> 17f8f98a93SRussell King #include <linux/mm.h> 18063b0a42SRussell King #include <linux/highmem.h> 19f8f98a93SRussell King 20f8f98a93SRussell King #include <asm/pgtable.h> 21f8f98a93SRussell King #include <asm/tlbflush.h> 221c9d3df5SRichard Purdie #include <asm/cacheflush.h> 23f8f98a93SRussell King 241b2e2b73SRussell King #include "mm.h" 251b2e2b73SRussell King 26f8f98a93SRussell King #define minicache_pgprot __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | \ 27bb30f36fSRussell King L_PTE_MT_MINICACHE) 28f8f98a93SRussell King 29bd31b859SThomas Gleixner static DEFINE_RAW_SPINLOCK(minicache_lock); 30f8f98a93SRussell King 31f8f98a93SRussell King /* 32063b0a42SRussell King * XScale mini-dcache optimised copy_user_highpage 33f8f98a93SRussell King * 34f8f98a93SRussell King * We flush the destination cache lines just before we write the data into the 35f8f98a93SRussell King * corresponding address. Since the Dcache is read-allocate, this removes the 36f8f98a93SRussell King * Dcache aliasing issue. The writes will be forwarded to the write buffer, 37f8f98a93SRussell King * and merged as appropriate. 38f8f98a93SRussell King */ 39446c92b2SUwe Kleine-König static void __naked 40f8f98a93SRussell King mc_copy_user_page(void *from, void *to) 41f8f98a93SRussell King { 42f8f98a93SRussell King /* 43f8f98a93SRussell King * Strangely enough, best performance is achieved 44f8f98a93SRussell King * when prefetching destination as well. (NP) 45f8f98a93SRussell King */ 46f8f98a93SRussell King asm volatile( 47f8f98a93SRussell King "stmfd sp!, {r4, r5, lr} \n\ 48f8f98a93SRussell King mov lr, %2 \n\ 49f8f98a93SRussell King pld [r0, #0] \n\ 50f8f98a93SRussell King pld [r0, #32] \n\ 51f8f98a93SRussell King pld [r1, #0] \n\ 52f8f98a93SRussell King pld [r1, #32] \n\ 53f8f98a93SRussell King 1: pld [r0, #64] \n\ 54f8f98a93SRussell King pld [r0, #96] \n\ 55f8f98a93SRussell King pld [r1, #64] \n\ 56f8f98a93SRussell King pld [r1, #96] \n\ 57f8f98a93SRussell King 2: ldrd r2, [r0], #8 \n\ 58f8f98a93SRussell King ldrd r4, [r0], #8 \n\ 59f8f98a93SRussell King mov ip, r1 \n\ 60f8f98a93SRussell King strd r2, [r1], #8 \n\ 61f8f98a93SRussell King ldrd r2, [r0], #8 \n\ 62f8f98a93SRussell King strd r4, [r1], #8 \n\ 63f8f98a93SRussell King ldrd r4, [r0], #8 \n\ 64f8f98a93SRussell King strd r2, [r1], #8 \n\ 65f8f98a93SRussell King strd r4, [r1], #8 \n\ 66f8f98a93SRussell King mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\ 67f8f98a93SRussell King ldrd r2, [r0], #8 \n\ 68f8f98a93SRussell King mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\ 69f8f98a93SRussell King ldrd r4, [r0], #8 \n\ 70f8f98a93SRussell King mov ip, r1 \n\ 71f8f98a93SRussell King strd r2, [r1], #8 \n\ 72f8f98a93SRussell King ldrd r2, [r0], #8 \n\ 73f8f98a93SRussell King strd r4, [r1], #8 \n\ 74f8f98a93SRussell King ldrd r4, [r0], #8 \n\ 75f8f98a93SRussell King strd r2, [r1], #8 \n\ 76f8f98a93SRussell King strd r4, [r1], #8 \n\ 77f8f98a93SRussell King mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\ 78f8f98a93SRussell King subs lr, lr, #1 \n\ 79f8f98a93SRussell King mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\ 80f8f98a93SRussell King bgt 1b \n\ 81f8f98a93SRussell King beq 2b \n\ 82f8f98a93SRussell King ldmfd sp!, {r4, r5, pc} " 83f8f98a93SRussell King : 84f8f98a93SRussell King : "r" (from), "r" (to), "I" (PAGE_SIZE / 64 - 1)); 85f8f98a93SRussell King } 86f8f98a93SRussell King 87063b0a42SRussell King void xscale_mc_copy_user_highpage(struct page *to, struct page *from, 88f00a75c0SRussell King unsigned long vaddr, struct vm_area_struct *vma) 89f8f98a93SRussell King { 905472e862SCong Wang void *kto = kmap_atomic(to); 911c9d3df5SRichard Purdie 92c0177800SCatalin Marinas if (!test_and_set_bit(PG_dcache_clean, &from->flags)) 93*cb9f753aSHuang Ying __flush_dcache_page(page_mapping_file(from), from); 941c9d3df5SRichard Purdie 95bd31b859SThomas Gleixner raw_spin_lock(&minicache_lock); 96f8f98a93SRussell King 9767ece144SRussell King set_top_pte(COPYPAGE_MINICACHE, mk_pte(from, minicache_pgprot)); 98f8f98a93SRussell King 99f8f98a93SRussell King mc_copy_user_page((void *)COPYPAGE_MINICACHE, kto); 100f8f98a93SRussell King 101bd31b859SThomas Gleixner raw_spin_unlock(&minicache_lock); 102063b0a42SRussell King 1035472e862SCong Wang kunmap_atomic(kto); 104f8f98a93SRussell King } 105f8f98a93SRussell King 106f8f98a93SRussell King /* 107f8f98a93SRussell King * XScale optimised clear_user_page 108f8f98a93SRussell King */ 109303c6443SRussell King void 110303c6443SRussell King xscale_mc_clear_user_highpage(struct page *page, unsigned long vaddr) 111f8f98a93SRussell King { 1125472e862SCong Wang void *ptr, *kaddr = kmap_atomic(page); 113f8f98a93SRussell King asm volatile( 11443ae286bSNicolas Pitre "mov r1, %2 \n\ 115f8f98a93SRussell King mov r2, #0 \n\ 116f8f98a93SRussell King mov r3, #0 \n\ 117303c6443SRussell King 1: mov ip, %0 \n\ 118303c6443SRussell King strd r2, [%0], #8 \n\ 119303c6443SRussell King strd r2, [%0], #8 \n\ 120303c6443SRussell King strd r2, [%0], #8 \n\ 121303c6443SRussell King strd r2, [%0], #8 \n\ 122f8f98a93SRussell King mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\ 123f8f98a93SRussell King subs r1, r1, #1 \n\ 124f8f98a93SRussell King mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\ 125303c6443SRussell King bne 1b" 12643ae286bSNicolas Pitre : "=r" (ptr) 12743ae286bSNicolas Pitre : "0" (kaddr), "I" (PAGE_SIZE / 32) 128303c6443SRussell King : "r1", "r2", "r3", "ip"); 1295472e862SCong Wang kunmap_atomic(kaddr); 130f8f98a93SRussell King } 131f8f98a93SRussell King 132f8f98a93SRussell King struct cpu_user_fns xscale_mc_user_fns __initdata = { 133303c6443SRussell King .cpu_clear_user_highpage = xscale_mc_clear_user_highpage, 134063b0a42SRussell King .cpu_copy_user_highpage = xscale_mc_copy_user_highpage, 135f8f98a93SRussell King }; 136