1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2f8f98a93SRussell King /*
3f8f98a93SRussell King * linux/arch/arm/lib/copypage-xscale.S
4f8f98a93SRussell King *
5f8f98a93SRussell King * Copyright (C) 1995-2005 Russell King
6f8f98a93SRussell King *
7f8f98a93SRussell King * This handles the mini data cache, as found on SA11x0 and XScale
8f8f98a93SRussell King * processors. When we copy a user page page, we map it in such a way
9f8f98a93SRussell King * that accesses to this page will not touch the main data cache, but
10f8f98a93SRussell King * will be cached in the mini data cache. This prevents us thrashing
11f8f98a93SRussell King * the main data cache on page faults.
12f8f98a93SRussell King */
13f8f98a93SRussell King #include <linux/init.h>
14f8f98a93SRussell King #include <linux/mm.h>
15063b0a42SRussell King #include <linux/highmem.h>
16842ca547SMatthew Wilcox (Oracle) #include <linux/pagemap.h>
17f8f98a93SRussell King
18f8f98a93SRussell King #include <asm/tlbflush.h>
191c9d3df5SRichard Purdie #include <asm/cacheflush.h>
20f8f98a93SRussell King
211b2e2b73SRussell King #include "mm.h"
221b2e2b73SRussell King
23f8f98a93SRussell King #define minicache_pgprot __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | \
24bb30f36fSRussell King L_PTE_MT_MINICACHE)
25f8f98a93SRussell King
26bd31b859SThomas Gleixner static DEFINE_RAW_SPINLOCK(minicache_lock);
27f8f98a93SRussell King
28f8f98a93SRussell King /*
29063b0a42SRussell King * XScale mini-dcache optimised copy_user_highpage
30f8f98a93SRussell King *
31f8f98a93SRussell King * We flush the destination cache lines just before we write the data into the
32f8f98a93SRussell King * corresponding address. Since the Dcache is read-allocate, this removes the
33f8f98a93SRussell King * Dcache aliasing issue. The writes will be forwarded to the write buffer,
34f8f98a93SRussell King * and merged as appropriate.
35f8f98a93SRussell King */
mc_copy_user_page(void * from,void * to)36b99afae1SNicolas Pitre static void mc_copy_user_page(void *from, void *to)
37f8f98a93SRussell King {
38b99afae1SNicolas Pitre int tmp;
39b99afae1SNicolas Pitre
40f8f98a93SRussell King /*
41f8f98a93SRussell King * Strangely enough, best performance is achieved
42f8f98a93SRussell King * when prefetching destination as well. (NP)
43f8f98a93SRussell King */
44b99afae1SNicolas Pitre asm volatile ("\
45c7b68049SArnd Bergmann .arch xscale \n\
46b99afae1SNicolas Pitre pld [%0, #0] \n\
47b99afae1SNicolas Pitre pld [%0, #32] \n\
48b99afae1SNicolas Pitre pld [%1, #0] \n\
49b99afae1SNicolas Pitre pld [%1, #32] \n\
50b99afae1SNicolas Pitre 1: pld [%0, #64] \n\
51b99afae1SNicolas Pitre pld [%0, #96] \n\
52b99afae1SNicolas Pitre pld [%1, #64] \n\
53b99afae1SNicolas Pitre pld [%1, #96] \n\
54bc2eca9aSNicolas Pitre 2: ldrd r2, r3, [%0], #8 \n\
55bc2eca9aSNicolas Pitre ldrd r4, r5, [%0], #8 \n\
56b99afae1SNicolas Pitre mov ip, %1 \n\
57bc2eca9aSNicolas Pitre strd r2, r3, [%1], #8 \n\
58bc2eca9aSNicolas Pitre ldrd r2, r3, [%0], #8 \n\
59bc2eca9aSNicolas Pitre strd r4, r5, [%1], #8 \n\
60bc2eca9aSNicolas Pitre ldrd r4, r5, [%0], #8 \n\
61bc2eca9aSNicolas Pitre strd r2, r3, [%1], #8 \n\
62bc2eca9aSNicolas Pitre strd r4, r5, [%1], #8 \n\
63f8f98a93SRussell King mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\
64bc2eca9aSNicolas Pitre ldrd r2, r3, [%0], #8 \n\
65f8f98a93SRussell King mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\
66bc2eca9aSNicolas Pitre ldrd r4, r5, [%0], #8 \n\
67b99afae1SNicolas Pitre mov ip, %1 \n\
68bc2eca9aSNicolas Pitre strd r2, r3, [%1], #8 \n\
69bc2eca9aSNicolas Pitre ldrd r2, r3, [%0], #8 \n\
70bc2eca9aSNicolas Pitre strd r4, r5, [%1], #8 \n\
71bc2eca9aSNicolas Pitre ldrd r4, r5, [%0], #8 \n\
72bc2eca9aSNicolas Pitre strd r2, r3, [%1], #8 \n\
73bc2eca9aSNicolas Pitre strd r4, r5, [%1], #8 \n\
74f8f98a93SRussell King mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\
75b99afae1SNicolas Pitre subs %2, %2, #1 \n\
76f8f98a93SRussell King mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\
77f8f98a93SRussell King bgt 1b \n\
78b99afae1SNicolas Pitre beq 2b "
79b99afae1SNicolas Pitre : "+&r" (from), "+&r" (to), "=&r" (tmp)
80b99afae1SNicolas Pitre : "2" (PAGE_SIZE / 64 - 1)
81b99afae1SNicolas Pitre : "r2", "r3", "r4", "r5", "ip");
82f8f98a93SRussell King }
83f8f98a93SRussell King
xscale_mc_copy_user_highpage(struct page * to,struct page * from,unsigned long vaddr,struct vm_area_struct * vma)84063b0a42SRussell King void xscale_mc_copy_user_highpage(struct page *to, struct page *from,
85f00a75c0SRussell King unsigned long vaddr, struct vm_area_struct *vma)
86f8f98a93SRussell King {
87*8b5989f3SMatthew Wilcox (Oracle) struct folio *src = page_folio(from);
885472e862SCong Wang void *kto = kmap_atomic(to);
891c9d3df5SRichard Purdie
90*8b5989f3SMatthew Wilcox (Oracle) if (!test_and_set_bit(PG_dcache_clean, &src->flags))
91*8b5989f3SMatthew Wilcox (Oracle) __flush_dcache_folio(folio_flush_mapping(src), src);
921c9d3df5SRichard Purdie
93bd31b859SThomas Gleixner raw_spin_lock(&minicache_lock);
94f8f98a93SRussell King
9567ece144SRussell King set_top_pte(COPYPAGE_MINICACHE, mk_pte(from, minicache_pgprot));
96f8f98a93SRussell King
97f8f98a93SRussell King mc_copy_user_page((void *)COPYPAGE_MINICACHE, kto);
98f8f98a93SRussell King
99bd31b859SThomas Gleixner raw_spin_unlock(&minicache_lock);
100063b0a42SRussell King
1015472e862SCong Wang kunmap_atomic(kto);
102f8f98a93SRussell King }
103f8f98a93SRussell King
104f8f98a93SRussell King /*
105f8f98a93SRussell King * XScale optimised clear_user_page
106f8f98a93SRussell King */
107303c6443SRussell King void
xscale_mc_clear_user_highpage(struct page * page,unsigned long vaddr)108303c6443SRussell King xscale_mc_clear_user_highpage(struct page *page, unsigned long vaddr)
109f8f98a93SRussell King {
1105472e862SCong Wang void *ptr, *kaddr = kmap_atomic(page);
111c7b68049SArnd Bergmann asm volatile("\
112c7b68049SArnd Bergmann .arch xscale \n\
113c7b68049SArnd Bergmann mov r1, %2 \n\
114f8f98a93SRussell King mov r2, #0 \n\
115f8f98a93SRussell King mov r3, #0 \n\
116303c6443SRussell King 1: mov ip, %0 \n\
117bc2eca9aSNicolas Pitre strd r2, r3, [%0], #8 \n\
118bc2eca9aSNicolas Pitre strd r2, r3, [%0], #8 \n\
119bc2eca9aSNicolas Pitre strd r2, r3, [%0], #8 \n\
120bc2eca9aSNicolas Pitre strd r2, r3, [%0], #8 \n\
121f8f98a93SRussell King mcr p15, 0, ip, c7, c10, 1 @ clean D line\n\
122f8f98a93SRussell King subs r1, r1, #1 \n\
123f8f98a93SRussell King mcr p15, 0, ip, c7, c6, 1 @ invalidate D line\n\
124303c6443SRussell King bne 1b"
12543ae286bSNicolas Pitre : "=r" (ptr)
12643ae286bSNicolas Pitre : "0" (kaddr), "I" (PAGE_SIZE / 32)
127303c6443SRussell King : "r1", "r2", "r3", "ip");
1285472e862SCong Wang kunmap_atomic(kaddr);
129f8f98a93SRussell King }
130f8f98a93SRussell King
131f8f98a93SRussell King struct cpu_user_fns xscale_mc_user_fns __initdata = {
132303c6443SRussell King .cpu_clear_user_highpage = xscale_mc_clear_user_highpage,
133063b0a42SRussell King .cpu_copy_user_highpage = xscale_mc_copy_user_highpage,
134f8f98a93SRussell King };
135