1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2d73e60b7SRussell King /*
3d73e60b7SRussell King * linux/arch/arm/mm/copypage-feroceon.S
4d73e60b7SRussell King *
5d73e60b7SRussell King * Copyright (C) 2008 Marvell Semiconductors
6d73e60b7SRussell King *
7063b0a42SRussell King * This handles copy_user_highpage and clear_user_page on Feroceon
8d73e60b7SRussell King * more optimally than the generic implementations.
9d73e60b7SRussell King */
10d73e60b7SRussell King #include <linux/init.h>
11063b0a42SRussell King #include <linux/highmem.h>
12d73e60b7SRussell King
feroceon_copy_user_page(void * kto,const void * kfrom)13b99afae1SNicolas Pitre static void feroceon_copy_user_page(void *kto, const void *kfrom)
14d73e60b7SRussell King {
15b99afae1SNicolas Pitre int tmp;
16b99afae1SNicolas Pitre
17b99afae1SNicolas Pitre asm volatile ("\
18*a2faac39SNick Desaulniers .arch armv5te \n\
19b99afae1SNicolas Pitre 1: ldmia %1!, {r2 - r7, ip, lr} \n\
20b99afae1SNicolas Pitre pld [%1, #0] \n\
21b99afae1SNicolas Pitre pld [%1, #32] \n\
22b99afae1SNicolas Pitre pld [%1, #64] \n\
23b99afae1SNicolas Pitre pld [%1, #96] \n\
24b99afae1SNicolas Pitre pld [%1, #128] \n\
25b99afae1SNicolas Pitre pld [%1, #160] \n\
26b99afae1SNicolas Pitre pld [%1, #192] \n\
27b99afae1SNicolas Pitre stmia %0, {r2 - r7, ip, lr} \n\
28b99afae1SNicolas Pitre ldmia %1!, {r2 - r7, ip, lr} \n\
29b99afae1SNicolas Pitre mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
30b99afae1SNicolas Pitre add %0, %0, #32 \n\
31b99afae1SNicolas Pitre stmia %0, {r2 - r7, ip, lr} \n\
32b99afae1SNicolas Pitre ldmia %1!, {r2 - r7, ip, lr} \n\
33b99afae1SNicolas Pitre mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
34b99afae1SNicolas Pitre add %0, %0, #32 \n\
35b99afae1SNicolas Pitre stmia %0, {r2 - r7, ip, lr} \n\
36b99afae1SNicolas Pitre ldmia %1!, {r2 - r7, ip, lr} \n\
37b99afae1SNicolas Pitre mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
38b99afae1SNicolas Pitre add %0, %0, #32 \n\
39b99afae1SNicolas Pitre stmia %0, {r2 - r7, ip, lr} \n\
40b99afae1SNicolas Pitre ldmia %1!, {r2 - r7, ip, lr} \n\
41b99afae1SNicolas Pitre mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
42b99afae1SNicolas Pitre add %0, %0, #32 \n\
43b99afae1SNicolas Pitre stmia %0, {r2 - r7, ip, lr} \n\
44b99afae1SNicolas Pitre ldmia %1!, {r2 - r7, ip, lr} \n\
45b99afae1SNicolas Pitre mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
46b99afae1SNicolas Pitre add %0, %0, #32 \n\
47b99afae1SNicolas Pitre stmia %0, {r2 - r7, ip, lr} \n\
48b99afae1SNicolas Pitre ldmia %1!, {r2 - r7, ip, lr} \n\
49b99afae1SNicolas Pitre mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
50b99afae1SNicolas Pitre add %0, %0, #32 \n\
51b99afae1SNicolas Pitre stmia %0, {r2 - r7, ip, lr} \n\
52b99afae1SNicolas Pitre ldmia %1!, {r2 - r7, ip, lr} \n\
53b99afae1SNicolas Pitre mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
54b99afae1SNicolas Pitre add %0, %0, #32 \n\
55b99afae1SNicolas Pitre stmia %0, {r2 - r7, ip, lr} \n\
56b99afae1SNicolas Pitre subs %2, %2, #(32 * 8) \n\
57b99afae1SNicolas Pitre mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
58b99afae1SNicolas Pitre add %0, %0, #32 \n\
59d73e60b7SRussell King bne 1b \n\
60b99afae1SNicolas Pitre mcr p15, 0, %2, c7, c10, 4 @ drain WB"
61b99afae1SNicolas Pitre : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp)
62b99afae1SNicolas Pitre : "2" (PAGE_SIZE)
63b99afae1SNicolas Pitre : "r2", "r3", "r4", "r5", "r6", "r7", "ip", "lr");
64d73e60b7SRussell King }
65d73e60b7SRussell King
feroceon_copy_user_highpage(struct page * to,struct page * from,unsigned long vaddr,struct vm_area_struct * vma)66063b0a42SRussell King void feroceon_copy_user_highpage(struct page *to, struct page *from,
67f00a75c0SRussell King unsigned long vaddr, struct vm_area_struct *vma)
68063b0a42SRussell King {
69063b0a42SRussell King void *kto, *kfrom;
70063b0a42SRussell King
715472e862SCong Wang kto = kmap_atomic(to);
725472e862SCong Wang kfrom = kmap_atomic(from);
732725898fSRussell King flush_cache_page(vma, vaddr, page_to_pfn(from));
74063b0a42SRussell King feroceon_copy_user_page(kto, kfrom);
755472e862SCong Wang kunmap_atomic(kfrom);
765472e862SCong Wang kunmap_atomic(kto);
77063b0a42SRussell King }
78063b0a42SRussell King
feroceon_clear_user_highpage(struct page * page,unsigned long vaddr)79303c6443SRussell King void feroceon_clear_user_highpage(struct page *page, unsigned long vaddr)
80d73e60b7SRussell King {
815472e862SCong Wang void *ptr, *kaddr = kmap_atomic(page);
8243ae286bSNicolas Pitre asm volatile ("\
8343ae286bSNicolas Pitre mov r1, %2 \n\
84d73e60b7SRussell King mov r2, #0 \n\
85d73e60b7SRussell King mov r3, #0 \n\
86d73e60b7SRussell King mov r4, #0 \n\
87d73e60b7SRussell King mov r5, #0 \n\
88d73e60b7SRussell King mov r6, #0 \n\
89d73e60b7SRussell King mov r7, #0 \n\
90d73e60b7SRussell King mov ip, #0 \n\
91d73e60b7SRussell King mov lr, #0 \n\
92303c6443SRussell King 1: stmia %0, {r2-r7, ip, lr} \n\
93d73e60b7SRussell King subs r1, r1, #1 \n\
94303c6443SRussell King mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\
9543ae286bSNicolas Pitre add %0, %0, #32 \n\
96d73e60b7SRussell King bne 1b \n\
97303c6443SRussell King mcr p15, 0, r1, c7, c10, 4 @ drain WB"
9843ae286bSNicolas Pitre : "=r" (ptr)
9943ae286bSNicolas Pitre : "0" (kaddr), "I" (PAGE_SIZE / 32)
100303c6443SRussell King : "r1", "r2", "r3", "r4", "r5", "r6", "r7", "ip", "lr");
1015472e862SCong Wang kunmap_atomic(kaddr);
102d73e60b7SRussell King }
103d73e60b7SRussell King
104d73e60b7SRussell King struct cpu_user_fns feroceon_user_fns __initdata = {
105303c6443SRussell King .cpu_clear_user_highpage = feroceon_clear_user_highpage,
106063b0a42SRussell King .cpu_copy_user_highpage = feroceon_copy_user_highpage,
107d73e60b7SRussell King };
108d73e60b7SRussell King
109