xref: /linux/arch/x86/lib/clear_page_64.S (revision 3a64d5b82eccc0dc629d43cde791a2c19bd67dfc) 
1/* SPDX-License-Identifier: GPL-2.0-only */
2#include <linux/export.h>
3#include <linux/linkage.h>
4#include <linux/cfi_types.h>
5#include <linux/objtool.h>
6#include <asm/asm.h>
7
8/*
9 * Zero page aligned region.
10 * %rdi	- dest
11 * %rcx	- length
12 */
13SYM_TYPED_FUNC_START(__clear_pages_unrolled)
14	shrq   $6, %rcx
15	.p2align 4
16.Lloop:
17	decq	%rcx
18#define PUT(x) movq %rax,x*8(%rdi)
19	movq %rax,(%rdi)
20	PUT(1)
21	PUT(2)
22	PUT(3)
23	PUT(4)
24	PUT(5)
25	PUT(6)
26	PUT(7)
27	leaq	64(%rdi),%rdi
28	jnz	.Lloop
29	nop
30	RET
31SYM_FUNC_END(__clear_pages_unrolled)
32EXPORT_SYMBOL_GPL(__clear_pages_unrolled)
33
34/*
35 * Default clear user-space.
36 * Input:
37 * rdi destination
38 * rcx count
39 * rax is zero
40 *
41 * Output:
42 * rcx: uncleared bytes or 0 if successful.
43 */
44SYM_FUNC_START(rep_stos_alternative)
45	ANNOTATE_NOENDBR
46	cmpq $64,%rcx
47	jae .Lunrolled
48
49	cmp $8,%ecx
50	jae .Lword
51
52	testl %ecx,%ecx
53	je .Lexit
54
55.Lclear_user_tail:
560:	movb %al,(%rdi)
57	inc %rdi
58	dec %rcx
59	jnz .Lclear_user_tail
60.Lexit:
61	RET
62
63	_ASM_EXTABLE_UA( 0b, .Lexit)
64
65.Lword:
661:	movq %rax,(%rdi)
67	addq $8,%rdi
68	sub $8,%ecx
69	je .Lexit
70	cmp $8,%ecx
71	jae .Lword
72	jmp .Lclear_user_tail
73
74	.p2align 4
75.Lunrolled:
7610:	movq %rax,(%rdi)
7711:	movq %rax,8(%rdi)
7812:	movq %rax,16(%rdi)
7913:	movq %rax,24(%rdi)
8014:	movq %rax,32(%rdi)
8115:	movq %rax,40(%rdi)
8216:	movq %rax,48(%rdi)
8317:	movq %rax,56(%rdi)
84	addq $64,%rdi
85	subq $64,%rcx
86	cmpq $64,%rcx
87	jae .Lunrolled
88	cmpl $8,%ecx
89	jae .Lword
90	testl %ecx,%ecx
91	jne .Lclear_user_tail
92	RET
93
94	/*
95	 * If we take an exception on any of the
96	 * word stores, we know that %rcx isn't zero,
97	 * so we can just go to the tail clearing to
98	 * get the exact count.
99	 *
100	 * The unrolled case might end up clearing
101	 * some bytes twice. Don't care.
102	 *
103	 * We could use the value in %rdi to avoid
104	 * a second fault on the exact count case,
105	 * but do we really care? No.
106	 *
107	 * Finally, we could try to align %rdi at the
108	 * top of the unrolling. But unaligned stores
109	 * just aren't that common or expensive.
110	 */
111	_ASM_EXTABLE_UA( 1b, .Lclear_user_tail)
112	_ASM_EXTABLE_UA(10b, .Lclear_user_tail)
113	_ASM_EXTABLE_UA(11b, .Lclear_user_tail)
114	_ASM_EXTABLE_UA(12b, .Lclear_user_tail)
115	_ASM_EXTABLE_UA(13b, .Lclear_user_tail)
116	_ASM_EXTABLE_UA(14b, .Lclear_user_tail)
117	_ASM_EXTABLE_UA(15b, .Lclear_user_tail)
118	_ASM_EXTABLE_UA(16b, .Lclear_user_tail)
119	_ASM_EXTABLE_UA(17b, .Lclear_user_tail)
120SYM_FUNC_END(rep_stos_alternative)
121EXPORT_SYMBOL(rep_stos_alternative)
122