xref: /linux/arch/sparc/lib/NGmemcpy.S (revision 6ee738610f41b59733f63718f0bdbcba7d3a3f12) 
1/* NGmemcpy.S: Niagara optimized memcpy.
2 *
3 * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
4 */
5
6#ifdef __KERNEL__
7#include <asm/asi.h>
8#include <asm/thread_info.h>
9#define GLOBAL_SPARE	%g7
10#define RESTORE_ASI(TMP)	\
11	ldub	[%g6 + TI_CURRENT_DS], TMP;  \
12	wr	TMP, 0x0, %asi;
13#else
14#define GLOBAL_SPARE	%g5
15#define RESTORE_ASI(TMP)	\
16	wr	%g0, ASI_PNF, %asi
17#endif
18
19#ifdef __sparc_v9__
20#define SAVE_AMOUNT	128
21#else
22#define SAVE_AMOUNT	64
23#endif
24
25#ifndef STORE_ASI
26#define STORE_ASI	ASI_BLK_INIT_QUAD_LDD_P
27#endif
28
29#ifndef EX_LD
30#define EX_LD(x)	x
31#endif
32
33#ifndef EX_ST
34#define EX_ST(x)	x
35#endif
36
37#ifndef EX_RETVAL
38#define EX_RETVAL(x)	x
39#endif
40
41#ifndef LOAD
42#ifndef MEMCPY_DEBUG
43#define LOAD(type,addr,dest)	type [addr], dest
44#else
45#define LOAD(type,addr,dest)	type##a [addr] 0x80, dest
46#endif
47#endif
48
49#ifndef LOAD_TWIN
50#define LOAD_TWIN(addr_reg,dest0,dest1)	\
51	ldda [addr_reg] ASI_BLK_INIT_QUAD_LDD_P, dest0
52#endif
53
54#ifndef STORE
55#define STORE(type,src,addr)	type src, [addr]
56#endif
57
58#ifndef STORE_INIT
59#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
60#define STORE_INIT(src,addr)	stxa src, [addr] %asi
61#else
62#define STORE_INIT(src,addr)	stx src, [addr + 0x00]
63#endif
64#endif
65
66#ifndef FUNC_NAME
67#define FUNC_NAME	NGmemcpy
68#endif
69
70#ifndef PREAMBLE
71#define PREAMBLE
72#endif
73
74#ifndef XCC
75#define XCC xcc
76#endif
77
78	.register	%g2,#scratch
79	.register	%g3,#scratch
80
81	.text
82	.align		64
83
84	.globl	FUNC_NAME
85	.type	FUNC_NAME,#function
86FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
87	PREAMBLE
88	save		%sp, -SAVE_AMOUNT, %sp
89	srlx		%i2, 31, %g2
90	cmp		%g2, 0
91	tne		%xcc, 5
92	mov		%i0, %o0
93	cmp		%i2, 0
94	be,pn		%XCC, 85f
95	 or		%o0, %i1, %i3
96	cmp		%i2, 16
97	blu,a,pn	%XCC, 80f
98	 or		%i3, %i2, %i3
99
100	/* 2 blocks (128 bytes) is the minimum we can do the block
101	 * copy with.  We need to ensure that we'll iterate at least
102	 * once in the block copy loop.  At worst we'll need to align
103	 * the destination to a 64-byte boundary which can chew up
104	 * to (64 - 1) bytes from the length before we perform the
105	 * block copy loop.
106	 */
107	cmp		%i2, (2 * 64)
108	blu,pt		%XCC, 70f
109	 andcc		%i3, 0x7, %g0
110
111	/* %o0:	dst
112	 * %i1:	src
113	 * %i2:	len  (known to be >= 128)
114	 *
115	 * The block copy loops will use %i4/%i5,%g2/%g3 as
116	 * temporaries while copying the data.
117	 */
118
119	LOAD(prefetch, %i1, #one_read)
120	wr		%g0, STORE_ASI, %asi
121
122	/* Align destination on 64-byte boundary.  */
123	andcc		%o0, (64 - 1), %i4
124	be,pt		%XCC, 2f
125	 sub		%i4, 64, %i4
126	sub		%g0, %i4, %i4	! bytes to align dst
127	sub		%i2, %i4, %i2
1281:	subcc		%i4, 1, %i4
129	EX_LD(LOAD(ldub, %i1, %g1))
130	EX_ST(STORE(stb, %g1, %o0))
131	add		%i1, 1, %i1
132	bne,pt		%XCC, 1b
133	add		%o0, 1, %o0
134
135	/* If the source is on a 16-byte boundary we can do
136	 * the direct block copy loop.  If it is 8-byte aligned
137	 * we can do the 16-byte loads offset by -8 bytes and the
138	 * init stores offset by one register.
139	 *
140	 * If the source is not even 8-byte aligned, we need to do
141	 * shifting and masking (basically integer faligndata).
142	 *
143	 * The careful bit with init stores is that if we store
144	 * to any part of the cache line we have to store the whole
145	 * cacheline else we can end up with corrupt L2 cache line
146	 * contents.  Since the loop works on 64-bytes of 64-byte
147	 * aligned store data at a time, this is easy to ensure.
148	 */
1492:
150	andcc		%i1, (16 - 1), %i4
151	andn		%i2, (64 - 1), %g1	! block copy loop iterator
152	be,pt		%XCC, 50f
153	 sub		%i2, %g1, %i2		! final sub-block copy bytes
154
155	cmp		%i4, 8
156	be,pt		%XCC, 10f
157	 sub		%i1, %i4, %i1
158
159	/* Neither 8-byte nor 16-byte aligned, shift and mask.  */
160	and		%i4, 0x7, GLOBAL_SPARE
161	sll		GLOBAL_SPARE, 3, GLOBAL_SPARE
162	mov		64, %i5
163	EX_LD(LOAD_TWIN(%i1, %g2, %g3))
164	sub		%i5, GLOBAL_SPARE, %i5
165	mov		16, %o4
166	mov		32, %o5
167	mov		48, %o7
168	mov		64, %i3
169
170	bg,pn	   	%XCC, 9f
171	 nop
172
173#define MIX_THREE_WORDS(WORD1, WORD2, WORD3, PRE_SHIFT, POST_SHIFT, TMP) \
174	sllx		WORD1, POST_SHIFT, WORD1; \
175	srlx		WORD2, PRE_SHIFT, TMP; \
176	sllx		WORD2, POST_SHIFT, WORD2; \
177	or		WORD1, TMP, WORD1; \
178	srlx		WORD3, PRE_SHIFT, TMP; \
179	or		WORD2, TMP, WORD2;
180
1818:	EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
182	MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
183	LOAD(prefetch, %i1 + %i3, #one_read)
184
185	EX_ST(STORE_INIT(%g2, %o0 + 0x00))
186	EX_ST(STORE_INIT(%g3, %o0 + 0x08))
187
188	EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
189	MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
190
191	EX_ST(STORE_INIT(%o2, %o0 + 0x10))
192	EX_ST(STORE_INIT(%o3, %o0 + 0x18))
193
194	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
195	MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
196
197	EX_ST(STORE_INIT(%g2, %o0 + 0x20))
198	EX_ST(STORE_INIT(%g3, %o0 + 0x28))
199
200	EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
201	add		%i1, 64, %i1
202	MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
203
204	EX_ST(STORE_INIT(%o2, %o0 + 0x30))
205	EX_ST(STORE_INIT(%o3, %o0 + 0x38))
206
207	subcc		%g1, 64, %g1
208	bne,pt		%XCC, 8b
209	 add		%o0, 64, %o0
210
211	ba,pt		%XCC, 60f
212	 add		%i1, %i4, %i1
213
2149:	EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
215	MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
216	LOAD(prefetch, %i1 + %i3, #one_read)
217
218	EX_ST(STORE_INIT(%g3, %o0 + 0x00))
219	EX_ST(STORE_INIT(%o2, %o0 + 0x08))
220
221	EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
222	MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
223
224	EX_ST(STORE_INIT(%o3, %o0 + 0x10))
225	EX_ST(STORE_INIT(%g2, %o0 + 0x18))
226
227	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
228	MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
229
230	EX_ST(STORE_INIT(%g3, %o0 + 0x20))
231	EX_ST(STORE_INIT(%o2, %o0 + 0x28))
232
233	EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
234	add		%i1, 64, %i1
235	MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
236
237	EX_ST(STORE_INIT(%o3, %o0 + 0x30))
238	EX_ST(STORE_INIT(%g2, %o0 + 0x38))
239
240	subcc		%g1, 64, %g1
241	bne,pt		%XCC, 9b
242	 add		%o0, 64, %o0
243
244	ba,pt		%XCC, 60f
245	 add		%i1, %i4, %i1
246
24710:	/* Destination is 64-byte aligned, source was only 8-byte
248	 * aligned but it has been subtracted by 8 and we perform
249	 * one twin load ahead, then add 8 back into source when
250	 * we finish the loop.
251	 */
252	EX_LD(LOAD_TWIN(%i1, %o4, %o5))
253	mov	16, %o7
254	mov	32, %g2
255	mov	48, %g3
256	mov	64, %o1
2571:	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
258	LOAD(prefetch, %i1 + %o1, #one_read)
259	EX_ST(STORE_INIT(%o5, %o0 + 0x00))	! initializes cache line
260	EX_ST(STORE_INIT(%o2, %o0 + 0x08))
261	EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
262	EX_ST(STORE_INIT(%o3, %o0 + 0x10))
263	EX_ST(STORE_INIT(%o4, %o0 + 0x18))
264	EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
265	EX_ST(STORE_INIT(%o5, %o0 + 0x20))
266	EX_ST(STORE_INIT(%o2, %o0 + 0x28))
267	EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5))
268	add		%i1, 64, %i1
269	EX_ST(STORE_INIT(%o3, %o0 + 0x30))
270	EX_ST(STORE_INIT(%o4, %o0 + 0x38))
271	subcc		%g1, 64, %g1
272	bne,pt		%XCC, 1b
273	 add		%o0, 64, %o0
274
275	ba,pt		%XCC, 60f
276	 add		%i1, 0x8, %i1
277
27850:	/* Destination is 64-byte aligned, and source is 16-byte
279	 * aligned.
280	 */
281	mov	16, %o7
282	mov	32, %g2
283	mov	48, %g3
284	mov	64, %o1
2851:	EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5))
286	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
287	LOAD(prefetch, %i1 + %o1, #one_read)
288	EX_ST(STORE_INIT(%o4, %o0 + 0x00))	! initializes cache line
289	EX_ST(STORE_INIT(%o5, %o0 + 0x08))
290	EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
291	EX_ST(STORE_INIT(%o2, %o0 + 0x10))
292	EX_ST(STORE_INIT(%o3, %o0 + 0x18))
293	EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
294	add	%i1, 64, %i1
295	EX_ST(STORE_INIT(%o4, %o0 + 0x20))
296	EX_ST(STORE_INIT(%o5, %o0 + 0x28))
297	EX_ST(STORE_INIT(%o2, %o0 + 0x30))
298	EX_ST(STORE_INIT(%o3, %o0 + 0x38))
299	subcc	%g1, 64, %g1
300	bne,pt	%XCC, 1b
301	 add	%o0, 64, %o0
302	/* fall through */
303
30460:
305	membar		#Sync
306
307	/* %i2 contains any final bytes still needed to be copied
308	 * over. If anything is left, we copy it one byte at a time.
309	 */
310	RESTORE_ASI(%i3)
311	brz,pt		%i2, 85f
312	 sub		%o0, %i1, %i3
313	ba,a,pt		%XCC, 90f
314
315	.align		64
31670: /* 16 < len <= 64 */
317	bne,pn		%XCC, 75f
318	 sub		%o0, %i1, %i3
319
32072:
321	andn		%i2, 0xf, %i4
322	and		%i2, 0xf, %i2
3231:	subcc		%i4, 0x10, %i4
324	EX_LD(LOAD(ldx, %i1, %o4))
325	add		%i1, 0x08, %i1
326	EX_LD(LOAD(ldx, %i1, %g1))
327	sub		%i1, 0x08, %i1
328	EX_ST(STORE(stx, %o4, %i1 + %i3))
329	add		%i1, 0x8, %i1
330	EX_ST(STORE(stx, %g1, %i1 + %i3))
331	bgu,pt		%XCC, 1b
332	 add		%i1, 0x8, %i1
33373:	andcc		%i2, 0x8, %g0
334	be,pt		%XCC, 1f
335	 nop
336	sub		%i2, 0x8, %i2
337	EX_LD(LOAD(ldx, %i1, %o4))
338	EX_ST(STORE(stx, %o4, %i1 + %i3))
339	add		%i1, 0x8, %i1
3401:	andcc		%i2, 0x4, %g0
341	be,pt		%XCC, 1f
342	 nop
343	sub		%i2, 0x4, %i2
344	EX_LD(LOAD(lduw, %i1, %i5))
345	EX_ST(STORE(stw, %i5, %i1 + %i3))
346	add		%i1, 0x4, %i1
3471:	cmp		%i2, 0
348	be,pt		%XCC, 85f
349	 nop
350	ba,pt		%xcc, 90f
351	 nop
352
35375:
354	andcc		%o0, 0x7, %g1
355	sub		%g1, 0x8, %g1
356	be,pn		%icc, 2f
357	 sub		%g0, %g1, %g1
358	sub		%i2, %g1, %i2
359
3601:	subcc		%g1, 1, %g1
361	EX_LD(LOAD(ldub, %i1, %i5))
362	EX_ST(STORE(stb, %i5, %i1 + %i3))
363	bgu,pt		%icc, 1b
364	 add		%i1, 1, %i1
365
3662:	add		%i1, %i3, %o0
367	andcc		%i1, 0x7, %g1
368	bne,pt		%icc, 8f
369	 sll		%g1, 3, %g1
370
371	cmp		%i2, 16
372	bgeu,pt		%icc, 72b
373	 nop
374	ba,a,pt		%xcc, 73b
375
3768:	mov		64, %i3
377	andn		%i1, 0x7, %i1
378	EX_LD(LOAD(ldx, %i1, %g2))
379	sub		%i3, %g1, %i3
380	andn		%i2, 0x7, %i4
381	sllx		%g2, %g1, %g2
3821:	add		%i1, 0x8, %i1
383	EX_LD(LOAD(ldx, %i1, %g3))
384	subcc		%i4, 0x8, %i4
385	srlx		%g3, %i3, %i5
386	or		%i5, %g2, %i5
387	EX_ST(STORE(stx, %i5, %o0))
388	add		%o0, 0x8, %o0
389	bgu,pt		%icc, 1b
390	 sllx		%g3, %g1, %g2
391
392	srl		%g1, 3, %g1
393	andcc		%i2, 0x7, %i2
394	be,pn		%icc, 85f
395	 add		%i1, %g1, %i1
396	ba,pt		%xcc, 90f
397	 sub		%o0, %i1, %i3
398
399	.align		64
40080: /* 0 < len <= 16 */
401	andcc		%i3, 0x3, %g0
402	bne,pn		%XCC, 90f
403	 sub		%o0, %i1, %i3
404
4051:
406	subcc		%i2, 4, %i2
407	EX_LD(LOAD(lduw, %i1, %g1))
408	EX_ST(STORE(stw, %g1, %i1 + %i3))
409	bgu,pt		%XCC, 1b
410	 add		%i1, 4, %i1
411
41285:	ret
413	 restore	EX_RETVAL(%i0), %g0, %o0
414
415	.align		32
41690:
417	subcc		%i2, 1, %i2
418	EX_LD(LOAD(ldub, %i1, %g1))
419	EX_ST(STORE(stb, %g1, %i1 + %i3))
420	bgu,pt		%XCC, 90b
421	 add		%i1, 1, %i1
422	ret
423	 restore	EX_RETVAL(%i0), %g0, %o0
424
425	.size		FUNC_NAME, .-FUNC_NAME
426