xref: /linux/arch/sparc/lib/NG2memcpy.S (revision f3539c12d8196ce0a1993364d30b3a18908470d1) 
1/* NG2memcpy.S: Niagara-2 optimized memcpy.
2 *
3 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
4 */
5
6#ifdef __KERNEL__
7#include <asm/visasm.h>
8#include <asm/asi.h>
9#define GLOBAL_SPARE	%g7
10#else
11#define ASI_PNF 0x82
12#define ASI_BLK_P 0xf0
13#define ASI_BLK_INIT_QUAD_LDD_P 0xe2
14#define FPRS_FEF  0x04
15#ifdef MEMCPY_DEBUG
16#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
17		     clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0;
18#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
19#else
20#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
21#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
22#endif
23#define GLOBAL_SPARE	%g5
24#endif
25
26#ifndef STORE_ASI
27#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
28#define STORE_ASI	ASI_BLK_INIT_QUAD_LDD_P
29#else
30#define STORE_ASI	0x80		/* ASI_P */
31#endif
32#endif
33
34#ifndef EX_LD
35#define EX_LD(x)	x
36#endif
37#ifndef EX_LD_FP
38#define EX_LD_FP(x)	x
39#endif
40
41#ifndef EX_ST
42#define EX_ST(x)	x
43#endif
44#ifndef EX_ST_FP
45#define EX_ST_FP(x)	x
46#endif
47
48#ifndef EX_RETVAL
49#define EX_RETVAL(x)	x
50#endif
51
52#ifndef LOAD
53#define LOAD(type,addr,dest)	type [addr], dest
54#endif
55
56#ifndef LOAD_BLK
57#define LOAD_BLK(addr,dest)	ldda [addr] ASI_BLK_P, dest
58#endif
59
60#ifndef STORE
61#ifndef MEMCPY_DEBUG
62#define STORE(type,src,addr)	type src, [addr]
63#else
64#define STORE(type,src,addr)	type##a src, [addr] 0x80
65#endif
66#endif
67
68#ifndef STORE_BLK
69#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_P
70#endif
71
72#ifndef STORE_INIT
73#define STORE_INIT(src,addr)	stxa src, [addr] STORE_ASI
74#endif
75
76#ifndef FUNC_NAME
77#define FUNC_NAME	NG2memcpy
78#endif
79
80#ifndef PREAMBLE
81#define PREAMBLE
82#endif
83
84#ifndef XCC
85#define XCC xcc
86#endif
87
88#define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \
89	faligndata	%x0, %x1, %f0; \
90	faligndata	%x1, %x2, %f2; \
91	faligndata	%x2, %x3, %f4; \
92	faligndata	%x3, %x4, %f6; \
93	faligndata	%x4, %x5, %f8; \
94	faligndata	%x5, %x6, %f10; \
95	faligndata	%x6, %x7, %f12; \
96	faligndata	%x7, %x8, %f14;
97
98#define FREG_MOVE_1(x0) \
99	fsrc2		%x0, %f0;
100#define FREG_MOVE_2(x0, x1) \
101	fsrc2		%x0, %f0; \
102	fsrc2		%x1, %f2;
103#define FREG_MOVE_3(x0, x1, x2) \
104	fsrc2		%x0, %f0; \
105	fsrc2		%x1, %f2; \
106	fsrc2		%x2, %f4;
107#define FREG_MOVE_4(x0, x1, x2, x3) \
108	fsrc2		%x0, %f0; \
109	fsrc2		%x1, %f2; \
110	fsrc2		%x2, %f4; \
111	fsrc2		%x3, %f6;
112#define FREG_MOVE_5(x0, x1, x2, x3, x4) \
113	fsrc2		%x0, %f0; \
114	fsrc2		%x1, %f2; \
115	fsrc2		%x2, %f4; \
116	fsrc2		%x3, %f6; \
117	fsrc2		%x4, %f8;
118#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \
119	fsrc2		%x0, %f0; \
120	fsrc2		%x1, %f2; \
121	fsrc2		%x2, %f4; \
122	fsrc2		%x3, %f6; \
123	fsrc2		%x4, %f8; \
124	fsrc2		%x5, %f10;
125#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \
126	fsrc2		%x0, %f0; \
127	fsrc2		%x1, %f2; \
128	fsrc2		%x2, %f4; \
129	fsrc2		%x3, %f6; \
130	fsrc2		%x4, %f8; \
131	fsrc2		%x5, %f10; \
132	fsrc2		%x6, %f12;
133#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \
134	fsrc2		%x0, %f0; \
135	fsrc2		%x1, %f2; \
136	fsrc2		%x2, %f4; \
137	fsrc2		%x3, %f6; \
138	fsrc2		%x4, %f8; \
139	fsrc2		%x5, %f10; \
140	fsrc2		%x6, %f12; \
141	fsrc2		%x7, %f14;
142#define FREG_LOAD_1(base, x0) \
143	EX_LD_FP(LOAD(ldd, base + 0x00, %x0))
144#define FREG_LOAD_2(base, x0, x1) \
145	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
146	EX_LD_FP(LOAD(ldd, base + 0x08, %x1));
147#define FREG_LOAD_3(base, x0, x1, x2) \
148	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
149	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
150	EX_LD_FP(LOAD(ldd, base + 0x10, %x2));
151#define FREG_LOAD_4(base, x0, x1, x2, x3) \
152	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
153	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
154	EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
155	EX_LD_FP(LOAD(ldd, base + 0x18, %x3));
156#define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
157	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
158	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
159	EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
160	EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
161	EX_LD_FP(LOAD(ldd, base + 0x20, %x4));
162#define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
163	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
164	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
165	EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
166	EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
167	EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
168	EX_LD_FP(LOAD(ldd, base + 0x28, %x5));
169#define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
170	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
171	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
172	EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
173	EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
174	EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
175	EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \
176	EX_LD_FP(LOAD(ldd, base + 0x30, %x6));
177
178	.register	%g2,#scratch
179	.register	%g3,#scratch
180
181	.text
182	.align		64
183
184	.globl	FUNC_NAME
185	.type	FUNC_NAME,#function
186FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
187	srlx		%o2, 31, %g2
188	cmp		%g2, 0
189	tne		%xcc, 5
190	PREAMBLE
191	mov		%o0, %o3
192	cmp		%o2, 0
193	be,pn		%XCC, 85f
194	 or		%o0, %o1, GLOBAL_SPARE
195	cmp		%o2, 16
196	blu,a,pn	%XCC, 80f
197	 or		GLOBAL_SPARE, %o2, GLOBAL_SPARE
198
199	/* 2 blocks (128 bytes) is the minimum we can do the block
200	 * copy with.  We need to ensure that we'll iterate at least
201	 * once in the block copy loop.  At worst we'll need to align
202	 * the destination to a 64-byte boundary which can chew up
203	 * to (64 - 1) bytes from the length before we perform the
204	 * block copy loop.
205	 *
206	 * However, the cut-off point, performance wise, is around
207	 * 4 64-byte blocks.
208	 */
209	cmp		%o2, (4 * 64)
210	blu,pt		%XCC, 75f
211	 andcc		GLOBAL_SPARE, 0x7, %g0
212
213	/* %o0:	dst
214	 * %o1:	src
215	 * %o2:	len  (known to be >= 128)
216	 *
217	 * The block copy loops can use %o4, %g2, %g3 as
218	 * temporaries while copying the data.  %o5 must
219	 * be preserved between VISEntryHalf and VISExitHalf
220	 */
221
222	LOAD(prefetch, %o1 + 0x000, #one_read)
223	LOAD(prefetch, %o1 + 0x040, #one_read)
224	LOAD(prefetch, %o1 + 0x080, #one_read)
225
226	/* Align destination on 64-byte boundary.  */
227	andcc		%o0, (64 - 1), %o4
228	be,pt		%XCC, 2f
229	 sub		%o4, 64, %o4
230	sub		%g0, %o4, %o4	! bytes to align dst
231	sub		%o2, %o4, %o2
2321:	subcc		%o4, 1, %o4
233	EX_LD(LOAD(ldub, %o1, %g1))
234	EX_ST(STORE(stb, %g1, %o0))
235	add		%o1, 1, %o1
236	bne,pt		%XCC, 1b
237	add		%o0, 1, %o0
238
2392:
240	/* Clobbers o5/g1/g2/g3/g7/icc/xcc.  We must preserve
241	 * o5 from here until we hit VISExitHalf.
242	 */
243	VISEntryHalf
244
245	membar		#Sync
246	alignaddr	%o1, %g0, %g0
247
248	add		%o1, (64 - 1), %o4
249	andn		%o4, (64 - 1), %o4
250	andn		%o2, (64 - 1), %g1
251	sub		%o2, %g1, %o2
252
253	and		%o1, (64 - 1), %g2
254	add		%o1, %g1, %o1
255	sub		%o0, %o4, %g3
256	brz,pt		%g2, 190f
257	 cmp		%g2, 32
258	blu,a		5f
259	 cmp		%g2, 16
260	cmp		%g2, 48
261	blu,a		4f
262	 cmp		%g2, 40
263	cmp		%g2, 56
264	blu		170f
265	 nop
266	ba,a,pt		%xcc, 180f
267
2684:	/* 32 <= low bits < 48 */
269	blu		150f
270	 nop
271	ba,a,pt		%xcc, 160f
2725:	/* 0 < low bits < 32 */
273	blu,a		6f
274	 cmp		%g2, 8
275	cmp		%g2, 24
276	blu		130f
277	 nop
278	ba,a,pt		%xcc, 140f
2796:	/* 0 < low bits < 16 */
280	bgeu		120f
281	 nop
282	/* fall through for 0 < low bits < 8 */
283110:	sub		%o4, 64, %g2
284	EX_LD_FP(LOAD_BLK(%g2, %f0))
2851:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
286	EX_LD_FP(LOAD_BLK(%o4, %f16))
287	FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
288	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
289	FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
290	subcc		%g1, 64, %g1
291	add		%o4, 64, %o4
292	bne,pt		%xcc, 1b
293	 LOAD(prefetch, %o4 + 64, #one_read)
294	ba,pt		%xcc, 195f
295	 nop
296
297120:	sub		%o4, 56, %g2
298	FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
2991:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
300	EX_LD_FP(LOAD_BLK(%o4, %f16))
301	FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
302	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
303	FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
304	subcc		%g1, 64, %g1
305	add		%o4, 64, %o4
306	bne,pt		%xcc, 1b
307	 LOAD(prefetch, %o4 + 64, #one_read)
308	ba,pt		%xcc, 195f
309	 nop
310
311130:	sub		%o4, 48, %g2
312	FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
3131:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
314	EX_LD_FP(LOAD_BLK(%o4, %f16))
315	FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
316	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
317	FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
318	subcc		%g1, 64, %g1
319	add		%o4, 64, %o4
320	bne,pt		%xcc, 1b
321	 LOAD(prefetch, %o4 + 64, #one_read)
322	ba,pt		%xcc, 195f
323	 nop
324
325140:	sub		%o4, 40, %g2
326	FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
3271:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
328	EX_LD_FP(LOAD_BLK(%o4, %f16))
329	FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
330	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
331	FREG_MOVE_5(f22, f24, f26, f28, f30)
332	subcc		%g1, 64, %g1
333	add		%o4, 64, %o4
334	bne,pt		%xcc, 1b
335	 LOAD(prefetch, %o4 + 64, #one_read)
336	ba,pt		%xcc, 195f
337	 nop
338
339150:	sub		%o4, 32, %g2
340	FREG_LOAD_4(%g2, f0, f2, f4, f6)
3411:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
342	EX_LD_FP(LOAD_BLK(%o4, %f16))
343	FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
344	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
345	FREG_MOVE_4(f24, f26, f28, f30)
346	subcc		%g1, 64, %g1
347	add		%o4, 64, %o4
348	bne,pt		%xcc, 1b
349	 LOAD(prefetch, %o4 + 64, #one_read)
350	ba,pt		%xcc, 195f
351	 nop
352
353160:	sub		%o4, 24, %g2
354	FREG_LOAD_3(%g2, f0, f2, f4)
3551:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
356	EX_LD_FP(LOAD_BLK(%o4, %f16))
357	FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
358	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
359	FREG_MOVE_3(f26, f28, f30)
360	subcc		%g1, 64, %g1
361	add		%o4, 64, %o4
362	bne,pt		%xcc, 1b
363	 LOAD(prefetch, %o4 + 64, #one_read)
364	ba,pt		%xcc, 195f
365	 nop
366
367170:	sub		%o4, 16, %g2
368	FREG_LOAD_2(%g2, f0, f2)
3691:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
370	EX_LD_FP(LOAD_BLK(%o4, %f16))
371	FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
372	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
373	FREG_MOVE_2(f28, f30)
374	subcc		%g1, 64, %g1
375	add		%o4, 64, %o4
376	bne,pt		%xcc, 1b
377	 LOAD(prefetch, %o4 + 64, #one_read)
378	ba,pt		%xcc, 195f
379	 nop
380
381180:	sub		%o4, 8, %g2
382	FREG_LOAD_1(%g2, f0)
3831:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
384	EX_LD_FP(LOAD_BLK(%o4, %f16))
385	FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
386	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
387	FREG_MOVE_1(f30)
388	subcc		%g1, 64, %g1
389	add		%o4, 64, %o4
390	bne,pt		%xcc, 1b
391	 LOAD(prefetch, %o4 + 64, #one_read)
392	ba,pt		%xcc, 195f
393	 nop
394
395190:
3961:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
397	subcc		%g1, 64, %g1
398	EX_LD_FP(LOAD_BLK(%o4, %f0))
399	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
400	add		%o4, 64, %o4
401	bne,pt		%xcc, 1b
402	 LOAD(prefetch, %o4 + 64, #one_read)
403
404195:
405	add		%o4, %g3, %o0
406	membar		#Sync
407
408	VISExitHalf
409
410	/* %o2 contains any final bytes still needed to be copied
411	 * over. If anything is left, we copy it one byte at a time.
412	 */
413	brz,pt		%o2, 85f
414	 sub		%o0, %o1, GLOBAL_SPARE
415	ba,a,pt		%XCC, 90f
416
417	.align		64
41875: /* 16 < len <= 64 */
419	bne,pn		%XCC, 75f
420	 sub		%o0, %o1, GLOBAL_SPARE
421
42272:
423	andn		%o2, 0xf, %o4
424	and		%o2, 0xf, %o2
4251:	subcc		%o4, 0x10, %o4
426	EX_LD(LOAD(ldx, %o1, %o5))
427	add		%o1, 0x08, %o1
428	EX_LD(LOAD(ldx, %o1, %g1))
429	sub		%o1, 0x08, %o1
430	EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
431	add		%o1, 0x8, %o1
432	EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE))
433	bgu,pt		%XCC, 1b
434	 add		%o1, 0x8, %o1
43573:	andcc		%o2, 0x8, %g0
436	be,pt		%XCC, 1f
437	 nop
438	sub		%o2, 0x8, %o2
439	EX_LD(LOAD(ldx, %o1, %o5))
440	EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
441	add		%o1, 0x8, %o1
4421:	andcc		%o2, 0x4, %g0
443	be,pt		%XCC, 1f
444	 nop
445	sub		%o2, 0x4, %o2
446	EX_LD(LOAD(lduw, %o1, %o5))
447	EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE))
448	add		%o1, 0x4, %o1
4491:	cmp		%o2, 0
450	be,pt		%XCC, 85f
451	 nop
452	ba,pt		%xcc, 90f
453	 nop
454
45575:
456	andcc		%o0, 0x7, %g1
457	sub		%g1, 0x8, %g1
458	be,pn		%icc, 2f
459	 sub		%g0, %g1, %g1
460	sub		%o2, %g1, %o2
461
4621:	subcc		%g1, 1, %g1
463	EX_LD(LOAD(ldub, %o1, %o5))
464	EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE))
465	bgu,pt		%icc, 1b
466	 add		%o1, 1, %o1
467
4682:	add		%o1, GLOBAL_SPARE, %o0
469	andcc		%o1, 0x7, %g1
470	bne,pt		%icc, 8f
471	 sll		%g1, 3, %g1
472
473	cmp		%o2, 16
474	bgeu,pt		%icc, 72b
475	 nop
476	ba,a,pt		%xcc, 73b
477
4788:	mov		64, GLOBAL_SPARE
479	andn		%o1, 0x7, %o1
480	EX_LD(LOAD(ldx, %o1, %g2))
481	sub		GLOBAL_SPARE, %g1, GLOBAL_SPARE
482	andn		%o2, 0x7, %o4
483	sllx		%g2, %g1, %g2
4841:	add		%o1, 0x8, %o1
485	EX_LD(LOAD(ldx, %o1, %g3))
486	subcc		%o4, 0x8, %o4
487	srlx		%g3, GLOBAL_SPARE, %o5
488	or		%o5, %g2, %o5
489	EX_ST(STORE(stx, %o5, %o0))
490	add		%o0, 0x8, %o0
491	bgu,pt		%icc, 1b
492	 sllx		%g3, %g1, %g2
493
494	srl		%g1, 3, %g1
495	andcc		%o2, 0x7, %o2
496	be,pn		%icc, 85f
497	 add		%o1, %g1, %o1
498	ba,pt		%xcc, 90f
499	 sub		%o0, %o1, GLOBAL_SPARE
500
501	.align		64
50280: /* 0 < len <= 16 */
503	andcc		GLOBAL_SPARE, 0x3, %g0
504	bne,pn		%XCC, 90f
505	 sub		%o0, %o1, GLOBAL_SPARE
506
5071:
508	subcc		%o2, 4, %o2
509	EX_LD(LOAD(lduw, %o1, %g1))
510	EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE))
511	bgu,pt		%XCC, 1b
512	 add		%o1, 4, %o1
513
51485:	retl
515	 mov		EX_RETVAL(%o3), %o0
516
517	.align		32
51890:
519	subcc		%o2, 1, %o2
520	EX_LD(LOAD(ldub, %o1, %g1))
521	EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE))
522	bgu,pt		%XCC, 90b
523	 add		%o1, 1, %o1
524	retl
525	 mov		EX_RETVAL(%o3), %o0
526
527	.size		FUNC_NAME, .-FUNC_NAME
528