xref: /linux/arch/sparc/lib/U1memcpy.S (revision e58e871becec2d3b04ed91c0c16fe8deac9c9dfa) 
1/* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy.
2 *
3 * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
4 * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
5 */
6
7#ifdef __KERNEL__
8#include <linux/linkage.h>
9#include <asm/visasm.h>
10#include <asm/asi.h>
11#include <asm/export.h>
12#define GLOBAL_SPARE	g7
13#else
14#define GLOBAL_SPARE	g5
15#define ASI_BLK_P 0xf0
16#define FPRS_FEF  0x04
17#ifdef MEMCPY_DEBUG
18#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
19		 clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
20#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
21#else
22#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
23#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
24#endif
25#endif
26
27#ifndef EX_LD
28#define EX_LD(x,y)	x
29#endif
30#ifndef EX_LD_FP
31#define EX_LD_FP(x,y)	x
32#endif
33
34#ifndef EX_ST
35#define EX_ST(x,y)	x
36#endif
37#ifndef EX_ST_FP
38#define EX_ST_FP(x,y)	x
39#endif
40
41#ifndef LOAD
42#define LOAD(type,addr,dest)	type [addr], dest
43#endif
44
45#ifndef LOAD_BLK
46#define LOAD_BLK(addr,dest)	ldda [addr] ASI_BLK_P, dest
47#endif
48
49#ifndef STORE
50#define STORE(type,src,addr)	type src, [addr]
51#endif
52
53#ifndef STORE_BLK
54#define STORE_BLK(src,addr)	stda src, [addr] ASI_BLK_P
55#endif
56
57#ifndef FUNC_NAME
58#define FUNC_NAME	memcpy
59#endif
60
61#ifndef PREAMBLE
62#define PREAMBLE
63#endif
64
65#ifndef XCC
66#define XCC xcc
67#endif
68
69#define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9)		\
70	faligndata		%f1, %f2, %f48;			\
71	faligndata		%f2, %f3, %f50;			\
72	faligndata		%f3, %f4, %f52;			\
73	faligndata		%f4, %f5, %f54;			\
74	faligndata		%f5, %f6, %f56;			\
75	faligndata		%f6, %f7, %f58;			\
76	faligndata		%f7, %f8, %f60;			\
77	faligndata		%f8, %f9, %f62;
78
79#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt)			\
80	EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp);			\
81	EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp);			\
82	add			%src, 0x40, %src;			\
83	subcc			%GLOBAL_SPARE, 0x40, %GLOBAL_SPARE;	\
84	be,pn			%xcc, jmptgt;				\
85	 add			%dest, 0x40, %dest;			\
86
87#define LOOP_CHUNK1(src, dest, branch_dest)		\
88	MAIN_LOOP_CHUNK(src, dest, f0,  f48, branch_dest)
89#define LOOP_CHUNK2(src, dest, branch_dest)		\
90	MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest)
91#define LOOP_CHUNK3(src, dest, branch_dest)		\
92	MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest)
93
94#define DO_SYNC			membar	#Sync;
95#define STORE_SYNC(dest, fsrc)				\
96	EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp);	\
97	add			%dest, 0x40, %dest;	\
98	DO_SYNC
99
100#define STORE_JUMP(dest, fsrc, target)			\
101	EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp);	\
102	add			%dest, 0x40, %dest;	\
103	ba,pt			%xcc, target;		\
104	 nop;
105
106#define FINISH_VISCHUNK(dest, f0, f1)			\
107	subcc			%g3, 8, %g3;		\
108	bl,pn			%xcc, 95f;		\
109	 faligndata		%f0, %f1, %f48;		\
110	EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp);	\
111	add			%dest, 8, %dest;
112
113#define UNEVEN_VISCHUNK_LAST(dest, f0, f1)	\
114	subcc			%g3, 8, %g3;	\
115	bl,pn			%xcc, 95f;	\
116	 fsrc2			%f0, %f1;
117
118#define UNEVEN_VISCHUNK(dest, f0, f1)		\
119	UNEVEN_VISCHUNK_LAST(dest, f0, f1)	\
120	ba,a,pt			%xcc, 93f;
121
122	.register	%g2,#scratch
123	.register	%g3,#scratch
124
125	.text
126#ifndef EX_RETVAL
127#define EX_RETVAL(x)	x
128ENTRY(U1_g1_1_fp)
129	VISExitHalf
130	add		%g1, 1, %g1
131	add		%g1, %g2, %g1
132	retl
133	 add		%g1, %o2, %o0
134ENDPROC(U1_g1_1_fp)
135ENTRY(U1_g2_0_fp)
136	VISExitHalf
137	retl
138	 add		%g2, %o2, %o0
139ENDPROC(U1_g2_0_fp)
140ENTRY(U1_g2_8_fp)
141	VISExitHalf
142	add		%g2, 8, %g2
143	retl
144	 add		%g2, %o2, %o0
145ENDPROC(U1_g2_8_fp)
146ENTRY(U1_gs_0_fp)
147	VISExitHalf
148	add		%GLOBAL_SPARE, %g3, %o0
149	retl
150	 add		%o0, %o2, %o0
151ENDPROC(U1_gs_0_fp)
152ENTRY(U1_gs_80_fp)
153	VISExitHalf
154	add		%GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
155	add		%GLOBAL_SPARE, %g3, %o0
156	retl
157	 add		%o0, %o2, %o0
158ENDPROC(U1_gs_80_fp)
159ENTRY(U1_gs_40_fp)
160	VISExitHalf
161	add		%GLOBAL_SPARE, 0x40, %GLOBAL_SPARE
162	add		%GLOBAL_SPARE, %g3, %o0
163	retl
164	 add		%o0, %o2, %o0
165ENDPROC(U1_gs_40_fp)
166ENTRY(U1_g3_0_fp)
167	VISExitHalf
168	retl
169	 add		%g3, %o2, %o0
170ENDPROC(U1_g3_0_fp)
171ENTRY(U1_g3_8_fp)
172	VISExitHalf
173	add		%g3, 8, %g3
174	retl
175	 add		%g3, %o2, %o0
176ENDPROC(U1_g3_8_fp)
177ENTRY(U1_o2_0_fp)
178	VISExitHalf
179	retl
180	 mov		%o2, %o0
181ENDPROC(U1_o2_0_fp)
182ENTRY(U1_o2_1_fp)
183	VISExitHalf
184	retl
185	 add		%o2, 1, %o0
186ENDPROC(U1_o2_1_fp)
187ENTRY(U1_gs_0)
188	VISExitHalf
189	retl
190	 add		%GLOBAL_SPARE, %o2, %o0
191ENDPROC(U1_gs_0)
192ENTRY(U1_gs_8)
193	VISExitHalf
194	add		%GLOBAL_SPARE, %o2, %GLOBAL_SPARE
195	retl
196	 add		%GLOBAL_SPARE, 0x8, %o0
197ENDPROC(U1_gs_8)
198ENTRY(U1_gs_10)
199	VISExitHalf
200	add		%GLOBAL_SPARE, %o2, %GLOBAL_SPARE
201	retl
202	 add		%GLOBAL_SPARE, 0x10, %o0
203ENDPROC(U1_gs_10)
204ENTRY(U1_o2_0)
205	retl
206	 mov		%o2, %o0
207ENDPROC(U1_o2_0)
208ENTRY(U1_o2_8)
209	retl
210	 add		%o2, 8, %o0
211ENDPROC(U1_o2_8)
212ENTRY(U1_o2_4)
213	retl
214	 add		%o2, 4, %o0
215ENDPROC(U1_o2_4)
216ENTRY(U1_o2_1)
217	retl
218	 add		%o2, 1, %o0
219ENDPROC(U1_o2_1)
220ENTRY(U1_g1_0)
221	retl
222	 add		%g1, %o2, %o0
223ENDPROC(U1_g1_0)
224ENTRY(U1_g1_1)
225	add		%g1, 1, %g1
226	retl
227	 add		%g1, %o2, %o0
228ENDPROC(U1_g1_1)
229ENTRY(U1_gs_0_o2_adj)
230	and		%o2, 7, %o2
231	retl
232	 add		%GLOBAL_SPARE, %o2, %o0
233ENDPROC(U1_gs_0_o2_adj)
234ENTRY(U1_gs_8_o2_adj)
235	and		%o2, 7, %o2
236	add		%GLOBAL_SPARE, 8, %GLOBAL_SPARE
237	retl
238	 add		%GLOBAL_SPARE, %o2, %o0
239ENDPROC(U1_gs_8_o2_adj)
240#endif
241
242	.align		64
243
244	.globl		FUNC_NAME
245	.type		FUNC_NAME,#function
246FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
247	srlx		%o2, 31, %g2
248	cmp		%g2, 0
249	tne		%xcc, 5
250	PREAMBLE
251	mov		%o0, %o4
252	cmp		%o2, 0
253	be,pn		%XCC, 85f
254	 or		%o0, %o1, %o3
255	cmp		%o2, 16
256	blu,a,pn	%XCC, 80f
257	 or		%o3, %o2, %o3
258
259	cmp		%o2, (5 * 64)
260	blu,pt		%XCC, 70f
261	 andcc		%o3, 0x7, %g0
262
263	/* Clobbers o5/g1/g2/g3/g7/icc/xcc.  */
264	VISEntry
265
266	/* Is 'dst' already aligned on an 64-byte boundary? */
267	andcc		%o0, 0x3f, %g2
268	be,pt		%XCC, 2f
269
270	/* Compute abs((dst & 0x3f) - 0x40) into %g2.  This is the number
271	 * of bytes to copy to make 'dst' 64-byte aligned.  We pre-
272	 * subtract this from 'len'.
273	 */
274	 sub		%o0, %o1, %GLOBAL_SPARE
275	sub		%g2, 0x40, %g2
276	sub		%g0, %g2, %g2
277	sub		%o2, %g2, %o2
278	andcc		%g2, 0x7, %g1
279	be,pt		%icc, 2f
280	 and		%g2, 0x38, %g2
281
2821:	subcc		%g1, 0x1, %g1
283	EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp)
284	EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp)
285	bgu,pt		%XCC, 1b
286	 add		%o1, 0x1, %o1
287
288	add		%o1, %GLOBAL_SPARE, %o0
289
2902:	cmp		%g2, 0x0
291	and		%o1, 0x7, %g1
292	be,pt		%icc, 3f
293	 alignaddr	%o1, %g0, %o1
294
295	EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp)
2961:	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp)
297	add		%o1, 0x8, %o1
298	subcc		%g2, 0x8, %g2
299	faligndata	%f4, %f6, %f0
300	EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
301	be,pn		%icc, 3f
302	 add		%o0, 0x8, %o0
303
304	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp)
305	add		%o1, 0x8, %o1
306	subcc		%g2, 0x8, %g2
307	faligndata	%f6, %f4, %f0
308	EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
309	bne,pt		%icc, 1b
310	 add		%o0, 0x8, %o0
311
312	/* Destination is 64-byte aligned.  */
3133:
314	membar		  #LoadStore | #StoreStore | #StoreLoad
315
316	subcc		%o2, 0x40, %GLOBAL_SPARE
317	add		%o1, %g1, %g1
318	andncc		%GLOBAL_SPARE, (0x40 - 1), %GLOBAL_SPARE
319	srl		%g1, 3, %g2
320	sub		%o2, %GLOBAL_SPARE, %g3
321	andn		%o1, (0x40 - 1), %o1
322	and		%g2, 7, %g2
323	andncc		%g3, 0x7, %g3
324	fsrc2		%f0, %f2
325	sub		%g3, 0x8, %g3
326	sub		%o2, %GLOBAL_SPARE, %o2
327
328	add		%g1, %GLOBAL_SPARE, %g1
329	subcc		%o2, %g3, %o2
330
331	EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp)
332	add		%o1, 0x40, %o1
333	add		%g1, %g3, %g1
334	EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp)
335	add		%o1, 0x40, %o1
336	sub		%GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
337	EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp)
338	add		%o1, 0x40, %o1
339
340	/* There are 8 instances of the unrolled loop,
341	 * one for each possible alignment of the
342	 * source buffer.  Each loop instance is 452
343	 * bytes.
344	 */
345	sll		%g2, 3, %o3
346	sub		%o3, %g2, %o3
347	sllx		%o3, 4, %o3
348	add		%o3, %g2, %o3
349	sllx		%o3, 2, %g2
3501:	rd		%pc, %o3
351	add		%o3, %lo(1f - 1b), %o3
352	jmpl		%o3 + %g2, %g0
353	 nop
354
355	.align		64
3561:	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
357	LOOP_CHUNK1(o1, o0, 1f)
358	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
359	LOOP_CHUNK2(o1, o0, 2f)
360	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
361	LOOP_CHUNK3(o1, o0, 3f)
362	ba,pt		%xcc, 1b+4
363	 faligndata	%f0, %f2, %f48
3641:	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
365	STORE_SYNC(o0, f48)
366	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
367	STORE_JUMP(o0, f48, 40f)
3682:	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
369	STORE_SYNC(o0, f48)
370	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
371	STORE_JUMP(o0, f48, 48f)
3723:	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
373	STORE_SYNC(o0, f48)
374	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
375	STORE_JUMP(o0, f48, 56f)
376
3771:	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
378	LOOP_CHUNK1(o1, o0, 1f)
379	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
380	LOOP_CHUNK2(o1, o0, 2f)
381	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
382	LOOP_CHUNK3(o1, o0, 3f)
383	ba,pt		%xcc, 1b+4
384	 faligndata	%f2, %f4, %f48
3851:	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
386	STORE_SYNC(o0, f48)
387	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
388	STORE_JUMP(o0, f48, 41f)
3892:	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
390	STORE_SYNC(o0, f48)
391	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
392	STORE_JUMP(o0, f48, 49f)
3933:	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
394	STORE_SYNC(o0, f48)
395	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
396	STORE_JUMP(o0, f48, 57f)
397
3981:	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
399	LOOP_CHUNK1(o1, o0, 1f)
400	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
401	LOOP_CHUNK2(o1, o0, 2f)
402	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
403	LOOP_CHUNK3(o1, o0, 3f)
404	ba,pt		%xcc, 1b+4
405	 faligndata	%f4, %f6, %f48
4061:	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
407	STORE_SYNC(o0, f48)
408	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
409	STORE_JUMP(o0, f48, 42f)
4102:	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
411	STORE_SYNC(o0, f48)
412	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
413	STORE_JUMP(o0, f48, 50f)
4143:	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
415	STORE_SYNC(o0, f48)
416	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
417	STORE_JUMP(o0, f48, 58f)
418
4191:	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
420	LOOP_CHUNK1(o1, o0, 1f)
421	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
422	LOOP_CHUNK2(o1, o0, 2f)
423	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
424	LOOP_CHUNK3(o1, o0, 3f)
425	ba,pt		%xcc, 1b+4
426	 faligndata	%f6, %f8, %f48
4271:	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
428	STORE_SYNC(o0, f48)
429	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
430	STORE_JUMP(o0, f48, 43f)
4312:	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
432	STORE_SYNC(o0, f48)
433	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
434	STORE_JUMP(o0, f48, 51f)
4353:	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
436	STORE_SYNC(o0, f48)
437	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
438	STORE_JUMP(o0, f48, 59f)
439
4401:	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
441	LOOP_CHUNK1(o1, o0, 1f)
442	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
443	LOOP_CHUNK2(o1, o0, 2f)
444	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
445	LOOP_CHUNK3(o1, o0, 3f)
446	ba,pt		%xcc, 1b+4
447	 faligndata	%f8, %f10, %f48
4481:	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
449	STORE_SYNC(o0, f48)
450	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
451	STORE_JUMP(o0, f48, 44f)
4522:	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
453	STORE_SYNC(o0, f48)
454	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
455	STORE_JUMP(o0, f48, 52f)
4563:	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
457	STORE_SYNC(o0, f48)
458	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
459	STORE_JUMP(o0, f48, 60f)
460
4611:	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
462	LOOP_CHUNK1(o1, o0, 1f)
463	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
464	LOOP_CHUNK2(o1, o0, 2f)
465	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
466	LOOP_CHUNK3(o1, o0, 3f)
467	ba,pt		%xcc, 1b+4
468	 faligndata	%f10, %f12, %f48
4691:	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
470	STORE_SYNC(o0, f48)
471	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
472	STORE_JUMP(o0, f48, 45f)
4732:	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
474	STORE_SYNC(o0, f48)
475	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
476	STORE_JUMP(o0, f48, 53f)
4773:	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
478	STORE_SYNC(o0, f48)
479	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
480	STORE_JUMP(o0, f48, 61f)
481
4821:	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
483	LOOP_CHUNK1(o1, o0, 1f)
484	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
485	LOOP_CHUNK2(o1, o0, 2f)
486	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
487	LOOP_CHUNK3(o1, o0, 3f)
488	ba,pt		%xcc, 1b+4
489	 faligndata	%f12, %f14, %f48
4901:	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
491	STORE_SYNC(o0, f48)
492	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
493	STORE_JUMP(o0, f48, 46f)
4942:	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
495	STORE_SYNC(o0, f48)
496	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
497	STORE_JUMP(o0, f48, 54f)
4983:	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
499	STORE_SYNC(o0, f48)
500	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
501	STORE_JUMP(o0, f48, 62f)
502
5031:	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
504	LOOP_CHUNK1(o1, o0, 1f)
505	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
506	LOOP_CHUNK2(o1, o0, 2f)
507	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
508	LOOP_CHUNK3(o1, o0, 3f)
509	ba,pt		%xcc, 1b+4
510	 faligndata	%f14, %f16, %f48
5111:	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
512	STORE_SYNC(o0, f48)
513	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
514	STORE_JUMP(o0, f48, 47f)
5152:	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
516	STORE_SYNC(o0, f48)
517	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
518	STORE_JUMP(o0, f48, 55f)
5193:	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
520	STORE_SYNC(o0, f48)
521	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
522	STORE_JUMP(o0, f48, 63f)
523
52440:	FINISH_VISCHUNK(o0, f0,  f2)
52541:	FINISH_VISCHUNK(o0, f2,  f4)
52642:	FINISH_VISCHUNK(o0, f4,  f6)
52743:	FINISH_VISCHUNK(o0, f6,  f8)
52844:	FINISH_VISCHUNK(o0, f8,  f10)
52945:	FINISH_VISCHUNK(o0, f10, f12)
53046:	FINISH_VISCHUNK(o0, f12, f14)
53147:	UNEVEN_VISCHUNK(o0, f14, f0)
53248:	FINISH_VISCHUNK(o0, f16, f18)
53349:	FINISH_VISCHUNK(o0, f18, f20)
53450:	FINISH_VISCHUNK(o0, f20, f22)
53551:	FINISH_VISCHUNK(o0, f22, f24)
53652:	FINISH_VISCHUNK(o0, f24, f26)
53753:	FINISH_VISCHUNK(o0, f26, f28)
53854:	FINISH_VISCHUNK(o0, f28, f30)
53955:	UNEVEN_VISCHUNK(o0, f30, f0)
54056:	FINISH_VISCHUNK(o0, f32, f34)
54157:	FINISH_VISCHUNK(o0, f34, f36)
54258:	FINISH_VISCHUNK(o0, f36, f38)
54359:	FINISH_VISCHUNK(o0, f38, f40)
54460:	FINISH_VISCHUNK(o0, f40, f42)
54561:	FINISH_VISCHUNK(o0, f42, f44)
54662:	FINISH_VISCHUNK(o0, f44, f46)
54763:	UNEVEN_VISCHUNK_LAST(o0, f46, f0)
548
54993:	EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp)
550	add		%o1, 8, %o1
551	subcc		%g3, 8, %g3
552	faligndata	%f0, %f2, %f8
553	EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
554	bl,pn		%xcc, 95f
555	 add		%o0, 8, %o0
556	EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp)
557	add		%o1, 8, %o1
558	subcc		%g3, 8, %g3
559	faligndata	%f2, %f0, %f8
560	EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
561	bge,pt		%xcc, 93b
562	 add		%o0, 8, %o0
563
56495:	brz,pt		%o2, 2f
565	 mov		%g1, %o1
566
5671:	EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp)
568	add		%o1, 1, %o1
569	subcc		%o2, 1, %o2
570	EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp)
571	bne,pt		%xcc, 1b
572	 add		%o0, 1, %o0
573
5742:	membar		#StoreLoad | #StoreStore
575	VISExit
576	retl
577	 mov		EX_RETVAL(%o4), %o0
578
579	.align		64
58070:	/* 16 < len <= (5 * 64) */
581	bne,pn		%XCC, 75f
582	 sub		%o0, %o1, %o3
583
58472:	andn		%o2, 0xf, %GLOBAL_SPARE
585	and		%o2, 0xf, %o2
5861:	EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0)
587	EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0)
588	subcc		%GLOBAL_SPARE, 0x10, %GLOBAL_SPARE
589	EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10)
590	add		%o1, 0x8, %o1
591	EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8)
592	bgu,pt		%XCC, 1b
593	 add		%o1, 0x8, %o1
59473:	andcc		%o2, 0x8, %g0
595	be,pt		%XCC, 1f
596	 nop
597	EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0)
598	sub		%o2, 0x8, %o2
599	EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8)
600	add		%o1, 0x8, %o1
6011:	andcc		%o2, 0x4, %g0
602	be,pt		%XCC, 1f
603	 nop
604	EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0)
605	sub		%o2, 0x4, %o2
606	EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4)
607	add		%o1, 0x4, %o1
6081:	cmp		%o2, 0
609	be,pt		%XCC, 85f
610	 nop
611	ba,pt		%xcc, 90f
612	 nop
613
61475:	andcc		%o0, 0x7, %g1
615	sub		%g1, 0x8, %g1
616	be,pn		%icc, 2f
617	 sub		%g0, %g1, %g1
618	sub		%o2, %g1, %o2
619
6201:	EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0)
621	subcc		%g1, 1, %g1
622	EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1)
623	bgu,pt		%icc, 1b
624	 add		%o1, 1, %o1
625
6262:	add		%o1, %o3, %o0
627	andcc		%o1, 0x7, %g1
628	bne,pt		%icc, 8f
629	 sll		%g1, 3, %g1
630
631	cmp		%o2, 16
632	bgeu,pt		%icc, 72b
633	 nop
634	ba,a,pt		%xcc, 73b
635
6368:	mov		64, %o3
637	andn		%o1, 0x7, %o1
638	EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0)
639	sub		%o3, %g1, %o3
640	andn		%o2, 0x7, %GLOBAL_SPARE
641	sllx		%g2, %g1, %g2
6421:	EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj)
643	subcc		%GLOBAL_SPARE, 0x8, %GLOBAL_SPARE
644	add		%o1, 0x8, %o1
645	srlx		%g3, %o3, %o5
646	or		%o5, %g2, %o5
647	EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj)
648	add		%o0, 0x8, %o0
649	bgu,pt		%icc, 1b
650	 sllx		%g3, %g1, %g2
651
652	srl		%g1, 3, %g1
653	andcc		%o2, 0x7, %o2
654	be,pn		%icc, 85f
655	 add		%o1, %g1, %o1
656	ba,pt		%xcc, 90f
657	 sub		%o0, %o1, %o3
658
659	.align		64
66080:	/* 0 < len <= 16 */
661	andcc		%o3, 0x3, %g0
662	bne,pn		%XCC, 90f
663	 sub		%o0, %o1, %o3
664
6651:	EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0)
666	subcc		%o2, 4, %o2
667	EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4)
668	bgu,pt		%XCC, 1b
669	 add		%o1, 4, %o1
670
67185:	retl
672	 mov		EX_RETVAL(%o4), %o0
673
674	.align		32
67590:	EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0)
676	subcc		%o2, 1, %o2
677	EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1)
678	bgu,pt		%XCC, 90b
679	 add		%o1, 1, %o1
680	retl
681	 mov		EX_RETVAL(%o4), %o0
682
683	.size		FUNC_NAME, .-FUNC_NAME
684EXPORT_SYMBOL(FUNC_NAME)
685