xref: /illumos-gate/usr/src/uts/sun4/ml/copy.S (revision 34bbc83afbf22a6f8e504cb99d76c97c017cb5f4)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#include <sys/param.h>
27#include <sys/errno.h>
28#include <sys/asm_linkage.h>
29#include <sys/vtrace.h>
30#include <sys/machthread.h>
31#include <sys/clock.h>
32#include <sys/asi.h>
33#include <sys/fsr.h>
34#include <sys/privregs.h>
35
36#include "assym.h"
37
38/*
39 * Error barrier:
40 * We use membar sync to establish an error barrier for
41 * deferred errors. Membar syncs are added before any update
42 * to t_lofault to ensure that deferred errors from earlier
43 * accesses will not be reported after the membar. This error
44 * isolation is important when we try to recover from async
45 * errors which tries to distinguish kernel accesses to user
46 * data.
47 */
48
49/*
50 * Copy a null terminated string from one point to another in
51 * the kernel address space.
52 * NOTE - don't use %o5 in this routine as copy{in,out}str uses it.
53 *
54 * copystr(from, to, maxlength, lencopied)
55 *	caddr_t from, to;
56 *	u_int maxlength, *lencopied;
57 */
58
59	ENTRY(copystr)
60	orcc	%o2, %g0, %o4		! save original count
61	bg,a	%ncc, 1f
62	  sub	%o0, %o1, %o0		! o0 gets the difference of src and dst
63
64	!
65	! maxlength <= 0
66	!
67	bz	%ncc, .cs_out		! maxlength = 0
68	mov	ENAMETOOLONG, %o0
69
70	b	2f			! maxlength < 0
71	mov	EFAULT, %o0		! return failure
72
73	!
74	! Do a byte by byte loop.
75	! We do this instead of a word by word copy because most strings
76	! are small and this takes a small number of cache lines.
77	!
780:
79	stb	%g1, [%o1]		! store byte
80	tst	%g1
81	bnz,pt	%icc, 1f
82	add	%o1, 1, %o1		! incr dst addr
83
84	ba,pt	%ncc, .cs_out		! last byte in string
85	mov	0, %o0			! ret code = 0
861:
87	subcc	%o2, 1, %o2		! test count
88	bgeu,a	%ncc, 0b
89	ldub	[%o0 + %o1], %g1	! delay slot, get source byte
90
91	mov	0, %o2			! max number of bytes moved
92	mov	ENAMETOOLONG, %o0	! ret code = ENAMETOOLONG
93.cs_out:
94	tst	%o3
95	bz	%ncc, 2f
96	sub	%o4, %o2, %o4		! compute length and store it
97	stn	%o4, [%o3]
982:
99	retl
100	nop
101	SET_SIZE(copystr)
102
103
104/*
105 * Copy a null terminated string from the user address space into
106 * the kernel address space.
107 */
108
109	ENTRY(copyinstr)
110	sethi	%hi(.copyinstr_err), %o4
111	ldn	[THREAD_REG + T_LOFAULT], %o5	! catch faults
112	or	%o4, %lo(.copyinstr_err), %o4
113	membar	#Sync				! sync error barrier
114	stn	%o4, [THREAD_REG + T_LOFAULT]
115
116	brz,a,pn %o2, .copyinstr_out
117	mov	ENAMETOOLONG, %o0
118
119	mov	%o2, %g3		! g3 is the current count
120	mov	%o1, %g4		! g4 is the dest addr
121
122	b	1f
123	sub	%o0, %o1, %g2		! g2 gets the difference of src and dst
124
125	!
126	! Do a byte by byte loop.
127	! We do this instead of a word by word copy because most strings
128	! are small and this takes a small number of cache lines.
129	!
1300:
131	stb	%g1, [%g4]		! store byte
132	tst	%g1
133	bnz,pt	%icc, 1f
134	add	%g4, 1, %g4		! incr dst addr
135
136	ba,pt	%ncc, .copyinstr_out	! last byte in string
137	mov	0, %o0			! ret code = 0
1381:
139	subcc	%g3, 1, %g3		! test count
140	bgeu,a	%ncc, 0b
141	lduba	[%g2+%g4]ASI_USER, %g1	! delay slot, get source byte
142
143	mov	0, %g3			! max number of bytes moved
144	ba,pt	%ncc, .copyinstr_out
145	  mov	ENAMETOOLONG, %o0	! ret code = ENAMETOOLONG
146
147/*
148 * Fault while trying to move from or to user space.
149 * Set and return error code.
150 */
151.copyinstr_err:
152	membar	#Sync			! sync error barrier
153	stn	%o5, [THREAD_REG + T_LOFAULT]
154	ldn	[THREAD_REG + T_COPYOPS], %o4
155	brz	%o4, 1f
156	nop
157	ldn	[%o4 + CP_COPYINSTR], %g1
158	jmp	%g1
159	nop
1601:
161	retl
162	mov	EFAULT, %o0
163.copyinstr_out:
164	tst	%o3			! want length?
165	bz	%ncc, 2f
166	sub	%o2, %g3, %o2		! compute length and store it
167	stn	%o2, [%o3]
1682:
169	membar	#Sync			! sync error barrier
170	retl
171	stn	%o5, [THREAD_REG + T_LOFAULT]	! stop catching faults
172	SET_SIZE(copyinstr)
173
174	ENTRY(copyinstr_noerr)
175	mov	%o2, %o4		! save original count
176
177	! maxlength is unsigned so the only error is if it's 0
178	brz,a,pn %o2, .copyinstr_noerr_out
179	mov	ENAMETOOLONG, %o0
180
181	b	1f
182	sub	%o0, %o1, %o0		! o0 gets the difference of src and dst
183
184	!
185	! Do a byte by byte loop.
186	! We do this instead of a word by word copy because most strings
187	! are small and this takes a small number of cache lines.
188	!
1890:
190	stb	%g1, [%o1]		! store byte
191	tst	%g1			! null byte?
192	bnz	1f
193	add	%o1, 1, %o1		! incr dst addr
194
195	ba,pt	%ncc, .copyinstr_noerr_out	! last byte in string
196	mov	0, %o0			! ret code = 0
1971:
198	subcc	%o2, 1, %o2		! test count
199	bgeu,a	%ncc, 0b
200	lduba	[%o0 + %o1]ASI_USER, %g1	! delay slot, get source byte
201
202	mov	0, %o2			! max number of bytes moved
203	b	.copyinstr_noerr_out
204	  mov	ENAMETOOLONG, %o0	! ret code = ENAMETOOLONG
205.copyinstr_noerr_out:
206	tst	%o3			! want length?
207	bz	%ncc, 2f
208	sub	%o4, %o2, %o4
209	stn	%o4, [%o3]
2102:
211	retl
212	nop
213	SET_SIZE(copyinstr_noerr)
214
215/*
216 * Copy a null terminated string from the kernel
217 * address space to the user address space.
218 */
219
220	ENTRY(copyoutstr)
221	sethi	%hi(.copyoutstr_err), %o5
222	ldn	[THREAD_REG + T_LOFAULT], %o4	! catch faults
223	or	%o5, %lo(.copyoutstr_err), %o5
224	membar	#Sync				! sync error barrier
225	stn	%o5, [THREAD_REG + T_LOFAULT]
226	mov	%o4, %o5
227
228	brz,a,pn %o2, .copyoutstr_out
229	mov	ENAMETOOLONG, %o0
230
231	mov	%o2, %g3		! g3 is the current count
232	mov	%o1, %g4		! g4 is the dest addr
233
234	b	1f
235	sub	%o0, %o1, %g2		! g2 gets the difference of src and dst
236
237	!
238	! Do a byte by byte loop.
239	! We do this instead of a word by word copy because most strings
240	! are small and this takes a small number of cache lines.
241	!
2420:
243	stba	%g1, [%g4]ASI_USER	! store byte
244	tst	%g1
245	bnz,pt	%icc, 1f
246	add	%g4, 1, %g4		! incr dst addr
247
248	ba,pt	%ncc, .copyoutstr_out	! last byte in string
249	mov	0, %o0			! ret code = 0
2501:
251	subcc	%g3, 1, %g3		! test count
252	bgeu,a	%ncc, 0b
253	ldub	[%g2 + %g4], %g1	! delay slot, get source byte
254
255	mov	0, %g3			! max number of bytes moved
256	ba,pt	%ncc, .copyoutstr_out
257	  mov	ENAMETOOLONG, %o0	! ret code = ENAMETOOLONG
258
259/*
260 * Fault while trying to move from or to user space.
261 * Set and return error code.
262 */
263.copyoutstr_err:
264	membar	#Sync			! sync error barrier
265	stn	%o5, [THREAD_REG + T_LOFAULT]
266	ldn	[THREAD_REG + T_COPYOPS], %o4
267	brz	%o4, 1f
268	nop
269	ldn	[%o4 + CP_COPYOUTSTR], %g1
270	jmp	%g1
271	nop
2721:
273	retl
274	mov	EFAULT, %o0
275.copyoutstr_out:
276	tst	%o3			! want length?
277	bz	%ncc, 2f
278	sub	%o2, %g3, %o2		! compute length and store it
279	stn	%o2, [%o3]
2802:
281	membar	#Sync			! sync error barrier
282	retl
283	stn	%o5, [THREAD_REG + T_LOFAULT]	! stop catching faults
284	SET_SIZE(copyoutstr)
285
286	ENTRY(copyoutstr_noerr)
287	mov	%o2, %o4		! save original count
288
289	brz,a,pn %o2, .copyoutstr_noerr_out
290	mov	ENAMETOOLONG, %o0
291
292	b	1f
293	sub	%o0, %o1, %o0		! o0 gets the difference of src and dst
294
295	!
296	! Do a byte by byte loop.
297	! We do this instead of a word by word copy because most strings
298	! are small and this takes a small number of cache lines.
299	!
3000:
301	stba	%g1, [%o1]ASI_USER	! store byte
302	tst	%g1			! null byte?
303	bnz	1f
304	add	%o1, 1, %o1		! incr dst addr
305
306	b	.copyoutstr_noerr_out	! last byte in string
307	mov	0, %o0			! ret code = 0
3081:
309	subcc	%o2, 1, %o2		! test count
310	bgeu,a	%ncc, 0b
311	ldub	[%o0+%o1], %g1	! delay slot, get source byte
312
313	mov	0, %o2			! max number of bytes moved
314	b	.copyoutstr_noerr_out
315	  mov	ENAMETOOLONG, %o0	! ret code = ENAMETOOLONG
316.copyoutstr_noerr_out:
317	tst	%o3			! want length?
318	bz	%ncc, 2f
319	sub	%o4, %o2, %o4
320	stn	%o4, [%o3]
3212:
322	retl
323	nop
324	SET_SIZE(copyoutstr_noerr)
325
326
327/*
328 * Copy a block of storage.  If the source and target regions overlap,
329 * one or both of the regions will be silently corrupted.
330 * No fault handler installed (to be called under on_fault())
331 */
332
333	ENTRY(ucopy)
334	save	%sp, -SA(MINFRAME), %sp ! get another window
335
336	subcc	%g0, %i2, %i3
337	add	%i0, %i2, %i0
338	bz,pn	%ncc, 5f
339	add	%i1, %i2, %i1
340	lduba	[%i0 + %i3]ASI_USER, %i4
3414:	stba	%i4, [%i1 + %i3]ASI_USER
342	inccc	%i3
343	bcc,a,pt %ncc, 4b
344	lduba  [%i0 + %i3]ASI_USER, %i4
3455:
346	ret
347	restore %g0, 0, %o0		! return (0)
348
349	SET_SIZE(ucopy)
350
351/*
352 * Copy a user-land string.  If the source and target regions overlap,
353 * one or both of the regions will be silently corrupted.
354 * No fault handler installed (to be called under on_fault())
355 */
356
357	ENTRY(ucopystr)
358	save	%sp, -SA(MINFRAME), %sp ! get another window
359
360	brz	%i2, 5f
361	clr	%i5
362
363	lduba	[%i0 + %i5]ASI_USER, %i4
3644:	stba	%i4, [%i1 + %i5]ASI_USER
365	brz,pn	%i4, 5f
366	inc	%i5
367	deccc	%i2
368	bnz,a,pt %ncc, 4b
369	lduba	[%i0 + %i5]ASI_USER, %i4
3705:
371	brnz,a,pt %i3, 6f
372	stn	%i5, [%i3]
3736:
374	ret
375	restore %g0, 0, %o0		! return (0)
376
377	SET_SIZE(ucopystr)
378