xref: /titanic_51/usr/src/uts/sun4u/ml/mach_copy.s (revision 7aec1d6e253b21f9e9b7ef68b4d81ab9859b51fe) 
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#pragma ident	"%Z%%M%	%I%	%E% SMI"
28
29#include <sys/param.h>
30#include <sys/errno.h>
31#include <sys/asm_linkage.h>
32#include <sys/vtrace.h>
33#include <sys/machthread.h>
34#include <sys/clock.h>
35#include <sys/asi.h>
36#include <sys/fsr.h>
37#include <sys/privregs.h>
38
39#if !defined(lint)
40#include "assym.h"
41#endif	/* lint */
42
43#define	FP_USED 1
44#define	LOFAULT_SET 2
45
46/*
47 * Error barrier:
48 * We use membar sync to establish an error barrier for
49 * deferred errors. Membar syncs are added before any update
50 * to t_lofault to ensure that deferred errors from earlier
51 * accesses will not be reported after the membar. This error
52 * isolation is important when we try to recover from async
53 * errors which tries to distinguish kernel accesses to user
54 * data.
55 */
56
57/*
58 * Zero a block of storage.
59 *
60 * uzero is used by the kernel to zero a block in user address space.
61 */
62
63#if defined(lint)
64
65/* ARGSUSED */
66int
67kzero(void *addr, size_t count)
68{ return(0); }
69
70/* ARGSUSED */
71void
72uzero(void *addr, size_t count)
73{}
74
75#else	/* lint */
76
77	ENTRY(uzero)
78	!
79	! Set a new lo_fault handler only if we came in with one
80	! already specified.
81	!
82	wr	%g0, ASI_USER, %asi
83	ldn	[THREAD_REG + T_LOFAULT], %o5
84	tst	%o5
85	bz,pt	%ncc, .do_zero
86	sethi	%hi(.zeroerr), %o2
87	or	%o2, %lo(.zeroerr), %o2
88	membar	#Sync
89	ba,pt	%ncc, .do_zero
90	stn	%o2, [THREAD_REG + T_LOFAULT]
91
92	ENTRY(kzero)
93	!
94	! Always set a lo_fault handler
95	!
96	wr	%g0, ASI_P, %asi
97	ldn	[THREAD_REG + T_LOFAULT], %o5
98	sethi	%hi(.zeroerr), %o2
99	or	%o5, LOFAULT_SET, %o5
100	or	%o2, %lo(.zeroerr), %o2
101	membar	#Sync
102	ba,pt	%ncc, .do_zero
103	stn	%o2, [THREAD_REG + T_LOFAULT]
104
105/*
106 * We got here because of a fault during kzero or if
107 * uzero or bzero was called with t_lofault non-zero.
108 * Otherwise we've already run screaming from the room.
109 * Errno value is in %g1. Note that we're here iff
110 * we did set t_lofault.
111 */
112.zeroerr:
113	!
114	! Undo asi register setting. Just set it to be the
115        ! kernel default without checking.
116	!
117	wr	%g0, ASI_P, %asi
118	!
119	! If saved t_lofault has FP_USED set, clear the %fprs register
120	!
121	btst	FP_USED, %o5
122	bz,pt	%ncc, 1f		! skip if not used
123	nop
124	membar #Sync
125	wr	%g0, %g0, %fprs		! clear fprs
126	andn	%o5, FP_USED, %o5	! turn off flag bit
127	!
128	! We did set t_lofault. It may well have been zero coming in.
129	!
1301:
131	tst	%o5
132	membar #Sync
133	bne,pn	%ncc, 3f
134	andncc	%o5, LOFAULT_SET, %o5
1352:
136	!
137	! Old handler was zero. Just return the error.
138	!
139	retl				! return
140	mov	%g1, %o0		! error code from %g1
1413:
142	!
143	! We're here because %o5 was non-zero. It was non-zero
144	! because either LOFAULT_SET was present, a previous fault
145	! handler was present or both. In all cases we need to reset
146	! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET
147	! before we either simply return the error or we invoke the
148	! previously specified handler.
149	!
150	be	%ncc, 2b
151	stn	%o5, [THREAD_REG + T_LOFAULT]
152	jmp	%o5			! goto real handler
153	  nop
154	SET_SIZE(kzero)
155	SET_SIZE(uzero)
156
157#endif	/* lint */
158
159/*
160 * Zero a block of storage.
161 */
162
163#if defined(lint)
164
165/* ARGSUSED */
166void
167bzero(void *addr, size_t count)
168{}
169
170#else	/* lint */
171
172	ENTRY(bzero)
173	wr	%g0, ASI_P, %asi
174
175	ldn	[THREAD_REG + T_LOFAULT], %o5	! save old vector
176	tst	%o5
177	bz,pt	%ncc, .do_zero
178	sethi	%hi(.zeroerr), %o2
179	or	%o2, %lo(.zeroerr), %o2
180	membar	#Sync				! sync error barrier
181	stn	%o2, [THREAD_REG + T_LOFAULT]	! install new vector
182
183.do_zero:
184	cmp	%o1, 15			! check for small counts
185	blu,pn	%ncc, .byteclr		! just clear bytes
186	nop
187
188	cmp	%o1, 192		! check for large counts
189	blu	%ncc, .bzero_small
190	nop
191
192	sethi	%hi(use_hw_bzero), %o2
193	ld	[%o2 + %lo(use_hw_bzero)], %o2
194	tst	%o2
195	bz	%icc, .bzero_small
196	nop
197
198	rd	%fprs, %o2		! check for unused fp
199	btst	FPRS_FEF, %o2
200	bnz	%icc, .bzero_small
201	nop
202
203	ldn	[THREAD_REG + T_LWP], %o2
204	tst	%o2
205	bz,pn	%ncc, .bzero_small
206	nop
207
208	! Check for block alignment
209	btst	(64-1), %o0
210	bz	%icc, .bzl_block
211	nop
212
213	! Check for double-word alignment
214	btst	(8-1), %o0
215	bz	%icc, .bzl_dword
216	nop
217
218	! Check for word alignment
219	btst	(4-1), %o0
220	bz	%icc, .bzl_word
221	nop
222
223	! Clear bytes until word aligned
224.bzl_byte:
225	stba	%g0, [%o0]%asi
226	add	%o0, 1, %o0
227	btst	(4-1), %o0
228	bnz	%icc, .bzl_byte
229	sub	%o1, 1, %o1
230
231	! Check for dword-aligned
232	btst	(8-1), %o0
233	bz	%icc, .bzl_dword
234	nop
235
236	! Clear words until double-word aligned
237.bzl_word:
238	sta	%g0, [%o0]%asi
239	add	%o0, 4, %o0
240	btst	(8-1), %o0
241	bnz	%icc, .bzl_word
242	sub	%o1, 4, %o1
243
244.bzl_dword:
245	! Clear dwords until block aligned
246	stxa	%g0, [%o0]%asi
247	add	%o0, 8, %o0
248	btst	(64-1), %o0
249	bnz	%icc, .bzl_dword
250	sub	%o1, 8, %o1
251
252.bzl_block:
253	membar	#StoreStore|#StoreLoad|#LoadStore
254	wr	%g0, FPRS_FEF, %fprs
255
256	! Set the lower bit in the saved t_lofault to indicate
257	! that we need to clear the %fprs register on the way
258	! out
259	or	%o5, FP_USED, %o5
260
261	! Clear block
262	fzero	%d0
263	fzero	%d2
264	fzero	%d4
265	fzero	%d6
266	fzero	%d8
267	fzero	%d10
268	fzero	%d12
269	fzero	%d14
270	rd	%asi, %o3
271	wr	%g0, ASI_BLK_P, %asi
272	cmp	%o3, ASI_P
273	bne,a	%icc, 1f
274	wr	%g0, ASI_BLK_AIUS, %asi
2751:
276	mov	256, %o3
277	ba,pt	%ncc, .bzl_doblock
278	nop
279
280.bzl_blkstart:
281      ! stda	%d0, [%o0+192]%asi  ! in dly slot of branch that got us here
282	stda	%d0, [%o0+128]%asi
283	stda	%d0, [%o0+64]%asi
284	stda	%d0, [%o0]%asi
285.bzl_zinst:
286	add	%o0, %o3, %o0
287	sub	%o1, %o3, %o1
288.bzl_doblock:
289	cmp	%o1, 256
290	bgeu,a	%ncc, .bzl_blkstart
291	stda	%d0, [%o0+192]%asi
292
293	cmp	%o1, 64
294	blu	%ncc, .bzl_finish
295
296	andn	%o1, (64-1), %o3
297	srl	%o3, 4, %o2		! using blocks, 1 instr / 16 words
298	set	.bzl_zinst, %o4
299	sub	%o4, %o2, %o4
300	jmp	%o4
301	nop
302
303.bzl_finish:
304	membar	#StoreLoad|#StoreStore
305	wr	%g0, %g0, %fprs
306	andn	%o5, FP_USED, %o5
307
308	rd	%asi, %o4
309	wr	%g0, ASI_P, %asi
310	cmp	%o4, ASI_BLK_P
311	bne,a	%icc, 1f
312	wr	%g0, ASI_USER, %asi
3131:
314
315.bzlf_dword:
316	! double words
317	cmp	%o1, 8
318	blu	%ncc, .bzlf_word
319	nop
320	stxa	%g0, [%o0]%asi
321	add	%o0, 8, %o0
322	sub	%o1, 8, %o1
323	ba,pt	%ncc, .bzlf_dword
324	nop
325
326.bzlf_word:
327	! words
328	cmp	%o1, 4
329	blu	%ncc, .bzlf_byte
330	nop
331	sta	%g0, [%o0]%asi
332	add	%o0, 4, %o0
333	sub	%o1, 4, %o1
334	ba,pt	%ncc, .bzlf_word
335	nop
336
3371:
338	add	%o0, 1, %o0		! increment address
339.bzlf_byte:
340	subcc	%o1, 1, %o1		! decrement count
341	bgeu,a	%ncc, 1b
342	stba	%g0, [%o0]%asi		! zero a byte
343
344	!
345	! If we used the FP registers, that bit was turned
346	! off after we were finished. We're just concerned with
347	! whether t_lofault was set when we came in. We end up
348	! here from either kzero() or bzero(). kzero() *always*
349	! sets a lofault handler. It ors LOFAULT_SET into %o5
350	! to indicate it has done this even if the value of %o5
351	! is otherwise zero. bzero() sets a lofault handler *only*
352	! if one was previously set. Accordingly we need to examine
353	! %o5 and if it is non-zero be sure to clear LOFAULT_SET
354	! before resetting the error handler.
355	!
356	tst	%o5
357	bz,pt	%ncc, 1f
358	andn	%o5, LOFAULT_SET, %o5
359	membar	#Sync				! sync error barrier
360	stn	%o5, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
3611:
362	retl
363	clr	%o0			! return (0)
364
365.bzero_small:
366
367	!
368	! Check for word alignment.
369	!
370	btst	3, %o0
371	bz	.bzero_probe
372	mov	0x100, %o3		! constant size of main loop
373	!
374	!
375	! clear bytes until word aligned
376	!
3771:	stba	%g0,[%o0]%asi
378	add	%o0, 1, %o0
379	btst	3, %o0
380	bnz	1b
381	sub	%o1, 1, %o1
382.bzero_probe:
383
384	!
385	! if needed move a word to become double-word aligned.
386	!
387	btst	7, %o0			! is double aligned?
388	bz	%icc, .bzero_nobuf
389	nop
390	sta	%g0, [%o0]%asi		! clr to double boundry
391	sub	%o1, 4, %o1
392	ba,pt	%ncc, .bzero_nobuf
393	add	%o0, 4, %o0
394
395	!stxa	%g0, [%o0+0xf8]%asi
396.bzero_blk:
397	stxa	%g0, [%o0+0xf0]%asi
398	stxa	%g0, [%o0+0xe8]%asi
399	stxa	%g0, [%o0+0xe0]%asi
400	stxa	%g0, [%o0+0xd8]%asi
401	stxa	%g0, [%o0+0xd0]%asi
402	stxa	%g0, [%o0+0xc8]%asi
403	stxa	%g0, [%o0+0xc0]%asi
404	stxa	%g0, [%o0+0xb8]%asi
405	stxa	%g0, [%o0+0xb0]%asi
406	stxa	%g0, [%o0+0xa8]%asi
407	stxa	%g0, [%o0+0xa0]%asi
408	stxa	%g0, [%o0+0x98]%asi
409	stxa	%g0, [%o0+0x90]%asi
410	stxa	%g0, [%o0+0x88]%asi
411	stxa	%g0, [%o0+0x80]%asi
412	stxa	%g0, [%o0+0x78]%asi
413	stxa	%g0, [%o0+0x70]%asi
414	stxa	%g0, [%o0+0x68]%asi
415	stxa	%g0, [%o0+0x60]%asi
416	stxa	%g0, [%o0+0x58]%asi
417	stxa	%g0, [%o0+0x50]%asi
418	stxa	%g0, [%o0+0x48]%asi
419	stxa	%g0, [%o0+0x40]%asi
420	stxa	%g0, [%o0+0x38]%asi
421	stxa	%g0, [%o0+0x30]%asi
422	stxa	%g0, [%o0+0x28]%asi
423	stxa	%g0, [%o0+0x20]%asi
424	stxa	%g0, [%o0+0x18]%asi
425	stxa	%g0, [%o0+0x10]%asi
426	stxa	%g0, [%o0+0x08]%asi
427	stxa	%g0, [%o0]%asi
428.zinst:
429	add	%o0, %o3, %o0		! increment source address
430	sub	%o1, %o3, %o1		! decrement count
431.bzero_nobuf:
432	cmp	%o1, 0x100		! can we do whole chunk?
433	bgeu,a	%ncc, .bzero_blk
434	stxa	%g0, [%o0+0xf8]%asi	! do first double of chunk
435
436	cmp	%o1, 7			! can we zero any more double words
437	bleu	%ncc, .byteclr		! too small go zero bytes
438
439	andn	%o1, 7, %o3		! %o3 bytes left, double-word aligned
440	srl	%o3, 1, %o2		! using doubles, need 1 instr / 2 words
441	set	.zinst, %o4		! address of clr instructions
442	sub	%o4, %o2, %o4		! jmp address relative to instr
443	jmp	%o4
444	nop
445	!
446	! do leftover bytes
447	!
4483:
449	add	%o0, 1, %o0		! increment address
450.byteclr:
451	subcc	%o1, 1, %o1		! decrement count
452	bgeu,a	%ncc, 3b
453	stba	%g0, [%o0]%asi		! zero a byte
454
455.bzero_finished:
456	!
457	! We're just concerned with whether t_lofault was set
458	! when we came in. We end up here from either kzero()
459	! or bzero(). kzero() *always* sets a lofault handler.
460	! It ors LOFAULT_SET into %o5 to indicate it has done
461	! this even if the value of %o5 is otherwise zero.
462	! bzero() sets a lofault handler *only* if one was
463	! previously set. Accordingly we need to examine
464	! %o5 and if it is non-zero be sure to clear LOFAULT_SET
465	! before resetting the error handler.
466	!
467	tst	%o5
468	bz	%ncc, 1f
469	andn	%o5, LOFAULT_SET, %o5
470	membar	#Sync				! sync error barrier
471	stn	%o5, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
4721:
473	retl
474	clr	%o0			! return (0)
475
476	SET_SIZE(bzero)
477#endif	/* lint */
478