xref: /illumos-gate/usr/src/uts/sun4u/ml/mach_copy.S (revision 55fea89dcaa64928bed4327112404dcb3e07b79f)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include <sys/param.h>
28#include <sys/errno.h>
29#include <sys/asm_linkage.h>
30#include <sys/vtrace.h>
31#include <sys/machthread.h>
32#include <sys/clock.h>
33#include <sys/asi.h>
34#include <sys/fsr.h>
35#include <sys/privregs.h>
36
37#include "assym.h"
38
39#define	FP_USED 1
40#define	LOFAULT_SET 2
41
42/*
43 * Error barrier:
44 * We use membar sync to establish an error barrier for
45 * deferred errors. Membar syncs are added before any update
46 * to t_lofault to ensure that deferred errors from earlier
47 * accesses will not be reported after the membar. This error
48 * isolation is important when we try to recover from async
49 * errors which tries to distinguish kernel accesses to user
50 * data.
51 */
52
53/*
54 * Zero a block of storage.
55 *
56 * uzero is used by the kernel to zero a block in user address space.
57 */
58
59	ENTRY(uzero)
60	!
61	! Set a new lo_fault handler only if we came in with one
62	! already specified.
63	!
64	wr	%g0, ASI_USER, %asi
65	ldn	[THREAD_REG + T_LOFAULT], %o5
66	tst	%o5
67	bz,pt	%ncc, .do_zero
68	sethi	%hi(.zeroerr), %o2
69	or	%o2, %lo(.zeroerr), %o2
70	membar	#Sync
71	ba,pt	%ncc, .do_zero
72	stn	%o2, [THREAD_REG + T_LOFAULT]
73
74	ENTRY(kzero)
75	!
76	! Always set a lo_fault handler
77	!
78	wr	%g0, ASI_P, %asi
79	ldn	[THREAD_REG + T_LOFAULT], %o5
80	sethi	%hi(.zeroerr), %o2
81	or	%o5, LOFAULT_SET, %o5
82	or	%o2, %lo(.zeroerr), %o2
83	membar	#Sync
84	ba,pt	%ncc, .do_zero
85	stn	%o2, [THREAD_REG + T_LOFAULT]
86
87/*
88 * We got here because of a fault during kzero or if
89 * uzero or bzero was called with t_lofault non-zero.
90 * Otherwise we've already run screaming from the room.
91 * Errno value is in %g1. Note that we're here iff
92 * we did set t_lofault.
93 */
94.zeroerr:
95	!
96	! Undo asi register setting. Just set it to be the
97        ! kernel default without checking.
98	!
99	wr	%g0, ASI_P, %asi
100	!
101	! If saved t_lofault has FP_USED set, clear the %fprs register
102	!
103	btst	FP_USED, %o5
104	bz,pt	%ncc, 1f		! skip if not used
105	nop
106	membar #Sync
107	wr	%g0, %g0, %fprs		! clear fprs
108	andn	%o5, FP_USED, %o5	! turn off flag bit
109	!
110	! We did set t_lofault. It may well have been zero coming in.
111	!
1121:
113	tst	%o5
114	membar #Sync
115	bne,pn	%ncc, 3f
116	andncc	%o5, LOFAULT_SET, %o5
1172:
118	!
119	! Old handler was zero. Just return the error.
120	!
121	retl				! return
122	mov	%g1, %o0		! error code from %g1
1233:
124	!
125	! We're here because %o5 was non-zero. It was non-zero
126	! because either LOFAULT_SET was present, a previous fault
127	! handler was present or both. In all cases we need to reset
128	! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET
129	! before we either simply return the error or we invoke the
130	! previously specified handler.
131	!
132	be	%ncc, 2b
133	stn	%o5, [THREAD_REG + T_LOFAULT]
134	jmp	%o5			! goto real handler
135	  nop
136	SET_SIZE(kzero)
137	SET_SIZE(uzero)
138
139/*
140 * Zero a block of storage.
141 */
142
143	ENTRY(bzero)
144	wr	%g0, ASI_P, %asi
145
146	ldn	[THREAD_REG + T_LOFAULT], %o5	! save old vector
147	tst	%o5
148	bz,pt	%ncc, .do_zero
149	sethi	%hi(.zeroerr), %o2
150	or	%o2, %lo(.zeroerr), %o2
151	membar	#Sync				! sync error barrier
152	stn	%o2, [THREAD_REG + T_LOFAULT]	! install new vector
153
154.do_zero:
155	cmp	%o1, 15			! check for small counts
156	blu,pn	%ncc, .byteclr		! just clear bytes
157	nop
158
159	cmp	%o1, 192		! check for large counts
160	blu	%ncc, .bzero_small
161	nop
162
163	sethi	%hi(use_hw_bzero), %o2
164	ld	[%o2 + %lo(use_hw_bzero)], %o2
165	tst	%o2
166	bz	%icc, .bzero_small
167	nop
168
169	rd	%fprs, %o2		! check for unused fp
170	btst	FPRS_FEF, %o2
171	bnz	%icc, .bzero_small
172	nop
173
174	ldn	[THREAD_REG + T_LWP], %o2
175	tst	%o2
176	bz,pn	%ncc, .bzero_small
177	nop
178
179	! Check for block alignment
180	btst	(64-1), %o0
181	bz	%icc, .bzl_block
182	nop
183
184	! Check for double-word alignment
185	btst	(8-1), %o0
186	bz	%icc, .bzl_dword
187	nop
188
189	! Check for word alignment
190	btst	(4-1), %o0
191	bz	%icc, .bzl_word
192	nop
193
194	! Clear bytes until word aligned
195.bzl_byte:
196	stba	%g0, [%o0]%asi
197	add	%o0, 1, %o0
198	btst	(4-1), %o0
199	bnz	%icc, .bzl_byte
200	sub	%o1, 1, %o1
201
202	! Check for dword-aligned
203	btst	(8-1), %o0
204	bz	%icc, .bzl_dword
205	nop
206
207	! Clear words until double-word aligned
208.bzl_word:
209	sta	%g0, [%o0]%asi
210	add	%o0, 4, %o0
211	btst	(8-1), %o0
212	bnz	%icc, .bzl_word
213	sub	%o1, 4, %o1
214
215.bzl_dword:
216	! Clear dwords until block aligned
217	stxa	%g0, [%o0]%asi
218	add	%o0, 8, %o0
219	btst	(64-1), %o0
220	bnz	%icc, .bzl_dword
221	sub	%o1, 8, %o1
222
223.bzl_block:
224	membar	#StoreStore|#StoreLoad|#LoadStore
225	wr	%g0, FPRS_FEF, %fprs
226
227	! Set the lower bit in the saved t_lofault to indicate
228	! that we need to clear the %fprs register on the way
229	! out
230	or	%o5, FP_USED, %o5
231
232	! Clear block
233	fzero	%d0
234	fzero	%d2
235	fzero	%d4
236	fzero	%d6
237	fzero	%d8
238	fzero	%d10
239	fzero	%d12
240	fzero	%d14
241	rd	%asi, %o3
242	wr	%g0, ASI_BLK_P, %asi
243	cmp	%o3, ASI_P
244	bne,a	%icc, 1f
245	wr	%g0, ASI_BLK_AIUS, %asi
2461:
247	mov	256, %o3
248	ba,pt	%ncc, .bzl_doblock
249	nop
250
251.bzl_blkstart:
252      ! stda	%d0, [%o0+192]%asi  ! in dly slot of branch that got us here
253	stda	%d0, [%o0+128]%asi
254	stda	%d0, [%o0+64]%asi
255	stda	%d0, [%o0]%asi
256.bzl_zinst:
257	add	%o0, %o3, %o0
258	sub	%o1, %o3, %o1
259.bzl_doblock:
260	cmp	%o1, 256
261	bgeu,a	%ncc, .bzl_blkstart
262	stda	%d0, [%o0+192]%asi
263
264	cmp	%o1, 64
265	blu	%ncc, .bzl_finish
266
267	andn	%o1, (64-1), %o3
268	srl	%o3, 4, %o2		! using blocks, 1 instr / 16 words
269	set	.bzl_zinst, %o4
270	sub	%o4, %o2, %o4
271	jmp	%o4
272	nop
273
274.bzl_finish:
275	membar	#StoreLoad|#StoreStore
276	wr	%g0, %g0, %fprs
277	andn	%o5, FP_USED, %o5
278
279	rd	%asi, %o4
280	wr	%g0, ASI_P, %asi
281	cmp	%o4, ASI_BLK_P
282	bne,a	%icc, 1f
283	wr	%g0, ASI_USER, %asi
2841:
285
286.bzlf_dword:
287	! double words
288	cmp	%o1, 8
289	blu	%ncc, .bzlf_word
290	nop
291	stxa	%g0, [%o0]%asi
292	add	%o0, 8, %o0
293	sub	%o1, 8, %o1
294	ba,pt	%ncc, .bzlf_dword
295	nop
296
297.bzlf_word:
298	! words
299	cmp	%o1, 4
300	blu	%ncc, .bzlf_byte
301	nop
302	sta	%g0, [%o0]%asi
303	add	%o0, 4, %o0
304	sub	%o1, 4, %o1
305	ba,pt	%ncc, .bzlf_word
306	nop
307
3081:
309	add	%o0, 1, %o0		! increment address
310.bzlf_byte:
311	subcc	%o1, 1, %o1		! decrement count
312	bgeu,a	%ncc, 1b
313	stba	%g0, [%o0]%asi		! zero a byte
314
315	!
316	! If we used the FP registers, that bit was turned
317	! off after we were finished. We're just concerned with
318	! whether t_lofault was set when we came in. We end up
319	! here from either kzero() or bzero(). kzero() *always*
320	! sets a lofault handler. It ors LOFAULT_SET into %o5
321	! to indicate it has done this even if the value of %o5
322	! is otherwise zero. bzero() sets a lofault handler *only*
323	! if one was previously set. Accordingly we need to examine
324	! %o5 and if it is non-zero be sure to clear LOFAULT_SET
325	! before resetting the error handler.
326	!
327	tst	%o5
328	bz,pt	%ncc, 1f
329	andn	%o5, LOFAULT_SET, %o5
330	membar	#Sync				! sync error barrier
331	stn	%o5, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
3321:
333	retl
334	clr	%o0			! return (0)
335
336.bzero_small:
337
338	!
339	! Check for word alignment.
340	!
341	btst	3, %o0
342	bz	.bzero_probe
343	mov	0x100, %o3		! constant size of main loop
344	!
345	!
346	! clear bytes until word aligned
347	!
3481:	stba	%g0,[%o0]%asi
349	add	%o0, 1, %o0
350	btst	3, %o0
351	bnz	1b
352	sub	%o1, 1, %o1
353.bzero_probe:
354
355	!
356	! if needed move a word to become double-word aligned.
357	!
358	btst	7, %o0			! is double aligned?
359	bz	%icc, .bzero_nobuf
360	nop
361	sta	%g0, [%o0]%asi		! clr to double boundry
362	sub	%o1, 4, %o1
363	ba,pt	%ncc, .bzero_nobuf
364	add	%o0, 4, %o0
365
366	!stxa	%g0, [%o0+0xf8]%asi
367.bzero_blk:
368	stxa	%g0, [%o0+0xf0]%asi
369	stxa	%g0, [%o0+0xe8]%asi
370	stxa	%g0, [%o0+0xe0]%asi
371	stxa	%g0, [%o0+0xd8]%asi
372	stxa	%g0, [%o0+0xd0]%asi
373	stxa	%g0, [%o0+0xc8]%asi
374	stxa	%g0, [%o0+0xc0]%asi
375	stxa	%g0, [%o0+0xb8]%asi
376	stxa	%g0, [%o0+0xb0]%asi
377	stxa	%g0, [%o0+0xa8]%asi
378	stxa	%g0, [%o0+0xa0]%asi
379	stxa	%g0, [%o0+0x98]%asi
380	stxa	%g0, [%o0+0x90]%asi
381	stxa	%g0, [%o0+0x88]%asi
382	stxa	%g0, [%o0+0x80]%asi
383	stxa	%g0, [%o0+0x78]%asi
384	stxa	%g0, [%o0+0x70]%asi
385	stxa	%g0, [%o0+0x68]%asi
386	stxa	%g0, [%o0+0x60]%asi
387	stxa	%g0, [%o0+0x58]%asi
388	stxa	%g0, [%o0+0x50]%asi
389	stxa	%g0, [%o0+0x48]%asi
390	stxa	%g0, [%o0+0x40]%asi
391	stxa	%g0, [%o0+0x38]%asi
392	stxa	%g0, [%o0+0x30]%asi
393	stxa	%g0, [%o0+0x28]%asi
394	stxa	%g0, [%o0+0x20]%asi
395	stxa	%g0, [%o0+0x18]%asi
396	stxa	%g0, [%o0+0x10]%asi
397	stxa	%g0, [%o0+0x08]%asi
398	stxa	%g0, [%o0]%asi
399.zinst:
400	add	%o0, %o3, %o0		! increment source address
401	sub	%o1, %o3, %o1		! decrement count
402.bzero_nobuf:
403	cmp	%o1, 0x100		! can we do whole chunk?
404	bgeu,a	%ncc, .bzero_blk
405	stxa	%g0, [%o0+0xf8]%asi	! do first double of chunk
406
407	cmp	%o1, 7			! can we zero any more double words
408	bleu	%ncc, .byteclr		! too small go zero bytes
409
410	andn	%o1, 7, %o3		! %o3 bytes left, double-word aligned
411	srl	%o3, 1, %o2		! using doubles, need 1 instr / 2 words
412	set	.zinst, %o4		! address of clr instructions
413	sub	%o4, %o2, %o4		! jmp address relative to instr
414	jmp	%o4
415	nop
416	!
417	! do leftover bytes
418	!
4193:
420	add	%o0, 1, %o0		! increment address
421.byteclr:
422	subcc	%o1, 1, %o1		! decrement count
423	bgeu,a	%ncc, 3b
424	stba	%g0, [%o0]%asi		! zero a byte
425
426.bzero_finished:
427	!
428	! We're just concerned with whether t_lofault was set
429	! when we came in. We end up here from either kzero()
430	! or bzero(). kzero() *always* sets a lofault handler.
431	! It ors LOFAULT_SET into %o5 to indicate it has done
432	! this even if the value of %o5 is otherwise zero.
433	! bzero() sets a lofault handler *only* if one was
434	! previously set. Accordingly we need to examine
435	! %o5 and if it is non-zero be sure to clear LOFAULT_SET
436	! before resetting the error handler.
437	!
438	tst	%o5
439	bz	%ncc, 1f
440	andn	%o5, LOFAULT_SET, %o5
441	membar	#Sync				! sync error barrier
442	stn	%o5, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
4431:
444	retl
445	clr	%o0			! return (0)
446
447	SET_SIZE(bzero)
448