xref: /titanic_50/usr/src/uts/sun4u/ml/trap_table.s (revision 733a5356058ae0150a67d61f0ad8e5260d2acae3)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#pragma ident	"%Z%%M%	%I%	%E% SMI"
28
29#if !defined(lint)
30#include "assym.h"
31#endif /* !lint */
32#include <sys/asm_linkage.h>
33#include <sys/privregs.h>
34#include <sys/sun4asi.h>
35#include <sys/cheetahregs.h>
36#include <sys/machtrap.h>
37#include <sys/machthread.h>
38#include <sys/pcb.h>
39#include <sys/pte.h>
40#include <sys/mmu.h>
41#include <sys/machpcb.h>
42#include <sys/async.h>
43#include <sys/intreg.h>
44#include <sys/scb.h>
45#include <sys/psr_compat.h>
46#include <sys/syscall.h>
47#include <sys/machparam.h>
48#include <sys/traptrace.h>
49#include <vm/hat_sfmmu.h>
50#include <sys/archsystm.h>
51#include <sys/utrap.h>
52#include <sys/clock.h>
53#include <sys/intr.h>
54#include <sys/fpu/fpu_simulator.h>
55#include <vm/seg_spt.h>
56
57/*
58 * WARNING: If you add a fast trap handler which can be invoked by a
59 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
60 * instead of "done" instruction to return back to the user mode. See
61 * comments for the "fast_trap_done" entry point for more information.
62 *
63 * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
64 * cases where you always want to process any pending interrupts before
65 * returning back to the user mode.
66 */
67#define	FAST_TRAP_DONE		\
68	ba,a	fast_trap_done
69
70#define	FAST_TRAP_DONE_CHK_INTR	\
71	ba,a	fast_trap_done_chk_intr
72
73/*
74 * SPARC V9 Trap Table
75 *
76 * Most of the trap handlers are made from common building
77 * blocks, and some are instantiated multiple times within
78 * the trap table. So, I build a bunch of macros, then
79 * populate the table using only the macros.
80 *
81 * Many macros branch to sys_trap.  Its calling convention is:
82 *	%g1		kernel trap handler
83 *	%g2, %g3	args for above
84 *	%g4		desire %pil
85 */
86
87#ifdef	TRAPTRACE
88
89/*
90 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
91 */
92#define	TT_TRACE(label)		\
93	ba	label		;\
94	rd	%pc, %g7
95#define	TT_TRACE_INS	2
96
97#define	TT_TRACE_L(label)	\
98	ba	label		;\
99	rd	%pc, %l4	;\
100	clr	%l4
101#define	TT_TRACE_L_INS	3
102
103#else
104
105#define	TT_TRACE(label)
106#define	TT_TRACE_INS	0
107
108#define	TT_TRACE_L(label)
109#define	TT_TRACE_L_INS	0
110
111#endif
112
113/*
114 * This macro is used to update per cpu mmu stats in perf critical
115 * paths. It is only enabled in debug kernels or if SFMMU_STAT_GATHER
116 * is defined.
117 */
118#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
119#define	HAT_PERCPU_DBSTAT(stat)			\
120	mov	stat, %g1			;\
121	ba	stat_mmu			;\
122	rd	%pc, %g7
123#else
124#define	HAT_PERCPU_DBSTAT(stat)
125#endif /* DEBUG || SFMMU_STAT_GATHER */
126
127/*
128 * This first set are funneled to trap() with %tt as the type.
129 * Trap will then either panic or send the user a signal.
130 */
131/*
132 * NOT is used for traps that just shouldn't happen.
133 * It comes in both single and quadruple flavors.
134 */
135#if !defined(lint)
136	.global	trap
137#endif /* !lint */
138#define	NOT			\
139	TT_TRACE(trace_gen)	;\
140	set	trap, %g1	;\
141	rdpr	%tt, %g3	;\
142	ba,pt	%xcc, sys_trap	;\
143	sub	%g0, 1, %g4	;\
144	.align	32
145#define	NOT4	NOT; NOT; NOT; NOT
146/*
147 * RED is for traps that use the red mode handler.
148 * We should never see these either.
149 */
150#define	RED	NOT
151/*
152 * BAD is used for trap vectors we don't have a kernel
153 * handler for.
154 * It also comes in single and quadruple versions.
155 */
156#define	BAD	NOT
157#define	BAD4	NOT4
158
159#define	DONE			\
160	done;			\
161	.align	32
162
163/*
164 * TRAP vectors to the trap() function.
165 * It's main use is for user errors.
166 */
167#if !defined(lint)
168	.global	trap
169#endif /* !lint */
170#define	TRAP(arg)		\
171	TT_TRACE(trace_gen)	;\
172	set	trap, %g1	;\
173	mov	arg, %g3	;\
174	ba,pt	%xcc, sys_trap	;\
175	sub	%g0, 1, %g4	;\
176	.align	32
177
178/*
179 * SYSCALL is used for system calls on both ILP32 and LP64 kernels
180 * depending on the "which" parameter (should be either syscall_trap
181 * or syscall_trap32).
182 */
183#define	SYSCALL(which)			\
184	TT_TRACE(trace_gen)		;\
185	set	(which), %g1		;\
186	ba,pt	%xcc, sys_trap		;\
187	sub	%g0, 1, %g4		;\
188	.align	32
189
190#define	FLUSHW()			\
191	set	trap, %g1		;\
192	mov	T_FLUSHW, %g3		;\
193	sub	%g0, 1, %g4		;\
194	save				;\
195	flushw				;\
196	restore				;\
197	FAST_TRAP_DONE			;\
198	.align	32
199
200/*
201 * GOTO just jumps to a label.
202 * It's used for things that can be fixed without going thru sys_trap.
203 */
204#define	GOTO(label)		\
205	.global	label		;\
206	ba,a	label		;\
207	.empty			;\
208	.align	32
209
210/*
211 * GOTO_TT just jumps to a label.
212 * correctable ECC error traps at  level 0 and 1 will use this macro.
213 * It's used for things that can be fixed without going thru sys_trap.
214 */
215#define	GOTO_TT(label, ttlabel)		\
216	.global	label		;\
217	TT_TRACE(ttlabel)	;\
218	ba,a	label		;\
219	.empty			;\
220	.align	32
221
222/*
223 * Privileged traps
224 * Takes breakpoint if privileged, calls trap() if not.
225 */
226#define	PRIV(label)			\
227	rdpr	%tstate, %g1		;\
228	btst	TSTATE_PRIV, %g1	;\
229	bnz	label			;\
230	rdpr	%tt, %g3		;\
231	set	trap, %g1		;\
232	ba,pt	%xcc, sys_trap		;\
233	sub	%g0, 1, %g4		;\
234	.align	32
235
236
237/*
238 * DTrace traps.
239 */
240#define	DTRACE_FASTTRAP			\
241	.global dtrace_fasttrap_probe				;\
242	.global dtrace_fasttrap_probe_ptr			;\
243	sethi	%hi(dtrace_fasttrap_probe_ptr), %g4		;\
244	ldn	[%g4 + %lo(dtrace_fasttrap_probe_ptr)], %g4	;\
245	set	dtrace_fasttrap_probe, %g1			;\
246	brnz,pn	%g4, user_trap					;\
247	sub	%g0, 1, %g4					;\
248	FAST_TRAP_DONE						;\
249	.align	32
250
251#define	DTRACE_PID			\
252	.global dtrace_pid_probe				;\
253	set	dtrace_pid_probe, %g1				;\
254	ba,pt	%xcc, user_trap					;\
255	sub	%g0, 1, %g4					;\
256	.align	32
257
258#define	DTRACE_RETURN			\
259	.global dtrace_return_probe				;\
260	set	dtrace_return_probe, %g1			;\
261	ba,pt	%xcc, user_trap					;\
262	sub	%g0, 1, %g4					;\
263	.align	32
264
265/*
266 * REGISTER WINDOW MANAGEMENT MACROS
267 */
268
269/*
270 * various convenient units of padding
271 */
272#define	SKIP(n)	.skip 4*(n)
273
274/*
275 * CLEAN_WINDOW is the simple handler for cleaning a register window.
276 */
277#define	CLEAN_WINDOW						\
278	TT_TRACE_L(trace_win)					;\
279	rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin	;\
280	clr %l0; clr %l1; clr %l2; clr %l3			;\
281	clr %l4; clr %l5; clr %l6; clr %l7			;\
282	clr %o0; clr %o1; clr %o2; clr %o3			;\
283	clr %o4; clr %o5; clr %o6; clr %o7			;\
284	retry; .align 128
285
286#if !defined(lint)
287
288/*
289 * If we get an unresolved tlb miss while in a window handler, the fault
290 * handler will resume execution at the last instruction of the window
291 * hander, instead of delivering the fault to the kernel.  Spill handlers
292 * use this to spill windows into the wbuf.
293 *
294 * The mixed handler works by checking %sp, and branching to the correct
295 * handler.  This is done by branching back to label 1: for 32b frames,
296 * or label 2: for 64b frames; which implies the handler order is: 32b,
297 * 64b, mixed.  The 1: and 2: labels are offset into the routines to
298 * allow the branchs' delay slots to contain useful instructions.
299 */
300
301/*
302 * SPILL_32bit spills a 32-bit-wide kernel register window.  It
303 * assumes that the kernel context and the nucleus context are the
304 * same.  The stack pointer is required to be eight-byte aligned even
305 * though this code only needs it to be four-byte aligned.
306 */
307#define	SPILL_32bit(tail)					\
308	srl	%sp, 0, %sp					;\
3091:	st	%l0, [%sp + 0]					;\
310	st	%l1, [%sp + 4]					;\
311	st	%l2, [%sp + 8]					;\
312	st	%l3, [%sp + 12]					;\
313	st	%l4, [%sp + 16]					;\
314	st	%l5, [%sp + 20]					;\
315	st	%l6, [%sp + 24]					;\
316	st	%l7, [%sp + 28]					;\
317	st	%i0, [%sp + 32]					;\
318	st	%i1, [%sp + 36]					;\
319	st	%i2, [%sp + 40]					;\
320	st	%i3, [%sp + 44]					;\
321	st	%i4, [%sp + 48]					;\
322	st	%i5, [%sp + 52]					;\
323	st	%i6, [%sp + 56]					;\
324	st	%i7, [%sp + 60]					;\
325	TT_TRACE_L(trace_win)					;\
326	saved							;\
327	retry							;\
328	SKIP(31-19-TT_TRACE_L_INS)				;\
329	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
330	.empty
331
332/*
333 * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
334 * wide address space via the designated asi.  It is used to spill
335 * non-kernel windows.  The stack pointer is required to be eight-byte
336 * aligned even though this code only needs it to be four-byte
337 * aligned.
338 */
339#define	SPILL_32bit_asi(asi_num, tail)				\
340	srl	%sp, 0, %sp					;\
3411:	sta	%l0, [%sp + %g0]asi_num				;\
342	mov	4, %g1						;\
343	sta	%l1, [%sp + %g1]asi_num				;\
344	mov	8, %g2						;\
345	sta	%l2, [%sp + %g2]asi_num				;\
346	mov	12, %g3						;\
347	sta	%l3, [%sp + %g3]asi_num				;\
348	add	%sp, 16, %g4					;\
349	sta	%l4, [%g4 + %g0]asi_num				;\
350	sta	%l5, [%g4 + %g1]asi_num				;\
351	sta	%l6, [%g4 + %g2]asi_num				;\
352	sta	%l7, [%g4 + %g3]asi_num				;\
353	add	%g4, 16, %g4					;\
354	sta	%i0, [%g4 + %g0]asi_num				;\
355	sta	%i1, [%g4 + %g1]asi_num				;\
356	sta	%i2, [%g4 + %g2]asi_num				;\
357	sta	%i3, [%g4 + %g3]asi_num				;\
358	add	%g4, 16, %g4					;\
359	sta	%i4, [%g4 + %g0]asi_num				;\
360	sta	%i5, [%g4 + %g1]asi_num				;\
361	sta	%i6, [%g4 + %g2]asi_num				;\
362	sta	%i7, [%g4 + %g3]asi_num				;\
363	TT_TRACE_L(trace_win)					;\
364	saved							;\
365	retry							;\
366	SKIP(31-25-TT_TRACE_L_INS)				;\
367	ba,a,pt %xcc, fault_32bit_/**/tail			;\
368	.empty
369
370/*
371 * SPILL_32bit_tt1 spills a 32-bit-wide register window into a 32-bit
372 * wide address space via the designated asi.  It is used to spill
373 * windows at tl>1 where performance isn't the primary concern and
374 * where we don't want to use unnecessary registers.  The stack
375 * pointer is required to be eight-byte aligned even though this code
376 * only needs it to be four-byte aligned.
377 */
378#define	SPILL_32bit_tt1(asi_num, tail)				\
379	mov	asi_num, %asi					;\
3801:	srl	%sp, 0, %sp					;\
381	sta	%l0, [%sp + 0]%asi				;\
382	sta	%l1, [%sp + 4]%asi				;\
383	sta	%l2, [%sp + 8]%asi				;\
384	sta	%l3, [%sp + 12]%asi				;\
385	sta	%l4, [%sp + 16]%asi				;\
386	sta	%l5, [%sp + 20]%asi				;\
387	sta	%l6, [%sp + 24]%asi				;\
388	sta	%l7, [%sp + 28]%asi				;\
389	sta	%i0, [%sp + 32]%asi				;\
390	sta	%i1, [%sp + 36]%asi				;\
391	sta	%i2, [%sp + 40]%asi				;\
392	sta	%i3, [%sp + 44]%asi				;\
393	sta	%i4, [%sp + 48]%asi				;\
394	sta	%i5, [%sp + 52]%asi				;\
395	sta	%i6, [%sp + 56]%asi				;\
396	sta	%i7, [%sp + 60]%asi				;\
397	TT_TRACE_L(trace_win)					;\
398	saved							;\
399	retry							;\
400	SKIP(31-20-TT_TRACE_L_INS)				;\
401	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
402	.empty
403
404
405/*
406 * FILL_32bit fills a 32-bit-wide kernel register window.  It assumes
407 * that the kernel context and the nucleus context are the same.  The
408 * stack pointer is required to be eight-byte aligned even though this
409 * code only needs it to be four-byte aligned.
410 */
411#define	FILL_32bit(tail)					\
412	srl	%sp, 0, %sp					;\
4131:	TT_TRACE_L(trace_win)					;\
414	ld	[%sp + 0], %l0					;\
415	ld	[%sp + 4], %l1					;\
416	ld	[%sp + 8], %l2					;\
417	ld	[%sp + 12], %l3					;\
418	ld	[%sp + 16], %l4					;\
419	ld	[%sp + 20], %l5					;\
420	ld	[%sp + 24], %l6					;\
421	ld	[%sp + 28], %l7					;\
422	ld	[%sp + 32], %i0					;\
423	ld	[%sp + 36], %i1					;\
424	ld	[%sp + 40], %i2					;\
425	ld	[%sp + 44], %i3					;\
426	ld	[%sp + 48], %i4					;\
427	ld	[%sp + 52], %i5					;\
428	ld	[%sp + 56], %i6					;\
429	ld	[%sp + 60], %i7					;\
430	restored						;\
431	retry							;\
432	SKIP(31-19-TT_TRACE_L_INS)				;\
433	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
434	.empty
435
436/*
437 * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
438 * wide address space via the designated asi.  It is used to fill
439 * non-kernel windows.  The stack pointer is required to be eight-byte
440 * aligned even though this code only needs it to be four-byte
441 * aligned.
442 */
443#define	FILL_32bit_asi(asi_num, tail)				\
444	srl	%sp, 0, %sp					;\
4451:	TT_TRACE_L(trace_win)					;\
446	mov	4, %g1						;\
447	lda	[%sp + %g0]asi_num, %l0				;\
448	mov	8, %g2						;\
449	lda	[%sp + %g1]asi_num, %l1				;\
450	mov	12, %g3						;\
451	lda	[%sp + %g2]asi_num, %l2				;\
452	lda	[%sp + %g3]asi_num, %l3				;\
453	add	%sp, 16, %g4					;\
454	lda	[%g4 + %g0]asi_num, %l4				;\
455	lda	[%g4 + %g1]asi_num, %l5				;\
456	lda	[%g4 + %g2]asi_num, %l6				;\
457	lda	[%g4 + %g3]asi_num, %l7				;\
458	add	%g4, 16, %g4					;\
459	lda	[%g4 + %g0]asi_num, %i0				;\
460	lda	[%g4 + %g1]asi_num, %i1				;\
461	lda	[%g4 + %g2]asi_num, %i2				;\
462	lda	[%g4 + %g3]asi_num, %i3				;\
463	add	%g4, 16, %g4					;\
464	lda	[%g4 + %g0]asi_num, %i4				;\
465	lda	[%g4 + %g1]asi_num, %i5				;\
466	lda	[%g4 + %g2]asi_num, %i6				;\
467	lda	[%g4 + %g3]asi_num, %i7				;\
468	restored						;\
469	retry							;\
470	SKIP(31-25-TT_TRACE_L_INS)				;\
471	ba,a,pt %xcc, fault_32bit_/**/tail			;\
472	.empty
473
474/*
475 * FILL_32bit_tt1 fills a 32-bit-wide register window from a 32-bit
476 * wide address space via the designated asi.  It is used to fill
477 * windows at tl>1 where performance isn't the primary concern and
478 * where we don't want to use unnecessary registers.  The stack
479 * pointer is required to be eight-byte aligned even though this code
480 * only needs it to be four-byte aligned.
481 */
482#define	FILL_32bit_tt1(asi_num, tail)				\
483	mov	asi_num, %asi					;\
4841:	srl	%sp, 0, %sp					;\
485	TT_TRACE_L(trace_win)					;\
486	lda	[%sp + 0]%asi, %l0				;\
487	lda	[%sp + 4]%asi, %l1				;\
488	lda	[%sp + 8]%asi, %l2				;\
489	lda	[%sp + 12]%asi, %l3				;\
490	lda	[%sp + 16]%asi, %l4				;\
491	lda	[%sp + 20]%asi, %l5				;\
492	lda	[%sp + 24]%asi, %l6				;\
493	lda	[%sp + 28]%asi, %l7				;\
494	lda	[%sp + 32]%asi, %i0				;\
495	lda	[%sp + 36]%asi, %i1				;\
496	lda	[%sp + 40]%asi, %i2				;\
497	lda	[%sp + 44]%asi, %i3				;\
498	lda	[%sp + 48]%asi, %i4				;\
499	lda	[%sp + 52]%asi, %i5				;\
500	lda	[%sp + 56]%asi, %i6				;\
501	lda	[%sp + 60]%asi, %i7				;\
502	restored						;\
503	retry							;\
504	SKIP(31-20-TT_TRACE_L_INS)				;\
505	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
506	.empty
507
508
509/*
510 * SPILL_64bit spills a 64-bit-wide kernel register window.  It
511 * assumes that the kernel context and the nucleus context are the
512 * same.  The stack pointer is required to be eight-byte aligned.
513 */
514#define	SPILL_64bit(tail)					\
5152:	stx	%l0, [%sp + V9BIAS64 + 0]			;\
516	stx	%l1, [%sp + V9BIAS64 + 8]			;\
517	stx	%l2, [%sp + V9BIAS64 + 16]			;\
518	stx	%l3, [%sp + V9BIAS64 + 24]			;\
519	stx	%l4, [%sp + V9BIAS64 + 32]			;\
520	stx	%l5, [%sp + V9BIAS64 + 40]			;\
521	stx	%l6, [%sp + V9BIAS64 + 48]			;\
522	stx	%l7, [%sp + V9BIAS64 + 56]			;\
523	stx	%i0, [%sp + V9BIAS64 + 64]			;\
524	stx	%i1, [%sp + V9BIAS64 + 72]			;\
525	stx	%i2, [%sp + V9BIAS64 + 80]			;\
526	stx	%i3, [%sp + V9BIAS64 + 88]			;\
527	stx	%i4, [%sp + V9BIAS64 + 96]			;\
528	stx	%i5, [%sp + V9BIAS64 + 104]			;\
529	stx	%i6, [%sp + V9BIAS64 + 112]			;\
530	stx	%i7, [%sp + V9BIAS64 + 120]			;\
531	TT_TRACE_L(trace_win)					;\
532	saved							;\
533	retry							;\
534	SKIP(31-18-TT_TRACE_L_INS)				;\
535	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
536	.empty
537
538/*
539 * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
540 * wide address space via the designated asi.  It is used to spill
541 * non-kernel windows.  The stack pointer is required to be eight-byte
542 * aligned.
543 */
544#define	SPILL_64bit_asi(asi_num, tail)				\
545	mov	0 + V9BIAS64, %g1				;\
5462:	stxa	%l0, [%sp + %g1]asi_num				;\
547	mov	8 + V9BIAS64, %g2				;\
548	stxa	%l1, [%sp + %g2]asi_num				;\
549	mov	16 + V9BIAS64, %g3				;\
550	stxa	%l2, [%sp + %g3]asi_num				;\
551	mov	24 + V9BIAS64, %g4				;\
552	stxa	%l3, [%sp + %g4]asi_num				;\
553	add	%sp, 32, %g5					;\
554	stxa	%l4, [%g5 + %g1]asi_num				;\
555	stxa	%l5, [%g5 + %g2]asi_num				;\
556	stxa	%l6, [%g5 + %g3]asi_num				;\
557	stxa	%l7, [%g5 + %g4]asi_num				;\
558	add	%g5, 32, %g5					;\
559	stxa	%i0, [%g5 + %g1]asi_num				;\
560	stxa	%i1, [%g5 + %g2]asi_num				;\
561	stxa	%i2, [%g5 + %g3]asi_num				;\
562	stxa	%i3, [%g5 + %g4]asi_num				;\
563	add	%g5, 32, %g5					;\
564	stxa	%i4, [%g5 + %g1]asi_num				;\
565	stxa	%i5, [%g5 + %g2]asi_num				;\
566	stxa	%i6, [%g5 + %g3]asi_num				;\
567	stxa	%i7, [%g5 + %g4]asi_num				;\
568	TT_TRACE_L(trace_win)					;\
569	saved							;\
570	retry							;\
571	SKIP(31-25-TT_TRACE_L_INS)				;\
572	ba,a,pt %xcc, fault_64bit_/**/tail			;\
573	.empty
574
575/*
576 * SPILL_64bit_tt1 spills a 64-bit-wide register window into a 64-bit
577 * wide address space via the designated asi.  It is used to spill
578 * windows at tl>1 where performance isn't the primary concern and
579 * where we don't want to use unnecessary registers.  The stack
580 * pointer is required to be eight-byte aligned.
581 */
582#define	SPILL_64bit_tt1(asi_num, tail)				\
583	mov	asi_num, %asi					;\
5842:	stxa	%l0, [%sp + V9BIAS64 + 0]%asi			;\
585	stxa	%l1, [%sp + V9BIAS64 + 8]%asi			;\
586	stxa	%l2, [%sp + V9BIAS64 + 16]%asi			;\
587	stxa	%l3, [%sp + V9BIAS64 + 24]%asi			;\
588	stxa	%l4, [%sp + V9BIAS64 + 32]%asi			;\
589	stxa	%l5, [%sp + V9BIAS64 + 40]%asi			;\
590	stxa	%l6, [%sp + V9BIAS64 + 48]%asi			;\
591	stxa	%l7, [%sp + V9BIAS64 + 56]%asi			;\
592	stxa	%i0, [%sp + V9BIAS64 + 64]%asi			;\
593	stxa	%i1, [%sp + V9BIAS64 + 72]%asi			;\
594	stxa	%i2, [%sp + V9BIAS64 + 80]%asi			;\
595	stxa	%i3, [%sp + V9BIAS64 + 88]%asi			;\
596	stxa	%i4, [%sp + V9BIAS64 + 96]%asi			;\
597	stxa	%i5, [%sp + V9BIAS64 + 104]%asi			;\
598	stxa	%i6, [%sp + V9BIAS64 + 112]%asi			;\
599	stxa	%i7, [%sp + V9BIAS64 + 120]%asi			;\
600	TT_TRACE_L(trace_win)					;\
601	saved							;\
602	retry							;\
603	SKIP(31-19-TT_TRACE_L_INS)				;\
604	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
605	.empty
606
607
608/*
609 * FILL_64bit fills a 64-bit-wide kernel register window.  It assumes
610 * that the kernel context and the nucleus context are the same.  The
611 * stack pointer is required to be eight-byte aligned.
612 */
613#define	FILL_64bit(tail)					\
6142:	TT_TRACE_L(trace_win)					;\
615	ldx	[%sp + V9BIAS64 + 0], %l0			;\
616	ldx	[%sp + V9BIAS64 + 8], %l1			;\
617	ldx	[%sp + V9BIAS64 + 16], %l2			;\
618	ldx	[%sp + V9BIAS64 + 24], %l3			;\
619	ldx	[%sp + V9BIAS64 + 32], %l4			;\
620	ldx	[%sp + V9BIAS64 + 40], %l5			;\
621	ldx	[%sp + V9BIAS64 + 48], %l6			;\
622	ldx	[%sp + V9BIAS64 + 56], %l7			;\
623	ldx	[%sp + V9BIAS64 + 64], %i0			;\
624	ldx	[%sp + V9BIAS64 + 72], %i1			;\
625	ldx	[%sp + V9BIAS64 + 80], %i2			;\
626	ldx	[%sp + V9BIAS64 + 88], %i3			;\
627	ldx	[%sp + V9BIAS64 + 96], %i4			;\
628	ldx	[%sp + V9BIAS64 + 104], %i5			;\
629	ldx	[%sp + V9BIAS64 + 112], %i6			;\
630	ldx	[%sp + V9BIAS64 + 120], %i7			;\
631	restored						;\
632	retry							;\
633	SKIP(31-18-TT_TRACE_L_INS)				;\
634	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
635	.empty
636
637/*
638 * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
639 * wide address space via the designated asi.  It is used to fill
640 * non-kernel windows.  The stack pointer is required to be eight-byte
641 * aligned.
642 */
643#define	FILL_64bit_asi(asi_num, tail)				\
644	mov	V9BIAS64 + 0, %g1				;\
6452:	TT_TRACE_L(trace_win)					;\
646	ldxa	[%sp + %g1]asi_num, %l0				;\
647	mov	V9BIAS64 + 8, %g2				;\
648	ldxa	[%sp + %g2]asi_num, %l1				;\
649	mov	V9BIAS64 + 16, %g3				;\
650	ldxa	[%sp + %g3]asi_num, %l2				;\
651	mov	V9BIAS64 + 24, %g4				;\
652	ldxa	[%sp + %g4]asi_num, %l3				;\
653	add	%sp, 32, %g5					;\
654	ldxa	[%g5 + %g1]asi_num, %l4				;\
655	ldxa	[%g5 + %g2]asi_num, %l5				;\
656	ldxa	[%g5 + %g3]asi_num, %l6				;\
657	ldxa	[%g5 + %g4]asi_num, %l7				;\
658	add	%g5, 32, %g5					;\
659	ldxa	[%g5 + %g1]asi_num, %i0				;\
660	ldxa	[%g5 + %g2]asi_num, %i1				;\
661	ldxa	[%g5 + %g3]asi_num, %i2				;\
662	ldxa	[%g5 + %g4]asi_num, %i3				;\
663	add	%g5, 32, %g5					;\
664	ldxa	[%g5 + %g1]asi_num, %i4				;\
665	ldxa	[%g5 + %g2]asi_num, %i5				;\
666	ldxa	[%g5 + %g3]asi_num, %i6				;\
667	ldxa	[%g5 + %g4]asi_num, %i7				;\
668	restored						;\
669	retry							;\
670	SKIP(31-25-TT_TRACE_L_INS)				;\
671	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
672	.empty
673
674/*
675 * FILL_64bit_tt1 fills a 64-bit-wide register window from a 64-bit
676 * wide address space via the designated asi.  It is used to fill
677 * windows at tl>1 where performance isn't the primary concern and
678 * where we don't want to use unnecessary registers.  The stack
679 * pointer is required to be eight-byte aligned.
680 */
681#define	FILL_64bit_tt1(asi_num, tail)				\
682	mov	asi_num, %asi					;\
683	TT_TRACE_L(trace_win)					;\
684	ldxa	[%sp + V9BIAS64 + 0]%asi, %l0			;\
685	ldxa	[%sp + V9BIAS64 + 8]%asi, %l1			;\
686	ldxa	[%sp + V9BIAS64 + 16]%asi, %l2			;\
687	ldxa	[%sp + V9BIAS64 + 24]%asi, %l3			;\
688	ldxa	[%sp + V9BIAS64 + 32]%asi, %l4			;\
689	ldxa	[%sp + V9BIAS64 + 40]%asi, %l5			;\
690	ldxa	[%sp + V9BIAS64 + 48]%asi, %l6			;\
691	ldxa	[%sp + V9BIAS64 + 56]%asi, %l7			;\
692	ldxa	[%sp + V9BIAS64 + 64]%asi, %i0			;\
693	ldxa	[%sp + V9BIAS64 + 72]%asi, %i1			;\
694	ldxa	[%sp + V9BIAS64 + 80]%asi, %i2			;\
695	ldxa	[%sp + V9BIAS64 + 88]%asi, %i3			;\
696	ldxa	[%sp + V9BIAS64 + 96]%asi, %i4			;\
697	ldxa	[%sp + V9BIAS64 + 104]%asi, %i5			;\
698	ldxa	[%sp + V9BIAS64 + 112]%asi, %i6			;\
699	ldxa	[%sp + V9BIAS64 + 120]%asi, %i7			;\
700	restored						;\
701	retry							;\
702	SKIP(31-19-TT_TRACE_L_INS)				;\
703	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
704	.empty
705
706#endif /* !lint */
707
708/*
709 * SPILL_mixed spills either size window, depending on
710 * whether %sp is even or odd, to a 32-bit address space.
711 * This may only be used in conjunction with SPILL_32bit/
712 * SPILL_64bit. New versions of SPILL_mixed_{tt1,asi} would be
713 * needed for use with SPILL_{32,64}bit_{tt1,asi}.  Particular
714 * attention should be paid to the instructions that belong
715 * in the delay slots of the branches depending on the type
716 * of spill handler being branched to.
717 * Clear upper 32 bits of %sp if it is odd.
718 * We won't need to clear them in 64 bit kernel.
719 */
720#define	SPILL_mixed						\
721	btst	1, %sp						;\
722	bz,a,pt	%xcc, 1b					;\
723	srl	%sp, 0, %sp					;\
724	ba,pt	%xcc, 2b					;\
725	nop							;\
726	.align	128
727
728/*
729 * FILL_mixed(ASI) fills either size window, depending on
730 * whether %sp is even or odd, from a 32-bit address space.
731 * This may only be used in conjunction with FILL_32bit/
732 * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
733 * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
734 * attention should be paid to the instructions that belong
735 * in the delay slots of the branches depending on the type
736 * of fill handler being branched to.
737 * Clear upper 32 bits of %sp if it is odd.
738 * We won't need to clear them in 64 bit kernel.
739 */
740#define	FILL_mixed						\
741	btst	1, %sp						;\
742	bz,a,pt	%xcc, 1b					;\
743	srl	%sp, 0, %sp					;\
744	ba,pt	%xcc, 2b					;\
745	nop							;\
746	.align	128
747
748
749/*
750 * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
751 * respectively, into the address space via the designated asi.  The
752 * unbiased stack pointer is required to be eight-byte aligned (even for
753 * the 32-bit case even though this code does not require such strict
754 * alignment).
755 *
756 * With SPARC v9 the spill trap takes precedence over the cleanwin trap
757 * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
758 * will cause cwp + 2 to be spilled but will not clean cwp + 1.  That
759 * window may contain kernel data so in user_rtt we set wstate to call
760 * these spill handlers on the first user spill trap.  These handler then
761 * spill the appropriate window but also back up a window and clean the
762 * window that didn't get a cleanwin trap.
763 */
764#define	SPILL_32clean(asi_num, tail)				\
765	srl	%sp, 0, %sp					;\
766	sta	%l0, [%sp + %g0]asi_num				;\
767	mov	4, %g1						;\
768	sta	%l1, [%sp + %g1]asi_num				;\
769	mov	8, %g2						;\
770	sta	%l2, [%sp + %g2]asi_num				;\
771	mov	12, %g3						;\
772	sta	%l3, [%sp + %g3]asi_num				;\
773	add	%sp, 16, %g4					;\
774	sta	%l4, [%g4 + %g0]asi_num				;\
775	sta	%l5, [%g4 + %g1]asi_num				;\
776	sta	%l6, [%g4 + %g2]asi_num				;\
777	sta	%l7, [%g4 + %g3]asi_num				;\
778	add	%g4, 16, %g4					;\
779	sta	%i0, [%g4 + %g0]asi_num				;\
780	sta	%i1, [%g4 + %g1]asi_num				;\
781	sta	%i2, [%g4 + %g2]asi_num				;\
782	sta	%i3, [%g4 + %g3]asi_num				;\
783	add	%g4, 16, %g4					;\
784	sta	%i4, [%g4 + %g0]asi_num				;\
785	sta	%i5, [%g4 + %g1]asi_num				;\
786	sta	%i6, [%g4 + %g2]asi_num				;\
787	sta	%i7, [%g4 + %g3]asi_num				;\
788	TT_TRACE_L(trace_win)					;\
789	b	.spill_clean					;\
790	  mov	WSTATE_USER32, %g7				;\
791	SKIP(31-25-TT_TRACE_L_INS)				;\
792	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
793	.empty
794
795#define	SPILL_64clean(asi_num, tail)				\
796	mov	0 + V9BIAS64, %g1				;\
797	stxa	%l0, [%sp + %g1]asi_num				;\
798	mov	8 + V9BIAS64, %g2				;\
799	stxa	%l1, [%sp + %g2]asi_num				;\
800	mov	16 + V9BIAS64, %g3				;\
801	stxa	%l2, [%sp + %g3]asi_num				;\
802	mov	24 + V9BIAS64, %g4				;\
803	stxa	%l3, [%sp + %g4]asi_num				;\
804	add	%sp, 32, %g5					;\
805	stxa	%l4, [%g5 + %g1]asi_num				;\
806	stxa	%l5, [%g5 + %g2]asi_num				;\
807	stxa	%l6, [%g5 + %g3]asi_num				;\
808	stxa	%l7, [%g5 + %g4]asi_num				;\
809	add	%g5, 32, %g5					;\
810	stxa	%i0, [%g5 + %g1]asi_num				;\
811	stxa	%i1, [%g5 + %g2]asi_num				;\
812	stxa	%i2, [%g5 + %g3]asi_num				;\
813	stxa	%i3, [%g5 + %g4]asi_num				;\
814	add	%g5, 32, %g5					;\
815	stxa	%i4, [%g5 + %g1]asi_num				;\
816	stxa	%i5, [%g5 + %g2]asi_num				;\
817	stxa	%i6, [%g5 + %g3]asi_num				;\
818	stxa	%i7, [%g5 + %g4]asi_num				;\
819	TT_TRACE_L(trace_win)					;\
820	b	.spill_clean					;\
821	  mov	WSTATE_USER64, %g7				;\
822	SKIP(31-25-TT_TRACE_L_INS)				;\
823	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
824	.empty
825
826
827/*
828 * Floating point disabled.
829 */
830#define	FP_DISABLED_TRAP		\
831	TT_TRACE(trace_gen)		;\
832	ba,pt	%xcc,.fp_disabled	;\
833	nop				;\
834	.align	32
835
836/*
837 * Floating point exceptions.
838 */
839#define	FP_IEEE_TRAP			\
840	TT_TRACE(trace_gen)		;\
841	ba,pt	%xcc,.fp_ieee_exception	;\
842	nop				;\
843	.align	32
844
845#define	FP_TRAP				\
846	TT_TRACE(trace_gen)		;\
847	ba,pt	%xcc,.fp_exception	;\
848	nop				;\
849	.align	32
850
851#if !defined(lint)
852/*
853 * asynchronous traps at level 0 and level 1
854 *
855 * The first instruction must be a membar for UltraSPARC-III
856 * to stop RED state entry if the store queue has many
857 * pending bad stores (PRM, Chapter 11).
858 */
859#define ASYNC_TRAP(ttype, ttlabel)\
860	membar	#Sync		;\
861	TT_TRACE(ttlabel)	;\
862	ba	async_err	;\
863	mov	ttype, %g5	;\
864	.align	32
865
866/*
867 * Defaults to BAD entry, but establishes label to be used for
868 * architecture-specific overwrite of trap table entry.
869 */
870#define	LABELED_BAD(table_name)		\
871	.global	table_name		;\
872table_name:				;\
873	BAD
874
875#endif /* !lint */
876
877/*
878 * illegal instruction trap
879 */
880#define	ILLTRAP_INSTR			  \
881	membar	#Sync			  ;\
882	TT_TRACE(trace_gen)		  ;\
883	or	%g0, P_UTRAP4, %g2	  ;\
884	or	%g0, T_UNIMP_INSTR, %g3   ;\
885	sethi	%hi(.check_v9utrap), %g4  ;\
886	jmp	%g4 + %lo(.check_v9utrap) ;\
887	nop				  ;\
888	.align	32
889
890/*
891 * tag overflow trap
892 */
893#define	TAG_OVERFLOW			  \
894	TT_TRACE(trace_gen)		  ;\
895	or	%g0, P_UTRAP10, %g2	  ;\
896	or	%g0, T_TAG_OVERFLOW, %g3  ;\
897	sethi	%hi(.check_v9utrap), %g4  ;\
898	jmp	%g4 + %lo(.check_v9utrap) ;\
899	nop				  ;\
900	.align	32
901
902/*
903 * divide by zero trap
904 */
905#define	DIV_BY_ZERO			  \
906	TT_TRACE(trace_gen)		  ;\
907	or	%g0, P_UTRAP11, %g2	  ;\
908	or	%g0, T_IDIV0, %g3	  ;\
909	sethi	%hi(.check_v9utrap), %g4  ;\
910	jmp	%g4 + %lo(.check_v9utrap) ;\
911	nop				  ;\
912	.align	32
913
914/*
915 * trap instruction for V9 user trap handlers
916 */
917#define	TRAP_INSTR			  \
918	TT_TRACE(trace_gen)		  ;\
919	or	%g0, T_SOFTWARE_TRAP, %g3 ;\
920	sethi	%hi(.check_v9utrap), %g4  ;\
921	jmp	%g4 + %lo(.check_v9utrap) ;\
922	nop				  ;\
923	.align	32
924#define	TRP4	TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
925
926/*
927 * LEVEL_INTERRUPT is for level N interrupts.
928 * VECTOR_INTERRUPT is for the vector trap.
929 */
930#define	LEVEL_INTERRUPT(level)		\
931	.global	tt_pil/**/level		;\
932tt_pil/**/level:			;\
933	ba,pt	%xcc, pil_interrupt	;\
934	mov	level, %g4		;\
935	.align	32
936
937#define	LEVEL14_INTERRUPT			\
938	ba	pil14_interrupt			;\
939	mov	PIL_14, %g4			;\
940	.align	32
941
942#define	VECTOR_INTERRUPT				\
943	ldxa	[%g0]ASI_INTR_RECEIVE_STATUS, %g1	;\
944	btst	IRSR_BUSY, %g1				;\
945	bnz,pt	%xcc, vec_interrupt			;\
946	nop						;\
947	ba,a,pt	%xcc, vec_intr_spurious			;\
948	.empty						;\
949	.align	32
950
951/*
952 * MMU Trap Handlers.
953 */
954#define	SWITCH_GLOBALS	/* mmu->alt, alt->mmu */			\
955	rdpr	%pstate, %g5						;\
956	wrpr	%g5, PSTATE_MG | PSTATE_AG, %pstate
957
958#define	IMMU_EXCEPTION							\
959	membar	#Sync							;\
960	SWITCH_GLOBALS							;\
961	wr	%g0, ASI_IMMU, %asi					;\
962	rdpr	%tpc, %g2						;\
963	ldxa	[MMU_SFSR]%asi, %g3					;\
964	ba,pt	%xcc, .mmu_exception_end				;\
965	mov	T_INSTR_EXCEPTION, %g1					;\
966	.align	32
967
968#define	DMMU_EXCEPTION							\
969	SWITCH_GLOBALS							;\
970	wr	%g0, ASI_DMMU, %asi					;\
971	ldxa	[MMU_TAG_ACCESS]%asi, %g2				;\
972	ldxa	[MMU_SFSR]%asi, %g3					;\
973	ba,pt	%xcc, .mmu_exception_end				;\
974	mov	T_DATA_EXCEPTION, %g1					;\
975	.align	32
976
977#define	DMMU_EXC_AG_PRIV						\
978	wr	%g0, ASI_DMMU, %asi					;\
979	ldxa	[MMU_SFAR]%asi, %g2					;\
980	ba,pt	%xcc, .mmu_priv_exception				;\
981	ldxa	[MMU_SFSR]%asi, %g3					;\
982	.align	32
983
984#define	DMMU_EXC_AG_NOT_ALIGNED						\
985	wr	%g0, ASI_DMMU, %asi					;\
986	ldxa	[MMU_SFAR]%asi, %g2					;\
987	ba,pt	%xcc, .mmu_exception_not_aligned			;\
988	ldxa	[MMU_SFSR]%asi, %g3					;\
989	.align	32
990
991/*
992 * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
993 */
994#define	DMMU_EXC_LDDF_NOT_ALIGNED					\
995	btst	1, %sp							;\
996	bnz,pt	%xcc, .lddf_exception_not_aligned			;\
997	wr	%g0, ASI_DMMU, %asi					;\
998	ldxa	[MMU_SFAR]%asi, %g2					;\
999	ba,pt	%xcc, .mmu_exception_not_aligned			;\
1000	ldxa	[MMU_SFSR]%asi, %g3					;\
1001	.align	32
1002
1003#define	DMMU_EXC_STDF_NOT_ALIGNED					\
1004	btst	1, %sp							;\
1005	bnz,pt	%xcc, .stdf_exception_not_aligned			;\
1006	wr	%g0, ASI_DMMU, %asi					;\
1007	ldxa	[MMU_SFAR]%asi, %g2					;\
1008	ba,pt	%xcc, .mmu_exception_not_aligned			;\
1009	ldxa	[MMU_SFSR]%asi, %g3					;\
1010	.align	32
1011
1012/*
1013 * Flush the TLB using either the primary, secondary, or nucleus flush
1014 * operation based on whether the ctx from the tag access register matches
1015 * the primary or secondary context (flush the nucleus if neither matches).
1016 *
1017 * Requires a membar #Sync before next ld/st.
1018 * exits with:
1019 * g2 = tag access register
1020 * g3 = ctx number
1021 */
1022#if TAGACC_CTX_MASK != CTXREG_CTX_MASK
1023#error "TAGACC_CTX_MASK != CTXREG_CTX_MASK"
1024#endif
1025#define	DTLB_DEMAP_ENTRY						\
1026	mov	MMU_TAG_ACCESS, %g1					;\
1027	mov	MMU_PCONTEXT, %g5					;\
1028	ldxa	[%g1]ASI_DMMU, %g2					;\
1029	sethi	%hi(TAGACC_CTX_MASK), %g4				;\
1030	or	%g4, %lo(TAGACC_CTX_MASK), %g4				;\
1031	and	%g2, %g4, %g3			/* g3 = ctx */		;\
1032	ldxa	[%g5]ASI_DMMU, %g6		/* g6 = primary ctx */	;\
1033	and	%g6, %g4, %g6			/* &= CTXREG_CTX_MASK */ ;\
1034	cmp	%g3, %g6						;\
1035	be,pt	%xcc, 1f						;\
1036	andn	%g2, %g4, %g1			/* ctx = primary */	;\
1037	mov	MMU_SCONTEXT, %g5					;\
1038	ldxa	[%g5]ASI_DMMU, %g6		/* g6 = secondary ctx */ ;\
1039	and	%g6, %g4, %g6			/* &= CTXREG_CTX_MASK */ ;\
1040	cmp	%g3, %g6						;\
1041	be,a,pt	%xcc, 1f						;\
1042	  or	%g1, DEMAP_SECOND, %g1					;\
1043	or	%g1, DEMAP_NUCLEUS, %g1					;\
10441:	stxa	%g0, [%g1]ASI_DTLB_DEMAP	/* MMU_DEMAP_PAGE */	;\
1045	membar	#Sync
1046
1047#if defined(cscope)
1048/*
1049 * Define labels to direct cscope quickly to labels that
1050 * are generated by macro expansion of DTLB_MISS().
1051 */
1052	.global	tt0_dtlbmiss
1053tt0_dtlbmiss:
1054	.global	tt1_dtlbmiss
1055tt1_dtlbmiss:
1056	nop
1057#endif
1058
1059/*
1060 * Needs to be exactly 32 instructions
1061 *
1062 * UTLB NOTE: If we don't hit on the 8k pointer then we branch
1063 * to a special 4M tsb handler. It would be nice if that handler
1064 * could live in this file but currently it seems better to allow
1065 * it to fall thru to sfmmu_tsb_miss.
1066 */
1067#define	DTLB_MISS(table_name)						;\
1068	.global	table_name/**/_dtlbmiss					;\
1069table_name/**/_dtlbmiss:						;\
1070	HAT_PERCPU_DBSTAT(TSBMISS_DTLBMISS) /* 3 instr ifdef DEBUG */	;\
1071	mov	MMU_TAG_ACCESS, %g6		/* select tag acc */	;\
1072	ldxa	[%g0]ASI_DMMU_TSB_8K, %g1	/* g1 = tsbe ptr */	;\
1073	ldxa	[%g6]ASI_DMMU, %g2		/* g2 = tag access */	;\
1074	sllx	%g2, TAGACC_CTX_LSHIFT, %g3				;\
1075	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctx */		;\
1076	cmp	%g3, INVALID_CONTEXT					;\
1077	ble,pn	%xcc, sfmmu_kdtlb_miss					;\
1078	  srlx	%g2, TAG_VALO_SHIFT, %g7	/* g7 = tsb tag */	;\
1079	brlz,pn %g1, sfmmu_udtlb_slowpath				;\
1080	  nop								;\
1081	ldda	[%g1]ASI_NQUAD_LD, %g4	/* g4 = tag, %g5 data */	;\
1082	cmp	%g4, %g7						;\
1083	bne,pn	%xcc, sfmmu_tsb_miss_tt		/* no 4M TSB, miss */	;\
1084	  mov	%g0, %g3		/* clear 4M tsbe ptr */		;\
1085	TT_TRACE(trace_tsbhit)		/* 2 instr ifdef TRAPTRACE */	;\
1086	stxa	%g5, [%g0]ASI_DTLB_IN	/* trapstat expects TTE */	;\
1087	retry				/* in %g5 */			;\
1088	unimp	0							;\
1089	unimp	0							;\
1090	unimp	0							;\
1091	unimp	0							;\
1092	unimp	0							;\
1093	unimp	0							;\
1094	unimp	0							;\
1095	unimp	0							;\
1096	unimp	0							;\
1097	unimp	0							;\
1098	unimp	0							;\
1099	.align 128
1100
1101#if defined(cscope)
1102/*
1103 * Define labels to direct cscope quickly to labels that
1104 * are generated by macro expansion of ITLB_MISS().
1105 */
1106	.global	tt0_itlbmiss
1107tt0_itlbmiss:
1108	.global	tt1_itlbmiss
1109tt1_itlbmiss:
1110	nop
1111#endif
1112
1113/*
1114 * Instruction miss handler.
1115 * ldda instructions will have their ASI patched
1116 * by sfmmu_patch_ktsb at runtime.
1117 * MUST be EXACTLY 32 instructions or we'll break.
1118 */
1119#define	ITLB_MISS(table_name)						 \
1120	.global	table_name/**/_itlbmiss					;\
1121table_name/**/_itlbmiss:						;\
1122	HAT_PERCPU_DBSTAT(TSBMISS_ITLBMISS) /* 3 instr ifdef DEBUG */	;\
1123	mov	MMU_TAG_ACCESS, %g6		/* select tag acc */	;\
1124	ldxa	[%g0]ASI_IMMU_TSB_8K, %g1	/* g1 = tsbe ptr */	;\
1125	ldxa	[%g6]ASI_IMMU, %g2		/* g2 = tag access */	;\
1126	sllx	%g2, TAGACC_CTX_LSHIFT, %g3				;\
1127	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctx */		;\
1128	cmp	%g3, INVALID_CONTEXT					;\
1129	ble,pn	%xcc, sfmmu_kitlb_miss					;\
1130	  srlx	%g2, TAG_VALO_SHIFT, %g7	/* g7 = tsb tag */	;\
1131	brlz,pn	%g1, sfmmu_uitlb_slowpath	/* if >1 TSB branch */	;\
1132	  nop								;\
1133	ldda	[%g1]ASI_NQUAD_LD, %g4	/* g4 = tag, g5 = data */	;\
1134	cmp	%g4, %g7						;\
1135	bne,pn	%xcc, sfmmu_tsb_miss_tt	/* br if 8k ptr miss */		;\
1136	  mov	%g0, %g3		/* no 4M TSB */			;\
1137	andcc	%g5, TTE_EXECPRM_INT, %g0 /* check execute bit */	;\
1138	bz,pn	%icc, exec_fault					;\
1139	  nop								;\
1140	TT_TRACE(trace_tsbhit)		/* 2 instr ifdef TRAPTRACE */	;\
1141	stxa	%g5, [%g0]ASI_ITLB_IN	/* trapstat expects %g5 */	;\
1142	retry								;\
1143	unimp	0							;\
1144	unimp	0							;\
1145	unimp	0							;\
1146	unimp	0							;\
1147	unimp	0							;\
1148	unimp	0							;\
1149	unimp	0							;\
1150	unimp	0							;\
1151	.align 128
1152
1153
1154/*
1155 * This macro is the first level handler for fast protection faults.
1156 * It first demaps the tlb entry which generated the fault and then
1157 * attempts to set the modify bit on the hash.  It needs to be
1158 * exactly 32 instructions.
1159 */
1160#define	DTLB_PROT							 \
1161	DTLB_DEMAP_ENTRY		/* 20 instructions */		;\
1162	/*								;\
1163	 * At this point:						;\
1164	 *   g1 = ????							;\
1165	 *   g2 = tag access register					;\
1166	 *   g3 = ctx number						;\
1167	 *   g4 = ????							;\
1168	 */								;\
1169	TT_TRACE(trace_dataprot)	/* 2 instr ifdef TRAPTRACE */	;\
1170					/* clobbers g1 and g6 */	;\
1171	ldxa	[%g0]ASI_DMMU_TSB_8K, %g1	/* g1 = tsbe ptr */	;\
1172	brnz,pt %g3, sfmmu_uprot_trap		/* user trap */		;\
1173	  nop								;\
1174	ba,a,pt	%xcc, sfmmu_kprot_trap		/* kernel trap */	;\
1175	unimp	0							;\
1176	unimp	0							;\
1177	unimp	0							;\
1178	unimp	0							;\
1179	unimp	0							;\
1180	unimp	0							;\
1181	.align 128
1182
1183#define	DMMU_EXCEPTION_TL1						;\
1184	SWITCH_GLOBALS							;\
1185	ba,a,pt	%xcc, mmu_trap_tl1					;\
1186	  nop								;\
1187	.align 32
1188
1189#define	MISALIGN_ADDR_TL1						;\
1190	ba,a,pt	%xcc, mmu_trap_tl1					;\
1191	  nop								;\
1192	.align 32
1193
1194/*
1195 * Trace a tsb hit
1196 * g1 = tsbe pointer (in/clobbered)
1197 * g2 = tag access register (in)
1198 * g3 - g4 = scratch (clobbered)
1199 * g5 = tsbe data (in)
1200 * g6 = scratch (clobbered)
1201 * g7 = pc we jumped here from (in)
1202 * ttextra = value to OR in to trap type (%tt) (in)
1203 */
1204#ifdef TRAPTRACE
1205#define TRACE_TSBHIT(ttextra)						 \
1206	membar	#Sync							;\
1207	sethi	%hi(FLUSH_ADDR), %g6					;\
1208	flush	%g6							;\
1209	TRACE_PTR(%g3, %g6)						;\
1210	GET_TRACE_TICK(%g6)						;\
1211	stxa	%g6, [%g3 + TRAP_ENT_TICK]%asi				;\
1212	stxa	%g2, [%g3 + TRAP_ENT_SP]%asi	/* tag access */	;\
1213	stxa	%g5, [%g3 + TRAP_ENT_F1]%asi	/* tsb data */		;\
1214	rdpr	%tnpc, %g6						;\
1215	stxa	%g6, [%g3 + TRAP_ENT_F2]%asi				;\
1216	stxa	%g1, [%g3 + TRAP_ENT_F3]%asi	/* tsb pointer */	;\
1217	stxa	%g0, [%g3 + TRAP_ENT_F4]%asi				;\
1218	rdpr	%tpc, %g6						;\
1219	stxa	%g6, [%g3 + TRAP_ENT_TPC]%asi				;\
1220	rdpr	%tl, %g6						;\
1221	stha	%g6, [%g3 + TRAP_ENT_TL]%asi				;\
1222	rdpr	%tt, %g6						;\
1223	or	%g6, (ttextra), %g6					;\
1224	stha	%g6, [%g3 + TRAP_ENT_TT]%asi				;\
1225	ldxa	[%g0]ASI_IMMU, %g1		/* tag target */	;\
1226	ldxa	[%g0]ASI_DMMU, %g4					;\
1227	cmp	%g6, FAST_IMMU_MISS_TT					;\
1228	movne	%icc, %g4, %g1						;\
1229	stxa	%g1, [%g3 + TRAP_ENT_TSTATE]%asi /* tsb tag */		;\
1230	stxa	%g0, [%g3 + TRAP_ENT_TR]%asi				;\
1231	TRACE_NEXT(%g3, %g4, %g6)
1232#else
1233#define TRACE_TSBHIT(ttextra)
1234#endif
1235
1236#if defined(lint)
1237
1238struct scb	trap_table;
1239struct scb	scb;		/* trap_table/scb are the same object */
1240
1241#else /* lint */
1242
1243/*
1244 * =======================================================================
1245 *		SPARC V9 TRAP TABLE
1246 *
1247 * The trap table is divided into two halves: the first half is used when
1248 * taking traps when TL=0; the second half is used when taking traps from
1249 * TL>0. Note that handlers in the second half of the table might not be able
1250 * to make the same assumptions as handlers in the first half of the table.
1251 *
1252 * Worst case trap nesting so far:
1253 *
1254 *	at TL=0 client issues software trap requesting service
1255 *	at TL=1 nucleus wants a register window
1256 *	at TL=2 register window clean/spill/fill takes a TLB miss
1257 *	at TL=3 processing TLB miss
1258 *	at TL=4 handle asynchronous error
1259 *
1260 * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
1261 *
1262 * =======================================================================
1263 */
1264	.section ".text"
1265	.align	4
1266	.global trap_table, scb, trap_table0, trap_table1, etrap_table
1267	.type	trap_table, #function
1268	.type	scb, #function
1269trap_table:
1270scb:
1271trap_table0:
1272	/* hardware traps */
1273	NOT;				/* 000	reserved */
1274	RED;				/* 001	power on reset */
1275	RED;				/* 002	watchdog reset */
1276	RED;				/* 003	externally initiated reset */
1277	RED;				/* 004	software initiated reset */
1278	RED;				/* 005	red mode exception */
1279	NOT; NOT;			/* 006 - 007 reserved */
1280	IMMU_EXCEPTION;			/* 008	instruction access exception */
1281	NOT;				/* 009	instruction access MMU miss */
1282	ASYNC_TRAP(T_INSTR_ERROR, trace_gen);
1283					/* 00A	instruction access error */
1284	NOT; NOT4;			/* 00B - 00F reserved */
1285	ILLTRAP_INSTR;			/* 010	illegal instruction */
1286	TRAP(T_PRIV_INSTR);		/* 011	privileged opcode */
1287	NOT;				/* 012	unimplemented LDD */
1288	NOT;				/* 013	unimplemented STD */
1289	NOT4; NOT4; NOT4;		/* 014 - 01F reserved */
1290	FP_DISABLED_TRAP;		/* 020	fp disabled */
1291	FP_IEEE_TRAP;			/* 021	fp exception ieee 754 */
1292	FP_TRAP;			/* 022	fp exception other */
1293	TAG_OVERFLOW;			/* 023	tag overflow */
1294	CLEAN_WINDOW;			/* 024 - 027 clean window */
1295	DIV_BY_ZERO;			/* 028	division by zero */
1296	NOT;				/* 029	internal processor error */
1297	NOT; NOT; NOT4;			/* 02A - 02F reserved */
1298	DMMU_EXCEPTION;			/* 030	data access exception */
1299	NOT;				/* 031	data access MMU miss */
1300	ASYNC_TRAP(T_DATA_ERROR, trace_gen);
1301					/* 032	data access error */
1302	NOT;				/* 033	data access protection */
1303	DMMU_EXC_AG_NOT_ALIGNED;	/* 034	mem address not aligned */
1304	DMMU_EXC_LDDF_NOT_ALIGNED;	/* 035	LDDF mem address not aligned */
1305	DMMU_EXC_STDF_NOT_ALIGNED;	/* 036	STDF mem address not aligned */
1306	DMMU_EXC_AG_PRIV;		/* 037	privileged action */
1307	NOT;				/* 038	LDQF mem address not aligned */
1308	NOT;				/* 039	STQF mem address not aligned */
1309	NOT; NOT; NOT4;			/* 03A - 03F reserved */
1310	NOT;				/* 040	async data error */
1311	LEVEL_INTERRUPT(1);		/* 041	interrupt level 1 */
1312	LEVEL_INTERRUPT(2);		/* 042	interrupt level 2 */
1313	LEVEL_INTERRUPT(3);		/* 043	interrupt level 3 */
1314	LEVEL_INTERRUPT(4);		/* 044	interrupt level 4 */
1315	LEVEL_INTERRUPT(5);		/* 045	interrupt level 5 */
1316	LEVEL_INTERRUPT(6);		/* 046	interrupt level 6 */
1317	LEVEL_INTERRUPT(7);		/* 047	interrupt level 7 */
1318	LEVEL_INTERRUPT(8);		/* 048	interrupt level 8 */
1319	LEVEL_INTERRUPT(9);		/* 049	interrupt level 9 */
1320	LEVEL_INTERRUPT(10);		/* 04A	interrupt level 10 */
1321	LEVEL_INTERRUPT(11);		/* 04B	interrupt level 11 */
1322	LEVEL_INTERRUPT(12);		/* 04C	interrupt level 12 */
1323	LEVEL_INTERRUPT(13);		/* 04D	interrupt level 13 */
1324	LEVEL14_INTERRUPT;		/* 04E	interrupt level 14 */
1325	LEVEL_INTERRUPT(15);		/* 04F	interrupt level 15 */
1326	NOT4; NOT4; NOT4; NOT4;		/* 050 - 05F reserved */
1327	VECTOR_INTERRUPT;		/* 060	interrupt vector */
1328	GOTO(kmdb_trap);		/* 061	PA watchpoint */
1329	GOTO(kmdb_trap);		/* 062	VA watchpoint */
1330	GOTO_TT(ce_err, trace_gen);	/* 063	corrected ECC error */
1331	ITLB_MISS(tt0);			/* 064	instruction access MMU miss */
1332	DTLB_MISS(tt0);			/* 068	data access MMU miss */
1333	DTLB_PROT;			/* 06C	data access protection */
1334	LABELED_BAD(tt0_fecc);		/* 070  fast ecache ECC error */
1335	LABELED_BAD(tt0_dperr);		/* 071  Cheetah+ dcache parity error */
1336	LABELED_BAD(tt0_iperr);		/* 072  Cheetah+ icache parity error */
1337	NOT;				/* 073  reserved */
1338	NOT4; NOT4; NOT4;		/* 074 - 07F reserved */
1339	NOT4;				/* 080	spill 0 normal */
1340	SPILL_32bit_asi(ASI_AIUP,sn0);	/* 084	spill 1 normal */
1341	SPILL_64bit_asi(ASI_AIUP,sn0);	/* 088	spill 2 normal */
1342	SPILL_32clean(ASI_AIUP,sn0);	/* 08C	spill 3 normal */
1343	SPILL_64clean(ASI_AIUP,sn0);	/* 090	spill 4 normal */
1344	SPILL_32bit(not);		/* 094	spill 5 normal */
1345	SPILL_64bit(not);		/* 098	spill 6 normal */
1346	SPILL_mixed;			/* 09C	spill 7 normal */
1347	NOT4;				/* 0A0	spill 0 other */
1348	SPILL_32bit_asi(ASI_AIUS,so0);	/* 0A4	spill 1 other */
1349	SPILL_64bit_asi(ASI_AIUS,so0);	/* 0A8	spill 2 other */
1350	SPILL_32bit_asi(ASI_AIUS,so0);	/* 0AC	spill 3 other */
1351	SPILL_64bit_asi(ASI_AIUS,so0);	/* 0B0	spill 4 other */
1352	NOT4;				/* 0B4	spill 5 other */
1353	NOT4;				/* 0B8	spill 6 other */
1354	NOT4;				/* 0BC	spill 7 other */
1355	NOT4;				/* 0C0	fill 0 normal */
1356	FILL_32bit_asi(ASI_AIUP,fn0);	/* 0C4	fill 1 normal */
1357	FILL_64bit_asi(ASI_AIUP,fn0);	/* 0C8	fill 2 normal */
1358	FILL_32bit_asi(ASI_AIUP,fn0);	/* 0CC	fill 3 normal */
1359	FILL_64bit_asi(ASI_AIUP,fn0);	/* 0D0	fill 4 normal */
1360	FILL_32bit(not);		/* 0D4	fill 5 normal */
1361	FILL_64bit(not);		/* 0D8	fill 6 normal */
1362	FILL_mixed;			/* 0DC	fill 7 normal */
1363	NOT4;				/* 0E0	fill 0 other */
1364	NOT4;				/* 0E4	fill 1 other */
1365	NOT4;				/* 0E8	fill 2 other */
1366	NOT4;				/* 0EC	fill 3 other */
1367	NOT4;				/* 0F0	fill 4 other */
1368	NOT4;				/* 0F4	fill 5 other */
1369	NOT4;				/* 0F8	fill 6 other */
1370	NOT4;				/* 0FC	fill 7 other */
1371	/* user traps */
1372	GOTO(syscall_trap_4x);		/* 100	old system call */
1373	TRAP(T_BREAKPOINT);		/* 101	user breakpoint */
1374	TRAP(T_DIV0);			/* 102	user divide by zero */
1375	FLUSHW();			/* 103	flush windows */
1376	GOTO(.clean_windows);		/* 104	clean windows */
1377	BAD;				/* 105	range check ?? */
1378	GOTO(.fix_alignment);		/* 106	do unaligned references */
1379	BAD;				/* 107	unused */
1380	SYSCALL(syscall_trap32);	/* 108	ILP32 system call on LP64 */
1381	GOTO(set_trap0_addr);		/* 109	set trap0 address */
1382	BAD; BAD; BAD4;			/* 10A - 10F unused */
1383	TRP4; TRP4; TRP4; TRP4;		/* 110 - 11F V9 user trap handlers */
1384	GOTO(.getcc);			/* 120	get condition codes */
1385	GOTO(.setcc);			/* 121	set condition codes */
1386	GOTO(.getpsr);			/* 122	get psr */
1387	GOTO(.setpsr);			/* 123	set psr (some fields) */
1388	GOTO(get_timestamp);		/* 124	get timestamp */
1389	GOTO(get_virtime);		/* 125	get lwp virtual time */
1390	PRIV(self_xcall);		/* 126	self xcall */
1391	GOTO(get_hrestime);		/* 127	get hrestime */
1392	BAD;				/* 128	ST_SETV9STACK */
1393	GOTO(.getlgrp);			/* 129  get lgrpid */
1394	BAD; BAD; BAD4;			/* 12A - 12F unused */
1395	BAD4; BAD4; 			/* 130 - 137 unused */
1396	DTRACE_PID;			/* 138  dtrace pid tracing provider */
1397	DTRACE_FASTTRAP;		/* 139  dtrace fasttrap provider */
1398	DTRACE_RETURN;			/* 13A	dtrace pid return probe */
1399	BAD; BAD4;			/* 13B - 13F unused */
1400	SYSCALL(syscall_trap)		/* 140  LP64 system call */
1401	BAD;				/* 141  unused */
1402#ifdef DEBUG_USER_TRAPTRACECTL
1403	GOTO(.traptrace_freeze);	/* 142  freeze traptrace */
1404	GOTO(.traptrace_unfreeze);	/* 143  unfreeze traptrace */
1405#else
1406	BAD; BAD;			/* 142 - 143 unused */
1407#endif
1408	BAD4; BAD4; BAD4;		/* 144 - 14F unused */
1409	BAD4; BAD4; BAD4; BAD4;		/* 150 - 15F unused */
1410	BAD4; BAD4; BAD4; BAD4;		/* 160 - 16F unused */
1411	BAD;				/* 170 - unused */
1412	BAD;				/* 171 - unused */
1413	BAD; BAD;			/* 172 - 173 unused */
1414	BAD4; BAD4;			/* 174 - 17B unused */
1415#ifdef	PTL1_PANIC_DEBUG
1416	mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
1417					/* 17C	test ptl1_panic */
1418#else
1419	BAD;				/* 17C  unused */
1420#endif	/* PTL1_PANIC_DEBUG */
1421	PRIV(kmdb_trap);		/* 17D	kmdb enter (L1-A) */
1422	PRIV(kmdb_trap);		/* 17E	kmdb breakpoint */
1423	PRIV(kctx_obp_bpt);		/* 17F	obp breakpoint */
1424	/* reserved */
1425	NOT4; NOT4; NOT4; NOT4;		/* 180 - 18F reserved */
1426	NOT4; NOT4; NOT4; NOT4;		/* 190 - 19F reserved */
1427	NOT4; NOT4; NOT4; NOT4;		/* 1A0 - 1AF reserved */
1428	NOT4; NOT4; NOT4; NOT4;		/* 1B0 - 1BF reserved */
1429	NOT4; NOT4; NOT4; NOT4;		/* 1C0 - 1CF reserved */
1430	NOT4; NOT4; NOT4; NOT4;		/* 1D0 - 1DF reserved */
1431	NOT4; NOT4; NOT4; NOT4;		/* 1E0 - 1EF reserved */
1432	NOT4; NOT4; NOT4; NOT4;		/* 1F0 - 1FF reserved */
1433trap_table1:
1434	NOT4; NOT4; NOT; NOT;		/* 000 - 009 unused */
1435	ASYNC_TRAP(T_INSTR_ERROR + T_TL1, trace_gen);
1436					/* 00A	instruction access error */
1437	NOT; NOT4;			/* 00B - 00F unused */
1438	NOT4; NOT4; NOT4; NOT4;		/* 010 - 01F unused */
1439	NOT4;				/* 020 - 023 unused */
1440	CLEAN_WINDOW;			/* 024 - 027 clean window */
1441	NOT4; NOT4;			/* 028 - 02F unused */
1442	DMMU_EXCEPTION_TL1;		/* 030 	data access exception */
1443	NOT;				/* 031 unused */
1444	ASYNC_TRAP(T_DATA_ERROR + T_TL1, trace_gen);
1445					/* 032	data access error */
1446	NOT;				/* 033	unused */
1447	MISALIGN_ADDR_TL1;		/* 034	mem address not aligned */
1448	NOT; NOT; NOT; NOT4; NOT4	/* 035 - 03F unused */
1449	NOT4; NOT4; NOT4; NOT4;		/* 040 - 04F unused */
1450	NOT4; NOT4; NOT4; NOT4;		/* 050 - 05F unused */
1451	NOT;				/* 060	unused */
1452	GOTO(kmdb_trap_tl1);		/* 061	PA watchpoint */
1453	GOTO(kmdb_trap_tl1);		/* 062	VA watchpoint */
1454	GOTO_TT(ce_err_tl1, trace_gen);	/* 063	corrected ECC error */
1455	ITLB_MISS(tt1);			/* 064	instruction access MMU miss */
1456	DTLB_MISS(tt1);			/* 068	data access MMU miss */
1457	DTLB_PROT;			/* 06C	data access protection */
1458	LABELED_BAD(tt1_fecc);		/* 070  fast ecache ECC error */
1459	LABELED_BAD(tt1_dperr);		/* 071  Cheetah+ dcache parity error */
1460	LABELED_BAD(tt1_iperr);		/* 072  Cheetah+ icache parity error */
1461	NOT;				/* 073  reserved */
1462	NOT4; NOT4; NOT4;		/* 074 - 07F reserved */
1463	NOT4;				/* 080	spill 0 normal */
1464	SPILL_32bit_tt1(ASI_AIUP,sn1);	/* 084	spill 1 normal */
1465	SPILL_64bit_tt1(ASI_AIUP,sn1);	/* 088	spill 2 normal */
1466	SPILL_32bit_tt1(ASI_AIUP,sn1);	/* 08C	spill 3 normal */
1467	SPILL_64bit_tt1(ASI_AIUP,sn1);	/* 090	spill 4 normal */
1468	SPILL_32bit(not);		/* 094	spill 5 normal */
1469	SPILL_64bit(not);		/* 098	spill 6 normal */
1470	SPILL_mixed;			/* 09C	spill 7 normal */
1471	NOT4;				/* 0A0	spill 0 other */
1472	SPILL_32bit_tt1(ASI_AIUS,so1);	/* 0A4	spill 1 other */
1473	SPILL_64bit_tt1(ASI_AIUS,so1);	/* 0A8	spill 2 other */
1474	SPILL_32bit_tt1(ASI_AIUS,so1);	/* 0AC	spill 3 other */
1475	SPILL_64bit_tt1(ASI_AIUS,so1);	/* 0B0  spill 4 other */
1476	NOT4;				/* 0B4  spill 5 other */
1477	NOT4;				/* 0B8  spill 6 other */
1478	NOT4;				/* 0BC  spill 7 other */
1479	NOT4;				/* 0C0	fill 0 normal */
1480	FILL_32bit_tt1(ASI_AIUP,fn1);	/* 0C4	fill 1 normal */
1481	FILL_64bit_tt1(ASI_AIUP,fn1);	/* 0C8	fill 2 normal */
1482	FILL_32bit_tt1(ASI_AIUP,fn1);	/* 0CC	fill 3 normal */
1483	FILL_64bit_tt1(ASI_AIUP,fn1);	/* 0D0	fill 4 normal */
1484	FILL_32bit(not);		/* 0D4	fill 5 normal */
1485	FILL_64bit(not);		/* 0D8	fill 6 normal */
1486	FILL_mixed;			/* 0DC	fill 7 normal */
1487	NOT4; NOT4; NOT4; NOT4;		/* 0E0 - 0EF unused */
1488	NOT4; NOT4; NOT4; NOT4;		/* 0F0 - 0FF unused */
1489	LABELED_BAD(tt1_swtrap0);	/* 100  fast ecache ECC error (cont) */
1490	LABELED_BAD(tt1_swtrap1);	/* 101  Ch+ D$ parity error (cont) */
1491	LABELED_BAD(tt1_swtrap2);	/* 102  Ch+ I$ parity error (cont) */
1492	NOT;				/* 103  reserved */
1493/*
1494 * We only reserve the above four special case soft traps for code running
1495 * at TL>0, so we can truncate the trap table here.
1496 */
1497etrap_table:
1498	.size	trap_table, (.-trap_table)
1499	.size	scb, (.-scb)
1500
1501/*
1502 * We get to exec_fault in the case of an instruction miss and tte
1503 * has no execute bit set.  We go to tl0 to handle it.
1504 *
1505 * g1 = tsbe pointer (in/clobbered)
1506 * g2 = tag access register (in)
1507 * g3 - g4 = scratch (clobbered)
1508 * g5 = tsbe data (in)
1509 * g6 = scratch (clobbered)
1510 * g7 = pc we jumped here from (in)
1511 */
1512	ALTENTRY(exec_fault)
1513	TRACE_TSBHIT(0x200)
1514	SWITCH_GLOBALS
1515	mov	MMU_TAG_ACCESS, %g4
1516	ldxa	[%g4]ASI_IMMU, %g2			! arg1 = addr
1517	mov	T_INSTR_MMU_MISS, %g3			! arg2 = traptype
1518	set	trap, %g1
1519	ba,pt	%xcc, sys_trap
1520	  mov	-1, %g4
1521
1522.mmu_exception_not_aligned:
1523	rdpr	%tstate, %g1
1524	btst	TSTATE_PRIV, %g1
1525	bnz,pn	%icc, 2f
1526	nop
1527	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1528	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1529	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1530	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1531	brz,pt	%g5, 2f
1532	nop
1533	ldn	[%g5 + P_UTRAP15], %g5			! unaligned utrap?
1534	brz,pn	%g5, 2f
1535	nop
1536	btst	1, %sp
1537	bz,pt	%xcc, 1f				! 32 bit user program
1538	nop
1539	ba,pt	%xcc, .setup_v9utrap			! 64 bit user program
1540	nop
15411:
1542	ba,pt	%xcc, .setup_utrap
1543	or	%g2, %g0, %g7
15442:
1545	ba,pt	%xcc, .mmu_exception_end
1546	mov	T_ALIGNMENT, %g1
1547
1548.mmu_priv_exception:
1549	rdpr	%tstate, %g1
1550	btst	TSTATE_PRIV, %g1
1551	bnz,pn	%icc, 1f
1552	nop
1553	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1554	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1555	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1556	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1557	brz,pt	%g5, 1f
1558	nop
1559	ldn	[%g5 + P_UTRAP16], %g5
1560	brnz,pt	%g5, .setup_v9utrap
1561	nop
15621:
1563	mov	T_PRIV_INSTR, %g1
1564
1565.mmu_exception_end:
1566	CPU_INDEX(%g4, %g5)
1567	set	cpu_core, %g5
1568	sllx	%g4, CPU_CORE_SHIFT, %g4
1569	add	%g4, %g5, %g4
1570	lduh	[%g4 + CPUC_DTRACE_FLAGS], %g5
1571	andcc	%g5, CPU_DTRACE_NOFAULT, %g0
1572	bz	%xcc, .mmu_exception_tlb_chk
1573	or	%g5, CPU_DTRACE_BADADDR, %g5
1574	stuh	%g5, [%g4 + CPUC_DTRACE_FLAGS]
1575	done
1576
1577.mmu_exception_tlb_chk:
1578	GET_CPU_IMPL(%g5)			! check SFSR.FT to see if this
1579	cmp	%g5, PANTHER_IMPL		! is a TLB parity error. But
1580	bne	2f				! we only do this check while
1581	mov	1, %g4				! running on Panther CPUs
1582	sllx	%g4, PN_SFSR_PARITY_SHIFT, %g4	! since US-I/II use the same
1583	andcc	%g3, %g4, %g0			! bit for something else which
1584	bz	2f				! will be handled later.
1585	nop
1586.mmu_exception_is_tlb_parity:
1587	.weak itlb_parity_trap
1588	.weak dtlb_parity_trap
1589	set	itlb_parity_trap, %g4
1590	cmp	%g1, T_INSTR_EXCEPTION		! branch to the itlb or
1591	be	3f				! dtlb parity handler
1592	nop					! if this trap is due
1593	set	dtlb_parity_trap, %g4
1594	cmp	%g1, T_DATA_EXCEPTION		! to a IMMU exception
1595	be	3f				! or DMMU exception.
1596	nop
15972:
1598	sllx	%g3, 32, %g3
1599	or	%g3, %g1, %g3
1600	set	trap, %g1
1601	ba,pt	%xcc, sys_trap
1602	sub	%g0, 1, %g4
16033:
1604	jmp	%g4				! off to the appropriate
1605	nop					! TLB parity handler
1606
1607.fp_disabled:
1608	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1609	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1610#ifdef SF_ERRATA_30 /* call causes fp-disabled */
1611	brz,a,pn %g1, 2f
1612	  nop
1613#endif
1614	rdpr	%tstate, %g4
1615	btst	TSTATE_PRIV, %g4
1616#ifdef SF_ERRATA_30 /* call causes fp-disabled */
1617	bnz,pn %icc, 2f
1618	  nop
1619#else
1620	bnz,a,pn %icc, ptl1_panic
1621	  mov	PTL1_BAD_FPTRAP, %g1
1622#endif
1623	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1624	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1625	brz,a,pt %g5, 2f
1626	  nop
1627	ldn	[%g5 + P_UTRAP7], %g5			! fp_disabled utrap?
1628	brz,a,pn %g5, 2f
1629	  nop
1630	btst	1, %sp
1631	bz,a,pt	%xcc, 1f				! 32 bit user program
1632	  nop
1633	ba,a,pt	%xcc, .setup_v9utrap			! 64 bit user program
1634	  nop
16351:
1636	ba,pt	%xcc, .setup_utrap
1637	  or	%g0, %g0, %g7
16382:
1639	set	fp_disabled, %g1
1640	ba,pt	%xcc, sys_trap
1641	  sub	%g0, 1, %g4
1642
1643.fp_ieee_exception:
1644	rdpr	%tstate, %g1
1645	btst	TSTATE_PRIV, %g1
1646	bnz,a,pn %icc, ptl1_panic
1647	  mov	PTL1_BAD_FPTRAP, %g1
1648	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1649	stx	%fsr, [%g1 + CPU_TMP1]
1650	ldx	[%g1 + CPU_TMP1], %g2
1651	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1652	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1653	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1654	brz,a,pt %g5, 1f
1655	  nop
1656	ldn	[%g5 + P_UTRAP8], %g5
1657	brnz,a,pt %g5, .setup_v9utrap
1658	  nop
16591:
1660	set	_fp_ieee_exception, %g1
1661	ba,pt	%xcc, sys_trap
1662	  sub	%g0, 1, %g4
1663
1664/*
1665 * Register Inputs:
1666 *	%g5		user trap handler
1667 *	%g7		misaligned addr - for alignment traps only
1668 */
1669.setup_utrap:
1670	set	trap, %g1			! setup in case we go
1671	mov	T_FLUSH_PCB, %g3		! through sys_trap on
1672	sub	%g0, 1, %g4			! the save instruction below
1673
1674	/*
1675	 * If the DTrace pid provider is single stepping a copied-out
1676	 * instruction, t->t_dtrace_step will be set. In that case we need
1677	 * to abort the single-stepping (since execution of the instruction
1678	 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1679	 */
1680	save	%sp, -SA(MINFRAME32), %sp	! window for trap handler
1681	CPU_ADDR(%g1, %g4)			! load CPU struct addr
1682	ldn	[%g1 + CPU_THREAD], %g1		! load thread pointer
1683	ldub	[%g1 + T_DTRACE_STEP], %g2	! load t->t_dtrace_step
1684	rdpr	%tnpc, %l2			! arg1 == tnpc
1685	brz,pt	%g2, 1f
1686	rdpr	%tpc, %l1			! arg0 == tpc
1687
1688	ldub	[%g1 + T_DTRACE_AST], %g2	! load t->t_dtrace_ast
1689	ldn	[%g1 + T_DTRACE_NPC], %l2	! arg1 = t->t_dtrace_npc (step)
1690	brz,pt	%g2, 1f
1691	st	%g0, [%g1 + T_DTRACE_FT]	! zero all pid provider flags
1692	stub	%g2, [%g1 + T_ASTFLAG]		! aston(t) if t->t_dtrace_ast
16931:
1694	mov	%g7, %l3			! arg2 == misaligned address
1695
1696	rdpr	%tstate, %g1			! cwp for trap handler
1697	rdpr	%cwp, %g4
1698	bclr	TSTATE_CWP_MASK, %g1
1699	wrpr	%g1, %g4, %tstate
1700	wrpr	%g0, %g5, %tnpc			! trap handler address
1701	FAST_TRAP_DONE
1702	/* NOTREACHED */
1703
1704.check_v9utrap:
1705	rdpr	%tstate, %g1
1706	btst	TSTATE_PRIV, %g1
1707	bnz,a,pn %icc, 3f
1708	  nop
1709	CPU_ADDR(%g4, %g1)				! load CPU struct addr
1710	ldn	[%g4 + CPU_THREAD], %g5			! load thread pointer
1711	ldn	[%g5 + T_PROCP], %g5			! load proc pointer
1712	ldn	[%g5 + P_UTRAPS], %g5			! are there utraps?
1713
1714	cmp	%g3, T_SOFTWARE_TRAP
1715	bne,a,pt %icc, 1f
1716	  nop
1717
1718	brz,pt %g5, 3f			! if p_utraps == NULL goto trap()
1719	  rdpr	%tt, %g3		! delay - get actual hw trap type
1720
1721	sub	%g3, 254, %g1		! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
1722	ba,pt	%icc, 2f
1723	  smul	%g1, CPTRSIZE, %g2
17241:
1725	brz,a,pt %g5, 3f		! if p_utraps == NULL goto trap()
1726	  nop
1727
1728	cmp	%g3, T_UNIMP_INSTR
1729	bne,a,pt %icc, 2f
1730	  nop
1731
1732	mov	1, %g1
1733	st	%g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
1734	rdpr	%tpc, %g1		! ld trapping instruction using
1735	lduwa	[%g1]ASI_AIUP, %g1	! "AS IF USER" ASI which could fault
1736	st	%g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
1737
1738	sethi	%hi(0xc1c00000), %g4	! setup mask for illtrap instruction
1739	andcc	%g1, %g4, %g4		! and instruction with mask
1740	bnz,a,pt %icc, 3f		! if %g4 == zero, %g1 is an ILLTRAP
1741	  nop				! fall thru to setup
17422:
1743	ldn	[%g5 + %g2], %g5
1744	brnz,a,pt %g5, .setup_v9utrap
1745	  nop
17463:
1747	set	trap, %g1
1748	ba,pt	%xcc, sys_trap
1749	  sub	%g0, 1, %g4
1750	/* NOTREACHED */
1751
1752/*
1753 * Register Inputs:
1754 *	%g5		user trap handler
1755 */
1756.setup_v9utrap:
1757	set	trap, %g1			! setup in case we go
1758	mov	T_FLUSH_PCB, %g3		! through sys_trap on
1759	sub	%g0, 1, %g4			! the save instruction below
1760
1761	/*
1762	 * If the DTrace pid provider is single stepping a copied-out
1763	 * instruction, t->t_dtrace_step will be set. In that case we need
1764	 * to abort the single-stepping (since execution of the instruction
1765	 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1766	 */
1767	save	%sp, -SA(MINFRAME64), %sp	! window for trap handler
1768	CPU_ADDR(%g1, %g4)			! load CPU struct addr
1769	ldn	[%g1 + CPU_THREAD], %g1		! load thread pointer
1770	ldub	[%g1 + T_DTRACE_STEP], %g2	! load t->t_dtrace_step
1771	rdpr	%tnpc, %l7			! arg1 == tnpc
1772	brz,pt	%g2, 1f
1773	rdpr	%tpc, %l6			! arg0 == tpc
1774
1775	ldub	[%g1 + T_DTRACE_AST], %g2	! load t->t_dtrace_ast
1776	ldn	[%g1 + T_DTRACE_NPC], %l7	! arg1 == t->t_dtrace_npc (step)
1777	brz,pt	%g2, 1f
1778	st	%g0, [%g1 + T_DTRACE_FT]	! zero all pid provider flags
1779	stub	%g2, [%g1 + T_ASTFLAG]		! aston(t) if t->t_dtrace_ast
17801:
1781	rdpr	%tstate, %g2			! cwp for trap handler
1782	rdpr	%cwp, %g4
1783	bclr	TSTATE_CWP_MASK, %g2
1784	wrpr	%g2, %g4, %tstate
1785
1786	ldn	[%g1 + T_PROCP], %g4		! load proc pointer
1787	ldn	[%g4 + P_AS], %g4		! load as pointer
1788	ldn	[%g4 + A_USERLIMIT], %g4	! load as userlimit
1789	cmp	%l7, %g4			! check for single-step set
1790	bne,pt	%xcc, 4f
1791	  nop
1792	ldn	[%g1 + T_LWP], %g1		! load klwp pointer
1793	ld	[%g1 + PCB_STEP], %g4		! load single-step flag
1794	cmp	%g4, STEP_ACTIVE		! step flags set in pcb?
1795	bne,pt	%icc, 4f
1796	  nop
1797	stn	%g5, [%g1 + PCB_TRACEPC]	! save trap handler addr in pcb
1798	mov	%l7, %g4			! on entry to precise user trap
1799	add	%l6, 4, %l7			! handler, %l6 == pc, %l7 == npc
1800						! at time of trap
1801	wrpr	%g0, %g4, %tnpc			! generate FLTBOUNDS,
1802						! %g4 == userlimit
1803	FAST_TRAP_DONE
1804	/* NOTREACHED */
18054:
1806	wrpr	%g0, %g5, %tnpc			! trap handler address
1807	FAST_TRAP_DONE_CHK_INTR
1808	/* NOTREACHED */
1809
1810.fp_exception:
1811	CPU_ADDR(%g1, %g4)
1812	stx	%fsr, [%g1 + CPU_TMP1]
1813	ldx	[%g1 + CPU_TMP1], %g2
1814
1815	/*
1816	 * Cheetah takes unfinished_FPop trap for certain range of operands
1817	 * to the "fitos" instruction. Instead of going through the slow
1818	 * software emulation path, we try to simulate the "fitos" instruction
1819	 * via "fitod" and "fdtos" provided the following conditions are met:
1820	 *
1821	 *	fpu_exists is set (if DEBUG)
1822	 *	not in privileged mode
1823	 *	ftt is unfinished_FPop
1824	 *	NXM IEEE trap is not enabled
1825	 *	instruction at %tpc is "fitos"
1826	 *
1827	 *  Usage:
1828	 *	%g1	per cpu address
1829	 *	%g2	%fsr
1830	 *	%g6	user instruction
1831	 *
1832	 * Note that we can take a memory access related trap while trying
1833	 * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
1834	 * flag to catch those traps and let the SFMMU code deal with page
1835	 * fault and data access exception.
1836	 */
1837#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
1838	sethi	%hi(fpu_exists), %g7
1839	ld	[%g7 + %lo(fpu_exists)], %g7
1840	brz,pn %g7, .fp_exception_cont
1841	  nop
1842#endif
1843	rdpr	%tstate, %g7			! branch if in privileged mode
1844	btst	TSTATE_PRIV, %g7
1845	bnz,pn	%xcc, .fp_exception_cont
1846	srl	%g2, FSR_FTT_SHIFT, %g7		! extract ftt from %fsr
1847	and	%g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
1848	cmp	%g7, FTT_UNFIN
1849	set	FSR_TEM_NX, %g5
1850	bne,pn	%xcc, .fp_exception_cont	! branch if NOT unfinished_FPop
1851	  andcc	%g2, %g5, %g0
1852	bne,pn	%xcc, .fp_exception_cont	! branch if FSR_TEM_NX enabled
1853	  rdpr	%tpc, %g5			! get faulting PC
1854
1855	or	%g0, 1, %g7
1856	st	%g7, [%g1 + CPU_TL1_HDLR]	! set tl1_hdlr flag
1857	lda	[%g5]ASI_USER, %g6		! get user's instruction
1858	st	%g0, [%g1 + CPU_TL1_HDLR]	! clear tl1_hdlr flag
1859
1860	set	FITOS_INSTR_MASK, %g7
1861	and	%g6, %g7, %g7
1862	set	FITOS_INSTR, %g5
1863	cmp	%g7, %g5
1864	bne,pn	%xcc, .fp_exception_cont	! branch if not FITOS_INSTR
1865	 nop
1866
1867	/*
1868	 * This is unfinished FPops trap for "fitos" instruction. We
1869	 * need to simulate "fitos" via "fitod" and "fdtos" instruction
1870	 * sequence.
1871	 *
1872	 * We need a temporary FP register to do the conversion. Since
1873	 * both source and destination operands for the "fitos" instruction
1874	 * have to be within %f0-%f31, we use an FP register from the upper
1875	 * half to guarantee that it won't collide with the source or the
1876	 * dest operand. However, we do have to save and restore its value.
1877	 *
1878	 * We use %d62 as a temporary FP register for the conversion and
1879	 * branch to appropriate instruction within the conversion tables
1880	 * based upon the rs2 and rd values.
1881	 */
1882
1883	std	%d62, [%g1 + CPU_TMP1]		! save original value
1884
1885	srl	%g6, FITOS_RS2_SHIFT, %g7
1886	and	%g7, FITOS_REG_MASK, %g7
1887	set	_fitos_fitod_table, %g4
1888	sllx	%g7, 2, %g7
1889	jmp	%g4 + %g7
1890	  ba,pt	%xcc, _fitos_fitod_done
1891	.empty
1892
1893_fitos_fitod_table:
1894	  fitod	%f0, %d62
1895	  fitod	%f1, %d62
1896	  fitod	%f2, %d62
1897	  fitod	%f3, %d62
1898	  fitod	%f4, %d62
1899	  fitod	%f5, %d62
1900	  fitod	%f6, %d62
1901	  fitod	%f7, %d62
1902	  fitod	%f8, %d62
1903	  fitod	%f9, %d62
1904	  fitod	%f10, %d62
1905	  fitod	%f11, %d62
1906	  fitod	%f12, %d62
1907	  fitod	%f13, %d62
1908	  fitod	%f14, %d62
1909	  fitod	%f15, %d62
1910	  fitod	%f16, %d62
1911	  fitod	%f17, %d62
1912	  fitod	%f18, %d62
1913	  fitod	%f19, %d62
1914	  fitod	%f20, %d62
1915	  fitod	%f21, %d62
1916	  fitod	%f22, %d62
1917	  fitod	%f23, %d62
1918	  fitod	%f24, %d62
1919	  fitod	%f25, %d62
1920	  fitod	%f26, %d62
1921	  fitod	%f27, %d62
1922	  fitod	%f28, %d62
1923	  fitod	%f29, %d62
1924	  fitod	%f30, %d62
1925	  fitod	%f31, %d62
1926_fitos_fitod_done:
1927
1928	/*
1929	 * Now convert data back into single precision
1930	 */
1931	srl	%g6, FITOS_RD_SHIFT, %g7
1932	and	%g7, FITOS_REG_MASK, %g7
1933	set	_fitos_fdtos_table, %g4
1934	sllx	%g7, 2, %g7
1935	jmp	%g4 + %g7
1936	  ba,pt	%xcc, _fitos_fdtos_done
1937	.empty
1938
1939_fitos_fdtos_table:
1940	  fdtos	%d62, %f0
1941	  fdtos	%d62, %f1
1942	  fdtos	%d62, %f2
1943	  fdtos	%d62, %f3
1944	  fdtos	%d62, %f4
1945	  fdtos	%d62, %f5
1946	  fdtos	%d62, %f6
1947	  fdtos	%d62, %f7
1948	  fdtos	%d62, %f8
1949	  fdtos	%d62, %f9
1950	  fdtos	%d62, %f10
1951	  fdtos	%d62, %f11
1952	  fdtos	%d62, %f12
1953	  fdtos	%d62, %f13
1954	  fdtos	%d62, %f14
1955	  fdtos	%d62, %f15
1956	  fdtos	%d62, %f16
1957	  fdtos	%d62, %f17
1958	  fdtos	%d62, %f18
1959	  fdtos	%d62, %f19
1960	  fdtos	%d62, %f20
1961	  fdtos	%d62, %f21
1962	  fdtos	%d62, %f22
1963	  fdtos	%d62, %f23
1964	  fdtos	%d62, %f24
1965	  fdtos	%d62, %f25
1966	  fdtos	%d62, %f26
1967	  fdtos	%d62, %f27
1968	  fdtos	%d62, %f28
1969	  fdtos	%d62, %f29
1970	  fdtos	%d62, %f30
1971	  fdtos	%d62, %f31
1972_fitos_fdtos_done:
1973
1974	ldd	[%g1 + CPU_TMP1], %d62		! restore %d62
1975
1976#if DEBUG
1977	/*
1978	 * Update FPop_unfinished trap kstat
1979	 */
1980	set	fpustat+FPUSTAT_UNFIN_KSTAT, %g7
1981	ldx	[%g7], %g5
19821:
1983	add	%g5, 1, %g6
1984
1985	casxa	[%g7] ASI_N, %g5, %g6
1986	cmp	%g5, %g6
1987	bne,a,pn %xcc, 1b
1988	  or	%g0, %g6, %g5
1989
1990	/*
1991	 * Update fpu_sim_fitos kstat
1992	 */
1993	set	fpuinfo+FPUINFO_FITOS_KSTAT, %g7
1994	ldx	[%g7], %g5
19951:
1996	add	%g5, 1, %g6
1997
1998	casxa	[%g7] ASI_N, %g5, %g6
1999	cmp	%g5, %g6
2000	bne,a,pn %xcc, 1b
2001	  or	%g0, %g6, %g5
2002#endif /* DEBUG */
2003
2004	FAST_TRAP_DONE
2005
2006.fp_exception_cont:
2007	/*
2008	 * Let _fp_exception deal with simulating FPop instruction.
2009	 * Note that we need to pass %fsr in %g2 (already read above).
2010	 */
2011
2012	set	_fp_exception, %g1
2013	ba,pt	%xcc, sys_trap
2014	sub	%g0, 1, %g4
2015
2016.clean_windows:
2017	set	trap, %g1
2018	mov	T_FLUSH_PCB, %g3
2019	sub	%g0, 1, %g4
2020	save
2021	flushw
2022	restore
2023	wrpr	%g0, %g0, %cleanwin	! no clean windows
2024
2025	CPU_ADDR(%g4, %g5)
2026	ldn	[%g4 + CPU_MPCB], %g4
2027	brz,a,pn %g4, 1f
2028	  nop
2029	ld	[%g4 + MPCB_WSTATE], %g5
2030	add	%g5, WSTATE_CLEAN_OFFSET, %g5
2031	wrpr	%g0, %g5, %wstate
20321:	FAST_TRAP_DONE
2033
2034/*
2035 * .spill_clean: clean the previous window, restore the wstate, and
2036 * "done".
2037 *
2038 * Entry: %g7 contains new wstate
2039 */
2040.spill_clean:
2041	sethi	%hi(nwin_minus_one), %g5
2042	ld	[%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
2043	rdpr	%cwp, %g6			! %g6 = %cwp
2044	deccc	%g6				! %g6--
2045	movneg	%xcc, %g5, %g6			! if (%g6<0) %g6 = nwin-1
2046	wrpr	%g6, %cwp
2047	TT_TRACE_L(trace_win)
2048	clr	%l0
2049	clr	%l1
2050	clr	%l2
2051	clr	%l3
2052	clr	%l4
2053	clr	%l5
2054	clr	%l6
2055	clr	%l7
2056	wrpr	%g0, %g7, %wstate
2057	saved
2058	retry			! restores correct %cwp
2059
2060.fix_alignment:
2061	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2062	ldn	[%g1 + CPU_THREAD], %g1	! load thread pointer
2063	ldn	[%g1 + T_PROCP], %g1
2064	mov	1, %g2
2065	stb	%g2, [%g1 + P_FIXALIGNMENT]
2066	FAST_TRAP_DONE
2067
2068#define	STDF_REG(REG, ADDR, TMP)		\
2069	sll	REG, 3, REG			;\
2070mark1:	set	start1, TMP			;\
2071	jmp	REG + TMP			;\
2072	  nop					;\
2073start1:	ba,pt	%xcc, done1			;\
2074	  std	%f0, [ADDR + CPU_TMP1]		;\
2075	ba,pt	%xcc, done1			;\
2076	  std	%f32, [ADDR + CPU_TMP1]		;\
2077	ba,pt	%xcc, done1			;\
2078	  std	%f2, [ADDR + CPU_TMP1]		;\
2079	ba,pt	%xcc, done1			;\
2080	  std	%f34, [ADDR + CPU_TMP1]		;\
2081	ba,pt	%xcc, done1			;\
2082	  std	%f4, [ADDR + CPU_TMP1]		;\
2083	ba,pt	%xcc, done1			;\
2084	  std	%f36, [ADDR + CPU_TMP1]		;\
2085	ba,pt	%xcc, done1			;\
2086	  std	%f6, [ADDR + CPU_TMP1]		;\
2087	ba,pt	%xcc, done1			;\
2088	  std	%f38, [ADDR + CPU_TMP1]		;\
2089	ba,pt	%xcc, done1			;\
2090	  std	%f8, [ADDR + CPU_TMP1]		;\
2091	ba,pt	%xcc, done1			;\
2092	  std	%f40, [ADDR + CPU_TMP1]		;\
2093	ba,pt	%xcc, done1			;\
2094	  std	%f10, [ADDR + CPU_TMP1]		;\
2095	ba,pt	%xcc, done1			;\
2096	  std	%f42, [ADDR + CPU_TMP1]		;\
2097	ba,pt	%xcc, done1			;\
2098	  std	%f12, [ADDR + CPU_TMP1]		;\
2099	ba,pt	%xcc, done1			;\
2100	  std	%f44, [ADDR + CPU_TMP1]		;\
2101	ba,pt	%xcc, done1			;\
2102	  std	%f14, [ADDR + CPU_TMP1]		;\
2103	ba,pt	%xcc, done1			;\
2104	  std	%f46, [ADDR + CPU_TMP1]		;\
2105	ba,pt	%xcc, done1			;\
2106	  std	%f16, [ADDR + CPU_TMP1]		;\
2107	ba,pt	%xcc, done1			;\
2108	  std	%f48, [ADDR + CPU_TMP1]		;\
2109	ba,pt	%xcc, done1			;\
2110	  std	%f18, [ADDR + CPU_TMP1]		;\
2111	ba,pt	%xcc, done1			;\
2112	  std	%f50, [ADDR + CPU_TMP1]		;\
2113	ba,pt	%xcc, done1			;\
2114	  std	%f20, [ADDR + CPU_TMP1]		;\
2115	ba,pt	%xcc, done1			;\
2116	  std	%f52, [ADDR + CPU_TMP1]		;\
2117	ba,pt	%xcc, done1			;\
2118	  std	%f22, [ADDR + CPU_TMP1]		;\
2119	ba,pt	%xcc, done1			;\
2120	  std	%f54, [ADDR + CPU_TMP1]		;\
2121	ba,pt	%xcc, done1			;\
2122	  std	%f24, [ADDR + CPU_TMP1]		;\
2123	ba,pt	%xcc, done1			;\
2124	  std	%f56, [ADDR + CPU_TMP1]		;\
2125	ba,pt	%xcc, done1			;\
2126	  std	%f26, [ADDR + CPU_TMP1]		;\
2127	ba,pt	%xcc, done1			;\
2128	  std	%f58, [ADDR + CPU_TMP1]		;\
2129	ba,pt	%xcc, done1			;\
2130	  std	%f28, [ADDR + CPU_TMP1]		;\
2131	ba,pt	%xcc, done1			;\
2132	  std	%f60, [ADDR + CPU_TMP1]		;\
2133	ba,pt	%xcc, done1			;\
2134	  std	%f30, [ADDR + CPU_TMP1]		;\
2135	ba,pt	%xcc, done1			;\
2136	  std	%f62, [ADDR + CPU_TMP1]		;\
2137done1:
2138
2139#define	LDDF_REG(REG, ADDR, TMP)		\
2140	sll	REG, 3, REG			;\
2141mark2:	set	start2, TMP			;\
2142	jmp	REG + TMP			;\
2143	  nop					;\
2144start2:	ba,pt	%xcc, done2			;\
2145	  ldd	[ADDR + CPU_TMP1], %f0		;\
2146	ba,pt	%xcc, done2			;\
2147	  ldd	[ADDR + CPU_TMP1], %f32		;\
2148	ba,pt	%xcc, done2			;\
2149	  ldd	[ADDR + CPU_TMP1], %f2		;\
2150	ba,pt	%xcc, done2			;\
2151	  ldd	[ADDR + CPU_TMP1], %f34		;\
2152	ba,pt	%xcc, done2			;\
2153	  ldd	[ADDR + CPU_TMP1], %f4		;\
2154	ba,pt	%xcc, done2			;\
2155	  ldd	[ADDR + CPU_TMP1], %f36		;\
2156	ba,pt	%xcc, done2			;\
2157	  ldd	[ADDR + CPU_TMP1], %f6		;\
2158	ba,pt	%xcc, done2			;\
2159	  ldd	[ADDR + CPU_TMP1], %f38		;\
2160	ba,pt	%xcc, done2			;\
2161	  ldd	[ADDR + CPU_TMP1], %f8		;\
2162	ba,pt	%xcc, done2			;\
2163	  ldd	[ADDR + CPU_TMP1], %f40		;\
2164	ba,pt	%xcc, done2			;\
2165	  ldd	[ADDR + CPU_TMP1], %f10		;\
2166	ba,pt	%xcc, done2			;\
2167	  ldd	[ADDR + CPU_TMP1], %f42		;\
2168	ba,pt	%xcc, done2			;\
2169	  ldd	[ADDR + CPU_TMP1], %f12		;\
2170	ba,pt	%xcc, done2			;\
2171	  ldd	[ADDR + CPU_TMP1], %f44		;\
2172	ba,pt	%xcc, done2			;\
2173	  ldd	[ADDR + CPU_TMP1], %f14		;\
2174	ba,pt	%xcc, done2			;\
2175	  ldd	[ADDR + CPU_TMP1], %f46		;\
2176	ba,pt	%xcc, done2			;\
2177	  ldd	[ADDR + CPU_TMP1], %f16		;\
2178	ba,pt	%xcc, done2			;\
2179	  ldd	[ADDR + CPU_TMP1], %f48		;\
2180	ba,pt	%xcc, done2			;\
2181	  ldd	[ADDR + CPU_TMP1], %f18		;\
2182	ba,pt	%xcc, done2			;\
2183	  ldd	[ADDR + CPU_TMP1], %f50		;\
2184	ba,pt	%xcc, done2			;\
2185	  ldd	[ADDR + CPU_TMP1], %f20		;\
2186	ba,pt	%xcc, done2			;\
2187	  ldd	[ADDR + CPU_TMP1], %f52		;\
2188	ba,pt	%xcc, done2			;\
2189	  ldd	[ADDR + CPU_TMP1], %f22		;\
2190	ba,pt	%xcc, done2			;\
2191	  ldd	[ADDR + CPU_TMP1], %f54		;\
2192	ba,pt	%xcc, done2			;\
2193	  ldd	[ADDR + CPU_TMP1], %f24		;\
2194	ba,pt	%xcc, done2			;\
2195	  ldd	[ADDR + CPU_TMP1], %f56		;\
2196	ba,pt	%xcc, done2			;\
2197	  ldd	[ADDR + CPU_TMP1], %f26		;\
2198	ba,pt	%xcc, done2			;\
2199	  ldd	[ADDR + CPU_TMP1], %f58		;\
2200	ba,pt	%xcc, done2			;\
2201	  ldd	[ADDR + CPU_TMP1], %f28		;\
2202	ba,pt	%xcc, done2			;\
2203	  ldd	[ADDR + CPU_TMP1], %f60		;\
2204	ba,pt	%xcc, done2			;\
2205	  ldd	[ADDR + CPU_TMP1], %f30		;\
2206	ba,pt	%xcc, done2			;\
2207	  ldd	[ADDR + CPU_TMP1], %f62		;\
2208done2:
2209
2210.lddf_exception_not_aligned:
2211	/*
2212	 * Cheetah overwrites SFAR on a DTLB miss, hence read it now.
2213	 */
2214	ldxa	[MMU_SFAR]%asi, %g5	! misaligned vaddr in %g5
2215
2216#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2217	sethi	%hi(fpu_exists), %g2		! check fpu_exists
2218	ld	[%g2 + %lo(fpu_exists)], %g2
2219	brz,a,pn %g2, 4f
2220	  nop
2221#endif
2222	CPU_ADDR(%g1, %g4)
2223	or	%g0, 1, %g4
2224	st	%g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2225
2226	rdpr	%tpc, %g2
2227	lda	[%g2]ASI_AIUP, %g6	! get the user's lddf instruction
2228	srl	%g6, 23, %g1		! using ldda or not?
2229	and	%g1, 1, %g1
2230	brz,a,pt %g1, 2f		! check for ldda instruction
2231	  nop
2232	srl	%g6, 13, %g1		! check immflag
2233	and	%g1, 1, %g1
2234	rdpr	%tstate, %g2		! %tstate in %g2
2235	brnz,a,pn %g1, 1f
2236	  srl	%g2, 31, %g1		! get asi from %tstate
2237	srl	%g6, 5, %g1		! get asi from instruction
2238	and	%g1, 0xFF, %g1		! imm_asi field
22391:
2240	cmp	%g1, ASI_P		! primary address space
2241	be,a,pt %icc, 2f
2242	  nop
2243	cmp	%g1, ASI_PNF		! primary no fault address space
2244	be,a,pt %icc, 2f
2245	  nop
2246	cmp	%g1, ASI_S		! secondary address space
2247	be,a,pt %icc, 2f
2248	  nop
2249	cmp	%g1, ASI_SNF		! secondary no fault address space
2250	bne,a,pn %icc, 3f
2251	  nop
22522:
2253	lduwa	[%g5]ASI_USER, %g7	! get first half of misaligned data
2254	add	%g5, 4, %g5		! increment misaligned data address
2255	lduwa	[%g5]ASI_USER, %g5	! get second half of misaligned data
2256
2257	sllx	%g7, 32, %g7
2258	or	%g5, %g7, %g5		! combine data
2259	CPU_ADDR(%g7, %g1)		! save data on a per-cpu basis
2260	stx	%g5, [%g7 + CPU_TMP1]	! save in cpu_tmp1
2261
2262	srl	%g6, 25, %g3		! %g6 has the instruction
2263	and	%g3, 0x1F, %g3		! %g3 has rd
2264	LDDF_REG(%g3, %g7, %g4)
2265
2266	CPU_ADDR(%g1, %g4)
2267	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2268	FAST_TRAP_DONE
22693:
2270	CPU_ADDR(%g1, %g4)
2271	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
22724:
2273	set	T_USER, %g3		! trap type in %g3
2274	or	%g3, T_LDDF_ALIGN, %g3
2275	mov	%g5, %g2		! misaligned vaddr in %g2
2276	set	fpu_trap, %g1		! goto C for the little and
2277	ba,pt	%xcc, sys_trap		! no fault little asi's
2278	  sub	%g0, 1, %g4
2279
2280.stdf_exception_not_aligned:
2281	/*
2282	 * Cheetah overwrites SFAR on a DTLB miss, hence read it now.
2283	 */
2284	ldxa	[MMU_SFAR]%asi, %g5	! misaligned vaddr in %g5
2285
2286#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2287	sethi	%hi(fpu_exists), %g7		! check fpu_exists
2288	ld	[%g7 + %lo(fpu_exists)], %g3
2289	brz,a,pn %g3, 4f
2290	  nop
2291#endif
2292	CPU_ADDR(%g1, %g4)
2293	or	%g0, 1, %g4
2294	st	%g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2295
2296	rdpr	%tpc, %g2
2297	lda	[%g2]ASI_AIUP, %g6	! get the user's stdf instruction
2298
2299	srl	%g6, 23, %g1		! using stda or not?
2300	and	%g1, 1, %g1
2301	brz,a,pt %g1, 2f		! check for stda instruction
2302	  nop
2303	srl	%g6, 13, %g1		! check immflag
2304	and	%g1, 1, %g1
2305	rdpr	%tstate, %g2		! %tstate in %g2
2306	brnz,a,pn %g1, 1f
2307	  srl	%g2, 31, %g1		! get asi from %tstate
2308	srl	%g6, 5, %g1		! get asi from instruction
2309	and	%g1, 0xFF, %g1		! imm_asi field
23101:
2311	cmp	%g1, ASI_P		! primary address space
2312	be,a,pt %icc, 2f
2313	  nop
2314	cmp	%g1, ASI_S		! secondary address space
2315	bne,a,pn %icc, 3f
2316	  nop
23172:
2318	srl	%g6, 25, %g6
2319	and	%g6, 0x1F, %g6		! %g6 has rd
2320	CPU_ADDR(%g7, %g1)
2321	STDF_REG(%g6, %g7, %g4)		! STDF_REG(REG, ADDR, TMP)
2322
2323	ldx	[%g7 + CPU_TMP1], %g6
2324	srlx	%g6, 32, %g7
2325	stuwa	%g7, [%g5]ASI_USER	! first half
2326	add	%g5, 4, %g5		! increment misaligned data address
2327	stuwa	%g6, [%g5]ASI_USER	! second half
2328
2329	CPU_ADDR(%g1, %g4)
2330	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2331	FAST_TRAP_DONE
23323:
2333	CPU_ADDR(%g1, %g4)
2334	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
23354:
2336	set	T_USER, %g3		! trap type in %g3
2337	or	%g3, T_STDF_ALIGN, %g3
2338	mov	%g5, %g2		! misaligned vaddr in %g2
2339	set	fpu_trap, %g1		! goto C for the little and
2340	ba,pt	%xcc, sys_trap		! nofault little asi's
2341	  sub	%g0, 1, %g4
2342
2343#ifdef DEBUG_USER_TRAPTRACECTL
2344
2345.traptrace_freeze:
2346	mov	%l0, %g1 ; mov	%l1, %g2 ; mov	%l2, %g3 ; mov	%l4, %g4
2347	TT_TRACE_L(trace_win)
2348	mov	%g4, %l4 ; mov	%g3, %l2 ; mov	%g2, %l1 ; mov	%g1, %l0
2349	set	trap_freeze, %g1
2350	mov	1, %g2
2351	st	%g2, [%g1]
2352	FAST_TRAP_DONE
2353
2354.traptrace_unfreeze:
2355	set	trap_freeze, %g1
2356	st	%g0, [%g1]
2357	mov	%l0, %g1 ; mov	%l1, %g2 ; mov	%l2, %g3 ; mov	%l4, %g4
2358	TT_TRACE_L(trace_win)
2359	mov	%g4, %l4 ; mov	%g3, %l2 ; mov	%g2, %l1 ; mov	%g1, %l0
2360	FAST_TRAP_DONE
2361
2362#endif /* DEBUG_USER_TRAPTRACECTL */
2363
2364.getcc:
2365	CPU_ADDR(%g1, %g2)
2366	stx	%o0, [%g1 + CPU_TMP1]		! save %o0
2367	stx	%o1, [%g1 + CPU_TMP2]		! save %o1
2368	rdpr	%tstate, %g3			! get tstate
2369	srlx	%g3, PSR_TSTATE_CC_SHIFT, %o0	! shift ccr to V8 psr
2370	set	PSR_ICC, %g2
2371	and	%o0, %g2, %o0			! mask out the rest
2372	srl	%o0, PSR_ICC_SHIFT, %o0		! right justify
2373	rdpr	%pstate, %o1
2374	wrpr	%o1, PSTATE_AG, %pstate		! get into normal globals
2375	mov	%o0, %g1			! move ccr to normal %g1
2376	wrpr	%g0, %o1, %pstate		! back into alternate globals
2377	ldx	[%g1 + CPU_TMP1], %o0		! restore %o0
2378	ldx	[%g1 + CPU_TMP2], %o1		! restore %o1
2379	FAST_TRAP_DONE
2380
2381.setcc:
2382	CPU_ADDR(%g1, %g2)
2383	stx	%o0, [%g1 + CPU_TMP1]		! save %o0
2384	stx	%o1, [%g1 + CPU_TMP2]		! save %o1
2385	rdpr	%pstate, %o0
2386	wrpr	%o0, PSTATE_AG, %pstate		! get into normal globals
2387	mov	%g1, %o1
2388	wrpr	%g0, %o0, %pstate		! back to alternates
2389	sll	%o1, PSR_ICC_SHIFT, %g2
2390	set	PSR_ICC, %g3
2391	and	%g2, %g3, %g2			! mask out rest
2392	sllx	%g2, PSR_TSTATE_CC_SHIFT, %g2
2393	rdpr	%tstate, %g3			! get tstate
2394	srl	%g3, 0, %g3			! clear upper word
2395	or	%g3, %g2, %g3			! or in new bits
2396	wrpr	%g3, %tstate
2397	ldx	[%g1 + CPU_TMP1], %o0		! restore %o0
2398	ldx	[%g1 + CPU_TMP2], %o1		! restore %o1
2399	FAST_TRAP_DONE
2400
2401/*
2402 * getpsr(void)
2403 * Note that the xcc part of the ccr is not provided.
2404 * The V8 code shows why the V9 trap is not faster:
2405 * #define GETPSR_TRAP() \
2406 *      mov %psr, %i0; jmp %l2; rett %l2+4; nop;
2407 */
2408
2409	.type	.getpsr, #function
2410.getpsr:
2411	rdpr	%tstate, %g1			! get tstate
2412	srlx	%g1, PSR_TSTATE_CC_SHIFT, %o0	! shift ccr to V8 psr
2413	set	PSR_ICC, %g2
2414	and	%o0, %g2, %o0			! mask out the rest
2415
2416	rd	%fprs, %g1			! get fprs
2417	and	%g1, FPRS_FEF, %g2		! mask out dirty upper/lower
2418	sllx	%g2, PSR_FPRS_FEF_SHIFT, %g2	! shift fef to V8 psr.ef
2419	or	%o0, %g2, %o0			! or result into psr.ef
2420
2421	set	V9_PSR_IMPLVER, %g2		! SI assigned impl/ver: 0xef
2422	or	%o0, %g2, %o0			! or psr.impl/ver
2423	FAST_TRAP_DONE
2424	SET_SIZE(.getpsr)
2425
2426/*
2427 * setpsr(newpsr)
2428 * Note that there is no support for ccr.xcc in the V9 code.
2429 */
2430
2431	.type	.setpsr, #function
2432.setpsr:
2433	rdpr	%tstate, %g1			! get tstate
2434!	setx	TSTATE_V8_UBITS, %g2
2435	or 	%g0, CCR_ICC, %g3
2436	sllx	%g3, TSTATE_CCR_SHIFT, %g2
2437
2438	andn	%g1, %g2, %g1			! zero current user bits
2439	set	PSR_ICC, %g2
2440	and	%g2, %o0, %g2			! clear all but psr.icc bits
2441	sllx	%g2, PSR_TSTATE_CC_SHIFT, %g3	! shift to tstate.ccr.icc
2442	wrpr	%g1, %g3, %tstate		! write tstate
2443
2444	set	PSR_EF, %g2
2445	and	%g2, %o0, %g2			! clear all but fp enable bit
2446	srlx	%g2, PSR_FPRS_FEF_SHIFT, %g4	! shift ef to V9 fprs.fef
2447	wr	%g0, %g4, %fprs			! write fprs
2448
2449	CPU_ADDR(%g1, %g2)			! load CPU struct addr to %g1
2450	ldn	[%g1 + CPU_THREAD], %g2		! load thread pointer
2451	ldn	[%g2 + T_LWP], %g3		! load klwp pointer
2452	ldn	[%g3 + LWP_FPU], %g2		! get lwp_fpu pointer
2453	stuw	%g4, [%g2 + FPU_FPRS]		! write fef value to fpu_fprs
2454	srlx	%g4, 2, %g4			! shift fef value to bit 0
2455	stub	%g4, [%g2 + FPU_EN]		! write fef value to fpu_en
2456	FAST_TRAP_DONE
2457	SET_SIZE(.setpsr)
2458
2459/*
2460 * getlgrp
2461 * get home lgrpid on which the calling thread is currently executing.
2462 */
2463	.type	.getlgrp, #function
2464.getlgrp:
2465	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2466	ld	[%g1 + CPU_ID], %o0	! load cpu_id
2467	ldn	[%g1 + CPU_THREAD], %g2	! load thread pointer
2468	ldn	[%g2 + T_LPL], %g2	! load lpl pointer
2469	ld	[%g2 + LPL_LGRPID], %g1	! load lpl_lgrpid
2470	sra	%g1, 0, %o1
2471	FAST_TRAP_DONE
2472	SET_SIZE(.getlgrp)
2473
2474/*
2475 * Entry for old 4.x trap (trap 0).
2476 */
2477	ENTRY_NP(syscall_trap_4x)
2478	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2479	ldn	[%g1 + CPU_THREAD], %g2	! load thread pointer
2480	ldn	[%g2 + T_LWP], %g2	! load klwp pointer
2481	ld	[%g2 + PCB_TRAP0], %g2	! lwp->lwp_pcb.pcb_trap0addr
2482	brz,pn	%g2, 1f			! has it been set?
2483	st	%l0, [%g1 + CPU_TMP1]	! delay - save some locals
2484	st	%l1, [%g1 + CPU_TMP2]
2485	rdpr	%tnpc, %l1		! save old tnpc
2486	wrpr	%g0, %g2, %tnpc		! setup tnpc
2487
2488	rdpr	%pstate, %l0
2489	wrpr	%l0, PSTATE_AG, %pstate	! switch to normal globals
2490	mov	%l1, %g6		! pass tnpc to user code in %g6
2491	wrpr	%l0, %g0, %pstate	! switch back to alternate globals
2492
2493	! Note that %g1 still contains CPU struct addr
2494	ld	[%g1 + CPU_TMP2], %l1	! restore locals
2495	ld	[%g1 + CPU_TMP1], %l0
2496	FAST_TRAP_DONE_CHK_INTR
24971:
2498	mov	%g1, %l0
2499	st	%l1, [%g1 + CPU_TMP2]
2500	rdpr	%pstate, %l1
2501	wrpr	%l1, PSTATE_AG, %pstate
2502	!
2503	! check for old syscall mmap which is the only different one which
2504	! must be the same.  Others are handled in the compatibility library.
2505	!
2506	cmp	%g1, OSYS_mmap	! compare to old 4.x mmap
2507	movz	%icc, SYS_mmap, %g1
2508	wrpr	%g0, %l1, %pstate
2509	ld	[%l0 + CPU_TMP2], %l1	! restore locals
2510	ld	[%l0 + CPU_TMP1], %l0
2511	SYSCALL(syscall_trap32)
2512	SET_SIZE(syscall_trap_4x)
2513
2514/*
2515 * Handler for software trap 9.
2516 * Set trap0 emulation address for old 4.x system call trap.
2517 * XXX - this should be a system call.
2518 */
2519	ENTRY_NP(set_trap0_addr)
2520	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2521	ldn	[%g1 + CPU_THREAD], %g2	! load thread pointer
2522	ldn	[%g2 + T_LWP], %g2	! load klwp pointer
2523	st	%l0, [%g1 + CPU_TMP1]	! save some locals
2524	st	%l1, [%g1 + CPU_TMP2]
2525	rdpr	%pstate, %l0
2526	wrpr	%l0, PSTATE_AG, %pstate
2527	mov	%g1, %l1
2528	wrpr	%g0, %l0, %pstate
2529	andn	%l1, 3, %l1		! force alignment
2530	st	%l1, [%g2 + PCB_TRAP0]	! lwp->lwp_pcb.pcb_trap0addr
2531	ld	[%g1 + CPU_TMP1], %l0	! restore locals
2532	ld	[%g1 + CPU_TMP2], %l1
2533	FAST_TRAP_DONE
2534	SET_SIZE(set_trap0_addr)
2535
2536/*
2537 * mmu_trap_tl1
2538 * trap handler for unexpected mmu traps.
2539 * simply checks if the trap was a user lddf/stdf alignment trap, in which
2540 * case we go to fpu_trap or a user trap from the window handler, in which
2541 * case we go save the state on the pcb.  Otherwise, we go to ptl1_panic.
2542 */
2543	.type	mmu_trap_tl1, #function
2544mmu_trap_tl1:
2545#ifdef	TRAPTRACE
2546	TRACE_PTR(%g5, %g6)
2547	GET_TRACE_TICK(%g6)
2548	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi
2549	rdpr	%tl, %g6
2550	stha	%g6, [%g5 + TRAP_ENT_TL]%asi
2551	rdpr	%tt, %g6
2552	stha	%g6, [%g5 + TRAP_ENT_TT]%asi
2553	rdpr	%tstate, %g6
2554	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi
2555	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
2556	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
2557	rdpr	%tpc, %g6
2558	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi
2559	set	MMU_SFAR, %g6
2560	ldxa	[%g6]ASI_DMMU, %g6
2561	stxa	%g6, [%g5 + TRAP_ENT_F1]%asi
2562	CPU_PADDR(%g7, %g6);
2563	add	%g7, CPU_TL1_HDLR, %g7
2564	lda	[%g7]ASI_MEM, %g6
2565	stxa	%g6, [%g5 + TRAP_ENT_F2]%asi
2566	set	0xdeadbeef, %g6
2567	stna	%g6, [%g5 + TRAP_ENT_F3]%asi
2568	stna	%g6, [%g5 + TRAP_ENT_F4]%asi
2569	TRACE_NEXT(%g5, %g6, %g7)
2570#endif /* TRAPTRACE */
2571
2572	GET_CPU_IMPL(%g5)
2573	cmp	%g5, PANTHER_IMPL
2574	bne	mmu_trap_tl1_4
2575	  nop
2576	rdpr	%tt, %g5
2577	cmp	%g5, T_DATA_EXCEPTION
2578	bne	mmu_trap_tl1_4
2579	  nop
2580	wr	%g0, ASI_DMMU, %asi
2581	ldxa	[MMU_SFSR]%asi, %g5
2582	mov	1, %g6
2583	sllx	%g6, PN_SFSR_PARITY_SHIFT, %g6
2584	andcc	%g5, %g6, %g0
2585	bz	mmu_trap_tl1_4
2586
2587	/*
2588	 * We are running on a Panther and have hit a DTLB parity error.
2589	 */
2590	ldxa	[MMU_TAG_ACCESS]%asi, %g2
2591	mov	%g5, %g3
2592	ba,pt	%xcc, .mmu_exception_is_tlb_parity
2593	mov	T_DATA_EXCEPTION, %g1
2594
2595mmu_trap_tl1_4:
2596	CPU_PADDR(%g7, %g6);
2597	add     %g7, CPU_TL1_HDLR, %g7		! %g7 = &cpu_m.tl1_hdlr (PA)
2598	/*
2599	 * AM is cleared on trap, so addresses are 64 bit
2600	 */
2601	lda     [%g7]ASI_MEM, %g6
2602	brz,a,pt %g6, 1f
2603	  nop
2604	/*
2605	 * We are going to update cpu_m.tl1_hdlr using physical address.
2606	 * Flush the D$ line, so that stale data won't be accessed later.
2607	 */
2608	CPU_ADDR(%g6, %g5)
2609	add     %g6, CPU_TL1_HDLR, %g6		! %g6 = &cpu_m.tl1_hdlr (VA)
2610	GET_CPU_IMPL(%g5)
2611	cmp	%g5, CHEETAH_IMPL
2612	bl,pn	%icc, 3f
2613	sethi	%hi(dcache_line_mask), %g5
2614	stxa	%g0, [%g7]ASI_DC_INVAL
2615	membar	#Sync
2616	ba,pt	%xcc, 2f
2617	nop
26183:
2619	ld	[%g5 + %lo(dcache_line_mask)], %g5
2620	and	%g6, %g5, %g5
2621	stxa	%g0, [%g5]ASI_DC_TAG
2622	membar	#Sync
26232:
2624	sta     %g0, [%g7]ASI_MEM
2625	SWITCH_GLOBALS				! back to mmu globals
2626	ba,a,pt	%xcc, sfmmu_mmu_trap		! handle page faults
26271:
2628	rdpr	%tt, %g5
2629	rdpr	%tl, %g7
2630	sub	%g7, 1, %g6
2631	wrpr	%g6, %tl
2632	rdpr	%tt, %g6
2633	wrpr	%g7, %tl
2634	and	%g6, WTRAP_TTMASK, %g6
2635	cmp	%g6, WTRAP_TYPE
2636	bne,a,pn %xcc, ptl1_panic
2637	mov	PTL1_BAD_MMUTRAP, %g1
2638	rdpr	%tpc, %g7
2639	/* tpc should be in the trap table */
2640	set	trap_table, %g6
2641	cmp	%g7, %g6
2642	blt,a,pn %xcc, ptl1_panic
2643	  mov	PTL1_BAD_MMUTRAP, %g1
2644	set	etrap_table, %g6
2645	cmp	%g7, %g6
2646	bge,a,pn %xcc, ptl1_panic
2647	  mov	PTL1_BAD_MMUTRAP, %g1
2648	cmp	%g5, T_ALIGNMENT
2649	move	%icc, MMU_SFAR, %g6
2650	movne	%icc, MMU_TAG_ACCESS, %g6
2651	ldxa	[%g6]ASI_DMMU, %g6
2652	andn	%g7, WTRAP_ALIGN, %g7	/* 128 byte aligned */
2653	add	%g7, WTRAP_FAULTOFF, %g7
2654	wrpr	%g0, %g7, %tnpc
2655	done
2656	SET_SIZE(mmu_trap_tl1)
2657
2658/*
2659 * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers.  These
2660 * traps are valid only when kmdb is loaded.  When the debugger is active,
2661 * the code below is rewritten to transfer control to the appropriate
2662 * debugger entry points.
2663 */
2664	.global	kmdb_trap
2665	.align	8
2666kmdb_trap:
2667	ba,a	trap_table0
2668	jmp	%g1 + 0
2669	nop
2670
2671	.global	kmdb_trap_tl1
2672	.align	8
2673kmdb_trap_tl1:
2674	ba,a	trap_table0
2675	jmp	%g1 + 0
2676	nop
2677
2678/*
2679 * This entry is copied from OBP's trap table during boot.
2680 */
2681	.global	obp_bpt
2682	.align	8
2683obp_bpt:
2684	NOT
2685
2686/*
2687 * if kernel, set PCONTEXT to 0 for debuggers
2688 * if user, clear nucleus page sizes
2689 */
2690	.global kctx_obp_bpt
2691kctx_obp_bpt:
2692	set	obp_bpt, %g2
26931:
2694	mov	MMU_PCONTEXT, %g1
2695	ldxa	[%g1]ASI_DMMU, %g1
2696	srlx	%g1, CTXREG_NEXT_SHIFT, %g3
2697	brz,pt	%g3, 3f			! nucleus pgsz is 0, no problem
2698	  sllx	%g3, CTXREG_NEXT_SHIFT, %g3
2699	set	CTXREG_CTX_MASK, %g4	! check Pcontext
2700	btst	%g4, %g1
2701	bz,a,pt	%xcc, 2f
2702	  clr	%g3			! kernel:  PCONTEXT=0
2703	xor	%g3, %g1, %g3		! user:	clr N_pgsz0/1 bits
27042:
2705	set	DEMAP_ALL_TYPE, %g1
2706	stxa	%g0, [%g1]ASI_DTLB_DEMAP
2707	stxa	%g0, [%g1]ASI_ITLB_DEMAP
2708	mov	MMU_PCONTEXT, %g1
2709	stxa	%g3, [%g1]ASI_DMMU
2710        membar  #Sync
2711	sethi	%hi(FLUSH_ADDR), %g1
2712	flush	%g1			! flush required by immu
27133:
2714	jmp	%g2
2715	  nop
2716
2717
2718#ifdef	TRAPTRACE
2719/*
2720 * TRAPTRACE support.
2721 * labels here are branched to with "rd %pc, %g7" in the delay slot.
2722 * Return is done by "jmp %g7 + 4".
2723 */
2724
2725trace_gen:
2726	TRACE_PTR(%g3, %g6)
2727	GET_TRACE_TICK(%g6)
2728	stxa	%g6, [%g3 + TRAP_ENT_TICK]%asi
2729	rdpr	%tl, %g6
2730	stha	%g6, [%g3 + TRAP_ENT_TL]%asi
2731	rdpr	%tt, %g6
2732	stha	%g6, [%g3 + TRAP_ENT_TT]%asi
2733	rdpr	%tstate, %g6
2734	stxa	%g6, [%g3 + TRAP_ENT_TSTATE]%asi
2735	stna	%sp, [%g3 + TRAP_ENT_SP]%asi
2736	rdpr	%tpc, %g6
2737	stna	%g6, [%g3 + TRAP_ENT_TPC]%asi
2738	TRACE_NEXT(%g3, %g4, %g5)
2739	jmp	%g7 + 4
2740	nop
2741
2742trace_win:
2743	TRACE_WIN_INFO(0, %l0, %l1, %l2)
2744	! Keep the locals as clean as possible, caller cleans %l4
2745	clr	%l2
2746	clr	%l1
2747	jmp	%l4 + 4
2748	  clr	%l0
2749
2750/*
2751 * Trace a tsb hit
2752 * g1 = tsbe pointer (in/clobbered)
2753 * g2 = tag access register (in)
2754 * g3 - g4 = scratch (clobbered)
2755 * g5 = tsbe data (in)
2756 * g6 = scratch (clobbered)
2757 * g7 = pc we jumped here from (in)
2758 */
2759
2760	! Do not disturb %g5, it will be used after the trace
2761	ALTENTRY(trace_tsbhit)
2762	TRACE_TSBHIT(0)
2763	jmp	%g7 + 4
2764	nop
2765
2766/*
2767 * Trace a TSB miss
2768 *
2769 * g1 = tsb8k pointer (in)
2770 * g2 = tag access register (in)
2771 * g3 = tsb4m pointer (in)
2772 * g4 = tsbe tag (in/clobbered)
2773 * g5 - g6 = scratch (clobbered)
2774 * g7 = pc we jumped here from (in)
2775 */
2776	.global	trace_tsbmiss
2777trace_tsbmiss:
2778	membar	#Sync
2779	sethi	%hi(FLUSH_ADDR), %g6
2780	flush	%g6
2781	TRACE_PTR(%g5, %g6)
2782	GET_TRACE_TICK(%g6)
2783	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi
2784	stxa	%g2, [%g5 + TRAP_ENT_SP]%asi		! tag access
2785	stxa	%g4, [%g5 + TRAP_ENT_F1]%asi		! tsb tag
2786	rdpr	%tnpc, %g6
2787	stxa	%g6, [%g5 + TRAP_ENT_F2]%asi
2788	stna	%g1, [%g5 + TRAP_ENT_F3]%asi		! tsb8k pointer
2789	srlx	%g1, 32, %g6
2790	stna	%g6, [%g5 + TRAP_ENT_F4]%asi		! huh?
2791	rdpr	%tpc, %g6
2792	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi
2793	rdpr	%tl, %g6
2794	stha	%g6, [%g5 + TRAP_ENT_TL]%asi
2795	rdpr	%tt, %g6
2796	or	%g6, TT_MMU_MISS, %g4
2797	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
2798	cmp	%g6, FAST_IMMU_MISS_TT
2799	be,a	%icc, 1f
2800	  ldxa	[%g0]ASI_IMMU, %g6
2801	ldxa	[%g0]ASI_DMMU, %g6
28021:	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi	! tag target
2803	stxa	%g3, [%g5 + TRAP_ENT_TR]%asi		! tsb4m pointer
2804	TRACE_NEXT(%g5, %g4, %g6)
2805	jmp	%g7 + 4
2806	nop
2807
2808/*
2809 * g2 = tag access register (in)
2810 * g3 = ctx number (in)
2811 */
2812trace_dataprot:
2813	membar	#Sync
2814	sethi	%hi(FLUSH_ADDR), %g6
2815	flush	%g6
2816	TRACE_PTR(%g1, %g6)
2817	GET_TRACE_TICK(%g6)
2818	stxa	%g6, [%g1 + TRAP_ENT_TICK]%asi
2819	rdpr	%tpc, %g6
2820	stna	%g6, [%g1 + TRAP_ENT_TPC]%asi
2821	rdpr	%tstate, %g6
2822	stxa	%g6, [%g1 + TRAP_ENT_TSTATE]%asi
2823	stxa	%g2, [%g1 + TRAP_ENT_SP]%asi		! tag access reg
2824	stxa	%g0, [%g1 + TRAP_ENT_TR]%asi
2825	stxa	%g0, [%g1 + TRAP_ENT_F1]%asi
2826	stxa	%g0, [%g1 + TRAP_ENT_F2]%asi
2827	stxa	%g0, [%g1 + TRAP_ENT_F3]%asi
2828	stxa	%g0, [%g1 + TRAP_ENT_F4]%asi
2829	rdpr	%tl, %g6
2830	stha	%g6, [%g1 + TRAP_ENT_TL]%asi
2831	rdpr	%tt, %g6
2832	stha	%g6, [%g1 + TRAP_ENT_TT]%asi
2833	TRACE_NEXT(%g1, %g4, %g5)
2834	jmp	%g7 + 4
2835	nop
2836
2837#endif /* TRAPTRACE */
2838
2839/*
2840 * expects offset into tsbmiss area in %g1 and return pc in %g7
2841 */
2842stat_mmu:
2843	CPU_INDEX(%g5, %g6)
2844	sethi	%hi(tsbmiss_area), %g6
2845	sllx	%g5, TSBMISS_SHIFT, %g5
2846	or	%g6, %lo(tsbmiss_area), %g6
2847	add	%g6, %g5, %g6		/* g6 = tsbmiss area */
2848	ld	[%g6 + %g1], %g5
2849	add	%g5, 1, %g5
2850	jmp	%g7 + 4
2851	st	%g5, [%g6 + %g1]
2852
2853
2854/*
2855 * fast_trap_done, fast_trap_done_chk_intr:
2856 *
2857 * Due to the design of UltraSPARC pipeline, pending interrupts are not
2858 * taken immediately after a RETRY or DONE instruction which causes IE to
2859 * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed
2860 * to execute first before taking any interrupts. If that instruction
2861 * results in other traps, and if the corresponding trap handler runs
2862 * entirely at TL=1 with interrupts disabled, then pending interrupts
2863 * won't be taken until after yet another instruction following the %tpc
2864 * or %tnpc.
2865 *
2866 * A malicious user program can use this feature to block out interrupts
2867 * for extended durations, which can result in send_mondo_timeout kernel
2868 * panic.
2869 *
2870 * This problem is addressed by servicing any pending interrupts via
2871 * sys_trap before returning back to the user mode from a fast trap
2872 * handler. The "done" instruction within a fast trap handler, which
2873 * runs entirely at TL=1 with interrupts disabled, is replaced with the
2874 * FAST_TRAP_DONE macro, which branches control to this fast_trap_done
2875 * entry point.
2876 *
2877 * We check for any pending interrupts here and force a sys_trap to
2878 * service those interrupts, if any. To minimize overhead, pending
2879 * interrupts are checked if the %tpc happens to be at 16K boundary,
2880 * which allows a malicious program to execute at most 4K consecutive
2881 * instructions before we service any pending interrupts. If a worst
2882 * case fast trap handler takes about 2 usec, then interrupts will be
2883 * blocked for at most 8 msec, less than a clock tick.
2884 *
2885 * For the cases where we don't know if the %tpc will cross a 16K
2886 * boundary, we can't use the above optimization and always process
2887 * any pending interrupts via fast_frap_done_chk_intr entry point.
2888 *
2889 * Entry Conditions:
2890 * 	%pstate		am:0 priv:1 ie:0
2891 * 			globals are AG (not normal globals)
2892 */
2893
2894	.global	fast_trap_done, fast_trap_done_chk_intr
2895fast_trap_done:
2896	rdpr	%tpc, %g5
2897	sethi	%hi(0xffffc000), %g6	! 1's complement of 0x3fff
2898	andncc	%g5, %g6, %g0		! check lower 14 bits of %tpc
2899	bz,a,pn	%icc, 1f		! branch if zero (lower 32 bits only)
2900	  ldxa	[%g0]ASI_INTR_RECEIVE_STATUS, %g5
2901	done
2902
2903fast_trap_done_chk_intr:
2904	ldxa	[%g0]ASI_INTR_RECEIVE_STATUS, %g5
2905
29061:	rd	SOFTINT, %g6
2907	and	%g5, IRSR_BUSY, %g5
2908	orcc	%g5, %g6, %g0
2909	bnz,pn	%xcc, 2f		! branch if any pending intr
2910	nop
2911	done
2912
29132:
2914	/*
2915	 * We get here if there are any pending interrupts.
2916	 * Adjust %tpc/%tnpc as we'll be resuming via "retry"
2917	 * instruction.
2918	 */
2919	rdpr	%tnpc, %g5
2920	wrpr	%g0, %g5, %tpc
2921	add	%g5, 4, %g5
2922	wrpr	%g0, %g5, %tnpc
2923
2924	/*
2925	 * Force a dummy sys_trap call so that interrupts can be serviced.
2926	 */
2927	set	fast_trap_dummy_call, %g1
2928	ba,pt	%xcc, sys_trap
2929	  mov	-1, %g4
2930
2931fast_trap_dummy_call:
2932	retl
2933	nop
2934
2935#endif	/* lint */
2936
2937