xref: /titanic_50/usr/src/uts/sun4u/ml/trap_table.s (revision b56bf881a9655cb27b53cba1468312f7c6dfb0a2)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#if !defined(lint)
27#include "assym.h"
28#endif /* !lint */
29#include <sys/asm_linkage.h>
30#include <sys/privregs.h>
31#include <sys/sun4asi.h>
32#include <sys/spitregs.h>
33#include <sys/cheetahregs.h>
34#include <sys/machtrap.h>
35#include <sys/machthread.h>
36#include <sys/machbrand.h>
37#include <sys/pcb.h>
38#include <sys/pte.h>
39#include <sys/mmu.h>
40#include <sys/machpcb.h>
41#include <sys/async.h>
42#include <sys/intreg.h>
43#include <sys/scb.h>
44#include <sys/psr_compat.h>
45#include <sys/syscall.h>
46#include <sys/machparam.h>
47#include <sys/traptrace.h>
48#include <vm/hat_sfmmu.h>
49#include <sys/archsystm.h>
50#include <sys/utrap.h>
51#include <sys/clock.h>
52#include <sys/intr.h>
53#include <sys/fpu/fpu_simulator.h>
54#include <vm/seg_spt.h>
55
56/*
57 * WARNING: If you add a fast trap handler which can be invoked by a
58 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
59 * instead of "done" instruction to return back to the user mode. See
60 * comments for the "fast_trap_done" entry point for more information.
61 *
62 * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
63 * cases where you always want to process any pending interrupts before
64 * returning back to the user mode.
65 */
66#define	FAST_TRAP_DONE		\
67	ba,a	fast_trap_done
68
69#define	FAST_TRAP_DONE_CHK_INTR	\
70	ba,a	fast_trap_done_chk_intr
71
72/*
73 * SPARC V9 Trap Table
74 *
75 * Most of the trap handlers are made from common building
76 * blocks, and some are instantiated multiple times within
77 * the trap table. So, I build a bunch of macros, then
78 * populate the table using only the macros.
79 *
80 * Many macros branch to sys_trap.  Its calling convention is:
81 *	%g1		kernel trap handler
82 *	%g2, %g3	args for above
83 *	%g4		desire %pil
84 */
85
86#ifdef	TRAPTRACE
87
88/*
89 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
90 */
91#define	TT_TRACE(label)		\
92	ba	label		;\
93	rd	%pc, %g7
94#define	TT_TRACE_INS	2
95
96#define	TT_TRACE_L(label)	\
97	ba	label		;\
98	rd	%pc, %l4	;\
99	clr	%l4
100#define	TT_TRACE_L_INS	3
101
102#else
103
104#define	TT_TRACE(label)
105#define	TT_TRACE_INS	0
106
107#define	TT_TRACE_L(label)
108#define	TT_TRACE_L_INS	0
109
110#endif
111
112/*
113 * This first set are funneled to trap() with %tt as the type.
114 * Trap will then either panic or send the user a signal.
115 */
116/*
117 * NOT is used for traps that just shouldn't happen.
118 * It comes in both single and quadruple flavors.
119 */
120#if !defined(lint)
121	.global	trap
122#endif /* !lint */
123#define	NOT			\
124	TT_TRACE(trace_gen)	;\
125	set	trap, %g1	;\
126	rdpr	%tt, %g3	;\
127	ba,pt	%xcc, sys_trap	;\
128	sub	%g0, 1, %g4	;\
129	.align	32
130#define	NOT4	NOT; NOT; NOT; NOT
131/*
132 * RED is for traps that use the red mode handler.
133 * We should never see these either.
134 */
135#define	RED	NOT
136/*
137 * BAD is used for trap vectors we don't have a kernel
138 * handler for.
139 * It also comes in single and quadruple versions.
140 */
141#define	BAD	NOT
142#define	BAD4	NOT4
143
144#define	DONE			\
145	done;			\
146	.align	32
147
148/*
149 * TRAP vectors to the trap() function.
150 * It's main use is for user errors.
151 */
152#if !defined(lint)
153	.global	trap
154#endif /* !lint */
155#define	TRAP(arg)		\
156	TT_TRACE(trace_gen)	;\
157	set	trap, %g1	;\
158	mov	arg, %g3	;\
159	ba,pt	%xcc, sys_trap	;\
160	sub	%g0, 1, %g4	;\
161	.align	32
162
163/*
164 * SYSCALL is used for unsupported syscall interfaces (with 'which'
165 * set to 'nosys') and legacy support of old SunOS 4.x syscalls (with
166 * 'which' set to 'syscall_trap32').
167 *
168 * The SYSCALL_TRAP* macros are used for syscall entry points.
169 * SYSCALL_TRAP is used to support LP64 syscalls and SYSCALL_TRAP32
170 * is used to support ILP32.  Each macro can only be used once
171 * since they each define a symbol.  The symbols are used as hot patch
172 * points by the brand infrastructure to dynamically enable and disable
173 * brand syscall interposition.  See the comments around BRAND_CALLBACK
174 * and brand_plat_interposition_enable() for more information.
175 */
176#define	SYSCALL_NOTT(which)		\
177	set	(which), %g1		;\
178	ba,pt	%xcc, sys_trap		;\
179	sub	%g0, 1, %g4		;\
180	.align	32
181
182#define	SYSCALL(which)			\
183	TT_TRACE(trace_gen)		;\
184	SYSCALL_NOTT(which)
185
186#define	SYSCALL_TRAP32				\
187	TT_TRACE(trace_gen)			;\
188	ALTENTRY(syscall_trap32_patch_point)	\
189	SYSCALL_NOTT(syscall_trap32)
190
191#define	SYSCALL_TRAP				\
192	TT_TRACE(trace_gen)			;\
193	ALTENTRY(syscall_trap_patch_point)	\
194	SYSCALL_NOTT(syscall_trap)
195
196#define	FLUSHW(h_name)			\
197	.global h_name			;\
198h_name:					;\
199	set	trap, %g1		;\
200	mov	T_FLUSHW, %g3		;\
201	sub	%g0, 1, %g4		;\
202	save				;\
203	flushw				;\
204	restore				;\
205	FAST_TRAP_DONE			;\
206	.align	32
207
208/*
209 * GOTO just jumps to a label.
210 * It's used for things that can be fixed without going thru sys_trap.
211 */
212#define	GOTO(label)		\
213	.global	label		;\
214	ba,a	label		;\
215	.empty			;\
216	.align	32
217
218/*
219 * GOTO_TT just jumps to a label.
220 * correctable ECC error traps at  level 0 and 1 will use this macro.
221 * It's used for things that can be fixed without going thru sys_trap.
222 */
223#define	GOTO_TT(label, ttlabel)		\
224	.global	label		;\
225	TT_TRACE(ttlabel)	;\
226	ba,a	label		;\
227	.empty			;\
228	.align	32
229
230/*
231 * Privileged traps
232 * Takes breakpoint if privileged, calls trap() if not.
233 */
234#define	PRIV(label)			\
235	rdpr	%tstate, %g1		;\
236	btst	TSTATE_PRIV, %g1	;\
237	bnz	label			;\
238	rdpr	%tt, %g3		;\
239	set	trap, %g1		;\
240	ba,pt	%xcc, sys_trap		;\
241	sub	%g0, 1, %g4		;\
242	.align	32
243
244
245/*
246 * DTrace traps.
247 */
248#define	DTRACE_PID			\
249	.global dtrace_pid_probe				;\
250	set	dtrace_pid_probe, %g1				;\
251	ba,pt	%xcc, user_trap					;\
252	sub	%g0, 1, %g4					;\
253	.align	32
254
255#define	DTRACE_RETURN			\
256	.global dtrace_return_probe				;\
257	set	dtrace_return_probe, %g1			;\
258	ba,pt	%xcc, user_trap					;\
259	sub	%g0, 1, %g4					;\
260	.align	32
261
262/*
263 * REGISTER WINDOW MANAGEMENT MACROS
264 */
265
266/*
267 * various convenient units of padding
268 */
269#define	SKIP(n)	.skip 4*(n)
270
271/*
272 * CLEAN_WINDOW is the simple handler for cleaning a register window.
273 */
274#define	CLEAN_WINDOW						\
275	TT_TRACE_L(trace_win)					;\
276	rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin	;\
277	clr %l0; clr %l1; clr %l2; clr %l3			;\
278	clr %l4; clr %l5; clr %l6; clr %l7			;\
279	clr %o0; clr %o1; clr %o2; clr %o3			;\
280	clr %o4; clr %o5; clr %o6; clr %o7			;\
281	retry; .align 128
282
283#if !defined(lint)
284
285/*
286 * If we get an unresolved tlb miss while in a window handler, the fault
287 * handler will resume execution at the last instruction of the window
288 * hander, instead of delivering the fault to the kernel.  Spill handlers
289 * use this to spill windows into the wbuf.
290 *
291 * The mixed handler works by checking %sp, and branching to the correct
292 * handler.  This is done by branching back to label 1: for 32b frames,
293 * or label 2: for 64b frames; which implies the handler order is: 32b,
294 * 64b, mixed.  The 1: and 2: labels are offset into the routines to
295 * allow the branchs' delay slots to contain useful instructions.
296 */
297
298/*
299 * SPILL_32bit spills a 32-bit-wide kernel register window.  It
300 * assumes that the kernel context and the nucleus context are the
301 * same.  The stack pointer is required to be eight-byte aligned even
302 * though this code only needs it to be four-byte aligned.
303 */
304#define	SPILL_32bit(tail)					\
305	srl	%sp, 0, %sp					;\
3061:	st	%l0, [%sp + 0]					;\
307	st	%l1, [%sp + 4]					;\
308	st	%l2, [%sp + 8]					;\
309	st	%l3, [%sp + 12]					;\
310	st	%l4, [%sp + 16]					;\
311	st	%l5, [%sp + 20]					;\
312	st	%l6, [%sp + 24]					;\
313	st	%l7, [%sp + 28]					;\
314	st	%i0, [%sp + 32]					;\
315	st	%i1, [%sp + 36]					;\
316	st	%i2, [%sp + 40]					;\
317	st	%i3, [%sp + 44]					;\
318	st	%i4, [%sp + 48]					;\
319	st	%i5, [%sp + 52]					;\
320	st	%i6, [%sp + 56]					;\
321	st	%i7, [%sp + 60]					;\
322	TT_TRACE_L(trace_win)					;\
323	saved							;\
324	retry							;\
325	SKIP(31-19-TT_TRACE_L_INS)				;\
326	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
327	.empty
328
329/*
330 * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
331 * wide address space via the designated asi.  It is used to spill
332 * non-kernel windows.  The stack pointer is required to be eight-byte
333 * aligned even though this code only needs it to be four-byte
334 * aligned.
335 */
336#define	SPILL_32bit_asi(asi_num, tail)				\
337	srl	%sp, 0, %sp					;\
3381:	sta	%l0, [%sp + %g0]asi_num				;\
339	mov	4, %g1						;\
340	sta	%l1, [%sp + %g1]asi_num				;\
341	mov	8, %g2						;\
342	sta	%l2, [%sp + %g2]asi_num				;\
343	mov	12, %g3						;\
344	sta	%l3, [%sp + %g3]asi_num				;\
345	add	%sp, 16, %g4					;\
346	sta	%l4, [%g4 + %g0]asi_num				;\
347	sta	%l5, [%g4 + %g1]asi_num				;\
348	sta	%l6, [%g4 + %g2]asi_num				;\
349	sta	%l7, [%g4 + %g3]asi_num				;\
350	add	%g4, 16, %g4					;\
351	sta	%i0, [%g4 + %g0]asi_num				;\
352	sta	%i1, [%g4 + %g1]asi_num				;\
353	sta	%i2, [%g4 + %g2]asi_num				;\
354	sta	%i3, [%g4 + %g3]asi_num				;\
355	add	%g4, 16, %g4					;\
356	sta	%i4, [%g4 + %g0]asi_num				;\
357	sta	%i5, [%g4 + %g1]asi_num				;\
358	sta	%i6, [%g4 + %g2]asi_num				;\
359	sta	%i7, [%g4 + %g3]asi_num				;\
360	TT_TRACE_L(trace_win)					;\
361	saved							;\
362	retry							;\
363	SKIP(31-25-TT_TRACE_L_INS)				;\
364	ba,a,pt %xcc, fault_32bit_/**/tail			;\
365	.empty
366
367/*
368 * SPILL_32bit_tt1 spills a 32-bit-wide register window into a 32-bit
369 * wide address space via the designated asi.  It is used to spill
370 * windows at tl>1 where performance isn't the primary concern and
371 * where we don't want to use unnecessary registers.  The stack
372 * pointer is required to be eight-byte aligned even though this code
373 * only needs it to be four-byte aligned.
374 */
375#define	SPILL_32bit_tt1(asi_num, tail)				\
376	mov	asi_num, %asi					;\
3771:	srl	%sp, 0, %sp					;\
378	sta	%l0, [%sp + 0]%asi				;\
379	sta	%l1, [%sp + 4]%asi				;\
380	sta	%l2, [%sp + 8]%asi				;\
381	sta	%l3, [%sp + 12]%asi				;\
382	sta	%l4, [%sp + 16]%asi				;\
383	sta	%l5, [%sp + 20]%asi				;\
384	sta	%l6, [%sp + 24]%asi				;\
385	sta	%l7, [%sp + 28]%asi				;\
386	sta	%i0, [%sp + 32]%asi				;\
387	sta	%i1, [%sp + 36]%asi				;\
388	sta	%i2, [%sp + 40]%asi				;\
389	sta	%i3, [%sp + 44]%asi				;\
390	sta	%i4, [%sp + 48]%asi				;\
391	sta	%i5, [%sp + 52]%asi				;\
392	sta	%i6, [%sp + 56]%asi				;\
393	sta	%i7, [%sp + 60]%asi				;\
394	TT_TRACE_L(trace_win)					;\
395	saved							;\
396	retry							;\
397	SKIP(31-20-TT_TRACE_L_INS)				;\
398	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
399	.empty
400
401
402/*
403 * FILL_32bit fills a 32-bit-wide kernel register window.  It assumes
404 * that the kernel context and the nucleus context are the same.  The
405 * stack pointer is required to be eight-byte aligned even though this
406 * code only needs it to be four-byte aligned.
407 */
408#define	FILL_32bit(tail)					\
409	srl	%sp, 0, %sp					;\
4101:	TT_TRACE_L(trace_win)					;\
411	ld	[%sp + 0], %l0					;\
412	ld	[%sp + 4], %l1					;\
413	ld	[%sp + 8], %l2					;\
414	ld	[%sp + 12], %l3					;\
415	ld	[%sp + 16], %l4					;\
416	ld	[%sp + 20], %l5					;\
417	ld	[%sp + 24], %l6					;\
418	ld	[%sp + 28], %l7					;\
419	ld	[%sp + 32], %i0					;\
420	ld	[%sp + 36], %i1					;\
421	ld	[%sp + 40], %i2					;\
422	ld	[%sp + 44], %i3					;\
423	ld	[%sp + 48], %i4					;\
424	ld	[%sp + 52], %i5					;\
425	ld	[%sp + 56], %i6					;\
426	ld	[%sp + 60], %i7					;\
427	restored						;\
428	retry							;\
429	SKIP(31-19-TT_TRACE_L_INS)				;\
430	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
431	.empty
432
433/*
434 * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
435 * wide address space via the designated asi.  It is used to fill
436 * non-kernel windows.  The stack pointer is required to be eight-byte
437 * aligned even though this code only needs it to be four-byte
438 * aligned.
439 */
440#define	FILL_32bit_asi(asi_num, tail)				\
441	srl	%sp, 0, %sp					;\
4421:	TT_TRACE_L(trace_win)					;\
443	mov	4, %g1						;\
444	lda	[%sp + %g0]asi_num, %l0				;\
445	mov	8, %g2						;\
446	lda	[%sp + %g1]asi_num, %l1				;\
447	mov	12, %g3						;\
448	lda	[%sp + %g2]asi_num, %l2				;\
449	lda	[%sp + %g3]asi_num, %l3				;\
450	add	%sp, 16, %g4					;\
451	lda	[%g4 + %g0]asi_num, %l4				;\
452	lda	[%g4 + %g1]asi_num, %l5				;\
453	lda	[%g4 + %g2]asi_num, %l6				;\
454	lda	[%g4 + %g3]asi_num, %l7				;\
455	add	%g4, 16, %g4					;\
456	lda	[%g4 + %g0]asi_num, %i0				;\
457	lda	[%g4 + %g1]asi_num, %i1				;\
458	lda	[%g4 + %g2]asi_num, %i2				;\
459	lda	[%g4 + %g3]asi_num, %i3				;\
460	add	%g4, 16, %g4					;\
461	lda	[%g4 + %g0]asi_num, %i4				;\
462	lda	[%g4 + %g1]asi_num, %i5				;\
463	lda	[%g4 + %g2]asi_num, %i6				;\
464	lda	[%g4 + %g3]asi_num, %i7				;\
465	restored						;\
466	retry							;\
467	SKIP(31-25-TT_TRACE_L_INS)				;\
468	ba,a,pt %xcc, fault_32bit_/**/tail			;\
469	.empty
470
471/*
472 * FILL_32bit_tt1 fills a 32-bit-wide register window from a 32-bit
473 * wide address space via the designated asi.  It is used to fill
474 * windows at tl>1 where performance isn't the primary concern and
475 * where we don't want to use unnecessary registers.  The stack
476 * pointer is required to be eight-byte aligned even though this code
477 * only needs it to be four-byte aligned.
478 */
479#define	FILL_32bit_tt1(asi_num, tail)				\
480	mov	asi_num, %asi					;\
4811:	srl	%sp, 0, %sp					;\
482	TT_TRACE_L(trace_win)					;\
483	lda	[%sp + 0]%asi, %l0				;\
484	lda	[%sp + 4]%asi, %l1				;\
485	lda	[%sp + 8]%asi, %l2				;\
486	lda	[%sp + 12]%asi, %l3				;\
487	lda	[%sp + 16]%asi, %l4				;\
488	lda	[%sp + 20]%asi, %l5				;\
489	lda	[%sp + 24]%asi, %l6				;\
490	lda	[%sp + 28]%asi, %l7				;\
491	lda	[%sp + 32]%asi, %i0				;\
492	lda	[%sp + 36]%asi, %i1				;\
493	lda	[%sp + 40]%asi, %i2				;\
494	lda	[%sp + 44]%asi, %i3				;\
495	lda	[%sp + 48]%asi, %i4				;\
496	lda	[%sp + 52]%asi, %i5				;\
497	lda	[%sp + 56]%asi, %i6				;\
498	lda	[%sp + 60]%asi, %i7				;\
499	restored						;\
500	retry							;\
501	SKIP(31-20-TT_TRACE_L_INS)				;\
502	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
503	.empty
504
505
506/*
507 * SPILL_64bit spills a 64-bit-wide kernel register window.  It
508 * assumes that the kernel context and the nucleus context are the
509 * same.  The stack pointer is required to be eight-byte aligned.
510 */
511#define	SPILL_64bit(tail)					\
5122:	stx	%l0, [%sp + V9BIAS64 + 0]			;\
513	stx	%l1, [%sp + V9BIAS64 + 8]			;\
514	stx	%l2, [%sp + V9BIAS64 + 16]			;\
515	stx	%l3, [%sp + V9BIAS64 + 24]			;\
516	stx	%l4, [%sp + V9BIAS64 + 32]			;\
517	stx	%l5, [%sp + V9BIAS64 + 40]			;\
518	stx	%l6, [%sp + V9BIAS64 + 48]			;\
519	stx	%l7, [%sp + V9BIAS64 + 56]			;\
520	stx	%i0, [%sp + V9BIAS64 + 64]			;\
521	stx	%i1, [%sp + V9BIAS64 + 72]			;\
522	stx	%i2, [%sp + V9BIAS64 + 80]			;\
523	stx	%i3, [%sp + V9BIAS64 + 88]			;\
524	stx	%i4, [%sp + V9BIAS64 + 96]			;\
525	stx	%i5, [%sp + V9BIAS64 + 104]			;\
526	stx	%i6, [%sp + V9BIAS64 + 112]			;\
527	stx	%i7, [%sp + V9BIAS64 + 120]			;\
528	TT_TRACE_L(trace_win)					;\
529	saved							;\
530	retry							;\
531	SKIP(31-18-TT_TRACE_L_INS)				;\
532	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
533	.empty
534
535/*
536 * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
537 * wide address space via the designated asi.  It is used to spill
538 * non-kernel windows.  The stack pointer is required to be eight-byte
539 * aligned.
540 */
541#define	SPILL_64bit_asi(asi_num, tail)				\
542	mov	0 + V9BIAS64, %g1				;\
5432:	stxa	%l0, [%sp + %g1]asi_num				;\
544	mov	8 + V9BIAS64, %g2				;\
545	stxa	%l1, [%sp + %g2]asi_num				;\
546	mov	16 + V9BIAS64, %g3				;\
547	stxa	%l2, [%sp + %g3]asi_num				;\
548	mov	24 + V9BIAS64, %g4				;\
549	stxa	%l3, [%sp + %g4]asi_num				;\
550	add	%sp, 32, %g5					;\
551	stxa	%l4, [%g5 + %g1]asi_num				;\
552	stxa	%l5, [%g5 + %g2]asi_num				;\
553	stxa	%l6, [%g5 + %g3]asi_num				;\
554	stxa	%l7, [%g5 + %g4]asi_num				;\
555	add	%g5, 32, %g5					;\
556	stxa	%i0, [%g5 + %g1]asi_num				;\
557	stxa	%i1, [%g5 + %g2]asi_num				;\
558	stxa	%i2, [%g5 + %g3]asi_num				;\
559	stxa	%i3, [%g5 + %g4]asi_num				;\
560	add	%g5, 32, %g5					;\
561	stxa	%i4, [%g5 + %g1]asi_num				;\
562	stxa	%i5, [%g5 + %g2]asi_num				;\
563	stxa	%i6, [%g5 + %g3]asi_num				;\
564	stxa	%i7, [%g5 + %g4]asi_num				;\
565	TT_TRACE_L(trace_win)					;\
566	saved							;\
567	retry							;\
568	SKIP(31-25-TT_TRACE_L_INS)				;\
569	ba,a,pt %xcc, fault_64bit_/**/tail			;\
570	.empty
571
572/*
573 * SPILL_64bit_tt1 spills a 64-bit-wide register window into a 64-bit
574 * wide address space via the designated asi.  It is used to spill
575 * windows at tl>1 where performance isn't the primary concern and
576 * where we don't want to use unnecessary registers.  The stack
577 * pointer is required to be eight-byte aligned.
578 */
579#define	SPILL_64bit_tt1(asi_num, tail)				\
580	mov	asi_num, %asi					;\
5812:	stxa	%l0, [%sp + V9BIAS64 + 0]%asi			;\
582	stxa	%l1, [%sp + V9BIAS64 + 8]%asi			;\
583	stxa	%l2, [%sp + V9BIAS64 + 16]%asi			;\
584	stxa	%l3, [%sp + V9BIAS64 + 24]%asi			;\
585	stxa	%l4, [%sp + V9BIAS64 + 32]%asi			;\
586	stxa	%l5, [%sp + V9BIAS64 + 40]%asi			;\
587	stxa	%l6, [%sp + V9BIAS64 + 48]%asi			;\
588	stxa	%l7, [%sp + V9BIAS64 + 56]%asi			;\
589	stxa	%i0, [%sp + V9BIAS64 + 64]%asi			;\
590	stxa	%i1, [%sp + V9BIAS64 + 72]%asi			;\
591	stxa	%i2, [%sp + V9BIAS64 + 80]%asi			;\
592	stxa	%i3, [%sp + V9BIAS64 + 88]%asi			;\
593	stxa	%i4, [%sp + V9BIAS64 + 96]%asi			;\
594	stxa	%i5, [%sp + V9BIAS64 + 104]%asi			;\
595	stxa	%i6, [%sp + V9BIAS64 + 112]%asi			;\
596	stxa	%i7, [%sp + V9BIAS64 + 120]%asi			;\
597	TT_TRACE_L(trace_win)					;\
598	saved							;\
599	retry							;\
600	SKIP(31-19-TT_TRACE_L_INS)				;\
601	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
602	.empty
603
604
605/*
606 * FILL_64bit fills a 64-bit-wide kernel register window.  It assumes
607 * that the kernel context and the nucleus context are the same.  The
608 * stack pointer is required to be eight-byte aligned.
609 */
610#define	FILL_64bit(tail)					\
6112:	TT_TRACE_L(trace_win)					;\
612	ldx	[%sp + V9BIAS64 + 0], %l0			;\
613	ldx	[%sp + V9BIAS64 + 8], %l1			;\
614	ldx	[%sp + V9BIAS64 + 16], %l2			;\
615	ldx	[%sp + V9BIAS64 + 24], %l3			;\
616	ldx	[%sp + V9BIAS64 + 32], %l4			;\
617	ldx	[%sp + V9BIAS64 + 40], %l5			;\
618	ldx	[%sp + V9BIAS64 + 48], %l6			;\
619	ldx	[%sp + V9BIAS64 + 56], %l7			;\
620	ldx	[%sp + V9BIAS64 + 64], %i0			;\
621	ldx	[%sp + V9BIAS64 + 72], %i1			;\
622	ldx	[%sp + V9BIAS64 + 80], %i2			;\
623	ldx	[%sp + V9BIAS64 + 88], %i3			;\
624	ldx	[%sp + V9BIAS64 + 96], %i4			;\
625	ldx	[%sp + V9BIAS64 + 104], %i5			;\
626	ldx	[%sp + V9BIAS64 + 112], %i6			;\
627	ldx	[%sp + V9BIAS64 + 120], %i7			;\
628	restored						;\
629	retry							;\
630	SKIP(31-18-TT_TRACE_L_INS)				;\
631	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
632	.empty
633
634/*
635 * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
636 * wide address space via the designated asi.  It is used to fill
637 * non-kernel windows.  The stack pointer is required to be eight-byte
638 * aligned.
639 */
640#define	FILL_64bit_asi(asi_num, tail)				\
641	mov	V9BIAS64 + 0, %g1				;\
6422:	TT_TRACE_L(trace_win)					;\
643	ldxa	[%sp + %g1]asi_num, %l0				;\
644	mov	V9BIAS64 + 8, %g2				;\
645	ldxa	[%sp + %g2]asi_num, %l1				;\
646	mov	V9BIAS64 + 16, %g3				;\
647	ldxa	[%sp + %g3]asi_num, %l2				;\
648	mov	V9BIAS64 + 24, %g4				;\
649	ldxa	[%sp + %g4]asi_num, %l3				;\
650	add	%sp, 32, %g5					;\
651	ldxa	[%g5 + %g1]asi_num, %l4				;\
652	ldxa	[%g5 + %g2]asi_num, %l5				;\
653	ldxa	[%g5 + %g3]asi_num, %l6				;\
654	ldxa	[%g5 + %g4]asi_num, %l7				;\
655	add	%g5, 32, %g5					;\
656	ldxa	[%g5 + %g1]asi_num, %i0				;\
657	ldxa	[%g5 + %g2]asi_num, %i1				;\
658	ldxa	[%g5 + %g3]asi_num, %i2				;\
659	ldxa	[%g5 + %g4]asi_num, %i3				;\
660	add	%g5, 32, %g5					;\
661	ldxa	[%g5 + %g1]asi_num, %i4				;\
662	ldxa	[%g5 + %g2]asi_num, %i5				;\
663	ldxa	[%g5 + %g3]asi_num, %i6				;\
664	ldxa	[%g5 + %g4]asi_num, %i7				;\
665	restored						;\
666	retry							;\
667	SKIP(31-25-TT_TRACE_L_INS)				;\
668	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
669	.empty
670
671/*
672 * FILL_64bit_tt1 fills a 64-bit-wide register window from a 64-bit
673 * wide address space via the designated asi.  It is used to fill
674 * windows at tl>1 where performance isn't the primary concern and
675 * where we don't want to use unnecessary registers.  The stack
676 * pointer is required to be eight-byte aligned.
677 */
678#define	FILL_64bit_tt1(asi_num, tail)				\
679	mov	asi_num, %asi					;\
680	TT_TRACE_L(trace_win)					;\
681	ldxa	[%sp + V9BIAS64 + 0]%asi, %l0			;\
682	ldxa	[%sp + V9BIAS64 + 8]%asi, %l1			;\
683	ldxa	[%sp + V9BIAS64 + 16]%asi, %l2			;\
684	ldxa	[%sp + V9BIAS64 + 24]%asi, %l3			;\
685	ldxa	[%sp + V9BIAS64 + 32]%asi, %l4			;\
686	ldxa	[%sp + V9BIAS64 + 40]%asi, %l5			;\
687	ldxa	[%sp + V9BIAS64 + 48]%asi, %l6			;\
688	ldxa	[%sp + V9BIAS64 + 56]%asi, %l7			;\
689	ldxa	[%sp + V9BIAS64 + 64]%asi, %i0			;\
690	ldxa	[%sp + V9BIAS64 + 72]%asi, %i1			;\
691	ldxa	[%sp + V9BIAS64 + 80]%asi, %i2			;\
692	ldxa	[%sp + V9BIAS64 + 88]%asi, %i3			;\
693	ldxa	[%sp + V9BIAS64 + 96]%asi, %i4			;\
694	ldxa	[%sp + V9BIAS64 + 104]%asi, %i5			;\
695	ldxa	[%sp + V9BIAS64 + 112]%asi, %i6			;\
696	ldxa	[%sp + V9BIAS64 + 120]%asi, %i7			;\
697	restored						;\
698	retry							;\
699	SKIP(31-19-TT_TRACE_L_INS)				;\
700	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
701	.empty
702
703#endif /* !lint */
704
705/*
706 * SPILL_mixed spills either size window, depending on
707 * whether %sp is even or odd, to a 32-bit address space.
708 * This may only be used in conjunction with SPILL_32bit/
709 * SPILL_64bit. New versions of SPILL_mixed_{tt1,asi} would be
710 * needed for use with SPILL_{32,64}bit_{tt1,asi}.  Particular
711 * attention should be paid to the instructions that belong
712 * in the delay slots of the branches depending on the type
713 * of spill handler being branched to.
714 * Clear upper 32 bits of %sp if it is odd.
715 * We won't need to clear them in 64 bit kernel.
716 */
717#define	SPILL_mixed						\
718	btst	1, %sp						;\
719	bz,a,pt	%xcc, 1b					;\
720	srl	%sp, 0, %sp					;\
721	ba,pt	%xcc, 2b					;\
722	nop							;\
723	.align	128
724
725/*
726 * FILL_mixed(ASI) fills either size window, depending on
727 * whether %sp is even or odd, from a 32-bit address space.
728 * This may only be used in conjunction with FILL_32bit/
729 * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
730 * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
731 * attention should be paid to the instructions that belong
732 * in the delay slots of the branches depending on the type
733 * of fill handler being branched to.
734 * Clear upper 32 bits of %sp if it is odd.
735 * We won't need to clear them in 64 bit kernel.
736 */
737#define	FILL_mixed						\
738	btst	1, %sp						;\
739	bz,a,pt	%xcc, 1b					;\
740	srl	%sp, 0, %sp					;\
741	ba,pt	%xcc, 2b					;\
742	nop							;\
743	.align	128
744
745
746/*
747 * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
748 * respectively, into the address space via the designated asi.  The
749 * unbiased stack pointer is required to be eight-byte aligned (even for
750 * the 32-bit case even though this code does not require such strict
751 * alignment).
752 *
753 * With SPARC v9 the spill trap takes precedence over the cleanwin trap
754 * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
755 * will cause cwp + 2 to be spilled but will not clean cwp + 1.  That
756 * window may contain kernel data so in user_rtt we set wstate to call
757 * these spill handlers on the first user spill trap.  These handler then
758 * spill the appropriate window but also back up a window and clean the
759 * window that didn't get a cleanwin trap.
760 */
761#define	SPILL_32clean(asi_num, tail)				\
762	srl	%sp, 0, %sp					;\
763	sta	%l0, [%sp + %g0]asi_num				;\
764	mov	4, %g1						;\
765	sta	%l1, [%sp + %g1]asi_num				;\
766	mov	8, %g2						;\
767	sta	%l2, [%sp + %g2]asi_num				;\
768	mov	12, %g3						;\
769	sta	%l3, [%sp + %g3]asi_num				;\
770	add	%sp, 16, %g4					;\
771	sta	%l4, [%g4 + %g0]asi_num				;\
772	sta	%l5, [%g4 + %g1]asi_num				;\
773	sta	%l6, [%g4 + %g2]asi_num				;\
774	sta	%l7, [%g4 + %g3]asi_num				;\
775	add	%g4, 16, %g4					;\
776	sta	%i0, [%g4 + %g0]asi_num				;\
777	sta	%i1, [%g4 + %g1]asi_num				;\
778	sta	%i2, [%g4 + %g2]asi_num				;\
779	sta	%i3, [%g4 + %g3]asi_num				;\
780	add	%g4, 16, %g4					;\
781	sta	%i4, [%g4 + %g0]asi_num				;\
782	sta	%i5, [%g4 + %g1]asi_num				;\
783	sta	%i6, [%g4 + %g2]asi_num				;\
784	sta	%i7, [%g4 + %g3]asi_num				;\
785	TT_TRACE_L(trace_win)					;\
786	b	.spill_clean					;\
787	  mov	WSTATE_USER32, %g7				;\
788	SKIP(31-25-TT_TRACE_L_INS)				;\
789	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
790	.empty
791
792#define	SPILL_64clean(asi_num, tail)				\
793	mov	0 + V9BIAS64, %g1				;\
794	stxa	%l0, [%sp + %g1]asi_num				;\
795	mov	8 + V9BIAS64, %g2				;\
796	stxa	%l1, [%sp + %g2]asi_num				;\
797	mov	16 + V9BIAS64, %g3				;\
798	stxa	%l2, [%sp + %g3]asi_num				;\
799	mov	24 + V9BIAS64, %g4				;\
800	stxa	%l3, [%sp + %g4]asi_num				;\
801	add	%sp, 32, %g5					;\
802	stxa	%l4, [%g5 + %g1]asi_num				;\
803	stxa	%l5, [%g5 + %g2]asi_num				;\
804	stxa	%l6, [%g5 + %g3]asi_num				;\
805	stxa	%l7, [%g5 + %g4]asi_num				;\
806	add	%g5, 32, %g5					;\
807	stxa	%i0, [%g5 + %g1]asi_num				;\
808	stxa	%i1, [%g5 + %g2]asi_num				;\
809	stxa	%i2, [%g5 + %g3]asi_num				;\
810	stxa	%i3, [%g5 + %g4]asi_num				;\
811	add	%g5, 32, %g5					;\
812	stxa	%i4, [%g5 + %g1]asi_num				;\
813	stxa	%i5, [%g5 + %g2]asi_num				;\
814	stxa	%i6, [%g5 + %g3]asi_num				;\
815	stxa	%i7, [%g5 + %g4]asi_num				;\
816	TT_TRACE_L(trace_win)					;\
817	b	.spill_clean					;\
818	  mov	WSTATE_USER64, %g7				;\
819	SKIP(31-25-TT_TRACE_L_INS)				;\
820	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
821	.empty
822
823
824/*
825 * Floating point disabled.
826 */
827#define	FP_DISABLED_TRAP		\
828	TT_TRACE(trace_gen)		;\
829	ba,pt	%xcc,.fp_disabled	;\
830	nop				;\
831	.align	32
832
833/*
834 * Floating point exceptions.
835 */
836#define	FP_IEEE_TRAP			\
837	TT_TRACE(trace_gen)		;\
838	ba,pt	%xcc,.fp_ieee_exception	;\
839	nop				;\
840	.align	32
841
842#define	FP_TRAP				\
843	TT_TRACE(trace_gen)		;\
844	ba,pt	%xcc,.fp_exception	;\
845	nop				;\
846	.align	32
847
848#if !defined(lint)
849/*
850 * asynchronous traps at level 0 and level 1
851 *
852 * The first instruction must be a membar for UltraSPARC-III
853 * to stop RED state entry if the store queue has many
854 * pending bad stores (PRM, Chapter 11).
855 */
856#define ASYNC_TRAP(ttype, ttlabel, table_name)\
857	.global	table_name	;\
858table_name:			;\
859	membar	#Sync		;\
860	TT_TRACE(ttlabel)	;\
861	ba	async_err	;\
862	mov	ttype, %g5	;\
863	.align	32
864
865/*
866 * Defaults to BAD entry, but establishes label to be used for
867 * architecture-specific overwrite of trap table entry.
868 */
869#define	LABELED_BAD(table_name)		\
870	.global	table_name		;\
871table_name:				;\
872	BAD
873
874#endif /* !lint */
875
876/*
877 * illegal instruction trap
878 */
879#define	ILLTRAP_INSTR			  \
880	membar	#Sync			  ;\
881	TT_TRACE(trace_gen)		  ;\
882	or	%g0, P_UTRAP4, %g2	  ;\
883	or	%g0, T_UNIMP_INSTR, %g3   ;\
884	sethi	%hi(.check_v9utrap), %g4  ;\
885	jmp	%g4 + %lo(.check_v9utrap) ;\
886	nop				  ;\
887	.align	32
888
889/*
890 * tag overflow trap
891 */
892#define	TAG_OVERFLOW			  \
893	TT_TRACE(trace_gen)		  ;\
894	or	%g0, P_UTRAP10, %g2	  ;\
895	or	%g0, T_TAG_OVERFLOW, %g3  ;\
896	sethi	%hi(.check_v9utrap), %g4  ;\
897	jmp	%g4 + %lo(.check_v9utrap) ;\
898	nop				  ;\
899	.align	32
900
901/*
902 * divide by zero trap
903 */
904#define	DIV_BY_ZERO			  \
905	TT_TRACE(trace_gen)		  ;\
906	or	%g0, P_UTRAP11, %g2	  ;\
907	or	%g0, T_IDIV0, %g3	  ;\
908	sethi	%hi(.check_v9utrap), %g4  ;\
909	jmp	%g4 + %lo(.check_v9utrap) ;\
910	nop				  ;\
911	.align	32
912
913/*
914 * trap instruction for V9 user trap handlers
915 */
916#define	TRAP_INSTR			  \
917	TT_TRACE(trace_gen)		  ;\
918	or	%g0, T_SOFTWARE_TRAP, %g3 ;\
919	sethi	%hi(.check_v9utrap), %g4  ;\
920	jmp	%g4 + %lo(.check_v9utrap) ;\
921	nop				  ;\
922	.align	32
923#define	TRP4	TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
924
925/*
926 * LEVEL_INTERRUPT is for level N interrupts.
927 * VECTOR_INTERRUPT is for the vector trap.
928 */
929#define	LEVEL_INTERRUPT(level)		\
930	.global	tt_pil/**/level		;\
931tt_pil/**/level:			;\
932	ba,pt	%xcc, pil_interrupt	;\
933	mov	level, %g4		;\
934	.align	32
935
936#define	LEVEL14_INTERRUPT			\
937	ba	pil14_interrupt			;\
938	mov	PIL_14, %g4			;\
939	.align	32
940
941#define        LEVEL15_INTERRUPT                       \
942       ba      pil15_interrupt                 ;\
943       mov     PIL_15, %g4                     ;\
944       .align  32
945
946#define	VECTOR_INTERRUPT				\
947	ldxa	[%g0]ASI_INTR_RECEIVE_STATUS, %g1	;\
948	btst	IRSR_BUSY, %g1				;\
949	bnz,pt	%xcc, vec_interrupt			;\
950	nop						;\
951	ba,a,pt	%xcc, vec_intr_spurious			;\
952	.empty						;\
953	.align	32
954
955/*
956 * MMU Trap Handlers.
957 */
958#define	SWITCH_GLOBALS	/* mmu->alt, alt->mmu */			\
959	rdpr	%pstate, %g5						;\
960	wrpr	%g5, PSTATE_MG | PSTATE_AG, %pstate
961
962#define	IMMU_EXCEPTION							\
963	membar	#Sync							;\
964	SWITCH_GLOBALS							;\
965	wr	%g0, ASI_IMMU, %asi					;\
966	rdpr	%tpc, %g2						;\
967	ldxa	[MMU_SFSR]%asi, %g3					;\
968	ba,pt	%xcc, .mmu_exception_end				;\
969	mov	T_INSTR_EXCEPTION, %g1					;\
970	.align	32
971
972#define	DMMU_EXCEPTION							\
973	SWITCH_GLOBALS							;\
974	wr	%g0, ASI_DMMU, %asi					;\
975	ldxa	[MMU_TAG_ACCESS]%asi, %g2				;\
976	ldxa	[MMU_SFSR]%asi, %g3					;\
977	ba,pt	%xcc, .mmu_exception_end				;\
978	mov	T_DATA_EXCEPTION, %g1					;\
979	.align	32
980
981#define	DMMU_EXC_AG_PRIV						\
982	wr	%g0, ASI_DMMU, %asi					;\
983	ldxa	[MMU_SFAR]%asi, %g2					;\
984	ba,pt	%xcc, .mmu_priv_exception				;\
985	ldxa	[MMU_SFSR]%asi, %g3					;\
986	.align	32
987
988#define	DMMU_EXC_AG_NOT_ALIGNED						\
989	wr	%g0, ASI_DMMU, %asi					;\
990	ldxa	[MMU_SFAR]%asi, %g2					;\
991	ba,pt	%xcc, .mmu_exception_not_aligned			;\
992	ldxa	[MMU_SFSR]%asi, %g3					;\
993	.align	32
994
995/*
996 * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
997 */
998#define	DMMU_EXC_LDDF_NOT_ALIGNED					\
999	btst	1, %sp							;\
1000	bnz,pt	%xcc, .lddf_exception_not_aligned			;\
1001	wr	%g0, ASI_DMMU, %asi					;\
1002	ldxa	[MMU_SFAR]%asi, %g2					;\
1003	ba,pt	%xcc, .mmu_exception_not_aligned			;\
1004	ldxa	[MMU_SFSR]%asi, %g3					;\
1005	.align	32
1006
1007#define	DMMU_EXC_STDF_NOT_ALIGNED					\
1008	btst	1, %sp							;\
1009	bnz,pt	%xcc, .stdf_exception_not_aligned			;\
1010	wr	%g0, ASI_DMMU, %asi					;\
1011	ldxa	[MMU_SFAR]%asi, %g2					;\
1012	ba,pt	%xcc, .mmu_exception_not_aligned			;\
1013	ldxa	[MMU_SFSR]%asi, %g3					;\
1014	.align	32
1015
1016/*
1017 * Flush the TLB using either the primary, secondary, or nucleus flush
1018 * operation based on whether the ctx from the tag access register matches
1019 * the primary or secondary context (flush the nucleus if neither matches).
1020 *
1021 * Requires a membar #Sync before next ld/st.
1022 * exits with:
1023 * g2 = tag access register
1024 * g3 = ctx number
1025 */
1026#if TAGACC_CTX_MASK != CTXREG_CTX_MASK
1027#error "TAGACC_CTX_MASK != CTXREG_CTX_MASK"
1028#endif
1029#define	DTLB_DEMAP_ENTRY						\
1030	mov	MMU_TAG_ACCESS, %g1					;\
1031	mov	MMU_PCONTEXT, %g5					;\
1032	ldxa	[%g1]ASI_DMMU, %g2					;\
1033	sethi	%hi(TAGACC_CTX_MASK), %g4				;\
1034	or	%g4, %lo(TAGACC_CTX_MASK), %g4				;\
1035	and	%g2, %g4, %g3			/* g3 = ctx */		;\
1036	ldxa	[%g5]ASI_DMMU, %g6		/* g6 = primary ctx */	;\
1037	and	%g6, %g4, %g6			/* &= CTXREG_CTX_MASK */ ;\
1038	cmp	%g3, %g6						;\
1039	be,pt	%xcc, 1f						;\
1040	andn	%g2, %g4, %g1			/* ctx = primary */	;\
1041	mov	MMU_SCONTEXT, %g5					;\
1042	ldxa	[%g5]ASI_DMMU, %g6		/* g6 = secondary ctx */ ;\
1043	and	%g6, %g4, %g6			/* &= CTXREG_CTX_MASK */ ;\
1044	cmp	%g3, %g6						;\
1045	be,a,pt	%xcc, 1f						;\
1046	  or	%g1, DEMAP_SECOND, %g1					;\
1047	or	%g1, DEMAP_NUCLEUS, %g1					;\
10481:	stxa	%g0, [%g1]ASI_DTLB_DEMAP	/* MMU_DEMAP_PAGE */	;\
1049	membar	#Sync
1050
1051#if defined(cscope)
1052/*
1053 * Define labels to direct cscope quickly to labels that
1054 * are generated by macro expansion of DTLB_MISS().
1055 */
1056	.global	tt0_dtlbmiss
1057tt0_dtlbmiss:
1058	.global	tt1_dtlbmiss
1059tt1_dtlbmiss:
1060	nop
1061#endif
1062
1063/*
1064 * Needs to be exactly 32 instructions
1065 *
1066 * UTLB NOTE: If we don't hit on the 8k pointer then we branch
1067 * to a special 4M tsb handler. It would be nice if that handler
1068 * could live in this file but currently it seems better to allow
1069 * it to fall thru to sfmmu_tsb_miss.
1070 */
1071#ifdef UTSB_PHYS
1072#define	DTLB_MISS(table_name)						;\
1073	.global	table_name/**/_dtlbmiss					;\
1074table_name/**/_dtlbmiss:						;\
1075	mov	MMU_TAG_ACCESS, %g6		/* select tag acc */	;\
1076	ldxa	[%g0]ASI_DMMU_TSB_8K, %g1	/* g1 = tsbe ptr */	;\
1077	ldxa	[%g6]ASI_DMMU, %g2		/* g2 = tag access */	;\
1078	sllx	%g2, TAGACC_CTX_LSHIFT, %g3				;\
1079	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctx */		;\
1080	cmp	%g3, INVALID_CONTEXT					;\
1081	ble,pn	%xcc, sfmmu_kdtlb_miss					;\
1082	  srax	%g2, PREDISM_BASESHIFT, %g6  /* g6 > 0 ISM predicted */ ;\
1083	brgz,pn %g6, sfmmu_udtlb_slowpath_ismpred			;\
1084	  srlx	%g2, TAG_VALO_SHIFT, %g7	/* g7 = tsb tag */	;\
1085	ldda	[%g1]ASI_QUAD_LDD_PHYS, %g4	/* g4 = tag, %g5 data */;\
1086	cmp	%g4, %g7						;\
1087	bne,pn %xcc, sfmmu_udtlb_slowpath_noismpred			;\
1088	  nop								;\
1089	TT_TRACE(trace_tsbhit)		/* 2 instr ifdef TRAPTRACE */	;\
1090	stxa	%g5, [%g0]ASI_DTLB_IN	/* trapstat expects TTE */	;\
1091	retry				/* in %g5 */			;\
1092	unimp	0							;\
1093	unimp	0							;\
1094	unimp	0							;\
1095	unimp	0							;\
1096	unimp	0							;\
1097	unimp	0							;\
1098	unimp	0							;\
1099	unimp	0							;\
1100	unimp	0							;\
1101	unimp	0							;\
1102	unimp	0							;\
1103	unimp	0							;\
1104	unimp	0							;\
1105	unimp	0							;\
1106	.align 128
1107
1108#else /* UTSB_PHYS */
1109#define	DTLB_MISS(table_name)						;\
1110	.global	table_name/**/_dtlbmiss					;\
1111table_name/**/_dtlbmiss:						;\
1112	mov	MMU_TAG_ACCESS, %g6		/* select tag acc */	;\
1113	ldxa	[%g0]ASI_DMMU_TSB_8K, %g1	/* g1 = tsbe ptr */	;\
1114	ldxa	[%g6]ASI_DMMU, %g2		/* g2 = tag access */	;\
1115	sllx	%g2, TAGACC_CTX_LSHIFT, %g3				;\
1116	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctx */		;\
1117	cmp	%g3, INVALID_CONTEXT					;\
1118	ble,pn	%xcc, sfmmu_kdtlb_miss					;\
1119	  srlx	%g2, TAG_VALO_SHIFT, %g7	/* g7 = tsb tag */	;\
1120	brlz,pn %g1, sfmmu_udtlb_slowpath				;\
1121	  nop								;\
1122	ldda	[%g1]ASI_NQUAD_LD, %g4	/* g4 = tag, %g5 data */	;\
1123	cmp	%g4, %g7						;\
1124	bne,pn	%xcc, sfmmu_tsb_miss_tt		/* no 4M TSB, miss */	;\
1125	  mov	-1, %g3		/* set 4M tsbe ptr to -1 */		;\
1126	TT_TRACE(trace_tsbhit)		/* 2 instr ifdef TRAPTRACE */	;\
1127	stxa	%g5, [%g0]ASI_DTLB_IN	/* trapstat expects TTE */	;\
1128	retry				/* in %g5 */			;\
1129	unimp	0							;\
1130	unimp	0							;\
1131	unimp	0							;\
1132	unimp	0							;\
1133	unimp	0							;\
1134        unimp   0                                                       ;\
1135        unimp   0                                                       ;\
1136        unimp   0                                                       ;\
1137        unimp   0                                                       ;\
1138        unimp   0                                                       ;\
1139        unimp   0                                                       ;\
1140        unimp   0                                                       ;\
1141        unimp   0                                                       ;\
1142        unimp   0                                                       ;\
1143	.align 128
1144#endif /* UTSB_PHYS */
1145
1146#if defined(cscope)
1147/*
1148 * Define labels to direct cscope quickly to labels that
1149 * are generated by macro expansion of ITLB_MISS().
1150 */
1151	.global	tt0_itlbmiss
1152tt0_itlbmiss:
1153	.global	tt1_itlbmiss
1154tt1_itlbmiss:
1155	nop
1156#endif
1157
1158/*
1159 * Instruction miss handler.
1160 * ldda instructions will have their ASI patched
1161 * by sfmmu_patch_ktsb at runtime.
1162 * MUST be EXACTLY 32 instructions or we'll break.
1163 */
1164#ifdef UTSB_PHYS
1165#define ITLB_MISS(table_name)                                            \
1166        .global table_name/**/_itlbmiss                                 ;\
1167table_name/**/_itlbmiss:                                                ;\
1168        mov     MMU_TAG_ACCESS, %g6             /* select tag acc */    ;\
1169        ldxa    [%g0]ASI_IMMU_TSB_8K, %g1       /* g1 = tsbe ptr */     ;\
1170        ldxa    [%g6]ASI_IMMU, %g2              /* g2 = tag access */   ;\
1171        sllx    %g2, TAGACC_CTX_LSHIFT, %g3                             ;\
1172        srlx    %g3, TAGACC_CTX_LSHIFT, %g3     /* g3 = ctx */          ;\
1173        cmp     %g3, INVALID_CONTEXT                                    ;\
1174        ble,pn  %xcc, sfmmu_kitlb_miss                                  ;\
1175          srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
1176        ldda    [%g1]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */   ;\
1177        cmp     %g4, %g7                                                ;\
1178        bne,pn  %xcc, sfmmu_uitlb_slowpath                              ;\
1179          andcc %g5, TTE_EXECPRM_INT, %g0 /* check execute bit */       ;\
1180        bz,pn   %icc, exec_fault                                        ;\
1181          nop                                                           ;\
1182        TT_TRACE(trace_tsbhit)          /* 2 instr ifdef TRAPTRACE */   ;\
1183        stxa    %g5, [%g0]ASI_ITLB_IN   /* trapstat expects %g5 */      ;\
1184        retry                                                           ;\
1185        unimp   0                                                       ;\
1186        unimp   0                                                       ;\
1187        unimp   0                                                       ;\
1188        unimp   0                                                       ;\
1189        unimp   0                                                       ;\
1190        unimp   0                                                       ;\
1191        unimp   0                                                       ;\
1192        unimp   0                                                       ;\
1193        unimp   0                                                       ;\
1194        unimp   0                                                       ;\
1195        unimp   0                                                       ;\
1196        unimp   0                                                       ;\
1197        unimp   0                                                       ;\
1198        unimp   0                                                       ;\
1199        .align 128
1200
1201#else /* UTSB_PHYS */
1202#define	ITLB_MISS(table_name)						 \
1203	.global	table_name/**/_itlbmiss					;\
1204table_name/**/_itlbmiss:						;\
1205	mov	MMU_TAG_ACCESS, %g6		/* select tag acc */	;\
1206	ldxa	[%g0]ASI_IMMU_TSB_8K, %g1	/* g1 = tsbe ptr */	;\
1207	ldxa	[%g6]ASI_IMMU, %g2		/* g2 = tag access */	;\
1208	sllx	%g2, TAGACC_CTX_LSHIFT, %g3				;\
1209	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctx */		;\
1210	cmp	%g3, INVALID_CONTEXT					;\
1211	ble,pn	%xcc, sfmmu_kitlb_miss					;\
1212	  srlx	%g2, TAG_VALO_SHIFT, %g7	/* g7 = tsb tag */	;\
1213	brlz,pn	%g1, sfmmu_uitlb_slowpath	/* if >1 TSB branch */	;\
1214	  nop								;\
1215	ldda	[%g1]ASI_NQUAD_LD, %g4	/* g4 = tag, g5 = data */	;\
1216	cmp	%g4, %g7						;\
1217	bne,pn	%xcc, sfmmu_tsb_miss_tt	/* br if 8k ptr miss */		;\
1218	  mov	-1, %g3		/* set 4M TSB ptr to -1 */		;\
1219	andcc	%g5, TTE_EXECPRM_INT, %g0 /* check execute bit */	;\
1220	bz,pn	%icc, exec_fault					;\
1221	  nop								;\
1222	TT_TRACE(trace_tsbhit)		/* 2 instr ifdef TRAPTRACE */	;\
1223	stxa	%g5, [%g0]ASI_ITLB_IN	/* trapstat expects %g5 */	;\
1224	retry								;\
1225	unimp	0							;\
1226	unimp	0							;\
1227	unimp	0							;\
1228	unimp	0							;\
1229	unimp	0							;\
1230        unimp   0                                                       ;\
1231        unimp   0                                                       ;\
1232        unimp   0                                                       ;\
1233        unimp   0                                                       ;\
1234        unimp   0                                                       ;\
1235        unimp   0                                                       ;\
1236	.align 128
1237#endif /* UTSB_PHYS */
1238
1239
1240/*
1241 * This macro is the first level handler for fast protection faults.
1242 * It first demaps the tlb entry which generated the fault and then
1243 * attempts to set the modify bit on the hash.  It needs to be
1244 * exactly 32 instructions.
1245 */
1246#define	DTLB_PROT							 \
1247	DTLB_DEMAP_ENTRY		/* 20 instructions */		;\
1248	/*								;\
1249	 * At this point:						;\
1250	 *   g1 = ????							;\
1251	 *   g2 = tag access register					;\
1252	 *   g3 = ctx number						;\
1253	 *   g4 = ????							;\
1254	 */								;\
1255	TT_TRACE(trace_dataprot)	/* 2 instr ifdef TRAPTRACE */	;\
1256					/* clobbers g1 and g6 */	;\
1257	ldxa	[%g0]ASI_DMMU_TSB_8K, %g1	/* g1 = tsbe ptr */	;\
1258	brnz,pt %g3, sfmmu_uprot_trap		/* user trap */		;\
1259	  nop								;\
1260	ba,a,pt	%xcc, sfmmu_kprot_trap		/* kernel trap */	;\
1261	unimp	0							;\
1262	unimp	0							;\
1263	unimp	0							;\
1264	unimp	0							;\
1265	unimp	0							;\
1266	unimp	0							;\
1267	.align 128
1268
1269#define	DMMU_EXCEPTION_TL1						;\
1270	SWITCH_GLOBALS							;\
1271	ba,a,pt	%xcc, mmu_trap_tl1					;\
1272	  nop								;\
1273	.align 32
1274
1275#define	MISALIGN_ADDR_TL1						;\
1276	ba,a,pt	%xcc, mmu_trap_tl1					;\
1277	  nop								;\
1278	.align 32
1279
1280/*
1281 * Trace a tsb hit
1282 * g1 = tsbe pointer (in/clobbered)
1283 * g2 = tag access register (in)
1284 * g3 - g4 = scratch (clobbered)
1285 * g5 = tsbe data (in)
1286 * g6 = scratch (clobbered)
1287 * g7 = pc we jumped here from (in)
1288 * ttextra = value to OR in to trap type (%tt) (in)
1289 */
1290#ifdef TRAPTRACE
1291#define TRACE_TSBHIT(ttextra)						 \
1292	membar	#Sync							;\
1293	sethi	%hi(FLUSH_ADDR), %g6					;\
1294	flush	%g6							;\
1295	TRACE_PTR(%g3, %g6)						;\
1296	GET_TRACE_TICK(%g6, %g4)					;\
1297	stxa	%g6, [%g3 + TRAP_ENT_TICK]%asi				;\
1298	stxa	%g2, [%g3 + TRAP_ENT_SP]%asi	/* tag access */	;\
1299	stxa	%g5, [%g3 + TRAP_ENT_F1]%asi	/* tsb data */		;\
1300	rdpr	%tnpc, %g6						;\
1301	stxa	%g6, [%g3 + TRAP_ENT_F2]%asi				;\
1302	stxa	%g1, [%g3 + TRAP_ENT_F3]%asi	/* tsb pointer */	;\
1303	stxa	%g0, [%g3 + TRAP_ENT_F4]%asi				;\
1304	rdpr	%tpc, %g6						;\
1305	stxa	%g6, [%g3 + TRAP_ENT_TPC]%asi				;\
1306	rdpr	%tl, %g6						;\
1307	stha	%g6, [%g3 + TRAP_ENT_TL]%asi				;\
1308	rdpr	%tt, %g6						;\
1309	or	%g6, (ttextra), %g6					;\
1310	stha	%g6, [%g3 + TRAP_ENT_TT]%asi				;\
1311	ldxa	[%g0]ASI_IMMU, %g1		/* tag target */	;\
1312	ldxa	[%g0]ASI_DMMU, %g4					;\
1313	cmp	%g6, FAST_IMMU_MISS_TT					;\
1314	movne	%icc, %g4, %g1						;\
1315	stxa	%g1, [%g3 + TRAP_ENT_TSTATE]%asi /* tsb tag */		;\
1316	stxa	%g0, [%g3 + TRAP_ENT_TR]%asi				;\
1317	TRACE_NEXT(%g3, %g4, %g6)
1318#else
1319#define TRACE_TSBHIT(ttextra)
1320#endif
1321
1322#if defined(lint)
1323
1324struct scb	trap_table;
1325struct scb	scb;		/* trap_table/scb are the same object */
1326
1327#else /* lint */
1328
1329/*
1330 * =======================================================================
1331 *		SPARC V9 TRAP TABLE
1332 *
1333 * The trap table is divided into two halves: the first half is used when
1334 * taking traps when TL=0; the second half is used when taking traps from
1335 * TL>0. Note that handlers in the second half of the table might not be able
1336 * to make the same assumptions as handlers in the first half of the table.
1337 *
1338 * Worst case trap nesting so far:
1339 *
1340 *	at TL=0 client issues software trap requesting service
1341 *	at TL=1 nucleus wants a register window
1342 *	at TL=2 register window clean/spill/fill takes a TLB miss
1343 *	at TL=3 processing TLB miss
1344 *	at TL=4 handle asynchronous error
1345 *
1346 * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
1347 *
1348 * =======================================================================
1349 */
1350	.section ".text"
1351	.align	4
1352	.global trap_table, scb, trap_table0, trap_table1, etrap_table
1353	.type	trap_table, #object
1354	.type	scb, #object
1355trap_table:
1356scb:
1357trap_table0:
1358	/* hardware traps */
1359	NOT;				/* 000	reserved */
1360	RED;				/* 001	power on reset */
1361	RED;				/* 002	watchdog reset */
1362	RED;				/* 003	externally initiated reset */
1363	RED;				/* 004	software initiated reset */
1364	RED;				/* 005	red mode exception */
1365	NOT; NOT;			/* 006 - 007 reserved */
1366	IMMU_EXCEPTION;			/* 008	instruction access exception */
1367	NOT;				/* 009	instruction access MMU miss */
1368	ASYNC_TRAP(T_INSTR_ERROR, trace_gen, tt0_iae);
1369					/* 00A	instruction access error */
1370	NOT; NOT4;			/* 00B - 00F reserved */
1371	ILLTRAP_INSTR;			/* 010	illegal instruction */
1372	TRAP(T_PRIV_INSTR);		/* 011	privileged opcode */
1373	NOT;				/* 012	unimplemented LDD */
1374	NOT;				/* 013	unimplemented STD */
1375	NOT4; NOT4; NOT4;		/* 014 - 01F reserved */
1376	FP_DISABLED_TRAP;		/* 020	fp disabled */
1377	FP_IEEE_TRAP;			/* 021	fp exception ieee 754 */
1378	FP_TRAP;			/* 022	fp exception other */
1379	TAG_OVERFLOW;			/* 023	tag overflow */
1380	CLEAN_WINDOW;			/* 024 - 027 clean window */
1381	DIV_BY_ZERO;			/* 028	division by zero */
1382	NOT;				/* 029	internal processor error */
1383	NOT; NOT; NOT4;			/* 02A - 02F reserved */
1384	DMMU_EXCEPTION;			/* 030	data access exception */
1385	NOT;				/* 031	data access MMU miss */
1386	ASYNC_TRAP(T_DATA_ERROR, trace_gen, tt0_dae);
1387					/* 032	data access error */
1388	NOT;				/* 033	data access protection */
1389	DMMU_EXC_AG_NOT_ALIGNED;	/* 034	mem address not aligned */
1390	DMMU_EXC_LDDF_NOT_ALIGNED;	/* 035	LDDF mem address not aligned */
1391	DMMU_EXC_STDF_NOT_ALIGNED;	/* 036	STDF mem address not aligned */
1392	DMMU_EXC_AG_PRIV;		/* 037	privileged action */
1393	NOT;				/* 038	LDQF mem address not aligned */
1394	NOT;				/* 039	STQF mem address not aligned */
1395	NOT; NOT; NOT4;			/* 03A - 03F reserved */
1396	LABELED_BAD(tt0_asdat);		/* 040	async data error */
1397	LEVEL_INTERRUPT(1);		/* 041	interrupt level 1 */
1398	LEVEL_INTERRUPT(2);		/* 042	interrupt level 2 */
1399	LEVEL_INTERRUPT(3);		/* 043	interrupt level 3 */
1400	LEVEL_INTERRUPT(4);		/* 044	interrupt level 4 */
1401	LEVEL_INTERRUPT(5);		/* 045	interrupt level 5 */
1402	LEVEL_INTERRUPT(6);		/* 046	interrupt level 6 */
1403	LEVEL_INTERRUPT(7);		/* 047	interrupt level 7 */
1404	LEVEL_INTERRUPT(8);		/* 048	interrupt level 8 */
1405	LEVEL_INTERRUPT(9);		/* 049	interrupt level 9 */
1406	LEVEL_INTERRUPT(10);		/* 04A	interrupt level 10 */
1407	LEVEL_INTERRUPT(11);		/* 04B	interrupt level 11 */
1408	LEVEL_INTERRUPT(12);		/* 04C	interrupt level 12 */
1409	LEVEL_INTERRUPT(13);		/* 04D	interrupt level 13 */
1410	LEVEL14_INTERRUPT;		/* 04E	interrupt level 14 */
1411	LEVEL15_INTERRUPT;		/* 04F	interrupt level 15 */
1412	NOT4; NOT4; NOT4; NOT4;		/* 050 - 05F reserved */
1413	VECTOR_INTERRUPT;		/* 060	interrupt vector */
1414	GOTO(kmdb_trap);		/* 061	PA watchpoint */
1415	GOTO(kmdb_trap);		/* 062	VA watchpoint */
1416	GOTO_TT(ce_err, trace_gen);	/* 063	corrected ECC error */
1417	ITLB_MISS(tt0);			/* 064	instruction access MMU miss */
1418	DTLB_MISS(tt0);			/* 068	data access MMU miss */
1419	DTLB_PROT;			/* 06C	data access protection */
1420	LABELED_BAD(tt0_fecc);		/* 070  fast ecache ECC error */
1421	LABELED_BAD(tt0_dperr);		/* 071  Cheetah+ dcache parity error */
1422	LABELED_BAD(tt0_iperr);		/* 072  Cheetah+ icache parity error */
1423	NOT;				/* 073  reserved */
1424	NOT4; NOT4; NOT4;		/* 074 - 07F reserved */
1425	NOT4;				/* 080	spill 0 normal */
1426	SPILL_32bit_asi(ASI_AIUP,sn0);	/* 084	spill 1 normal */
1427	SPILL_64bit_asi(ASI_AIUP,sn0);	/* 088	spill 2 normal */
1428	SPILL_32clean(ASI_AIUP,sn0);	/* 08C	spill 3 normal */
1429	SPILL_64clean(ASI_AIUP,sn0);	/* 090	spill 4 normal */
1430	SPILL_32bit(not);		/* 094	spill 5 normal */
1431	SPILL_64bit(not);		/* 098	spill 6 normal */
1432	SPILL_mixed;			/* 09C	spill 7 normal */
1433	NOT4;				/* 0A0	spill 0 other */
1434	SPILL_32bit_asi(ASI_AIUS,so0);	/* 0A4	spill 1 other */
1435	SPILL_64bit_asi(ASI_AIUS,so0);	/* 0A8	spill 2 other */
1436	SPILL_32bit_asi(ASI_AIUS,so0);	/* 0AC	spill 3 other */
1437	SPILL_64bit_asi(ASI_AIUS,so0);	/* 0B0	spill 4 other */
1438	NOT4;				/* 0B4	spill 5 other */
1439	NOT4;				/* 0B8	spill 6 other */
1440	NOT4;				/* 0BC	spill 7 other */
1441	NOT4;				/* 0C0	fill 0 normal */
1442	FILL_32bit_asi(ASI_AIUP,fn0);	/* 0C4	fill 1 normal */
1443	FILL_64bit_asi(ASI_AIUP,fn0);	/* 0C8	fill 2 normal */
1444	FILL_32bit_asi(ASI_AIUP,fn0);	/* 0CC	fill 3 normal */
1445	FILL_64bit_asi(ASI_AIUP,fn0);	/* 0D0	fill 4 normal */
1446	FILL_32bit(not);		/* 0D4	fill 5 normal */
1447	FILL_64bit(not);		/* 0D8	fill 6 normal */
1448	FILL_mixed;			/* 0DC	fill 7 normal */
1449	NOT4;				/* 0E0	fill 0 other */
1450	NOT4;				/* 0E4	fill 1 other */
1451	NOT4;				/* 0E8	fill 2 other */
1452	NOT4;				/* 0EC	fill 3 other */
1453	NOT4;				/* 0F0	fill 4 other */
1454	NOT4;				/* 0F4	fill 5 other */
1455	NOT4;				/* 0F8	fill 6 other */
1456	NOT4;				/* 0FC	fill 7 other */
1457	/* user traps */
1458	GOTO(syscall_trap_4x);		/* 100	old system call */
1459	TRAP(T_BREAKPOINT);		/* 101	user breakpoint */
1460	TRAP(T_DIV0);			/* 102	user divide by zero */
1461	FLUSHW(tt0_flushw);		/* 103	flush windows */
1462	GOTO(.clean_windows);		/* 104	clean windows */
1463	BAD;				/* 105	range check ?? */
1464	GOTO(.fix_alignment);		/* 106	do unaligned references */
1465	BAD;				/* 107	unused */
1466	SYSCALL_TRAP32;			/* 108	ILP32 system call on LP64 */
1467	GOTO(set_trap0_addr);		/* 109	set trap0 address */
1468	BAD; BAD; BAD4;			/* 10A - 10F unused */
1469	TRP4; TRP4; TRP4; TRP4;		/* 110 - 11F V9 user trap handlers */
1470	GOTO(.getcc);			/* 120	get condition codes */
1471	GOTO(.setcc);			/* 121	set condition codes */
1472	GOTO(.getpsr);			/* 122	get psr */
1473	GOTO(.setpsr);			/* 123	set psr (some fields) */
1474	GOTO(get_timestamp);		/* 124	get timestamp */
1475	GOTO(get_virtime);		/* 125	get lwp virtual time */
1476	PRIV(self_xcall);		/* 126	self xcall */
1477	GOTO(get_hrestime);		/* 127	get hrestime */
1478	BAD;				/* 128	ST_SETV9STACK */
1479	GOTO(.getlgrp);			/* 129  get lgrpid */
1480	BAD; BAD; BAD4;			/* 12A - 12F unused */
1481	BAD4; BAD4; 			/* 130 - 137 unused */
1482	DTRACE_PID;			/* 138  dtrace pid tracing provider */
1483	BAD;				/* 139  unused */
1484	DTRACE_RETURN;			/* 13A	dtrace pid return probe */
1485	BAD; BAD4;			/* 13B - 13F unused */
1486	SYSCALL_TRAP;			/* 140  LP64 system call */
1487	SYSCALL(nosys);			/* 141  unused system call trap */
1488#ifdef DEBUG_USER_TRAPTRACECTL
1489	GOTO(.traptrace_freeze);	/* 142  freeze traptrace */
1490	GOTO(.traptrace_unfreeze);	/* 143  unfreeze traptrace */
1491#else
1492	SYSCALL(nosys);			/* 142  unused system call trap */
1493	SYSCALL(nosys);			/* 143  unused system call trap */
1494#endif
1495	BAD4; BAD4; BAD4;		/* 144 - 14F unused */
1496	BAD4; BAD4; BAD4; BAD4;		/* 150 - 15F unused */
1497	BAD4; BAD4; BAD4; BAD4;		/* 160 - 16F unused */
1498	BAD;				/* 170 - unused */
1499	BAD;				/* 171 - unused */
1500	BAD; BAD;			/* 172 - 173 unused */
1501	BAD4; BAD4;			/* 174 - 17B unused */
1502#ifdef	PTL1_PANIC_DEBUG
1503	mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
1504					/* 17C	test ptl1_panic */
1505#else
1506	BAD;				/* 17C  unused */
1507#endif	/* PTL1_PANIC_DEBUG */
1508	PRIV(kmdb_trap);		/* 17D	kmdb enter (L1-A) */
1509	PRIV(kmdb_trap);		/* 17E	kmdb breakpoint */
1510	PRIV(kctx_obp_bpt);		/* 17F	obp breakpoint */
1511	/* reserved */
1512	NOT4; NOT4; NOT4; NOT4;		/* 180 - 18F reserved */
1513	NOT4; NOT4; NOT4; NOT4;		/* 190 - 19F reserved */
1514	NOT4; NOT4; NOT4; NOT4;		/* 1A0 - 1AF reserved */
1515	NOT4; NOT4; NOT4; NOT4;		/* 1B0 - 1BF reserved */
1516	NOT4; NOT4; NOT4; NOT4;		/* 1C0 - 1CF reserved */
1517	NOT4; NOT4; NOT4; NOT4;		/* 1D0 - 1DF reserved */
1518	NOT4; NOT4; NOT4; NOT4;		/* 1E0 - 1EF reserved */
1519	NOT4; NOT4; NOT4; NOT4;		/* 1F0 - 1FF reserved */
1520trap_table1:
1521	NOT4; NOT4; NOT; NOT;		/* 000 - 009 unused */
1522	ASYNC_TRAP(T_INSTR_ERROR + T_TL1, trace_gen, tt1_iae);
1523					/* 00A	instruction access error */
1524	NOT; NOT4;			/* 00B - 00F unused */
1525	NOT4; NOT4; NOT4; NOT4;		/* 010 - 01F unused */
1526	NOT4;				/* 020 - 023 unused */
1527	CLEAN_WINDOW;			/* 024 - 027 clean window */
1528	NOT4; NOT4;			/* 028 - 02F unused */
1529	DMMU_EXCEPTION_TL1;		/* 030 	data access exception */
1530	NOT;				/* 031 unused */
1531	ASYNC_TRAP(T_DATA_ERROR + T_TL1, trace_gen, tt1_dae);
1532					/* 032	data access error */
1533	NOT;				/* 033	unused */
1534	MISALIGN_ADDR_TL1;		/* 034	mem address not aligned */
1535	NOT; NOT; NOT; NOT4; NOT4	/* 035 - 03F unused */
1536	LABELED_BAD(tt1_asdat);		/* 040	async data error */
1537	NOT; NOT; NOT;			/* 041 - 043 unused */
1538	NOT4; NOT4; NOT4;		/* 044 - 04F unused */
1539	NOT4; NOT4; NOT4; NOT4;		/* 050 - 05F unused */
1540	NOT;				/* 060	unused */
1541	GOTO(kmdb_trap_tl1);		/* 061	PA watchpoint */
1542	GOTO(kmdb_trap_tl1);		/* 062	VA watchpoint */
1543	GOTO_TT(ce_err_tl1, trace_gen);	/* 063	corrected ECC error */
1544	ITLB_MISS(tt1);			/* 064	instruction access MMU miss */
1545	DTLB_MISS(tt1);			/* 068	data access MMU miss */
1546	DTLB_PROT;			/* 06C	data access protection */
1547	LABELED_BAD(tt1_fecc);		/* 070  fast ecache ECC error */
1548	LABELED_BAD(tt1_dperr);		/* 071  Cheetah+ dcache parity error */
1549	LABELED_BAD(tt1_iperr);		/* 072  Cheetah+ icache parity error */
1550	NOT;				/* 073  reserved */
1551	NOT4; NOT4; NOT4;		/* 074 - 07F reserved */
1552	NOT4;				/* 080	spill 0 normal */
1553	SPILL_32bit_tt1(ASI_AIUP,sn1);	/* 084	spill 1 normal */
1554	SPILL_64bit_tt1(ASI_AIUP,sn1);	/* 088	spill 2 normal */
1555	SPILL_32bit_tt1(ASI_AIUP,sn1);	/* 08C	spill 3 normal */
1556	SPILL_64bit_tt1(ASI_AIUP,sn1);	/* 090	spill 4 normal */
1557	SPILL_32bit(not);		/* 094	spill 5 normal */
1558	SPILL_64bit(not);		/* 098	spill 6 normal */
1559	SPILL_mixed;			/* 09C	spill 7 normal */
1560	NOT4;				/* 0A0	spill 0 other */
1561	SPILL_32bit_tt1(ASI_AIUS,so1);	/* 0A4	spill 1 other */
1562	SPILL_64bit_tt1(ASI_AIUS,so1);	/* 0A8	spill 2 other */
1563	SPILL_32bit_tt1(ASI_AIUS,so1);	/* 0AC	spill 3 other */
1564	SPILL_64bit_tt1(ASI_AIUS,so1);	/* 0B0  spill 4 other */
1565	NOT4;				/* 0B4  spill 5 other */
1566	NOT4;				/* 0B8  spill 6 other */
1567	NOT4;				/* 0BC  spill 7 other */
1568	NOT4;				/* 0C0	fill 0 normal */
1569	FILL_32bit_tt1(ASI_AIUP,fn1);	/* 0C4	fill 1 normal */
1570	FILL_64bit_tt1(ASI_AIUP,fn1);	/* 0C8	fill 2 normal */
1571	FILL_32bit_tt1(ASI_AIUP,fn1);	/* 0CC	fill 3 normal */
1572	FILL_64bit_tt1(ASI_AIUP,fn1);	/* 0D0	fill 4 normal */
1573	FILL_32bit(not);		/* 0D4	fill 5 normal */
1574	FILL_64bit(not);		/* 0D8	fill 6 normal */
1575	FILL_mixed;			/* 0DC	fill 7 normal */
1576	NOT4; NOT4; NOT4; NOT4;		/* 0E0 - 0EF unused */
1577	NOT4; NOT4; NOT4; NOT4;		/* 0F0 - 0FF unused */
1578	LABELED_BAD(tt1_swtrap0);	/* 100  fast ecache ECC error (cont) */
1579	LABELED_BAD(tt1_swtrap1);	/* 101  Ch+ D$ parity error (cont) */
1580	LABELED_BAD(tt1_swtrap2);	/* 102  Ch+ I$ parity error (cont) */
1581	NOT;				/* 103  reserved */
1582/*
1583 * We only reserve the above four special case soft traps for code running
1584 * at TL>0, so we can truncate the trap table here.
1585 */
1586etrap_table:
1587	.size	trap_table, (.-trap_table)
1588	.size	scb, (.-scb)
1589
1590/*
1591 * We get to exec_fault in the case of an instruction miss and tte
1592 * has no execute bit set.  We go to tl0 to handle it.
1593 *
1594 * g1 = tsbe pointer (in/clobbered)
1595 * g2 = tag access register (in)
1596 * g3 - g4 = scratch (clobbered)
1597 * g5 = tsbe data (in)
1598 * g6 = scratch (clobbered)
1599 */
1600	ALTENTRY(exec_fault)
1601	TRACE_TSBHIT(0x200)
1602	SWITCH_GLOBALS
1603	mov	MMU_TAG_ACCESS, %g4
1604	ldxa	[%g4]ASI_IMMU, %g2			! arg1 = addr
1605	mov	T_INSTR_MMU_MISS, %g3			! arg2 = traptype
1606	set	trap, %g1
1607	ba,pt	%xcc, sys_trap
1608	  mov	-1, %g4
1609
1610.mmu_exception_not_aligned:
1611	rdpr	%tstate, %g1
1612	btst	TSTATE_PRIV, %g1
1613	bnz,pn	%icc, 2f
1614	nop
1615	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1616	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1617	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1618	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1619	brz,pt	%g5, 2f
1620	nop
1621	ldn	[%g5 + P_UTRAP15], %g5			! unaligned utrap?
1622	brz,pn	%g5, 2f
1623	nop
1624	btst	1, %sp
1625	bz,pt	%xcc, 1f				! 32 bit user program
1626	nop
1627	ba,pt	%xcc, .setup_v9utrap			! 64 bit user program
1628	nop
16291:
1630	ba,pt	%xcc, .setup_utrap
1631	or	%g2, %g0, %g7
16322:
1633	ba,pt	%xcc, .mmu_exception_end
1634	mov	T_ALIGNMENT, %g1
1635
1636.mmu_priv_exception:
1637	rdpr	%tstate, %g1
1638	btst	TSTATE_PRIV, %g1
1639	bnz,pn	%icc, 1f
1640	nop
1641	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1642	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1643	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1644	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1645	brz,pt	%g5, 1f
1646	nop
1647	ldn	[%g5 + P_UTRAP16], %g5
1648	brnz,pt	%g5, .setup_v9utrap
1649	nop
16501:
1651	mov	T_PRIV_INSTR, %g1
1652
1653.mmu_exception_end:
1654	CPU_INDEX(%g4, %g5)
1655	set	cpu_core, %g5
1656	sllx	%g4, CPU_CORE_SHIFT, %g4
1657	add	%g4, %g5, %g4
1658	lduh	[%g4 + CPUC_DTRACE_FLAGS], %g5
1659	andcc	%g5, CPU_DTRACE_NOFAULT, %g0
1660	bz	%xcc, .mmu_exception_tlb_chk
1661	or	%g5, CPU_DTRACE_BADADDR, %g5
1662	stuh	%g5, [%g4 + CPUC_DTRACE_FLAGS]
1663	done
1664
1665.mmu_exception_tlb_chk:
1666	GET_CPU_IMPL(%g5)			! check SFSR.FT to see if this
1667	cmp	%g5, PANTHER_IMPL		! is a TLB parity error. But
1668	bne	2f				! we only do this check while
1669	mov	1, %g4				! running on Panther CPUs
1670	sllx	%g4, PN_SFSR_PARITY_SHIFT, %g4	! since US-I/II use the same
1671	andcc	%g3, %g4, %g0			! bit for something else which
1672	bz	2f				! will be handled later.
1673	nop
1674.mmu_exception_is_tlb_parity:
1675	.weak itlb_parity_trap
1676	.weak dtlb_parity_trap
1677	set	itlb_parity_trap, %g4
1678	cmp	%g1, T_INSTR_EXCEPTION		! branch to the itlb or
1679	be	3f				! dtlb parity handler
1680	nop					! if this trap is due
1681	set	dtlb_parity_trap, %g4
1682	cmp	%g1, T_DATA_EXCEPTION		! to a IMMU exception
1683	be	3f				! or DMMU exception.
1684	nop
16852:
1686	sllx	%g3, 32, %g3
1687	or	%g3, %g1, %g3
1688	set	trap, %g1
1689	ba,pt	%xcc, sys_trap
1690	sub	%g0, 1, %g4
16913:
1692	jmp	%g4				! off to the appropriate
1693	nop					! TLB parity handler
1694
1695.fp_disabled:
1696	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1697	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1698#ifdef SF_ERRATA_30 /* call causes fp-disabled */
1699	brz,a,pn %g1, 2f
1700	  nop
1701#endif
1702	rdpr	%tstate, %g4
1703	btst	TSTATE_PRIV, %g4
1704#ifdef SF_ERRATA_30 /* call causes fp-disabled */
1705	bnz,pn %icc, 2f
1706	  nop
1707#else
1708	bnz,a,pn %icc, ptl1_panic
1709	  mov	PTL1_BAD_FPTRAP, %g1
1710#endif
1711	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1712	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1713	brz,a,pt %g5, 2f
1714	  nop
1715	ldn	[%g5 + P_UTRAP7], %g5			! fp_disabled utrap?
1716	brz,a,pn %g5, 2f
1717	  nop
1718	btst	1, %sp
1719	bz,a,pt	%xcc, 1f				! 32 bit user program
1720	  nop
1721	ba,a,pt	%xcc, .setup_v9utrap			! 64 bit user program
1722	  nop
17231:
1724	ba,pt	%xcc, .setup_utrap
1725	  or	%g0, %g0, %g7
17262:
1727	set	fp_disabled, %g1
1728	ba,pt	%xcc, sys_trap
1729	  sub	%g0, 1, %g4
1730
1731.fp_ieee_exception:
1732	rdpr	%tstate, %g1
1733	btst	TSTATE_PRIV, %g1
1734	bnz,a,pn %icc, ptl1_panic
1735	  mov	PTL1_BAD_FPTRAP, %g1
1736	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1737	stx	%fsr, [%g1 + CPU_TMP1]
1738	ldx	[%g1 + CPU_TMP1], %g2
1739	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1740	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1741	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1742	brz,a,pt %g5, 1f
1743	  nop
1744	ldn	[%g5 + P_UTRAP8], %g5
1745	brnz,a,pt %g5, .setup_v9utrap
1746	  nop
17471:
1748	set	_fp_ieee_exception, %g1
1749	ba,pt	%xcc, sys_trap
1750	  sub	%g0, 1, %g4
1751
1752/*
1753 * Register Inputs:
1754 *	%g5		user trap handler
1755 *	%g7		misaligned addr - for alignment traps only
1756 */
1757.setup_utrap:
1758	set	trap, %g1			! setup in case we go
1759	mov	T_FLUSH_PCB, %g3		! through sys_trap on
1760	sub	%g0, 1, %g4			! the save instruction below
1761
1762	/*
1763	 * If the DTrace pid provider is single stepping a copied-out
1764	 * instruction, t->t_dtrace_step will be set. In that case we need
1765	 * to abort the single-stepping (since execution of the instruction
1766	 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1767	 */
1768	save	%sp, -SA(MINFRAME32), %sp	! window for trap handler
1769	CPU_ADDR(%g1, %g4)			! load CPU struct addr
1770	ldn	[%g1 + CPU_THREAD], %g1		! load thread pointer
1771	ldub	[%g1 + T_DTRACE_STEP], %g2	! load t->t_dtrace_step
1772	rdpr	%tnpc, %l2			! arg1 == tnpc
1773	brz,pt	%g2, 1f
1774	rdpr	%tpc, %l1			! arg0 == tpc
1775
1776	ldub	[%g1 + T_DTRACE_AST], %g2	! load t->t_dtrace_ast
1777	ldn	[%g1 + T_DTRACE_NPC], %l2	! arg1 = t->t_dtrace_npc (step)
1778	brz,pt	%g2, 1f
1779	st	%g0, [%g1 + T_DTRACE_FT]	! zero all pid provider flags
1780	stub	%g2, [%g1 + T_ASTFLAG]		! aston(t) if t->t_dtrace_ast
17811:
1782	mov	%g7, %l3			! arg2 == misaligned address
1783
1784	rdpr	%tstate, %g1			! cwp for trap handler
1785	rdpr	%cwp, %g4
1786	bclr	TSTATE_CWP_MASK, %g1
1787	wrpr	%g1, %g4, %tstate
1788	wrpr	%g0, %g5, %tnpc			! trap handler address
1789	FAST_TRAP_DONE
1790	/* NOTREACHED */
1791
1792.check_v9utrap:
1793	rdpr	%tstate, %g1
1794	btst	TSTATE_PRIV, %g1
1795	bnz,a,pn %icc, 3f
1796	  nop
1797	CPU_ADDR(%g4, %g1)				! load CPU struct addr
1798	ldn	[%g4 + CPU_THREAD], %g5			! load thread pointer
1799	ldn	[%g5 + T_PROCP], %g5			! load proc pointer
1800	ldn	[%g5 + P_UTRAPS], %g5			! are there utraps?
1801
1802	cmp	%g3, T_SOFTWARE_TRAP
1803	bne,a,pt %icc, 1f
1804	  nop
1805
1806	brz,pt %g5, 3f			! if p_utraps == NULL goto trap()
1807	  rdpr	%tt, %g3		! delay - get actual hw trap type
1808
1809	sub	%g3, 254, %g1		! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
1810	ba,pt	%icc, 2f
1811	  smul	%g1, CPTRSIZE, %g2
18121:
1813	brz,a,pt %g5, 3f		! if p_utraps == NULL goto trap()
1814	  nop
1815
1816	cmp	%g3, T_UNIMP_INSTR
1817	bne,a,pt %icc, 2f
1818	  nop
1819
1820	mov	1, %g1
1821	st	%g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
1822	rdpr	%tpc, %g1		! ld trapping instruction using
1823	lduwa	[%g1]ASI_AIUP, %g1	! "AS IF USER" ASI which could fault
1824	st	%g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
1825
1826	sethi	%hi(0xc1c00000), %g4	! setup mask for illtrap instruction
1827	andcc	%g1, %g4, %g4		! and instruction with mask
1828	bnz,a,pt %icc, 3f		! if %g4 == zero, %g1 is an ILLTRAP
1829	  nop				! fall thru to setup
18302:
1831	ldn	[%g5 + %g2], %g5
1832	brnz,a,pt %g5, .setup_v9utrap
1833	  nop
18343:
1835	set	trap, %g1
1836	ba,pt	%xcc, sys_trap
1837	  sub	%g0, 1, %g4
1838	/* NOTREACHED */
1839
1840/*
1841 * Register Inputs:
1842 *	%g5		user trap handler
1843 */
1844.setup_v9utrap:
1845	set	trap, %g1			! setup in case we go
1846	mov	T_FLUSH_PCB, %g3		! through sys_trap on
1847	sub	%g0, 1, %g4			! the save instruction below
1848
1849	/*
1850	 * If the DTrace pid provider is single stepping a copied-out
1851	 * instruction, t->t_dtrace_step will be set. In that case we need
1852	 * to abort the single-stepping (since execution of the instruction
1853	 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1854	 */
1855	save	%sp, -SA(MINFRAME64), %sp	! window for trap handler
1856	CPU_ADDR(%g1, %g4)			! load CPU struct addr
1857	ldn	[%g1 + CPU_THREAD], %g1		! load thread pointer
1858	ldub	[%g1 + T_DTRACE_STEP], %g2	! load t->t_dtrace_step
1859	rdpr	%tnpc, %l7			! arg1 == tnpc
1860	brz,pt	%g2, 1f
1861	rdpr	%tpc, %l6			! arg0 == tpc
1862
1863	ldub	[%g1 + T_DTRACE_AST], %g2	! load t->t_dtrace_ast
1864	ldn	[%g1 + T_DTRACE_NPC], %l7	! arg1 == t->t_dtrace_npc (step)
1865	brz,pt	%g2, 1f
1866	st	%g0, [%g1 + T_DTRACE_FT]	! zero all pid provider flags
1867	stub	%g2, [%g1 + T_ASTFLAG]		! aston(t) if t->t_dtrace_ast
18681:
1869	rdpr	%tstate, %g2			! cwp for trap handler
1870	rdpr	%cwp, %g4
1871	bclr	TSTATE_CWP_MASK, %g2
1872	wrpr	%g2, %g4, %tstate
1873
1874	ldn	[%g1 + T_PROCP], %g4		! load proc pointer
1875	ldn	[%g4 + P_AS], %g4		! load as pointer
1876	ldn	[%g4 + A_USERLIMIT], %g4	! load as userlimit
1877	cmp	%l7, %g4			! check for single-step set
1878	bne,pt	%xcc, 4f
1879	  nop
1880	ldn	[%g1 + T_LWP], %g1		! load klwp pointer
1881	ld	[%g1 + PCB_STEP], %g4		! load single-step flag
1882	cmp	%g4, STEP_ACTIVE		! step flags set in pcb?
1883	bne,pt	%icc, 4f
1884	  nop
1885	stn	%g5, [%g1 + PCB_TRACEPC]	! save trap handler addr in pcb
1886	mov	%l7, %g4			! on entry to precise user trap
1887	add	%l6, 4, %l7			! handler, %l6 == pc, %l7 == npc
1888						! at time of trap
1889	wrpr	%g0, %g4, %tnpc			! generate FLTBOUNDS,
1890						! %g4 == userlimit
1891	FAST_TRAP_DONE
1892	/* NOTREACHED */
18934:
1894	wrpr	%g0, %g5, %tnpc			! trap handler address
1895	FAST_TRAP_DONE_CHK_INTR
1896	/* NOTREACHED */
1897
1898.fp_exception:
1899	CPU_ADDR(%g1, %g4)
1900	stx	%fsr, [%g1 + CPU_TMP1]
1901	ldx	[%g1 + CPU_TMP1], %g2
1902
1903	/*
1904	 * Cheetah takes unfinished_FPop trap for certain range of operands
1905	 * to the "fitos" instruction. Instead of going through the slow
1906	 * software emulation path, we try to simulate the "fitos" instruction
1907	 * via "fitod" and "fdtos" provided the following conditions are met:
1908	 *
1909	 *	fpu_exists is set (if DEBUG)
1910	 *	not in privileged mode
1911	 *	ftt is unfinished_FPop
1912	 *	NXM IEEE trap is not enabled
1913	 *	instruction at %tpc is "fitos"
1914	 *
1915	 *  Usage:
1916	 *	%g1	per cpu address
1917	 *	%g2	%fsr
1918	 *	%g6	user instruction
1919	 *
1920	 * Note that we can take a memory access related trap while trying
1921	 * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
1922	 * flag to catch those traps and let the SFMMU code deal with page
1923	 * fault and data access exception.
1924	 */
1925#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
1926	sethi	%hi(fpu_exists), %g7
1927	ld	[%g7 + %lo(fpu_exists)], %g7
1928	brz,pn %g7, .fp_exception_cont
1929	  nop
1930#endif
1931	rdpr	%tstate, %g7			! branch if in privileged mode
1932	btst	TSTATE_PRIV, %g7
1933	bnz,pn	%xcc, .fp_exception_cont
1934	srl	%g2, FSR_FTT_SHIFT, %g7		! extract ftt from %fsr
1935	and	%g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
1936	cmp	%g7, FTT_UNFIN
1937	set	FSR_TEM_NX, %g5
1938	bne,pn	%xcc, .fp_exception_cont	! branch if NOT unfinished_FPop
1939	  andcc	%g2, %g5, %g0
1940	bne,pn	%xcc, .fp_exception_cont	! branch if FSR_TEM_NX enabled
1941	  rdpr	%tpc, %g5			! get faulting PC
1942
1943	or	%g0, 1, %g7
1944	st	%g7, [%g1 + CPU_TL1_HDLR]	! set tl1_hdlr flag
1945	lda	[%g5]ASI_USER, %g6		! get user's instruction
1946	st	%g0, [%g1 + CPU_TL1_HDLR]	! clear tl1_hdlr flag
1947
1948	set	FITOS_INSTR_MASK, %g7
1949	and	%g6, %g7, %g7
1950	set	FITOS_INSTR, %g5
1951	cmp	%g7, %g5
1952	bne,pn	%xcc, .fp_exception_cont	! branch if not FITOS_INSTR
1953	 nop
1954
1955	/*
1956	 * This is unfinished FPops trap for "fitos" instruction. We
1957	 * need to simulate "fitos" via "fitod" and "fdtos" instruction
1958	 * sequence.
1959	 *
1960	 * We need a temporary FP register to do the conversion. Since
1961	 * both source and destination operands for the "fitos" instruction
1962	 * have to be within %f0-%f31, we use an FP register from the upper
1963	 * half to guarantee that it won't collide with the source or the
1964	 * dest operand. However, we do have to save and restore its value.
1965	 *
1966	 * We use %d62 as a temporary FP register for the conversion and
1967	 * branch to appropriate instruction within the conversion tables
1968	 * based upon the rs2 and rd values.
1969	 */
1970
1971	std	%d62, [%g1 + CPU_TMP1]		! save original value
1972
1973	srl	%g6, FITOS_RS2_SHIFT, %g7
1974	and	%g7, FITOS_REG_MASK, %g7
1975	set	_fitos_fitod_table, %g4
1976	sllx	%g7, 2, %g7
1977	jmp	%g4 + %g7
1978	  ba,pt	%xcc, _fitos_fitod_done
1979	.empty
1980
1981_fitos_fitod_table:
1982	  fitod	%f0, %d62
1983	  fitod	%f1, %d62
1984	  fitod	%f2, %d62
1985	  fitod	%f3, %d62
1986	  fitod	%f4, %d62
1987	  fitod	%f5, %d62
1988	  fitod	%f6, %d62
1989	  fitod	%f7, %d62
1990	  fitod	%f8, %d62
1991	  fitod	%f9, %d62
1992	  fitod	%f10, %d62
1993	  fitod	%f11, %d62
1994	  fitod	%f12, %d62
1995	  fitod	%f13, %d62
1996	  fitod	%f14, %d62
1997	  fitod	%f15, %d62
1998	  fitod	%f16, %d62
1999	  fitod	%f17, %d62
2000	  fitod	%f18, %d62
2001	  fitod	%f19, %d62
2002	  fitod	%f20, %d62
2003	  fitod	%f21, %d62
2004	  fitod	%f22, %d62
2005	  fitod	%f23, %d62
2006	  fitod	%f24, %d62
2007	  fitod	%f25, %d62
2008	  fitod	%f26, %d62
2009	  fitod	%f27, %d62
2010	  fitod	%f28, %d62
2011	  fitod	%f29, %d62
2012	  fitod	%f30, %d62
2013	  fitod	%f31, %d62
2014_fitos_fitod_done:
2015
2016	/*
2017	 * Now convert data back into single precision
2018	 */
2019	srl	%g6, FITOS_RD_SHIFT, %g7
2020	and	%g7, FITOS_REG_MASK, %g7
2021	set	_fitos_fdtos_table, %g4
2022	sllx	%g7, 2, %g7
2023	jmp	%g4 + %g7
2024	  ba,pt	%xcc, _fitos_fdtos_done
2025	.empty
2026
2027_fitos_fdtos_table:
2028	  fdtos	%d62, %f0
2029	  fdtos	%d62, %f1
2030	  fdtos	%d62, %f2
2031	  fdtos	%d62, %f3
2032	  fdtos	%d62, %f4
2033	  fdtos	%d62, %f5
2034	  fdtos	%d62, %f6
2035	  fdtos	%d62, %f7
2036	  fdtos	%d62, %f8
2037	  fdtos	%d62, %f9
2038	  fdtos	%d62, %f10
2039	  fdtos	%d62, %f11
2040	  fdtos	%d62, %f12
2041	  fdtos	%d62, %f13
2042	  fdtos	%d62, %f14
2043	  fdtos	%d62, %f15
2044	  fdtos	%d62, %f16
2045	  fdtos	%d62, %f17
2046	  fdtos	%d62, %f18
2047	  fdtos	%d62, %f19
2048	  fdtos	%d62, %f20
2049	  fdtos	%d62, %f21
2050	  fdtos	%d62, %f22
2051	  fdtos	%d62, %f23
2052	  fdtos	%d62, %f24
2053	  fdtos	%d62, %f25
2054	  fdtos	%d62, %f26
2055	  fdtos	%d62, %f27
2056	  fdtos	%d62, %f28
2057	  fdtos	%d62, %f29
2058	  fdtos	%d62, %f30
2059	  fdtos	%d62, %f31
2060_fitos_fdtos_done:
2061
2062	ldd	[%g1 + CPU_TMP1], %d62		! restore %d62
2063
2064#if DEBUG
2065	/*
2066	 * Update FPop_unfinished trap kstat
2067	 */
2068	set	fpustat+FPUSTAT_UNFIN_KSTAT, %g7
2069	ldx	[%g7], %g5
20701:
2071	add	%g5, 1, %g6
2072
2073	casxa	[%g7] ASI_N, %g5, %g6
2074	cmp	%g5, %g6
2075	bne,a,pn %xcc, 1b
2076	  or	%g0, %g6, %g5
2077
2078	/*
2079	 * Update fpu_sim_fitos kstat
2080	 */
2081	set	fpuinfo+FPUINFO_FITOS_KSTAT, %g7
2082	ldx	[%g7], %g5
20831:
2084	add	%g5, 1, %g6
2085
2086	casxa	[%g7] ASI_N, %g5, %g6
2087	cmp	%g5, %g6
2088	bne,a,pn %xcc, 1b
2089	  or	%g0, %g6, %g5
2090#endif /* DEBUG */
2091
2092	FAST_TRAP_DONE
2093
2094.fp_exception_cont:
2095	/*
2096	 * Let _fp_exception deal with simulating FPop instruction.
2097	 * Note that we need to pass %fsr in %g2 (already read above).
2098	 */
2099
2100	set	_fp_exception, %g1
2101	ba,pt	%xcc, sys_trap
2102	sub	%g0, 1, %g4
2103
2104	.global opl_cleanw_patch
2105opl_cleanw_patch:
2106.clean_windows:
2107	set	trap, %g1
2108	mov	T_FLUSH_PCB, %g3
2109	sub	%g0, 1, %g4
2110	save
2111	flushw
2112	restore
2113	wrpr	%g0, %g0, %cleanwin	! no clean windows
2114
2115	CPU_ADDR(%g4, %g5)
2116	ldn	[%g4 + CPU_MPCB], %g4
2117	brz,a,pn %g4, 1f
2118	  nop
2119	ld	[%g4 + MPCB_WSTATE], %g5
2120	add	%g5, WSTATE_CLEAN_OFFSET, %g5
2121	wrpr	%g0, %g5, %wstate
21221:	FAST_TRAP_DONE
2123
2124/*
2125 * .spill_clean: clean the previous window, restore the wstate, and
2126 * "done".
2127 *
2128 * Entry: %g7 contains new wstate
2129 */
2130.spill_clean:
2131	sethi	%hi(nwin_minus_one), %g5
2132	ld	[%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
2133	rdpr	%cwp, %g6			! %g6 = %cwp
2134	deccc	%g6				! %g6--
2135	movneg	%xcc, %g5, %g6			! if (%g6<0) %g6 = nwin-1
2136	wrpr	%g6, %cwp
2137	TT_TRACE_L(trace_win)
2138	clr	%l0
2139	clr	%l1
2140	clr	%l2
2141	clr	%l3
2142	clr	%l4
2143	clr	%l5
2144	clr	%l6
2145	clr	%l7
2146	wrpr	%g0, %g7, %wstate
2147	saved
2148	retry			! restores correct %cwp
2149
2150.fix_alignment:
2151	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2152	ldn	[%g1 + CPU_THREAD], %g1	! load thread pointer
2153	ldn	[%g1 + T_PROCP], %g1
2154	mov	1, %g2
2155	stb	%g2, [%g1 + P_FIXALIGNMENT]
2156	FAST_TRAP_DONE
2157
2158#define	STDF_REG(REG, ADDR, TMP)		\
2159	sll	REG, 3, REG			;\
2160mark1:	set	start1, TMP			;\
2161	jmp	REG + TMP			;\
2162	  nop					;\
2163start1:	ba,pt	%xcc, done1			;\
2164	  std	%f0, [ADDR + CPU_TMP1]		;\
2165	ba,pt	%xcc, done1			;\
2166	  std	%f32, [ADDR + CPU_TMP1]		;\
2167	ba,pt	%xcc, done1			;\
2168	  std	%f2, [ADDR + CPU_TMP1]		;\
2169	ba,pt	%xcc, done1			;\
2170	  std	%f34, [ADDR + CPU_TMP1]		;\
2171	ba,pt	%xcc, done1			;\
2172	  std	%f4, [ADDR + CPU_TMP1]		;\
2173	ba,pt	%xcc, done1			;\
2174	  std	%f36, [ADDR + CPU_TMP1]		;\
2175	ba,pt	%xcc, done1			;\
2176	  std	%f6, [ADDR + CPU_TMP1]		;\
2177	ba,pt	%xcc, done1			;\
2178	  std	%f38, [ADDR + CPU_TMP1]		;\
2179	ba,pt	%xcc, done1			;\
2180	  std	%f8, [ADDR + CPU_TMP1]		;\
2181	ba,pt	%xcc, done1			;\
2182	  std	%f40, [ADDR + CPU_TMP1]		;\
2183	ba,pt	%xcc, done1			;\
2184	  std	%f10, [ADDR + CPU_TMP1]		;\
2185	ba,pt	%xcc, done1			;\
2186	  std	%f42, [ADDR + CPU_TMP1]		;\
2187	ba,pt	%xcc, done1			;\
2188	  std	%f12, [ADDR + CPU_TMP1]		;\
2189	ba,pt	%xcc, done1			;\
2190	  std	%f44, [ADDR + CPU_TMP1]		;\
2191	ba,pt	%xcc, done1			;\
2192	  std	%f14, [ADDR + CPU_TMP1]		;\
2193	ba,pt	%xcc, done1			;\
2194	  std	%f46, [ADDR + CPU_TMP1]		;\
2195	ba,pt	%xcc, done1			;\
2196	  std	%f16, [ADDR + CPU_TMP1]		;\
2197	ba,pt	%xcc, done1			;\
2198	  std	%f48, [ADDR + CPU_TMP1]		;\
2199	ba,pt	%xcc, done1			;\
2200	  std	%f18, [ADDR + CPU_TMP1]		;\
2201	ba,pt	%xcc, done1			;\
2202	  std	%f50, [ADDR + CPU_TMP1]		;\
2203	ba,pt	%xcc, done1			;\
2204	  std	%f20, [ADDR + CPU_TMP1]		;\
2205	ba,pt	%xcc, done1			;\
2206	  std	%f52, [ADDR + CPU_TMP1]		;\
2207	ba,pt	%xcc, done1			;\
2208	  std	%f22, [ADDR + CPU_TMP1]		;\
2209	ba,pt	%xcc, done1			;\
2210	  std	%f54, [ADDR + CPU_TMP1]		;\
2211	ba,pt	%xcc, done1			;\
2212	  std	%f24, [ADDR + CPU_TMP1]		;\
2213	ba,pt	%xcc, done1			;\
2214	  std	%f56, [ADDR + CPU_TMP1]		;\
2215	ba,pt	%xcc, done1			;\
2216	  std	%f26, [ADDR + CPU_TMP1]		;\
2217	ba,pt	%xcc, done1			;\
2218	  std	%f58, [ADDR + CPU_TMP1]		;\
2219	ba,pt	%xcc, done1			;\
2220	  std	%f28, [ADDR + CPU_TMP1]		;\
2221	ba,pt	%xcc, done1			;\
2222	  std	%f60, [ADDR + CPU_TMP1]		;\
2223	ba,pt	%xcc, done1			;\
2224	  std	%f30, [ADDR + CPU_TMP1]		;\
2225	ba,pt	%xcc, done1			;\
2226	  std	%f62, [ADDR + CPU_TMP1]		;\
2227done1:
2228
2229#define	LDDF_REG(REG, ADDR, TMP)		\
2230	sll	REG, 3, REG			;\
2231mark2:	set	start2, TMP			;\
2232	jmp	REG + TMP			;\
2233	  nop					;\
2234start2:	ba,pt	%xcc, done2			;\
2235	  ldd	[ADDR + CPU_TMP1], %f0		;\
2236	ba,pt	%xcc, done2			;\
2237	  ldd	[ADDR + CPU_TMP1], %f32		;\
2238	ba,pt	%xcc, done2			;\
2239	  ldd	[ADDR + CPU_TMP1], %f2		;\
2240	ba,pt	%xcc, done2			;\
2241	  ldd	[ADDR + CPU_TMP1], %f34		;\
2242	ba,pt	%xcc, done2			;\
2243	  ldd	[ADDR + CPU_TMP1], %f4		;\
2244	ba,pt	%xcc, done2			;\
2245	  ldd	[ADDR + CPU_TMP1], %f36		;\
2246	ba,pt	%xcc, done2			;\
2247	  ldd	[ADDR + CPU_TMP1], %f6		;\
2248	ba,pt	%xcc, done2			;\
2249	  ldd	[ADDR + CPU_TMP1], %f38		;\
2250	ba,pt	%xcc, done2			;\
2251	  ldd	[ADDR + CPU_TMP1], %f8		;\
2252	ba,pt	%xcc, done2			;\
2253	  ldd	[ADDR + CPU_TMP1], %f40		;\
2254	ba,pt	%xcc, done2			;\
2255	  ldd	[ADDR + CPU_TMP1], %f10		;\
2256	ba,pt	%xcc, done2			;\
2257	  ldd	[ADDR + CPU_TMP1], %f42		;\
2258	ba,pt	%xcc, done2			;\
2259	  ldd	[ADDR + CPU_TMP1], %f12		;\
2260	ba,pt	%xcc, done2			;\
2261	  ldd	[ADDR + CPU_TMP1], %f44		;\
2262	ba,pt	%xcc, done2			;\
2263	  ldd	[ADDR + CPU_TMP1], %f14		;\
2264	ba,pt	%xcc, done2			;\
2265	  ldd	[ADDR + CPU_TMP1], %f46		;\
2266	ba,pt	%xcc, done2			;\
2267	  ldd	[ADDR + CPU_TMP1], %f16		;\
2268	ba,pt	%xcc, done2			;\
2269	  ldd	[ADDR + CPU_TMP1], %f48		;\
2270	ba,pt	%xcc, done2			;\
2271	  ldd	[ADDR + CPU_TMP1], %f18		;\
2272	ba,pt	%xcc, done2			;\
2273	  ldd	[ADDR + CPU_TMP1], %f50		;\
2274	ba,pt	%xcc, done2			;\
2275	  ldd	[ADDR + CPU_TMP1], %f20		;\
2276	ba,pt	%xcc, done2			;\
2277	  ldd	[ADDR + CPU_TMP1], %f52		;\
2278	ba,pt	%xcc, done2			;\
2279	  ldd	[ADDR + CPU_TMP1], %f22		;\
2280	ba,pt	%xcc, done2			;\
2281	  ldd	[ADDR + CPU_TMP1], %f54		;\
2282	ba,pt	%xcc, done2			;\
2283	  ldd	[ADDR + CPU_TMP1], %f24		;\
2284	ba,pt	%xcc, done2			;\
2285	  ldd	[ADDR + CPU_TMP1], %f56		;\
2286	ba,pt	%xcc, done2			;\
2287	  ldd	[ADDR + CPU_TMP1], %f26		;\
2288	ba,pt	%xcc, done2			;\
2289	  ldd	[ADDR + CPU_TMP1], %f58		;\
2290	ba,pt	%xcc, done2			;\
2291	  ldd	[ADDR + CPU_TMP1], %f28		;\
2292	ba,pt	%xcc, done2			;\
2293	  ldd	[ADDR + CPU_TMP1], %f60		;\
2294	ba,pt	%xcc, done2			;\
2295	  ldd	[ADDR + CPU_TMP1], %f30		;\
2296	ba,pt	%xcc, done2			;\
2297	  ldd	[ADDR + CPU_TMP1], %f62		;\
2298done2:
2299
2300.lddf_exception_not_aligned:
2301	/*
2302	 * Cheetah overwrites SFAR on a DTLB miss, hence read it now.
2303	 */
2304	ldxa	[MMU_SFAR]%asi, %g5	! misaligned vaddr in %g5
2305
2306#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2307	sethi	%hi(fpu_exists), %g2		! check fpu_exists
2308	ld	[%g2 + %lo(fpu_exists)], %g2
2309	brz,a,pn %g2, 4f
2310	  nop
2311#endif
2312	CPU_ADDR(%g1, %g4)
2313	or	%g0, 1, %g4
2314	st	%g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2315
2316	rdpr	%tpc, %g2
2317	lda	[%g2]ASI_AIUP, %g6	! get the user's lddf instruction
2318	srl	%g6, 23, %g1		! using ldda or not?
2319	and	%g1, 1, %g1
2320	brz,a,pt %g1, 2f		! check for ldda instruction
2321	  nop
2322	srl	%g6, 13, %g1		! check immflag
2323	and	%g1, 1, %g1
2324	rdpr	%tstate, %g2		! %tstate in %g2
2325	brnz,a,pn %g1, 1f
2326	  srl	%g2, 31, %g1		! get asi from %tstate
2327	srl	%g6, 5, %g1		! get asi from instruction
2328	and	%g1, 0xFF, %g1		! imm_asi field
23291:
2330	cmp	%g1, ASI_P		! primary address space
2331	be,a,pt %icc, 2f
2332	  nop
2333	cmp	%g1, ASI_PNF		! primary no fault address space
2334	be,a,pt %icc, 2f
2335	  nop
2336	cmp	%g1, ASI_S		! secondary address space
2337	be,a,pt %icc, 2f
2338	  nop
2339	cmp	%g1, ASI_SNF		! secondary no fault address space
2340	bne,a,pn %icc, 3f
2341	  nop
23422:
2343	lduwa	[%g5]ASI_USER, %g7	! get first half of misaligned data
2344	add	%g5, 4, %g5		! increment misaligned data address
2345	lduwa	[%g5]ASI_USER, %g5	! get second half of misaligned data
2346
2347	sllx	%g7, 32, %g7
2348	or	%g5, %g7, %g5		! combine data
2349	CPU_ADDR(%g7, %g1)		! save data on a per-cpu basis
2350	stx	%g5, [%g7 + CPU_TMP1]	! save in cpu_tmp1
2351
2352	srl	%g6, 25, %g3		! %g6 has the instruction
2353	and	%g3, 0x1F, %g3		! %g3 has rd
2354	LDDF_REG(%g3, %g7, %g4)
2355
2356	CPU_ADDR(%g1, %g4)
2357	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2358	FAST_TRAP_DONE
23593:
2360	CPU_ADDR(%g1, %g4)
2361	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
23624:
2363	set	T_USER, %g3		! trap type in %g3
2364	or	%g3, T_LDDF_ALIGN, %g3
2365	mov	%g5, %g2		! misaligned vaddr in %g2
2366	set	fpu_trap, %g1		! goto C for the little and
2367	ba,pt	%xcc, sys_trap		! no fault little asi's
2368	  sub	%g0, 1, %g4
2369
2370.stdf_exception_not_aligned:
2371	/*
2372	 * Cheetah overwrites SFAR on a DTLB miss, hence read it now.
2373	 */
2374	ldxa	[MMU_SFAR]%asi, %g5	! misaligned vaddr in %g5
2375
2376#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2377	sethi	%hi(fpu_exists), %g7		! check fpu_exists
2378	ld	[%g7 + %lo(fpu_exists)], %g3
2379	brz,a,pn %g3, 4f
2380	  nop
2381#endif
2382	CPU_ADDR(%g1, %g4)
2383	or	%g0, 1, %g4
2384	st	%g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2385
2386	rdpr	%tpc, %g2
2387	lda	[%g2]ASI_AIUP, %g6	! get the user's stdf instruction
2388
2389	srl	%g6, 23, %g1		! using stda or not?
2390	and	%g1, 1, %g1
2391	brz,a,pt %g1, 2f		! check for stda instruction
2392	  nop
2393	srl	%g6, 13, %g1		! check immflag
2394	and	%g1, 1, %g1
2395	rdpr	%tstate, %g2		! %tstate in %g2
2396	brnz,a,pn %g1, 1f
2397	  srl	%g2, 31, %g1		! get asi from %tstate
2398	srl	%g6, 5, %g1		! get asi from instruction
2399	and	%g1, 0xFF, %g1		! imm_asi field
24001:
2401	cmp	%g1, ASI_P		! primary address space
2402	be,a,pt %icc, 2f
2403	  nop
2404	cmp	%g1, ASI_S		! secondary address space
2405	bne,a,pn %icc, 3f
2406	  nop
24072:
2408	srl	%g6, 25, %g6
2409	and	%g6, 0x1F, %g6		! %g6 has rd
2410	CPU_ADDR(%g7, %g1)
2411	STDF_REG(%g6, %g7, %g4)		! STDF_REG(REG, ADDR, TMP)
2412
2413	ldx	[%g7 + CPU_TMP1], %g6
2414	srlx	%g6, 32, %g7
2415	stuwa	%g7, [%g5]ASI_USER	! first half
2416	add	%g5, 4, %g5		! increment misaligned data address
2417	stuwa	%g6, [%g5]ASI_USER	! second half
2418
2419	CPU_ADDR(%g1, %g4)
2420	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2421	FAST_TRAP_DONE
24223:
2423	CPU_ADDR(%g1, %g4)
2424	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
24254:
2426	set	T_USER, %g3		! trap type in %g3
2427	or	%g3, T_STDF_ALIGN, %g3
2428	mov	%g5, %g2		! misaligned vaddr in %g2
2429	set	fpu_trap, %g1		! goto C for the little and
2430	ba,pt	%xcc, sys_trap		! nofault little asi's
2431	  sub	%g0, 1, %g4
2432
2433#ifdef DEBUG_USER_TRAPTRACECTL
2434
2435.traptrace_freeze:
2436	mov	%l0, %g1 ; mov	%l1, %g2 ; mov	%l2, %g3 ; mov	%l4, %g4
2437	TT_TRACE_L(trace_win)
2438	mov	%g4, %l4 ; mov	%g3, %l2 ; mov	%g2, %l1 ; mov	%g1, %l0
2439	set	trap_freeze, %g1
2440	mov	1, %g2
2441	st	%g2, [%g1]
2442	FAST_TRAP_DONE
2443
2444.traptrace_unfreeze:
2445	set	trap_freeze, %g1
2446	st	%g0, [%g1]
2447	mov	%l0, %g1 ; mov	%l1, %g2 ; mov	%l2, %g3 ; mov	%l4, %g4
2448	TT_TRACE_L(trace_win)
2449	mov	%g4, %l4 ; mov	%g3, %l2 ; mov	%g2, %l1 ; mov	%g1, %l0
2450	FAST_TRAP_DONE
2451
2452#endif /* DEBUG_USER_TRAPTRACECTL */
2453
2454.getcc:
2455	CPU_ADDR(%g1, %g2)
2456	stx	%o0, [%g1 + CPU_TMP1]		! save %o0
2457	stx	%o1, [%g1 + CPU_TMP2]		! save %o1
2458	rdpr	%tstate, %g3			! get tstate
2459	srlx	%g3, PSR_TSTATE_CC_SHIFT, %o0	! shift ccr to V8 psr
2460	set	PSR_ICC, %g2
2461	and	%o0, %g2, %o0			! mask out the rest
2462	srl	%o0, PSR_ICC_SHIFT, %o0		! right justify
2463	rdpr	%pstate, %o1
2464	wrpr	%o1, PSTATE_AG, %pstate		! get into normal globals
2465	mov	%o0, %g1			! move ccr to normal %g1
2466	wrpr	%g0, %o1, %pstate		! back into alternate globals
2467	ldx	[%g1 + CPU_TMP1], %o0		! restore %o0
2468	ldx	[%g1 + CPU_TMP2], %o1		! restore %o1
2469	FAST_TRAP_DONE
2470
2471.setcc:
2472	CPU_ADDR(%g1, %g2)
2473	stx	%o0, [%g1 + CPU_TMP1]		! save %o0
2474	stx	%o1, [%g1 + CPU_TMP2]		! save %o1
2475	rdpr	%pstate, %o0
2476	wrpr	%o0, PSTATE_AG, %pstate		! get into normal globals
2477	mov	%g1, %o1
2478	wrpr	%g0, %o0, %pstate		! back to alternates
2479	sll	%o1, PSR_ICC_SHIFT, %g2
2480	set	PSR_ICC, %g3
2481	and	%g2, %g3, %g2			! mask out rest
2482	sllx	%g2, PSR_TSTATE_CC_SHIFT, %g2
2483	rdpr	%tstate, %g3			! get tstate
2484	srl	%g3, 0, %g3			! clear upper word
2485	or	%g3, %g2, %g3			! or in new bits
2486	wrpr	%g3, %tstate
2487	ldx	[%g1 + CPU_TMP1], %o0		! restore %o0
2488	ldx	[%g1 + CPU_TMP2], %o1		! restore %o1
2489	FAST_TRAP_DONE
2490
2491/*
2492 * getpsr(void)
2493 * Note that the xcc part of the ccr is not provided.
2494 * The V8 code shows why the V9 trap is not faster:
2495 * #define GETPSR_TRAP() \
2496 *      mov %psr, %i0; jmp %l2; rett %l2+4; nop;
2497 */
2498
2499	.type	.getpsr, #function
2500.getpsr:
2501	rdpr	%tstate, %g1			! get tstate
2502	srlx	%g1, PSR_TSTATE_CC_SHIFT, %o0	! shift ccr to V8 psr
2503	set	PSR_ICC, %g2
2504	and	%o0, %g2, %o0			! mask out the rest
2505
2506	rd	%fprs, %g1			! get fprs
2507	and	%g1, FPRS_FEF, %g2		! mask out dirty upper/lower
2508	sllx	%g2, PSR_FPRS_FEF_SHIFT, %g2	! shift fef to V8 psr.ef
2509	or	%o0, %g2, %o0			! or result into psr.ef
2510
2511	set	V9_PSR_IMPLVER, %g2		! SI assigned impl/ver: 0xef
2512	or	%o0, %g2, %o0			! or psr.impl/ver
2513	FAST_TRAP_DONE
2514	SET_SIZE(.getpsr)
2515
2516/*
2517 * setpsr(newpsr)
2518 * Note that there is no support for ccr.xcc in the V9 code.
2519 */
2520
2521	.type	.setpsr, #function
2522.setpsr:
2523	rdpr	%tstate, %g1			! get tstate
2524!	setx	TSTATE_V8_UBITS, %g2
2525	or 	%g0, CCR_ICC, %g3
2526	sllx	%g3, TSTATE_CCR_SHIFT, %g2
2527
2528	andn	%g1, %g2, %g1			! zero current user bits
2529	set	PSR_ICC, %g2
2530	and	%g2, %o0, %g2			! clear all but psr.icc bits
2531	sllx	%g2, PSR_TSTATE_CC_SHIFT, %g3	! shift to tstate.ccr.icc
2532	wrpr	%g1, %g3, %tstate		! write tstate
2533
2534	set	PSR_EF, %g2
2535	and	%g2, %o0, %g2			! clear all but fp enable bit
2536	srlx	%g2, PSR_FPRS_FEF_SHIFT, %g4	! shift ef to V9 fprs.fef
2537	wr	%g0, %g4, %fprs			! write fprs
2538
2539	CPU_ADDR(%g1, %g2)			! load CPU struct addr to %g1
2540	ldn	[%g1 + CPU_THREAD], %g2		! load thread pointer
2541	ldn	[%g2 + T_LWP], %g3		! load klwp pointer
2542	ldn	[%g3 + LWP_FPU], %g2		! get lwp_fpu pointer
2543	stuw	%g4, [%g2 + FPU_FPRS]		! write fef value to fpu_fprs
2544	srlx	%g4, 2, %g4			! shift fef value to bit 0
2545	stub	%g4, [%g2 + FPU_EN]		! write fef value to fpu_en
2546	FAST_TRAP_DONE
2547	SET_SIZE(.setpsr)
2548
2549/*
2550 * getlgrp
2551 * get home lgrpid on which the calling thread is currently executing.
2552 */
2553	.type	.getlgrp, #function
2554.getlgrp:
2555	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2556	ld	[%g1 + CPU_ID], %o0	! load cpu_id
2557	ldn	[%g1 + CPU_THREAD], %g2	! load thread pointer
2558	ldn	[%g2 + T_LPL], %g2	! load lpl pointer
2559	ld	[%g2 + LPL_LGRPID], %g1	! load lpl_lgrpid
2560	sra	%g1, 0, %o1
2561	FAST_TRAP_DONE
2562	SET_SIZE(.getlgrp)
2563
2564/*
2565 * Entry for old 4.x trap (trap 0).
2566 */
2567	ENTRY_NP(syscall_trap_4x)
2568	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2569	ldn	[%g1 + CPU_THREAD], %g2	! load thread pointer
2570	ldn	[%g2 + T_LWP], %g2	! load klwp pointer
2571	ld	[%g2 + PCB_TRAP0], %g2	! lwp->lwp_pcb.pcb_trap0addr
2572	brz,pn	%g2, 1f			! has it been set?
2573	st	%l0, [%g1 + CPU_TMP1]	! delay - save some locals
2574	st	%l1, [%g1 + CPU_TMP2]
2575	rdpr	%tnpc, %l1		! save old tnpc
2576	wrpr	%g0, %g2, %tnpc		! setup tnpc
2577
2578	rdpr	%pstate, %l0
2579	wrpr	%l0, PSTATE_AG, %pstate	! switch to normal globals
2580	mov	%l1, %g6		! pass tnpc to user code in %g6
2581	wrpr	%l0, %g0, %pstate	! switch back to alternate globals
2582
2583	! Note that %g1 still contains CPU struct addr
2584	ld	[%g1 + CPU_TMP2], %l1	! restore locals
2585	ld	[%g1 + CPU_TMP1], %l0
2586	FAST_TRAP_DONE_CHK_INTR
25871:
2588	mov	%g1, %l0
2589	st	%l1, [%g1 + CPU_TMP2]
2590	rdpr	%pstate, %l1
2591	wrpr	%l1, PSTATE_AG, %pstate
2592	!
2593	! check for old syscall mmap which is the only different one which
2594	! must be the same.  Others are handled in the compatibility library.
2595	!
2596	cmp	%g1, OSYS_mmap	! compare to old 4.x mmap
2597	movz	%icc, SYS_mmap, %g1
2598	wrpr	%g0, %l1, %pstate
2599	ld	[%l0 + CPU_TMP2], %l1	! restore locals
2600	ld	[%l0 + CPU_TMP1], %l0
2601	SYSCALL(syscall_trap32)
2602	SET_SIZE(syscall_trap_4x)
2603
2604/*
2605 * Handler for software trap 9.
2606 * Set trap0 emulation address for old 4.x system call trap.
2607 * XXX - this should be a system call.
2608 */
2609	ENTRY_NP(set_trap0_addr)
2610	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2611	ldn	[%g1 + CPU_THREAD], %g2	! load thread pointer
2612	ldn	[%g2 + T_LWP], %g2	! load klwp pointer
2613	st	%l0, [%g1 + CPU_TMP1]	! save some locals
2614	st	%l1, [%g1 + CPU_TMP2]
2615	rdpr	%pstate, %l0
2616	wrpr	%l0, PSTATE_AG, %pstate
2617	mov	%g1, %l1
2618	wrpr	%g0, %l0, %pstate
2619	andn	%l1, 3, %l1		! force alignment
2620	st	%l1, [%g2 + PCB_TRAP0]	! lwp->lwp_pcb.pcb_trap0addr
2621	ld	[%g1 + CPU_TMP1], %l0	! restore locals
2622	ld	[%g1 + CPU_TMP2], %l1
2623	FAST_TRAP_DONE
2624	SET_SIZE(set_trap0_addr)
2625
2626/*
2627 * mmu_trap_tl1
2628 * trap handler for unexpected mmu traps.
2629 * simply checks if the trap was a user lddf/stdf alignment trap, in which
2630 * case we go to fpu_trap or a user trap from the window handler, in which
2631 * case we go save the state on the pcb.  Otherwise, we go to ptl1_panic.
2632 */
2633	.type	mmu_trap_tl1, #function
2634mmu_trap_tl1:
2635#ifdef	TRAPTRACE
2636	TRACE_PTR(%g5, %g6)
2637	GET_TRACE_TICK(%g6, %g7)
2638	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi
2639	rdpr	%tl, %g6
2640	stha	%g6, [%g5 + TRAP_ENT_TL]%asi
2641	rdpr	%tt, %g6
2642	stha	%g6, [%g5 + TRAP_ENT_TT]%asi
2643	rdpr	%tstate, %g6
2644	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi
2645	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
2646	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
2647	rdpr	%tpc, %g6
2648	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi
2649	set	MMU_SFAR, %g6
2650	ldxa	[%g6]ASI_DMMU, %g6
2651	stxa	%g6, [%g5 + TRAP_ENT_F1]%asi
2652	CPU_PADDR(%g7, %g6);
2653	add	%g7, CPU_TL1_HDLR, %g7
2654	lda	[%g7]ASI_MEM, %g6
2655	stxa	%g6, [%g5 + TRAP_ENT_F2]%asi
2656	set	0xdeadbeef, %g6
2657	stna	%g6, [%g5 + TRAP_ENT_F3]%asi
2658	stna	%g6, [%g5 + TRAP_ENT_F4]%asi
2659	TRACE_NEXT(%g5, %g6, %g7)
2660#endif /* TRAPTRACE */
2661
2662	GET_CPU_IMPL(%g5)
2663	cmp	%g5, PANTHER_IMPL
2664	bne	mmu_trap_tl1_4
2665	  nop
2666	rdpr	%tt, %g5
2667	cmp	%g5, T_DATA_EXCEPTION
2668	bne	mmu_trap_tl1_4
2669	  nop
2670	wr	%g0, ASI_DMMU, %asi
2671	ldxa	[MMU_SFSR]%asi, %g5
2672	mov	1, %g6
2673	sllx	%g6, PN_SFSR_PARITY_SHIFT, %g6
2674	andcc	%g5, %g6, %g0
2675	bz	mmu_trap_tl1_4
2676
2677	/*
2678	 * We are running on a Panther and have hit a DTLB parity error.
2679	 */
2680	ldxa	[MMU_TAG_ACCESS]%asi, %g2
2681	mov	%g5, %g3
2682	ba,pt	%xcc, .mmu_exception_is_tlb_parity
2683	mov	T_DATA_EXCEPTION, %g1
2684
2685mmu_trap_tl1_4:
2686	CPU_PADDR(%g7, %g6);
2687	add     %g7, CPU_TL1_HDLR, %g7		! %g7 = &cpu_m.tl1_hdlr (PA)
2688	/*
2689	 * AM is cleared on trap, so addresses are 64 bit
2690	 */
2691	lda     [%g7]ASI_MEM, %g6
2692	brz,a,pt %g6, 1f
2693	  nop
2694	/*
2695	 * We are going to update cpu_m.tl1_hdlr using physical address.
2696	 * Flush the D$ line, so that stale data won't be accessed later.
2697	 */
2698	CPU_ADDR(%g6, %g5)
2699	add     %g6, CPU_TL1_HDLR, %g6		! %g6 = &cpu_m.tl1_hdlr (VA)
2700	GET_CPU_IMPL(%g5)
2701	cmp	%g5, CHEETAH_IMPL
2702	bl,pt	%icc, 3f
2703	 cmp	%g5, SPITFIRE_IMPL
2704	stxa	%g0, [%g7]ASI_DC_INVAL
2705	membar	#Sync
2706	ba,pt	%xcc, 2f
2707	 nop
27083:
2709	bl,pt	%icc, 2f
2710	 sethi	%hi(dcache_line_mask), %g5
2711	ld	[%g5 + %lo(dcache_line_mask)], %g5
2712	and	%g6, %g5, %g5
2713	stxa	%g0, [%g5]ASI_DC_TAG
2714	membar	#Sync
27152:
2716	sta     %g0, [%g7]ASI_MEM
2717	SWITCH_GLOBALS				! back to mmu globals
2718	ba,a,pt	%xcc, sfmmu_mmu_trap		! handle page faults
27191:
2720	rdpr	%tt, %g5
2721	rdpr	%tl, %g7
2722	sub	%g7, 1, %g6
2723	wrpr	%g6, %tl
2724	rdpr	%tt, %g6
2725	wrpr	%g7, %tl
2726	and	%g6, WTRAP_TTMASK, %g6
2727	cmp	%g6, WTRAP_TYPE
2728	bne,a,pn %xcc, ptl1_panic
2729	mov	PTL1_BAD_MMUTRAP, %g1
2730	rdpr	%tpc, %g7
2731	/* tpc should be in the trap table */
2732	set	trap_table, %g6
2733	cmp	%g7, %g6
2734	blt,a,pn %xcc, ptl1_panic
2735	  mov	PTL1_BAD_MMUTRAP, %g1
2736	set	etrap_table, %g6
2737	cmp	%g7, %g6
2738	bge,a,pn %xcc, ptl1_panic
2739	  mov	PTL1_BAD_MMUTRAP, %g1
2740	cmp	%g5, T_ALIGNMENT
2741	move	%icc, MMU_SFAR, %g6
2742	movne	%icc, MMU_TAG_ACCESS, %g6
2743	ldxa	[%g6]ASI_DMMU, %g6
2744	andn	%g7, WTRAP_ALIGN, %g7	/* 128 byte aligned */
2745	add	%g7, WTRAP_FAULTOFF, %g7
2746	wrpr	%g0, %g7, %tnpc
2747	done
2748	SET_SIZE(mmu_trap_tl1)
2749
2750/*
2751 * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers.  These
2752 * traps are valid only when kmdb is loaded.  When the debugger is active,
2753 * the code below is rewritten to transfer control to the appropriate
2754 * debugger entry points.
2755 */
2756	.global	kmdb_trap
2757	.align	8
2758kmdb_trap:
2759	ba,a	trap_table0
2760	jmp	%g1 + 0
2761	nop
2762
2763	.global	kmdb_trap_tl1
2764	.align	8
2765kmdb_trap_tl1:
2766	ba,a	trap_table0
2767	jmp	%g1 + 0
2768	nop
2769
2770/*
2771 * This entry is copied from OBP's trap table during boot.
2772 */
2773	.global	obp_bpt
2774	.align	8
2775obp_bpt:
2776	NOT
2777
2778/*
2779 * if kernel, set PCONTEXT to 0 for debuggers
2780 * if user, clear nucleus page sizes
2781 */
2782	.global kctx_obp_bpt
2783kctx_obp_bpt:
2784	set	obp_bpt, %g2
27851:
2786#ifndef _OPL
2787	mov	MMU_PCONTEXT, %g1
2788	ldxa	[%g1]ASI_DMMU, %g1
2789	srlx	%g1, CTXREG_NEXT_SHIFT, %g3
2790	brz,pt	%g3, 3f			! nucleus pgsz is 0, no problem
2791	  sllx	%g3, CTXREG_NEXT_SHIFT, %g3
2792	set	CTXREG_CTX_MASK, %g4	! check Pcontext
2793	btst	%g4, %g1
2794	bz,a,pt	%xcc, 2f
2795	  clr	%g3			! kernel:  PCONTEXT=0
2796	xor	%g3, %g1, %g3		! user:	clr N_pgsz0/1 bits
27972:
2798	set	DEMAP_ALL_TYPE, %g1
2799	stxa	%g0, [%g1]ASI_DTLB_DEMAP
2800	stxa	%g0, [%g1]ASI_ITLB_DEMAP
2801	mov	MMU_PCONTEXT, %g1
2802	stxa	%g3, [%g1]ASI_DMMU
2803        membar  #Sync
2804	sethi	%hi(FLUSH_ADDR), %g1
2805	flush	%g1			! flush required by immu
2806#endif /* _OPL */
28073:
2808	jmp	%g2
2809	  nop
2810
2811
2812#ifdef	TRAPTRACE
2813/*
2814 * TRAPTRACE support.
2815 * labels here are branched to with "rd %pc, %g7" in the delay slot.
2816 * Return is done by "jmp %g7 + 4".
2817 */
2818
2819trace_gen:
2820	TRACE_PTR(%g3, %g6)
2821	GET_TRACE_TICK(%g6, %g4)
2822	stxa	%g6, [%g3 + TRAP_ENT_TICK]%asi
2823	rdpr	%tl, %g6
2824	stha	%g6, [%g3 + TRAP_ENT_TL]%asi
2825	rdpr	%tt, %g6
2826	stha	%g6, [%g3 + TRAP_ENT_TT]%asi
2827	rdpr	%tstate, %g6
2828	stxa	%g6, [%g3 + TRAP_ENT_TSTATE]%asi
2829	stna	%sp, [%g3 + TRAP_ENT_SP]%asi
2830	rdpr	%tpc, %g6
2831	stna	%g6, [%g3 + TRAP_ENT_TPC]%asi
2832	TRACE_NEXT(%g3, %g4, %g5)
2833	jmp	%g7 + 4
2834	nop
2835
2836trace_win:
2837	TRACE_WIN_INFO(0, %l0, %l1, %l2)
2838	! Keep the locals as clean as possible, caller cleans %l4
2839	clr	%l2
2840	clr	%l1
2841	jmp	%l4 + 4
2842	  clr	%l0
2843
2844/*
2845 * Trace a tsb hit
2846 * g1 = tsbe pointer (in/clobbered)
2847 * g2 = tag access register (in)
2848 * g3 - g4 = scratch (clobbered)
2849 * g5 = tsbe data (in)
2850 * g6 = scratch (clobbered)
2851 * g7 = pc we jumped here from (in)
2852 */
2853
2854	! Do not disturb %g5, it will be used after the trace
2855	ALTENTRY(trace_tsbhit)
2856	TRACE_TSBHIT(0)
2857	jmp	%g7 + 4
2858	nop
2859
2860/*
2861 * Trace a TSB miss
2862 *
2863 * g1 = tsb8k pointer (in)
2864 * g2 = tag access register (in)
2865 * g3 = tsb4m pointer (in)
2866 * g4 = tsbe tag (in/clobbered)
2867 * g5 - g6 = scratch (clobbered)
2868 * g7 = pc we jumped here from (in)
2869 */
2870	.global	trace_tsbmiss
2871trace_tsbmiss:
2872	membar	#Sync
2873	sethi	%hi(FLUSH_ADDR), %g6
2874	flush	%g6
2875	TRACE_PTR(%g5, %g6)
2876	stxa	%g2, [%g5 + TRAP_ENT_SP]%asi		! tag access
2877	stxa	%g4, [%g5 + TRAP_ENT_F1]%asi		! tsb tag
2878	GET_TRACE_TICK(%g6, %g4)
2879	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi
2880	rdpr	%tnpc, %g6
2881	stxa	%g6, [%g5 + TRAP_ENT_F2]%asi
2882	stna	%g1, [%g5 + TRAP_ENT_F3]%asi		! tsb8k pointer
2883	srlx	%g1, 32, %g6
2884	stna	%g6, [%g5 + TRAP_ENT_F4]%asi		! huh?
2885	rdpr	%tpc, %g6
2886	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi
2887	rdpr	%tl, %g6
2888	stha	%g6, [%g5 + TRAP_ENT_TL]%asi
2889	rdpr	%tt, %g6
2890	or	%g6, TT_MMU_MISS, %g4
2891	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
2892	cmp	%g6, FAST_IMMU_MISS_TT
2893	be,a	%icc, 1f
2894	  ldxa	[%g0]ASI_IMMU, %g6
2895	ldxa	[%g0]ASI_DMMU, %g6
28961:	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi	! tag target
2897	stxa	%g3, [%g5 + TRAP_ENT_TR]%asi		! tsb4m pointer
2898	TRACE_NEXT(%g5, %g4, %g6)
2899	jmp	%g7 + 4
2900	nop
2901
2902/*
2903 * g2 = tag access register (in)
2904 * g3 = ctx number (in)
2905 */
2906trace_dataprot:
2907	membar	#Sync
2908	sethi	%hi(FLUSH_ADDR), %g6
2909	flush	%g6
2910	TRACE_PTR(%g1, %g6)
2911	GET_TRACE_TICK(%g6, %g5)
2912	stxa	%g6, [%g1 + TRAP_ENT_TICK]%asi
2913	rdpr	%tpc, %g6
2914	stna	%g6, [%g1 + TRAP_ENT_TPC]%asi
2915	rdpr	%tstate, %g6
2916	stxa	%g6, [%g1 + TRAP_ENT_TSTATE]%asi
2917	stxa	%g2, [%g1 + TRAP_ENT_SP]%asi		! tag access reg
2918	stxa	%g0, [%g1 + TRAP_ENT_TR]%asi
2919	stxa	%g0, [%g1 + TRAP_ENT_F1]%asi
2920	stxa	%g0, [%g1 + TRAP_ENT_F2]%asi
2921	stxa	%g0, [%g1 + TRAP_ENT_F3]%asi
2922	stxa	%g0, [%g1 + TRAP_ENT_F4]%asi
2923	rdpr	%tl, %g6
2924	stha	%g6, [%g1 + TRAP_ENT_TL]%asi
2925	rdpr	%tt, %g6
2926	stha	%g6, [%g1 + TRAP_ENT_TT]%asi
2927	TRACE_NEXT(%g1, %g4, %g5)
2928	jmp	%g7 + 4
2929	nop
2930
2931#endif /* TRAPTRACE */
2932
2933       .align  32
2934       .global pil15_epilogue
2935pil15_epilogue:
2936       ba      pil_interrupt_common
2937       nop
2938       .align  32
2939
2940/*
2941 * fast_trap_done, fast_trap_done_chk_intr:
2942 *
2943 * Due to the design of UltraSPARC pipeline, pending interrupts are not
2944 * taken immediately after a RETRY or DONE instruction which causes IE to
2945 * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed
2946 * to execute first before taking any interrupts. If that instruction
2947 * results in other traps, and if the corresponding trap handler runs
2948 * entirely at TL=1 with interrupts disabled, then pending interrupts
2949 * won't be taken until after yet another instruction following the %tpc
2950 * or %tnpc.
2951 *
2952 * A malicious user program can use this feature to block out interrupts
2953 * for extended durations, which can result in send_mondo_timeout kernel
2954 * panic.
2955 *
2956 * This problem is addressed by servicing any pending interrupts via
2957 * sys_trap before returning back to the user mode from a fast trap
2958 * handler. The "done" instruction within a fast trap handler, which
2959 * runs entirely at TL=1 with interrupts disabled, is replaced with the
2960 * FAST_TRAP_DONE macro, which branches control to this fast_trap_done
2961 * entry point.
2962 *
2963 * We check for any pending interrupts here and force a sys_trap to
2964 * service those interrupts, if any. To minimize overhead, pending
2965 * interrupts are checked if the %tpc happens to be at 16K boundary,
2966 * which allows a malicious program to execute at most 4K consecutive
2967 * instructions before we service any pending interrupts. If a worst
2968 * case fast trap handler takes about 2 usec, then interrupts will be
2969 * blocked for at most 8 msec, less than a clock tick.
2970 *
2971 * For the cases where we don't know if the %tpc will cross a 16K
2972 * boundary, we can't use the above optimization and always process
2973 * any pending interrupts via fast_frap_done_chk_intr entry point.
2974 *
2975 * Entry Conditions:
2976 * 	%pstate		am:0 priv:1 ie:0
2977 * 			globals are AG (not normal globals)
2978 */
2979
2980	.global	fast_trap_done, fast_trap_done_chk_intr
2981fast_trap_done:
2982	rdpr	%tpc, %g5
2983	sethi	%hi(0xffffc000), %g6	! 1's complement of 0x3fff
2984	andncc	%g5, %g6, %g0		! check lower 14 bits of %tpc
2985	bz,a,pn	%icc, 1f		! branch if zero (lower 32 bits only)
2986	  ldxa	[%g0]ASI_INTR_RECEIVE_STATUS, %g5
2987	done
2988
2989	ALTENTRY(fast_trap_done_check_interrupts)
2990fast_trap_done_chk_intr:
2991	ldxa	[%g0]ASI_INTR_RECEIVE_STATUS, %g5
2992
29931:	rd	SOFTINT, %g6
2994	and	%g5, IRSR_BUSY, %g5
2995	orcc	%g5, %g6, %g0
2996	bnz,pn	%xcc, 2f		! branch if any pending intr
2997	nop
2998	done
2999
30002:
3001	/*
3002	 * We get here if there are any pending interrupts.
3003	 * Adjust %tpc/%tnpc as we'll be resuming via "retry"
3004	 * instruction.
3005	 */
3006	rdpr	%tnpc, %g5
3007	wrpr	%g0, %g5, %tpc
3008	add	%g5, 4, %g5
3009	wrpr	%g0, %g5, %tnpc
3010
3011	/*
3012	 * Force a dummy sys_trap call so that interrupts can be serviced.
3013	 */
3014	set	fast_trap_dummy_call, %g1
3015	ba,pt	%xcc, sys_trap
3016	  mov	-1, %g4
3017
3018fast_trap_dummy_call:
3019	retl
3020	nop
3021
3022/*
3023 * Currently the brand syscall interposition code is not enabled by
3024 * default.  Instead, when a branded zone is first booted the brand
3025 * infrastructure will patch the trap table so that the syscall
3026 * entry points are redirected to syscall_wrapper32 and syscall_wrapper
3027 * for ILP32 and LP64 syscalls respectively.  this is done in
3028 * brand_plat_interposition_enable().  Note that the syscall wrappers
3029 * below do not collect any trap trace data since the syscall hot patch
3030 * points are reached after trap trace data has already been collected.
3031 */
3032#define	BRAND_CALLBACK(callback_id)					    \
3033	CPU_ADDR(%g2, %g1)		/* load CPU struct addr to %g2	*/ ;\
3034	ldn	[%g2 + CPU_THREAD], %g3	/* load thread pointer		*/ ;\
3035	ldn	[%g3 + T_PROCP], %g3	/* get proc pointer		*/ ;\
3036	ldn	[%g3 + P_BRAND], %g3	/* get brand pointer		*/ ;\
3037	brz	%g3, 1f			/* No brand?  No callback. 	*/ ;\
3038	nop 								   ;\
3039	ldn	[%g3 + B_MACHOPS], %g3	/* get machops list		*/ ;\
3040	ldn	[%g3 + (callback_id << 3)], %g3 			   ;\
3041	brz	%g3, 1f							   ;\
3042	/*								    \
3043	 * This isn't pretty.  We want a low-latency way for the callback   \
3044	 * routine to decline to do anything.  We just pass in an address   \
3045	 * the routine can directly jmp back to, pretending that nothing    \
3046	 * has happened.						    \
3047	 * 								    \
3048	 * %g1: return address (where the brand handler jumps back to)	    \
3049	 * %g2: address of CPU structure				    \
3050	 * %g3: address of brand handler (where we will jump to)	    \
3051	 */								    \
3052	mov	%pc, %g1						   ;\
3053	add	%g1, 16, %g1						   ;\
3054	jmp	%g3							   ;\
3055	nop								   ;\
30561:
3057
3058	ENTRY_NP(syscall_wrapper32)
3059	BRAND_CALLBACK(BRAND_CB_SYSCALL32)
3060	SYSCALL_NOTT(syscall_trap32)
3061	SET_SIZE(syscall_wrapper32)
3062
3063	ENTRY_NP(syscall_wrapper)
3064	BRAND_CALLBACK(BRAND_CB_SYSCALL)
3065	SYSCALL_NOTT(syscall_trap)
3066	SET_SIZE(syscall_wrapper)
3067
3068#endif	/* lint */
3069