/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #if !defined(lint) #include "assym.h" #endif /* !lint */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * WARNING: If you add a fast trap handler which can be invoked by a * non-privileged user, you may have to use the FAST_TRAP_DONE macro * instead of "done" instruction to return back to the user mode. See * comments for the "fast_trap_done" entry point for more information. * * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the * cases where you always want to process any pending interrupts before * returning back to the user mode. */ #define FAST_TRAP_DONE \ ba,a fast_trap_done #define FAST_TRAP_DONE_CHK_INTR \ ba,a fast_trap_done_chk_intr /* * SPARC V9 Trap Table * * Most of the trap handlers are made from common building * blocks, and some are instantiated multiple times within * the trap table. So, I build a bunch of macros, then * populate the table using only the macros. * * Many macros branch to sys_trap. Its calling convention is: * %g1 kernel trap handler * %g2, %g3 args for above * %g4 desire %pil */ #ifdef TRAPTRACE /* * Tracing macro. Adds two instructions if TRAPTRACE is defined. */ #define TT_TRACE(label) \ ba label ;\ rd %pc, %g7 #define TT_TRACE_INS 2 #define TT_TRACE_L(label) \ ba label ;\ rd %pc, %l4 ;\ clr %l4 #define TT_TRACE_L_INS 3 #else #define TT_TRACE(label) #define TT_TRACE_INS 0 #define TT_TRACE_L(label) #define TT_TRACE_L_INS 0 #endif #ifdef NIAGARA_ERRATUM_42 #define NI_GL_RESET_TT0 \ wrpr %g0, 2, %gl ;\ wrpr %g0, 1, %gl #define NI_GL_RESET_TT0_INS 2 #else /* NIAGARA_ERRATUM_42 */ #define NI_GL_RESET_TT0 #define NI_GL_RESET_TT0_INS 0 #endif /* NIAGARA_ERRATUM_42 */ /* * This macro is used to update per cpu mmu stats in perf critical * paths. It is only enabled in debug kernels or if SFMMU_STAT_GATHER * is defined. */ #if defined(DEBUG) || defined(SFMMU_STAT_GATHER) #define HAT_PERCPU_DBSTAT(stat) \ mov stat, %g1 ;\ ba stat_mmu ;\ rd %pc, %g7 #else #define HAT_PERCPU_DBSTAT(stat) #endif /* DEBUG || SFMMU_STAT_GATHER */ /* * This first set are funneled to trap() with %tt as the type. * Trap will then either panic or send the user a signal. */ /* * NOT is used for traps that just shouldn't happen. * It comes in both single and quadruple flavors. */ #if !defined(lint) .global trap #endif /* !lint */ #define NOT \ TT_TRACE(trace_gen) ;\ set trap, %g1 ;\ rdpr %tt, %g3 ;\ ba,pt %xcc, sys_trap ;\ sub %g0, 1, %g4 ;\ .align 32 #define NOT4 NOT; NOT; NOT; NOT #define NOTP \ TT_TRACE(trace_gen) ;\ ba,pt %xcc, ptl1_panic ;\ mov PTL1_BAD_TRAP, %g1 ;\ .align 32 #define NOTP4 NOTP; NOTP; NOTP; NOTP /* * RED is for traps that use the red mode handler. * We should never see these either. */ #define RED NOT /* * BAD is used for trap vectors we don't have a kernel * handler for. * It also comes in single and quadruple versions. */ #define BAD NOT #define BAD4 NOT4 #define DONE \ done; \ .align 32 /* * TRAP vectors to the trap() function. * It's main use is for user errors. */ #if !defined(lint) .global trap #endif /* !lint */ #define TRAP(arg) \ TT_TRACE(trace_gen) ;\ set trap, %g1 ;\ mov arg, %g3 ;\ ba,pt %xcc, sys_trap ;\ sub %g0, 1, %g4 ;\ .align 32 /* * SYSCALL is used for system calls on both ILP32 and LP64 kernels * depending on the "which" parameter (should be either syscall_trap * or syscall_trap32). */ #define SYSCALL(which) \ TT_TRACE(trace_gen) ;\ set (which), %g1 ;\ ba,pt %xcc, sys_trap ;\ sub %g0, 1, %g4 ;\ .align 32 /* * GOTO just jumps to a label. * It's used for things that can be fixed without going thru sys_trap. */ #define GOTO(label) \ .global label ;\ ba,a label ;\ .empty ;\ .align 32 /* * GOTO_TT just jumps to a label. * correctable ECC error traps at level 0 and 1 will use this macro. * It's used for things that can be fixed without going thru sys_trap. */ #define GOTO_TT(label, ttlabel) \ .global label ;\ TT_TRACE(ttlabel) ;\ ba,a label ;\ .empty ;\ .align 32 /* * Privileged traps * Takes breakpoint if privileged, calls trap() if not. */ #define PRIV(label) \ rdpr %tstate, %g1 ;\ btst TSTATE_PRIV, %g1 ;\ bnz label ;\ rdpr %tt, %g3 ;\ set trap, %g1 ;\ ba,pt %xcc, sys_trap ;\ sub %g0, 1, %g4 ;\ .align 32 /* * DTrace traps. */ #define DTRACE_FASTTRAP \ .global dtrace_fasttrap_probe ;\ .global dtrace_fasttrap_probe_ptr ;\ sethi %hi(dtrace_fasttrap_probe_ptr), %g4 ;\ ldn [%g4 + %lo(dtrace_fasttrap_probe_ptr)], %g4 ;\ set dtrace_fasttrap_probe, %g1 ;\ brnz,pn %g4, user_trap ;\ sub %g0, 1, %g4 ;\ FAST_TRAP_DONE ;\ .align 32 #define DTRACE_PID \ .global dtrace_pid_probe ;\ set dtrace_pid_probe, %g1 ;\ ba,pt %xcc, user_trap ;\ sub %g0, 1, %g4 ;\ .align 32 #define DTRACE_RETURN \ .global dtrace_return_probe ;\ set dtrace_return_probe, %g1 ;\ ba,pt %xcc, user_trap ;\ sub %g0, 1, %g4 ;\ .align 32 /* * REGISTER WINDOW MANAGEMENT MACROS */ /* * various convenient units of padding */ #define SKIP(n) .skip 4*(n) /* * CLEAN_WINDOW is the simple handler for cleaning a register window. */ #define CLEAN_WINDOW \ TT_TRACE_L(trace_win) ;\ rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin ;\ clr %l0; clr %l1; clr %l2; clr %l3 ;\ clr %l4; clr %l5; clr %l6; clr %l7 ;\ clr %o0; clr %o1; clr %o2; clr %o3 ;\ clr %o4; clr %o5; clr %o6; clr %o7 ;\ retry; .align 128 #if !defined(lint) /* * If we get an unresolved tlb miss while in a window handler, the fault * handler will resume execution at the last instruction of the window * hander, instead of delivering the fault to the kernel. Spill handlers * use this to spill windows into the wbuf. * * The mixed handler works by checking %sp, and branching to the correct * handler. This is done by branching back to label 1: for 32b frames, * or label 2: for 64b frames; which implies the handler order is: 32b, * 64b, mixed. The 1: and 2: labels are offset into the routines to * allow the branchs' delay slots to contain useful instructions. */ /* * SPILL_32bit spills a 32-bit-wide kernel register window. It * assumes that the kernel context and the nucleus context are the * same. The stack pointer is required to be eight-byte aligned even * though this code only needs it to be four-byte aligned. */ #define SPILL_32bit(tail) \ NI_GL_RESET_TT0 ;\ srl %sp, 0, %sp ;\ 1: st %l0, [%sp + 0] ;\ st %l1, [%sp + 4] ;\ st %l2, [%sp + 8] ;\ st %l3, [%sp + 12] ;\ st %l4, [%sp + 16] ;\ st %l5, [%sp + 20] ;\ st %l6, [%sp + 24] ;\ st %l7, [%sp + 28] ;\ st %i0, [%sp + 32] ;\ st %i1, [%sp + 36] ;\ st %i2, [%sp + 40] ;\ st %i3, [%sp + 44] ;\ st %i4, [%sp + 48] ;\ st %i5, [%sp + 52] ;\ st %i6, [%sp + 56] ;\ st %i7, [%sp + 60] ;\ TT_TRACE_L(trace_win) ;\ saved ;\ retry ;\ SKIP(31-19-NI_GL_RESET_TT0_INS-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_32bit_/**/tail ;\ .empty /* * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit * wide address space via the designated asi. It is used to spill * non-kernel windows. The stack pointer is required to be eight-byte * aligned even though this code only needs it to be four-byte * aligned. */ #define SPILL_32bit_asi(asi_num, tail) \ NI_GL_RESET_TT0 ;\ srl %sp, 0, %sp ;\ 1: sta %l0, [%sp + %g0]asi_num ;\ mov 4, %g1 ;\ sta %l1, [%sp + %g1]asi_num ;\ mov 8, %g2 ;\ sta %l2, [%sp + %g2]asi_num ;\ mov 12, %g3 ;\ sta %l3, [%sp + %g3]asi_num ;\ add %sp, 16, %g4 ;\ sta %l4, [%g4 + %g0]asi_num ;\ sta %l5, [%g4 + %g1]asi_num ;\ sta %l6, [%g4 + %g2]asi_num ;\ sta %l7, [%g4 + %g3]asi_num ;\ add %g4, 16, %g4 ;\ sta %i0, [%g4 + %g0]asi_num ;\ sta %i1, [%g4 + %g1]asi_num ;\ sta %i2, [%g4 + %g2]asi_num ;\ sta %i3, [%g4 + %g3]asi_num ;\ add %g4, 16, %g4 ;\ sta %i4, [%g4 + %g0]asi_num ;\ sta %i5, [%g4 + %g1]asi_num ;\ sta %i6, [%g4 + %g2]asi_num ;\ sta %i7, [%g4 + %g3]asi_num ;\ TT_TRACE_L(trace_win) ;\ saved ;\ retry ;\ SKIP(31-25-NI_GL_RESET_TT0_INS-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_32bit_/**/tail ;\ .empty #define SPILL_32bit_tt1(asi_num, tail) \ ba,a,pt %xcc, fault_32bit_/**/tail ;\ .empty ;\ .align 128 /* * FILL_32bit fills a 32-bit-wide kernel register window. It assumes * that the kernel context and the nucleus context are the same. The * stack pointer is required to be eight-byte aligned even though this * code only needs it to be four-byte aligned. */ #define FILL_32bit(tail) \ srl %sp, 0, %sp ;\ 1: TT_TRACE_L(trace_win) ;\ ld [%sp + 0], %l0 ;\ ld [%sp + 4], %l1 ;\ ld [%sp + 8], %l2 ;\ ld [%sp + 12], %l3 ;\ ld [%sp + 16], %l4 ;\ ld [%sp + 20], %l5 ;\ ld [%sp + 24], %l6 ;\ ld [%sp + 28], %l7 ;\ ld [%sp + 32], %i0 ;\ ld [%sp + 36], %i1 ;\ ld [%sp + 40], %i2 ;\ ld [%sp + 44], %i3 ;\ ld [%sp + 48], %i4 ;\ ld [%sp + 52], %i5 ;\ ld [%sp + 56], %i6 ;\ ld [%sp + 60], %i7 ;\ restored ;\ retry ;\ SKIP(31-19-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_32bit_/**/tail ;\ .empty /* * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit * wide address space via the designated asi. It is used to fill * non-kernel windows. The stack pointer is required to be eight-byte * aligned even though this code only needs it to be four-byte * aligned. */ #define FILL_32bit_asi(asi_num, tail) \ srl %sp, 0, %sp ;\ 1: TT_TRACE_L(trace_win) ;\ mov 4, %g1 ;\ lda [%sp + %g0]asi_num, %l0 ;\ mov 8, %g2 ;\ lda [%sp + %g1]asi_num, %l1 ;\ mov 12, %g3 ;\ lda [%sp + %g2]asi_num, %l2 ;\ lda [%sp + %g3]asi_num, %l3 ;\ add %sp, 16, %g4 ;\ lda [%g4 + %g0]asi_num, %l4 ;\ lda [%g4 + %g1]asi_num, %l5 ;\ lda [%g4 + %g2]asi_num, %l6 ;\ lda [%g4 + %g3]asi_num, %l7 ;\ add %g4, 16, %g4 ;\ lda [%g4 + %g0]asi_num, %i0 ;\ lda [%g4 + %g1]asi_num, %i1 ;\ lda [%g4 + %g2]asi_num, %i2 ;\ lda [%g4 + %g3]asi_num, %i3 ;\ add %g4, 16, %g4 ;\ lda [%g4 + %g0]asi_num, %i4 ;\ lda [%g4 + %g1]asi_num, %i5 ;\ lda [%g4 + %g2]asi_num, %i6 ;\ lda [%g4 + %g3]asi_num, %i7 ;\ restored ;\ retry ;\ SKIP(31-25-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_32bit_/**/tail ;\ .empty /* * SPILL_64bit spills a 64-bit-wide kernel register window. It * assumes that the kernel context and the nucleus context are the * same. The stack pointer is required to be eight-byte aligned. */ #define SPILL_64bit(tail) \ NI_GL_RESET_TT0 ;\ 2: stx %l0, [%sp + V9BIAS64 + 0] ;\ stx %l1, [%sp + V9BIAS64 + 8] ;\ stx %l2, [%sp + V9BIAS64 + 16] ;\ stx %l3, [%sp + V9BIAS64 + 24] ;\ stx %l4, [%sp + V9BIAS64 + 32] ;\ stx %l5, [%sp + V9BIAS64 + 40] ;\ stx %l6, [%sp + V9BIAS64 + 48] ;\ stx %l7, [%sp + V9BIAS64 + 56] ;\ stx %i0, [%sp + V9BIAS64 + 64] ;\ stx %i1, [%sp + V9BIAS64 + 72] ;\ stx %i2, [%sp + V9BIAS64 + 80] ;\ stx %i3, [%sp + V9BIAS64 + 88] ;\ stx %i4, [%sp + V9BIAS64 + 96] ;\ stx %i5, [%sp + V9BIAS64 + 104] ;\ stx %i6, [%sp + V9BIAS64 + 112] ;\ stx %i7, [%sp + V9BIAS64 + 120] ;\ TT_TRACE_L(trace_win) ;\ saved ;\ retry ;\ SKIP(31-18-NI_GL_RESET_TT0_INS-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_64bit_/**/tail ;\ .empty #define SPILL_64bit_ktt1(tail) \ ba,a,pt %xcc, fault_64bit_/**/tail ;\ .empty ;\ .align 128 #define SPILL_mixed_ktt1(tail) \ btst 1, %sp ;\ bz,a,pt %xcc, fault_32bit_/**/tail ;\ srl %sp, 0, %sp ;\ ba,a,pt %xcc, fault_64bit_/**/tail ;\ .empty ;\ .align 128 /* * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit * wide address space via the designated asi. It is used to spill * non-kernel windows. The stack pointer is required to be eight-byte * aligned. */ #define SPILL_64bit_asi(asi_num, tail) \ NI_GL_RESET_TT0 ;\ mov 0 + V9BIAS64, %g1 ;\ 2: stxa %l0, [%sp + %g1]asi_num ;\ mov 8 + V9BIAS64, %g2 ;\ stxa %l1, [%sp + %g2]asi_num ;\ mov 16 + V9BIAS64, %g3 ;\ stxa %l2, [%sp + %g3]asi_num ;\ mov 24 + V9BIAS64, %g4 ;\ stxa %l3, [%sp + %g4]asi_num ;\ add %sp, 32, %g5 ;\ stxa %l4, [%g5 + %g1]asi_num ;\ stxa %l5, [%g5 + %g2]asi_num ;\ stxa %l6, [%g5 + %g3]asi_num ;\ stxa %l7, [%g5 + %g4]asi_num ;\ add %g5, 32, %g5 ;\ stxa %i0, [%g5 + %g1]asi_num ;\ stxa %i1, [%g5 + %g2]asi_num ;\ stxa %i2, [%g5 + %g3]asi_num ;\ stxa %i3, [%g5 + %g4]asi_num ;\ add %g5, 32, %g5 ;\ stxa %i4, [%g5 + %g1]asi_num ;\ stxa %i5, [%g5 + %g2]asi_num ;\ stxa %i6, [%g5 + %g3]asi_num ;\ stxa %i7, [%g5 + %g4]asi_num ;\ TT_TRACE_L(trace_win) ;\ saved ;\ retry ;\ SKIP(31-25-NI_GL_RESET_TT0_INS-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_64bit_/**/tail ;\ .empty #define SPILL_64bit_tt1(asi_num, tail) \ ba,a,pt %xcc, fault_64bit_/**/tail ;\ .empty ;\ .align 128 /* * FILL_64bit fills a 64-bit-wide kernel register window. It assumes * that the kernel context and the nucleus context are the same. The * stack pointer is required to be eight-byte aligned. */ #define FILL_64bit(tail) \ 2: TT_TRACE_L(trace_win) ;\ ldx [%sp + V9BIAS64 + 0], %l0 ;\ ldx [%sp + V9BIAS64 + 8], %l1 ;\ ldx [%sp + V9BIAS64 + 16], %l2 ;\ ldx [%sp + V9BIAS64 + 24], %l3 ;\ ldx [%sp + V9BIAS64 + 32], %l4 ;\ ldx [%sp + V9BIAS64 + 40], %l5 ;\ ldx [%sp + V9BIAS64 + 48], %l6 ;\ ldx [%sp + V9BIAS64 + 56], %l7 ;\ ldx [%sp + V9BIAS64 + 64], %i0 ;\ ldx [%sp + V9BIAS64 + 72], %i1 ;\ ldx [%sp + V9BIAS64 + 80], %i2 ;\ ldx [%sp + V9BIAS64 + 88], %i3 ;\ ldx [%sp + V9BIAS64 + 96], %i4 ;\ ldx [%sp + V9BIAS64 + 104], %i5 ;\ ldx [%sp + V9BIAS64 + 112], %i6 ;\ ldx [%sp + V9BIAS64 + 120], %i7 ;\ restored ;\ retry ;\ SKIP(31-18-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_64bit_/**/tail ;\ .empty /* * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit * wide address space via the designated asi. It is used to fill * non-kernel windows. The stack pointer is required to be eight-byte * aligned. */ #define FILL_64bit_asi(asi_num, tail) \ mov V9BIAS64 + 0, %g1 ;\ 2: TT_TRACE_L(trace_win) ;\ ldxa [%sp + %g1]asi_num, %l0 ;\ mov V9BIAS64 + 8, %g2 ;\ ldxa [%sp + %g2]asi_num, %l1 ;\ mov V9BIAS64 + 16, %g3 ;\ ldxa [%sp + %g3]asi_num, %l2 ;\ mov V9BIAS64 + 24, %g4 ;\ ldxa [%sp + %g4]asi_num, %l3 ;\ add %sp, 32, %g5 ;\ ldxa [%g5 + %g1]asi_num, %l4 ;\ ldxa [%g5 + %g2]asi_num, %l5 ;\ ldxa [%g5 + %g3]asi_num, %l6 ;\ ldxa [%g5 + %g4]asi_num, %l7 ;\ add %g5, 32, %g5 ;\ ldxa [%g5 + %g1]asi_num, %i0 ;\ ldxa [%g5 + %g2]asi_num, %i1 ;\ ldxa [%g5 + %g3]asi_num, %i2 ;\ ldxa [%g5 + %g4]asi_num, %i3 ;\ add %g5, 32, %g5 ;\ ldxa [%g5 + %g1]asi_num, %i4 ;\ ldxa [%g5 + %g2]asi_num, %i5 ;\ ldxa [%g5 + %g3]asi_num, %i6 ;\ ldxa [%g5 + %g4]asi_num, %i7 ;\ restored ;\ retry ;\ SKIP(31-25-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_64bit_/**/tail ;\ .empty #endif /* !lint */ /* * SPILL_mixed spills either size window, depending on * whether %sp is even or odd, to a 32-bit address space. * This may only be used in conjunction with SPILL_32bit/ * FILL_64bit. * Clear upper 32 bits of %sp if it is odd. * We won't need to clear them in 64 bit kernel. */ #define SPILL_mixed \ NI_GL_RESET_TT0 ;\ btst 1, %sp ;\ bz,a,pt %xcc, 1b ;\ srl %sp, 0, %sp ;\ ba,pt %xcc, 2b ;\ nop ;\ .align 128 /* * FILL_mixed(ASI) fills either size window, depending on * whether %sp is even or odd, from a 32-bit address space. * This may only be used in conjunction with FILL_32bit/ * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular * attention should be paid to the instructions that belong * in the delay slots of the branches depending on the type * of fill handler being branched to. * Clear upper 32 bits of %sp if it is odd. * We won't need to clear them in 64 bit kernel. */ #define FILL_mixed \ btst 1, %sp ;\ bz,a,pt %xcc, 1b ;\ srl %sp, 0, %sp ;\ ba,pt %xcc, 2b ;\ nop ;\ .align 128 /* * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows, * respectively, into the address space via the designated asi. The * unbiased stack pointer is required to be eight-byte aligned (even for * the 32-bit case even though this code does not require such strict * alignment). * * With SPARC v9 the spill trap takes precedence over the cleanwin trap * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save * will cause cwp + 2 to be spilled but will not clean cwp + 1. That * window may contain kernel data so in user_rtt we set wstate to call * these spill handlers on the first user spill trap. These handler then * spill the appropriate window but also back up a window and clean the * window that didn't get a cleanwin trap. */ #define SPILL_32clean(asi_num, tail) \ NI_GL_RESET_TT0 ;\ srl %sp, 0, %sp ;\ sta %l0, [%sp + %g0]asi_num ;\ mov 4, %g1 ;\ sta %l1, [%sp + %g1]asi_num ;\ mov 8, %g2 ;\ sta %l2, [%sp + %g2]asi_num ;\ mov 12, %g3 ;\ sta %l3, [%sp + %g3]asi_num ;\ add %sp, 16, %g4 ;\ sta %l4, [%g4 + %g0]asi_num ;\ sta %l5, [%g4 + %g1]asi_num ;\ sta %l6, [%g4 + %g2]asi_num ;\ sta %l7, [%g4 + %g3]asi_num ;\ add %g4, 16, %g4 ;\ sta %i0, [%g4 + %g0]asi_num ;\ sta %i1, [%g4 + %g1]asi_num ;\ sta %i2, [%g4 + %g2]asi_num ;\ sta %i3, [%g4 + %g3]asi_num ;\ add %g4, 16, %g4 ;\ sta %i4, [%g4 + %g0]asi_num ;\ sta %i5, [%g4 + %g1]asi_num ;\ sta %i6, [%g4 + %g2]asi_num ;\ sta %i7, [%g4 + %g3]asi_num ;\ TT_TRACE_L(trace_win) ;\ b .spill_clean ;\ mov WSTATE_USER32, %g7 ;\ SKIP(31-25-NI_GL_RESET_TT0_INS-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_32bit_/**/tail ;\ .empty #define SPILL_64clean(asi_num, tail) \ NI_GL_RESET_TT0 ;\ mov 0 + V9BIAS64, %g1 ;\ stxa %l0, [%sp + %g1]asi_num ;\ mov 8 + V9BIAS64, %g2 ;\ stxa %l1, [%sp + %g2]asi_num ;\ mov 16 + V9BIAS64, %g3 ;\ stxa %l2, [%sp + %g3]asi_num ;\ mov 24 + V9BIAS64, %g4 ;\ stxa %l3, [%sp + %g4]asi_num ;\ add %sp, 32, %g5 ;\ stxa %l4, [%g5 + %g1]asi_num ;\ stxa %l5, [%g5 + %g2]asi_num ;\ stxa %l6, [%g5 + %g3]asi_num ;\ stxa %l7, [%g5 + %g4]asi_num ;\ add %g5, 32, %g5 ;\ stxa %i0, [%g5 + %g1]asi_num ;\ stxa %i1, [%g5 + %g2]asi_num ;\ stxa %i2, [%g5 + %g3]asi_num ;\ stxa %i3, [%g5 + %g4]asi_num ;\ add %g5, 32, %g5 ;\ stxa %i4, [%g5 + %g1]asi_num ;\ stxa %i5, [%g5 + %g2]asi_num ;\ stxa %i6, [%g5 + %g3]asi_num ;\ stxa %i7, [%g5 + %g4]asi_num ;\ TT_TRACE_L(trace_win) ;\ b .spill_clean ;\ mov WSTATE_USER64, %g7 ;\ SKIP(31-25-NI_GL_RESET_TT0_INS-TT_TRACE_L_INS) ;\ ba,a,pt %xcc, fault_64bit_/**/tail ;\ .empty /* * Floating point disabled. */ #define FP_DISABLED_TRAP \ TT_TRACE(trace_gen) ;\ ba,pt %xcc,.fp_disabled ;\ nop ;\ .align 32 /* * Floating point exceptions. */ #define FP_IEEE_TRAP \ NI_GL_RESET_TT0 ;\ TT_TRACE(trace_gen) ;\ ba,pt %xcc,.fp_ieee_exception ;\ nop ;\ .align 32 #define FP_TRAP \ NI_GL_RESET_TT0 ;\ TT_TRACE(trace_gen) ;\ ba,pt %xcc,.fp_exception ;\ nop ;\ .align 32 #if !defined(lint) /* * ECACHE_ECC error traps at level 0 and level 1 */ #define ECACHE_ECC(table_name) \ .global table_name ;\ table_name: ;\ membar #Sync ;\ set trap, %g1 ;\ rdpr %tt, %g3 ;\ ba,pt %xcc, sys_trap ;\ sub %g0, 1, %g4 ;\ .align 32 #endif /* !lint */ /* * illegal instruction trap */ #define ILLTRAP_INSTR \ membar #Sync ;\ TT_TRACE(trace_gen) ;\ or %g0, P_UTRAP4, %g2 ;\ or %g0, T_UNIMP_INSTR, %g3 ;\ sethi %hi(.check_v9utrap), %g4 ;\ jmp %g4 + %lo(.check_v9utrap) ;\ nop ;\ .align 32 /* * tag overflow trap */ #define TAG_OVERFLOW \ TT_TRACE(trace_gen) ;\ or %g0, P_UTRAP10, %g2 ;\ or %g0, T_TAG_OVERFLOW, %g3 ;\ sethi %hi(.check_v9utrap), %g4 ;\ jmp %g4 + %lo(.check_v9utrap) ;\ nop ;\ .align 32 /* * divide by zero trap */ #define DIV_BY_ZERO \ TT_TRACE(trace_gen) ;\ or %g0, P_UTRAP11, %g2 ;\ or %g0, T_IDIV0, %g3 ;\ sethi %hi(.check_v9utrap), %g4 ;\ jmp %g4 + %lo(.check_v9utrap) ;\ nop ;\ .align 32 /* * trap instruction for V9 user trap handlers */ #define TRAP_INSTR \ TT_TRACE(trace_gen) ;\ or %g0, T_SOFTWARE_TRAP, %g3 ;\ sethi %hi(.check_v9utrap), %g4 ;\ jmp %g4 + %lo(.check_v9utrap) ;\ nop ;\ .align 32 #define TRP4 TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR /* * LEVEL_INTERRUPT is for level N interrupts. * VECTOR_INTERRUPT is for the vector trap. */ #define LEVEL_INTERRUPT(level) \ .global tt_pil/**/level ;\ tt_pil/**/level: ;\ ba,pt %xcc, pil_interrupt ;\ mov level, %g4 ;\ .align 32 #define LEVEL14_INTERRUPT \ ba pil14_interrupt ;\ mov PIL_14, %g4 ;\ .align 32 #define CPU_MONDO \ ba,a,pt %xcc, cpu_mondo ;\ .align 32 #define DEV_MONDO \ ba,a,pt %xcc, dev_mondo ;\ .align 32 /* * MMU Trap Handlers. */ #define SFSR_CTX_SHIFT 16 #define IMMU_EXCEPTION \ MMU_FAULT_STATUS_AREA(%g3) ;\ rdpr %tpc, %g2 ;\ ldx [%g3 + MMFSA_I_TYPE], %g1 ;\ ldx [%g3 + MMFSA_I_CTX], %g3 ;\ sllx %g3, SFSR_CTX_SHIFT, %g3 ;\ or %g3, %g1, %g3 ;\ ba,pt %xcc, .mmu_exception_end ;\ mov T_INSTR_EXCEPTION, %g1 ;\ .align 32 #define DMMU_EXCEPTION \ MMU_FAULT_STATUS_AREA(%g3) ;\ ldx [%g3 + MMFSA_D_ADDR], %g2 ;\ ldx [%g3 + MMFSA_D_TYPE], %g1 ;\ ldx [%g3 + MMFSA_D_CTX], %g3 ;\ sllx %g3, SFSR_CTX_SHIFT, %g3 ;\ or %g3, %g1, %g3 ;\ ba,pt %xcc, .mmu_exception_end ;\ mov T_DATA_EXCEPTION, %g1 ;\ .align 32 #define DMMU_EXC_AG_PRIV \ MMU_FAULT_STATUS_AREA(%g3) ;\ ldx [%g3 + MMFSA_D_ADDR], %g2 ;\ /* Fault type not available in MMU fault status area */ ;\ mov MMFSA_F_PRVACT, %g1 ;\ ldx [%g3 + MMFSA_D_CTX], %g3 ;\ sllx %g3, SFSR_CTX_SHIFT, %g3 ;\ ba,pt %xcc, .mmu_priv_exception ;\ or %g3, %g1, %g3 ;\ .align 32 #define DMMU_EXC_AG_NOT_ALIGNED \ MMU_FAULT_STATUS_AREA(%g3) ;\ ldx [%g3 + MMFSA_D_ADDR], %g2 ;\ /* Fault type not available in MMU fault status area */ ;\ mov MMFSA_F_UNALIGN, %g1 ;\ ldx [%g3 + MMFSA_D_CTX], %g3 ;\ sllx %g3, SFSR_CTX_SHIFT, %g3 ;\ ba,pt %xcc, .mmu_exception_not_aligned ;\ or %g3, %g1, %g3 ;\ .align 32 /* * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2) */ #define DMMU_EXC_LDDF_NOT_ALIGNED \ ba,a,pt %xcc, .dmmu_exc_lddf_not_aligned ;\ .align 32 #define DMMU_EXC_STDF_NOT_ALIGNED \ ba,a,pt %xcc, .dmmu_exc_stdf_not_aligned ;\ .align 32 #if TAGACC_CTX_MASK != CTXREG_CTX_MASK #error "TAGACC_CTX_MASK != CTXREG_CTX_MASK" #endif #if defined(cscope) /* * Define labels to direct cscope quickly to labels that * are generated by macro expansion of DTLB_MISS(). */ .global tt0_dtlbmiss tt0_dtlbmiss: .global tt1_dtlbmiss tt1_dtlbmiss: nop #endif /* * Data miss handler (must be exactly 32 instructions) * * This handler is invoked only if the hypervisor has been instructed * not to do any TSB walk. * * Kernel and invalid context cases are handled by the sfmmu_kdtlb_miss * handler. * * User TLB miss handling depends upon whether a user process has one or * two TSBs. User TSB information (physical base and size code) is kept * in two dedicated scratchpad registers. Absence of a user TSB (primarily * second TSB) is indicated by a negative value (-1) in that register. */ #define DTLB_MISS(table_name) ;\ .global table_name/**/_dtlbmiss ;\ table_name/**/_dtlbmiss: ;\ HAT_PERCPU_DBSTAT(TSBMISS_DTLBMISS) /* 3 instr ifdef DEBUG */ ;\ MMU_FAULT_STATUS_AREA(%g7) ;\ ldx [%g7 + MMFSA_D_ADDR], %g2 /* address */ ;\ ldx [%g7 + MMFSA_D_CTX], %g3 /* g3 = ctx */ ;\ or %g2, %g3, %g2 /* XXXQ temp */ ;\ cmp %g3, INVALID_CONTEXT ;\ ble,pn %xcc, sfmmu_kdtlb_miss ;\ srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ mov SCRATCHPAD_UTSBREG2, %g1 ;\ ldxa [%g1]ASI_SCRATCHPAD, %g1 /* get 2nd tsbreg */ ;\ brgez,pn %g1, sfmmu_udtlb_slowpath /* brnach if 2 TSBs */ ;\ nop ;\ GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) /* 11 instr */ ;\ ba,pt %xcc, sfmmu_udtlb_fastpath /* no 4M TSB, miss */ ;\ srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ .align 128 #if defined(cscope) /* * Define labels to direct cscope quickly to labels that * are generated by macro expansion of ITLB_MISS(). */ .global tt0_itlbmiss tt0_itlbmiss: .global tt1_itlbmiss tt1_itlbmiss: nop #endif /* * Instruction miss handler. * * This handler is invoked only if the hypervisor has been instructed * not to do any TSB walk. * * ldda instructions will have their ASI patched * by sfmmu_patch_ktsb at runtime. * MUST be EXACTLY 32 instructions or we'll break. */ #define ITLB_MISS(table_name) \ .global table_name/**/_itlbmiss ;\ table_name/**/_itlbmiss: ;\ HAT_PERCPU_DBSTAT(TSBMISS_ITLBMISS) /* 3 instr ifdef DEBUG */ ;\ MMU_FAULT_STATUS_AREA(%g7) ;\ ldx [%g7 + MMFSA_I_ADDR], %g2 /* g2 = address */ ;\ ldx [%g7 + MMFSA_I_CTX], %g3 /* g3 = ctx */ ;\ or %g2, %g3, %g2 /* XXXQ temp */ ;\ cmp %g3, INVALID_CONTEXT ;\ ble,pn %xcc, sfmmu_kitlb_miss ;\ srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ mov SCRATCHPAD_UTSBREG2, %g1 ;\ ldxa [%g1]ASI_SCRATCHPAD, %g1 /* get 2nd tsbreg */ ;\ brgez,pn %g1, sfmmu_uitlb_slowpath /* branch if 2 TSBS */ ;\ nop ;\ GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) /* 11 instr */ ;\ ba,pt %xcc, sfmmu_uitlb_fastpath /* no 4M TSB, miss */ ;\ srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ .align 128 #define DTSB_MISS \ GOTO_TT(sfmmu_slow_dmmu_miss,trace_dmmu) #define ITSB_MISS \ GOTO_TT(sfmmu_slow_immu_miss,trace_immu) /* * This macro is the first level handler for fast protection faults. * It first demaps the tlb entry which generated the fault and then * attempts to set the modify bit on the hash. It needs to be * exactly 32 instructions. */ #define DTLB_PROT \ MMU_FAULT_STATUS_AREA(%g7) ;\ ldx [%g7 + MMFSA_D_ADDR], %g2 /* address */ ;\ ldx [%g7 + MMFSA_D_CTX], %g3 /* %g3 = ctx */ ;\ or %g2, %g3, %g2 /* XXXQ temp */ ;\ /* ;\ * g2 = tag access register ;\ * g3 = ctx number ;\ */ ;\ TT_TRACE(trace_dataprot) /* 2 instr ifdef TRAPTRACE */ ;\ /* clobbers g1 and g6 XXXQ? */ ;\ brnz,pt %g3, sfmmu_uprot_trap /* user trap */ ;\ nop ;\ ba,a,pt %xcc, sfmmu_kprot_trap /* kernel trap */ ;\ .align 128 #define DMMU_EXCEPTION_TL1 ;\ ba,a,pt %xcc, mmu_trap_tl1 ;\ .align 32 #define MISALIGN_ADDR_TL1 ;\ ba,a,pt %xcc, mmu_trap_tl1 ;\ .align 32 /* * Trace a tsb hit * g1 = tsbe pointer (in/clobbered) * g2 = tag access register (in) * g3 - g4 = scratch (clobbered) * g5 = tsbe data (in) * g6 = scratch (clobbered) * g7 = pc we jumped here from (in) * ttextra = value to OR in to trap type (%tt) (in) */ #ifdef TRAPTRACE #define TRACE_TSBHIT(ttextra) \ membar #Sync ;\ sethi %hi(FLUSH_ADDR), %g6 ;\ flush %g6 ;\ TRACE_PTR(%g3, %g6) ;\ GET_TRACE_TICK(%g6) ;\ stxa %g6, [%g3 + TRAP_ENT_TICK]%asi ;\ stna %g2, [%g3 + TRAP_ENT_SP]%asi /* tag access */ ;\ stna %g5, [%g3 + TRAP_ENT_F1]%asi /* tsb data */ ;\ rdpr %tnpc, %g6 ;\ stna %g6, [%g3 + TRAP_ENT_F2]%asi ;\ stna %g1, [%g3 + TRAP_ENT_F3]%asi /* tsb pointer */ ;\ stna %g0, [%g3 + TRAP_ENT_F4]%asi ;\ rdpr %tpc, %g6 ;\ stna %g6, [%g3 + TRAP_ENT_TPC]%asi ;\ TRACE_SAVE_TL_GL_REGS(%g3, %g6) ;\ rdpr %tt, %g6 ;\ or %g6, (ttextra), %g1 ;\ stha %g1, [%g3 + TRAP_ENT_TT]%asi ;\ MMU_FAULT_STATUS_AREA(%g4) ;\ ldx [%g4 + MMFSA_I_ADDR], %g1 ;\ ldx [%g4 + MMFSA_D_ADDR], %g4 ;\ cmp %g6, FAST_IMMU_MISS_TT ;\ move %icc, %g1, %g4 ;\ cmp %g6, T_INSTR_MMU_MISS ;\ move %icc, %g1, %g4 ;\ stxa %g4, [%g3 + TRAP_ENT_TSTATE]%asi /* fault addr */ ;\ MMU_FAULT_STATUS_AREA(%g4) ;\ ldx [%g4 + MMFSA_I_CTX], %g1 ;\ ldx [%g4 + MMFSA_D_CTX], %g4 ;\ cmp %g6, FAST_IMMU_MISS_TT ;\ move %icc, %g1, %g4 ;\ cmp %g6, T_INSTR_MMU_MISS ;\ move %icc, %g1, %g4 ;\ stna %g4, [%g3 + TRAP_ENT_TR]%asi ;\ TRACE_NEXT(%g3, %g4, %g6) #else #define TRACE_TSBHIT(ttextra) #endif #if defined(lint) struct scb trap_table; struct scb scb; /* trap_table/scb are the same object */ #else /* lint */ /* * ======================================================================= * SPARC V9 TRAP TABLE * * The trap table is divided into two halves: the first half is used when * taking traps when TL=0; the second half is used when taking traps from * TL>0. Note that handlers in the second half of the table might not be able * to make the same assumptions as handlers in the first half of the table. * * Worst case trap nesting so far: * * at TL=0 client issues software trap requesting service * at TL=1 nucleus wants a register window * at TL=2 register window clean/spill/fill takes a TLB miss * at TL=3 processing TLB miss * at TL=4 handle asynchronous error * * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode". * * ======================================================================= */ .section ".text" .align 4 .global trap_table, scb, trap_table0, trap_table1, etrap_table .type trap_table, #function .type trap_table0, #function .type trap_table1, #function .type scb, #function trap_table: scb: trap_table0: /* hardware traps */ NOT; /* 000 reserved */ RED; /* 001 power on reset */ RED; /* 002 watchdog reset */ RED; /* 003 externally initiated reset */ RED; /* 004 software initiated reset */ RED; /* 005 red mode exception */ NOT; NOT; /* 006 - 007 reserved */ IMMU_EXCEPTION; /* 008 instruction access exception */ ITSB_MISS; /* 009 instruction access MMU miss */ NOT; /* 00A reserved */ NOT; NOT4; /* 00B - 00F reserved */ ILLTRAP_INSTR; /* 010 illegal instruction */ TRAP(T_PRIV_INSTR); /* 011 privileged opcode */ NOT; /* 012 unimplemented LDD */ NOT; /* 013 unimplemented STD */ NOT4; NOT4; NOT4; /* 014 - 01F reserved */ FP_DISABLED_TRAP; /* 020 fp disabled */ FP_IEEE_TRAP; /* 021 fp exception ieee 754 */ FP_TRAP; /* 022 fp exception other */ TAG_OVERFLOW; /* 023 tag overflow */ CLEAN_WINDOW; /* 024 - 027 clean window */ DIV_BY_ZERO; /* 028 division by zero */ NOT; /* 029 internal processor error */ NOT; NOT; NOT4; /* 02A - 02F reserved */ DMMU_EXCEPTION; /* 030 data access exception */ DTSB_MISS; /* 031 data access MMU miss */ NOT; /* 032 reserved */ NOT; /* 033 data access protection */ DMMU_EXC_AG_NOT_ALIGNED; /* 034 mem address not aligned */ DMMU_EXC_LDDF_NOT_ALIGNED; /* 035 LDDF mem address not aligned */ DMMU_EXC_STDF_NOT_ALIGNED; /* 036 STDF mem address not aligned */ DMMU_EXC_AG_PRIV; /* 037 privileged action */ NOT; /* 038 LDQF mem address not aligned */ NOT; /* 039 STQF mem address not aligned */ NOT; NOT; NOT4; /* 03A - 03F reserved */ NOT; /* 040 async data error */ LEVEL_INTERRUPT(1); /* 041 interrupt level 1 */ LEVEL_INTERRUPT(2); /* 042 interrupt level 2 */ LEVEL_INTERRUPT(3); /* 043 interrupt level 3 */ LEVEL_INTERRUPT(4); /* 044 interrupt level 4 */ LEVEL_INTERRUPT(5); /* 045 interrupt level 5 */ LEVEL_INTERRUPT(6); /* 046 interrupt level 6 */ LEVEL_INTERRUPT(7); /* 047 interrupt level 7 */ LEVEL_INTERRUPT(8); /* 048 interrupt level 8 */ LEVEL_INTERRUPT(9); /* 049 interrupt level 9 */ LEVEL_INTERRUPT(10); /* 04A interrupt level 10 */ LEVEL_INTERRUPT(11); /* 04B interrupt level 11 */ LEVEL_INTERRUPT(12); /* 04C interrupt level 12 */ LEVEL_INTERRUPT(13); /* 04D interrupt level 13 */ LEVEL14_INTERRUPT; /* 04E interrupt level 14 */ LEVEL_INTERRUPT(15); /* 04F interrupt level 15 */ NOT4; NOT4; NOT4; NOT4; /* 050 - 05F reserved */ NOT; /* 060 interrupt vector */ GOTO(kmdb_trap); /* 061 PA watchpoint */ GOTO(kmdb_trap); /* 062 VA watchpoint */ NOT; /* 063 reserved */ ITLB_MISS(tt0); /* 064 instruction access MMU miss */ DTLB_MISS(tt0); /* 068 data access MMU miss */ DTLB_PROT; /* 06C data access protection */ NOT; /* 070 reserved */ NOT; /* 071 reserved */ NOT; /* 072 reserved */ NOT; /* 073 reserved */ NOT4; NOT4 /* 074 - 07B reserved */ CPU_MONDO; /* 07C cpu_mondo */ DEV_MONDO; /* 07D dev_mondo */ GOTO_TT(resumable_error, trace_gen); /* 07E resumable error */ GOTO_TT(nonresumable_error, trace_gen); /* 07F non-reasumable error */ NOT4; /* 080 spill 0 normal */ SPILL_32bit_asi(ASI_AIUP,sn0); /* 084 spill 1 normal */ SPILL_64bit_asi(ASI_AIUP,sn0); /* 088 spill 2 normal */ SPILL_32clean(ASI_AIUP,sn0); /* 08C spill 3 normal */ SPILL_64clean(ASI_AIUP,sn0); /* 090 spill 4 normal */ SPILL_32bit(not); /* 094 spill 5 normal */ SPILL_64bit(not); /* 098 spill 6 normal */ SPILL_mixed; /* 09C spill 7 normal */ NOT4; /* 0A0 spill 0 other */ SPILL_32bit_asi(ASI_AIUS,so0); /* 0A4 spill 1 other */ SPILL_64bit_asi(ASI_AIUS,so0); /* 0A8 spill 2 other */ SPILL_32bit_asi(ASI_AIUS,so0); /* 0AC spill 3 other */ SPILL_64bit_asi(ASI_AIUS,so0); /* 0B0 spill 4 other */ NOT4; /* 0B4 spill 5 other */ NOT4; /* 0B8 spill 6 other */ NOT4; /* 0BC spill 7 other */ NOT4; /* 0C0 fill 0 normal */ FILL_32bit_asi(ASI_AIUP,fn0); /* 0C4 fill 1 normal */ FILL_64bit_asi(ASI_AIUP,fn0); /* 0C8 fill 2 normal */ FILL_32bit_asi(ASI_AIUP,fn0); /* 0CC fill 3 normal */ FILL_64bit_asi(ASI_AIUP,fn0); /* 0D0 fill 4 normal */ FILL_32bit(not); /* 0D4 fill 5 normal */ FILL_64bit(not); /* 0D8 fill 6 normal */ FILL_mixed; /* 0DC fill 7 normal */ NOT4; /* 0E0 fill 0 other */ NOT4; /* 0E4 fill 1 other */ NOT4; /* 0E8 fill 2 other */ NOT4; /* 0EC fill 3 other */ NOT4; /* 0F0 fill 4 other */ NOT4; /* 0F4 fill 5 other */ NOT4; /* 0F8 fill 6 other */ NOT4; /* 0FC fill 7 other */ /* user traps */ GOTO(syscall_trap_4x); /* 100 old system call */ TRAP(T_BREAKPOINT); /* 101 user breakpoint */ TRAP(T_DIV0); /* 102 user divide by zero */ GOTO(.flushw); /* 103 flush windows */ GOTO(.clean_windows); /* 104 clean windows */ BAD; /* 105 range check ?? */ GOTO(.fix_alignment); /* 106 do unaligned references */ BAD; /* 107 unused */ SYSCALL(syscall_trap32); /* 108 ILP32 system call on LP64 */ GOTO(set_trap0_addr); /* 109 set trap0 address */ BAD; BAD; BAD4; /* 10A - 10F unused */ TRP4; TRP4; TRP4; TRP4; /* 110 - 11F V9 user trap handlers */ GOTO(.getcc); /* 120 get condition codes */ GOTO(.setcc); /* 121 set condition codes */ GOTO(.getpsr); /* 122 get psr */ GOTO(.setpsr); /* 123 set psr (some fields) */ GOTO(get_timestamp); /* 124 get timestamp */ GOTO(get_virtime); /* 125 get lwp virtual time */ PRIV(self_xcall); /* 126 self xcall */ GOTO(get_hrestime); /* 127 get hrestime */ BAD; /* 128 ST_SETV9STACK */ GOTO(.getlgrp); /* 129 get lgrpid */ BAD; BAD; BAD4; /* 12A - 12F unused */ BAD4; BAD4; /* 130 - 137 unused */ DTRACE_PID; /* 138 dtrace pid tracing provider */ DTRACE_FASTTRAP; /* 139 dtrace fasttrap provider */ DTRACE_RETURN; /* 13A dtrace pid return probe */ BAD; BAD4; /* 13B - 13F unused */ SYSCALL(syscall_trap) /* 140 LP64 system call */ BAD; /* 141 unused */ #ifdef DEBUG_USER_TRAPTRACECTL GOTO(.traptrace_freeze); /* 142 freeze traptrace */ GOTO(.traptrace_unfreeze); /* 143 unfreeze traptrace */ #else BAD; BAD; /* 142 - 143 unused */ #endif BAD4; BAD4; BAD4; /* 144 - 14F unused */ BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */ BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */ BAD; /* 170 - unused */ BAD; /* 171 - unused */ BAD; BAD; /* 172 - 173 unused */ BAD4; BAD4; /* 174 - 17B unused */ #ifdef PTL1_PANIC_DEBUG mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic); /* 17C test ptl1_panic */ #else BAD; /* 17C unused */ #endif /* PTL1_PANIC_DEBUG */ PRIV(kmdb_trap); /* 17D kmdb enter (L1-A) */ PRIV(kmdb_trap); /* 17E kmdb breakpoint */ PRIV(obp_bpt); /* 17F obp breakpoint */ /* reserved */ NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */ NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */ NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */ .size trap_table0, (.-trap_table0) trap_table1: NOT4; NOT4; /* 000 - 007 unused */ NOT; /* 008 instruction access exception */ ITSB_MISS; /* 009 instruction access MMU miss */ NOT; /* 00A reserved */ NOT; NOT4; /* 00B - 00F unused */ NOT4; NOT4; NOT4; NOT4; /* 010 - 01F unused */ NOT4; /* 020 - 023 unused */ CLEAN_WINDOW; /* 024 - 027 clean window */ NOT4; NOT4; /* 028 - 02F unused */ DMMU_EXCEPTION_TL1; /* 030 data access exception */ DTSB_MISS; /* 031 data access MMU miss */ NOT; /* 032 reserved */ NOT; /* 033 unused */ MISALIGN_ADDR_TL1; /* 034 mem address not aligned */ NOT; NOT; NOT; NOT4; NOT4 /* 035 - 03F unused */ NOT4; NOT4; NOT4; NOT4; /* 040 - 04F unused */ NOT4; NOT4; NOT4; NOT4; /* 050 - 05F unused */ NOT; /* 060 unused */ GOTO(kmdb_trap_tl1); /* 061 PA watchpoint */ GOTO(kmdb_trap_tl1); /* 062 VA watchpoint */ NOT; /* 063 reserved */ ITLB_MISS(tt1); /* 064 instruction access MMU miss */ DTLB_MISS(tt1); /* 068 data access MMU miss */ DTLB_PROT; /* 06C data access protection */ NOT; /* 070 reserved */ NOT; /* 071 reserved */ NOT; /* 072 reserved */ NOT; /* 073 reserved */ NOT4; NOT4; NOT4; /* 074 - 07F reserved */ NOTP4; /* 080 spill 0 normal */ SPILL_32bit_tt1(ASI_AIUP,sn1); /* 084 spill 1 normal */ SPILL_64bit_tt1(ASI_AIUP,sn1); /* 088 spill 2 normal */ SPILL_32bit_tt1(ASI_AIUP,sn1); /* 08C spill 3 normal */ SPILL_64bit_tt1(ASI_AIUP,sn1); /* 090 spill 4 normal */ NOTP4; /* 094 spill 5 normal */ SPILL_64bit_ktt1(sk); /* 098 spill 6 normal */ SPILL_mixed_ktt1(sk); /* 09C spill 7 normal */ NOTP4; /* 0A0 spill 0 other */ SPILL_32bit_tt1(ASI_AIUS,so1); /* 0A4 spill 1 other */ SPILL_64bit_tt1(ASI_AIUS,so1); /* 0A8 spill 2 other */ SPILL_32bit_tt1(ASI_AIUS,so1); /* 0AC spill 3 other */ SPILL_64bit_tt1(ASI_AIUS,so1); /* 0B0 spill 4 other */ NOTP4; /* 0B4 spill 5 other */ NOTP4; /* 0B8 spill 6 other */ NOTP4; /* 0BC spill 7 other */ NOT4; /* 0C0 fill 0 normal */ NOT4; /* 0C4 fill 1 normal */ NOT4; /* 0C8 fill 2 normal */ NOT4; /* 0CC fill 3 normal */ NOT4; /* 0D0 fill 4 normal */ NOT4; /* 0D4 fill 5 normal */ NOT4; /* 0D8 fill 6 normal */ NOT4; /* 0DC fill 7 normal */ NOT4; NOT4; NOT4; NOT4; /* 0E0 - 0EF unused */ NOT4; NOT4; NOT4; NOT4; /* 0F0 - 0FF unused */ /* * Code running at TL>0 does not use soft traps, so * we can truncate the table here. * However: * sun4v uses (hypervisor) ta instructions at TL > 0, so * provide a safety net for now. */ /* soft traps */ BAD4; BAD4; BAD4; BAD4; /* 100 - 10F unused */ BAD4; BAD4; BAD4; BAD4; /* 110 - 11F unused */ BAD4; BAD4; BAD4; BAD4; /* 120 - 12F unused */ BAD4; BAD4; BAD4; BAD4; /* 130 - 13F unused */ BAD4; BAD4; BAD4; BAD4; /* 140 - 14F unused */ BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */ BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */ BAD4; BAD4; BAD4; BAD4; /* 170 - 17F unused */ /* reserved */ NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */ NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */ NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */ NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */ etrap_table: .size trap_table1, (.-trap_table1) .size trap_table, (.-trap_table) .size scb, (.-scb) /* * We get to exec_fault in the case of an instruction miss and tte * has no execute bit set. We go to tl0 to handle it. * * g1 = tsbe pointer (in/clobbered) * g2 = tag access register (in) * g3 - g4 = scratch (clobbered) * g5 = tsbe data (in) * g6 = scratch (clobbered) * g7 = pc we jumped here from (in) */ ALTENTRY(exec_fault) TRACE_TSBHIT(TT_MMU_EXEC) MMU_FAULT_STATUS_AREA(%g4) ldx [%g4 + MMFSA_I_ADDR], %g2 /* g2 = address */ ldx [%g4 + MMFSA_I_CTX], %g3 /* g3 = ctx */ srlx %g2, MMU_PAGESHIFT, %g2 ! align address to page boundry sllx %g2, MMU_PAGESHIFT, %g2 or %g2, %g3, %g2 /* XXXQ temp */ mov T_INSTR_MMU_MISS, %g3 ! arg2 = traptype set trap, %g1 ba,pt %xcc, sys_trap mov -1, %g4 .mmu_exception_not_aligned: /* %g2 = sfar, %g3 = sfsr */ rdpr %tstate, %g1 btst TSTATE_PRIV, %g1 bnz,pn %icc, 2f nop CPU_ADDR(%g1, %g4) ! load CPU struct addr ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer ldn [%g1 + T_PROCP], %g1 ! load proc pointer ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? brz,pt %g5, 2f nop ldn [%g5 + P_UTRAP15], %g5 ! unaligned utrap? brz,pn %g5, 2f nop btst 1, %sp bz,pt %xcc, 1f ! 32 bit user program nop ba,pt %xcc, .setup_v9utrap ! 64 bit user program nop 1: ba,pt %xcc, .setup_utrap or %g2, %g0, %g7 2: ba,pt %xcc, .mmu_exception_end mov T_ALIGNMENT, %g1 .mmu_priv_exception: rdpr %tstate, %g1 btst TSTATE_PRIV, %g1 bnz,pn %icc, 1f nop CPU_ADDR(%g1, %g4) ! load CPU struct addr ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer ldn [%g1 + T_PROCP], %g1 ! load proc pointer ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? brz,pt %g5, 1f nop ldn [%g5 + P_UTRAP16], %g5 brnz,pt %g5, .setup_v9utrap nop 1: mov T_PRIV_INSTR, %g1 .mmu_exception_end: CPU_INDEX(%g4, %g5) set cpu_core, %g5 sllx %g4, CPU_CORE_SHIFT, %g4 add %g4, %g5, %g4 lduh [%g4 + CPUC_DTRACE_FLAGS], %g5 andcc %g5, CPU_DTRACE_NOFAULT, %g0 bz 1f or %g5, CPU_DTRACE_BADADDR, %g5 stuh %g5, [%g4 + CPUC_DTRACE_FLAGS] done 1: sllx %g3, 32, %g3 or %g3, %g1, %g3 set trap, %g1 ba,pt %xcc, sys_trap sub %g0, 1, %g4 .fp_disabled: CPU_ADDR(%g1, %g4) ! load CPU struct addr ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer rdpr %tstate, %g4 btst TSTATE_PRIV, %g4 bnz,a,pn %icc, ptl1_panic mov PTL1_BAD_FPTRAP, %g1 ldn [%g1 + T_PROCP], %g1 ! load proc pointer ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? brz,a,pt %g5, 2f nop ldn [%g5 + P_UTRAP7], %g5 ! fp_disabled utrap? brz,a,pn %g5, 2f nop btst 1, %sp bz,a,pt %xcc, 1f ! 32 bit user program nop ba,a,pt %xcc, .setup_v9utrap ! 64 bit user program nop 1: ba,pt %xcc, .setup_utrap or %g0, %g0, %g7 2: set fp_disabled, %g1 ba,pt %xcc, sys_trap sub %g0, 1, %g4 .fp_ieee_exception: rdpr %tstate, %g1 btst TSTATE_PRIV, %g1 bnz,a,pn %icc, ptl1_panic mov PTL1_BAD_FPTRAP, %g1 CPU_ADDR(%g1, %g4) ! load CPU struct addr stx %fsr, [%g1 + CPU_TMP1] ldx [%g1 + CPU_TMP1], %g2 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer ldn [%g1 + T_PROCP], %g1 ! load proc pointer ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? brz,a,pt %g5, 1f nop ldn [%g5 + P_UTRAP8], %g5 brnz,a,pt %g5, .setup_v9utrap nop 1: set _fp_ieee_exception, %g1 ba,pt %xcc, sys_trap sub %g0, 1, %g4 /* * Register Inputs: * %g5 user trap handler * %g7 misaligned addr - for alignment traps only */ .setup_utrap: set trap, %g1 ! setup in case we go mov T_FLUSH_PCB, %g3 ! through sys_trap on sub %g0, 1, %g4 ! the save instruction below /* * If the DTrace pid provider is single stepping a copied-out * instruction, t->t_dtrace_step will be set. In that case we need * to abort the single-stepping (since execution of the instruction * was interrupted) and use the value of t->t_dtrace_npc as the %npc. */ save %sp, -SA(MINFRAME32), %sp ! window for trap handler CPU_ADDR(%g1, %g4) ! load CPU struct addr ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step rdpr %tnpc, %l2 ! arg1 == tnpc brz,pt %g2, 1f rdpr %tpc, %l1 ! arg0 == tpc ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast ldn [%g1 + T_DTRACE_NPC], %l2 ! arg1 = t->t_dtrace_npc (step) brz,pt %g2, 1f st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast 1: mov %g7, %l3 ! arg2 == misaligned address rdpr %tstate, %g1 ! cwp for trap handler rdpr %cwp, %g4 bclr TSTATE_CWP_MASK, %g1 wrpr %g1, %g4, %tstate wrpr %g0, %g5, %tnpc ! trap handler address FAST_TRAP_DONE /* NOTREACHED */ .check_v9utrap: rdpr %tstate, %g1 btst TSTATE_PRIV, %g1 bnz,a,pn %icc, 3f nop CPU_ADDR(%g4, %g1) ! load CPU struct addr ldn [%g4 + CPU_THREAD], %g5 ! load thread pointer ldn [%g5 + T_PROCP], %g5 ! load proc pointer ldn [%g5 + P_UTRAPS], %g5 ! are there utraps? cmp %g3, T_SOFTWARE_TRAP bne,a,pt %icc, 1f nop brz,pt %g5, 3f ! if p_utraps == NULL goto trap() rdpr %tt, %g3 ! delay - get actual hw trap type sub %g3, 254, %g1 ! UT_TRAP_INSTRUCTION_16 = p_utraps[18] ba,pt %icc, 2f smul %g1, CPTRSIZE, %g2 1: brz,a,pt %g5, 3f ! if p_utraps == NULL goto trap() nop cmp %g3, T_UNIMP_INSTR bne,a,pt %icc, 2f nop mov 1, %g1 st %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR rdpr %tpc, %g1 ! ld trapping instruction using lduwa [%g1]ASI_AIUP, %g1 ! "AS IF USER" ASI which could fault st %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR sethi %hi(0xc1c00000), %g4 ! setup mask for illtrap instruction andcc %g1, %g4, %g4 ! and instruction with mask bnz,a,pt %icc, 3f ! if %g4 == zero, %g1 is an ILLTRAP nop ! fall thru to setup 2: ldn [%g5 + %g2], %g5 brnz,a,pt %g5, .setup_v9utrap nop 3: set trap, %g1 ba,pt %xcc, sys_trap sub %g0, 1, %g4 /* NOTREACHED */ /* * Register Inputs: * %g5 user trap handler */ .setup_v9utrap: set trap, %g1 ! setup in case we go mov T_FLUSH_PCB, %g3 ! through sys_trap on sub %g0, 1, %g4 ! the save instruction below /* * If the DTrace pid provider is single stepping a copied-out * instruction, t->t_dtrace_step will be set. In that case we need * to abort the single-stepping (since execution of the instruction * was interrupted) and use the value of t->t_dtrace_npc as the %npc. */ save %sp, -SA(MINFRAME64), %sp ! window for trap handler CPU_ADDR(%g1, %g4) ! load CPU struct addr ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step rdpr %tnpc, %l7 ! arg1 == tnpc brz,pt %g2, 1f rdpr %tpc, %l6 ! arg0 == tpc ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast ldn [%g1 + T_DTRACE_NPC], %l7 ! arg1 == t->t_dtrace_npc (step) brz,pt %g2, 1f st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast 1: rdpr %tstate, %g2 ! cwp for trap handler rdpr %cwp, %g4 bclr TSTATE_CWP_MASK, %g2 wrpr %g2, %g4, %tstate ldn [%g1 + T_PROCP], %g4 ! load proc pointer ldn [%g4 + P_AS], %g4 ! load as pointer ldn [%g4 + A_USERLIMIT], %g4 ! load as userlimit cmp %l7, %g4 ! check for single-step set bne,pt %xcc, 4f nop ldn [%g1 + T_LWP], %g1 ! load klwp pointer ld [%g1 + PCB_STEP], %g4 ! load single-step flag cmp %g4, STEP_ACTIVE ! step flags set in pcb? bne,pt %icc, 4f nop stn %g5, [%g1 + PCB_TRACEPC] ! save trap handler addr in pcb mov %l7, %g4 ! on entry to precise user trap add %l6, 4, %l7 ! handler, %l6 == pc, %l7 == npc ! at time of trap wrpr %g0, %g4, %tnpc ! generate FLTBOUNDS, ! %g4 == userlimit FAST_TRAP_DONE /* NOTREACHED */ 4: wrpr %g0, %g5, %tnpc ! trap handler address FAST_TRAP_DONE_CHK_INTR /* NOTREACHED */ .fp_exception: CPU_ADDR(%g1, %g4) stx %fsr, [%g1 + CPU_TMP1] ldx [%g1 + CPU_TMP1], %g2 /* * Cheetah takes unfinished_FPop trap for certain range of operands * to the "fitos" instruction. Instead of going through the slow * software emulation path, we try to simulate the "fitos" instruction * via "fitod" and "fdtos" provided the following conditions are met: * * fpu_exists is set (if DEBUG) * not in privileged mode * ftt is unfinished_FPop * NXM IEEE trap is not enabled * instruction at %tpc is "fitos" * * Usage: * %g1 per cpu address * %g2 %fsr * %g6 user instruction * * Note that we can take a memory access related trap while trying * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR * flag to catch those traps and let the SFMMU code deal with page * fault and data access exception. */ #if defined(DEBUG) || defined(NEED_FPU_EXISTS) sethi %hi(fpu_exists), %g7 ld [%g7 + %lo(fpu_exists)], %g7 brz,pn %g7, .fp_exception_cont nop #endif rdpr %tstate, %g7 ! branch if in privileged mode btst TSTATE_PRIV, %g7 bnz,pn %xcc, .fp_exception_cont srl %g2, FSR_FTT_SHIFT, %g7 ! extract ftt from %fsr and %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7 cmp %g7, FTT_UNFIN set FSR_TEM_NX, %g5 bne,pn %xcc, .fp_exception_cont ! branch if NOT unfinished_FPop andcc %g2, %g5, %g0 bne,pn %xcc, .fp_exception_cont ! branch if FSR_TEM_NX enabled rdpr %tpc, %g5 ! get faulting PC or %g0, 1, %g7 st %g7, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag lda [%g5]ASI_USER, %g6 ! get user's instruction st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag set FITOS_INSTR_MASK, %g7 and %g6, %g7, %g7 set FITOS_INSTR, %g5 cmp %g7, %g5 bne,pn %xcc, .fp_exception_cont ! branch if not FITOS_INSTR nop /* * This is unfinished FPops trap for "fitos" instruction. We * need to simulate "fitos" via "fitod" and "fdtos" instruction * sequence. * * We need a temporary FP register to do the conversion. Since * both source and destination operands for the "fitos" instruction * have to be within %f0-%f31, we use an FP register from the upper * half to guarantee that it won't collide with the source or the * dest operand. However, we do have to save and restore its value. * * We use %d62 as a temporary FP register for the conversion and * branch to appropriate instruction within the conversion tables * based upon the rs2 and rd values. */ std %d62, [%g1 + CPU_TMP1] ! save original value srl %g6, FITOS_RS2_SHIFT, %g7 and %g7, FITOS_REG_MASK, %g7 set _fitos_fitod_table, %g4 sllx %g7, 2, %g7 jmp %g4 + %g7 ba,pt %xcc, _fitos_fitod_done .empty _fitos_fitod_table: fitod %f0, %d62 fitod %f1, %d62 fitod %f2, %d62 fitod %f3, %d62 fitod %f4, %d62 fitod %f5, %d62 fitod %f6, %d62 fitod %f7, %d62 fitod %f8, %d62 fitod %f9, %d62 fitod %f10, %d62 fitod %f11, %d62 fitod %f12, %d62 fitod %f13, %d62 fitod %f14, %d62 fitod %f15, %d62 fitod %f16, %d62 fitod %f17, %d62 fitod %f18, %d62 fitod %f19, %d62 fitod %f20, %d62 fitod %f21, %d62 fitod %f22, %d62 fitod %f23, %d62 fitod %f24, %d62 fitod %f25, %d62 fitod %f26, %d62 fitod %f27, %d62 fitod %f28, %d62 fitod %f29, %d62 fitod %f30, %d62 fitod %f31, %d62 _fitos_fitod_done: /* * Now convert data back into single precision */ srl %g6, FITOS_RD_SHIFT, %g7 and %g7, FITOS_REG_MASK, %g7 set _fitos_fdtos_table, %g4 sllx %g7, 2, %g7 jmp %g4 + %g7 ba,pt %xcc, _fitos_fdtos_done .empty _fitos_fdtos_table: fdtos %d62, %f0 fdtos %d62, %f1 fdtos %d62, %f2 fdtos %d62, %f3 fdtos %d62, %f4 fdtos %d62, %f5 fdtos %d62, %f6 fdtos %d62, %f7 fdtos %d62, %f8 fdtos %d62, %f9 fdtos %d62, %f10 fdtos %d62, %f11 fdtos %d62, %f12 fdtos %d62, %f13 fdtos %d62, %f14 fdtos %d62, %f15 fdtos %d62, %f16 fdtos %d62, %f17 fdtos %d62, %f18 fdtos %d62, %f19 fdtos %d62, %f20 fdtos %d62, %f21 fdtos %d62, %f22 fdtos %d62, %f23 fdtos %d62, %f24 fdtos %d62, %f25 fdtos %d62, %f26 fdtos %d62, %f27 fdtos %d62, %f28 fdtos %d62, %f29 fdtos %d62, %f30 fdtos %d62, %f31 _fitos_fdtos_done: ldd [%g1 + CPU_TMP1], %d62 ! restore %d62 #if DEBUG /* * Update FPop_unfinished trap kstat */ set fpustat+FPUSTAT_UNFIN_KSTAT, %g7 ldx [%g7], %g5 1: add %g5, 1, %g6 casxa [%g7] ASI_N, %g5, %g6 cmp %g5, %g6 bne,a,pn %xcc, 1b or %g0, %g6, %g5 /* * Update fpu_sim_fitos kstat */ set fpuinfo+FPUINFO_FITOS_KSTAT, %g7 ldx [%g7], %g5 1: add %g5, 1, %g6 casxa [%g7] ASI_N, %g5, %g6 cmp %g5, %g6 bne,a,pn %xcc, 1b or %g0, %g6, %g5 #endif /* DEBUG */ FAST_TRAP_DONE .fp_exception_cont: /* * Let _fp_exception deal with simulating FPop instruction. * Note that we need to pass %fsr in %g2 (already read above). */ set _fp_exception, %g1 ba,pt %xcc, sys_trap sub %g0, 1, %g4 /* * Register windows */ /* * FILL_32bit_flushw/FILL_64bit_flushw fills a 32/64-bit-wide register window * from a 32/64-bit * wide address space via the designated asi. * It is used to fill windows in user_flushw to avoid going above TL 2. */ /* TODO: Use the faster FILL based on FILL_32bit_asi/FILL_64bit_asi */ #define FILL_32bit_flushw(asi_num) \ mov asi_num, %asi ;\ rdpr %cwp, %g2 ;\ sub %g2, 1, %g2 ;\ wrpr %g2, %cwp ;\ 1: srl %sp, 0, %sp ;\ lda [%sp + 0]%asi, %l0 ;\ lda [%sp + 4]%asi, %l1 ;\ lda [%sp + 8]%asi, %l2 ;\ lda [%sp + 12]%asi, %l3 ;\ lda [%sp + 16]%asi, %l4 ;\ lda [%sp + 20]%asi, %l5 ;\ lda [%sp + 24]%asi, %l6 ;\ lda [%sp + 28]%asi, %l7 ;\ lda [%sp + 32]%asi, %i0 ;\ lda [%sp + 36]%asi, %i1 ;\ lda [%sp + 40]%asi, %i2 ;\ lda [%sp + 44]%asi, %i3 ;\ lda [%sp + 48]%asi, %i4 ;\ lda [%sp + 52]%asi, %i5 ;\ lda [%sp + 56]%asi, %i6 ;\ lda [%sp + 60]%asi, %i7 ;\ restored ;\ add %g2, 1, %g2 ;\ wrpr %g2, %cwp #define FILL_64bit_flushw(asi_num) \ mov asi_num, %asi ;\ rdpr %cwp, %g2 ;\ sub %g2, 1, %g2 ;\ wrpr %g2, %cwp ;\ ldxa [%sp + V9BIAS64 + 0]%asi, %l0 ;\ ldxa [%sp + V9BIAS64 + 8]%asi, %l1 ;\ ldxa [%sp + V9BIAS64 + 16]%asi, %l2 ;\ ldxa [%sp + V9BIAS64 + 24]%asi, %l3 ;\ ldxa [%sp + V9BIAS64 + 32]%asi, %l4 ;\ ldxa [%sp + V9BIAS64 + 40]%asi, %l5 ;\ ldxa [%sp + V9BIAS64 + 48]%asi, %l6 ;\ ldxa [%sp + V9BIAS64 + 56]%asi, %l7 ;\ ldxa [%sp + V9BIAS64 + 64]%asi, %i0 ;\ ldxa [%sp + V9BIAS64 + 72]%asi, %i1 ;\ ldxa [%sp + V9BIAS64 + 80]%asi, %i2 ;\ ldxa [%sp + V9BIAS64 + 88]%asi, %i3 ;\ ldxa [%sp + V9BIAS64 + 96]%asi, %i4 ;\ ldxa [%sp + V9BIAS64 + 104]%asi, %i5 ;\ ldxa [%sp + V9BIAS64 + 112]%asi, %i6 ;\ ldxa [%sp + V9BIAS64 + 120]%asi, %i7 ;\ restored ;\ add %g2, 1, %g2 ;\ wrpr %g2, %cwp .flushw: rdpr %tnpc, %g1 wrpr %g1, %tpc add %g1, 4, %g1 wrpr %g1, %tnpc set trap, %g1 mov T_FLUSH_PCB, %g3 ba,pt %xcc, sys_trap sub %g0, 1, %g4 .clean_windows: set trap, %g1 mov T_FLUSH_PCB, %g3 sub %g0, 1, %g4 save flushw rdpr %canrestore, %g2 brnz %g2, 1f nop rdpr %wstate, %g2 btst 1, %g2 beq 2f nop FILL_32bit_flushw(ASI_AIUP) ba,a 1f .empty 2: FILL_64bit_flushw(ASI_AIUP) 1: restore wrpr %g0, %g0, %cleanwin ! no clean windows CPU_ADDR(%g4, %g5) ldn [%g4 + CPU_MPCB], %g4 brz,a,pn %g4, 1f nop ld [%g4 + MPCB_WSTATE], %g5 add %g5, WSTATE_CLEAN_OFFSET, %g5 wrpr %g0, %g5, %wstate 1: FAST_TRAP_DONE /* * .spill_clean: clean the previous window, restore the wstate, and * "done". * * Entry: %g7 contains new wstate */ .spill_clean: sethi %hi(nwin_minus_one), %g5 ld [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1 rdpr %cwp, %g6 ! %g6 = %cwp deccc %g6 ! %g6-- movneg %xcc, %g5, %g6 ! if (%g6<0) %g6 = nwin-1 wrpr %g6, %cwp TT_TRACE_L(trace_win) clr %l0 clr %l1 clr %l2 clr %l3 clr %l4 clr %l5 clr %l6 clr %l7 wrpr %g0, %g7, %wstate saved retry ! restores correct %cwp .fix_alignment: CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer ldn [%g1 + T_PROCP], %g1 mov 1, %g2 stb %g2, [%g1 + P_FIXALIGNMENT] FAST_TRAP_DONE #define STDF_REG(REG, ADDR, TMP) \ sll REG, 3, REG ;\ mark1: set start1, TMP ;\ jmp REG + TMP ;\ nop ;\ start1: ba,pt %xcc, done1 ;\ std %f0, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f32, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f2, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f34, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f4, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f36, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f6, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f38, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f8, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f40, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f10, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f42, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f12, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f44, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f14, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f46, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f16, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f48, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f18, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f50, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f20, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f52, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f22, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f54, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f24, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f56, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f26, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f58, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f28, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f60, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f30, [ADDR + CPU_TMP1] ;\ ba,pt %xcc, done1 ;\ std %f62, [ADDR + CPU_TMP1] ;\ done1: #define LDDF_REG(REG, ADDR, TMP) \ sll REG, 3, REG ;\ mark2: set start2, TMP ;\ jmp REG + TMP ;\ nop ;\ start2: ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f0 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f32 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f2 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f34 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f4 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f36 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f6 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f38 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f8 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f40 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f10 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f42 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f12 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f44 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f14 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f46 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f16 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f48 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f18 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f50 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f20 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f52 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f22 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f54 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f24 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f56 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f26 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f58 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f28 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f60 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f30 ;\ ba,pt %xcc, done2 ;\ ldd [ADDR + CPU_TMP1], %f62 ;\ done2: .lddf_exception_not_aligned: /* %g2 = sfar, %g3 = sfsr */ mov %g2, %g5 ! stash sfar #if defined(DEBUG) || defined(NEED_FPU_EXISTS) sethi %hi(fpu_exists), %g2 ! check fpu_exists ld [%g2 + %lo(fpu_exists)], %g2 brz,a,pn %g2, 4f nop #endif CPU_ADDR(%g1, %g4) or %g0, 1, %g4 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag rdpr %tpc, %g2 lda [%g2]ASI_AIUP, %g6 ! get the user's lddf instruction srl %g6, 23, %g1 ! using ldda or not? and %g1, 1, %g1 brz,a,pt %g1, 2f ! check for ldda instruction nop srl %g6, 13, %g1 ! check immflag and %g1, 1, %g1 rdpr %tstate, %g2 ! %tstate in %g2 brnz,a,pn %g1, 1f srl %g2, 31, %g1 ! get asi from %tstate srl %g6, 5, %g1 ! get asi from instruction and %g1, 0xFF, %g1 ! imm_asi field 1: cmp %g1, ASI_P ! primary address space be,a,pt %icc, 2f nop cmp %g1, ASI_PNF ! primary no fault address space be,a,pt %icc, 2f nop cmp %g1, ASI_S ! secondary address space be,a,pt %icc, 2f nop cmp %g1, ASI_SNF ! secondary no fault address space bne,a,pn %icc, 3f nop 2: lduwa [%g5]ASI_USER, %g7 ! get first half of misaligned data add %g5, 4, %g5 ! increment misaligned data address lduwa [%g5]ASI_USER, %g5 ! get second half of misaligned data sllx %g7, 32, %g7 or %g5, %g7, %g5 ! combine data CPU_ADDR(%g7, %g1) ! save data on a per-cpu basis stx %g5, [%g7 + CPU_TMP1] ! save in cpu_tmp1 srl %g6, 25, %g3 ! %g6 has the instruction and %g3, 0x1F, %g3 ! %g3 has rd LDDF_REG(%g3, %g7, %g4) CPU_ADDR(%g1, %g4) st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag FAST_TRAP_DONE 3: CPU_ADDR(%g1, %g4) st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 4: set T_USER, %g3 ! trap type in %g3 or %g3, T_LDDF_ALIGN, %g3 mov %g5, %g2 ! misaligned vaddr in %g2 set fpu_trap, %g1 ! goto C for the little and ba,pt %xcc, sys_trap ! no fault little asi's sub %g0, 1, %g4 .stdf_exception_not_aligned: /* %g2 = sfar, %g3 = sfsr */ mov %g2, %g5 #if defined(DEBUG) || defined(NEED_FPU_EXISTS) sethi %hi(fpu_exists), %g7 ! check fpu_exists ld [%g7 + %lo(fpu_exists)], %g3 brz,a,pn %g3, 4f nop #endif CPU_ADDR(%g1, %g4) or %g0, 1, %g4 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag rdpr %tpc, %g2 lda [%g2]ASI_AIUP, %g6 ! get the user's stdf instruction srl %g6, 23, %g1 ! using stda or not? and %g1, 1, %g1 brz,a,pt %g1, 2f ! check for stda instruction nop srl %g6, 13, %g1 ! check immflag and %g1, 1, %g1 rdpr %tstate, %g2 ! %tstate in %g2 brnz,a,pn %g1, 1f srl %g2, 31, %g1 ! get asi from %tstate srl %g6, 5, %g1 ! get asi from instruction and %g1, 0xff, %g1 ! imm_asi field 1: cmp %g1, ASI_P ! primary address space be,a,pt %icc, 2f nop cmp %g1, ASI_S ! secondary address space bne,a,pn %icc, 3f nop 2: srl %g6, 25, %g6 and %g6, 0x1F, %g6 ! %g6 has rd CPU_ADDR(%g7, %g1) STDF_REG(%g6, %g7, %g4) ! STDF_REG(REG, ADDR, TMP) ldx [%g7 + CPU_TMP1], %g6 srlx %g6, 32, %g7 stuwa %g7, [%g5]ASI_USER ! first half add %g5, 4, %g5 ! increment misaligned data address stuwa %g6, [%g5]ASI_USER ! second half CPU_ADDR(%g1, %g4) st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag FAST_TRAP_DONE 3: CPU_ADDR(%g1, %g4) st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 4: set T_USER, %g3 ! trap type in %g3 or %g3, T_STDF_ALIGN, %g3 mov %g5, %g2 ! misaligned vaddr in %g2 set fpu_trap, %g1 ! goto C for the little and ba,pt %xcc, sys_trap ! nofault little asi's sub %g0, 1, %g4 #ifdef DEBUG_USER_TRAPTRACECTL .traptrace_freeze: mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4 TT_TRACE_L(trace_win) mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0 set trap_freeze, %g1 mov 1, %g2 st %g2, [%g1] FAST_TRAP_DONE .traptrace_unfreeze: set trap_freeze, %g1 st %g0, [%g1] mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4 TT_TRACE_L(trace_win) mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0 FAST_TRAP_DONE #endif /* DEBUG_USER_TRAPTRACECTL */ .getcc: CPU_ADDR(%g1, %g2) stx %o0, [%g1 + CPU_TMP1] ! save %o0 rdpr %tstate, %g3 ! get tstate srlx %g3, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr set PSR_ICC, %g2 and %o0, %g2, %o0 ! mask out the rest srl %o0, PSR_ICC_SHIFT, %o0 ! right justify wrpr %g0, 0, %gl mov %o0, %g1 ! move ccr to normal %g1 wrpr %g0, 1, %gl ! cannot assume globals retained their values after increasing %gl CPU_ADDR(%g1, %g2) ldx [%g1 + CPU_TMP1], %o0 ! restore %o0 FAST_TRAP_DONE .setcc: CPU_ADDR(%g1, %g2) stx %o0, [%g1 + CPU_TMP1] ! save %o0 wrpr %g0, 0, %gl mov %g1, %o0 wrpr %g0, 1, %gl ! cannot assume globals retained their values after increasing %gl CPU_ADDR(%g1, %g2) sll %o0, PSR_ICC_SHIFT, %g2 set PSR_ICC, %g3 and %g2, %g3, %g2 ! mask out rest sllx %g2, PSR_TSTATE_CC_SHIFT, %g2 rdpr %tstate, %g3 ! get tstate srl %g3, 0, %g3 ! clear upper word or %g3, %g2, %g3 ! or in new bits wrpr %g3, %tstate ldx [%g1 + CPU_TMP1], %o0 ! restore %o0 FAST_TRAP_DONE /* * getpsr(void) * Note that the xcc part of the ccr is not provided. * The V8 code shows why the V9 trap is not faster: * #define GETPSR_TRAP() \ * mov %psr, %i0; jmp %l2; rett %l2+4; nop; */ .type .getpsr, #function .getpsr: rdpr %tstate, %g1 ! get tstate srlx %g1, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr set PSR_ICC, %g2 and %o0, %g2, %o0 ! mask out the rest rd %fprs, %g1 ! get fprs and %g1, FPRS_FEF, %g2 ! mask out dirty upper/lower sllx %g2, PSR_FPRS_FEF_SHIFT, %g2 ! shift fef to V8 psr.ef or %o0, %g2, %o0 ! or result into psr.ef set V9_PSR_IMPLVER, %g2 ! SI assigned impl/ver: 0xef or %o0, %g2, %o0 ! or psr.impl/ver FAST_TRAP_DONE SET_SIZE(.getpsr) /* * setpsr(newpsr) * Note that there is no support for ccr.xcc in the V9 code. */ .type .setpsr, #function .setpsr: rdpr %tstate, %g1 ! get tstate ! setx TSTATE_V8_UBITS, %g2 or %g0, CCR_ICC, %g3 sllx %g3, TSTATE_CCR_SHIFT, %g2 andn %g1, %g2, %g1 ! zero current user bits set PSR_ICC, %g2 and %g2, %o0, %g2 ! clear all but psr.icc bits sllx %g2, PSR_TSTATE_CC_SHIFT, %g3 ! shift to tstate.ccr.icc wrpr %g1, %g3, %tstate ! write tstate set PSR_EF, %g2 and %g2, %o0, %g2 ! clear all but fp enable bit srlx %g2, PSR_FPRS_FEF_SHIFT, %g4 ! shift ef to V9 fprs.fef wr %g0, %g4, %fprs ! write fprs CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer ldn [%g2 + T_LWP], %g3 ! load klwp pointer ldn [%g3 + LWP_FPU], %g2 ! get lwp_fpu pointer stuw %g4, [%g2 + FPU_FPRS] ! write fef value to fpu_fprs srlx %g4, 2, %g4 ! shift fef value to bit 0 stub %g4, [%g2 + FPU_EN] ! write fef value to fpu_en FAST_TRAP_DONE SET_SIZE(.setpsr) /* * getlgrp * get home lgrpid on which the calling thread is currently executing. */ .type .getlgrp, #function .getlgrp: ! Thanks for the incredibly helpful comments CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 ld [%g1 + CPU_ID], %o0 ! load cpu_id ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer ldn [%g2 + T_LPL], %g2 ! load lpl pointer ld [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid sra %g1, 0, %o1 FAST_TRAP_DONE SET_SIZE(.getlgrp) /* * Entry for old 4.x trap (trap 0). */ ENTRY_NP(syscall_trap_4x) CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer ldn [%g2 + T_LWP], %g2 ! load klwp pointer ld [%g2 + PCB_TRAP0], %g2 ! lwp->lwp_pcb.pcb_trap0addr brz,pn %g2, 1f ! has it been set? st %l0, [%g1 + CPU_TMP1] ! delay - save some locals st %l1, [%g1 + CPU_TMP2] rdpr %tnpc, %l1 ! save old tnpc wrpr %g0, %g2, %tnpc ! setup tnpc mov %g1, %l0 ! save CPU struct addr wrpr %g0, 0, %gl mov %l1, %g6 ! pass tnpc to user code in %g6 wrpr %g0, 1, %gl ld [%l0 + CPU_TMP2], %l1 ! restore locals ld [%l0 + CPU_TMP1], %l0 FAST_TRAP_DONE_CHK_INTR 1: ! ! check for old syscall mmap which is the only different one which ! must be the same. Others are handled in the compatibility library. ! mov %g1, %l0 ! save CPU struct addr wrpr %g0, 0, %gl cmp %g1, OSYS_mmap ! compare to old 4.x mmap movz %icc, SYS_mmap, %g1 wrpr %g0, 1, %gl ld [%l0 + CPU_TMP1], %l0 SYSCALL(syscall_trap32) SET_SIZE(syscall_trap_4x) /* * Handler for software trap 9. * Set trap0 emulation address for old 4.x system call trap. * XXX - this should be a system call. */ ENTRY_NP(set_trap0_addr) CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 st %l0, [%g1 + CPU_TMP1] ! save some locals st %l1, [%g1 + CPU_TMP2] mov %g1, %l0 ! preserve CPU addr wrpr %g0, 0, %gl mov %g1, %l1 wrpr %g0, 1, %gl ! cannot assume globals retained their values after increasing %gl ldn [%l0 + CPU_THREAD], %g2 ! load thread pointer ldn [%g2 + T_LWP], %g2 ! load klwp pointer andn %l1, 3, %l1 ! force alignment st %l1, [%g2 + PCB_TRAP0] ! lwp->lwp_pcb.pcb_trap0addr ld [%l0 + CPU_TMP2], %l1 ! restore locals ld [%l0 + CPU_TMP1], %l0 FAST_TRAP_DONE SET_SIZE(set_trap0_addr) /* * mmu_trap_tl1 * trap handler for unexpected mmu traps. * simply checks if the trap was a user lddf/stdf alignment trap, in which * case we go to fpu_trap or a user trap from the window handler, in which * case we go save the state on the pcb. Otherwise, we go to ptl1_panic. */ .type mmu_trap_tl1, #function mmu_trap_tl1: #ifdef TRAPTRACE TRACE_PTR(%g5, %g6) GET_TRACE_TICK(%g6) stxa %g6, [%g5 + TRAP_ENT_TICK]%asi TRACE_SAVE_TL_GL_REGS(%g5, %g6) rdpr %tt, %g6 stha %g6, [%g5 + TRAP_ENT_TT]%asi rdpr %tstate, %g6 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi stna %sp, [%g5 + TRAP_ENT_SP]%asi stna %g0, [%g5 + TRAP_ENT_TR]%asi rdpr %tpc, %g6 stna %g6, [%g5 + TRAP_ENT_TPC]%asi MMU_FAULT_STATUS_AREA(%g6) ldx [%g6 + MMFSA_D_ADDR], %g6 stna %g6, [%g5 + TRAP_ENT_F1]%asi ! MMU fault address CPU_PADDR(%g7, %g6); add %g7, CPU_TL1_HDLR, %g7 lda [%g7]ASI_MEM, %g6 stna %g6, [%g5 + TRAP_ENT_F2]%asi MMU_FAULT_STATUS_AREA(%g6) ldx [%g6 + MMFSA_D_TYPE], %g7 ! XXXQ should be a MMFSA_F_ constant? ldx [%g6 + MMFSA_D_CTX], %g6 sllx %g6, SFSR_CTX_SHIFT, %g6 or %g6, %g7, %g6 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! MMU context/type set 0xdeadbeef, %g6 stna %g6, [%g5 + TRAP_ENT_F4]%asi TRACE_NEXT(%g5, %g6, %g7) #endif /* TRAPTRACE */ CPU_PADDR(%g7, %g6); add %g7, CPU_TL1_HDLR, %g7 ! %g7 = &cpu_m.tl1_hdlr (PA) lda [%g7]ASI_MEM, %g6 brz,a,pt %g6, 1f nop sta %g0, [%g7]ASI_MEM ! XXXQ need to setup registers for sfmmu_mmu_trap? ba,a,pt %xcc, sfmmu_mmu_trap ! handle page faults 1: rdpr %tpc, %g7 /* in user_rtt? */ set rtt_fill_start, %g6 cmp %g7, %g6 blu,pn %xcc, 6f .empty set rtt_fill_end, %g6 cmp %g7, %g6 bgeu,pn %xcc, 6f nop set fault_rtt_fn1, %g7 ba,a 7f 6: ! check to see if the trap pc is in a window spill/fill handling rdpr %tpc, %g7 /* tpc should be in the trap table */ set trap_table, %g6 cmp %g7, %g6 blu,a,pn %xcc, ptl1_panic mov PTL1_BAD_MMUTRAP, %g1 set etrap_table, %g6 cmp %g7, %g6 bgeu,a,pn %xcc, ptl1_panic mov PTL1_BAD_MMUTRAP, %g1 ! pc is inside the trap table, convert to trap type srl %g7, 5, %g6 ! XXXQ need #define and %g6, 0x1ff, %g6 ! XXXQ need #define ! and check for a window trap type and %g6, WTRAP_TTMASK, %g6 cmp %g6, WTRAP_TYPE bne,a,pn %xcc, ptl1_panic mov PTL1_BAD_MMUTRAP, %g1 andn %g7, WTRAP_ALIGN, %g7 /* 128 byte aligned */ add %g7, WTRAP_FAULTOFF, %g7 7: ! Arguments are passed in the global set active after the ! 'done' instruction. Before switching sets, must save ! the calculated next pc wrpr %g0, %g7, %tnpc wrpr %g0, 1, %gl rdpr %tt, %g5 cmp %g5, T_ALIGNMENT MMU_FAULT_STATUS_AREA(%g4) ldx [%g4 + MMFSA_D_ADDR], %g6 done SET_SIZE(mmu_trap_tl1) /* * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers. These * traps are valid only when kmdb is loaded. When the debugger is active, * the code below is rewritten to transfer control to the appropriate * debugger entry points. */ .global kmdb_trap .align 8 kmdb_trap: ba,a trap_table0 jmp %g1 + 0 nop .global kmdb_trap_tl1 .align 8 kmdb_trap_tl1: ba,a trap_table0 jmp %g1 + 0 nop /* * This entry is copied from OBP's trap table during boot. */ .global obp_bpt .align 8 obp_bpt: NOT #ifdef TRAPTRACE /* * TRAPTRACE support. * labels here are branched to with "rd %pc, %g7" in the delay slot. * Return is done by "jmp %g7 + 4". */ trace_dmmu: TRACE_PTR(%g3, %g6) GET_TRACE_TICK(%g6) stxa %g6, [%g3 + TRAP_ENT_TICK]%asi TRACE_SAVE_TL_GL_REGS(%g3, %g6) rdpr %tt, %g6 stha %g6, [%g3 + TRAP_ENT_TT]%asi rdpr %tstate, %g6 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi stna %sp, [%g3 + TRAP_ENT_SP]%asi rdpr %tpc, %g6 stna %g6, [%g3 + TRAP_ENT_TPC]%asi MMU_FAULT_STATUS_AREA(%g6) ldx [%g6 + MMFSA_D_ADDR], %g4 stxa %g4, [%g3 + TRAP_ENT_TR]%asi ldx [%g6 + MMFSA_D_CTX], %g4 stxa %g4, [%g3 + TRAP_ENT_F1]%asi ldx [%g6 + MMFSA_D_TYPE], %g4 stxa %g4, [%g3 + TRAP_ENT_F2]%asi stxa %g6, [%g3 + TRAP_ENT_F3]%asi stna %g0, [%g3 + TRAP_ENT_F4]%asi TRACE_NEXT(%g3, %g4, %g5) jmp %g7 + 4 nop trace_immu: TRACE_PTR(%g3, %g6) GET_TRACE_TICK(%g6) stxa %g6, [%g3 + TRAP_ENT_TICK]%asi TRACE_SAVE_TL_GL_REGS(%g3, %g6) rdpr %tt, %g6 stha %g6, [%g3 + TRAP_ENT_TT]%asi rdpr %tstate, %g6 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi stna %sp, [%g3 + TRAP_ENT_SP]%asi rdpr %tpc, %g6 stna %g6, [%g3 + TRAP_ENT_TPC]%asi MMU_FAULT_STATUS_AREA(%g6) ldx [%g6 + MMFSA_I_ADDR], %g4 stxa %g4, [%g3 + TRAP_ENT_TR]%asi ldx [%g6 + MMFSA_I_CTX], %g4 stxa %g4, [%g3 + TRAP_ENT_F1]%asi ldx [%g6 + MMFSA_I_TYPE], %g4 stxa %g4, [%g3 + TRAP_ENT_F2]%asi stxa %g6, [%g3 + TRAP_ENT_F3]%asi stna %g0, [%g3 + TRAP_ENT_F4]%asi TRACE_NEXT(%g3, %g4, %g5) jmp %g7 + 4 nop trace_gen: TRACE_PTR(%g3, %g6) GET_TRACE_TICK(%g6) stxa %g6, [%g3 + TRAP_ENT_TICK]%asi TRACE_SAVE_TL_GL_REGS(%g3, %g6) rdpr %tt, %g6 stha %g6, [%g3 + TRAP_ENT_TT]%asi rdpr %tstate, %g6 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi stna %sp, [%g3 + TRAP_ENT_SP]%asi rdpr %tpc, %g6 stna %g6, [%g3 + TRAP_ENT_TPC]%asi stna %g0, [%g3 + TRAP_ENT_TR]%asi stna %g0, [%g3 + TRAP_ENT_F1]%asi stna %g0, [%g3 + TRAP_ENT_F2]%asi stna %g0, [%g3 + TRAP_ENT_F3]%asi stna %g0, [%g3 + TRAP_ENT_F4]%asi TRACE_NEXT(%g3, %g4, %g5) jmp %g7 + 4 nop trace_win: TRACE_WIN_INFO(0, %l0, %l1, %l2) ! Keep the locals as clean as possible, caller cleans %l4 clr %l2 clr %l1 jmp %l4 + 4 clr %l0 /* * Trace a tsb hit * g1 = tsbe pointer (in/clobbered) * g2 = tag access register (in) * g3 - g4 = scratch (clobbered) * g5 = tsbe data (in) * g6 = scratch (clobbered) * g7 = pc we jumped here from (in) */ ! Do not disturb %g5, it will be used after the trace ALTENTRY(trace_tsbhit) TRACE_TSBHIT(0) jmp %g7 + 4 nop /* * Trace a TSB miss * * g1 = tsb8k pointer (in) * g2 = tag access register (in) * g3 = tsb4m pointer (in) * g4 = tsbe tag (in/clobbered) * g5 - g6 = scratch (clobbered) * g7 = pc we jumped here from (in) */ .global trace_tsbmiss trace_tsbmiss: membar #Sync sethi %hi(FLUSH_ADDR), %g6 flush %g6 TRACE_PTR(%g5, %g6) GET_TRACE_TICK(%g6) stxa %g6, [%g5 + TRAP_ENT_TICK]%asi stna %g2, [%g5 + TRAP_ENT_SP]%asi ! tag access stna %g4, [%g5 + TRAP_ENT_F1]%asi ! XXX? tsb tag rdpr %tnpc, %g6 stna %g6, [%g5 + TRAP_ENT_F2]%asi stna %g1, [%g5 + TRAP_ENT_F3]%asi ! tsb8k pointer srlx %g1, 32, %g6 stna %g6, [%g5 + TRAP_ENT_F4]%asi ! huh? rdpr %tpc, %g6 stna %g6, [%g5 + TRAP_ENT_TPC]%asi TRACE_SAVE_TL_GL_REGS(%g5, %g6) rdpr %tt, %g6 or %g6, TT_MMU_MISS, %g4 stha %g4, [%g5 + TRAP_ENT_TT]%asi MMU_FAULT_STATUS_AREA(%g4) cmp %g6, FAST_IMMU_MISS_TT be,a %icc, 1f ldx [%g4 + MMFSA_I_ADDR], %g6 cmp %g6, T_INSTR_MMU_MISS be,a %icc, 1f ldx [%g4 + MMFSA_I_ADDR], %g6 ldx [%g4 + MMFSA_D_ADDR], %g6 1: stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! tag target stna %g3, [%g5 + TRAP_ENT_TR]%asi ! tsb4m pointer TRACE_NEXT(%g5, %g4, %g6) jmp %g7 + 4 nop /* * g2 = tag access register (in) * g3 = ctx number (in) */ trace_dataprot: membar #Sync sethi %hi(FLUSH_ADDR), %g6 flush %g6 TRACE_PTR(%g1, %g6) GET_TRACE_TICK(%g6) stxa %g6, [%g1 + TRAP_ENT_TICK]%asi rdpr %tpc, %g6 stna %g6, [%g1 + TRAP_ENT_TPC]%asi rdpr %tstate, %g6 stxa %g6, [%g1 + TRAP_ENT_TSTATE]%asi stna %g2, [%g1 + TRAP_ENT_SP]%asi ! tag access reg stna %g0, [%g1 + TRAP_ENT_TR]%asi stna %g0, [%g1 + TRAP_ENT_F1]%asi stna %g0, [%g1 + TRAP_ENT_F2]%asi stna %g0, [%g1 + TRAP_ENT_F3]%asi stna %g0, [%g1 + TRAP_ENT_F4]%asi TRACE_SAVE_TL_GL_REGS(%g1, %g6) rdpr %tt, %g6 stha %g6, [%g1 + TRAP_ENT_TT]%asi TRACE_NEXT(%g1, %g4, %g5) jmp %g7 + 4 nop #endif /* TRAPTRACE */ .dmmu_exc_lddf_not_aligned: MMU_FAULT_STATUS_AREA(%g3) ldx [%g3 + MMFSA_D_ADDR], %g2 /* Fault type not available in MMU fault status area */ mov MMFSA_F_UNALIGN, %g1 ldx [%g3 + MMFSA_D_CTX], %g3 sllx %g3, SFSR_CTX_SHIFT, %g3 btst 1, %sp bnz,pt %xcc, .lddf_exception_not_aligned or %g3, %g1, %g3 ba,a,pt %xcc, .mmu_exception_not_aligned .dmmu_exc_stdf_not_aligned: MMU_FAULT_STATUS_AREA(%g3) ldx [%g3 + MMFSA_D_ADDR], %g2 /* Fault type not available in MMU fault status area */ mov MMFSA_F_UNALIGN, %g1 ldx [%g3 + MMFSA_D_CTX], %g3 sllx %g3, SFSR_CTX_SHIFT, %g3 btst 1, %sp bnz,pt %xcc, .stdf_exception_not_aligned or %g3, %g1, %g3 ba,a,pt %xcc, .mmu_exception_not_aligned /* * expects offset into tsbmiss area in %g1 and return pc in %g7 */ stat_mmu: CPU_INDEX(%g5, %g6) sethi %hi(tsbmiss_area), %g6 sllx %g5, TSBMISS_SHIFT, %g5 or %g6, %lo(tsbmiss_area), %g6 add %g6, %g5, %g6 /* g6 = tsbmiss area */ ld [%g6 + %g1], %g5 add %g5, 1, %g5 jmp %g7 + 4 st %g5, [%g6 + %g1] /* * fast_trap_done, fast_trap_done_chk_intr: * * Due to the design of UltraSPARC pipeline, pending interrupts are not * taken immediately after a RETRY or DONE instruction which causes IE to * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed * to execute first before taking any interrupts. If that instruction * results in other traps, and if the corresponding trap handler runs * entirely at TL=1 with interrupts disabled, then pending interrupts * won't be taken until after yet another instruction following the %tpc * or %tnpc. * * A malicious user program can use this feature to block out interrupts * for extended durations, which can result in send_mondo_timeout kernel * panic. * * This problem is addressed by servicing any pending interrupts via * sys_trap before returning back to the user mode from a fast trap * handler. The "done" instruction within a fast trap handler, which * runs entirely at TL=1 with interrupts disabled, is replaced with the * FAST_TRAP_DONE macro, which branches control to this fast_trap_done * entry point. * * We check for any pending interrupts here and force a sys_trap to * service those interrupts, if any. To minimize overhead, pending * interrupts are checked if the %tpc happens to be at 16K boundary, * which allows a malicious program to execute at most 4K consecutive * instructions before we service any pending interrupts. If a worst * case fast trap handler takes about 2 usec, then interrupts will be * blocked for at most 8 msec, less than a clock tick. * * For the cases where we don't know if the %tpc will cross a 16K * boundary, we can't use the above optimization and always process * any pending interrupts via fast_frap_done_chk_intr entry point. * * Entry Conditions: * %pstate am:0 priv:1 ie:0 * globals are AG (not normal globals) */ .global fast_trap_done, fast_trap_done_chk_intr fast_trap_done: rdpr %tpc, %g5 sethi %hi(0xffffc000), %g6 ! 1's complement of 0x3fff andncc %g5, %g6, %g0 ! check lower 14 bits of %tpc bz,pn %icc, 1f ! branch if zero (lower 32 bits only) nop done fast_trap_done_chk_intr: 1: rd SOFTINT, %g6 brnz,pn %g6, 2f ! branch if any pending intr nop done 2: /* * We get here if there are any pending interrupts. * Adjust %tpc/%tnpc as we'll be resuming via "retry" * instruction. */ rdpr %tnpc, %g5 wrpr %g0, %g5, %tpc add %g5, 4, %g5 wrpr %g0, %g5, %tnpc /* * Force a dummy sys_trap call so that interrupts can be serviced. */ set fast_trap_dummy_call, %g1 ba,pt %xcc, sys_trap mov -1, %g4 fast_trap_dummy_call: retl nop #endif /* lint */