/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Assembly code support for the Cheetah+ module */ #pragma ident "%Z%%M% %I% %E% SMI" #if !defined(lint) #include "assym.h" #endif /* lint */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef TRAPTRACE #include #endif /* TRAPTRACE */ #if !defined(lint) /* BEGIN CSTYLED */ /* * Cheetah+ version to reflush an Ecache line by index. * * By default we assume the Ecache is 2-way so we flush both * ways. Even if the cache is direct-mapped no harm will come * from performing the flush twice, apart from perhaps a performance * penalty. * * XXX - scr2 not used. */ #define ECACHE_REFLUSH_LINE(ec_set_size, index, scr2) \ ldxa [index]ASI_EC_DIAG, %g0; \ ldxa [index + ec_set_size]ASI_EC_DIAG, %g0; /* * Cheetah+ version of ecache_flush_line. Uses Cheetah+ Ecache Displacement * Flush feature. */ #define ECACHE_FLUSH_LINE(physaddr, ec_set_size, scr1, scr2) \ sub ec_set_size, 1, scr1; \ and physaddr, scr1, scr1; \ set CHP_ECACHE_IDX_DISP_FLUSH, scr2; \ or scr2, scr1, scr1; \ ECACHE_REFLUSH_LINE(ec_set_size, scr1, scr2) /* END CSTYLED */ /* * Panther version to reflush a line from both the L2 cache and L3 * cache by the respective indexes. Flushes all ways of the line from * each cache. * * l2_index Index into the L2$ of the line to be flushed. This * register will not be modified by this routine. * l3_index Index into the L3$ of the line to be flushed. This * register will not be modified by this routine. * scr2 scratch register. * scr3 scratch register. * */ #define PN_ECACHE_REFLUSH_LINE(l2_index, l3_index, scr2, scr3) \ set PN_L2_MAX_SET, scr2; \ set PN_L2_SET_SIZE, scr3; \ 1: \ ldxa [l2_index + scr2]ASI_L2_TAG, %g0; \ cmp scr2, %g0; \ bg,a 1b; \ sub scr2, scr3, scr2; \ mov 6, scr2; \ 7: \ cmp scr2, %g0; \ bg,a 7b; \ sub scr2, 1, scr2; \ set PN_L3_MAX_SET, scr2; \ set PN_L3_SET_SIZE, scr3; \ 2: \ ldxa [l3_index + scr2]ASI_EC_DIAG, %g0; \ cmp scr2, %g0; \ bg,a 2b; \ sub scr2, scr3, scr2; /* * Panther version of ecache_flush_line. Flushes the line corresponding * to physaddr from both the L2 cache and the L3 cache. * * physaddr Input: Physical address to flush. * Output: Physical address to flush (preserved). * l2_idx_out Input: scratch register. * Output: Index into the L2$ of the line to be flushed. * l3_idx_out Input: scratch register. * Output: Index into the L3$ of the line to be flushed. * scr3 scratch register. * scr4 scratch register. * */ #define PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4) \ set PN_L3_SET_SIZE, l2_idx_out; \ sub l2_idx_out, 1, l2_idx_out; \ and physaddr, l2_idx_out, l3_idx_out; \ set PN_L3_IDX_DISP_FLUSH, l2_idx_out; \ or l2_idx_out, l3_idx_out, l3_idx_out; \ set PN_L2_SET_SIZE, l2_idx_out; \ sub l2_idx_out, 1, l2_idx_out; \ and physaddr, l2_idx_out, l2_idx_out; \ set PN_L2_IDX_DISP_FLUSH, scr3; \ or l2_idx_out, scr3, l2_idx_out; \ PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4) #endif /* !lint */ /* * Fast ECC error at TL>0 handler * We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. * For a complete description of the Fast ECC at TL>0 handling see the * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in * us3_common_asm.s */ #if defined(lint) void fast_ecc_tl1_err(void) {} #else /* lint */ .section ".text" .align 64 ENTRY_NP(fast_ecc_tl1_err) /* * This macro turns off the D$/I$ if they are on and saves their * original state in ch_err_tl1_tmp, saves all the %g registers in the * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves * the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will * point to the ch_err_tl1_data structure and the original D$/I$ state * will be saved in ch_err_tl1_tmp. All %g registers except for %g1 * will be available. */ CH_ERR_TL1_FECC_ENTER; /* * Get the diagnostic logout data. %g4 must be initialized to * current CEEN state, %g5 must point to logout structure in * ch_err_tl1_data_t. %g3 will contain the nesting count upon * return. */ ldxa [%g0]ASI_ESTATE_ERR, %g4 and %g4, EN_REG_CEEN, %g4 add %g1, CH_ERR_TL1_LOGOUT, %g5 DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4) /* * If the logout nesting count is exceeded, we're probably * not making any progress, try to panic instead. */ cmp %g3, CLO_NESTING_MAX bge fecc_tl1_err nop /* * Save the current CEEN and NCEEN state in %g7 and turn them off * before flushing the Ecache. */ ldxa [%g0]ASI_ESTATE_ERR, %g7 andn %g7, EN_REG_CEEN | EN_REG_NCEEN, %g5 stxa %g5, [%g0]ASI_ESTATE_ERR membar #Sync /* * Flush the Ecache, using the largest possible cache size with the * smallest possible line size since we can't get the actual sizes * from the cpu_node due to DTLB misses. */ PN_L2_FLUSHALL(%g3, %g4, %g5) set CH_ECACHE_MAX_SIZE, %g4 set CH_ECACHE_MIN_LSIZE, %g5 GET_CPU_IMPL(%g6) cmp %g6, PANTHER_IMPL bne %xcc, 2f nop set PN_L3_SIZE, %g4 2: mov %g6, %g3 CHP_ECACHE_FLUSHALL(%g4, %g5, %g3) /* * Restore CEEN and NCEEN to the previous state. */ stxa %g7, [%g0]ASI_ESTATE_ERR membar #Sync /* * If we turned off the D$, then flush it and turn it back on. */ ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 andcc %g3, CH_ERR_TSTATE_DC_ON, %g0 bz %xcc, 3f nop /* * Flush the D$. */ ASM_LD(%g4, dcache_size) ASM_LD(%g5, dcache_linesize) CH_DCACHE_FLUSHALL(%g4, %g5, %g6) /* * Turn the D$ back on. */ ldxa [%g0]ASI_DCU, %g3 or %g3, DCU_DC, %g3 stxa %g3, [%g0]ASI_DCU membar #Sync 3: /* * If we turned off the I$, then flush it and turn it back on. */ ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 andcc %g3, CH_ERR_TSTATE_IC_ON, %g0 bz %xcc, 4f nop /* * Flush the I$. Panther has different I$ parameters, and we * can't access the logout I$ params without possibly generating * a MMU miss. */ GET_CPU_IMPL(%g6) set PN_ICACHE_SIZE, %g3 set CH_ICACHE_SIZE, %g4 mov CH_ICACHE_LSIZE, %g5 cmp %g6, PANTHER_IMPL movz %xcc, %g3, %g4 movz %xcc, PN_ICACHE_LSIZE, %g5 CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3) /* * Turn the I$ back on. Changing DCU_IC requires flush. */ ldxa [%g0]ASI_DCU, %g3 or %g3, DCU_IC, %g3 stxa %g3, [%g0]ASI_DCU flush %g0 4: #ifdef TRAPTRACE /* * Get current trap trace entry physical pointer. */ CPU_INDEX(%g6, %g5) sll %g6, TRAPTR_SIZE_SHIFT, %g6 set trap_trace_ctl, %g5 add %g6, %g5, %g6 ld [%g6 + TRAPTR_LIMIT], %g5 tst %g5 be %icc, skip_traptrace nop ldx [%g6 + TRAPTR_PBASE], %g5 ld [%g6 + TRAPTR_OFFSET], %g4 add %g5, %g4, %g5 /* * Create trap trace entry. */ rd %asi, %g7 wr %g0, TRAPTR_ASI, %asi rd STICK, %g4 stxa %g4, [%g5 + TRAP_ENT_TICK]%asi rdpr %tl, %g4 stha %g4, [%g5 + TRAP_ENT_TL]%asi rdpr %tt, %g4 stha %g4, [%g5 + TRAP_ENT_TT]%asi rdpr %tpc, %g4 stna %g4, [%g5 + TRAP_ENT_TPC]%asi rdpr %tstate, %g4 stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi stna %sp, [%g5 + TRAP_ENT_SP]%asi stna %g0, [%g5 + TRAP_ENT_TR]%asi wr %g0, %g7, %asi ldxa [%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4 wr %g0, TRAPTR_ASI, %asi stna %g3, [%g5 + TRAP_ENT_F1]%asi stna %g4, [%g5 + TRAP_ENT_F2]%asi wr %g0, %g7, %asi ldxa [%g1 + CH_ERR_TL1_AFAR]%asi, %g3 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4 wr %g0, TRAPTR_ASI, %asi stna %g3, [%g5 + TRAP_ENT_F3]%asi stna %g4, [%g5 + TRAP_ENT_F4]%asi wr %g0, %g7, %asi /* * Advance trap trace pointer. */ ld [%g6 + TRAPTR_OFFSET], %g5 ld [%g6 + TRAPTR_LIMIT], %g4 st %g5, [%g6 + TRAPTR_LAST_OFFSET] add %g5, TRAP_ENT_SIZE, %g5 sub %g4, TRAP_ENT_SIZE, %g4 cmp %g5, %g4 movge %icc, 0, %g5 st %g5, [%g6 + TRAPTR_OFFSET] skip_traptrace: #endif /* TRAPTRACE */ /* * If nesting count is not zero, skip all the AFSR/AFAR * handling and just do the necessary cache-flushing. */ ldxa [%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2 brnz %g2, 6f nop /* * If a UCU or L3_UCU followed by a WDU has occurred go ahead * and panic since a UE will occur (on the retry) before the * UCU and WDU messages are enqueued. */ ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g3 set 1, %g4 sllx %g4, C_AFSR_UCU_SHIFT, %g4 btst %g4, %g3 ! UCU in original shadow AFSR? bnz %xcc, 5f mov 1, %g4 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR_EXT]%asi, %g3 sllx %g4, C_AFSR_L3_UCU_SHIFT, %g4 btst %g4, %g3 ! L3_UCU in original shadow AFSR_EXT? bz %xcc, 6f nop 5: ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4 ! original AFSR ldxa [%g0]ASI_AFSR, %g3 ! current AFSR or %g3, %g4, %g3 ! %g3 = original + current AFSR set 1, %g4 sllx %g4, C_AFSR_WDU_SHIFT, %g4 btst %g4, %g3 ! WDU in original or current AFSR? bnz %xcc, fecc_tl1_err nop 6: /* * We fall into this macro if we've successfully logged the error in * the ch_err_tl1_data structure and want the PIL15 softint to pick * it up and log it. %g1 must point to the ch_err_tl1_data structure. * Restores the %g registers and issues retry. */ CH_ERR_TL1_EXIT; /* * Establish panic exit label. */ CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err); SET_SIZE(fast_ecc_tl1_err) #endif /* lint */ #if defined(lint) /* * scrubphys - Pass in the aligned physical memory address * that you want to scrub, along with the ecache set size. * * 1) Displacement flush the E$ line corresponding to %addr. * The first ldxa guarantees that the %addr is no longer in * M, O, or E (goes to I or S (if instruction fetch also happens). * 2) "Write" the data using a CAS %addr,%g0,%g0. * The casxa guarantees a transition from I to M or S to M. * 3) Displacement flush the E$ line corresponding to %addr. * The second ldxa pushes the M line out of the ecache, into the * writeback buffers, on the way to memory. * 4) The "membar #Sync" pushes the cache line out of the writeback * buffers onto the bus, on the way to dram finally. * * This is a modified version of the algorithm suggested by Gary Lauterbach. * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line * as modified, but then we found out that for spitfire, if it misses in the * E$ it will probably install as an M, but if it hits in the E$, then it * will stay E, if the store doesn't happen. So the first displacement flush * should ensure that the CAS will miss in the E$. Arrgh. */ /* ARGSUSED */ void scrubphys(uint64_t paddr, int ecache_set_size) {} #else /* lint */ ENTRY(scrubphys) rdpr %pstate, %o4 andn %o4, PSTATE_IE | PSTATE_AM, %o5 wrpr %o5, %g0, %pstate ! clear IE, AM bits GET_CPU_IMPL(%o5) ! Panther Ecache is flushed differently cmp %o5, PANTHER_IMPL bne scrubphys_1 nop PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o5) casxa [%o0]ASI_MEM, %g0, %g0 PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0) b scrubphys_2 nop scrubphys_1: ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) casxa [%o0]ASI_MEM, %g0, %g0 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) scrubphys_2: wrpr %g0, %o4, %pstate ! restore earlier pstate register value retl membar #Sync ! move the data out of the load buffer SET_SIZE(scrubphys) #endif /* lint */ #if defined(lint) /* * clearphys - Pass in the physical memory address of the checkblock * that you want to push out, cleared with a recognizable pattern, * from the ecache. * * To ensure that the ecc gets recalculated after the bad data is cleared, * we must write out enough data to fill the w$ line (64 bytes). So we read * in an entire ecache subblock's worth of data, and write it back out. * Then we overwrite the 16 bytes of bad data with the pattern. */ /* ARGSUSED */ void clearphys(uint64_t paddr, int ecache_set_size, int ecache_linesize) { } #else /* lint */ ENTRY(clearphys) /* turn off IE, AM bits */ rdpr %pstate, %o4 andn %o4, PSTATE_IE | PSTATE_AM, %o5 wrpr %o5, %g0, %pstate /* turn off NCEEN */ ldxa [%g0]ASI_ESTATE_ERR, %o5 andn %o5, EN_REG_NCEEN, %o3 stxa %o3, [%g0]ASI_ESTATE_ERR membar #Sync /* align address passed with 64 bytes subblock size */ mov CH_ECACHE_SUBBLK_SIZE, %o2 andn %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1 /* move the good data into the W$ */ clearphys_1: subcc %o2, 8, %o2 ldxa [%g1 + %o2]ASI_MEM, %g2 bge clearphys_1 stxa %g2, [%g1 + %o2]ASI_MEM /* now overwrite the bad data */ setx 0xbadecc00badecc01, %g1, %g2 stxa %g2, [%o0]ASI_MEM mov 8, %g1 stxa %g2, [%o0 + %g1]ASI_MEM GET_CPU_IMPL(%o3) ! Panther Ecache is flushed differently cmp %o3, PANTHER_IMPL bne clearphys_2 nop PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %g1) casxa [%o0]ASI_MEM, %g0, %g0 PN_ECACHE_REFLUSH_LINE(%o1, %o2, %o3, %o0) b clearphys_3 nop clearphys_2: ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) casxa [%o0]ASI_MEM, %g0, %g0 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) clearphys_3: /* clear the AFSR */ ldxa [%g0]ASI_AFSR, %o1 stxa %o1, [%g0]ASI_AFSR membar #Sync /* turn NCEEN back on */ stxa %o5, [%g0]ASI_ESTATE_ERR membar #Sync /* return and re-enable IE and AM */ retl wrpr %g0, %o4, %pstate SET_SIZE(clearphys) #endif /* lint */ #if defined(lint) /* * Cheetah+ Ecache displacement flush the specified line from the E$ * * For Panther, this means flushing the specified line from both the * L2 cache and L3 cache. * * Register usage: * %o0 - 64 bit physical address for flushing * %o1 - Ecache set size */ /*ARGSUSED*/ void ecache_flush_line(uint64_t flushaddr, int ec_set_size) { } #else /* lint */ ENTRY(ecache_flush_line) GET_CPU_IMPL(%o3) ! Panther Ecache is flushed differently cmp %o3, PANTHER_IMPL bne ecache_flush_line_1 nop PN_ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3, %o4) b ecache_flush_line_2 nop ecache_flush_line_1: ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) ecache_flush_line_2: retl nop SET_SIZE(ecache_flush_line) #endif /* lint */ #if defined(lint) void set_afsr_ext(uint64_t afsr_ext) { afsr_ext = afsr_ext; } #else /* lint */ ENTRY(set_afsr_ext) set ASI_AFSR_EXT_VA, %o1 stxa %o0, [%o1]ASI_AFSR ! afsr_ext reg membar #Sync retl nop SET_SIZE(set_afsr_ext) #endif /* lint */ #if defined(lint) /* * The CPU jumps here from the MMU exception handler if an ITLB parity * error is detected and we are running on Panther. * * In this routine we collect diagnostic information and write it to our * logout structure (if possible) and clear all ITLB entries that may have * caused our parity trap. * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0 * and log any error messages. As for parameters to cpu_tlb_parity_error, we * send two: * * %g2 - Contains the VA whose lookup in the ITLB caused the parity error * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct, * regardless of whether or not we actually used the logout struct. * * In the TL0 handler (cpu_tlb_parity_error) we will compare those two * parameters to the data contained in the logout structure in order to * determine whether the logout information is valid for this particular * error or not. */ void itlb_parity_trap(void) {} #else /* lint */ ENTRY_NP(itlb_parity_trap) /* * Collect important information about the trap which will be * used as a parameter to the TL0 handler. */ wr %g0, ASI_IMMU, %asi rdpr %tpc, %g2 ! VA that caused the IMMU trap ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page size set PN_ITLB_PGSZ_MASK, %g4 and %g3, %g4, %g3 ldxa [MMU_TAG_ACCESS]%asi, %g4 set TAGREAD_CTX_MASK, %g5 and %g4, %g5, %g4 or %g4, %g3, %g3 ! 'or' in the trap context and mov 1, %g4 ! add the IMMU flag to complete sllx %g4, PN_TLO_INFO_IMMU_SHIFT, %g4 or %g4, %g3, %g3 ! the tlo_info field for logout stxa %g0,[MMU_SFSR]%asi ! clear the SFSR membar #Sync /* * at this point: * %g2 - contains the VA whose lookup caused the trap * %g3 - contains the tlo_info field * * Next, we calculate the TLB index value for the failing VA. */ mov %g2, %g4 ! We need the ITLB index set PN_ITLB_PGSZ_MASK, %g5 and %g3, %g5, %g5 srlx %g5, PN_ITLB_PGSZ_SHIFT, %g5 PN_GET_TLB_INDEX(%g4, %g5) ! %g4 has the index sllx %g4, PN_TLB_ACC_IDX_SHIFT, %g4 ! shift the index into place set PN_ITLB_T512, %g5 or %g4, %g5, %g4 ! and add in the TLB ID /* * at this point: * %g2 - contains the VA whose lookup caused the trap * %g3 - contains the tlo_info field * %g4 - contains the TLB access index value for the * VA/PgSz in question * * Check to see if the logout structure is available. */ set CHPR_TLB_LOGOUT, %g6 GET_CPU_PRIVATE_PTR(%g6, %g1, %g5, itlb_parity_trap_1) set LOGOUT_INVALID_U32, %g6 sllx %g6, 32, %g6 ! if our logout structure is set LOGOUT_INVALID_L32, %g5 ! unavailable or if it is or %g5, %g6, %g5 ! already being used, then we ldx [%g1 + PN_TLO_ADDR], %g6 ! don't collect any diagnostic cmp %g6, %g5 ! information before clearing bne itlb_parity_trap_1 ! and logging the error. nop /* * Record the logout information. %g4 contains our index + TLB ID * for use in ASI_ITLB_ACCESS and ASI_ITLB_TAGREAD. %g1 contains * the pointer to our logout struct. */ stx %g3, [%g1 + PN_TLO_INFO] stx %g2, [%g1 + PN_TLO_ADDR] stx %g2, [%g1 + PN_TLO_PC] ! %tpc == fault addr for IMMU add %g1, PN_TLO_ITLB_TTE, %g1 ! move up the pointer ldxa [%g4]ASI_ITLB_ACCESS, %g5 ! read the data stx %g5, [%g1 + CH_TLO_TTE_DATA] ! store it away ldxa [%g4]ASI_ITLB_TAGREAD, %g5 ! read the tag stx %g5, [%g1 + CH_TLO_TTE_TAG] ! store it away set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 or %g4, %g6, %g4 add %g1, CH_TLO_TTE_SIZE, %g1 ! move up the pointer ldxa [%g4]ASI_ITLB_ACCESS, %g5 ! read the data stx %g5, [%g1 + CH_TLO_TTE_DATA] ! store it away ldxa [%g4]ASI_ITLB_TAGREAD, %g5 ! read the tag stx %g5, [%g1 + CH_TLO_TTE_TAG] ! store it away andn %g4, %g6, %g4 ! back to way 0 itlb_parity_trap_1: /* * at this point: * %g2 - contains the VA whose lookup caused the trap * %g3 - contains the tlo_info field * %g4 - contains the TLB access index value for the * VA/PgSz in question * * Here we will clear the errors from the TLB. */ set MMU_TAG_ACCESS, %g5 ! We write a TTE tag value of stxa %g0, [%g5]ASI_IMMU ! 0 as it will be invalid. stxa %g0, [%g4]ASI_ITLB_ACCESS ! Write the data and tag membar #Sync set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 or %g4, %g6, %g4 stxa %g0, [%g4]ASI_ITLB_ACCESS ! Write same data and tag membar #Sync sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a flush %g6 ! flush after writing MMU regs /* * at this point: * %g2 - contains the VA whose lookup caused the trap * %g3 - contains the tlo_info field * * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're * already at PIL 15. */ set cpu_tlb_parity_error, %g1 rdpr %pil, %g4 cmp %g4, PIL_14 movl %icc, PIL_14, %g4 ba sys_trap nop SET_SIZE(itlb_parity_trap) #endif /* lint */ #if defined(lint) /* * The CPU jumps here from the MMU exception handler if a DTLB parity * error is detected and we are running on Panther. * * In this routine we collect diagnostic information and write it to our * logout structure (if possible) and clear all DTLB entries that may have * caused our parity trap. * Then we call cpu_tlb_parity_error via systrap in order to drop down to TL0 * and log any error messages. As for parameters to cpu_tlb_parity_error, we * send two: * * %g2 - Contains the VA whose lookup in the DTLB caused the parity error * %g3 - Contains the tlo_info field of the pn_tlb_logout logout struct, * regardless of whether or not we actually used the logout struct. * * In the TL0 handler (cpu_tlb_parity_error) we will compare those two * parameters to the data contained in the logout structure in order to * determine whether the logout information is valid for this particular * error or not. */ void dtlb_parity_trap(void) {} #else /* lint */ ENTRY_NP(dtlb_parity_trap) /* * Collect important information about the trap which will be * used as a parameter to the TL0 handler. */ wr %g0, ASI_DMMU, %asi ldxa [MMU_SFAR]%asi, %g2 ! VA that caused the IMMU trap ldxa [MMU_TAG_ACCESS_EXT]%asi, %g3 ! read the trap VA page sizes set PN_DTLB_PGSZ_MASK, %g4 and %g3, %g4, %g3 ldxa [MMU_TAG_ACCESS]%asi, %g4 set TAGREAD_CTX_MASK, %g5 ! 'or' in the trap context and %g4, %g5, %g4 ! to complete the tlo_info or %g4, %g3, %g3 ! field for logout stxa %g0,[MMU_SFSR]%asi ! clear the SFSR membar #Sync /* * at this point: * %g2 - contains the VA whose lookup caused the trap * %g3 - contains the tlo_info field * * Calculate the TLB index values for the failing VA. Since the T512 * TLBs can be configured for different page sizes, we need to find * the index into each one separately. */ mov %g2, %g4 ! First we get the DTLB_0 index set PN_DTLB_PGSZ0_MASK, %g5 and %g3, %g5, %g5 srlx %g5, PN_DTLB_PGSZ0_SHIFT, %g5 PN_GET_TLB_INDEX(%g4, %g5) ! %g4 has the DTLB_0 index sllx %g4, PN_TLB_ACC_IDX_SHIFT, %g4 ! shift the index into place set PN_DTLB_T512_0, %g5 or %g4, %g5, %g4 ! and add in the TLB ID mov %g2, %g7 ! Next we get the DTLB_1 index set PN_DTLB_PGSZ1_MASK, %g5 and %g3, %g5, %g5 srlx %g5, PN_DTLB_PGSZ1_SHIFT, %g5 PN_GET_TLB_INDEX(%g7, %g5) ! %g7 has the DTLB_1 index sllx %g7, PN_TLB_ACC_IDX_SHIFT, %g7 ! shift the index into place set PN_DTLB_T512_1, %g5 or %g7, %g5, %g7 ! and add in the TLB ID /* * at this point: * %g2 - contains the VA whose lookup caused the trap * %g3 - contains the tlo_info field * %g4 - contains the T512_0 access index value for the * VA/PgSz in question * %g7 - contains the T512_1 access index value for the * VA/PgSz in question * * If this trap happened at TL>0, then we don't want to mess * with the normal logout struct since that could caused a TLB * miss. */ rdpr %tl, %g6 ! read current trap level cmp %g6, 1 ! skip over the tl>1 code ble dtlb_parity_trap_1 ! if TL <= 1. nop /* * If we are here, then the trap happened at TL>1. Simply * update our tlo_info field and then skip to the TLB flush * code. */ mov 1, %g6 sllx %g6, PN_TLO_INFO_TL1_SHIFT, %g6 or %g6, %g3, %g3 ba dtlb_parity_trap_2 nop dtlb_parity_trap_1: /* * at this point: * %g2 - contains the VA whose lookup caused the trap * %g3 - contains the tlo_info field * %g4 - contains the T512_0 access index value for the * VA/PgSz in question * %g7 - contains the T512_1 access index value for the * VA/PgSz in question * * Check to see if the logout structure is available. */ set CHPR_TLB_LOGOUT, %g6 GET_CPU_PRIVATE_PTR(%g6, %g1, %g5, dtlb_parity_trap_2) set LOGOUT_INVALID_U32, %g6 sllx %g6, 32, %g6 ! if our logout structure is set LOGOUT_INVALID_L32, %g5 ! unavailable or if it is or %g5, %g6, %g5 ! already being used, then we ldx [%g1 + PN_TLO_ADDR], %g6 ! don't collect any diagnostic cmp %g6, %g5 ! information before clearing bne dtlb_parity_trap_2 ! and logging the error. nop /* * Record the logout information. %g4 contains our DTLB_0 * index + TLB ID and %g7 contains our DTLB_1 index + TLB ID * both of which will be used for ASI_DTLB_ACCESS and * ASI_DTLB_TAGREAD. %g1 contains the pointer to our logout * struct. */ stx %g3, [%g1 + PN_TLO_INFO] stx %g2, [%g1 + PN_TLO_ADDR] rdpr %tpc, %g5 stx %g5, [%g1 + PN_TLO_PC] add %g1, PN_TLO_DTLB_TTE, %g1 ! move up the pointer ldxa [%g4]ASI_DTLB_ACCESS, %g5 ! read the data from DTLB_0 stx %g5, [%g1 + CH_TLO_TTE_DATA] ! way 0 and store it away ldxa [%g4]ASI_DTLB_TAGREAD, %g5 ! read the tag from DTLB_0 stx %g5, [%g1 + CH_TLO_TTE_TAG] ! way 0 and store it away ldxa [%g7]ASI_DTLB_ACCESS, %g5 ! now repeat for DTLB_1 way 0 stx %g5, [%g1 + (CH_TLO_TTE_DATA + (CH_TLO_TTE_SIZE * 2))] ldxa [%g7]ASI_DTLB_TAGREAD, %g5 stx %g5, [%g1 + (CH_TLO_TTE_TAG + (CH_TLO_TTE_SIZE * 2))] set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 or %g4, %g6, %g4 ! of each TLB. or %g7, %g6, %g7 add %g1, CH_TLO_TTE_SIZE, %g1 ! move up the pointer ldxa [%g4]ASI_DTLB_ACCESS, %g5 ! read the data from DTLB_0 stx %g5, [%g1 + CH_TLO_TTE_DATA] ! way 1 and store it away ldxa [%g4]ASI_DTLB_TAGREAD, %g5 ! read the tag from DTLB_0 stx %g5, [%g1 + CH_TLO_TTE_TAG] ! way 1 and store it away ldxa [%g7]ASI_DTLB_ACCESS, %g5 ! now repeat for DTLB_1 way 1 stx %g5, [%g1 + (CH_TLO_TTE_DATA + (CH_TLO_TTE_SIZE * 2))] ldxa [%g7]ASI_DTLB_TAGREAD, %g5 stx %g5, [%g1 + (CH_TLO_TTE_TAG + (CH_TLO_TTE_SIZE * 2))] andn %g4, %g6, %g4 ! back to way 0 andn %g7, %g6, %g7 ! back to way 0 dtlb_parity_trap_2: /* * at this point: * %g2 - contains the VA whose lookup caused the trap * %g3 - contains the tlo_info field * %g4 - contains the T512_0 access index value for the * VA/PgSz in question * %g7 - contains the T512_1 access index value for the * VA/PgSz in question * * Here we will clear the errors from the DTLB. */ set MMU_TAG_ACCESS, %g5 ! We write a TTE tag value of stxa %g0, [%g5]ASI_DMMU ! 0 as it will be invalid. stxa %g0, [%g4]ASI_DTLB_ACCESS ! Write the data and tag. stxa %g0, [%g7]ASI_DTLB_ACCESS ! Now repeat for DTLB_1 way 0 membar #Sync set PN_TLB_ACC_WAY_BIT, %g6 ! same thing again for way 1 or %g4, %g6, %g4 or %g7, %g6, %g7 stxa %g0, [%g4]ASI_DTLB_ACCESS ! Write same data and tag. stxa %g0, [%g7]ASI_DTLB_ACCESS ! Now repeat for DTLB_1 way 0 membar #Sync sethi %hi(FLUSH_ADDR), %g6 ! PRM says we need to issue a flush %g6 ! flush after writing MMU regs /* * at this point: * %g2 - contains the VA whose lookup caused the trap * %g3 - contains the tlo_info field * * Call cpu_tlb_parity_error via systrap at PIL 14 unless we're * already at PIL 15. We do this even for TL>1 traps since * those will lead to a system panic. */ set cpu_tlb_parity_error, %g1 rdpr %pil, %g4 cmp %g4, PIL_14 movl %icc, PIL_14, %g4 ba sys_trap nop SET_SIZE(dtlb_parity_trap) #endif /* lint */ #if defined(lint) /* * Calculates the Panther TLB index based on a virtual address and page size * * Register usage: * %o0 - virtual address whose index we want * %o1 - Page Size of the TLB in question as encoded in the * ASI_[D|I]MMU_TAG_ACCESS_EXT register. */ uint64_t pn_get_tlb_index(uint64_t va, uint64_t pg_sz) { return ((va + pg_sz)-(va + pg_sz)); } #else /* lint */ ENTRY(pn_get_tlb_index) PN_GET_TLB_INDEX(%o0, %o1) retl nop SET_SIZE(pn_get_tlb_index) #endif /* lint */ #if defined(lint) /* * For Panther CPUs we need to flush the IPB after any I$ or D$ * parity errors are detected. */ void flush_ipb(void) { return; } #else /* lint */ ENTRY(flush_ipb) clr %o0 flush_ipb_1: stxa %g0, [%o0]ASI_IPB_TAG membar #Sync cmp %o0, PN_IPB_TAG_ADDR_MAX blt flush_ipb_1 add %o0, PN_IPB_TAG_ADDR_LINESIZE, %o0 sethi %hi(FLUSH_ADDR), %o0 flush %o0 retl nop SET_SIZE(flush_ipb) #endif /* lint */