/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #if !defined(lint) #include "assym.h" #endif /* lint */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef TRAPTRACE #include #endif /* TRAPTRACE */ #ifndef lint /* BEGIN CSTYLED */ #define DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3) \ ldxa [%g0]ASI_LSU, tmp1 ;\ btst LSU_DC, tmp1 /* is dcache enabled? */ ;\ bz,pn %icc, 1f ;\ sethi %hi(dcache_linesize), tmp1 ;\ ld [tmp1 + %lo(dcache_linesize)], tmp1 ;\ sethi %hi(dflush_type), tmp2 ;\ ld [tmp2 + %lo(dflush_type)], tmp2 ;\ cmp tmp2, FLUSHPAGE_TYPE ;\ be,pt %icc, 2f ;\ sllx arg1, SF_DC_VBIT_SHIFT, arg1 /* tag to compare */ ;\ sethi %hi(dcache_size), tmp3 ;\ ld [tmp3 + %lo(dcache_size)], tmp3 ;\ cmp tmp2, FLUSHMATCH_TYPE ;\ be,pt %icc, 3f ;\ nop ;\ /* \ * flushtype = FLUSHALL_TYPE, flush the whole thing \ * tmp3 = cache size \ * tmp1 = cache line size \ */ \ sub tmp3, tmp1, tmp2 ;\ 4: \ stxa %g0, [tmp2]ASI_DC_TAG ;\ membar #Sync ;\ cmp %g0, tmp2 ;\ bne,pt %icc, 4b ;\ sub tmp2, tmp1, tmp2 ;\ ba,pt %icc, 1f ;\ nop ;\ /* \ * flushtype = FLUSHPAGE_TYPE \ * arg1 = tag to compare against \ * arg2 = virtual color \ * tmp1 = cache line size \ * tmp2 = tag from cache \ * tmp3 = counter \ */ \ 2: \ set MMU_PAGESIZE, tmp3 ;\ sllx arg2, MMU_PAGESHIFT, arg2 /* color to dcache page */ ;\ sub tmp3, tmp1, tmp3 ;\ 4: \ ldxa [arg2 + tmp3]ASI_DC_TAG, tmp2 /* read tag */ ;\ btst SF_DC_VBIT_MASK, tmp2 ;\ bz,pn %icc, 5f /* branch if no valid sub-blocks */ ;\ andn tmp2, SF_DC_VBIT_MASK, tmp2 /* clear out v bits */ ;\ cmp tmp2, arg1 ;\ bne,pn %icc, 5f /* br if tag miss */ ;\ nop ;\ stxa %g0, [arg2 + tmp3]ASI_DC_TAG ;\ membar #Sync ;\ 5: \ cmp %g0, tmp3 ;\ bnz,pt %icc, 4b /* branch if not done */ ;\ sub tmp3, tmp1, tmp3 ;\ ba,pt %icc, 1f ;\ nop ;\ /* \ * flushtype = FLUSHMATCH_TYPE \ * arg1 = tag to compare against \ * tmp1 = cache line size \ * tmp3 = cache size \ * arg2 = counter \ * tmp2 = cache tag \ */ \ 3: \ sub tmp3, tmp1, arg2 ;\ 4: \ ldxa [arg2]ASI_DC_TAG, tmp2 /* read tag */ ;\ btst SF_DC_VBIT_MASK, tmp2 ;\ bz,pn %icc, 5f /* br if no valid sub-blocks */ ;\ andn tmp2, SF_DC_VBIT_MASK, tmp2 /* clear out v bits */ ;\ cmp tmp2, arg1 ;\ bne,pn %icc, 5f /* branch if tag miss */ ;\ nop ;\ stxa %g0, [arg2]ASI_DC_TAG ;\ membar #Sync ;\ 5: \ cmp %g0, arg2 ;\ bne,pt %icc, 4b /* branch if not done */ ;\ sub arg2, tmp1, arg2 ;\ 1: /* * macro that flushes the entire dcache color */ #define DCACHE_FLUSHCOLOR(arg, tmp1, tmp2) \ ldxa [%g0]ASI_LSU, tmp1; \ btst LSU_DC, tmp1; /* is dcache enabled? */ \ bz,pn %icc, 1f; \ sethi %hi(dcache_linesize), tmp1; \ ld [tmp1 + %lo(dcache_linesize)], tmp1; \ set MMU_PAGESIZE, tmp2; \ /* \ * arg = virtual color \ * tmp2 = page size \ * tmp1 = cache line size \ */ \ sllx arg, MMU_PAGESHIFT, arg; /* color to dcache page */ \ sub tmp2, tmp1, tmp2; \ 2: \ stxa %g0, [arg + tmp2]ASI_DC_TAG; \ membar #Sync; \ cmp %g0, tmp2; \ bne,pt %icc, 2b; \ sub tmp2, tmp1, tmp2; \ 1: /* * macro that flushes the entire dcache */ #define DCACHE_FLUSHALL(size, linesize, tmp) \ ldxa [%g0]ASI_LSU, tmp; \ btst LSU_DC, tmp; /* is dcache enabled? */ \ bz,pn %icc, 1f; \ \ sub size, linesize, tmp; \ 2: \ stxa %g0, [tmp]ASI_DC_TAG; \ membar #Sync; \ cmp %g0, tmp; \ bne,pt %icc, 2b; \ sub tmp, linesize, tmp; \ 1: /* * macro that flushes the entire icache */ #define ICACHE_FLUSHALL(size, linesize, tmp) \ ldxa [%g0]ASI_LSU, tmp; \ btst LSU_IC, tmp; \ bz,pn %icc, 1f; \ \ sub size, linesize, tmp; \ 2: \ stxa %g0, [tmp]ASI_IC_TAG; \ membar #Sync; \ cmp %g0, tmp; \ bne,pt %icc, 2b; \ sub tmp, linesize, tmp; \ 1: #ifdef SF_ERRATA_32 #define SF_WORKAROUND(tmp1, tmp2) \ sethi %hi(FLUSH_ADDR), tmp2 ;\ set MMU_PCONTEXT, tmp1 ;\ stxa %g0, [tmp1]ASI_DMMU ;\ flush tmp2 ; #else #define SF_WORKAROUND(tmp1, tmp2) #endif /* SF_ERRATA_32 */ /* * arg1 = vaddr * arg2 = ctxnum * - disable interrupts and clear address mask * to access 64 bit physaddr * - Blow out the TLB, flush user page. * . use secondary context. */ #define VTAG_FLUSHUPAGE(lbl, arg1, arg2, tmp1, tmp2, tmp3, tmp4) \ rdpr %pstate, tmp1 ;\ andn tmp1, PSTATE_IE, tmp2 ;\ wrpr tmp2, 0, %pstate ;\ sethi %hi(FLUSH_ADDR), tmp2 ;\ set MMU_SCONTEXT, tmp3 ;\ ldxa [tmp3]ASI_DMMU, tmp4 ;\ or DEMAP_SECOND | DEMAP_PAGE_TYPE, arg1, arg1 ;\ cmp tmp4, arg2 ;\ be,a,pt %icc, lbl/**/4 ;\ nop ;\ stxa arg2, [tmp3]ASI_DMMU ;\ lbl/**/4: ;\ stxa %g0, [arg1]ASI_DTLB_DEMAP ;\ stxa %g0, [arg1]ASI_ITLB_DEMAP ;\ flush tmp2 ;\ be,a,pt %icc, lbl/**/5 ;\ nop ;\ stxa tmp4, [tmp3]ASI_DMMU ;\ flush tmp2 ;\ lbl/**/5: ;\ wrpr %g0, tmp1, %pstate /* * macro that flushes all the user entries in dtlb * arg1 = dtlb entries * - Before first compare: * tmp4 = tte * tmp5 = vaddr * tmp6 = cntxnum */ #define DTLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \ tmp4, tmp5, tmp6) \ lbl/**/0: ;\ sllx arg1, 3, tmp3 ;\ SF_WORKAROUND(tmp1, tmp2) ;\ ldxa [tmp3]ASI_DTLB_ACCESS, tmp4 ;\ srlx tmp4, 6, tmp4 ;\ andcc tmp4, 1, %g0 ;\ bnz,pn %xcc, lbl/**/1 ;\ srlx tmp4, 57, tmp4 ;\ andcc tmp4, 1, %g0 ;\ beq,pn %xcc, lbl/**/1 ;\ nop ;\ set TAGREAD_CTX_MASK, tmp1 ;\ ldxa [tmp3]ASI_DTLB_TAGREAD, tmp2 ;\ and tmp2, tmp1, tmp6 ;\ andn tmp2, tmp1, tmp5 ;\ set KCONTEXT, tmp4 ;\ cmp tmp6, tmp4 ;\ be lbl/**/1 ;\ nop ;\ VTAG_FLUSHUPAGE(VD/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\ lbl/**/1: ;\ brgz,pt arg1, lbl/**/0 ;\ sub arg1, 1, arg1 /* * macro that flushes all the user entries in itlb * arg1 = itlb entries * - Before first compare: * tmp4 = tte * tmp5 = vaddr * tmp6 = cntxnum */ #define ITLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \ tmp4, tmp5, tmp6) \ lbl/**/0: ;\ sllx arg1, 3, tmp3 ;\ SF_WORKAROUND(tmp1, tmp2) ;\ ldxa [tmp3]ASI_ITLB_ACCESS, tmp4 ;\ srlx tmp4, 6, tmp4 ;\ andcc tmp4, 1, %g0 ;\ bnz,pn %xcc, lbl/**/1 ;\ srlx tmp4, 57, tmp4 ;\ andcc tmp4, 1, %g0 ;\ beq,pn %xcc, lbl/**/1 ;\ nop ;\ set TAGREAD_CTX_MASK, tmp1 ;\ ldxa [tmp3]ASI_ITLB_TAGREAD, tmp2 ;\ and tmp2, tmp1, tmp6 ;\ andn tmp2, tmp1, tmp5 ;\ set KCONTEXT, tmp4 ;\ cmp tmp6, tmp4 ;\ be lbl/**/1 ;\ nop ;\ VTAG_FLUSHUPAGE(VI/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\ lbl/**/1: ;\ brgz,pt arg1, lbl/**/0 ;\ sub arg1, 1, arg1 /* * Macro for getting to offset from 'cpu_private' ptr. The 'cpu_private' * ptr is in the machcpu structure. * r_or_s: Register or symbol off offset from 'cpu_private' ptr. * scr1: Scratch, ptr is returned in this register. * scr2: Scratch */ #define GET_CPU_PRIVATE_PTR(r_or_s, scr1, scr2, label) \ CPU_ADDR(scr1, scr2); \ ldn [scr1 + CPU_PRIVATE], scr1; \ cmp scr1, 0; \ be label; \ nop; \ add scr1, r_or_s, scr1; \ #ifdef HUMMINGBIRD /* * UltraSPARC-IIe processor supports both 4-way set associative and * direct map E$. For performance reasons, we flush E$ by placing it * in direct map mode for data load/store and restore the state after * we are done flushing it. Keep interrupts off while flushing in this * manner. * * We flush the entire ecache by starting at one end and loading each * successive ecache line for the 2*ecache-size range. We have to repeat * the flush operation to guarantee that the entire ecache has been * flushed. * * For flushing a specific physical address, we start at the aliased * address and load at set-size stride, wrapping around at 2*ecache-size * boundary and skipping the physical address being flushed. It takes * 10 loads to guarantee that the physical address has been flushed. */ #define HB_ECACHE_FLUSH_CNT 2 #define HB_PHYS_FLUSH_CNT 10 /* #loads to flush specific paddr */ #endif /* HUMMINGBIRD */ /* END CSTYLED */ #endif /* !lint */ /* * Spitfire MMU and Cache operations. */ #if defined(lint) /*ARGSUSED*/ void vtag_flushpage(caddr_t vaddr, uint64_t sfmmup) {} /*ARGSUSED*/ void vtag_flushall(void) {} /*ARGSUSED*/ void vtag_flushall_uctxs(void) {} /*ARGSUSED*/ void vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup) {} /*ARGSUSED*/ void vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt) {} /*ARGSUSED*/ void vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2) {} /*ARGSUSED*/ void vac_flushpage(pfn_t pfnum, int vcolor) {} /*ARGSUSED*/ void vac_flushpage_tl1(uint64_t pfnum, uint64_t vcolor) {} /*ARGSUSED*/ void init_mondo(xcfunc_t *func, uint64_t arg1, uint64_t arg2) {} /*ARGSUSED*/ void init_mondo_nocheck(xcfunc_t *func, uint64_t arg1, uint64_t arg2) {} /*ARGSUSED*/ void flush_instr_mem(caddr_t vaddr, size_t len) {} /*ARGSUSED*/ void flush_ecache(uint64_t physaddr, size_t size, size_t linesize) {} /*ARGSUSED*/ void get_ecache_dtag(uint32_t ecache_idx, uint64_t *ecache_data, uint64_t *ecache_tag, uint64_t *oafsr, uint64_t *acc_afsr) {} /* ARGSUSED */ uint64_t get_ecache_tag(uint32_t id, uint64_t *nafsr, uint64_t *acc_afsr) { return ((uint64_t)0); } /* ARGSUSED */ uint64_t check_ecache_line(uint32_t id, uint64_t *acc_afsr) { return ((uint64_t)0); } /*ARGSUSED*/ void kdi_flush_idcache(int dcache_size, int dcache_lsize, int icache_size, int icache_lsize) {} #else /* lint */ ENTRY_NP(vtag_flushpage) /* * flush page from the tlb * * %o0 = vaddr * %o1 = sfmmup */ rdpr %pstate, %o5 #ifdef DEBUG PANIC_IF_INTR_DISABLED_PSTR(%o5, sfdi_label1, %g1) #endif /* DEBUG */ /* * disable ints */ andn %o5, PSTATE_IE, %o4 wrpr %o4, 0, %pstate /* * Then, blow out the tlb * Interrupts are disabled to prevent the secondary ctx register * from changing underneath us. */ sethi %hi(ksfmmup), %o3 ldx [%o3 + %lo(ksfmmup)], %o3 cmp %o3, %o1 bne,pt %xcc, 1f ! if not kernel as, go to 1 sethi %hi(FLUSH_ADDR), %o3 /* * For KCONTEXT demaps use primary. type = page implicitly */ stxa %g0, [%o0]ASI_DTLB_DEMAP /* dmmu flush for KCONTEXT */ stxa %g0, [%o0]ASI_ITLB_DEMAP /* immu flush for KCONTEXT */ flush %o3 b 5f nop 1: /* * User demap. We need to set the secondary context properly. * %o0 = vaddr * %o1 = sfmmup * %o3 = FLUSH_ADDR */ SFMMU_CPU_CNUM(%o1, %g1, %g2) /* %g1 = sfmmu cnum on this CPU */ set MMU_SCONTEXT, %o4 ldxa [%o4]ASI_DMMU, %o2 /* rd old ctxnum */ or DEMAP_SECOND | DEMAP_PAGE_TYPE, %o0, %o0 cmp %o2, %g1 be,pt %icc, 4f nop stxa %g1, [%o4]ASI_DMMU /* wr new ctxum */ 4: stxa %g0, [%o0]ASI_DTLB_DEMAP stxa %g0, [%o0]ASI_ITLB_DEMAP flush %o3 be,pt %icc, 5f nop stxa %o2, [%o4]ASI_DMMU /* restore old ctxnum */ flush %o3 5: retl wrpr %g0, %o5, %pstate /* enable interrupts */ SET_SIZE(vtag_flushpage) .seg ".text" .flushallmsg: .asciz "sfmmu_asm: unimplemented flush operation" ENTRY_NP(vtag_flushall) sethi %hi(.flushallmsg), %o0 call panic or %o0, %lo(.flushallmsg), %o0 SET_SIZE(vtag_flushall) ENTRY_NP(vtag_flushall_uctxs) /* * flush entire DTLB/ITLB. */ CPU_INDEX(%g1, %g2) mulx %g1, CPU_NODE_SIZE, %g1 set cpunodes, %g2 add %g1, %g2, %g1 lduh [%g1 + ITLB_SIZE], %g2 ! %g2 = # entries in ITLB lduh [%g1 + DTLB_SIZE], %g1 ! %g1 = # entries in DTLB sub %g2, 1, %g2 ! %g2 = # entries in ITLB - 1 sub %g1, 1, %g1 ! %g1 = # entries in DTLB - 1 ! ! Flush itlb's ! ITLB_FLUSH_UNLOCKED_UCTXS(I, %g2, %g3, %g4, %o2, %o3, %o4, %o5) ! ! Flush dtlb's ! DTLB_FLUSH_UNLOCKED_UCTXS(D, %g1, %g3, %g4, %o2, %o3, %o4, %o5) membar #Sync retl nop SET_SIZE(vtag_flushall_uctxs) ENTRY_NP(vtag_flushpage_tl1) /* * x-trap to flush page from tlb and tsb * * %g1 = vaddr, zero-extended on 32-bit kernel * %g2 = sfmmup * * assumes TSBE_TAG = 0 */ srln %g1, MMU_PAGESHIFT, %g1 slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ SFMMU_CPU_CNUM(%g2, %g3, %g4) /* %g3 = sfmmu cnum on this CPU */ /* We need to set the secondary context properly. */ set MMU_SCONTEXT, %g4 ldxa [%g4]ASI_DMMU, %g5 /* rd old ctxnum */ or DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1 stxa %g3, [%g4]ASI_DMMU /* wr new ctxum */ stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ membar #Sync retry SET_SIZE(vtag_flushpage_tl1) ENTRY_NP(vtag_flush_pgcnt_tl1) /* * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb * * %g1 = vaddr, zero-extended on 32-bit kernel * %g2 = * * NOTE: this handler relies on the fact that no * interrupts or traps can occur during the loop * issuing the TLB_DEMAP operations. It is assumed * that interrupts are disabled and this code is * fetching from the kernel locked text address. * * assumes TSBE_TAG = 0 */ srln %g1, MMU_PAGESHIFT, %g1 slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ or DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1 set SFMMU_PGCNT_MASK, %g4 and %g4, %g2, %g3 /* g3 = pgcnt - 1 */ add %g3, 1, %g3 /* g3 = pgcnt */ andn %g2, SFMMU_PGCNT_MASK, %g2 /* g2 = sfmmup */ SFMMU_CPU_CNUM(%g2, %g5, %g6) ! %g5 = sfmmu cnum on this CPU /* We need to set the secondary context properly. */ set MMU_SCONTEXT, %g4 ldxa [%g4]ASI_DMMU, %g6 /* read old ctxnum */ stxa %g5, [%g4]ASI_DMMU /* write new ctxum */ set MMU_PAGESIZE, %g2 /* g2 = pgsize */ sethi %hi(FLUSH_ADDR), %g5 1: stxa %g0, [%g1]ASI_DTLB_DEMAP stxa %g0, [%g1]ASI_ITLB_DEMAP flush %g5 deccc %g3 /* decr pgcnt */ bnz,pt %icc,1b add %g1, %g2, %g1 /* go to nextpage */ stxa %g6, [%g4]ASI_DMMU /* restore old ctxnum */ membar #Sync retry SET_SIZE(vtag_flush_pgcnt_tl1) ! Not implemented on US1/US2 ENTRY_NP(vtag_flushall_tl1) retry SET_SIZE(vtag_flushall_tl1) /* * vac_flushpage(pfnum, color) * Flush 1 8k page of the D-$ with physical page = pfnum * Algorithm: * The spitfire dcache is a 16k direct mapped virtual indexed, * physically tagged cache. Given the pfnum we read all cache * lines for the corresponding page in the cache (determined by * the color). Each cache line is compared with * the tag created from the pfnum. If the tags match we flush * the line. */ .seg ".data" .align 8 .global dflush_type dflush_type: .word FLUSHPAGE_TYPE .seg ".text" ENTRY(vac_flushpage) /* * flush page from the d$ * * %o0 = pfnum, %o1 = color */ DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4) retl nop SET_SIZE(vac_flushpage) ENTRY_NP(vac_flushpage_tl1) /* * x-trap to flush page from the d$ * * %g1 = pfnum, %g2 = color */ DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5) retry SET_SIZE(vac_flushpage_tl1) ENTRY(vac_flushcolor) /* * %o0 = vcolor */ DCACHE_FLUSHCOLOR(%o0, %o1, %o2) retl nop SET_SIZE(vac_flushcolor) ENTRY(vac_flushcolor_tl1) /* * %g1 = vcolor */ DCACHE_FLUSHCOLOR(%g1, %g2, %g3) retry SET_SIZE(vac_flushcolor_tl1) .global _dispatch_status_busy _dispatch_status_busy: .asciz "ASI_INTR_DISPATCH_STATUS error: busy" .align 4 /* * Determine whether or not the IDSR is busy. * Entry: no arguments * Returns: 1 if busy, 0 otherwise */ ENTRY(idsr_busy) ldxa [%g0]ASI_INTR_DISPATCH_STATUS, %g1 clr %o0 btst IDSR_BUSY, %g1 bz,a,pt %xcc, 1f mov 1, %o0 1: retl nop SET_SIZE(idsr_busy) /* * Setup interrupt dispatch data registers * Entry: * %o0 - function or inumber to call * %o1, %o2 - arguments (2 uint64_t's) */ .seg "text" ENTRY(init_mondo) #ifdef DEBUG ! ! IDSR should not be busy at the moment ! ldxa [%g0]ASI_INTR_DISPATCH_STATUS, %g1 btst IDSR_BUSY, %g1 bz,pt %xcc, 1f nop sethi %hi(_dispatch_status_busy), %o0 call panic or %o0, %lo(_dispatch_status_busy), %o0 #endif /* DEBUG */ ALTENTRY(init_mondo_nocheck) ! ! interrupt vector dispach data reg 0 ! 1: mov IDDR_0, %g1 mov IDDR_1, %g2 mov IDDR_2, %g3 stxa %o0, [%g1]ASI_INTR_DISPATCH ! ! interrupt vector dispach data reg 1 ! stxa %o1, [%g2]ASI_INTR_DISPATCH ! ! interrupt vector dispach data reg 2 ! stxa %o2, [%g3]ASI_INTR_DISPATCH retl membar #Sync ! allowed to be in the delay slot SET_SIZE(init_mondo) /* * Ship mondo to upaid */ ENTRY_NP(shipit) sll %o0, IDCR_PID_SHIFT, %g1 ! IDCR<18:14> = upa id or %g1, IDCR_OFFSET, %g1 ! IDCR<13:0> = 0x70 stxa %g0, [%g1]ASI_INTR_DISPATCH ! interrupt vector dispatch #if defined(SF_ERRATA_54) membar #Sync ! store must occur before load mov 0x20, %g3 ! UDBH Control Register Read ldxa [%g3]ASI_SDB_INTR_R, %g0 #endif retl membar #Sync SET_SIZE(shipit) /* * flush_instr_mem: * Flush a portion of the I-$ starting at vaddr * %o0 vaddr * %o1 bytes to be flushed */ ENTRY(flush_instr_mem) membar #StoreStore ! Ensure the stores ! are globally visible 1: flush %o0 subcc %o1, ICACHE_FLUSHSZ, %o1 ! bytes = bytes-0x20 bgu,pt %ncc, 1b add %o0, ICACHE_FLUSHSZ, %o0 ! vaddr = vaddr+0x20 retl nop SET_SIZE(flush_instr_mem) /* * flush_ecache: * Flush the entire e$ using displacement flush by reading through a * physically contiguous area. We use mmu bypass asi (ASI_MEM) while * reading this physical address range so that data doesn't go to d$. * incoming arguments: * %o0 - 64 bit physical address * %o1 - size of address range to read * %o2 - ecache linesize */ ENTRY(flush_ecache) #ifndef HUMMINGBIRD b 2f nop 1: ldxa [%o0 + %o1]ASI_MEM, %g0 ! start reading from physaddr + size 2: subcc %o1, %o2, %o1 bcc,a,pt %ncc, 1b nop #else /* HUMMINGBIRD */ /* * UltraSPARC-IIe processor supports both 4-way set associative * and direct map E$. For performance reasons, we flush E$ by * placing it in direct map mode for data load/store and restore * the state after we are done flushing it. It takes 2 iterations * to guarantee that the entire ecache has been flushed. * * Keep the interrupts disabled while flushing E$ in this manner. */ rdpr %pstate, %g4 ! current pstate (restored later) andn %g4, PSTATE_IE, %g5 wrpr %g0, %g5, %pstate ! disable interrupts ! Place E$ in direct map mode for data access or %g0, 1, %g5 sllx %g5, HB_UPA_DMAP_DATA_BIT, %g5 ldxa [%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later) or %g1, %g5, %g5 membar #Sync stxa %g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access membar #Sync ! flush entire ecache HB_ECACHE_FLUSH_CNT times mov HB_ECACHE_FLUSH_CNT-1, %g5 2: sub %o1, %o2, %g3 ! start from last entry 1: ldxa [%o0 + %g3]ASI_MEM, %g0 ! start reading from physaddr + size subcc %g3, %o2, %g3 bgeu,a,pt %ncc, 1b nop brgz,a,pt %g5, 2b dec %g5 membar #Sync stxa %g1, [%g0]ASI_UPA_CONFIG ! restore UPA config reg membar #Sync wrpr %g0, %g4, %pstate ! restore earlier pstate #endif /* HUMMINGBIRD */ retl nop SET_SIZE(flush_ecache) /* * void kdi_flush_idcache(int dcache_size, int dcache_linesize, * int icache_size, int icache_linesize) */ ENTRY(kdi_flush_idcache) DCACHE_FLUSHALL(%o0, %o1, %g1) ICACHE_FLUSHALL(%o2, %o3, %g1) membar #Sync retl nop SET_SIZE(kdi_flush_idcache) /* * void get_ecache_dtag(uint32_t ecache_idx, uint64_t *data, uint64_t *tag, * uint64_t *oafsr, uint64_t *acc_afsr) * * Get ecache data and tag. The ecache_idx argument is assumed to be aligned * on a 64-byte boundary. The corresponding AFSR value is also read for each * 8 byte ecache data obtained. The ecache data is assumed to be a pointer * to an array of 16 uint64_t's (e$data & afsr value). The action to read the * data and tag should be atomic to make sense. We will be executing at PIL15 * and will disable IE, so nothing can occur between the two reads. We also * assume that the execution of this code does not interfere with what we are * reading - not really possible, but we'll live with it for now. * We also pass the old AFSR value before clearing it, and caller will take * appropriate actions if the important bits are non-zero. * * If the caller wishes to track the AFSR in cases where the CP bit is * set, an address should be passed in for acc_afsr. Otherwise, this * argument may be null. * * Register Usage: * i0: In: 32-bit e$ index * i1: In: addr of e$ data * i2: In: addr of e$ tag * i3: In: addr of old afsr * i4: In: addr of accumulated afsr - may be null */ ENTRY(get_ecache_dtag) save %sp, -SA(MINFRAME), %sp or %g0, 1, %l4 sllx %l4, 39, %l4 ! set bit 39 for e$ data access or %i0, %l4, %g6 ! %g6 = e$ addr for data read sllx %l4, 1, %l4 ! set bit 40 for e$ tag access or %i0, %l4, %l4 ! %l4 = e$ addr for tag read rdpr %pstate, %i5 andn %i5, PSTATE_IE | PSTATE_AM, %i0 wrpr %i0, %g0, %pstate ! clear IE, AM bits ldxa [%g0]ASI_ESTATE_ERR, %g1 stxa %g0, [%g0]ASI_ESTATE_ERR ! disable errors membar #Sync ldxa [%g0]ASI_AFSR, %i0 ! grab the old-afsr before tag read stx %i0, [%i3] ! write back the old-afsr ldxa [%l4]ASI_EC_R, %g0 ! read tag into E$ tag reg ldxa [%g0]ASI_EC_DIAG, %i0 ! read tag from E$ tag reg stx %i0, [%i2] ! write back tag result clr %i2 ! loop count brz %i4, 1f ! acc_afsr == NULL? ldxa [%g0]ASI_AFSR, %i0 ! grab the old-afsr before clearing srlx %i0, P_AFSR_CP_SHIFT, %l0 btst 1, %l0 bz 1f nop ldx [%i4], %g4 or %g4, %i0, %g4 ! aggregate AFSR in cpu private stx %g4, [%i4] 1: stxa %i0, [%g0]ASI_AFSR ! clear AFSR membar #Sync ldxa [%g6]ASI_EC_R, %i0 ! read the 8byte E$data stx %i0, [%i1] ! save the E$data add %g6, 8, %g6 add %i1, 8, %i1 ldxa [%g0]ASI_AFSR, %i0 ! read AFSR for this 16byte read srlx %i0, P_AFSR_CP_SHIFT, %l0 btst 1, %l0 bz 2f stx %i0, [%i1] ! save the AFSR brz %i4, 2f ! acc_afsr == NULL? nop ldx [%i4], %g4 or %g4, %i0, %g4 ! aggregate AFSR in cpu private stx %g4, [%i4] 2: add %i2, 8, %i2 cmp %i2, 64 bl,a 1b add %i1, 8, %i1 stxa %i0, [%g0]ASI_AFSR ! clear AFSR membar #Sync stxa %g1, [%g0]ASI_ESTATE_ERR ! restore error enable membar #Sync wrpr %g0, %i5, %pstate ret restore SET_SIZE(get_ecache_dtag) #endif /* lint */ #if defined(lint) /* * The ce_err function handles trap type 0x63 (corrected_ECC_error) at tl=0. * Steps: 1. GET AFSR 2. Get AFAR <40:4> 3. Get datapath error status * 4. Clear datapath error bit(s) 5. Clear AFSR error bit * 6. package data in %g2 and %g3 7. call cpu_ce_error vis sys_trap * %g2: [ 52:43 UDB lower | 42:33 UDB upper | 32:0 afsr ] - arg #3/arg #1 * %g3: [ 40:4 afar ] - sys_trap->have_win: arg #4/arg #2 */ void ce_err(void) {} void ce_err_tl1(void) {} /* * The async_err function handles trap types 0x0A (instruction_access_error) * and 0x32 (data_access_error) at TL = 0 and TL > 0. When we branch here, * %g5 will have the trap type (with 0x200 set if we're at TL > 0). * * Steps: 1. Get AFSR 2. Get AFAR <40:4> 3. If not UE error skip UDP registers. * 4. Else get and clear datapath error bit(s) 4. Clear AFSR error bits * 6. package data in %g2 and %g3 7. disable all cpu errors, because * trap is likely to be fatal 8. call cpu_async_error vis sys_trap * * %g3: [ 63:53 tt | 52:43 UDB_L | 42:33 UDB_U | 32:0 afsr ] - arg #3/arg #1 * %g2: [ 40:4 afar ] - sys_trap->have_win: arg #4/arg #2 */ void async_err(void) {} /* * The clr_datapath function clears any error bits set in the UDB regs. */ void clr_datapath(void) {} /* * The get_udb_errors() function gets the current value of the * Datapath Error Registers. */ /*ARGSUSED*/ void get_udb_errors(uint64_t *udbh, uint64_t *udbl) { *udbh = 0; *udbl = 0; } #else /* lint */ ENTRY_NP(ce_err) ldxa [%g0]ASI_AFSR, %g3 ! save afsr in g3 ! ! Check for a UE... From Kevin.Normoyle: ! We try to switch to the trap for the UE, but since that's ! a hardware pipeline, we might get to the CE trap before we ! can switch. The UDB and AFSR registers will have both the ! UE and CE bits set but the UDB syndrome and the AFAR will be ! for the UE. ! or %g0, 1, %g1 ! put 1 in g1 sllx %g1, 21, %g1 ! shift left to <21> afsr UE andcc %g1, %g3, %g0 ! check for UE in afsr bnz async_err ! handle the UE, not the CE or %g0, 0x63, %g5 ! pass along the CE ttype ! ! Disable further CE traps to avoid recursion (stack overflow) ! and staying above XCALL_PIL for extended periods. ! ldxa [%g0]ASI_ESTATE_ERR, %g2 andn %g2, 0x1, %g2 ! clear bit 0 - CEEN stxa %g2, [%g0]ASI_ESTATE_ERR membar #Sync ! required ! ! handle the CE ldxa [%g0]ASI_AFAR, %g2 ! save afar in g2 set P_DER_H, %g4 ! put P_DER_H in g4 ldxa [%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into g5 or %g0, 1, %g6 ! put 1 in g6 sllx %g6, 8, %g6 ! shift g6 to <8> sdb CE andcc %g5, %g6, %g1 ! check for CE in upper half sllx %g5, 33, %g5 ! shift upper bits to <42:33> or %g3, %g5, %g3 ! or with afsr bits bz,a 1f ! no error, goto 1f nop stxa %g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg error bit membar #Sync ! membar sync required 1: set P_DER_L, %g4 ! put P_DER_L in g4 ldxa [%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g6 andcc %g5, %g6, %g1 ! check for CE in lower half sllx %g5, 43, %g5 ! shift upper bits to <52:43> or %g3, %g5, %g3 ! or with afsr bits bz,a 2f ! no error, goto 2f nop stxa %g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg error bit membar #Sync ! membar sync required 2: or %g0, 1, %g4 ! put 1 in g4 sllx %g4, 20, %g4 ! shift left to <20> afsr CE stxa %g4, [%g0]ASI_AFSR ! use g4 to clear afsr CE error membar #Sync ! membar sync required set cpu_ce_error, %g1 ! put *cpu_ce_error() in g1 rdpr %pil, %g6 ! read pil into %g6 subcc %g6, PIL_15, %g0 movneg %icc, PIL_14, %g4 ! run at pil 14 unless already at 15 sethi %hi(sys_trap), %g5 jmp %g5 + %lo(sys_trap) ! goto sys_trap movge %icc, PIL_15, %g4 ! already at pil 15 SET_SIZE(ce_err) ENTRY_NP(ce_err_tl1) #ifndef TRAPTRACE ldxa [%g0]ASI_AFSR, %g7 stxa %g7, [%g0]ASI_AFSR membar #Sync retry #else set ce_trap_tl1, %g1 sethi %hi(dis_err_panic1), %g4 jmp %g4 + %lo(dis_err_panic1) nop #endif SET_SIZE(ce_err_tl1) #ifdef TRAPTRACE .celevel1msg: .asciz "Softerror with trap tracing at tl1: AFAR 0x%08x.%08x AFSR 0x%08x.%08x"; ENTRY_NP(ce_trap_tl1) ! upper 32 bits of AFSR already in o3 mov %o4, %o0 ! save AFAR upper 32 bits mov %o2, %o4 ! lower 32 bits of AFSR mov %o1, %o2 ! lower 32 bits of AFAR mov %o0, %o1 ! upper 32 bits of AFAR set .celevel1msg, %o0 call panic nop SET_SIZE(ce_trap_tl1) #endif ! ! async_err is the assembly glue code to get us from the actual trap ! into the CPU module's C error handler. Note that we also branch ! here from ce_err() above. ! ENTRY_NP(async_err) stxa %g0, [%g0]ASI_ESTATE_ERR ! disable ecc and other cpu errors membar #Sync ! membar sync required ldxa [%g0]ASI_AFSR, %g3 ! save afsr in g3 ldxa [%g0]ASI_AFAR, %g2 ! save afar in g2 sllx %g5, 53, %g5 ! move ttype to <63:53> or %g3, %g5, %g3 ! or to afsr in g3 or %g0, 1, %g1 ! put 1 in g1 sllx %g1, 21, %g1 ! shift left to <21> afsr UE andcc %g1, %g3, %g0 ! check for UE in afsr bz,a,pn %icc, 2f ! if !UE skip sdb read/clear nop set P_DER_H, %g4 ! put P_DER_H in g4 ldxa [%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into 56 or %g0, 1, %g6 ! put 1 in g6 sllx %g6, 9, %g6 ! shift g6 to <9> sdb UE andcc %g5, %g6, %g1 ! check for UE in upper half sllx %g5, 33, %g5 ! shift upper bits to <42:33> or %g3, %g5, %g3 ! or with afsr bits bz,a 1f ! no error, goto 1f nop stxa %g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg UE error bit membar #Sync ! membar sync required 1: set P_DER_L, %g4 ! put P_DER_L in g4 ldxa [%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g5 andcc %g5, %g6, %g1 ! check for UE in lower half sllx %g5, 43, %g5 ! shift upper bits to <52:43> or %g3, %g5, %g3 ! or with afsr bits bz,a 2f ! no error, goto 2f nop stxa %g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg UE error bit membar #Sync ! membar sync required 2: stxa %g3, [%g0]ASI_AFSR ! clear all the sticky bits membar #Sync ! membar sync required RESET_USER_RTT_REGS(%g4, %g5, async_err_resetskip) async_err_resetskip: set cpu_async_error, %g1 ! put cpu_async_error in g1 sethi %hi(sys_trap), %g5 jmp %g5 + %lo(sys_trap) ! goto sys_trap or %g0, PIL_15, %g4 ! run at pil 15 SET_SIZE(async_err) ENTRY_NP(dis_err_panic1) stxa %g0, [%g0]ASI_ESTATE_ERR ! disable all error traps membar #Sync ! save destination routine is in g1 ldxa [%g0]ASI_AFAR, %g2 ! read afar ldxa [%g0]ASI_AFSR, %g3 ! read afsr set P_DER_H, %g4 ! put P_DER_H in g4 ldxa [%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into g5 sllx %g5, 33, %g5 ! shift upper bits to <42:33> or %g3, %g5, %g3 ! or with afsr bits set P_DER_L, %g4 ! put P_DER_L in g4 ldxa [%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g5 sllx %g5, 43, %g5 ! shift upper bits to <52:43> or %g3, %g5, %g3 ! or with afsr bits RESET_USER_RTT_REGS(%g4, %g5, dis_err_panic1_resetskip) dis_err_panic1_resetskip: sethi %hi(sys_trap), %g5 jmp %g5 + %lo(sys_trap) ! goto sys_trap sub %g0, 1, %g4 SET_SIZE(dis_err_panic1) ENTRY(clr_datapath) set P_DER_H, %o4 ! put P_DER_H in o4 ldxa [%o4]ASI_SDB_INTR_R, %o5 ! read sdb upper half into o3 or %g0, 0x3, %o2 ! put 0x3 in o2 sllx %o2, 8, %o2 ! shift o2 to <9:8> sdb andcc %o5, %o2, %o1 ! check for UE,CE in upper half bz,a 1f ! no error, goto 1f nop stxa %o1, [%o4]ASI_SDB_INTR_W ! clear sdb reg UE,CE error bits membar #Sync ! membar sync required 1: set P_DER_L, %o4 ! put P_DER_L in o4 ldxa [%o4]ASI_SDB_INTR_R, %o5 ! read sdb lower half into o5 andcc %o5, %o2, %o1 ! check for UE,CE in lower half bz,a 2f ! no error, goto 2f nop stxa %o1, [%o4]ASI_SDB_INTR_W ! clear sdb reg UE,CE error bits membar #Sync 2: retl nop SET_SIZE(clr_datapath) ENTRY(get_udb_errors) set P_DER_H, %o3 ldxa [%o3]ASI_SDB_INTR_R, %o2 stx %o2, [%o0] set P_DER_L, %o3 ldxa [%o3]ASI_SDB_INTR_R, %o2 retl stx %o2, [%o1] SET_SIZE(get_udb_errors) #endif /* lint */ #if defined(lint) /* * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the * tte, the virtual address, and the ctxnum of the specified tlb entry. They * should only be used in places where you have no choice but to look at the * tlb itself. * * Note: These two routines are required by the Estar "cpr" loadable module. */ /*ARGSUSED*/ void itlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag) {} /*ARGSUSED*/ void dtlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag) {} #else /* lint */ /* * NB - In Spitfire cpus, when reading a tte from the hardware, we * need to clear [42-41] because the general definitions in pte.h * define the PA to be [42-13] whereas Spitfire really uses [40-13]. * When cloning these routines for other cpus the "andn" below is not * necessary. */ ENTRY_NP(itlb_rd_entry) sllx %o0, 3, %o0 #if defined(SF_ERRATA_32) sethi %hi(FLUSH_ADDR), %g2 set MMU_PCONTEXT, %g1 stxa %g0, [%g1]ASI_DMMU ! KCONTEXT flush %g2 #endif ldxa [%o0]ASI_ITLB_ACCESS, %g1 set TTE_SPITFIRE_PFNHI_CLEAR, %g2 ! spitfire only sllx %g2, TTE_SPITFIRE_PFNHI_SHIFT, %g2 ! see comment above andn %g1, %g2, %g1 ! for details stx %g1, [%o1] ldxa [%o0]ASI_ITLB_TAGREAD, %g2 set TAGREAD_CTX_MASK, %o4 andn %g2, %o4, %o5 retl stx %o5, [%o2] SET_SIZE(itlb_rd_entry) ENTRY_NP(dtlb_rd_entry) sllx %o0, 3, %o0 #if defined(SF_ERRATA_32) sethi %hi(FLUSH_ADDR), %g2 set MMU_PCONTEXT, %g1 stxa %g0, [%g1]ASI_DMMU ! KCONTEXT flush %g2 #endif ldxa [%o0]ASI_DTLB_ACCESS, %g1 set TTE_SPITFIRE_PFNHI_CLEAR, %g2 ! spitfire only sllx %g2, TTE_SPITFIRE_PFNHI_SHIFT, %g2 ! see comment above andn %g1, %g2, %g1 ! itlb_rd_entry stx %g1, [%o1] ldxa [%o0]ASI_DTLB_TAGREAD, %g2 set TAGREAD_CTX_MASK, %o4 andn %g2, %o4, %o5 retl stx %o5, [%o2] SET_SIZE(dtlb_rd_entry) #endif /* lint */ #if defined(lint) /* * routines to get and set the LSU register */ uint64_t get_lsu(void) { return ((uint64_t)0); } /*ARGSUSED*/ void set_lsu(uint64_t lsu) {} #else /* lint */ ENTRY(set_lsu) stxa %o0, [%g0]ASI_LSU ! store to LSU retl membar #Sync SET_SIZE(set_lsu) ENTRY(get_lsu) retl ldxa [%g0]ASI_LSU, %o0 ! load LSU SET_SIZE(get_lsu) #endif /* lint */ #ifndef lint /* * Clear the NPT (non-privileged trap) bit in the %tick * registers. In an effort to make the change in the * tick counter as consistent as possible, we disable * all interrupts while we're changing the registers. We also * ensure that the read and write instructions are in the same * line in the instruction cache. */ ENTRY_NP(cpu_clearticknpt) rdpr %pstate, %g1 /* save processor state */ andn %g1, PSTATE_IE, %g3 /* turn off */ wrpr %g0, %g3, %pstate /* interrupts */ rdpr %tick, %g2 /* get tick register */ brgez,pn %g2, 1f /* if NPT bit off, we're done */ mov 1, %g3 /* create mask */ sllx %g3, 63, %g3 /* for NPT bit */ ba,a,pt %xcc, 2f .align 64 /* Align to I$ boundary */ 2: rdpr %tick, %g2 /* get tick register */ wrpr %g3, %g2, %tick /* write tick register, */ /* clearing NPT bit */ #if defined(BB_ERRATA_1) rdpr %tick, %g0 /* read (s)tick (BB_ERRATA_1) */ #endif 1: jmp %g4 + 4 wrpr %g0, %g1, %pstate /* restore processor state */ SET_SIZE(cpu_clearticknpt) /* * get_ecache_tag() * Register Usage: * %o0: In: 32-bit E$ index * Out: 64-bit E$ tag value * %o1: In: 64-bit AFSR value after clearing sticky bits * %o2: In: address of cpu private afsr storage */ ENTRY(get_ecache_tag) or %g0, 1, %o4 sllx %o4, 40, %o4 ! set bit 40 for e$ tag access or %o0, %o4, %o4 ! %o4 = e$ addr for tag read rdpr %pstate, %o5 andn %o5, PSTATE_IE | PSTATE_AM, %o0 wrpr %o0, %g0, %pstate ! clear IE, AM bits ldxa [%g0]ASI_ESTATE_ERR, %g1 stxa %g0, [%g0]ASI_ESTATE_ERR ! Turn off Error enable membar #Sync ldxa [%g0]ASI_AFSR, %o0 srlx %o0, P_AFSR_CP_SHIFT, %o3 btst 1, %o3 bz 1f nop ldx [%o2], %g4 or %g4, %o0, %g4 ! aggregate AFSR in cpu private stx %g4, [%o2] 1: stxa %o0, [%g0]ASI_AFSR ! clear AFSR membar #Sync ldxa [%o4]ASI_EC_R, %g0 ldxa [%g0]ASI_EC_DIAG, %o0 ! read tag from e$ tag reg ldxa [%g0]ASI_AFSR, %o3 srlx %o3, P_AFSR_CP_SHIFT, %o4 btst 1, %o4 bz 2f stx %o3, [%o1] ! AFSR after sticky clear ldx [%o2], %g4 or %g4, %o3, %g4 ! aggregate AFSR in cpu private stx %g4, [%o2] 2: membar #Sync stxa %g1, [%g0]ASI_ESTATE_ERR ! Turn error enable back on membar #Sync retl wrpr %g0, %o5, %pstate SET_SIZE(get_ecache_tag) /* * check_ecache_line() * Register Usage: * %o0: In: 32-bit E$ index * Out: 64-bit accumulated AFSR * %o1: In: address of cpu private afsr storage */ ENTRY(check_ecache_line) or %g0, 1, %o4 sllx %o4, 39, %o4 ! set bit 39 for e$ data access or %o0, %o4, %o4 ! %o4 = e$ addr for data read rdpr %pstate, %o5 andn %o5, PSTATE_IE | PSTATE_AM, %o0 wrpr %o0, %g0, %pstate ! clear IE, AM bits ldxa [%g0]ASI_ESTATE_ERR, %g1 stxa %g0, [%g0]ASI_ESTATE_ERR ! Turn off Error enable membar #Sync ldxa [%g0]ASI_AFSR, %o0 srlx %o0, P_AFSR_CP_SHIFT, %o2 btst 1, %o2 bz 1f clr %o2 ! loop count ldx [%o1], %o3 or %o3, %o0, %o3 ! aggregate AFSR in cpu private stx %o3, [%o1] 1: stxa %o0, [%g0]ASI_AFSR ! clear AFSR membar #Sync 2: ldxa [%o4]ASI_EC_R, %g0 ! Read the E$ data 8bytes each add %o2, 1, %o2 cmp %o2, 8 bl,a 2b add %o4, 8, %o4 membar #Sync ldxa [%g0]ASI_AFSR, %o0 ! read accumulated AFSR srlx %o0, P_AFSR_CP_SHIFT, %o2 btst 1, %o2 bz 3f nop ldx [%o1], %o3 or %o3, %o0, %o3 ! aggregate AFSR in cpu private stx %o3, [%o1] 3: stxa %o0, [%g0]ASI_AFSR ! clear AFSR membar #Sync stxa %g1, [%g0]ASI_ESTATE_ERR ! Turn error enable back on membar #Sync retl wrpr %g0, %o5, %pstate SET_SIZE(check_ecache_line) #endif /* lint */ #if defined(lint) uint64_t read_and_clear_afsr() { return ((uint64_t)0); } #else /* lint */ ENTRY(read_and_clear_afsr) ldxa [%g0]ASI_AFSR, %o0 retl stxa %o0, [%g0]ASI_AFSR ! clear AFSR SET_SIZE(read_and_clear_afsr) #endif /* lint */ #if defined(lint) /* ARGSUSED */ void scrubphys(uint64_t paddr, int ecache_size) { } #else /* lint */ /* * scrubphys - Pass in the aligned physical memory address that you want * to scrub, along with the ecache size. * * 1) Displacement flush the E$ line corresponding to %addr. * The first ldxa guarantees that the %addr is no longer in * M, O, or E (goes to I or S (if instruction fetch also happens). * 2) "Write" the data using a CAS %addr,%g0,%g0. * The casxa guarantees a transition from I to M or S to M. * 3) Displacement flush the E$ line corresponding to %addr. * The second ldxa pushes the M line out of the ecache, into the * writeback buffers, on the way to memory. * 4) The "membar #Sync" pushes the cache line out of the writeback * buffers onto the bus, on the way to dram finally. * * This is a modified version of the algorithm suggested by Gary Lauterbach. * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line * as modified, but then we found out that for spitfire, if it misses in the * E$ it will probably install as an M, but if it hits in the E$, then it * will stay E, if the store doesn't happen. So the first displacement flush * should ensure that the CAS will miss in the E$. Arrgh. */ ENTRY(scrubphys) or %o1, %g0, %o2 ! put ecache size in %o2 #ifndef HUMMINGBIRD xor %o0, %o2, %o1 ! calculate alias address add %o2, %o2, %o3 ! 2 * ecachesize in case ! addr == ecache_flushaddr sub %o3, 1, %o3 ! -1 == mask and %o1, %o3, %o1 ! and with xor'd address set ecache_flushaddr, %o3 ldx [%o3], %o3 rdpr %pstate, %o4 andn %o4, PSTATE_IE | PSTATE_AM, %o5 wrpr %o5, %g0, %pstate ! clear IE, AM bits ldxa [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias casxa [%o0]ASI_MEM, %g0, %g0 ldxa [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias #else /* HUMMINGBIRD */ /* * UltraSPARC-IIe processor supports both 4-way set associative * and direct map E$. We need to reconfigure E$ to direct map * mode for data load/store before displacement flush. Also, we * need to flush all 4 sets of the E$ to ensure that the physaddr * has been flushed. Keep the interrupts disabled while flushing * E$ in this manner. * * For flushing a specific physical address, we start at the * aliased address and load at set-size stride, wrapping around * at 2*ecache-size boundary and skipping fault physical address. * It takes 10 loads to guarantee that the physical address has * been flushed. * * Usage: * %o0 physaddr * %o5 physaddr - ecache_flushaddr * %g1 UPA config (restored later) * %g2 E$ set size * %g3 E$ flush address range mask (i.e. 2 * E$ -1) * %g4 #loads to flush phys address * %g5 temp */ sethi %hi(ecache_associativity), %g5 ld [%g5 + %lo(ecache_associativity)], %g5 udivx %o2, %g5, %g2 ! set size (i.e. ecache_size/#sets) xor %o0, %o2, %o1 ! calculate alias address add %o2, %o2, %g3 ! 2 * ecachesize in case ! addr == ecache_flushaddr sub %g3, 1, %g3 ! 2 * ecachesize -1 == mask and %o1, %g3, %o1 ! and with xor'd address sethi %hi(ecache_flushaddr), %o3 ldx [%o3 + %lo(ecache_flushaddr)], %o3 rdpr %pstate, %o4 andn %o4, PSTATE_IE | PSTATE_AM, %o5 wrpr %o5, %g0, %pstate ! clear IE, AM bits ! Place E$ in direct map mode for data access or %g0, 1, %g5 sllx %g5, HB_UPA_DMAP_DATA_BIT, %g5 ldxa [%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later) or %g1, %g5, %g5 membar #Sync stxa %g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access membar #Sync ! Displace cache line from each set of E$ starting at the ! aliased address. at set-size stride, wrapping at 2*ecache_size ! and skipping load from physaddr. We need 10 loads to flush the ! physaddr from E$. mov HB_PHYS_FLUSH_CNT-1, %g4 ! #loads to flush phys addr sub %o0, %o3, %o5 ! physaddr - ecache_flushaddr or %o1, %g0, %g5 ! starting aliased offset 2: ldxa [%g5 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias 1: add %g5, %g2, %g5 ! calculate offset in next set and %g5, %g3, %g5 ! force offset within aliased range cmp %g5, %o5 ! skip loads from physaddr be,pn %ncc, 1b nop brgz,pt %g4, 2b dec %g4 casxa [%o0]ASI_MEM, %g0, %g0 ! Flush %o0 from ecahe again. ! Need single displacement flush at offset %o1 this time as ! the E$ is already in direct map mode. ldxa [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias membar #Sync stxa %g1, [%g0]ASI_UPA_CONFIG ! restore UPA config (DM bits) membar #Sync #endif /* HUMMINGBIRD */ wrpr %g0, %o4, %pstate ! restore earlier pstate register value retl membar #Sync ! move the data out of the load buffer SET_SIZE(scrubphys) #endif /* lint */ #if defined(lint) /* * clearphys - Pass in the aligned physical memory address that you want * to push out, as a 64 byte block of zeros, from the ecache zero-filled. * Since this routine does not bypass the ecache, it is possible that * it could generate a UE error while trying to clear the a bad line. * This routine clears and restores the error enable flag. * TBD - Hummingbird may need similar protection */ /* ARGSUSED */ void clearphys(uint64_t paddr, int ecache_size, int ecache_linesize) { } #else /* lint */ ENTRY(clearphys) or %o2, %g0, %o3 ! ecache linesize or %o1, %g0, %o2 ! ecache size #ifndef HUMMINGBIRD or %o3, %g0, %o4 ! save ecache linesize xor %o0, %o2, %o1 ! calculate alias address add %o2, %o2, %o3 ! 2 * ecachesize sub %o3, 1, %o3 ! -1 == mask and %o1, %o3, %o1 ! and with xor'd address set ecache_flushaddr, %o3 ldx [%o3], %o3 or %o4, %g0, %o2 ! saved ecache linesize rdpr %pstate, %o4 andn %o4, PSTATE_IE | PSTATE_AM, %o5 wrpr %o5, %g0, %pstate ! clear IE, AM bits ldxa [%g0]ASI_ESTATE_ERR, %g1 stxa %g0, [%g0]ASI_ESTATE_ERR ! disable errors membar #Sync ! need to put zeros in the cache line before displacing it sub %o2, 8, %o2 ! get offset of last double word in ecache line 1: stxa %g0, [%o0 + %o2]ASI_MEM ! put zeros in the ecache line sub %o2, 8, %o2 brgez,a,pt %o2, 1b nop ldxa [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias casxa [%o0]ASI_MEM, %g0, %g0 ldxa [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias stxa %g1, [%g0]ASI_ESTATE_ERR ! restore error enable membar #Sync #else /* HUMMINGBIRD... */ /* * UltraSPARC-IIe processor supports both 4-way set associative * and direct map E$. We need to reconfigure E$ to direct map * mode for data load/store before displacement flush. Also, we * need to flush all 4 sets of the E$ to ensure that the physaddr * has been flushed. Keep the interrupts disabled while flushing * E$ in this manner. * * For flushing a specific physical address, we start at the * aliased address and load at set-size stride, wrapping around * at 2*ecache-size boundary and skipping fault physical address. * It takes 10 loads to guarantee that the physical address has * been flushed. * * Usage: * %o0 physaddr * %o5 physaddr - ecache_flushaddr * %g1 UPA config (restored later) * %g2 E$ set size * %g3 E$ flush address range mask (i.e. 2 * E$ -1) * %g4 #loads to flush phys address * %g5 temp */ or %o3, %g0, %o4 ! save ecache linesize sethi %hi(ecache_associativity), %g5 ld [%g5 + %lo(ecache_associativity)], %g5 udivx %o2, %g5, %g2 ! set size (i.e. ecache_size/#sets) xor %o0, %o2, %o1 ! calculate alias address add %o2, %o2, %g3 ! 2 * ecachesize sub %g3, 1, %g3 ! 2 * ecachesize -1 == mask and %o1, %g3, %o1 ! and with xor'd address sethi %hi(ecache_flushaddr), %o3 ldx [%o3 +%lo(ecache_flushaddr)], %o3 or %o4, %g0, %o2 ! saved ecache linesize rdpr %pstate, %o4 andn %o4, PSTATE_IE | PSTATE_AM, %o5 wrpr %o5, %g0, %pstate ! clear IE, AM bits ! Place E$ in direct map mode for data access or %g0, 1, %g5 sllx %g5, HB_UPA_DMAP_DATA_BIT, %g5 ldxa [%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later) or %g1, %g5, %g5 membar #Sync stxa %g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access membar #Sync ! need to put zeros in the cache line before displacing it sub %o2, 8, %o2 ! get offset of last double word in ecache line 1: stxa %g0, [%o0 + %o2]ASI_MEM ! put zeros in the ecache line sub %o2, 8, %o2 brgez,a,pt %o2, 1b nop ! Displace cache line from each set of E$ starting at the ! aliased address. at set-size stride, wrapping at 2*ecache_size ! and skipping load from physaddr. We need 10 loads to flush the ! physaddr from E$. mov HB_PHYS_FLUSH_CNT-1, %g4 ! #loads to flush phys addr sub %o0, %o3, %o5 ! physaddr - ecache_flushaddr or %o1, %g0, %g5 ! starting offset 2: ldxa [%g5 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias 3: add %g5, %g2, %g5 ! calculate offset in next set and %g5, %g3, %g5 ! force offset within aliased range cmp %g5, %o5 ! skip loads from physaddr be,pn %ncc, 3b nop brgz,pt %g4, 2b dec %g4 casxa [%o0]ASI_MEM, %g0, %g0 ! Flush %o0 from ecahe again. ! Need single displacement flush at offset %o1 this time as ! the E$ is already in direct map mode. ldxa [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias membar #Sync stxa %g1, [%g0]ASI_UPA_CONFIG ! restore UPA config (DM bits) membar #Sync #endif /* HUMMINGBIRD... */ retl wrpr %g0, %o4, %pstate ! restore earlier pstate register value SET_SIZE(clearphys) #endif /* lint */ #if defined(lint) /* ARGSUSED */ void flushecacheline(uint64_t paddr, int ecache_size) { } #else /* lint */ /* * flushecacheline - This is a simpler version of scrubphys * which simply does a displacement flush of the line in * question. This routine is mainly used in handling async * errors where we want to get rid of a bad line in ecache. * Note that if the line is modified and it has suffered * data corruption - we are guarantee that the hw will write * a UE back to mark the page poisoned. */ ENTRY(flushecacheline) or %o1, %g0, %o2 ! put ecache size in %o2 #ifndef HUMMINGBIRD xor %o0, %o2, %o1 ! calculate alias address add %o2, %o2, %o3 ! 2 * ecachesize in case ! addr == ecache_flushaddr sub %o3, 1, %o3 ! -1 == mask and %o1, %o3, %o1 ! and with xor'd address set ecache_flushaddr, %o3 ldx [%o3], %o3 rdpr %pstate, %o4 andn %o4, PSTATE_IE | PSTATE_AM, %o5 wrpr %o5, %g0, %pstate ! clear IE, AM bits ldxa [%g0]ASI_ESTATE_ERR, %g1 stxa %g0, [%g0]ASI_ESTATE_ERR ! disable errors membar #Sync ldxa [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias membar #Sync stxa %g1, [%g0]ASI_ESTATE_ERR ! restore error enable membar #Sync #else /* HUMMINGBIRD */ /* * UltraSPARC-IIe processor supports both 4-way set associative * and direct map E$. We need to reconfigure E$ to direct map * mode for data load/store before displacement flush. Also, we * need to flush all 4 sets of the E$ to ensure that the physaddr * has been flushed. Keep the interrupts disabled while flushing * E$ in this manner. * * For flushing a specific physical address, we start at the * aliased address and load at set-size stride, wrapping around * at 2*ecache-size boundary and skipping fault physical address. * It takes 10 loads to guarantee that the physical address has * been flushed. * * Usage: * %o0 physaddr * %o5 physaddr - ecache_flushaddr * %g1 error enable register * %g2 E$ set size * %g3 E$ flush address range mask (i.e. 2 * E$ -1) * %g4 UPA config (restored later) * %g5 temp */ sethi %hi(ecache_associativity), %g5 ld [%g5 + %lo(ecache_associativity)], %g5 udivx %o2, %g5, %g2 ! set size (i.e. ecache_size/#sets) xor %o0, %o2, %o1 ! calculate alias address add %o2, %o2, %g3 ! 2 * ecachesize in case ! addr == ecache_flushaddr sub %g3, 1, %g3 ! 2 * ecachesize -1 == mask and %o1, %g3, %o1 ! and with xor'd address sethi %hi(ecache_flushaddr), %o3 ldx [%o3 + %lo(ecache_flushaddr)], %o3 rdpr %pstate, %o4 andn %o4, PSTATE_IE | PSTATE_AM, %o5 wrpr %o5, %g0, %pstate ! clear IE, AM bits ! Place E$ in direct map mode for data access or %g0, 1, %g5 sllx %g5, HB_UPA_DMAP_DATA_BIT, %g5 ldxa [%g0]ASI_UPA_CONFIG, %g4 ! current UPA config (restored later) or %g4, %g5, %g5 membar #Sync stxa %g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access membar #Sync ldxa [%g0]ASI_ESTATE_ERR, %g1 stxa %g0, [%g0]ASI_ESTATE_ERR ! disable errors membar #Sync ! Displace cache line from each set of E$ starting at the ! aliased address. at set-size stride, wrapping at 2*ecache_size ! and skipping load from physaddr. We need 10 loads to flush the ! physaddr from E$. mov HB_PHYS_FLUSH_CNT-1, %g5 ! #loads to flush physaddr sub %o0, %o3, %o5 ! physaddr - ecache_flushaddr 2: ldxa [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias 3: add %o1, %g2, %o1 ! calculate offset in next set and %o1, %g3, %o1 ! force offset within aliased range cmp %o1, %o5 ! skip loads from physaddr be,pn %ncc, 3b nop brgz,pt %g5, 2b dec %g5 membar #Sync stxa %g1, [%g0]ASI_ESTATE_ERR ! restore error enable membar #Sync stxa %g4, [%g0]ASI_UPA_CONFIG ! restore UPA config (DM bits) membar #Sync #endif /* HUMMINGBIRD */ retl wrpr %g0, %o4, %pstate SET_SIZE(flushecacheline) #endif /* lint */ #if defined(lint) /* ARGSUSED */ void ecache_scrubreq_tl1(uint64_t inum, uint64_t dummy) { } #else /* lint */ /* * ecache_scrubreq_tl1 is the crosstrap handler called at ecache_calls_a_sec Hz * from the clock CPU. It atomically increments the outstanding request * counter and, if there was not already an outstanding request, * branches to setsoftint_tl1 to enqueue an intr_vec for the given inum. */ ! Register usage: ! ! Arguments: ! %g1 - inum ! ! Internal: ! %g2, %g3, %g5 - scratch ! %g4 - ptr. to spitfire_scrub_misc ec_scrub_outstanding. ! %g6 - setsoftint_tl1 address ENTRY_NP(ecache_scrubreq_tl1) set SFPR_SCRUB_MISC + EC_SCRUB_OUTSTANDING, %g2 GET_CPU_PRIVATE_PTR(%g2, %g4, %g5, 1f); ld [%g4], %g2 ! cpu's ec_scrub_outstanding. set setsoftint_tl1, %g6 ! ! no need to use atomic instructions for the following ! increment - we're at tl1 ! add %g2, 0x1, %g3 brnz,pn %g2, 1f ! no need to enqueue more intr_vec st %g3, [%g4] ! delay - store incremented counter jmp %g6 ! setsoftint_tl1(%g1) - queue intr_vec nop ! not reached 1: retry SET_SIZE(ecache_scrubreq_tl1) #endif /* lint */ #if defined(lint) /*ARGSUSED*/ void write_ec_tag_parity(uint32_t id) {} #else /* lint */ /* * write_ec_tag_parity(), which zero's the ecache tag, * marks the state as invalid and writes good parity to the tag. * Input %o1= 32 bit E$ index */ ENTRY(write_ec_tag_parity) or %g0, 1, %o4 sllx %o4, 39, %o4 ! set bit 40 for e$ tag access or %o0, %o4, %o4 ! %o4 = ecache addr for tag write rdpr %pstate, %o5 andn %o5, PSTATE_IE | PSTATE_AM, %o1 wrpr %o1, %g0, %pstate ! clear IE, AM bits ldxa [%g0]ASI_ESTATE_ERR, %g1 stxa %g0, [%g0]ASI_ESTATE_ERR ! Turn off Error enable membar #Sync ba 1f nop /* * Align on the ecache boundary in order to force * ciritical code section onto the same ecache line. */ .align 64 1: set S_EC_PARITY, %o3 ! clear tag, state invalid sllx %o3, S_ECPAR_SHIFT, %o3 ! and with good tag parity stxa %o3, [%g0]ASI_EC_DIAG ! update with the above info stxa %g0, [%o4]ASI_EC_W membar #Sync stxa %g1, [%g0]ASI_ESTATE_ERR ! Turn error enable back on membar #Sync retl wrpr %g0, %o5, %pstate SET_SIZE(write_ec_tag_parity) #endif /* lint */ #if defined(lint) /*ARGSUSED*/ void write_hb_ec_tag_parity(uint32_t id) {} #else /* lint */ /* * write_hb_ec_tag_parity(), which zero's the ecache tag, * marks the state as invalid and writes good parity to the tag. * Input %o1= 32 bit E$ index */ ENTRY(write_hb_ec_tag_parity) or %g0, 1, %o4 sllx %o4, 39, %o4 ! set bit 40 for e$ tag access or %o0, %o4, %o4 ! %o4 = ecache addr for tag write rdpr %pstate, %o5 andn %o5, PSTATE_IE | PSTATE_AM, %o1 wrpr %o1, %g0, %pstate ! clear IE, AM bits ldxa [%g0]ASI_ESTATE_ERR, %g1 stxa %g0, [%g0]ASI_ESTATE_ERR ! Turn off Error enable membar #Sync ba 1f nop /* * Align on the ecache boundary in order to force * ciritical code section onto the same ecache line. */ .align 64 1: #ifdef HUMMINGBIRD set HB_EC_PARITY, %o3 ! clear tag, state invalid sllx %o3, HB_ECPAR_SHIFT, %o3 ! and with good tag parity #else /* !HUMMINGBIRD */ set SB_EC_PARITY, %o3 ! clear tag, state invalid sllx %o3, SB_ECPAR_SHIFT, %o3 ! and with good tag parity #endif /* !HUMMINGBIRD */ stxa %o3, [%g0]ASI_EC_DIAG ! update with the above info stxa %g0, [%o4]ASI_EC_W membar #Sync stxa %g1, [%g0]ASI_ESTATE_ERR ! Turn error enable back on membar #Sync retl wrpr %g0, %o5, %pstate SET_SIZE(write_hb_ec_tag_parity) #endif /* lint */ #define VIS_BLOCKSIZE 64 #if defined(lint) /*ARGSUSED*/ int dtrace_blksuword32(uintptr_t addr, uint32_t *data, int tryagain) { return (0); } #else ENTRY(dtrace_blksuword32) save %sp, -SA(MINFRAME + 4), %sp rdpr %pstate, %l1 andn %l1, PSTATE_IE, %l2 ! disable interrupts to wrpr %g0, %l2, %pstate ! protect our FPU diddling rd %fprs, %l0 andcc %l0, FPRS_FEF, %g0 bz,a,pt %xcc, 1f ! if the fpu is disabled wr %g0, FPRS_FEF, %fprs ! ... enable the fpu st %f0, [%fp + STACK_BIAS - 4] ! save %f0 to the stack 1: set 0f, %l5 /* * We're about to write a block full or either total garbage * (not kernel data, don't worry) or user floating-point data * (so it only _looks_ like garbage). */ ld [%i1], %f0 ! modify the block membar #Sync stn %l5, [THREAD_REG + T_LOFAULT] ! set up the lofault handler stda %d0, [%i0]ASI_BLK_COMMIT_S ! store the modified block membar #Sync stn %g0, [THREAD_REG + T_LOFAULT] ! remove the lofault handler bz,a,pt %xcc, 1f wr %g0, %l0, %fprs ! restore %fprs ld [%fp + STACK_BIAS - 4], %f0 ! restore %f0 1: wrpr %g0, %l1, %pstate ! restore interrupts ret restore %g0, %g0, %o0 0: membar #Sync stn %g0, [THREAD_REG + T_LOFAULT] ! remove the lofault handler bz,a,pt %xcc, 1f wr %g0, %l0, %fprs ! restore %fprs ld [%fp + STACK_BIAS - 4], %f0 ! restore %f0 1: wrpr %g0, %l1, %pstate ! restore interrupts /* * If tryagain is set (%i2) we tail-call dtrace_blksuword32_err() * which deals with watchpoints. Otherwise, just return -1. */ brnz,pt %i2, 1f nop ret restore %g0, -1, %o0 1: call dtrace_blksuword32_err restore SET_SIZE(dtrace_blksuword32) #endif /* lint */