xref: /titanic_44/usr/src/uts/sun4u/cpu/us3_jalapeno_asm.s (revision 750ba2244c23d03e3f43b4967911885b36bbac55)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Assembly code support for the jalapeno module
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#if !defined(lint)
#include "assym.h"
#endif	/* lint */

#include <sys/asm_linkage.h>
#include <sys/mmu.h>
#include <vm/hat_sfmmu.h>
#include <sys/machparam.h>
#include <sys/machcpuvar.h>
#include <sys/machthread.h>
#include <sys/machtrap.h>
#include <sys/privregs.h>
#include <sys/asm_linkage.h>
#include <sys/trap.h>
#include <sys/cheetahregs.h>
#include <sys/us3_module.h>
#include <sys/xc_impl.h>
#include <sys/intreg.h>
#include <sys/async.h>
#include <sys/clock.h>
#include <sys/cheetahasm.h>

#ifdef TRAPTRACE
#include <sys/traptrace.h>
#endif /* TRAPTRACE */

#if !defined(lint)

/* BEGIN CSTYLED */

#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)

#define	CHK_JP_ERRATA85_ENABLED(scr, label)				\
	ASM_LD(scr, jp_errata_85_active);				\
	cmp	scr, 1;							\
	bne	%icc, label;						\
	nop

#define	SET_64BIT_PA(dest, scr, hi32, lo32)				\
	set	hi32, scr;						\
	sllx	scr, 32, scr;						\
	sethi	%hi(lo32), dest;					\
	or	dest, %lo(lo32), dest;					\
	or	scr, dest, dest

/*
 * Macro to trigger Jalapeno/Tomatillo speed change
 *   j_chng_pa - scratch register
 *   scr - scratch register
 */
#define	JP_ESTAR_TRIGGER(j_chng_pa, scr)				\
	SET_64BIT_PA(j_chng_pa, scr, TOM_HIGH_PA, M_T_J_CHNG_INIT_PA);	\
	ldxa	[j_chng_pa]ASI_IO, scr;					\
5:									\
	and	scr, TOM_TRIGGER_MASK, scr;				\
	cmp	scr, TOM_TRIGGER;					\
	be,pt %icc, 5b;			/* wait while 10 */		\
	ldxa	[j_chng_pa]ASI_IO, scr;					\
	andn	scr, TOM_TRIGGER_MASK, scr;				\
	stxa	scr, [j_chng_pa]ASI_IO;	/* clear j_chng[1:0] */		\
	or	scr, TOM_TRIGGER, scr;					\
	stxa	scr, [j_chng_pa]ASI_IO;	/* trigger j_chng */		\
	ldxa	[j_chng_pa]ASI_IO, scr;					\
6:									\
	and	scr, TOM_TRIGGER_MASK, scr;				\
	cmp	scr, TOM_TRIGGER;					\
	be,pt %icc, 6b;			/* wait while 10 */		\
	ldxa	[j_chng_pa]ASI_IO, scr;					\
	andn	scr, TOM_TRIGGER_MASK, scr;				\
	stxa	scr, [j_chng_pa]ASI_IO;	/* deassert j_chng */

/*
 * Macro to set Jalapeno CPU speed
 *   speed - new speed constant
 *   scr1  - scratch register
 *   scr2  - scratch register
 */
#define	SET_JP_SPEED(speed, scr1, scr2)					\
	ldxa	[%g0]ASI_JBUS_CONFIG, scr1;				\
	set	JBUS_CONFIG_ECLK_MASK, scr2;				\
	andn	scr1, scr2, scr1;					\
	set	speed, scr2;						\
	or	scr1, scr2, scr1;					\
	stxa	scr1, [%g0]ASI_JBUS_CONFIG;

/*
 * macro to set Master Tomatillo speed
 *   speed - tomatillo speed constant
 *   tpa   - tomatillo estar control register PA
 *   scr  - scratch register
 */
#define	SET_TOM_SPEED(speed, tpa, scr)					\
	ldxa	[tpa]ASI_IO, scr;					\
	andn	scr, TOM_ESTAR_ELCK_MASK, scr;				\
	or	scr, speed, scr;					\
	stxa	scr, [tpa]ASI_IO;

/*
 * macro to check and set Slave Tomatillo speed
 *   speed - tomatillo speed constant
 *   scr1   - scratch register
 *   scr2   - scratch register
 */

#define	SET_SLAVE_T_SPEED(speed, scr1, scr2)				\
	ldxa	[%g0]ASI_JBUS_CONFIG, scr2;				\
	srlx	scr2, JBUS_SLAVE_T_PORT_BIT, scr2;			\
	btst	1, scr2;						\
	bz,pt	%icc, 4f;						\
	nop;								\
	SET_64BIT_PA(scr1, scr2, TOM_HIGH_PA, S_T_ESTAR_CTRL_PA);	\
	SET_TOM_SPEED(speed, scr1, scr2);				\
4:


/*
 * macro to adjust ASI_MCU_CTL_REG1[26:25] fsm bits according to
 * new cpu speed: fsm[1:0]=11b for full speed, fsm[1:0]=0 for estar speed
 *    value - fsm bit value constant
 *    scr1  - scratch register
 *    scr2  - scratch register
 */
#define	JP_ADJUST_FSM(value, scr1, scr2)				\
	ldxa	[%g0]ASI_MCU_CTRL, scr1;				\
	set	JP_MCU_FSM_MASK, scr2;					\
	andn	scr1, scr2, scr1;					\
	set	value, scr2;						\
	or	scr1, scr2, scr1;					\
	stxa	scr1, [%g0]ASI_MCU_CTRL;				\
	membar	#Sync;

/*
 * JP_FORCE_FULL_SPEED and its fellow macros are for Jalapeno
 * workstation to work around Errata 85. The front portion of
 * it packs JP speed(14..13) and Tomatillo speed(5..0) into one
 * register.
 *
 * Current code assumes that these two fields are non-overlapping.
 * If that assumption changes, then this code won't work. If so, we
 * force a compile time error by not defining the JP_FORCE_FULL_SPEED
 * and JP_RESTORE_SPEED macros below.
 */

#if !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK)

/*
 * Macro to force Jalapeno/Tomatillo to full speed
 *   old_lvl - register used to save original cpu, tomatillo speed
 *   scr2 - scratch register
 *   scr3 - scratch register
 *   scr4 - scratch register
 */
#define	JP_FORCE_FULL_SPEED(old_lvl, scr2, scr3, scr4)			\
	ldxa	[%g0]ASI_JBUS_CONFIG, old_lvl;				\
	set	JBUS_CONFIG_ECLK_MASK, scr4;				\
	and	old_lvl, scr4, old_lvl;					\
	SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA);	\
	ldxa	[scr2]ASI_IO, scr3;					\
	set	TOM_ESTAR_ELCK_MASK, scr4;				\
	and	scr3, scr4, scr3;					\
	or	old_lvl, scr3, old_lvl;					\
	/* original jp and tomatillo speed saved in old_lvl */		\
									\
	/* either intended or currently at full speed */		\
	set	JBUS_CONFIG_ECLK_MASK, scr4;				\
	andcc	old_lvl, scr4, %g0;					\
	bz,pt	%icc, 8f;						\
	nop;								\
	/* go through 1/2 speed. */					\
	SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4);			\
	SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3);			\
	SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4);			\
	JP_ADJUST_FSM(0, scr3, scr4);					\
	set	jp_estar_tl0_data, scr3;				\
	ldx	[scr3], %g0;						\
	membar	#Sync;		/* or busy wait 1us */			\
	JP_ESTAR_TRIGGER(scr3, scr4);					\
8:									\
	/* bring to 1:1 speed */					\
	SET_JP_SPEED(JBUS_CONFIG_ECLK_1, scr3, scr4);			\
	SET_TOM_SPEED(TOM_FULL_SPEED, scr2, scr3);			\
	SET_SLAVE_T_SPEED(TOM_FULL_SPEED, scr3, scr4);			\
	JP_ADJUST_FSM(JP_MCU_FSM_MASK, scr3, scr4);			\
	JP_ESTAR_TRIGGER(scr3, scr4)


/*
 * Macro to restore Jalapeno/Tomatillo to original speed
 *     old_lvl - register contains saved original cpu, tomatillo speed
 *     scr2 - scratch register
 *     scr3 - scratch register
 *     scr4 - scratch register
 *
 * If trap had occured in the middle of ppm cpu speed transtion, then
 * old_lvl[31:10] contains the intended new speed written into jbus_config.
 * if old_lvl[9:0] is inconsistent with old_lvl[31:10], then the trap surely
 * interrupted the ppm cpu speed transition, otherwise nothing for sure.
 * We'll restore the intended/then-current speed, that should cause no
 * trouble to subsequent ppm cpu speed change code.
 */
#define	JP_RESTORE_SPEED(old_lvl, scr2, scr3, scr4)			\
	srlx	old_lvl, JBUS_CONFIG_ECLK_SHIFT, scr2;			\
	and	scr2, 3, scr2;						\
	add	scr2, 1, scr2;						\
	cmp	scr2, 3;						\
	bne,pt	%icc, 7f;						\
	  nop;								\
	set	TOM_SLOW_SPEED, scr2;					\
	/* scr2 contains tom speed according to intended jp speed */	\
7:									\
	andn	old_lvl, TOM_ESTAR_ELCK_MASK, old_lvl;			\
	or	scr2, old_lvl, old_lvl;					\
	/* updated old_lvl to contain intended jp and tom speed */	\
	andcc	old_lvl, TOM_FULL_SPEED, %g0;				\
	bnz,pt	%icc, 9f;	/* intended full, already at full */	\
	nop;								\
									\
	/* go to half speed */						\
	SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4);			\
	SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA);	\
	SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3);			\
	SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4);			\
	JP_ADJUST_FSM(0, scr3, scr4);					\
	set	jp_estar_tl0_data, scr3;				\
	ldx	[scr3], %g0;						\
	membar	#Sync;							\
	JP_ESTAR_TRIGGER(scr3, scr4);					\
	andcc	old_lvl, TOM_SLOW_SPEED, %g0;				\
	bz,pt	%icc, 9f;	/* intended 1:2, already at 1:2 */	\
	  nop;								\
									\
	/* go to 1:32 speed */						\
	SET_JP_SPEED(JBUS_CONFIG_ECLK_32, scr3, scr4);			\
	SET_TOM_SPEED(TOM_SLOW_SPEED, scr2, scr3);			\
	SET_SLAVE_T_SPEED(TOM_SLOW_SPEED, scr3, scr4);			\
	JP_ESTAR_TRIGGER(scr3, scr4);					\
9:

#endif /* !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK) */
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

/*
 * Jalapeno version to reflush an Ecache line by index.
 * Will flush all 4 ways (with only one scratch register).
 * Note that the code will be faster if we use 2 scratch registers.
 */
#define	ECACHE_REFLUSH_LINE(ec_set_size, index, scr1)			\
	JP_EC_DIAG_ACCESS_MEMBAR;					\
	ldxa	[index]ASI_EC_DIAG, %g0;				\
	JP_EC_DIAG_ACCESS_MEMBAR;					\
	mov	1, scr1;						\
	sllx	scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1;			\
	add	scr1, index, scr1;					\
	JP_EC_DIAG_ACCESS_MEMBAR;					\
	ldxa	[scr1]ASI_EC_DIAG, %g0;					\
	JP_EC_DIAG_ACCESS_MEMBAR;					\
	mov	2, scr1;						\
	sllx	scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1;			\
	add	scr1, index, scr1;					\
	JP_EC_DIAG_ACCESS_MEMBAR;					\
	ldxa	[scr1]ASI_EC_DIAG, %g0;					\
	JP_EC_DIAG_ACCESS_MEMBAR;					\
	mov	3, scr1;						\
	sllx	scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1;			\
	add	scr1, index, scr1;					\
	JP_EC_DIAG_ACCESS_MEMBAR;					\
	ldxa	[scr1]ASI_EC_DIAG, %g0;					\
	JP_EC_DIAG_ACCESS_MEMBAR

/*
 * Jalapeno version of ecache_flush_line.  Uses Jalapeno Ecache Displacement
 * Flush feature to flush all 4 sets/ways.
 */
#define	ECACHE_FLUSH_LINE(physaddr, ec_set_size, scr1, scr2)		\
	CPU_INDEX(scr1, scr2);						\
	sllx	scr1, JP_ECFLUSH_PORTID_SHIFT, scr1;			\
	set	JP_ECACHE_IDX_DISP_FLUSH, scr2;				\
	or	scr2, scr1, scr2;					\
	sub	ec_set_size, 1, scr1;					\
	and	physaddr, scr1, scr1;					\
	or	scr2, scr1, scr1;					\
	ECACHE_REFLUSH_LINE(ec_set_size, scr1, scr2)

/*
 * Macro for getting ecache size from cpunodes structure
 *  scr1:    Scratch, ecache size returned in this
 *  scr2:    Scratch
 */
#define	GET_ECACHE_SIZE(scr1, scr2)					\
	CPU_INDEX(scr1, scr2);						\
	mulx	scr1, CPU_NODE_SIZE, scr1;				\
	set	cpunodes + ECACHE_SIZE, scr2;				\
	ld	[scr1 + scr2], scr1

/* END CSTYLED */

#endif	/* !lint */

#if defined(lint)

/* ARGSUSED */
void
shipit(int upaid, int bn)
{ return; }

#else	/* lint */

/*
 * Ship mondo to aid using implicit busy/nack pair (bn ignored)
 */
	ENTRY_NP(shipit)
	sll	%o0, IDCR_PID_SHIFT, %g1	! IDCR<18:14> = agent id
	or	%g1, IDCR_OFFSET, %g1		! IDCR<13:0> = 0x70
	stxa	%g0, [%g1]ASI_INTR_DISPATCH	! interrupt vector dispatch
	membar	#Sync
	retl
	nop
	SET_SIZE(shipit)

#endif	/* lint */


/*
 * flush_ecache:
 *	%o0 - 64 bit physical address
 *	%o1 - ecache size
 *	%o2 - ecache linesize
 */
#if defined(lint)

/*ARGSUSED*/
void
flush_ecache(uint64_t physaddr, size_t ecache_size, size_t ecache_linesize)
{}

#else /* !lint */

	ENTRY(flush_ecache)
#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_1);
	JP_FORCE_FULL_SPEED(%o3, %g1, %g2, %g3);	/* %o3: saved speed */
flush_ecache_1:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	/*
	 * Flush the entire Ecache using displacement flush.
	 */
	ECACHE_FLUSHALL(%o1, %o2, %o0, %o4)

#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_2);
	JP_RESTORE_SPEED(%o3, %g1, %g2, %g3);		/* %o3: saved speed */
flush_ecache_2:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	retl
	nop
	SET_SIZE(flush_ecache)

#endif /* lint */


#if defined(lint)

void
fast_ecc_err(void)
{}

#else	/* lint */

	.section ".text"
	.align	64
	ENTRY_NP(fast_ecc_err)

	/*
	 * Turn off CEEN and NCEEN.
	 */
	ldxa	[%g0]ASI_ESTATE_ERR, %g3
	andn	%g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
	stxa	%g4, [%g0]ASI_ESTATE_ERR
	membar	#Sync			! membar sync required

	/*
	 * Do the CPU log out capture.
	 *   %g3 = "failed?" return value.
	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
	 *         into this macro via %g4. Output only valid if cpu_private
	 *         struct has not been initialized.
	 *   CHPR_FECCTL0_LOGOUT = cpu logout structure offset input
	 *   %g4 = Trap information stored in the cpu logout flags field
	 *   %g5 = scr1
	 *   %g6 = scr2
	 *   %g3 = scr3
	 *   %g4 = scr4
	 */
	and	%g3, EN_REG_CEEN, %g4		! store the CEEN value, TL=0
	set	CHPR_FECCTL0_LOGOUT, %g6
	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)

	/*
	 * Flush the Ecache to get the error out of the Ecache.  If the UCC
	 * or UCU is on a dirty line, then the following flush will turn
	 * that into a WDC or WDU, respectively.
	 */
	CPU_INDEX(%g4, %g5)
	mulx	%g4, CPU_NODE_SIZE, %g4
	set	cpunodes, %g5
	add	%g4, %g5, %g4
	ld	[%g4 + ECACHE_LINESIZE], %g5
	ld	[%g4 + ECACHE_SIZE], %g4
#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_1);
        set     jp_estar_tl0_data, %g6
        stx     %g2, [%g6 + 0]
        stx     %g3, [%g6 + 8]
	JP_FORCE_FULL_SPEED(%g2, %g3, %g6, %g7)		/* %g2: saved speed */
fast_ecc_err_1:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */
	ECACHE_FLUSHALL(%g4, %g5, %g6, %g7)
#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_2);
	JP_RESTORE_SPEED(%g2, %g3, %g6, %g7)		/* %g2: saved speed */
        set     jp_estar_tl0_data, %g6
        ldx     [%g6 + 0], %g2
        ldx     [%g6 + 8], %g3
fast_ecc_err_2:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	/*
	 * Flush the Dcache.  Since bad data could have been installed in
	 * the Dcache we must flush it before re-enabling it.
	 */
	ASM_LD(%g5, dcache_size)
	ASM_LD(%g6, dcache_linesize)
	CH_DCACHE_FLUSHALL(%g5, %g6, %g7)

	/*
	 * Flush the Icache.  Since we turned off the Icache to capture the
	 * Icache line it is now stale or corrupted and we must flush it
	 * before re-enabling it.
	 */
	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_4);
	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
	ba,pt	%icc, 5f
	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
fast_ecc_err_4:
	ASM_LD(%g5, icache_size)
	ASM_LD(%g6, icache_linesize)
5:
	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)

	/*
	 * Restore the Dcache and Icache to the previous state.
	 */
	stxa	%g1, [%g0]ASI_DCU
	flush	%g0	/* flush required after changing the IC bit */

	/*
	 * Make sure our CPU logout operation was successful.
	 */
	cmp	%g3, %g0
	be	8f
	  nop

	/*
	 * If the logout structure had been busy, how many times have
	 * we tried to use it and failed (nesting count)? If we have
	 * already recursed a substantial number of times, then we can
	 * assume things are not going to get better by themselves and
	 * so it would be best to panic.
	 */
	cmp	%g3, CLO_NESTING_MAX
	blt	7f
	  nop

        call ptl1_panic
          mov   PTL1_BAD_ECC, %g1

7:
	/*
	 * Otherwise, if the logout structure was busy but we have not
	 * nested more times than our maximum value, then we simply
	 * issue a retry. Our TL=0 trap handler code will check and
	 * clear the AFSR after it is done logging what is currently
	 * in the logout struct and handle this event at that time.
	 */
	retry
8:
	/*
	 * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're
	 * already at PIL 15.
	 */
	set	cpu_fast_ecc_error, %g1
	rdpr	%pil, %g4
	cmp	%g4, PIL_14
	ba	sys_trap
	  movl	%icc, PIL_14, %g4

	SET_SIZE(fast_ecc_err)

#endif	/* lint */


/*
 * Fast ECC error at TL>0 handler
 * We get here via trap 70 at TL>0->Software trap 0 at TL>0.  We enter
 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
 * For a complete description of the Fast ECC at TL>0 handling see the
 * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in
 * us3_common_asm.s
 */
#if defined(lint)

void
fast_ecc_tl1_err(void)
{}

#else	/* lint */

	.section ".text"
	.align	64
	ENTRY_NP(fast_ecc_tl1_err)

	/*
	 * This macro turns off the D$/I$ if they are on and saves their
	 * original state in ch_err_tl1_tmp, saves all the %g registers in the
	 * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves
	 * the %tpc in ch_err_tl1_tpc.  At the end of this macro, %g1 will
	 * point to the ch_err_tl1_data structure and the original D$/I$ state
	 * will be saved in ch_err_tl1_tmp.  All %g registers except for %g1
	 * will be available.
	 */
	CH_ERR_TL1_FECC_ENTER;

	/*
	 * Get the diagnostic logout data.  %g4 must be initialized to
	 * current CEEN state, %g5 must point to logout structure in
	 * ch_err_tl1_data_t.  %g3 will contain the nesting count upon
	 * return.
	 */
	ldxa	[%g0]ASI_ESTATE_ERR, %g4
	and	%g4, EN_REG_CEEN, %g4
	add	%g1, CH_ERR_TL1_LOGOUT, %g5
	DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4)

	/*
	 * If the logout nesting count is exceeded, we're probably
	 * not making any progress, try to panic instead.
	 */
	cmp	%g3, CLO_NESTING_MAX
	bge	fecc_tl1_err
	  nop

	/*
	 * Save the current CEEN and NCEEN state in %g7 and turn them off
	 * before flushing the Ecache.
	 */
	ldxa	[%g0]ASI_ESTATE_ERR, %g7
	andn	%g7, EN_REG_CEEN | EN_REG_NCEEN, %g5
	stxa	%g5, [%g0]ASI_ESTATE_ERR
	membar	#Sync

	/*
	 * Flush the Ecache, using the largest possible cache size with the
	 * smallest possible line size since we can't get the actual sizes
	 * from the cpu_node due to DTLB misses.
	 */
	set	JP_ECACHE_MAX_SIZE, %g4
#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_1);
        set     jp_estar_tl1_data, %g6
        stx     %g2, [%g6 + 0]
        stx     %g3, [%g6 + 8]
	JP_FORCE_FULL_SPEED(%g2, %g3, %g5, %g6)
fast_ecc_tl1_err_1:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */
	ECACHE_FLUSHALL(%g4, JP_ECACHE_MAX_LSIZE, %g5, %g6)
#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_2);
	JP_RESTORE_SPEED(%g2, %g3, %g5, %g6)
        set     jp_estar_tl1_data, %g6
        ldx     [%g6 + 0], %g2
        ldx     [%g6 + 8], %g3
fast_ecc_tl1_err_2:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	/*
	 * Restore CEEN and NCEEN to the previous state.
	 */
	stxa	%g7, [%g0]ASI_ESTATE_ERR
	membar	#Sync

	/*
	 * If we turned off the D$, then flush it and turn it back on.
	 */
	ldxa	[%g1 + CH_ERR_TL1_TMP]%asi, %g3
	andcc	%g3, CH_ERR_TSTATE_DC_ON, %g0
	bz	%xcc, 3f
	  nop

	/*
	 * Flush the D$.
	 */
	ASM_LD(%g4, dcache_size)
	ASM_LD(%g5, dcache_linesize)
	CH_DCACHE_FLUSHALL(%g4, %g5, %g6)

	/*
	 * Turn the D$ back on.
	 */
	ldxa	[%g0]ASI_DCU, %g3
	or	%g3, DCU_DC, %g3
	stxa	%g3, [%g0]ASI_DCU
	membar	#Sync
3:
	/*
	 * If we turned off the I$, then flush it and turn it back on.
	 */
	ldxa	[%g1 + CH_ERR_TL1_TMP]%asi, %g3
	andcc	%g3, CH_ERR_TSTATE_IC_ON, %g0
	bz	%xcc, 4f
	  nop

	/*
	 * Flush the I$.
	 */
	ASM_LD(%g4, icache_size)
	ASM_LD(%g5, icache_linesize)
	CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3)

	/*
	 * Turn the I$ back on.  Changing DCU_IC requires flush.
	 */
	ldxa	[%g0]ASI_DCU, %g3
	or	%g3, DCU_IC, %g3
	stxa	%g3, [%g0]ASI_DCU
	flush	%g0
4:

#ifdef TRAPTRACE
	/*
	 * Get current trap trace entry physical pointer.
	 */
	CPU_INDEX(%g6, %g5)
	sll	%g6, TRAPTR_SIZE_SHIFT, %g6
	set	trap_trace_ctl, %g5
	add	%g6, %g5, %g6
	ld	[%g6 + TRAPTR_LIMIT], %g5
	tst	%g5
	be	%icc, skip_traptrace
	  nop
	ldx	[%g6 + TRAPTR_PBASE], %g5
	ld	[%g6 + TRAPTR_OFFSET], %g4
	add	%g5, %g4, %g5

	/*
	 * Create trap trace entry.
	 */
	rd	%asi, %g7
	wr	%g0, TRAPTR_ASI, %asi
	rd	STICK, %g4
	stxa	%g4, [%g5 + TRAP_ENT_TICK]%asi
	rdpr	%tl, %g4
	stha	%g4, [%g5 + TRAP_ENT_TL]%asi
	rdpr	%tt, %g4
	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
	rdpr	%tpc, %g4
	stna	%g4, [%g5 + TRAP_ENT_TPC]%asi
	rdpr	%tstate, %g4
	stxa	%g4, [%g5 + TRAP_ENT_TSTATE]%asi
	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
	wr	%g0, %g7, %asi
	ldxa	[%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3
	ldxa	[%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4
	wr	%g0, TRAPTR_ASI, %asi
	stna	%g3, [%g5 + TRAP_ENT_F1]%asi
	stna	%g4, [%g5 + TRAP_ENT_F2]%asi
	wr	%g0, %g7, %asi
	ldxa	[%g1 + CH_ERR_TL1_AFAR]%asi, %g3
	ldxa	[%g1 + CH_ERR_TL1_AFSR]%asi, %g4
	wr	%g0, TRAPTR_ASI, %asi
	stna	%g3, [%g5 + TRAP_ENT_F3]%asi
	stna	%g4, [%g5 + TRAP_ENT_F4]%asi
	wr	%g0, %g7, %asi

	/*
	 * Advance trap trace pointer.
	 */
	ld	[%g6 + TRAPTR_OFFSET], %g5
	ld	[%g6 + TRAPTR_LIMIT], %g4
	st	%g5, [%g6 + TRAPTR_LAST_OFFSET]
	add	%g5, TRAP_ENT_SIZE, %g5
	sub	%g4, TRAP_ENT_SIZE, %g4
	cmp	%g5, %g4
	movge	%icc, 0, %g5
	st	%g5, [%g6 + TRAPTR_OFFSET]
skip_traptrace:
#endif	/* TRAPTRACE */

	/*
	 * If nesting count is not zero, skip all the AFSR/AFAR
	 * handling and just do the necessary cache-flushing.
	 */
	ldxa	[%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2
	brnz	%g2, 6f
	  nop

	/*
	 * If a UCU followed by a WDU has occurred go ahead and panic
	 * since a UE will occur (on the retry) before the UCU and WDU
	 * messages are enqueued.
	 */
	ldxa	[%g1 + CH_ERR_TL1_AFSR]%asi, %g3
	set	1, %g4
	sllx	%g4, C_AFSR_UCU_SHIFT, %g4
	btst	%g4, %g3		! UCU in original AFSR?
	bz	%xcc, 6f
	  nop
	ldxa	[%g0]ASI_AFSR, %g4	! current AFSR
	or	%g3, %g4, %g3		! %g3 = original + current AFSR
	set	1, %g4
	sllx	%g4, C_AFSR_WDU_SHIFT, %g4
	btst	%g4, %g3		! WDU in original or current AFSR?
	bnz	%xcc, fecc_tl1_err
	  nop

6:
	/*
	 * We fall into this macro if we've successfully logged the error in
	 * the ch_err_tl1_data structure and want the PIL15 softint to pick
	 * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
	 * Restores the %g registers and issues retry.
	 */
	CH_ERR_TL1_EXIT;
	/*
	 * Establish panic exit label.
	 */
	CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err);

	SET_SIZE(fast_ecc_tl1_err)

#endif	/* lint */


#if defined(lint)

uint64_t
get_jbus_config(void)
{ return (0); }

/* ARGSUSED */
void
set_jbus_config(uint64_t jbus_config)
{}

/* ARGSUSED */
void
set_mcu_ctl_reg1(uint64_t mcu_ctl)
{}

uint64_t
get_mcu_ctl_reg1(void)
{ return (0); }

#else	/* lint */

	ENTRY(get_jbus_config)
	ldxa	[%g0]ASI_JBUS_CONFIG, %o0
	retl
	nop
	SET_SIZE(get_jbus_config)

	ENTRY(set_jbus_config)
	stxa	%o0, [%g0]ASI_JBUS_CONFIG
	membar	#Sync
	retl
	nop
	SET_SIZE(set_jbus_config)


	ENTRY(get_mcu_ctl_reg1)
	ldxa	[%g0]ASI_MCU_CTRL, %o0	! MCU control reg1 is at offset 0
	retl
	nop
	SET_SIZE(get_mcu_ctl_reg1)


	ENTRY(set_mcu_ctl_reg1)
	stxa	%o0, [%g0]ASI_MCU_CTRL	! MCU control reg1 is at offset 0
	membar	#Sync
	retl
	nop
	SET_SIZE(set_mcu_ctl_reg1)

#endif	/* lint */


#if defined(lint)
/*
 * scrubphys - Pass in the aligned physical memory address
 * that you want to scrub, along with the ecache set size.
 *
 *	1) Displacement flush the E$ line corresponding to %addr.
 *	   The first ldxa guarantees that the %addr is no longer in
 *	   M, O, or E (goes to I or S (if instruction fetch also happens).
 *	2) "Write" the data using a CAS %addr,%g0,%g0.
 *	   The casxa guarantees a transition from I to M or S to M.
 *	3) Displacement flush the E$ line corresponding to %addr.
 *	   The second ldxa pushes the M line out of the ecache, into the
 *	   writeback buffers, on the way to memory.
 *	4) The "membar #Sync" pushes the cache line out of the writeback
 *	   buffers onto the bus, on the way to dram finally.
 *
 * This is a modified version of the algorithm suggested by Gary Lauterbach.
 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line
 * as modified, but then we found out that for spitfire, if it misses in the
 * E$ it will probably install as an M, but if it hits in the E$, then it
 * will stay E, if the store doesn't happen. So the first displacement flush
 * should ensure that the CAS will miss in the E$.  Arrgh.
 */
/* ARGSUSED */
void
scrubphys(uint64_t paddr, int ecache_set_size)
{}

#else	/* lint */
	ENTRY(scrubphys)
	rdpr	%pstate, %o4
	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
	wrpr	%o5, %g0, %pstate	! clear IE, AM bits

#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_1);
	JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3)		/* %o5: saved speed */
scrubphys_1:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
	casxa	[%o0]ASI_MEM, %g0, %g0
	ECACHE_REFLUSH_LINE(%o1, %o2, %o3)

#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_2);
	JP_RESTORE_SPEED(%o5, %g1, %g2, %g3)		/* %o5: saved speed */
scrubphys_2:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	wrpr	%g0, %o4, %pstate	! restore earlier pstate register value

	retl
	membar	#Sync			! move the data out of the load buffer
	SET_SIZE(scrubphys)

#endif	/* lint */


#if defined(lint)
/*
 * clearphys - Pass in the physical memory address of the checkblock
 * that you want to push out, cleared with a recognizable pattern,
 * from the ecache.
 *
 * To ensure that the ecc gets recalculated after the bad data is cleared,
 * we must write out enough data to fill the w$ line (64 bytes). So we read
 * in an entire ecache subblock's worth of data, and write it back out.
 * Then we overwrite the 16 bytes of bad data with the pattern.
 */
/* ARGSUSED */
void
clearphys(uint64_t paddr, int ecache_set_size, int ecache_linesize)
{
}

#else	/* lint */
	ENTRY(clearphys)
	/* turn off IE, AM bits */
	rdpr	%pstate, %o4
	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
	wrpr	%o5, %g0, %pstate

	/* turn off NCEEN */
	ldxa	[%g0]ASI_ESTATE_ERR, %o5
	andn	%o5, EN_REG_NCEEN, %o3
	stxa	%o3, [%g0]ASI_ESTATE_ERR
	membar	#Sync

	/* align address passed with 64 bytes subblock size */
	mov	CH_ECACHE_SUBBLK_SIZE, %o2
	andn	%o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1

	/* move the good data into the W$ */
1:
	subcc	%o2, 8, %o2
	ldxa	[%g1 + %o2]ASI_MEM, %g2
	bge	1b
	  stxa	%g2, [%g1 + %o2]ASI_MEM

	/* now overwrite the bad data */
	setx	0xbadecc00badecc01, %g1, %g2
	stxa	%g2, [%o0]ASI_MEM
	mov	8, %g1
	stxa	%g2, [%o0 + %g1]ASI_MEM

#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g1, clearphys_1);
	JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3)		/* %o5: saved speed */
clearphys_1:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)
	casxa	[%o0]ASI_MEM, %g0, %g0
	ECACHE_REFLUSH_LINE(%o1, %o2, %o3)

#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g1, clearphys_2);
	JP_RESTORE_SPEED(%o5, %g1, %g2, %g3)		/* %o5: saved speed */
clearphys_2:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	/* clear the AFSR */
	ldxa	[%g0]ASI_AFSR, %o1
	stxa	%o1, [%g0]ASI_AFSR
	membar	#Sync

	/* turn NCEEN back on */
	stxa	%o5, [%g0]ASI_ESTATE_ERR
	membar	#Sync

	/* return and re-enable IE and AM */
	retl
	  wrpr	%g0, %o4, %pstate
	SET_SIZE(clearphys)

#endif	/* lint */


#if defined(lint)
/*
 * Jalapeno Ecache displacement flush the specified line from the E$
 *
 * Register usage:
 *	%o0 - 64 bit physical address for flushing
 *	%o1 - Ecache set size
 */
/*ARGSUSED*/
void
ecache_flush_line(uint64_t flushaddr, int ec_set_size)
{
}
#else	/* lint */
	ENTRY(ecache_flush_line)

#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_1);
	JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3)		/* %o5: saved speed */
ecache_flush_line_1:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3)

#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
	CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_2);
	JP_RESTORE_SPEED(%o5, %g1, %g2, %g3)		/* %o5: saved speed */
ecache_flush_line_2:
#endif	/* JALAPENO && JALAPENO_ERRATA_85 */

	retl
	  nop
	SET_SIZE(ecache_flush_line)
#endif	/* lint */


/*
 * Perform necessary cpu workaround to ensure jbus ordering.
 * Called only from Fire systems.
 * CPU's internal "invalidate FIFOs" are flushed.
 */

#if defined(lint)
void
jbus_stst_order()
{}
#else	/* lint */

#define	VIS_BLOCKSIZE	64

	.seg    ".data"
	.align  VIS_BLOCKSIZE
	.type   sync_buf, #object
sync_buf:
	.skip   VIS_BLOCKSIZE
	.size   sync_buf, VIS_BLOCKSIZE

	ENTRY(jbus_stst_order)
	set	sync_buf, %o1

	rd	%fprs, %o2			! %o2 = saved fprs
	or	%o2, FPRS_FEF, %o3
	wr	%g0, %o3, %fprs			! make sure fp is enabled
	stda    %d0, [%o1]ASI_BLK_COMMIT_P
	wr	%o2, 0, %fprs			! restore fprs

	retl
	membar  #Sync
	SET_SIZE(jbus_stst_order)

#endif	/* lint */

#if defined(lint)
/*
 * This routine will not be called in Jalapeno systems.
 */
void
flush_ipb(void)
{ return; }

#else	/* lint */

	ENTRY(flush_ipb)
	retl
	nop
	SET_SIZE(flush_ipb)

#endif	/* lint */