xref: /titanic_50/usr/src/uts/sun4u/cpu/spitfire.c (revision f841f6ad96ea6675d6c6b35c749eaac601799fdf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/systm.h>
30 #include <sys/archsystm.h>
31 #include <sys/machparam.h>
32 #include <sys/machsystm.h>
33 #include <sys/cpu.h>
34 #include <sys/elf_SPARC.h>
35 #include <vm/hat_sfmmu.h>
36 #include <vm/page.h>
37 #include <sys/cpuvar.h>
38 #include <sys/spitregs.h>
39 #include <sys/async.h>
40 #include <sys/cmn_err.h>
41 #include <sys/debug.h>
42 #include <sys/dditypes.h>
43 #include <sys/sunddi.h>
44 #include <sys/cpu_module.h>
45 #include <sys/prom_debug.h>
46 #include <sys/vmsystm.h>
47 #include <sys/prom_plat.h>
48 #include <sys/sysmacros.h>
49 #include <sys/intreg.h>
50 #include <sys/machtrap.h>
51 #include <sys/ontrap.h>
52 #include <sys/ivintr.h>
53 #include <sys/atomic.h>
54 #include <sys/panic.h>
55 #include <sys/ndifm.h>
56 #include <sys/fm/protocol.h>
57 #include <sys/fm/util.h>
58 #include <sys/fm/cpu/UltraSPARC-II.h>
59 #include <sys/ddi.h>
60 #include <sys/ecc_kstat.h>
61 #include <sys/watchpoint.h>
62 #include <sys/dtrace.h>
63 #include <sys/errclassify.h>
64 
65 uint_t	cpu_impl_dual_pgsz = 0;
66 
67 /*
68  * Structure for the 8 byte ecache data dump and the associated AFSR state.
69  * There will be 8 of these structures used to dump an ecache line (64 bytes).
70  */
71 typedef struct sf_ec_data_elm {
72 	uint64_t ec_d8;
73 	uint64_t ec_afsr;
74 } ec_data_t;
75 
76 /*
77  * Define spitfire (Ultra I/II) specific asynchronous error structure
78  */
79 typedef struct spitfire_async_flt {
80 	struct async_flt cmn_asyncflt;	/* common - see sun4u/sys/async.h */
81 	ushort_t flt_type;		/* types of faults - cpu specific */
82 	ec_data_t flt_ec_data[8];	/* for E$ or mem dump/state */
83 	uint64_t flt_ec_tag;		/* E$ tag info */
84 	int flt_ec_lcnt;		/* number of bad E$ lines */
85 	ushort_t flt_sdbh;		/* UDBH reg */
86 	ushort_t flt_sdbl;		/* UDBL reg */
87 } spitf_async_flt;
88 
89 /*
90  * Prototypes for support routines in spitfire_asm.s:
91  */
92 extern void flush_ecache(uint64_t physaddr, size_t size, size_t linesize);
93 extern uint64_t get_lsu(void);
94 extern void set_lsu(uint64_t ncc);
95 extern void get_ecache_dtag(uint32_t ecache_idx, uint64_t *data, uint64_t *tag,
96 				uint64_t *oafsr, uint64_t *acc_afsr);
97 extern uint64_t check_ecache_line(uint32_t id, uint64_t *acc_afsr);
98 extern uint64_t get_ecache_tag(uint32_t id, uint64_t *nafsr,
99 				uint64_t *acc_afsr);
100 extern uint64_t read_and_clear_afsr();
101 extern void write_ec_tag_parity(uint32_t id);
102 extern void write_hb_ec_tag_parity(uint32_t id);
103 
104 /*
105  * Spitfire module routines:
106  */
107 static void cpu_async_log_err(void *flt);
108 /*PRINTFLIKE6*/
109 static void cpu_aflt_log(int ce_code, int tagnum, spitf_async_flt *spflt,
110     uint_t logflags, const char *endstr, const char *fmt, ...);
111 
112 static void cpu_read_paddr(struct async_flt *aflt, short verbose, short ce_err);
113 static void cpu_ce_log_status(spitf_async_flt *spf_flt, char *unum);
114 static void cpu_log_ecmem_info(spitf_async_flt *spf_flt);
115 
116 static void log_ce_err(struct async_flt *aflt, char *unum);
117 static void log_ue_err(struct async_flt *aflt, char *unum);
118 static void check_misc_err(spitf_async_flt *spf_flt);
119 static ushort_t ecc_gen(uint_t high_bytes, uint_t low_bytes);
120 static int check_ecc(struct async_flt *aflt);
121 static uint_t get_cpu_status(uint64_t arg);
122 static uint64_t clear_errors(spitf_async_flt *spf_flt, uint64_t *acc_afsr);
123 static void scan_ecache(uint64_t *afar, ec_data_t *data, uint64_t *tag,
124 		int *m, uint64_t *afsr);
125 static void ecache_kstat_init(struct cpu *cp);
126 static void ecache_scrub_log(ec_data_t *ec_data, uint64_t ec_tag,
127 		uint64_t paddr, int mpb, uint64_t);
128 static uint64_t ecache_scrub_misc_err(int, uint64_t);
129 static void ecache_scrub_tag_err(uint64_t, uchar_t, uint32_t);
130 static void ecache_page_retire(void *);
131 static int ecc_kstat_update(kstat_t *ksp, int rw);
132 static int ce_count_unum(int status, int len, char *unum);
133 static void add_leaky_bucket_timeout(void);
134 static int synd_to_synd_code(int synd_status, ushort_t synd);
135 
136 extern uint_t read_all_memscrub;
137 extern void memscrub_run(void);
138 
139 static uchar_t	isus2i;			/* set if sabre */
140 static uchar_t	isus2e;			/* set if hummingbird */
141 
142 /*
143  * Default ecache mask and shift settings for Spitfire.  If we detect a
144  * different CPU implementation, we will modify these values at boot time.
145  */
146 static uint64_t cpu_ec_tag_mask		= S_ECTAG_MASK;
147 static uint64_t cpu_ec_state_mask	= S_ECSTATE_MASK;
148 static uint64_t cpu_ec_par_mask		= S_ECPAR_MASK;
149 static int cpu_ec_par_shift		= S_ECPAR_SHIFT;
150 static int cpu_ec_tag_shift		= S_ECTAG_SHIFT;
151 static int cpu_ec_state_shift		= S_ECSTATE_SHIFT;
152 static uchar_t cpu_ec_state_exl		= S_ECSTATE_EXL;
153 static uchar_t cpu_ec_state_mod		= S_ECSTATE_MOD;
154 static uchar_t cpu_ec_state_shr		= S_ECSTATE_SHR;
155 static uchar_t cpu_ec_state_own		= S_ECSTATE_OWN;
156 
157 /*
158  * Default ecache state bits for Spitfire.  These individual bits indicate if
159  * the given line is in any of the valid or modified states, respectively.
160  * Again, we modify these at boot if we detect a different CPU.
161  */
162 static uchar_t cpu_ec_state_valid	= S_ECSTATE_VALID;
163 static uchar_t cpu_ec_state_dirty	= S_ECSTATE_DIRTY;
164 static uchar_t cpu_ec_parity		= S_EC_PARITY;
165 static uchar_t cpu_ec_state_parity	= S_ECSTATE_PARITY;
166 
167 /*
168  * This table is used to determine which bit(s) is(are) bad when an ECC
169  * error occurrs.  The array is indexed an 8-bit syndrome.  The entries
170  * of this array have the following semantics:
171  *
172  *      00-63   The number of the bad bit, when only one bit is bad.
173  *      64      ECC bit C0 is bad.
174  *      65      ECC bit C1 is bad.
175  *      66      ECC bit C2 is bad.
176  *      67      ECC bit C3 is bad.
177  *      68      ECC bit C4 is bad.
178  *      69      ECC bit C5 is bad.
179  *      70      ECC bit C6 is bad.
180  *      71      ECC bit C7 is bad.
181  *      72      Two bits are bad.
182  *      73      Three bits are bad.
183  *      74      Four bits are bad.
184  *      75      More than Four bits are bad.
185  *      76      NO bits are bad.
186  * Based on "Galaxy Memory Subsystem SPECIFICATION" rev 0.6, pg. 28.
187  */
188 
189 #define	C0	64
190 #define	C1	65
191 #define	C2	66
192 #define	C3	67
193 #define	C4	68
194 #define	C5	69
195 #define	C6	70
196 #define	C7	71
197 #define	M2	72
198 #define	M3	73
199 #define	M4	74
200 #define	MX	75
201 #define	NA	76
202 
203 #define	SYND_IS_SINGLE_BIT_DATA(synd_code)	((synd_code >= 0) && \
204 						    (synd_code < C0))
205 #define	SYND_IS_SINGLE_BIT_CHK(synd_code)	((synd_code >= C0) && \
206 						    (synd_code <= C7))
207 
208 static char ecc_syndrome_tab[] =
209 {
210 	NA, C0, C1, M2, C2, M2, M2, M3, C3, M2, M2, M3, M2, M3, M3, M4,
211 	C4, M2, M2, 32, M2, 57, MX, M2, M2, 37, 49, M2, 40, M2, M2, 44,
212 	C5, M2, M2, 33, M2, 61,  4, M2, M2, MX, 53, M2, 45, M2, M2, 41,
213 	M2,  0,  1, M2, 10, M2, M2, MX, 15, M2, M2, MX, M2, M3, M3, M2,
214 	C6, M2, M2, 42, M2, 59, 39, M2, M2, MX, 51, M2, 34, M2, M2, 46,
215 	M2, 25, 29, M2, 27, M4, M2, MX, 31, M2, M4, MX, M2, MX, MX, M2,
216 	M2, MX, 36, M2,  7, M2, M2, 54, MX, M2, M2, 62, M2, 48, 56, M2,
217 	M3, M2, M2, MX, M2, MX, 22, M2, M2, 18, MX, M2, M3, M2, M2, MX,
218 	C7, M2, M2, 47, M2, 63, MX, M2, M2,  6, 55, M2, 35, M2, M2, 43,
219 	M2,  5, MX, M2, MX, M2, M2, 50, 38, M2, M2, 58, M2, 52, 60, M2,
220 	M2, 17, 21, M2, 19, M4, M2, MX, 23, M2, M4, MX, M2, MX, MX, M2,
221 	M3, M2, M2, MX, M2, MX, 30, M2, M2, 26, MX, M2, M3, M2, M2, MX,
222 	M2,  8, 13, M2,  2, M2, M2, M3,  3, M2, M2, M3, M2, MX, MX, M2,
223 	M3, M2, M2, M3, M2, MX, 16, M2, M2, 20, MX, M2, MX, M2, M2, MX,
224 	M3, M2, M2, M3, M2, MX, 24, M2, M2, 28, MX, M2, MX, M2, M2, MX,
225 	M4, 12,  9, M2, 14, M2, M2, MX, 11, M2, M2, MX, M2, MX, MX, M4
226 };
227 
228 #define	SYND_TBL_SIZE 256
229 
230 /*
231  * Hack for determining UDBH/UDBL, for later cpu-specific error reporting.
232  * Cannot use bit 3 in afar, because it is a valid bit on a Sabre/Hummingbird.
233  */
234 #define	UDBL_REG	0x8000
235 #define	UDBL(synd)	((synd & UDBL_REG) >> 15)
236 #define	SYND(synd)	(synd & 0x7FFF)
237 
238 /*
239  * These error types are specific to Spitfire and are used internally for the
240  * spitfire fault structure flt_type field.
241  */
242 #define	CPU_UE_ERR		0	/* uncorrectable errors - UEs */
243 #define	CPU_EDP_LDP_ERR		1	/* LDP or EDP parity error */
244 #define	CPU_WP_ERR		2	/* WP parity error */
245 #define	CPU_BTO_BERR_ERR	3	/* bus timeout errors */
246 #define	CPU_PANIC_CP_ERR	4	/* cp error from panic polling */
247 #define	CPU_TRAPPING_CP_ERR	5	/* for sabre/hbird only, cp error */
248 #define	CPU_BADLINE_CI_ERR	6	/* E$ clean_bad line when idle */
249 #define	CPU_BADLINE_CB_ERR	7	/* E$ clean_bad line when busy */
250 #define	CPU_BADLINE_DI_ERR	8	/* E$ dirty_bad line when idle */
251 #define	CPU_BADLINE_DB_ERR	9	/* E$ dirty_bad line when busy */
252 #define	CPU_ORPHAN_CP_ERR	10	/* Orphan CP error */
253 #define	CPU_ECACHE_ADDR_PAR_ERR	11	/* Ecache Address parity error */
254 #define	CPU_ECACHE_STATE_ERR	12	/* Ecache state error */
255 #define	CPU_ECACHE_ETP_ETS_ERR	13	/* ETP set but ETS is zero */
256 #define	CPU_ECACHE_TAG_ERR	14	/* Scrub the E$ tag, if state clean */
257 #define	CPU_ADDITIONAL_ERR	15	/* Additional errors occurred */
258 
259 /*
260  * Macro to access the "Spitfire cpu private" data structure.
261  */
262 #define	CPU_PRIVATE_PTR(cp, x)	(&(((spitfire_private_t *)CPU_PRIVATE(cp))->x))
263 
264 /*
265  * set to 0 to disable automatic retiring of pages on
266  * DIMMs that have excessive soft errors
267  */
268 int automatic_page_removal = 1;
269 
270 /*
271  * Heuristic for figuring out which module to replace.
272  * Relative likelihood that this P_SYND indicates that this module is bad.
273  * We call it a "score", though, not a relative likelihood.
274  *
275  * Step 1.
276  * Assign a score to each byte of P_SYND according to the following rules:
277  * If no bits on (0x00) or all bits on (0xFF), then give it a 5.
278  * If one bit on, give it a 95.
279  * If seven bits on, give it a 10.
280  * If two bits on:
281  *   in different nybbles, a 90
282  *   in same nybble, but unaligned, 85
283  *   in same nybble and as an aligned pair, 80
284  * If six bits on, look at the bits that are off:
285  *   in same nybble and as an aligned pair, 15
286  *   in same nybble, but unaligned, 20
287  *   in different nybbles, a 25
288  * If three bits on:
289  *   in diferent nybbles, no aligned pairs, 75
290  *   in diferent nybbles, one aligned pair, 70
291  *   in the same nybble, 65
292  * If five bits on, look at the bits that are off:
293  *   in the same nybble, 30
294  *   in diferent nybbles, one aligned pair, 35
295  *   in diferent nybbles, no aligned pairs, 40
296  * If four bits on:
297  *   all in one nybble, 45
298  *   as two aligned pairs, 50
299  *   one aligned pair, 55
300  *   no aligned pairs, 60
301  *
302  * Step 2:
303  * Take the higher of the two scores (one for each byte) as the score
304  * for the module.
305  *
306  * Print the score for each module, and field service should replace the
307  * module with the highest score.
308  */
309 
310 /*
311  * In the table below, the first row/column comment indicates the
312  * number of bits on in that nybble; the second row/column comment is
313  * the hex digit.
314  */
315 
316 static int
317 p_synd_score_table[256] = {
318 	/* 0   1   1   2   1   2   2   3   1   2   2   3   2   3   3   4 */
319 	/* 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  A,  B,  C,  D,  E,  F */
320 /* 0 0 */  5, 95, 95, 80, 95, 85, 85, 65, 95, 85, 85, 65, 80, 65, 65, 45,
321 /* 1 1 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30,
322 /* 1 2 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30,
323 /* 2 3 */ 80, 70, 70, 50, 70, 55, 55, 35, 70, 55, 55, 35, 50, 35, 35, 15,
324 /* 1 4 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30,
325 /* 2 5 */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20,
326 /* 2 6 */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20,
327 /* 3 7 */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10,
328 /* 1 8 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30,
329 /* 2 9 */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20,
330 /* 2 A */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20,
331 /* 3 B */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10,
332 /* 2 C */ 80, 70, 70, 50, 70, 55, 55, 35, 70, 55, 55, 35, 50, 35, 35, 15,
333 /* 3 D */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10,
334 /* 3 E */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10,
335 /* 4 F */ 45, 30, 30, 15, 30, 20, 20, 10, 30, 20, 20, 10, 15, 10, 10,  5,
336 };
337 
338 int
339 ecc_psynd_score(ushort_t p_synd)
340 {
341 	int i, j, a, b;
342 
343 	i = p_synd & 0xFF;
344 	j = (p_synd >> 8) & 0xFF;
345 
346 	a = p_synd_score_table[i];
347 	b = p_synd_score_table[j];
348 
349 	return (a > b ? a : b);
350 }
351 
352 /*
353  * Async Fault Logging
354  *
355  * To ease identifying, reading, and filtering async fault log messages, the
356  * label [AFT#] is now prepended to each async fault message.  These messages
357  * and the logging rules are implemented by cpu_aflt_log(), below.
358  *
359  * [AFT0] - Tag for log messages that are associated with corrected ECC errors.
360  *          This includes both corrected ECC memory and ecache faults.
361  *
362  * [AFT1] - Tag for log messages that are not ECC corrected (i.e. everything
363  *          else except CE errors) with a priority of 1 (highest).  This tag
364  *          is also used for panic messages that result from an async fault.
365  *
366  * [AFT2] - These are lower priority diagnostic messages for uncorrected ECC
367  * [AFT3]   or parity errors.  For example, AFT2 is used for the actual dump
368  *          of the E-$ data and tags.
369  *
370  * In a non-DEBUG kernel, AFT > 1 logs will be sent to the system log but not
371  * printed on the console.  To send all AFT logs to both the log and the
372  * console, set aft_verbose = 1.
373  */
374 
375 #define	CPU_FLTCPU		0x0001	/* print flt_inst as a CPU id */
376 #define	CPU_SPACE		0x0002	/* print flt_status (data or instr) */
377 #define	CPU_ERRID		0x0004	/* print flt_id */
378 #define	CPU_TL			0x0008	/* print flt_tl */
379 #define	CPU_ERRID_FIRST 	0x0010	/* print flt_id first in message */
380 #define	CPU_AFSR		0x0020	/* print flt_stat as decoded %afsr */
381 #define	CPU_AFAR		0x0040	/* print flt_addr as %afar */
382 #define	CPU_AF_PSYND		0x0080	/* print flt_stat %afsr.PSYND */
383 #define	CPU_AF_ETS		0x0100	/* print flt_stat %afsr.ETS */
384 #define	CPU_UDBH		0x0200	/* print flt_sdbh and syndrome */
385 #define	CPU_UDBL		0x0400	/* print flt_sdbl and syndrome */
386 #define	CPU_FAULTPC		0x0800	/* print flt_pc */
387 #define	CPU_SYND		0x1000	/* print flt_synd and unum */
388 
389 #define	CMN_LFLAGS	(CPU_FLTCPU | CPU_SPACE | CPU_ERRID | CPU_TL |	\
390 				CPU_AFSR | CPU_AFAR | CPU_AF_PSYND |	\
391 				CPU_AF_ETS | CPU_UDBH | CPU_UDBL |	\
392 				CPU_FAULTPC)
393 #define	UE_LFLAGS	(CMN_LFLAGS | CPU_SYND)
394 #define	CE_LFLAGS	(UE_LFLAGS & ~CPU_UDBH & ~CPU_UDBL & ~CPU_TL &	\
395 				~CPU_SPACE)
396 #define	PARERR_LFLAGS	(CMN_LFLAGS)
397 #define	WP_LFLAGS	(CMN_LFLAGS & ~CPU_SPACE & ~CPU_TL)
398 #define	CP_LFLAGS	(CMN_LFLAGS & ~CPU_SPACE & ~CPU_TL &		\
399 				~CPU_FLTCPU & ~CPU_FAULTPC)
400 #define	BERRTO_LFLAGS	(CMN_LFLAGS)
401 #define	NO_LFLAGS	(0)
402 
403 #define	AFSR_FMTSTR0	"\020\1ME"
404 #define	AFSR_FMTSTR1	"\020\040PRIV\037ISAP\036ETP\035IVUE\034TO"	\
405 			"\033BERR\032LDP\031CP\030WP\027EDP\026UE\025CE"
406 #define	UDB_FMTSTR	"\020\012UE\011CE"
407 
408 /*
409  * Save the cache bootup state for use when internal
410  * caches are to be re-enabled after an error occurs.
411  */
412 uint64_t	cache_boot_state = 0;
413 
414 /*
415  * PA[31:0] represent Displacement in UPA configuration space.
416  */
417 uint_t	root_phys_addr_lo_mask = 0xffffffff;
418 
419 /*
420  * Spitfire legacy globals
421  */
422 int	itlb_entries;
423 int	dtlb_entries;
424 
425 void
426 cpu_setup(void)
427 {
428 	extern int page_retire_messages;
429 	extern int page_retire_first_ue;
430 	extern int at_flags;
431 #if defined(SF_ERRATA_57)
432 	extern caddr_t errata57_limit;
433 #endif
434 	extern int disable_text_largepages;
435 	extern int disable_initdata_largepages;
436 
437 	cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT);
438 
439 	at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1;
440 
441 	/*
442 	 * Spitfire isn't currently FMA-aware, so we have to enable the
443 	 * page retirement messages. We also change the default policy
444 	 * for UE retirement to allow clearing of transient errors.
445 	 */
446 	page_retire_messages = 1;
447 	page_retire_first_ue = 0;
448 
449 	/*
450 	 * save the cache bootup state.
451 	 */
452 	cache_boot_state = get_lsu() & (LSU_IC | LSU_DC);
453 
454 	if (use_page_coloring) {
455 		do_pg_coloring = 1;
456 		if (use_virtual_coloring)
457 			do_virtual_coloring = 1;
458 	}
459 
460 	/*
461 	 * Tune pp_slots to use up to 1/8th of the tlb entries.
462 	 */
463 	pp_slots = MIN(8, MAXPP_SLOTS);
464 
465 	/*
466 	 * Block stores invalidate all pages of the d$ so pagecopy
467 	 * et. al. do not need virtual translations with virtual
468 	 * coloring taken into consideration.
469 	 */
470 	pp_consistent_coloring = 0;
471 
472 	isa_list =
473 	    "sparcv9+vis sparcv9 "
474 	    "sparcv8plus+vis sparcv8plus "
475 	    "sparcv8 sparcv8-fsmuld sparcv7 sparc";
476 
477 	cpu_hwcap_flags = AV_SPARC_VIS;
478 
479 	/*
480 	 * On Spitfire, there's a hole in the address space
481 	 * that we must never map (the hardware only support 44-bits of
482 	 * virtual address).  Later CPUs are expected to have wider
483 	 * supported address ranges.
484 	 *
485 	 * See address map on p23 of the UltraSPARC 1 user's manual.
486 	 */
487 	hole_start = (caddr_t)0x80000000000ull;
488 	hole_end = (caddr_t)0xfffff80000000000ull;
489 
490 	/*
491 	 * A spitfire call bug requires us to be a further 4Gbytes of
492 	 * firewall from the spec.
493 	 *
494 	 * See Spitfire Errata #21
495 	 */
496 	hole_start = (caddr_t)((uintptr_t)hole_start - (1ul << 32));
497 	hole_end = (caddr_t)((uintptr_t)hole_end + (1ul << 32));
498 
499 	/*
500 	 * The kpm mapping window.
501 	 * kpm_size:
502 	 *	The size of a single kpm range.
503 	 *	The overall size will be: kpm_size * vac_colors.
504 	 * kpm_vbase:
505 	 *	The virtual start address of the kpm range within the kernel
506 	 *	virtual address space. kpm_vbase has to be kpm_size aligned.
507 	 */
508 	kpm_size = (size_t)(2ull * 1024 * 1024 * 1024 * 1024); /* 2TB */
509 	kpm_size_shift = 41;
510 	kpm_vbase = (caddr_t)0xfffffa0000000000ull; /* 16EB - 6TB */
511 
512 #if defined(SF_ERRATA_57)
513 	errata57_limit = (caddr_t)0x80000000ul;
514 #endif
515 
516 	/*
517 	 * Allow only 8K, 64K and 4M pages for text by default.
518 	 * Allow only 8K and 64K page for initialized data segments by
519 	 * default.
520 	 */
521 	disable_text_largepages = (1 << TTE512K) | (1 << TTE32M) |
522 	    (1 << TTE256M);
523 	disable_initdata_largepages = (1 << TTE512K) | (1 << TTE4M) |
524 	    (1 << TTE32M) | (1 << TTE256M);
525 }
526 
527 static int
528 getintprop(pnode_t node, char *name, int deflt)
529 {
530 	int	value;
531 
532 	switch (prom_getproplen(node, name)) {
533 	case 0:
534 		value = 1;	/* boolean properties */
535 		break;
536 
537 	case sizeof (int):
538 		(void) prom_getprop(node, name, (caddr_t)&value);
539 		break;
540 
541 	default:
542 		value = deflt;
543 		break;
544 	}
545 
546 	return (value);
547 }
548 
549 /*
550  * Set the magic constants of the implementation.
551  */
552 void
553 cpu_fiximp(pnode_t dnode)
554 {
555 	extern int vac_size, vac_shift;
556 	extern uint_t vac_mask;
557 	extern int dcache_line_mask;
558 	int i, a;
559 	static struct {
560 		char	*name;
561 		int	*var;
562 	} prop[] = {
563 		"dcache-size",		&dcache_size,
564 		"dcache-line-size",	&dcache_linesize,
565 		"icache-size",		&icache_size,
566 		"icache-line-size",	&icache_linesize,
567 		"ecache-size",		&ecache_size,
568 		"ecache-line-size",	&ecache_alignsize,
569 		"ecache-associativity", &ecache_associativity,
570 		"#itlb-entries",	&itlb_entries,
571 		"#dtlb-entries",	&dtlb_entries,
572 		};
573 
574 	for (i = 0; i < sizeof (prop) / sizeof (prop[0]); i++) {
575 		if ((a = getintprop(dnode, prop[i].name, -1)) != -1) {
576 			*prop[i].var = a;
577 		}
578 	}
579 
580 	ecache_setsize = ecache_size / ecache_associativity;
581 
582 	vac_size = S_VAC_SIZE;
583 	vac_mask = MMU_PAGEMASK & (vac_size - 1);
584 	i = 0; a = vac_size;
585 	while (a >>= 1)
586 		++i;
587 	vac_shift = i;
588 	shm_alignment = vac_size;
589 	vac = 1;
590 
591 	dcache_line_mask = (dcache_size - 1) & ~(dcache_linesize - 1);
592 
593 	/*
594 	 * UltraSPARC I & II have ecache sizes running
595 	 * as follows: .25 MB, .5 MB, 1 MB, 2 MB, 4 MB
596 	 * and 8 MB. Adjust the copyin/copyout limits
597 	 * according to the cache size. The magic number
598 	 * of VIS_COPY_THRESHOLD comes from the copyin/copyout code
599 	 * and its floor of VIS_COPY_THRESHOLD bytes before it will use
600 	 * VIS instructions.
601 	 *
602 	 * We assume that all CPUs on the system have the same size
603 	 * ecache. We're also called very early in the game.
604 	 * /etc/system will be parsed *after* we're called so
605 	 * these values can be overwritten.
606 	 */
607 
608 	hw_copy_limit_1 = VIS_COPY_THRESHOLD;
609 	if (ecache_size <= 524288) {
610 		hw_copy_limit_2 = VIS_COPY_THRESHOLD;
611 		hw_copy_limit_4 = VIS_COPY_THRESHOLD;
612 		hw_copy_limit_8 = VIS_COPY_THRESHOLD;
613 	} else if (ecache_size == 1048576) {
614 		hw_copy_limit_2 = 1024;
615 		hw_copy_limit_4 = 1280;
616 		hw_copy_limit_8 = 1536;
617 	} else if (ecache_size == 2097152) {
618 		hw_copy_limit_2 = 1536;
619 		hw_copy_limit_4 = 2048;
620 		hw_copy_limit_8 = 2560;
621 	} else if (ecache_size == 4194304) {
622 		hw_copy_limit_2 = 2048;
623 		hw_copy_limit_4 = 2560;
624 		hw_copy_limit_8 = 3072;
625 	} else {
626 		hw_copy_limit_2 = 2560;
627 		hw_copy_limit_4 = 3072;
628 		hw_copy_limit_8 = 3584;
629 	}
630 }
631 
632 /*
633  * Called by setcpudelay
634  */
635 void
636 cpu_init_tick_freq(void)
637 {
638 	/*
639 	 * Determine the cpu frequency by calling
640 	 * tod_get_cpufrequency. Use an approximate freqency
641 	 * value computed by the prom if the tod module
642 	 * is not initialized and loaded yet.
643 	 */
644 	if (tod_ops.tod_get_cpufrequency != NULL) {
645 		mutex_enter(&tod_lock);
646 		sys_tick_freq = tod_ops.tod_get_cpufrequency();
647 		mutex_exit(&tod_lock);
648 	} else {
649 #if defined(HUMMINGBIRD)
650 		/*
651 		 * the hummingbird version of %stick is used as the basis for
652 		 * low level timing; this provides an independent constant-rate
653 		 * clock for general system use, and frees power mgmt to set
654 		 * various cpu clock speeds.
655 		 */
656 		if (system_clock_freq == 0)
657 			cmn_err(CE_PANIC, "invalid system_clock_freq 0x%lx",
658 			    system_clock_freq);
659 		sys_tick_freq = system_clock_freq;
660 #else /* SPITFIRE */
661 		sys_tick_freq = cpunodes[CPU->cpu_id].clock_freq;
662 #endif
663 	}
664 }
665 
666 
667 void shipit(int upaid);
668 extern uint64_t xc_tick_limit;
669 extern uint64_t xc_tick_jump_limit;
670 
671 #ifdef SEND_MONDO_STATS
672 uint64_t x_early[NCPU][64];
673 #endif
674 
675 /*
676  * Note: A version of this function is used by the debugger via the KDI,
677  * and must be kept in sync with this version.  Any changes made to this
678  * function to support new chips or to accomodate errata must also be included
679  * in the KDI-specific version.  See spitfire_kdi.c.
680  */
681 void
682 send_one_mondo(int cpuid)
683 {
684 	uint64_t idsr, starttick, endtick;
685 	int upaid, busy, nack;
686 	uint64_t tick, tick_prev;
687 	ulong_t ticks;
688 
689 	CPU_STATS_ADDQ(CPU, sys, xcalls, 1);
690 	upaid = CPUID_TO_UPAID(cpuid);
691 	tick = starttick = gettick();
692 	shipit(upaid);
693 	endtick = starttick + xc_tick_limit;
694 	busy = nack = 0;
695 	for (;;) {
696 		idsr = getidsr();
697 		if (idsr == 0)
698 			break;
699 		/*
700 		 * When we detect an irregular tick jump, we adjust
701 		 * the timer window to the current tick value.
702 		 */
703 		tick_prev = tick;
704 		tick = gettick();
705 		ticks = tick - tick_prev;
706 		if (ticks > xc_tick_jump_limit) {
707 			endtick = tick + xc_tick_limit;
708 		} else if (tick > endtick) {
709 			if (panic_quiesce)
710 				return;
711 			cmn_err(CE_PANIC,
712 			"send mondo timeout (target 0x%x) [%d NACK %d BUSY]",
713 			upaid, nack, busy);
714 		}
715 		if (idsr & IDSR_BUSY) {
716 			busy++;
717 			continue;
718 		}
719 		drv_usecwait(1);
720 		shipit(upaid);
721 		nack++;
722 		busy = 0;
723 	}
724 #ifdef SEND_MONDO_STATS
725 	x_early[getprocessorid()][highbit(gettick() - starttick) - 1]++;
726 #endif
727 }
728 
729 void
730 send_mondo_set(cpuset_t set)
731 {
732 	int i;
733 
734 	for (i = 0; i < NCPU; i++)
735 		if (CPU_IN_SET(set, i)) {
736 			send_one_mondo(i);
737 			CPUSET_DEL(set, i);
738 			if (CPUSET_ISNULL(set))
739 				break;
740 		}
741 }
742 
743 void
744 syncfpu(void)
745 {
746 }
747 
748 /*
749  * Determine the size of the CPU module's error structure in bytes.  This is
750  * called once during boot to initialize the error queues.
751  */
752 int
753 cpu_aflt_size(void)
754 {
755 	/*
756 	 * We need to determine whether this is a sabre, Hummingbird or a
757 	 * Spitfire/Blackbird impl and set the appropriate state variables for
758 	 * ecache tag manipulation.  We can't do this in cpu_setup() as it is
759 	 * too early in the boot flow and the cpunodes are not initialized.
760 	 * This routine will be called once after cpunodes[] is ready, so do
761 	 * it here.
762 	 */
763 	if (cpunodes[CPU->cpu_id].implementation == SABRE_IMPL) {
764 		isus2i = 1;
765 		cpu_ec_tag_mask = SB_ECTAG_MASK;
766 		cpu_ec_state_mask = SB_ECSTATE_MASK;
767 		cpu_ec_par_mask = SB_ECPAR_MASK;
768 		cpu_ec_par_shift = SB_ECPAR_SHIFT;
769 		cpu_ec_tag_shift = SB_ECTAG_SHIFT;
770 		cpu_ec_state_shift = SB_ECSTATE_SHIFT;
771 		cpu_ec_state_exl = SB_ECSTATE_EXL;
772 		cpu_ec_state_mod = SB_ECSTATE_MOD;
773 
774 		/* These states do not exist in sabre - set to 0xFF */
775 		cpu_ec_state_shr = 0xFF;
776 		cpu_ec_state_own = 0xFF;
777 
778 		cpu_ec_state_valid = SB_ECSTATE_VALID;
779 		cpu_ec_state_dirty = SB_ECSTATE_DIRTY;
780 		cpu_ec_state_parity = SB_ECSTATE_PARITY;
781 		cpu_ec_parity = SB_EC_PARITY;
782 	} else if (cpunodes[CPU->cpu_id].implementation == HUMMBRD_IMPL) {
783 		isus2e = 1;
784 		cpu_ec_tag_mask = HB_ECTAG_MASK;
785 		cpu_ec_state_mask = HB_ECSTATE_MASK;
786 		cpu_ec_par_mask = HB_ECPAR_MASK;
787 		cpu_ec_par_shift = HB_ECPAR_SHIFT;
788 		cpu_ec_tag_shift = HB_ECTAG_SHIFT;
789 		cpu_ec_state_shift = HB_ECSTATE_SHIFT;
790 		cpu_ec_state_exl = HB_ECSTATE_EXL;
791 		cpu_ec_state_mod = HB_ECSTATE_MOD;
792 
793 		/* These states do not exist in hummingbird - set to 0xFF */
794 		cpu_ec_state_shr = 0xFF;
795 		cpu_ec_state_own = 0xFF;
796 
797 		cpu_ec_state_valid = HB_ECSTATE_VALID;
798 		cpu_ec_state_dirty = HB_ECSTATE_DIRTY;
799 		cpu_ec_state_parity = HB_ECSTATE_PARITY;
800 		cpu_ec_parity = HB_EC_PARITY;
801 	}
802 
803 	return (sizeof (spitf_async_flt));
804 }
805 
806 
807 /*
808  * Correctable ecc error trap handler
809  */
810 /*ARGSUSED*/
811 void
812 cpu_ce_error(struct regs *rp, ulong_t p_afar, ulong_t p_afsr,
813 	uint_t p_afsr_high, uint_t p_afar_high)
814 {
815 	ushort_t sdbh, sdbl;
816 	ushort_t e_syndh, e_syndl;
817 	spitf_async_flt spf_flt;
818 	struct async_flt *ecc;
819 	int queue = 1;
820 
821 	uint64_t t_afar = p_afar;
822 	uint64_t t_afsr = p_afsr;
823 
824 	/*
825 	 * Note: the Spitfire data buffer error registers
826 	 * (upper and lower halves) are or'ed into the upper
827 	 * word of the afsr by ce_err().
828 	 */
829 	sdbh = (ushort_t)((t_afsr >> 33) & 0x3FF);
830 	sdbl = (ushort_t)((t_afsr >> 43) & 0x3FF);
831 
832 	e_syndh = (uchar_t)(sdbh & (uint_t)P_DER_E_SYND);
833 	e_syndl = (uchar_t)(sdbl & (uint_t)P_DER_E_SYND);
834 
835 	t_afsr &= S_AFSR_MASK;
836 	t_afar &= SABRE_AFAR_PA;	/* must use Sabre AFAR mask */
837 
838 	/* Setup the async fault structure */
839 	bzero(&spf_flt, sizeof (spitf_async_flt));
840 	ecc = (struct async_flt *)&spf_flt;
841 	ecc->flt_id = gethrtime_waitfree();
842 	ecc->flt_stat = t_afsr;
843 	ecc->flt_addr = t_afar;
844 	ecc->flt_status = ECC_C_TRAP;
845 	ecc->flt_bus_id = getprocessorid();
846 	ecc->flt_inst = CPU->cpu_id;
847 	ecc->flt_pc = (caddr_t)rp->r_pc;
848 	ecc->flt_func = log_ce_err;
849 	ecc->flt_in_memory =
850 		(pf_is_memory(ecc->flt_addr >> MMU_PAGESHIFT)) ? 1: 0;
851 	spf_flt.flt_sdbh = sdbh;
852 	spf_flt.flt_sdbl = sdbl;
853 
854 	/*
855 	 * Check for fatal conditions.
856 	 */
857 	check_misc_err(&spf_flt);
858 
859 	/*
860 	 * Pananoid checks for valid AFSR and UDBs
861 	 */
862 	if ((t_afsr & P_AFSR_CE) == 0) {
863 		cpu_aflt_log(CE_PANIC, 1, &spf_flt, CMN_LFLAGS,
864 			"** Panic due to CE bit not set in the AFSR",
865 			"  Corrected Memory Error on");
866 	}
867 
868 	/*
869 	 * We want to skip logging only if ALL the following
870 	 * conditions are true:
871 	 *
872 	 *	1. There is only one error
873 	 *	2. That error is a correctable memory error
874 	 *	3. The error is caused by the memory scrubber (in which case
875 	 *	    the error will have occurred under on_trap protection)
876 	 *	4. The error is on a retired page
877 	 *
878 	 * Note: OT_DATA_EC is used places other than the memory scrubber.
879 	 * However, none of those errors should occur on a retired page.
880 	 */
881 	if ((ecc->flt_stat & (S_AFSR_ALL_ERRS & ~P_AFSR_ME)) == P_AFSR_CE &&
882 	    curthread->t_ontrap != NULL) {
883 
884 		if (curthread->t_ontrap->ot_prot & OT_DATA_EC) {
885 			if (page_retire_check(ecc->flt_addr, NULL) == 0) {
886 				queue = 0;
887 			}
888 		}
889 	}
890 
891 	if (((sdbh & P_DER_CE) == 0) && ((sdbl & P_DER_CE) == 0)) {
892 		cpu_aflt_log(CE_PANIC, 1, &spf_flt, CMN_LFLAGS,
893 			"** Panic due to CE bits not set in the UDBs",
894 			" Corrected Memory Error on");
895 	}
896 
897 	if ((sdbh >> 8) & 1) {
898 		ecc->flt_synd = e_syndh;
899 		ce_scrub(ecc);
900 		if (queue) {
901 			cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CE, ecc,
902 			    sizeof (*ecc), ce_queue, ERRORQ_ASYNC);
903 		}
904 	}
905 
906 	if ((sdbl >> 8) & 1) {
907 		ecc->flt_addr = t_afar | 0x8;	/* Sabres do not have a UDBL */
908 		ecc->flt_synd = e_syndl | UDBL_REG;
909 		ce_scrub(ecc);
910 		if (queue) {
911 			cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CE, ecc,
912 			    sizeof (*ecc), ce_queue, ERRORQ_ASYNC);
913 		}
914 	}
915 
916 	/*
917 	 * Re-enable all error trapping (CEEN currently cleared).
918 	 */
919 	clr_datapath();
920 	set_asyncflt(P_AFSR_CE);
921 	set_error_enable(EER_ENABLE);
922 }
923 
924 /*
925  * Cpu specific CE logging routine
926  */
927 static void
928 log_ce_err(struct async_flt *aflt, char *unum)
929 {
930 	spitf_async_flt spf_flt;
931 
932 	if ((aflt->flt_stat & P_AFSR_CE) && (ce_verbose_memory == 0)) {
933 		return;
934 	}
935 
936 	spf_flt.cmn_asyncflt = *aflt;
937 	cpu_aflt_log(CE_CONT, 0, &spf_flt, CE_LFLAGS, unum,
938 	    " Corrected Memory Error detected by");
939 }
940 
941 /*
942  * Spitfire does not perform any further CE classification refinement
943  */
944 /*ARGSUSED*/
945 int
946 ce_scrub_xdiag_recirc(struct async_flt *ecc, errorq_t *eqp, errorq_elem_t *eqep,
947     size_t afltoffset)
948 {
949 	return (0);
950 }
951 
952 char *
953 flt_to_error_type(struct async_flt *aflt)
954 {
955 	if (aflt->flt_status & ECC_INTERMITTENT)
956 		return (ERR_TYPE_DESC_INTERMITTENT);
957 	if (aflt->flt_status & ECC_PERSISTENT)
958 		return (ERR_TYPE_DESC_PERSISTENT);
959 	if (aflt->flt_status & ECC_STICKY)
960 		return (ERR_TYPE_DESC_STICKY);
961 	return (ERR_TYPE_DESC_UNKNOWN);
962 }
963 
964 /*
965  * Called by correctable ecc error logging code to print out
966  * the stick/persistent/intermittent status of the error.
967  */
968 static void
969 cpu_ce_log_status(spitf_async_flt *spf_flt, char *unum)
970 {
971 	ushort_t status;
972 	char *status1_str = "Memory";
973 	char *status2_str = "Intermittent";
974 	struct async_flt *aflt = (struct async_flt *)spf_flt;
975 
976 	status = aflt->flt_status;
977 
978 	if (status & ECC_ECACHE)
979 		status1_str = "Ecache";
980 
981 	if (status & ECC_STICKY)
982 		status2_str = "Sticky";
983 	else if (status & ECC_PERSISTENT)
984 		status2_str = "Persistent";
985 
986 	cpu_aflt_log(CE_CONT, 0, spf_flt, CPU_ERRID_FIRST,
987 		NULL, " Corrected %s Error on %s is %s",
988 		status1_str, unum, status2_str);
989 }
990 
991 /*
992  * check for a valid ce syndrome, then call the
993  * displacement flush scrubbing code, and then check the afsr to see if
994  * the error was persistent or intermittent. Reread the afar/afsr to see
995  * if the error was not scrubbed successfully, and is therefore sticky.
996  */
997 /*ARGSUSED1*/
998 void
999 cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t triedcpulogout)
1000 {
1001 	uint64_t eer, afsr;
1002 	ushort_t status;
1003 
1004 	ASSERT(getpil() > LOCK_LEVEL);
1005 
1006 	/*
1007 	 * It is possible that the flt_addr is not a valid
1008 	 * physical address. To deal with this, we disable
1009 	 * NCEEN while we scrub that address. If this causes
1010 	 * a TIMEOUT/BERR, we know this is an invalid
1011 	 * memory location.
1012 	 */
1013 	kpreempt_disable();
1014 	eer = get_error_enable();
1015 	if (eer & (EER_CEEN | EER_NCEEN))
1016 	    set_error_enable(eer & ~(EER_CEEN | EER_NCEEN));
1017 
1018 	/*
1019 	 * To check if the error detected by IO is persistent, sticky or
1020 	 * intermittent.
1021 	 */
1022 	if (ecc->flt_status & ECC_IOBUS) {
1023 		ecc->flt_stat = P_AFSR_CE;
1024 	}
1025 
1026 	scrubphys(P2ALIGN(ecc->flt_addr, 64),
1027 	    cpunodes[CPU->cpu_id].ecache_size);
1028 
1029 	get_asyncflt(&afsr);
1030 	if (afsr & (P_AFSR_TO | P_AFSR_BERR)) {
1031 		/*
1032 		 * Must ensure that we don't get the TIMEOUT/BERR
1033 		 * when we reenable NCEEN, so we clear the AFSR.
1034 		 */
1035 		set_asyncflt(afsr & (P_AFSR_TO | P_AFSR_BERR));
1036 		if (eer & (EER_CEEN | EER_NCEEN))
1037 		    set_error_enable(eer);
1038 		kpreempt_enable();
1039 		return;
1040 	}
1041 
1042 	if (eer & EER_NCEEN)
1043 	    set_error_enable(eer & ~EER_CEEN);
1044 
1045 	/*
1046 	 * Check and clear any ECC errors from the scrub.  If the scrub did
1047 	 * not trip over the error, mark it intermittent.  If the scrub did
1048 	 * trip the error again and it did not scrub away, mark it sticky.
1049 	 * Otherwise mark it persistent.
1050 	 */
1051 	if (check_ecc(ecc) != 0) {
1052 		cpu_read_paddr(ecc, 0, 1);
1053 
1054 		if (check_ecc(ecc) != 0)
1055 			status = ECC_STICKY;
1056 		else
1057 			status = ECC_PERSISTENT;
1058 	} else
1059 		status = ECC_INTERMITTENT;
1060 
1061 	if (eer & (EER_CEEN | EER_NCEEN))
1062 	    set_error_enable(eer);
1063 	kpreempt_enable();
1064 
1065 	ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY);
1066 	ecc->flt_status |= status;
1067 }
1068 
1069 /*
1070  * get the syndrome and unum, and then call the routines
1071  * to check the other cpus and iobuses, and then do the error logging.
1072  */
1073 /*ARGSUSED1*/
1074 void
1075 cpu_ce_log_err(struct async_flt *ecc, errorq_elem_t *eqep)
1076 {
1077 	char unum[UNUM_NAMLEN];
1078 	int len = 0;
1079 	int ce_verbose = 0;
1080 	int err;
1081 
1082 	ASSERT(ecc->flt_func != NULL);
1083 
1084 	/* Get the unum string for logging purposes */
1085 	(void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, ecc, unum,
1086 	    UNUM_NAMLEN, &len);
1087 
1088 	/* Call specific error logging routine */
1089 	(void) (*ecc->flt_func)(ecc, unum);
1090 
1091 	/*
1092 	 * Count errors per unum.
1093 	 * Non-memory errors are all counted via a special unum string.
1094 	 */
1095 	if ((err = ce_count_unum(ecc->flt_status, len, unum)) != PR_OK &&
1096 	    automatic_page_removal) {
1097 		(void) page_retire(ecc->flt_addr, err);
1098 	}
1099 
1100 	if (ecc->flt_panic) {
1101 		ce_verbose = 1;
1102 	} else if ((ecc->flt_class == BUS_FAULT) ||
1103 	    (ecc->flt_stat & P_AFSR_CE)) {
1104 		ce_verbose = (ce_verbose_memory > 0);
1105 	} else {
1106 		ce_verbose = 1;
1107 	}
1108 
1109 	if (ce_verbose) {
1110 		spitf_async_flt sflt;
1111 		int synd_code;
1112 
1113 		sflt.cmn_asyncflt = *ecc;	/* for cpu_aflt_log() */
1114 
1115 		cpu_ce_log_status(&sflt, unum);
1116 
1117 		synd_code = synd_to_synd_code(AFLT_STAT_VALID,
1118 				SYND(ecc->flt_synd));
1119 
1120 		if (SYND_IS_SINGLE_BIT_DATA(synd_code)) {
1121 			cpu_aflt_log(CE_CONT, 0, &sflt, CPU_ERRID_FIRST,
1122 			    NULL, " ECC Data Bit %2d was in error "
1123 			    "and corrected", synd_code);
1124 		} else if (SYND_IS_SINGLE_BIT_CHK(synd_code)) {
1125 			cpu_aflt_log(CE_CONT, 0, &sflt, CPU_ERRID_FIRST,
1126 			    NULL, " ECC Check Bit %2d was in error "
1127 			    "and corrected", synd_code - C0);
1128 		} else {
1129 			/*
1130 			 * These are UE errors - we shouldn't be getting CE
1131 			 * traps for these; handle them in case of bad h/w.
1132 			 */
1133 			switch (synd_code) {
1134 			case M2:
1135 				cpu_aflt_log(CE_CONT, 0, &sflt,
1136 				    CPU_ERRID_FIRST, NULL,
1137 				    " Two ECC Bits were in error");
1138 				break;
1139 			case M3:
1140 				cpu_aflt_log(CE_CONT, 0, &sflt,
1141 				    CPU_ERRID_FIRST, NULL,
1142 				    " Three ECC Bits were in error");
1143 				break;
1144 			case M4:
1145 				cpu_aflt_log(CE_CONT, 0, &sflt,
1146 				    CPU_ERRID_FIRST, NULL,
1147 				    " Four ECC Bits were in error");
1148 				break;
1149 			case MX:
1150 				cpu_aflt_log(CE_CONT, 0, &sflt,
1151 				    CPU_ERRID_FIRST, NULL,
1152 				    " More than Four ECC bits were "
1153 				    "in error");
1154 				break;
1155 			default:
1156 				cpu_aflt_log(CE_CONT, 0, &sflt,
1157 				    CPU_ERRID_FIRST, NULL,
1158 				    " Unknown fault syndrome %d",
1159 				    synd_code);
1160 				break;
1161 			}
1162 		}
1163 	}
1164 
1165 	/* Display entire cache line, if valid address */
1166 	if (ce_show_data && ecc->flt_addr != AFLT_INV_ADDR)
1167 		read_ecc_data(ecc, 1, 1);
1168 }
1169 
1170 /*
1171  * We route all errors through a single switch statement.
1172  */
1173 void
1174 cpu_ue_log_err(struct async_flt *aflt)
1175 {
1176 
1177 	switch (aflt->flt_class) {
1178 	case CPU_FAULT:
1179 		cpu_async_log_err(aflt);
1180 		break;
1181 
1182 	case BUS_FAULT:
1183 		bus_async_log_err(aflt);
1184 		break;
1185 
1186 	default:
1187 		cmn_err(CE_WARN, "discarding async error 0x%p with invalid "
1188 		    "fault class (0x%x)", (void *)aflt, aflt->flt_class);
1189 		break;
1190 	}
1191 }
1192 
1193 /* Values for action variable in cpu_async_error() */
1194 #define	ACTION_NONE		0
1195 #define	ACTION_TRAMPOLINE	1
1196 #define	ACTION_AST_FLAGS	2
1197 
1198 /*
1199  * Access error trap handler for asynchronous cpu errors.  This routine is
1200  * called to handle a data or instruction access error.  All fatal errors are
1201  * completely handled by this routine (by panicking).  Non fatal error logging
1202  * is queued for later processing either via AST or softint at a lower PIL.
1203  * In case of panic, the error log queue will also be processed as part of the
1204  * panic flow to ensure all errors are logged.  This routine is called with all
1205  * errors disabled at PIL15.  The AFSR bits are cleared and the UDBL and UDBH
1206  * error bits are also cleared.  The hardware has also disabled the I and
1207  * D-caches for us, so we must re-enable them before returning.
1208  *
1209  * A summary of the handling of tl=0 UE/LDP/EDP/TO/BERR/WP/CP:
1210  *
1211  *		_______________________________________________________________
1212  *		|        Privileged tl0		|         Unprivileged	      |
1213  *		| Protected	| Unprotected	| Protected	| Unprotected |
1214  *		|on_trap|lofault|		|		|	      |
1215  * -------------|-------|-------+---------------+---------------+-------------|
1216  *		|	|	|		|		|	      |
1217  * UE/LDP/EDP	| L,T,p	| L,R,p	| L,P		| n/a		| L,R,p	      |
1218  *		|	|	|		|		|	      |
1219  * TO/BERR	| T	| S	| L,P		| n/a		| S	      |
1220  *		|	|	|		|		|	      |
1221  * WP		| L,M,p | L,M,p	| L,M,p		| n/a		| L,M,p       |
1222  *		|	|	|		|		|	      |
1223  * CP (IIi/IIe)	| L,P	| L,P	| L,P		| n/a		| L,P	      |
1224  * ____________________________________________________________________________
1225  *
1226  *
1227  * Action codes:
1228  *
1229  * L - log
1230  * M - kick off memscrubber if flt_in_memory
1231  * P - panic
1232  * p - panic if US-IIi or US-IIe (Sabre); overrides R and M
1233  * R - i)  if aft_panic is set, panic
1234  *     ii) otherwise, send hwerr event to contract and SIGKILL to process
1235  * S - send SIGBUS to process
1236  * T - trampoline
1237  *
1238  * Special cases:
1239  *
1240  * 1) if aft_testfatal is set, all faults result in a panic regardless
1241  *    of type (even WP), protection (even on_trap), or privilege.
1242  */
1243 /*ARGSUSED*/
1244 void
1245 cpu_async_error(struct regs *rp, ulong_t p_afar, ulong_t p_afsr,
1246 	uint_t p_afsr_high, uint_t p_afar_high)
1247 {
1248 	ushort_t sdbh, sdbl, ttype, tl;
1249 	spitf_async_flt spf_flt;
1250 	struct async_flt *aflt;
1251 	char pr_reason[28];
1252 	uint64_t oafsr;
1253 	uint64_t acc_afsr = 0;			/* accumulated afsr */
1254 	int action = ACTION_NONE;
1255 	uint64_t t_afar = p_afar;
1256 	uint64_t t_afsr = p_afsr;
1257 	int expected = DDI_FM_ERR_UNEXPECTED;
1258 	ddi_acc_hdl_t *hp;
1259 
1260 	/*
1261 	 * We need to look at p_flag to determine if the thread detected an
1262 	 * error while dumping core.  We can't grab p_lock here, but it's ok
1263 	 * because we just need a consistent snapshot and we know that everyone
1264 	 * else will store a consistent set of bits while holding p_lock.  We
1265 	 * don't have to worry about a race because SDOCORE is set once prior
1266 	 * to doing i/o from the process's address space and is never cleared.
1267 	 */
1268 	uint_t pflag = ttoproc(curthread)->p_flag;
1269 
1270 	pr_reason[0] = '\0';
1271 
1272 	/*
1273 	 * Note: the Spitfire data buffer error registers
1274 	 * (upper and lower halves) are or'ed into the upper
1275 	 * word of the afsr by async_err() if P_AFSR_UE is set.
1276 	 */
1277 	sdbh = (ushort_t)((t_afsr >> 33) & 0x3FF);
1278 	sdbl = (ushort_t)((t_afsr >> 43) & 0x3FF);
1279 
1280 	/*
1281 	 * Grab the ttype encoded in <63:53> of the saved
1282 	 * afsr passed from async_err()
1283 	 */
1284 	ttype = (ushort_t)((t_afsr >> 53) & 0x1FF);
1285 	tl = (ushort_t)(t_afsr >> 62);
1286 
1287 	t_afsr &= S_AFSR_MASK;
1288 	t_afar &= SABRE_AFAR_PA;	/* must use Sabre AFAR mask */
1289 
1290 	/*
1291 	 * Initialize most of the common and CPU-specific structure.  We derive
1292 	 * aflt->flt_priv from %tstate, instead of from the AFSR.PRIV bit.  The
1293 	 * initial setting of aflt->flt_panic is based on TL: we must panic if
1294 	 * the error occurred at TL > 0.  We also set flt_panic if the test/demo
1295 	 * tuneable aft_testfatal is set (not the default).
1296 	 */
1297 	bzero(&spf_flt, sizeof (spitf_async_flt));
1298 	aflt = (struct async_flt *)&spf_flt;
1299 	aflt->flt_id = gethrtime_waitfree();
1300 	aflt->flt_stat = t_afsr;
1301 	aflt->flt_addr = t_afar;
1302 	aflt->flt_bus_id = getprocessorid();
1303 	aflt->flt_inst = CPU->cpu_id;
1304 	aflt->flt_pc = (caddr_t)rp->r_pc;
1305 	aflt->flt_prot = AFLT_PROT_NONE;
1306 	aflt->flt_class = CPU_FAULT;
1307 	aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0;
1308 	aflt->flt_tl = (uchar_t)tl;
1309 	aflt->flt_panic = (tl != 0 || aft_testfatal != 0);
1310 	aflt->flt_core = (pflag & SDOCORE) ? 1 : 0;
1311 
1312 	/*
1313 	 * Set flt_status based on the trap type.  If we end up here as the
1314 	 * result of a UE detected by the CE handling code, leave status 0.
1315 	 */
1316 	switch (ttype) {
1317 	case T_DATA_ERROR:
1318 		aflt->flt_status = ECC_D_TRAP;
1319 		break;
1320 	case T_INSTR_ERROR:
1321 		aflt->flt_status = ECC_I_TRAP;
1322 		break;
1323 	}
1324 
1325 	spf_flt.flt_sdbh = sdbh;
1326 	spf_flt.flt_sdbl = sdbl;
1327 
1328 	/*
1329 	 * Check for fatal async errors.
1330 	 */
1331 	check_misc_err(&spf_flt);
1332 
1333 	/*
1334 	 * If the trap occurred in privileged mode at TL=0, we need to check to
1335 	 * see if we were executing in the kernel under on_trap() or t_lofault
1336 	 * protection.  If so, modify the saved registers so that we return
1337 	 * from the trap to the appropriate trampoline routine.
1338 	 */
1339 	if (aflt->flt_priv && tl == 0) {
1340 		if (curthread->t_ontrap != NULL) {
1341 			on_trap_data_t *otp = curthread->t_ontrap;
1342 
1343 			if (otp->ot_prot & OT_DATA_EC) {
1344 				aflt->flt_prot = AFLT_PROT_EC;
1345 				otp->ot_trap |= OT_DATA_EC;
1346 				rp->r_pc = otp->ot_trampoline;
1347 				rp->r_npc = rp->r_pc + 4;
1348 				action = ACTION_TRAMPOLINE;
1349 			}
1350 
1351 			if ((t_afsr & (P_AFSR_TO | P_AFSR_BERR)) &&
1352 			    (otp->ot_prot & OT_DATA_ACCESS)) {
1353 				aflt->flt_prot = AFLT_PROT_ACCESS;
1354 				otp->ot_trap |= OT_DATA_ACCESS;
1355 				rp->r_pc = otp->ot_trampoline;
1356 				rp->r_npc = rp->r_pc + 4;
1357 				action = ACTION_TRAMPOLINE;
1358 				/*
1359 				 * for peeks and caut_gets errors are expected
1360 				 */
1361 				hp = (ddi_acc_hdl_t *)otp->ot_handle;
1362 				if (!hp)
1363 					expected = DDI_FM_ERR_PEEK;
1364 				else if (hp->ah_acc.devacc_attr_access ==
1365 				    DDI_CAUTIOUS_ACC)
1366 					expected = DDI_FM_ERR_EXPECTED;
1367 			}
1368 
1369 		} else if (curthread->t_lofault) {
1370 			aflt->flt_prot = AFLT_PROT_COPY;
1371 			rp->r_g1 = EFAULT;
1372 			rp->r_pc = curthread->t_lofault;
1373 			rp->r_npc = rp->r_pc + 4;
1374 			action = ACTION_TRAMPOLINE;
1375 		}
1376 	}
1377 
1378 	/*
1379 	 * Determine if this error needs to be treated as fatal.  Note that
1380 	 * multiple errors detected upon entry to this trap handler does not
1381 	 * necessarily warrant a panic.  We only want to panic if the trap
1382 	 * happened in privileged mode and not under t_ontrap or t_lofault
1383 	 * protection.  The exception is WP: if we *only* get WP, it is not
1384 	 * fatal even if the trap occurred in privileged mode, except on Sabre.
1385 	 *
1386 	 * aft_panic, if set, effectively makes us treat usermode
1387 	 * UE/EDP/LDP faults as if they were privileged - so we we will
1388 	 * panic instead of sending a contract event.  A lofault-protected
1389 	 * fault will normally follow the contract event; if aft_panic is
1390 	 * set this will be changed to a panic.
1391 	 *
1392 	 * For usermode BERR/BTO errors, eg from processes performing device
1393 	 * control through mapped device memory, we need only deliver
1394 	 * a SIGBUS to the offending process.
1395 	 *
1396 	 * Some additional flt_panic reasons (eg, WP on Sabre) will be
1397 	 * checked later; for now we implement the common reasons.
1398 	 */
1399 	if (aflt->flt_prot == AFLT_PROT_NONE) {
1400 		/*
1401 		 * Beware - multiple bits may be set in AFSR
1402 		 */
1403 		if (t_afsr & (P_AFSR_UE | P_AFSR_LDP | P_AFSR_EDP)) {
1404 			if (aflt->flt_priv || aft_panic)
1405 				aflt->flt_panic = 1;
1406 		}
1407 
1408 		if (t_afsr & (P_AFSR_TO | P_AFSR_BERR)) {
1409 			if (aflt->flt_priv)
1410 				aflt->flt_panic = 1;
1411 		}
1412 	} else if (aflt->flt_prot == AFLT_PROT_COPY && aft_panic) {
1413 		aflt->flt_panic = 1;
1414 	}
1415 
1416 	/*
1417 	 * UE/BERR/TO: Call our bus nexus friends to check for
1418 	 * IO errors that may have resulted in this trap.
1419 	 */
1420 	if (t_afsr & (P_AFSR_TO | P_AFSR_BERR | P_AFSR_UE)) {
1421 		cpu_run_bus_error_handlers(aflt, expected);
1422 	}
1423 
1424 	/*
1425 	 * Handle UE: If the UE is in memory, we need to flush the bad line from
1426 	 * the E-cache.  We also need to query the bus nexus for fatal errors.
1427 	 * For sabre, we will panic on UEs. Attempts to do diagnostic read on
1428 	 * caches may introduce more parity errors (especially when the module
1429 	 * is bad) and in sabre there is no guarantee that such errors
1430 	 * (if introduced) are written back as poisoned data.
1431 	 */
1432 	if (t_afsr & P_AFSR_UE) {
1433 		int i;
1434 
1435 		(void) strcat(pr_reason, "UE ");
1436 
1437 		spf_flt.flt_type = CPU_UE_ERR;
1438 		aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >>
1439 			MMU_PAGESHIFT)) ? 1: 0;
1440 
1441 		/*
1442 		 * With UE, we have the PA of the fault.
1443 		 * Let do a diagnostic read to get the ecache
1444 		 * data and tag info of the bad line for logging.
1445 		 */
1446 		if (aflt->flt_in_memory) {
1447 			uint32_t ec_set_size;
1448 			uchar_t state;
1449 			uint32_t ecache_idx;
1450 			uint64_t faultpa = P2ALIGN(aflt->flt_addr, 64);
1451 
1452 			/* touch the line to put it in ecache */
1453 			acc_afsr |= read_and_clear_afsr();
1454 			(void) lddphys(faultpa);
1455 			acc_afsr |= (read_and_clear_afsr() &
1456 				    ~(P_AFSR_EDP | P_AFSR_UE));
1457 
1458 			ec_set_size = cpunodes[CPU->cpu_id].ecache_size /
1459 			    ecache_associativity;
1460 
1461 			for (i = 0; i < ecache_associativity; i++) {
1462 				ecache_idx = i * ec_set_size +
1463 				    (aflt->flt_addr % ec_set_size);
1464 				get_ecache_dtag(P2ALIGN(ecache_idx, 64),
1465 					(uint64_t *)&spf_flt.flt_ec_data[0],
1466 					&spf_flt.flt_ec_tag, &oafsr, &acc_afsr);
1467 				acc_afsr |= oafsr;
1468 
1469 				state = (uchar_t)((spf_flt.flt_ec_tag &
1470 				    cpu_ec_state_mask) >> cpu_ec_state_shift);
1471 
1472 				if ((state & cpu_ec_state_valid) &&
1473 				    ((spf_flt.flt_ec_tag & cpu_ec_tag_mask) ==
1474 				    ((uint64_t)aflt->flt_addr >>
1475 				    cpu_ec_tag_shift)))
1476 					break;
1477 			}
1478 
1479 			/*
1480 			 * Check to see if the ecache tag is valid for the
1481 			 * fault PA. In the very unlikely event where the
1482 			 * line could be victimized, no ecache info will be
1483 			 * available. If this is the case, capture the line
1484 			 * from memory instead.
1485 			 */
1486 			if ((state & cpu_ec_state_valid) == 0 ||
1487 			    (spf_flt.flt_ec_tag & cpu_ec_tag_mask) !=
1488 			    ((uint64_t)aflt->flt_addr >> cpu_ec_tag_shift)) {
1489 				for (i = 0; i < 8; i++, faultpa += 8) {
1490 				    ec_data_t *ecdptr;
1491 
1492 					ecdptr = &spf_flt.flt_ec_data[i];
1493 					acc_afsr |= read_and_clear_afsr();
1494 					ecdptr->ec_d8 = lddphys(faultpa);
1495 					acc_afsr |= (read_and_clear_afsr() &
1496 						    ~(P_AFSR_EDP | P_AFSR_UE));
1497 					ecdptr->ec_afsr = 0;
1498 							/* null afsr value */
1499 				}
1500 
1501 				/*
1502 				 * Mark tag invalid to indicate mem dump
1503 				 * when we print out the info.
1504 				 */
1505 				spf_flt.flt_ec_tag = AFLT_INV_ADDR;
1506 			}
1507 			spf_flt.flt_ec_lcnt = 1;
1508 
1509 			/*
1510 			 * Flush out the bad line
1511 			 */
1512 			flushecacheline(P2ALIGN(aflt->flt_addr, 64),
1513 				cpunodes[CPU->cpu_id].ecache_size);
1514 
1515 			acc_afsr |= clear_errors(NULL, NULL);
1516 		}
1517 
1518 		/*
1519 		 * Ask our bus nexus friends if they have any fatal errors. If
1520 		 * so, they will log appropriate error messages and panic as a
1521 		 * result. We then queue an event for each UDB that reports a
1522 		 * UE. Each UE reported in a UDB will have its own log message.
1523 		 *
1524 		 * Note from kbn: In the case where there are multiple UEs
1525 		 * (ME bit is set) - the AFAR address is only accurate to
1526 		 * the 16-byte granularity. One cannot tell whether the AFAR
1527 		 * belongs to the UDBH or UDBL syndromes. In this case, we
1528 		 * always report the AFAR address to be 16-byte aligned.
1529 		 *
1530 		 * If we're on a Sabre, there is no SDBL, but it will always
1531 		 * read as zero, so the sdbl test below will safely fail.
1532 		 */
1533 		if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL || isus2i || isus2e)
1534 			aflt->flt_panic = 1;
1535 
1536 		if (sdbh & P_DER_UE) {
1537 			aflt->flt_synd = sdbh & P_DER_E_SYND;
1538 			cpu_errorq_dispatch(FM_EREPORT_CPU_USII_UE,
1539 			    (void *)&spf_flt, sizeof (spf_flt), ue_queue,
1540 			    aflt->flt_panic);
1541 		}
1542 		if (sdbl & P_DER_UE) {
1543 			aflt->flt_synd = sdbl & P_DER_E_SYND;
1544 			aflt->flt_synd |= UDBL_REG;	/* indicates UDBL */
1545 			if (!(aflt->flt_stat & P_AFSR_ME))
1546 				aflt->flt_addr |= 0x8;
1547 			cpu_errorq_dispatch(FM_EREPORT_CPU_USII_UE,
1548 			    (void *)&spf_flt, sizeof (spf_flt), ue_queue,
1549 			    aflt->flt_panic);
1550 		}
1551 
1552 		/*
1553 		 * We got a UE and are panicking, save the fault PA in a known
1554 		 * location so that the platform specific panic code can check
1555 		 * for copyback errors.
1556 		 */
1557 		if (aflt->flt_panic && aflt->flt_in_memory) {
1558 			panic_aflt = *aflt;
1559 		}
1560 	}
1561 
1562 	/*
1563 	 * Handle EDP and LDP: Locate the line with bad parity and enqueue an
1564 	 * async error for logging. For Sabre, we panic on EDP or LDP.
1565 	 */
1566 	if (t_afsr & (P_AFSR_EDP | P_AFSR_LDP)) {
1567 		spf_flt.flt_type = CPU_EDP_LDP_ERR;
1568 
1569 		if (t_afsr & P_AFSR_EDP)
1570 			(void) strcat(pr_reason, "EDP ");
1571 
1572 		if (t_afsr & P_AFSR_LDP)
1573 			(void) strcat(pr_reason, "LDP ");
1574 
1575 		/*
1576 		 * Here we have no PA to work with.
1577 		 * Scan each line in the ecache to look for
1578 		 * the one with bad parity.
1579 		 */
1580 		aflt->flt_addr = AFLT_INV_ADDR;
1581 		scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0],
1582 			&spf_flt.flt_ec_tag, &spf_flt.flt_ec_lcnt, &oafsr);
1583 		acc_afsr |= (oafsr & ~P_AFSR_WP);
1584 
1585 		/*
1586 		 * If we found a bad PA, update the state to indicate if it is
1587 		 * memory or I/O space.  This code will be important if we ever
1588 		 * support cacheable frame buffers.
1589 		 */
1590 		if (aflt->flt_addr != AFLT_INV_ADDR) {
1591 			aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >>
1592 				MMU_PAGESHIFT)) ? 1 : 0;
1593 		}
1594 
1595 		if (isus2i || isus2e)
1596 			aflt->flt_panic = 1;
1597 
1598 		cpu_errorq_dispatch((t_afsr & P_AFSR_EDP) ?
1599 		    FM_EREPORT_CPU_USII_EDP : FM_EREPORT_CPU_USII_LDP,
1600 		    (void *)&spf_flt, sizeof (spf_flt), ue_queue,
1601 		    aflt->flt_panic);
1602 	}
1603 
1604 	/*
1605 	 * Timeout and bus error handling.  There are two cases to consider:
1606 	 *
1607 	 * (1) If we are in the kernel protected by ddi_peek or ddi_poke,we
1608 	 * have already modified the saved registers so that we will return
1609 	 * from the trap to the appropriate trampoline routine; otherwise panic.
1610 	 *
1611 	 * (2) In user mode, we can simply use our AST mechanism to deliver
1612 	 * a SIGBUS.  We do not log the occurence - processes performing
1613 	 * device control would generate lots of uninteresting messages.
1614 	 */
1615 	if (t_afsr & (P_AFSR_TO | P_AFSR_BERR)) {
1616 		if (t_afsr & P_AFSR_TO)
1617 			(void) strcat(pr_reason, "BTO ");
1618 
1619 		if (t_afsr & P_AFSR_BERR)
1620 			(void) strcat(pr_reason, "BERR ");
1621 
1622 		spf_flt.flt_type = CPU_BTO_BERR_ERR;
1623 		if (aflt->flt_priv && aflt->flt_prot == AFLT_PROT_NONE) {
1624 			cpu_errorq_dispatch((t_afsr & P_AFSR_TO) ?
1625 			    FM_EREPORT_CPU_USII_TO : FM_EREPORT_CPU_USII_BERR,
1626 			    (void *)&spf_flt, sizeof (spf_flt), ue_queue,
1627 			    aflt->flt_panic);
1628 		}
1629 	}
1630 
1631 	/*
1632 	 * Handle WP: WP happens when the ecache is victimized and a parity
1633 	 * error was detected on a writeback.  The data in question will be
1634 	 * poisoned as a UE will be written back.  The PA is not logged and
1635 	 * it is possible that it doesn't belong to the trapped thread.  The
1636 	 * WP trap is not fatal, but it could be fatal to someone that
1637 	 * subsequently accesses the toxic page.  We set read_all_memscrub
1638 	 * to force the memscrubber to read all of memory when it awakens.
1639 	 * For Sabre/Hummingbird, WP is fatal because the HW doesn't write a
1640 	 * UE back to poison the data.
1641 	 */
1642 	if (t_afsr & P_AFSR_WP) {
1643 		(void) strcat(pr_reason, "WP ");
1644 		if (isus2i || isus2e) {
1645 			aflt->flt_panic = 1;
1646 		} else {
1647 			read_all_memscrub = 1;
1648 		}
1649 		spf_flt.flt_type = CPU_WP_ERR;
1650 		cpu_errorq_dispatch(FM_EREPORT_CPU_USII_WP,
1651 		    (void *)&spf_flt, sizeof (spf_flt), ue_queue,
1652 		    aflt->flt_panic);
1653 	}
1654 
1655 	/*
1656 	 * Handle trapping CP error: In Sabre/Hummingbird, parity error in
1657 	 * the ecache on a copyout due to a PCI DMA read is signaled as a CP.
1658 	 * This is fatal.
1659 	 */
1660 
1661 	if (t_afsr & P_AFSR_CP) {
1662 		if (isus2i || isus2e) {
1663 			(void) strcat(pr_reason, "CP ");
1664 			aflt->flt_panic = 1;
1665 			spf_flt.flt_type = CPU_TRAPPING_CP_ERR;
1666 			cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CP,
1667 			    (void *)&spf_flt, sizeof (spf_flt), ue_queue,
1668 			    aflt->flt_panic);
1669 		} else {
1670 			/*
1671 			 * Orphan CP: Happens due to signal integrity problem
1672 			 * on a CPU, where a CP is reported, without reporting
1673 			 * its associated UE. This is handled by locating the
1674 			 * bad parity line and would kick off the memscrubber
1675 			 * to find the UE if in memory or in another's cache.
1676 			 */
1677 			spf_flt.flt_type = CPU_ORPHAN_CP_ERR;
1678 			(void) strcat(pr_reason, "ORPHAN_CP ");
1679 
1680 			/*
1681 			 * Here we have no PA to work with.
1682 			 * Scan each line in the ecache to look for
1683 			 * the one with bad parity.
1684 			 */
1685 			aflt->flt_addr = AFLT_INV_ADDR;
1686 			scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0],
1687 				&spf_flt.flt_ec_tag, &spf_flt.flt_ec_lcnt,
1688 				&oafsr);
1689 			acc_afsr |= oafsr;
1690 
1691 			/*
1692 			 * If we found a bad PA, update the state to indicate
1693 			 * if it is memory or I/O space.
1694 			 */
1695 			if (aflt->flt_addr != AFLT_INV_ADDR) {
1696 				aflt->flt_in_memory =
1697 					(pf_is_memory(aflt->flt_addr >>
1698 						MMU_PAGESHIFT)) ? 1 : 0;
1699 			}
1700 			read_all_memscrub = 1;
1701 			cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CP,
1702 			    (void *)&spf_flt, sizeof (spf_flt), ue_queue,
1703 			    aflt->flt_panic);
1704 
1705 		}
1706 	}
1707 
1708 	/*
1709 	 * If we queued an error other than WP or CP and we are going to return
1710 	 * from the trap and the error was in user mode or inside of a
1711 	 * copy routine, set AST flag so the queue will be drained before
1712 	 * returning to user mode.
1713 	 *
1714 	 * For UE/LDP/EDP, the AST processing will SIGKILL the process
1715 	 * and send an event to its process contract.
1716 	 *
1717 	 * For BERR/BTO, the AST processing will SIGBUS the process.  There
1718 	 * will have been no error queued in this case.
1719 	 */
1720 	if ((t_afsr &
1721 	    (P_AFSR_UE | P_AFSR_LDP | P_AFSR_EDP | P_AFSR_BERR | P_AFSR_TO)) &&
1722 	    (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY)) {
1723 			int pcb_flag = 0;
1724 
1725 			if (t_afsr & (P_AFSR_UE | P_AFSR_LDP | P_AFSR_EDP))
1726 				pcb_flag |= ASYNC_HWERR;
1727 
1728 			if (t_afsr & P_AFSR_BERR)
1729 				pcb_flag |= ASYNC_BERR;
1730 
1731 			if (t_afsr & P_AFSR_TO)
1732 				pcb_flag |= ASYNC_BTO;
1733 
1734 			ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag;
1735 			aston(curthread);
1736 			action = ACTION_AST_FLAGS;
1737 	}
1738 
1739 	/*
1740 	 * In response to a deferred error, we must do one of three things:
1741 	 * (1) set the AST flags, (2) trampoline, or (3) panic.  action is
1742 	 * set in cases (1) and (2) - check that either action is set or
1743 	 * (3) is true.
1744 	 *
1745 	 * On II, the WP writes poisoned data back to memory, which will
1746 	 * cause a UE and a panic or reboot when read.  In this case, we
1747 	 * don't need to panic at this time.  On IIi and IIe,
1748 	 * aflt->flt_panic is already set above.
1749 	 */
1750 	ASSERT((aflt->flt_panic != 0) || (action != ACTION_NONE) ||
1751 	    (t_afsr & P_AFSR_WP));
1752 
1753 	/*
1754 	 * Make a final sanity check to make sure we did not get any more async
1755 	 * errors and accumulate the afsr.
1756 	 */
1757 	flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size * 2,
1758 	    cpunodes[CPU->cpu_id].ecache_linesize);
1759 	(void) clear_errors(&spf_flt, NULL);
1760 
1761 	/*
1762 	 * Take care of a special case: If there is a UE in the ecache flush
1763 	 * area, we'll see it in flush_ecache().  This will trigger the
1764 	 * CPU_ADDITIONAL_ERRORS case below.
1765 	 *
1766 	 * This could occur if the original error was a UE in the flush area,
1767 	 * or if the original error was an E$ error that was flushed out of
1768 	 * the E$ in scan_ecache().
1769 	 *
1770 	 * If it's at the same address that we're already logging, then it's
1771 	 * probably one of these cases.  Clear the bit so we don't trip over
1772 	 * it on the additional errors case, which could cause an unnecessary
1773 	 * panic.
1774 	 */
1775 	if ((aflt->flt_stat & P_AFSR_UE) && aflt->flt_addr == t_afar)
1776 		acc_afsr |= aflt->flt_stat & ~P_AFSR_UE;
1777 	else
1778 		acc_afsr |= aflt->flt_stat;
1779 
1780 	/*
1781 	 * Check the acumulated afsr for the important bits.
1782 	 * Make sure the spf_flt.flt_type value is set, and
1783 	 * enque an error.
1784 	 */
1785 	if (acc_afsr &
1786 	    (P_AFSR_LEVEL1 | P_AFSR_IVUE | P_AFSR_ETP | P_AFSR_ISAP)) {
1787 		if (acc_afsr & (P_AFSR_UE | P_AFSR_EDP | P_AFSR_LDP |
1788 		    P_AFSR_BERR | P_AFSR_TO | P_AFSR_IVUE | P_AFSR_ETP |
1789 		    P_AFSR_ISAP))
1790 			aflt->flt_panic = 1;
1791 
1792 		spf_flt.flt_type = CPU_ADDITIONAL_ERR;
1793 		aflt->flt_stat = acc_afsr;
1794 		cpu_errorq_dispatch(FM_EREPORT_CPU_USII_UNKNOWN,
1795 		    (void *)&spf_flt, sizeof (spf_flt), ue_queue,
1796 		    aflt->flt_panic);
1797 	}
1798 
1799 	/*
1800 	 * If aflt->flt_panic is set at this point, we need to panic as the
1801 	 * result of a trap at TL > 0, or an error we determined to be fatal.
1802 	 * We've already enqueued the error in one of the if-clauses above,
1803 	 * and it will be dequeued and logged as part of the panic flow.
1804 	 */
1805 	if (aflt->flt_panic) {
1806 		cpu_aflt_log(CE_PANIC, 1, &spf_flt, CPU_ERRID_FIRST,
1807 		    "See previous message(s) for details", " %sError(s)",
1808 		    pr_reason);
1809 	}
1810 
1811 	/*
1812 	 * Before returning, we must re-enable errors, and
1813 	 * reset the caches to their boot-up state.
1814 	 */
1815 	set_lsu(get_lsu() | cache_boot_state);
1816 	set_error_enable(EER_ENABLE);
1817 }
1818 
1819 /*
1820  * Check for miscellaneous fatal errors and call CE_PANIC if any are seen.
1821  * This routine is shared by the CE and UE handling code.
1822  */
1823 static void
1824 check_misc_err(spitf_async_flt *spf_flt)
1825 {
1826 	struct async_flt *aflt = (struct async_flt *)spf_flt;
1827 	char *fatal_str = NULL;
1828 
1829 	/*
1830 	 * The ISAP and ETP errors are supposed to cause a POR
1831 	 * from the system, so in theory we never, ever see these messages.
1832 	 * ISAP, ETP and IVUE are considered to be fatal.
1833 	 */
1834 	if (aflt->flt_stat & P_AFSR_ISAP)
1835 		fatal_str = " System Address Parity Error on";
1836 	else if (aflt->flt_stat & P_AFSR_ETP)
1837 		fatal_str = " Ecache Tag Parity Error on";
1838 	else if (aflt->flt_stat & P_AFSR_IVUE)
1839 		fatal_str = " Interrupt Vector Uncorrectable Error on";
1840 	if (fatal_str != NULL) {
1841 		cpu_aflt_log(CE_PANIC, 1, spf_flt, CMN_LFLAGS,
1842 			NULL, fatal_str);
1843 	}
1844 }
1845 
1846 /*
1847  * Routine to convert a syndrome into a syndrome code.
1848  */
1849 static int
1850 synd_to_synd_code(int synd_status, ushort_t synd)
1851 {
1852 	if (synd_status != AFLT_STAT_VALID)
1853 		return (-1);
1854 
1855 	/*
1856 	 * Use the 8-bit syndrome to index the ecc_syndrome_tab
1857 	 * to get the code indicating which bit(s) is(are) bad.
1858 	 */
1859 	if ((synd == 0) || (synd >= SYND_TBL_SIZE))
1860 		return (-1);
1861 	else
1862 		return (ecc_syndrome_tab[synd]);
1863 }
1864 
1865 /* ARGSUSED */
1866 int
1867 cpu_get_mem_sid(char *unum, char *buf, int buflen, int *lenp)
1868 {
1869 	return (ENOTSUP);
1870 }
1871 
1872 /* ARGSUSED */
1873 int
1874 cpu_get_mem_offset(uint64_t flt_addr, uint64_t *offp)
1875 {
1876 	return (ENOTSUP);
1877 }
1878 
1879 /* ARGSUSED */
1880 int
1881 cpu_get_mem_addr(char *unum, char *sid, uint64_t offset, uint64_t *addrp)
1882 {
1883 	return (ENOTSUP);
1884 }
1885 
1886 /*
1887  * Routine to return a string identifying the physical name
1888  * associated with a memory/cache error.
1889  */
1890 /* ARGSUSED */
1891 int
1892 cpu_get_mem_unum(int synd_status, ushort_t synd, uint64_t afsr,
1893     uint64_t afar, int cpuid, int flt_in_memory, ushort_t flt_status,
1894     char *buf, int buflen, int *lenp)
1895 {
1896 	short synd_code;
1897 	int ret;
1898 
1899 	if (flt_in_memory) {
1900 		synd_code = synd_to_synd_code(synd_status, synd);
1901 		if (synd_code == -1) {
1902 			ret = EINVAL;
1903 		} else if (prom_get_unum(synd_code, P2ALIGN(afar, 8),
1904 		    buf, buflen, lenp) != 0) {
1905 			ret = EIO;
1906 		} else if (*lenp <= 1) {
1907 			ret = EINVAL;
1908 		} else {
1909 			ret = 0;
1910 		}
1911 	} else {
1912 		ret = ENOTSUP;
1913 	}
1914 
1915 	if (ret != 0) {
1916 		buf[0] = '\0';
1917 		*lenp = 0;
1918 	}
1919 
1920 	return (ret);
1921 }
1922 
1923 /*
1924  * Wrapper for cpu_get_mem_unum() routine that takes an
1925  * async_flt struct rather than explicit arguments.
1926  */
1927 int
1928 cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt,
1929     char *buf, int buflen, int *lenp)
1930 {
1931 	return (cpu_get_mem_unum(synd_status, SYND(aflt->flt_synd),
1932 		aflt->flt_stat, aflt->flt_addr, aflt->flt_bus_id,
1933 		aflt->flt_in_memory, aflt->flt_status, buf, buflen, lenp));
1934 }
1935 
1936 /*
1937  * This routine is a more generic interface to cpu_get_mem_unum(),
1938  * that may be used by other modules (e.g. mm).
1939  */
1940 int
1941 cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar,
1942 		char *buf, int buflen, int *lenp)
1943 {
1944 	int synd_status, flt_in_memory, ret;
1945 	char unum[UNUM_NAMLEN];
1946 
1947 	/*
1948 	 * Check for an invalid address.
1949 	 */
1950 	if (afar == (uint64_t)-1)
1951 		return (ENXIO);
1952 
1953 	if (synd == (uint64_t)-1)
1954 		synd_status = AFLT_STAT_INVALID;
1955 	else
1956 		synd_status = AFLT_STAT_VALID;
1957 
1958 	flt_in_memory = (pf_is_memory(afar >> MMU_PAGESHIFT)) ? 1 : 0;
1959 
1960 	if ((ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, *afsr, afar,
1961 	    CPU->cpu_id, flt_in_memory, 0, unum, UNUM_NAMLEN, lenp))
1962 	    != 0)
1963 		return (ret);
1964 
1965 	if (*lenp >= buflen)
1966 		return (ENAMETOOLONG);
1967 
1968 	(void) strncpy(buf, unum, buflen);
1969 
1970 	return (0);
1971 }
1972 
1973 /*
1974  * Routine to return memory information associated
1975  * with a physical address and syndrome.
1976  */
1977 /* ARGSUSED */
1978 int
1979 cpu_get_mem_info(uint64_t synd, uint64_t afar,
1980     uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep,
1981     int *segsp, int *banksp, int *mcidp)
1982 {
1983 	return (ENOTSUP);
1984 }
1985 
1986 /*
1987  * Routine to return a string identifying the physical
1988  * name associated with a cpuid.
1989  */
1990 /* ARGSUSED */
1991 int
1992 cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp)
1993 {
1994 	return (ENOTSUP);
1995 }
1996 
1997 /*
1998  * This routine returns the size of the kernel's FRU name buffer.
1999  */
2000 size_t
2001 cpu_get_name_bufsize()
2002 {
2003 	return (UNUM_NAMLEN);
2004 }
2005 
2006 /*
2007  * Cpu specific log func for UEs.
2008  */
2009 static void
2010 log_ue_err(struct async_flt *aflt, char *unum)
2011 {
2012 	spitf_async_flt *spf_flt = (spitf_async_flt *)aflt;
2013 	int len = 0;
2014 
2015 #ifdef DEBUG
2016 	int afsr_priv = (aflt->flt_stat & P_AFSR_PRIV) ? 1 : 0;
2017 
2018 	/*
2019 	 * Paranoid Check for priv mismatch
2020 	 * Only applicable for UEs
2021 	 */
2022 	if (afsr_priv != aflt->flt_priv) {
2023 		/*
2024 		 * The priv bits in %tstate and %afsr did not match; we expect
2025 		 * this to be very rare, so flag it with a message.
2026 		 */
2027 		cpu_aflt_log(CE_WARN, 2, spf_flt, CPU_ERRID_FIRST, NULL,
2028 		    ": PRIV bit in TSTATE and AFSR mismatched; "
2029 		    "TSTATE.PRIV=%d used", (aflt->flt_priv) ? 1 : 0);
2030 
2031 		/* update saved afsr to reflect the correct priv */
2032 		aflt->flt_stat &= ~P_AFSR_PRIV;
2033 		if (aflt->flt_priv)
2034 			aflt->flt_stat |= P_AFSR_PRIV;
2035 	}
2036 #endif /* DEBUG */
2037 
2038 	(void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, unum,
2039 	    UNUM_NAMLEN, &len);
2040 
2041 	cpu_aflt_log(CE_WARN, 1, spf_flt, UE_LFLAGS, unum,
2042 	    " Uncorrectable Memory Error on");
2043 
2044 	if (SYND(aflt->flt_synd) == 0x3) {
2045 		cpu_aflt_log(CE_WARN, 1, spf_flt, CPU_ERRID_FIRST, NULL,
2046 		    " Syndrome 0x3 indicates that this may not be a "
2047 		    "memory module problem");
2048 	}
2049 
2050 	if (aflt->flt_in_memory)
2051 		cpu_log_ecmem_info(spf_flt);
2052 }
2053 
2054 
2055 /*
2056  * The cpu_async_log_err() function is called via the ue_drain() function to
2057  * handle logging for CPU events that are dequeued.  As such, it can be invoked
2058  * from softint context, from AST processing in the trap() flow, or from the
2059  * panic flow.  We decode the CPU-specific data, and log appropriate messages.
2060  */
2061 static void
2062 cpu_async_log_err(void *flt)
2063 {
2064 	spitf_async_flt *spf_flt = (spitf_async_flt *)flt;
2065 	struct async_flt *aflt = (struct async_flt *)flt;
2066 	char unum[UNUM_NAMLEN];
2067 	char *space;
2068 	char *ecache_scrub_logstr = NULL;
2069 
2070 	switch (spf_flt->flt_type) {
2071 	    case CPU_UE_ERR:
2072 		/*
2073 		 * We want to skip logging only if ALL the following
2074 		 * conditions are true:
2075 		 *
2076 		 *	1. We are not panicking
2077 		 *	2. There is only one error
2078 		 *	3. That error is a memory error
2079 		 *	4. The error is caused by the memory scrubber (in
2080 		 *	   which case the error will have occurred under
2081 		 *	   on_trap protection)
2082 		 *	5. The error is on a retired page
2083 		 *
2084 		 * Note 1: AFLT_PROT_EC is used places other than the memory
2085 		 * scrubber.  However, none of those errors should occur
2086 		 * on a retired page.
2087 		 *
2088 		 * Note 2: In the CE case, these errors are discarded before
2089 		 * the errorq.  In the UE case, we must wait until now --
2090 		 * softcall() grabs a mutex, which we can't do at a high PIL.
2091 		 */
2092 		if (!panicstr &&
2093 		    (aflt->flt_stat & S_AFSR_ALL_ERRS) == P_AFSR_UE &&
2094 		    aflt->flt_prot == AFLT_PROT_EC) {
2095 			if (page_retire_check(aflt->flt_addr, NULL) == 0) {
2096 				/* Zero the address to clear the error */
2097 				softcall(ecc_page_zero, (void *)aflt->flt_addr);
2098 				return;
2099 			}
2100 		}
2101 
2102 		/*
2103 		 * Log the UE and check for causes of this UE error that
2104 		 * don't cause a trap (Copyback error).  cpu_async_error()
2105 		 * has already checked the i/o buses for us.
2106 		 */
2107 		log_ue_err(aflt, unum);
2108 		if (aflt->flt_in_memory)
2109 			cpu_check_allcpus(aflt);
2110 		break;
2111 
2112 	    case CPU_EDP_LDP_ERR:
2113 		if (aflt->flt_stat & P_AFSR_EDP)
2114 			cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS,
2115 			    NULL, " EDP event on");
2116 
2117 		if (aflt->flt_stat & P_AFSR_LDP)
2118 			cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS,
2119 			    NULL, " LDP event on");
2120 
2121 		/* Log ecache info if exist */
2122 		if (spf_flt->flt_ec_lcnt > 0) {
2123 			cpu_log_ecmem_info(spf_flt);
2124 
2125 			cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST,
2126 			    NULL, " AFAR was derived from E$Tag");
2127 		} else {
2128 			cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST,
2129 			    NULL, " No error found in ecache (No fault "
2130 			    "PA available)");
2131 		}
2132 		break;
2133 
2134 	    case CPU_WP_ERR:
2135 		/*
2136 		 * If the memscrub thread hasn't yet read
2137 		 * all of memory, as we requested in the
2138 		 * trap handler, then give it a kick to
2139 		 * make sure it does.
2140 		 */
2141 		if (!isus2i && !isus2e && read_all_memscrub)
2142 			memscrub_run();
2143 
2144 		cpu_aflt_log(CE_WARN, 1, spf_flt, WP_LFLAGS, NULL,
2145 		    " WP event on");
2146 		return;
2147 
2148 	    case CPU_BTO_BERR_ERR:
2149 		/*
2150 		 * A bus timeout or error occurred that was in user mode or not
2151 		 * in a protected kernel code region.
2152 		 */
2153 		if (aflt->flt_stat & P_AFSR_BERR) {
2154 			cpu_aflt_log(CE_WARN, aflt->flt_panic ? 1 : 2,
2155 			    spf_flt, BERRTO_LFLAGS, NULL,
2156 			    " Bus Error on System Bus in %s mode from",
2157 			    aflt->flt_priv ? "privileged" : "user");
2158 		}
2159 
2160 		if (aflt->flt_stat & P_AFSR_TO) {
2161 			cpu_aflt_log(CE_WARN, aflt->flt_panic ? 1 : 2,
2162 			    spf_flt, BERRTO_LFLAGS, NULL,
2163 			    " Timeout on System Bus in %s mode from",
2164 			    aflt->flt_priv ? "privileged" : "user");
2165 		}
2166 
2167 		return;
2168 
2169 	    case CPU_PANIC_CP_ERR:
2170 		/*
2171 		 * Process the Copyback (CP) error info (if any) obtained from
2172 		 * polling all the cpus in the panic flow. This case is only
2173 		 * entered if we are panicking.
2174 		 */
2175 		ASSERT(panicstr != NULL);
2176 		ASSERT(aflt->flt_id == panic_aflt.flt_id);
2177 
2178 		/* See which space - this info may not exist */
2179 		if (panic_aflt.flt_status & ECC_D_TRAP)
2180 			space = "Data ";
2181 		else if (panic_aflt.flt_status & ECC_I_TRAP)
2182 			space = "Instruction ";
2183 		else
2184 			space = "";
2185 
2186 		cpu_aflt_log(CE_WARN, 1, spf_flt, CP_LFLAGS, NULL,
2187 		    " AFAR was derived from UE report,"
2188 		    " CP event on CPU%d (caused %saccess error on %s%d)",
2189 		    aflt->flt_inst, space, (panic_aflt.flt_status & ECC_IOBUS) ?
2190 		    "IOBUS" : "CPU", panic_aflt.flt_bus_id);
2191 
2192 		if (spf_flt->flt_ec_lcnt > 0)
2193 			cpu_log_ecmem_info(spf_flt);
2194 		else
2195 			cpu_aflt_log(CE_WARN, 2, spf_flt, CPU_ERRID_FIRST,
2196 			    NULL, " No cache dump available");
2197 
2198 		return;
2199 
2200 	    case CPU_TRAPPING_CP_ERR:
2201 		/*
2202 		 * For sabre only.  This is a copyback ecache parity error due
2203 		 * to a PCI DMA read.  We should be panicking if we get here.
2204 		 */
2205 		ASSERT(panicstr != NULL);
2206 		cpu_aflt_log(CE_WARN, 1, spf_flt, CP_LFLAGS, NULL,
2207 		    " AFAR was derived from UE report,"
2208 		    " CP event on CPU%d (caused Data access error "
2209 		    "on PCIBus)", aflt->flt_inst);
2210 		return;
2211 
2212 		/*
2213 		 * We log the ecache lines of the following states,
2214 		 * clean_bad_idle, clean_bad_busy, dirty_bad_idle and
2215 		 * dirty_bad_busy if ecache_scrub_verbose is set and panic
2216 		 * in addition to logging if ecache_scrub_panic is set.
2217 		 */
2218 	    case CPU_BADLINE_CI_ERR:
2219 		ecache_scrub_logstr = "CBI";
2220 		/* FALLTHRU */
2221 
2222 	    case CPU_BADLINE_CB_ERR:
2223 		if (ecache_scrub_logstr == NULL)
2224 			ecache_scrub_logstr = "CBB";
2225 		/* FALLTHRU */
2226 
2227 	    case CPU_BADLINE_DI_ERR:
2228 		if (ecache_scrub_logstr == NULL)
2229 			ecache_scrub_logstr = "DBI";
2230 		/* FALLTHRU */
2231 
2232 	    case CPU_BADLINE_DB_ERR:
2233 		if (ecache_scrub_logstr == NULL)
2234 			ecache_scrub_logstr = "DBB";
2235 
2236 		cpu_aflt_log(CE_NOTE, 2, spf_flt,
2237 			(CPU_ERRID_FIRST | CPU_FLTCPU), NULL,
2238 			" %s event on", ecache_scrub_logstr);
2239 		cpu_log_ecmem_info(spf_flt);
2240 
2241 		return;
2242 
2243 	    case CPU_ORPHAN_CP_ERR:
2244 		/*
2245 		 * Orphan CPs, where the CP bit is set, but when a CPU
2246 		 * doesn't report a UE.
2247 		 */
2248 		if (read_all_memscrub)
2249 			memscrub_run();
2250 
2251 		cpu_aflt_log(CE_NOTE, 2, spf_flt, (CP_LFLAGS | CPU_FLTCPU),
2252 			NULL, " Orphan CP event on");
2253 
2254 		/* Log ecache info if exist */
2255 		if (spf_flt->flt_ec_lcnt > 0)
2256 			cpu_log_ecmem_info(spf_flt);
2257 		else
2258 			cpu_aflt_log(CE_NOTE, 2, spf_flt,
2259 				(CP_LFLAGS | CPU_FLTCPU), NULL,
2260 				" No error found in ecache (No fault "
2261 				"PA available");
2262 		return;
2263 
2264 	    case CPU_ECACHE_ADDR_PAR_ERR:
2265 		cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL,
2266 				" E$ Tag Address Parity error on");
2267 		cpu_log_ecmem_info(spf_flt);
2268 		return;
2269 
2270 	    case CPU_ECACHE_STATE_ERR:
2271 		cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL,
2272 				" E$ Tag State Parity error on");
2273 		cpu_log_ecmem_info(spf_flt);
2274 		return;
2275 
2276 	    case CPU_ECACHE_TAG_ERR:
2277 		cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL,
2278 				" E$ Tag scrub event on");
2279 		cpu_log_ecmem_info(spf_flt);
2280 		return;
2281 
2282 	    case CPU_ECACHE_ETP_ETS_ERR:
2283 		cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL,
2284 				" AFSR.ETP is set and AFSR.ETS is zero on");
2285 		cpu_log_ecmem_info(spf_flt);
2286 		return;
2287 
2288 
2289 	    case CPU_ADDITIONAL_ERR:
2290 		cpu_aflt_log(CE_WARN, 1, spf_flt, CMN_LFLAGS & ~CPU_SPACE, NULL,
2291 		    " Additional errors detected during error processing on");
2292 		return;
2293 
2294 	    default:
2295 		cmn_err(CE_WARN, "cpu_async_log_err: fault %p has unknown "
2296 		    "fault type %x", (void *)spf_flt, spf_flt->flt_type);
2297 		return;
2298 	}
2299 
2300 	/* ... fall through from the UE, EDP, or LDP cases */
2301 
2302 	if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) {
2303 		if (!panicstr) {
2304 			(void) page_retire(aflt->flt_addr, PR_UE);
2305 		} else {
2306 			/*
2307 			 * Clear UEs on panic so that we don't
2308 			 * get haunted by them during panic or
2309 			 * after reboot
2310 			 */
2311 			clearphys(P2ALIGN(aflt->flt_addr, 64),
2312 			    cpunodes[CPU->cpu_id].ecache_size,
2313 			    cpunodes[CPU->cpu_id].ecache_linesize);
2314 
2315 			(void) clear_errors(NULL, NULL);
2316 		}
2317 	}
2318 
2319 	/*
2320 	 * Log final recover message
2321 	 */
2322 	if (!panicstr) {
2323 		if (!aflt->flt_priv) {
2324 			cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST,
2325 			    NULL, " Above Error is in User Mode"
2326 			    "\n    and is fatal: "
2327 			    "will SIGKILL process and notify contract");
2328 		} else if (aflt->flt_prot == AFLT_PROT_COPY && aflt->flt_core) {
2329 			cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST,
2330 			    NULL, " Above Error detected while dumping core;"
2331 			    "\n    core file will be truncated");
2332 		} else if (aflt->flt_prot == AFLT_PROT_COPY) {
2333 			cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST,
2334 			    NULL, " Above Error is due to Kernel access"
2335 			    "\n    to User space and is fatal: "
2336 			    "will SIGKILL process and notify contract");
2337 		} else if (aflt->flt_prot == AFLT_PROT_EC) {
2338 			cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST, NULL,
2339 			    " Above Error detected by protected Kernel code"
2340 			    "\n    that will try to clear error from system");
2341 		}
2342 	}
2343 }
2344 
2345 
2346 /*
2347  * Check all cpus for non-trapping UE-causing errors
2348  * In Ultra I/II, we look for copyback errors (CPs)
2349  */
2350 void
2351 cpu_check_allcpus(struct async_flt *aflt)
2352 {
2353 	spitf_async_flt cp;
2354 	spitf_async_flt *spf_cpflt = &cp;
2355 	struct async_flt *cpflt = (struct async_flt *)&cp;
2356 	int pix;
2357 
2358 	cpflt->flt_id = aflt->flt_id;
2359 	cpflt->flt_addr = aflt->flt_addr;
2360 
2361 	for (pix = 0; pix < NCPU; pix++) {
2362 		if (CPU_XCALL_READY(pix)) {
2363 			xc_one(pix, (xcfunc_t *)get_cpu_status,
2364 			    (uint64_t)cpflt, 0);
2365 
2366 			if (cpflt->flt_stat & P_AFSR_CP) {
2367 				char *space;
2368 
2369 				/* See which space - this info may not exist */
2370 				if (aflt->flt_status & ECC_D_TRAP)
2371 					space = "Data ";
2372 				else if (aflt->flt_status & ECC_I_TRAP)
2373 					space = "Instruction ";
2374 				else
2375 					space = "";
2376 
2377 				cpu_aflt_log(CE_WARN, 1, spf_cpflt, CP_LFLAGS,
2378 				    NULL, " AFAR was derived from UE report,"
2379 				    " CP event on CPU%d (caused %saccess "
2380 				    "error on %s%d)", pix, space,
2381 				    (aflt->flt_status & ECC_IOBUS) ?
2382 				    "IOBUS" : "CPU", aflt->flt_bus_id);
2383 
2384 				if (spf_cpflt->flt_ec_lcnt > 0)
2385 					cpu_log_ecmem_info(spf_cpflt);
2386 				else
2387 					cpu_aflt_log(CE_WARN, 2, spf_cpflt,
2388 					    CPU_ERRID_FIRST, NULL,
2389 					    " No cache dump available");
2390 			}
2391 		}
2392 	}
2393 }
2394 
2395 #ifdef DEBUG
2396 int test_mp_cp = 0;
2397 #endif
2398 
2399 /*
2400  * Cross-call callback routine to tell a CPU to read its own %afsr to check
2401  * for copyback errors and capture relevant information.
2402  */
2403 static uint_t
2404 get_cpu_status(uint64_t arg)
2405 {
2406 	struct async_flt *aflt = (struct async_flt *)arg;
2407 	spitf_async_flt *spf_flt = (spitf_async_flt *)arg;
2408 	uint64_t afsr;
2409 	uint32_t ec_idx;
2410 	uint64_t sdbh, sdbl;
2411 	int i;
2412 	uint32_t ec_set_size;
2413 	uchar_t valid;
2414 	ec_data_t ec_data[8];
2415 	uint64_t ec_tag, flt_addr_tag, oafsr;
2416 	uint64_t *acc_afsr = NULL;
2417 
2418 	get_asyncflt(&afsr);
2419 	if (CPU_PRIVATE(CPU) != NULL) {
2420 		acc_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr);
2421 		afsr |= *acc_afsr;
2422 		*acc_afsr = 0;
2423 	}
2424 
2425 #ifdef DEBUG
2426 	if (test_mp_cp)
2427 		afsr |= P_AFSR_CP;
2428 #endif
2429 	aflt->flt_stat = afsr;
2430 
2431 	if (afsr & P_AFSR_CP) {
2432 		/*
2433 		 * Capture the UDBs
2434 		 */
2435 		get_udb_errors(&sdbh, &sdbl);
2436 		spf_flt->flt_sdbh = (ushort_t)(sdbh & 0x3FF);
2437 		spf_flt->flt_sdbl = (ushort_t)(sdbl & 0x3FF);
2438 
2439 		/*
2440 		 * Clear CP bit before capturing ecache data
2441 		 * and AFSR info.
2442 		 */
2443 		set_asyncflt(P_AFSR_CP);
2444 
2445 		/*
2446 		 * See if we can capture the ecache line for the
2447 		 * fault PA.
2448 		 *
2449 		 * Return a valid matching ecache line, if any.
2450 		 * Otherwise, return the first matching ecache
2451 		 * line marked invalid.
2452 		 */
2453 		flt_addr_tag = aflt->flt_addr >> cpu_ec_tag_shift;
2454 		ec_set_size = cpunodes[CPU->cpu_id].ecache_size /
2455 		    ecache_associativity;
2456 		spf_flt->flt_ec_lcnt = 0;
2457 
2458 		for (i = 0, ec_idx = (aflt->flt_addr % ec_set_size);
2459 		    i < ecache_associativity; i++, ec_idx += ec_set_size) {
2460 			get_ecache_dtag(P2ALIGN(ec_idx, 64),
2461 				(uint64_t *)&ec_data[0], &ec_tag, &oafsr,
2462 				    acc_afsr);
2463 
2464 			if ((ec_tag & cpu_ec_tag_mask) != flt_addr_tag)
2465 				continue;
2466 
2467 			valid = cpu_ec_state_valid &
2468 			    (uchar_t)((ec_tag & cpu_ec_state_mask) >>
2469 			    cpu_ec_state_shift);
2470 
2471 			if (valid || spf_flt->flt_ec_lcnt == 0) {
2472 				spf_flt->flt_ec_tag = ec_tag;
2473 				bcopy(&ec_data, &spf_flt->flt_ec_data,
2474 				    sizeof (ec_data));
2475 				spf_flt->flt_ec_lcnt = 1;
2476 
2477 				if (valid)
2478 					break;
2479 			}
2480 		}
2481 	}
2482 	return (0);
2483 }
2484 
2485 /*
2486  * CPU-module callback for the non-panicking CPUs.  This routine is invoked
2487  * from panic_idle() as part of the other CPUs stopping themselves when a
2488  * panic occurs.  We need to be VERY careful what we do here, since panicstr
2489  * is NOT set yet and we cannot blow through locks.  If panic_aflt is set
2490  * (panic_aflt.flt_id is non-zero), we need to read our %afsr to look for
2491  * CP error information.
2492  */
2493 void
2494 cpu_async_panic_callb(void)
2495 {
2496 	spitf_async_flt cp;
2497 	struct async_flt *aflt = (struct async_flt *)&cp;
2498 	uint64_t *scrub_afsr;
2499 
2500 	if (panic_aflt.flt_id != 0) {
2501 		aflt->flt_addr = panic_aflt.flt_addr;
2502 		(void) get_cpu_status((uint64_t)aflt);
2503 
2504 		if (CPU_PRIVATE(CPU) != NULL) {
2505 			scrub_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr);
2506 			if (*scrub_afsr & P_AFSR_CP) {
2507 				aflt->flt_stat |= *scrub_afsr;
2508 				*scrub_afsr = 0;
2509 			}
2510 		}
2511 		if (aflt->flt_stat & P_AFSR_CP) {
2512 			aflt->flt_id = panic_aflt.flt_id;
2513 			aflt->flt_panic = 1;
2514 			aflt->flt_inst = CPU->cpu_id;
2515 			aflt->flt_class = CPU_FAULT;
2516 			cp.flt_type = CPU_PANIC_CP_ERR;
2517 			cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CP,
2518 			    (void *)&cp, sizeof (cp), ue_queue,
2519 			    aflt->flt_panic);
2520 		}
2521 	}
2522 }
2523 
2524 /*
2525  * Turn off all cpu error detection, normally only used for panics.
2526  */
2527 void
2528 cpu_disable_errors(void)
2529 {
2530 	xt_all(set_error_enable_tl1, EER_DISABLE, EER_SET_ABSOLUTE);
2531 }
2532 
2533 /*
2534  * Enable errors.
2535  */
2536 void
2537 cpu_enable_errors(void)
2538 {
2539 	xt_all(set_error_enable_tl1, EER_ENABLE, EER_SET_ABSOLUTE);
2540 }
2541 
2542 static void
2543 cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err)
2544 {
2545 	uint64_t aligned_addr = P2ALIGN(ecc->flt_addr, 8);
2546 	int i, loop = 1;
2547 	ushort_t ecc_0;
2548 	uint64_t paddr;
2549 	uint64_t data;
2550 
2551 	if (verbose)
2552 		loop = 8;
2553 	for (i = 0; i < loop; i++) {
2554 		paddr = aligned_addr + (i * 8);
2555 		data = lddphys(paddr);
2556 		if (verbose) {
2557 			if (ce_err) {
2558 			    ecc_0 = ecc_gen((uint32_t)(data>>32),
2559 			    (uint32_t)data);
2560 			    cpu_aflt_log(CE_CONT, 0, NULL, NO_LFLAGS,
2561 				NULL, "    Paddr 0x%" PRIx64 ", "
2562 				"Data 0x%08x.%08x, ECC 0x%x", paddr,
2563 				(uint32_t)(data>>32), (uint32_t)data, ecc_0);
2564 			} else {
2565 				cpu_aflt_log(CE_CONT, 0, NULL, NO_LFLAGS,
2566 				    NULL, "    Paddr 0x%" PRIx64 ", "
2567 				    "Data 0x%08x.%08x", paddr,
2568 				    (uint32_t)(data>>32), (uint32_t)data);
2569 			}
2570 		}
2571 	}
2572 }
2573 
2574 static struct {		/* sec-ded-s4ed ecc code */
2575 	uint_t hi, lo;
2576 } ecc_code[8] = {
2577 	{ 0xee55de23U, 0x16161161U },
2578 	{ 0x55eede93U, 0x61612212U },
2579 	{ 0xbb557b8cU, 0x49494494U },
2580 	{ 0x55bb7b6cU, 0x94948848U },
2581 	{ 0x16161161U, 0xee55de23U },
2582 	{ 0x61612212U, 0x55eede93U },
2583 	{ 0x49494494U, 0xbb557b8cU },
2584 	{ 0x94948848U, 0x55bb7b6cU }
2585 };
2586 
2587 static ushort_t
2588 ecc_gen(uint_t high_bytes, uint_t low_bytes)
2589 {
2590 	int i, j;
2591 	uchar_t checker, bit_mask;
2592 	struct {
2593 		uint_t hi, lo;
2594 	} hex_data, masked_data[8];
2595 
2596 	hex_data.hi = high_bytes;
2597 	hex_data.lo = low_bytes;
2598 
2599 	/* mask out bits according to sec-ded-s4ed ecc code */
2600 	for (i = 0; i < 8; i++) {
2601 		masked_data[i].hi = hex_data.hi & ecc_code[i].hi;
2602 		masked_data[i].lo = hex_data.lo & ecc_code[i].lo;
2603 	}
2604 
2605 	/*
2606 	 * xor all bits in masked_data[i] to get bit_i of checker,
2607 	 * where i = 0 to 7
2608 	 */
2609 	checker = 0;
2610 	for (i = 0; i < 8; i++) {
2611 		bit_mask = 1 << i;
2612 		for (j = 0; j < 32; j++) {
2613 			if (masked_data[i].lo & 1) checker ^= bit_mask;
2614 			if (masked_data[i].hi & 1) checker ^= bit_mask;
2615 			masked_data[i].hi >>= 1;
2616 			masked_data[i].lo >>= 1;
2617 		}
2618 	}
2619 	return (checker);
2620 }
2621 
2622 /*
2623  * Flush the entire ecache using displacement flush by reading through a
2624  * physical address range as large as the ecache.
2625  */
2626 void
2627 cpu_flush_ecache(void)
2628 {
2629 	flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size * 2,
2630 	    cpunodes[CPU->cpu_id].ecache_linesize);
2631 }
2632 
2633 /*
2634  * read and display the data in the cache line where the
2635  * original ce error occurred.
2636  * This routine is mainly used for debugging new hardware.
2637  */
2638 void
2639 read_ecc_data(struct async_flt *ecc, short verbose, short ce_err)
2640 {
2641 	kpreempt_disable();
2642 	/* disable ECC error traps */
2643 	set_error_enable(EER_ECC_DISABLE);
2644 
2645 	/*
2646 	 * flush the ecache
2647 	 * read the data
2648 	 * check to see if an ECC error occured
2649 	 */
2650 	flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size * 2,
2651 	    cpunodes[CPU->cpu_id].ecache_linesize);
2652 	set_lsu(get_lsu() | cache_boot_state);
2653 	cpu_read_paddr(ecc, verbose, ce_err);
2654 	(void) check_ecc(ecc);
2655 
2656 	/* enable ECC error traps */
2657 	set_error_enable(EER_ENABLE);
2658 	kpreempt_enable();
2659 }
2660 
2661 /*
2662  * Check the AFSR bits for UE/CE persistence.
2663  * If UE or CE errors are detected, the routine will
2664  * clears all the AFSR sticky bits (except CP for
2665  * spitfire/blackbird) and the UDBs.
2666  * if ce_debug or ue_debug is set, log any ue/ce errors detected.
2667  */
2668 static int
2669 check_ecc(struct async_flt *ecc)
2670 {
2671 	uint64_t t_afsr;
2672 	uint64_t t_afar;
2673 	uint64_t udbh;
2674 	uint64_t udbl;
2675 	ushort_t udb;
2676 	int persistent = 0;
2677 
2678 	/*
2679 	 * Capture the AFSR, AFAR and UDBs info
2680 	 */
2681 	get_asyncflt(&t_afsr);
2682 	get_asyncaddr(&t_afar);
2683 	t_afar &= SABRE_AFAR_PA;
2684 	get_udb_errors(&udbh, &udbl);
2685 
2686 	if ((t_afsr & P_AFSR_UE) || (t_afsr & P_AFSR_CE)) {
2687 		/*
2688 		 * Clear the errors
2689 		 */
2690 		clr_datapath();
2691 
2692 		if (isus2i || isus2e)
2693 			set_asyncflt(t_afsr);
2694 		else
2695 			set_asyncflt(t_afsr & ~P_AFSR_CP);
2696 
2697 		/*
2698 		 * determine whether to check UDBH or UDBL for persistence
2699 		 */
2700 		if (ecc->flt_synd & UDBL_REG) {
2701 			udb = (ushort_t)udbl;
2702 			t_afar |= 0x8;
2703 		} else {
2704 			udb = (ushort_t)udbh;
2705 		}
2706 
2707 		if (ce_debug || ue_debug) {
2708 			spitf_async_flt spf_flt; /* for logging */
2709 			struct async_flt *aflt =
2710 				(struct async_flt *)&spf_flt;
2711 
2712 			/* Package the info nicely in the spf_flt struct */
2713 			bzero(&spf_flt, sizeof (spitf_async_flt));
2714 			aflt->flt_stat = t_afsr;
2715 			aflt->flt_addr = t_afar;
2716 			spf_flt.flt_sdbh = (ushort_t)(udbh & 0x3FF);
2717 			spf_flt.flt_sdbl = (ushort_t)(udbl & 0x3FF);
2718 
2719 			cpu_aflt_log(CE_CONT, 0, &spf_flt, (CPU_AFSR |
2720 			    CPU_AFAR | CPU_UDBH | CPU_UDBL), NULL,
2721 			    " check_ecc: Dumping captured error states ...");
2722 		}
2723 
2724 		/*
2725 		 * if the fault addresses don't match, not persistent
2726 		 */
2727 		if (t_afar != ecc->flt_addr) {
2728 			return (persistent);
2729 		}
2730 
2731 		/*
2732 		 * check for UE persistence
2733 		 * since all DIMMs in the bank are identified for a UE,
2734 		 * there's no reason to check the syndrome
2735 		 */
2736 		if ((ecc->flt_stat & P_AFSR_UE) && (t_afsr & P_AFSR_UE)) {
2737 			persistent = 1;
2738 		}
2739 
2740 		/*
2741 		 * check for CE persistence
2742 		 */
2743 		if ((ecc->flt_stat & P_AFSR_CE) && (t_afsr & P_AFSR_CE)) {
2744 			if ((udb & P_DER_E_SYND) ==
2745 			    (ecc->flt_synd & P_DER_E_SYND)) {
2746 				persistent = 1;
2747 			}
2748 		}
2749 	}
2750 	return (persistent);
2751 }
2752 
2753 #ifdef HUMMINGBIRD
2754 #define	HB_FULL_DIV		1
2755 #define	HB_HALF_DIV		2
2756 #define	HB_LOWEST_DIV		8
2757 #define	HB_ECLK_INVALID		0xdeadbad
2758 static uint64_t hb_eclk[HB_LOWEST_DIV + 1] = {
2759 	HB_ECLK_INVALID, HB_ECLK_1, HB_ECLK_2, HB_ECLK_INVALID,
2760 	HB_ECLK_4, HB_ECLK_INVALID, HB_ECLK_6, HB_ECLK_INVALID,
2761 	HB_ECLK_8 };
2762 
2763 #define	HB_SLOW_DOWN		0
2764 #define	HB_SPEED_UP		1
2765 
2766 #define	SET_ESTAR_MODE(mode)					\
2767 	stdphysio(HB_ESTAR_MODE, (mode));			\
2768 	/*							\
2769 	 * PLL logic requires minimum of 16 clock		\
2770 	 * cycles to lock to the new clock speed.		\
2771 	 * Wait 1 usec to satisfy this requirement.		\
2772 	 */							\
2773 	drv_usecwait(1);
2774 
2775 #define	CHANGE_REFRESH_COUNT(direction, cur_div, new_div)	\
2776 {								\
2777 	volatile uint64_t data;					\
2778 	uint64_t count, new_count;				\
2779 	clock_t delay;						\
2780 	data = lddphysio(HB_MEM_CNTRL0);			\
2781 	count = (data & HB_REFRESH_COUNT_MASK) >> 		\
2782 	    HB_REFRESH_COUNT_SHIFT;				\
2783 	new_count = (HB_REFRESH_INTERVAL *			\
2784 	    cpunodes[CPU->cpu_id].clock_freq) /			\
2785 	    (HB_REFRESH_CLOCKS_PER_COUNT * (new_div) * NANOSEC);\
2786 	data = (data & ~HB_REFRESH_COUNT_MASK) |		\
2787 	    (new_count << HB_REFRESH_COUNT_SHIFT);		\
2788 	stdphysio(HB_MEM_CNTRL0, data);				\
2789 	data = lddphysio(HB_MEM_CNTRL0);        		\
2790 	/*							\
2791 	 * If we are slowing down the cpu and Memory		\
2792 	 * Self Refresh is not enabled, it is required		\
2793 	 * to wait for old refresh count to count-down and	\
2794 	 * new refresh count to go into effect (let new value	\
2795 	 * counts down once).					\
2796 	 */							\
2797 	if ((direction) == HB_SLOW_DOWN &&			\
2798 	    (data & HB_SELF_REFRESH_MASK) == 0) {		\
2799 		/*						\
2800 		 * Each count takes 64 cpu clock cycles		\
2801 		 * to decrement.  Wait for current refresh	\
2802 		 * count plus new refresh count at current	\
2803 		 * cpu speed to count down to zero.  Round	\
2804 		 * up the delay time.				\
2805 		 */						\
2806 		delay = ((HB_REFRESH_CLOCKS_PER_COUNT *		\
2807 		    (count + new_count) * MICROSEC * (cur_div)) /\
2808 		    cpunodes[CPU->cpu_id].clock_freq) + 1;	\
2809 		drv_usecwait(delay);				\
2810 	}							\
2811 }
2812 
2813 #define	SET_SELF_REFRESH(bit)					\
2814 {								\
2815 	volatile uint64_t data;					\
2816 	data = lddphysio(HB_MEM_CNTRL0);			\
2817 	data = (data & ~HB_SELF_REFRESH_MASK) |			\
2818 	    ((bit) << HB_SELF_REFRESH_SHIFT);			\
2819 	stdphysio(HB_MEM_CNTRL0, data);				\
2820 	data = lddphysio(HB_MEM_CNTRL0);			\
2821 }
2822 #endif	/* HUMMINGBIRD */
2823 
2824 /* ARGSUSED */
2825 void
2826 cpu_change_speed(uint64_t new_divisor, uint64_t arg2)
2827 {
2828 #ifdef HUMMINGBIRD
2829 	uint64_t cur_mask, cur_divisor = 0;
2830 	volatile uint64_t reg;
2831 	int index;
2832 
2833 	if ((new_divisor < HB_FULL_DIV || new_divisor > HB_LOWEST_DIV) ||
2834 	    (hb_eclk[new_divisor] == HB_ECLK_INVALID)) {
2835 		cmn_err(CE_WARN, "cpu_change_speed: bad divisor 0x%lx",
2836 		    new_divisor);
2837 		return;
2838 	}
2839 
2840 	reg = lddphysio(HB_ESTAR_MODE);
2841 	cur_mask = reg & HB_ECLK_MASK;
2842 	for (index = HB_FULL_DIV; index <= HB_LOWEST_DIV; index++) {
2843 		if (hb_eclk[index] == cur_mask) {
2844 			cur_divisor = index;
2845 			break;
2846 		}
2847 	}
2848 
2849 	if (cur_divisor == 0)
2850 		cmn_err(CE_PANIC, "cpu_change_speed: current divisor "
2851 		    "can't be determined!");
2852 
2853 	/*
2854 	 * If we are already at the requested divisor speed, just
2855 	 * return.
2856 	 */
2857 	if (cur_divisor == new_divisor)
2858 		return;
2859 
2860 	if (cur_divisor == HB_FULL_DIV && new_divisor == HB_HALF_DIV) {
2861 		CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, cur_divisor, new_divisor);
2862 		SET_ESTAR_MODE(hb_eclk[new_divisor]);
2863 		SET_SELF_REFRESH(HB_SELF_REFRESH_ENABLE);
2864 
2865 	} else if (cur_divisor == HB_HALF_DIV && new_divisor == HB_FULL_DIV) {
2866 		SET_SELF_REFRESH(HB_SELF_REFRESH_DISABLE);
2867 		SET_ESTAR_MODE(hb_eclk[new_divisor]);
2868 		/* LINTED: E_FALSE_LOGICAL_EXPR */
2869 		CHANGE_REFRESH_COUNT(HB_SPEED_UP, cur_divisor, new_divisor);
2870 
2871 	} else if (cur_divisor == HB_FULL_DIV && new_divisor > HB_HALF_DIV) {
2872 		/*
2873 		 * Transition to 1/2 speed first, then to
2874 		 * lower speed.
2875 		 */
2876 		CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, cur_divisor, HB_HALF_DIV);
2877 		SET_ESTAR_MODE(hb_eclk[HB_HALF_DIV]);
2878 		SET_SELF_REFRESH(HB_SELF_REFRESH_ENABLE);
2879 
2880 		CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, HB_HALF_DIV, new_divisor);
2881 		SET_ESTAR_MODE(hb_eclk[new_divisor]);
2882 
2883 	} else if (cur_divisor > HB_HALF_DIV && new_divisor == HB_FULL_DIV) {
2884 		/*
2885 		 * Transition to 1/2 speed first, then to
2886 		 * full speed.
2887 		 */
2888 		SET_ESTAR_MODE(hb_eclk[HB_HALF_DIV]);
2889 		/* LINTED: E_FALSE_LOGICAL_EXPR */
2890 		CHANGE_REFRESH_COUNT(HB_SPEED_UP, cur_divisor, HB_HALF_DIV);
2891 
2892 		SET_SELF_REFRESH(HB_SELF_REFRESH_DISABLE);
2893 		SET_ESTAR_MODE(hb_eclk[new_divisor]);
2894 		/* LINTED: E_FALSE_LOGICAL_EXPR */
2895 		CHANGE_REFRESH_COUNT(HB_SPEED_UP, HB_HALF_DIV, new_divisor);
2896 
2897 	} else if (cur_divisor < new_divisor) {
2898 		CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, cur_divisor, new_divisor);
2899 		SET_ESTAR_MODE(hb_eclk[new_divisor]);
2900 
2901 	} else if (cur_divisor > new_divisor) {
2902 		SET_ESTAR_MODE(hb_eclk[new_divisor]);
2903 		/* LINTED: E_FALSE_LOGICAL_EXPR */
2904 		CHANGE_REFRESH_COUNT(HB_SPEED_UP, cur_divisor, new_divisor);
2905 	}
2906 	CPU->cpu_m.divisor = (uchar_t)new_divisor;
2907 #endif
2908 }
2909 
2910 /*
2911  * Clear the AFSR sticky bits and the UDBs. For Sabre/Spitfire/Blackbird,
2912  * we clear all the sticky bits. If a non-null pointer to a async fault
2913  * structure argument is passed in, the captured error state (AFSR, AFAR, UDBs)
2914  * info will be returned in the structure.  If a non-null pointer to a
2915  * uint64_t is passed in, this will be updated if the CP bit is set in the
2916  * AFSR.  The afsr will be returned.
2917  */
2918 static uint64_t
2919 clear_errors(spitf_async_flt *spf_flt, uint64_t *acc_afsr)
2920 {
2921 	struct async_flt *aflt = (struct async_flt *)spf_flt;
2922 	uint64_t afsr;
2923 	uint64_t udbh, udbl;
2924 
2925 	get_asyncflt(&afsr);
2926 
2927 	if ((acc_afsr != NULL) && (afsr & P_AFSR_CP))
2928 		*acc_afsr |= afsr;
2929 
2930 	if (spf_flt != NULL) {
2931 		aflt->flt_stat = afsr;
2932 		get_asyncaddr(&aflt->flt_addr);
2933 		aflt->flt_addr &= SABRE_AFAR_PA;
2934 
2935 		get_udb_errors(&udbh, &udbl);
2936 		spf_flt->flt_sdbh = (ushort_t)(udbh & 0x3FF);
2937 		spf_flt->flt_sdbl = (ushort_t)(udbl & 0x3FF);
2938 	}
2939 
2940 	set_asyncflt(afsr);		/* clear afsr */
2941 	clr_datapath();			/* clear udbs */
2942 	return (afsr);
2943 }
2944 
2945 /*
2946  * Scan the ecache to look for bad lines.  If found, the afsr, afar, e$ data
2947  * tag of the first bad line will be returned. We also return the old-afsr
2948  * (before clearing the sticky bits). The linecnt data will be updated to
2949  * indicate the number of bad lines detected.
2950  */
2951 static void
2952 scan_ecache(uint64_t *t_afar, ec_data_t *ecache_data,
2953 	uint64_t *ecache_tag, int *linecnt, uint64_t *t_afsr)
2954 {
2955 	ec_data_t t_ecdata[8];
2956 	uint64_t t_etag, oafsr;
2957 	uint64_t pa = AFLT_INV_ADDR;
2958 	uint32_t i, j, ecache_sz;
2959 	uint64_t acc_afsr = 0;
2960 	uint64_t *cpu_afsr = NULL;
2961 
2962 	if (CPU_PRIVATE(CPU) != NULL)
2963 		cpu_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr);
2964 
2965 	*linecnt = 0;
2966 	ecache_sz = cpunodes[CPU->cpu_id].ecache_size;
2967 
2968 	for (i = 0; i < ecache_sz; i += 64) {
2969 		get_ecache_dtag(i, (uint64_t *)&t_ecdata[0], &t_etag, &oafsr,
2970 		    cpu_afsr);
2971 		acc_afsr |= oafsr;
2972 
2973 		/*
2974 		 * Scan through the whole 64 bytes line in 8 8-byte chunks
2975 		 * looking for the first occurrence of an EDP error.  The AFSR
2976 		 * info is captured for each 8-byte chunk.  Note that for
2977 		 * Spitfire/Blackbird, the AFSR.PSYND is captured by h/w in
2978 		 * 16-byte chunk granularity (i.e. the AFSR will be the same
2979 		 * for the high and low 8-byte words within the 16-byte chunk).
2980 		 * For Sabre/Hummingbird, the AFSR.PSYND is captured in 8-byte
2981 		 * granularity and only PSYND bits [7:0] are used.
2982 		 */
2983 		for (j = 0; j < 8; j++) {
2984 			ec_data_t *ecdptr = &t_ecdata[j];
2985 
2986 			if (ecdptr->ec_afsr & P_AFSR_EDP) {
2987 				uint64_t errpa;
2988 				ushort_t psynd;
2989 				uint32_t ec_set_size = ecache_sz /
2990 				    ecache_associativity;
2991 
2992 				/*
2993 				 * For Spitfire/Blackbird, we need to look at
2994 				 * the PSYND to make sure that this 8-byte chunk
2995 				 * is the right one.  PSYND bits [15:8] belong
2996 				 * to the upper 8-byte (even) chunk.  Bits
2997 				 * [7:0] belong to the lower 8-byte chunk (odd).
2998 				 */
2999 				psynd = ecdptr->ec_afsr & P_AFSR_P_SYND;
3000 				if (!isus2i && !isus2e) {
3001 					if (j & 0x1)
3002 						psynd = psynd & 0xFF;
3003 					else
3004 						psynd = psynd >> 8;
3005 
3006 					if (!psynd)
3007 						continue; /* wrong chunk */
3008 				}
3009 
3010 				/* Construct the PA */
3011 				errpa = ((t_etag & cpu_ec_tag_mask) <<
3012 				    cpu_ec_tag_shift) | ((i | (j << 3)) %
3013 				    ec_set_size);
3014 
3015 				/* clean up the cache line */
3016 				flushecacheline(P2ALIGN(errpa, 64),
3017 					cpunodes[CPU->cpu_id].ecache_size);
3018 
3019 				oafsr = clear_errors(NULL, cpu_afsr);
3020 				acc_afsr |= oafsr;
3021 
3022 				(*linecnt)++;
3023 
3024 				/*
3025 				 * Capture the PA for the first bad line found.
3026 				 * Return the ecache dump and tag info.
3027 				 */
3028 				if (pa == AFLT_INV_ADDR) {
3029 					int k;
3030 
3031 					pa = errpa;
3032 					for (k = 0; k < 8; k++)
3033 						ecache_data[k] = t_ecdata[k];
3034 					*ecache_tag = t_etag;
3035 				}
3036 				break;
3037 			}
3038 		}
3039 	}
3040 	*t_afar = pa;
3041 	*t_afsr = acc_afsr;
3042 }
3043 
3044 static void
3045 cpu_log_ecmem_info(spitf_async_flt *spf_flt)
3046 {
3047 	struct async_flt *aflt = (struct async_flt *)spf_flt;
3048 	uint64_t ecache_tag = spf_flt->flt_ec_tag;
3049 	char linestr[30];
3050 	char *state_str;
3051 	int i;
3052 
3053 	/*
3054 	 * Check the ecache tag to make sure it
3055 	 * is valid. If invalid, a memory dump was
3056 	 * captured instead of a ecache dump.
3057 	 */
3058 	if (spf_flt->flt_ec_tag != AFLT_INV_ADDR) {
3059 		uchar_t eparity = (uchar_t)
3060 		    ((ecache_tag & cpu_ec_par_mask) >> cpu_ec_par_shift);
3061 
3062 		uchar_t estate = (uchar_t)
3063 		    ((ecache_tag & cpu_ec_state_mask) >> cpu_ec_state_shift);
3064 
3065 		if (estate == cpu_ec_state_shr)
3066 			state_str = "Shared";
3067 		else if (estate == cpu_ec_state_exl)
3068 			state_str = "Exclusive";
3069 		else if (estate == cpu_ec_state_own)
3070 			state_str = "Owner";
3071 		else if (estate == cpu_ec_state_mod)
3072 			state_str = "Modified";
3073 		else
3074 			state_str = "Invalid";
3075 
3076 		if (spf_flt->flt_ec_lcnt > 1) {
3077 			(void) snprintf(linestr, sizeof (linestr),
3078 			    "Badlines found=%d", spf_flt->flt_ec_lcnt);
3079 		} else {
3080 			linestr[0] = '\0';
3081 		}
3082 
3083 		cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, NULL,
3084 		    " PA=0x%08x.%08x\n    E$tag 0x%08x.%08x E$State: %s "
3085 		    "E$parity 0x%02x %s", (uint32_t)(aflt->flt_addr >> 32),
3086 		    (uint32_t)aflt->flt_addr, (uint32_t)(ecache_tag >> 32),
3087 		    (uint32_t)ecache_tag, state_str,
3088 		    (uint32_t)eparity, linestr);
3089 	} else {
3090 		cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, NULL,
3091 		    " E$tag != PA from AFAR; E$line was victimized"
3092 		    "\n    dumping memory from PA 0x%08x.%08x instead",
3093 		    (uint32_t)(P2ALIGN(aflt->flt_addr, 64) >> 32),
3094 		    (uint32_t)P2ALIGN(aflt->flt_addr, 64));
3095 	}
3096 
3097 	/*
3098 	 * Dump out all 8 8-byte ecache data captured
3099 	 * For each 8-byte data captured, we check the
3100 	 * captured afsr's parity syndrome to find out
3101 	 * which 8-byte chunk is bad. For memory dump, the
3102 	 * AFSR values were initialized to 0.
3103 	 */
3104 	for (i = 0; i < 8; i++) {
3105 		ec_data_t *ecdptr;
3106 		uint_t offset;
3107 		ushort_t psynd;
3108 		ushort_t bad;
3109 		uint64_t edp;
3110 
3111 		offset = i << 3;	/* multiply by 8 */
3112 		ecdptr = &spf_flt->flt_ec_data[i];
3113 		psynd = ecdptr->ec_afsr & P_AFSR_P_SYND;
3114 		edp = ecdptr->ec_afsr & P_AFSR_EDP;
3115 
3116 		/*
3117 		 * For Sabre/Hummingbird, parity synd is captured only
3118 		 * in [7:0] of AFSR.PSYND for each 8-byte chunk.
3119 		 * For spitfire/blackbird, AFSR.PSYND is captured
3120 		 * in 16-byte granularity. [15:8] represent
3121 		 * the upper 8 byte and [7:0] the lower 8 byte.
3122 		 */
3123 		if (isus2i || isus2e || (i & 0x1))
3124 			bad = (psynd & 0xFF);		/* check bits [7:0] */
3125 		else
3126 			bad = (psynd & 0xFF00);		/* check bits [15:8] */
3127 
3128 		if (bad && edp) {
3129 			cpu_aflt_log(CE_CONT, 2, spf_flt, NO_LFLAGS, NULL,
3130 			    " E$Data (0x%02x): 0x%08x.%08x "
3131 			    "*Bad* PSYND=0x%04x", offset,
3132 			    (uint32_t)(ecdptr->ec_d8 >> 32),
3133 			    (uint32_t)ecdptr->ec_d8, psynd);
3134 		} else {
3135 			cpu_aflt_log(CE_CONT, 2, spf_flt, NO_LFLAGS, NULL,
3136 			    " E$Data (0x%02x): 0x%08x.%08x", offset,
3137 			    (uint32_t)(ecdptr->ec_d8 >> 32),
3138 			    (uint32_t)ecdptr->ec_d8);
3139 		}
3140 	}
3141 }
3142 
3143 /*
3144  * Common logging function for all cpu async errors.  This function allows the
3145  * caller to generate a single cmn_err() call that logs the appropriate items
3146  * from the fault structure, and implements our rules for AFT logging levels.
3147  *
3148  *	ce_code: cmn_err() code (e.g. CE_PANIC, CE_WARN, CE_CONT)
3149  *	tagnum: 0, 1, 2, .. generate the [AFT#] tag
3150  *	spflt: pointer to spitfire async fault structure
3151  *	logflags: bitflags indicating what to output
3152  *	endstr: a end string to appear at the end of this log
3153  *	fmt: a format string to appear at the beginning of the log
3154  *
3155  * The logflags allows the construction of predetermined output from the spflt
3156  * structure.  The individual data items always appear in a consistent order.
3157  * Note that either or both of the spflt structure pointer and logflags may be
3158  * NULL or zero respectively, indicating that the predetermined output
3159  * substrings are not requested in this log.  The output looks like this:
3160  *
3161  *	[AFT#] <CPU_ERRID_FIRST><fmt string><CPU_FLTCPU>
3162  *	<CPU_SPACE><CPU_ERRID>
3163  *	newline+4spaces<CPU_AFSR><CPU_AFAR>
3164  *	newline+4spaces<CPU_AF_PSYND><CPU_AF_ETS><CPU_FAULTPC>
3165  *	newline+4spaces<CPU_UDBH><CPU_UDBL>
3166  *	newline+4spaces<CPU_SYND>
3167  *	newline+4spaces<endstr>
3168  *
3169  * Note that <endstr> may not start on a newline if we are logging <CPU_PSYND>;
3170  * it is assumed that <endstr> will be the unum string in this case.  The size
3171  * of our intermediate formatting buf[] is based on the worst case of all flags
3172  * being enabled.  We pass the caller's varargs directly to vcmn_err() for
3173  * formatting so we don't need additional stack space to format them here.
3174  */
3175 /*PRINTFLIKE6*/
3176 static void
3177 cpu_aflt_log(int ce_code, int tagnum, spitf_async_flt *spflt, uint_t logflags,
3178 	const char *endstr, const char *fmt, ...)
3179 {
3180 	struct async_flt *aflt = (struct async_flt *)spflt;
3181 	char buf[400], *p, *q; /* see comments about buf[] size above */
3182 	va_list ap;
3183 	int console_log_flag;
3184 
3185 	if ((aflt == NULL) || ((aflt->flt_class == CPU_FAULT) &&
3186 				(aflt->flt_stat & P_AFSR_LEVEL1)) ||
3187 	    (aflt->flt_panic)) {
3188 		console_log_flag = (tagnum < 2) || aft_verbose;
3189 	} else {
3190 		int verbose = ((aflt->flt_class == BUS_FAULT) ||
3191 		    (aflt->flt_stat & P_AFSR_CE)) ?
3192 		    ce_verbose_memory : ce_verbose_other;
3193 
3194 		if (!verbose)
3195 			return;
3196 
3197 		console_log_flag = (verbose > 1);
3198 	}
3199 
3200 	if (console_log_flag)
3201 		(void) sprintf(buf, "[AFT%d]", tagnum);
3202 	else
3203 		(void) sprintf(buf, "![AFT%d]", tagnum);
3204 
3205 	p = buf + strlen(buf);	/* current buffer position */
3206 	q = buf + sizeof (buf);	/* pointer past end of buffer */
3207 
3208 	if (spflt != NULL && (logflags & CPU_ERRID_FIRST)) {
3209 		(void) snprintf(p, (size_t)(q - p), " errID 0x%08x.%08x",
3210 		    (uint32_t)(aflt->flt_id >> 32), (uint32_t)aflt->flt_id);
3211 		p += strlen(p);
3212 	}
3213 
3214 	/*
3215 	 * Copy the caller's format string verbatim into buf[].  It will be
3216 	 * formatted by the call to vcmn_err() at the end of this function.
3217 	 */
3218 	if (fmt != NULL && p < q) {
3219 		(void) strncpy(p, fmt, (size_t)(q - p - 1));
3220 		buf[sizeof (buf) - 1] = '\0';
3221 		p += strlen(p);
3222 	}
3223 
3224 	if (spflt != NULL) {
3225 		if (logflags & CPU_FLTCPU) {
3226 			(void) snprintf(p, (size_t)(q - p), " CPU%d",
3227 			    aflt->flt_inst);
3228 			p += strlen(p);
3229 		}
3230 
3231 		if (logflags & CPU_SPACE) {
3232 			if (aflt->flt_status & ECC_D_TRAP)
3233 				(void) snprintf(p, (size_t)(q - p),
3234 				    " Data access");
3235 			else if (aflt->flt_status & ECC_I_TRAP)
3236 				(void) snprintf(p, (size_t)(q - p),
3237 				    " Instruction access");
3238 			p += strlen(p);
3239 		}
3240 
3241 		if (logflags & CPU_TL) {
3242 			(void) snprintf(p, (size_t)(q - p), " at TL%s",
3243 			    aflt->flt_tl ? ">0" : "=0");
3244 			p += strlen(p);
3245 		}
3246 
3247 		if (logflags & CPU_ERRID) {
3248 			(void) snprintf(p, (size_t)(q - p),
3249 			    ", errID 0x%08x.%08x",
3250 			    (uint32_t)(aflt->flt_id >> 32),
3251 			    (uint32_t)aflt->flt_id);
3252 			p += strlen(p);
3253 		}
3254 
3255 		if (logflags & CPU_AFSR) {
3256 			(void) snprintf(p, (size_t)(q - p),
3257 			    "\n    AFSR 0x%08b.%08b",
3258 			    (uint32_t)(aflt->flt_stat >> 32), AFSR_FMTSTR0,
3259 			    (uint32_t)aflt->flt_stat, AFSR_FMTSTR1);
3260 			p += strlen(p);
3261 		}
3262 
3263 		if (logflags & CPU_AFAR) {
3264 			(void) snprintf(p, (size_t)(q - p), " AFAR 0x%08x.%08x",
3265 			    (uint32_t)(aflt->flt_addr >> 32),
3266 			    (uint32_t)aflt->flt_addr);
3267 			p += strlen(p);
3268 		}
3269 
3270 		if (logflags & CPU_AF_PSYND) {
3271 			ushort_t psynd = (ushort_t)
3272 			    (aflt->flt_stat & P_AFSR_P_SYND);
3273 
3274 			(void) snprintf(p, (size_t)(q - p),
3275 			    "\n    AFSR.PSYND 0x%04x(Score %02d)",
3276 			    psynd, ecc_psynd_score(psynd));
3277 			p += strlen(p);
3278 		}
3279 
3280 		if (logflags & CPU_AF_ETS) {
3281 			(void) snprintf(p, (size_t)(q - p), " AFSR.ETS 0x%02x",
3282 			    (uchar_t)((aflt->flt_stat & P_AFSR_ETS) >> 16));
3283 			p += strlen(p);
3284 		}
3285 
3286 		if (logflags & CPU_FAULTPC) {
3287 			(void) snprintf(p, (size_t)(q - p), " Fault_PC 0x%p",
3288 			    (void *)aflt->flt_pc);
3289 			p += strlen(p);
3290 		}
3291 
3292 		if (logflags & CPU_UDBH) {
3293 			(void) snprintf(p, (size_t)(q - p),
3294 			    "\n    UDBH 0x%04b UDBH.ESYND 0x%02x",
3295 			    spflt->flt_sdbh, UDB_FMTSTR,
3296 			    spflt->flt_sdbh & 0xFF);
3297 			p += strlen(p);
3298 		}
3299 
3300 		if (logflags & CPU_UDBL) {
3301 			(void) snprintf(p, (size_t)(q - p),
3302 			    " UDBL 0x%04b UDBL.ESYND 0x%02x",
3303 			    spflt->flt_sdbl, UDB_FMTSTR,
3304 			    spflt->flt_sdbl & 0xFF);
3305 			p += strlen(p);
3306 		}
3307 
3308 		if (logflags & CPU_SYND) {
3309 			ushort_t synd = SYND(aflt->flt_synd);
3310 
3311 			(void) snprintf(p, (size_t)(q - p),
3312 			    "\n    %s Syndrome 0x%x Memory Module ",
3313 			    UDBL(aflt->flt_synd) ? "UDBL" : "UDBH", synd);
3314 			p += strlen(p);
3315 		}
3316 	}
3317 
3318 	if (endstr != NULL) {
3319 		if (!(logflags & CPU_SYND))
3320 			(void) snprintf(p, (size_t)(q - p), "\n    %s", endstr);
3321 		else
3322 			(void) snprintf(p, (size_t)(q - p), "%s", endstr);
3323 		p += strlen(p);
3324 	}
3325 
3326 	if (ce_code == CE_CONT && (p < q - 1))
3327 		(void) strcpy(p, "\n"); /* add final \n if needed */
3328 
3329 	va_start(ap, fmt);
3330 	vcmn_err(ce_code, buf, ap);
3331 	va_end(ap);
3332 }
3333 
3334 /*
3335  * Ecache Scrubbing
3336  *
3337  * The basic idea is to prevent lines from sitting in the ecache long enough
3338  * to build up soft errors which can lead to ecache parity errors.
3339  *
3340  * The following rules are observed when flushing the ecache:
3341  *
3342  * 1. When the system is busy, flush bad clean lines
3343  * 2. When the system is idle, flush all clean lines
3344  * 3. When the system is idle, flush good dirty lines
3345  * 4. Never flush bad dirty lines.
3346  *
3347  *	modify	parity	busy   idle
3348  *	----------------------------
3349  *	clean	good		X
3350  * 	clean	bad	X	X
3351  * 	dirty	good		X
3352  *	dirty	bad
3353  *
3354  * Bad or good refers to whether a line has an E$ parity error or not.
3355  * Clean or dirty refers to the state of the modified bit.  We currently
3356  * default the scan rate to 100 (scan 10% of the cache per second).
3357  *
3358  * The following are E$ states and actions.
3359  *
3360  * We encode our state as a 3-bit number, consisting of:
3361  *	ECACHE_STATE_MODIFIED	(0=clean, 1=dirty)
3362  *	ECACHE_STATE_PARITY	(0=good,  1=bad)
3363  *	ECACHE_STATE_BUSY	(0=idle,  1=busy)
3364  *
3365  * We associate a flushing and a logging action with each state.
3366  *
3367  * E$ actions are different for Spitfire and Sabre/Hummingbird modules.
3368  * MIRROR_FLUSH indicates that an E$ line will be flushed for the mirrored
3369  * E$ only, in addition to value being set by ec_flush.
3370  */
3371 
3372 #define	ALWAYS_FLUSH		0x1	/* flush E$ line on all E$ types */
3373 #define	NEVER_FLUSH		0x0	/* never the flush the E$ line */
3374 #define	MIRROR_FLUSH		0xF	/* flush E$ line on mirrored E$ only */
3375 
3376 struct {
3377 	char	ec_flush;		/* whether to flush or not */
3378 	char	ec_log;			/* ecache logging */
3379 	char	ec_log_type;		/* log type info */
3380 } ec_action[] = {	/* states of the E$ line in M P B */
3381 	{ ALWAYS_FLUSH, 0, 0 },			 /* 0 0 0 clean_good_idle */
3382 	{ MIRROR_FLUSH, 0, 0 },			 /* 0 0 1 clean_good_busy */
3383 	{ ALWAYS_FLUSH, 1, CPU_BADLINE_CI_ERR }, /* 0 1 0 clean_bad_idle */
3384 	{ ALWAYS_FLUSH, 1, CPU_BADLINE_CB_ERR }, /* 0 1 1 clean_bad_busy */
3385 	{ ALWAYS_FLUSH, 0, 0 },			 /* 1 0 0 dirty_good_idle */
3386 	{ MIRROR_FLUSH, 0, 0 },			 /* 1 0 1 dirty_good_busy */
3387 	{ NEVER_FLUSH, 1, CPU_BADLINE_DI_ERR },	 /* 1 1 0 dirty_bad_idle */
3388 	{ NEVER_FLUSH, 1, CPU_BADLINE_DB_ERR }	 /* 1 1 1 dirty_bad_busy */
3389 };
3390 
3391 /*
3392  * Offsets into the ec_action[] that determines clean_good_busy and
3393  * dirty_good_busy lines.
3394  */
3395 #define	ECACHE_CGB_LINE		1	/* E$ clean_good_busy line */
3396 #define	ECACHE_DGB_LINE		5	/* E$ dirty_good_busy line */
3397 
3398 /*
3399  * We are flushing lines which are Clean_Good_Busy and also the lines
3400  * Dirty_Good_Busy. And we only follow it for non-mirrored E$.
3401  */
3402 #define	CGB(x, m)	(((x) == ECACHE_CGB_LINE) && (m != ECACHE_CPU_MIRROR))
3403 #define	DGB(x, m)	(((x) == ECACHE_DGB_LINE) && (m != ECACHE_CPU_MIRROR))
3404 
3405 #define	ECACHE_STATE_MODIFIED	0x4
3406 #define	ECACHE_STATE_PARITY	0x2
3407 #define	ECACHE_STATE_BUSY	0x1
3408 
3409 /*
3410  * If ecache is mirrored ecache_calls_a_sec and ecache_scan_rate are reduced.
3411  */
3412 int ecache_calls_a_sec_mirrored = 1;
3413 int ecache_lines_per_call_mirrored = 1;
3414 
3415 int ecache_scrub_enable = 1;	/* ecache scrubbing is on by default */
3416 int ecache_scrub_verbose = 1;		/* prints clean and dirty lines */
3417 int ecache_scrub_panic = 0;		/* panics on a clean and dirty line */
3418 int ecache_calls_a_sec = 100;		/* scrubber calls per sec */
3419 int ecache_scan_rate = 100;		/* scan rate (in tenths of a percent) */
3420 int ecache_idle_factor = 1;		/* increase the scan rate when idle */
3421 int ecache_flush_clean_good_busy = 50;	/* flush rate (in percent) */
3422 int ecache_flush_dirty_good_busy = 100;	/* flush rate (in percent) */
3423 
3424 volatile int ec_timeout_calls = 1;	/* timeout calls */
3425 
3426 /*
3427  * Interrupt number and pil for ecache scrubber cross-trap calls.
3428  */
3429 static uint_t ecache_scrub_inum;
3430 uint_t ecache_scrub_pil = PIL_9;
3431 
3432 /*
3433  * Kstats for the E$ scrubber.
3434  */
3435 typedef struct ecache_kstat {
3436 	kstat_named_t clean_good_idle;		/* # of lines scrubbed */
3437 	kstat_named_t clean_good_busy;		/* # of lines skipped */
3438 	kstat_named_t clean_bad_idle;		/* # of lines scrubbed */
3439 	kstat_named_t clean_bad_busy;		/* # of lines scrubbed */
3440 	kstat_named_t dirty_good_idle;		/* # of lines scrubbed */
3441 	kstat_named_t dirty_good_busy;		/* # of lines skipped */
3442 	kstat_named_t dirty_bad_idle;		/* # of lines skipped */
3443 	kstat_named_t dirty_bad_busy;		/* # of lines skipped */
3444 	kstat_named_t invalid_lines;		/* # of invalid lines */
3445 	kstat_named_t clean_good_busy_flush;    /* # of lines scrubbed */
3446 	kstat_named_t dirty_good_busy_flush;    /* # of lines scrubbed */
3447 	kstat_named_t tags_cleared;		/* # of E$ tags cleared */
3448 } ecache_kstat_t;
3449 
3450 static ecache_kstat_t ec_kstat_template = {
3451 	{ "clean_good_idle", KSTAT_DATA_ULONG },
3452 	{ "clean_good_busy", KSTAT_DATA_ULONG },
3453 	{ "clean_bad_idle", KSTAT_DATA_ULONG },
3454 	{ "clean_bad_busy", KSTAT_DATA_ULONG },
3455 	{ "dirty_good_idle", KSTAT_DATA_ULONG },
3456 	{ "dirty_good_busy", KSTAT_DATA_ULONG },
3457 	{ "dirty_bad_idle", KSTAT_DATA_ULONG },
3458 	{ "dirty_bad_busy", KSTAT_DATA_ULONG },
3459 	{ "invalid_lines", KSTAT_DATA_ULONG },
3460 	{ "clean_good_busy_flush", KSTAT_DATA_ULONG },
3461 	{ "dirty_good_busy_flush", KSTAT_DATA_ULONG },
3462 	{ "ecache_tags_cleared", KSTAT_DATA_ULONG }
3463 };
3464 
3465 struct kmem_cache *sf_private_cache;
3466 
3467 /*
3468  * Called periodically on each CPU to scan the ecache once a sec.
3469  * adjusting the ecache line index appropriately
3470  */
3471 void
3472 scrub_ecache_line()
3473 {
3474 	spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(CPU, sfpr_scrub_misc);
3475 	int cpuid = CPU->cpu_id;
3476 	uint32_t index = ssmp->ecache_flush_index;
3477 	uint64_t ec_size = cpunodes[cpuid].ecache_size;
3478 	size_t ec_linesize = cpunodes[cpuid].ecache_linesize;
3479 	int nlines = ssmp->ecache_nlines;
3480 	uint32_t ec_set_size = ec_size / ecache_associativity;
3481 	int ec_mirror = ssmp->ecache_mirror;
3482 	ecache_kstat_t *ec_ksp = (ecache_kstat_t *)ssmp->ecache_ksp->ks_data;
3483 
3484 	int line, scan_lines, flush_clean_busy = 0, flush_dirty_busy = 0;
3485 	int mpb;		/* encode Modified, Parity, Busy for action */
3486 	uchar_t state;
3487 	uint64_t ec_tag, paddr, oafsr, tafsr, nafsr;
3488 	uint64_t *acc_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr);
3489 	ec_data_t ec_data[8];
3490 	kstat_named_t *ec_knp;
3491 
3492 	switch (ec_mirror) {
3493 		default:
3494 		case ECACHE_CPU_NON_MIRROR:
3495 			/*
3496 			 * The E$ scan rate is expressed in units of tenths of
3497 			 * a percent.  ecache_scan_rate = 1000 (100%) means the
3498 			 * whole cache is scanned every second.
3499 			 */
3500 			scan_lines = (nlines * ecache_scan_rate) /
3501 					(1000 * ecache_calls_a_sec);
3502 			if (!(ssmp->ecache_busy)) {
3503 				if (ecache_idle_factor > 0) {
3504 					scan_lines *= ecache_idle_factor;
3505 				}
3506 			} else {
3507 				flush_clean_busy = (scan_lines *
3508 					ecache_flush_clean_good_busy) / 100;
3509 				flush_dirty_busy = (scan_lines *
3510 					ecache_flush_dirty_good_busy) / 100;
3511 			}
3512 
3513 			ec_timeout_calls = (ecache_calls_a_sec ?
3514 						ecache_calls_a_sec : 1);
3515 			break;
3516 
3517 		case ECACHE_CPU_MIRROR:
3518 			scan_lines = ecache_lines_per_call_mirrored;
3519 			ec_timeout_calls = (ecache_calls_a_sec_mirrored ?
3520 					ecache_calls_a_sec_mirrored : 1);
3521 			break;
3522 	}
3523 
3524 	/*
3525 	 * The ecache scrubber algorithm operates by reading and
3526 	 * decoding the E$ tag to determine whether the corresponding E$ line
3527 	 * can be scrubbed. There is a implicit assumption in the scrubber
3528 	 * logic that the E$ tag is valid. Unfortunately, this assertion is
3529 	 * flawed since the E$ tag may also be corrupted and have parity errors
3530 	 * The scrubber logic is enhanced to check the validity of the E$ tag
3531 	 * before scrubbing. When a parity error is detected in the E$ tag,
3532 	 * it is possible to recover and scrub the tag under certain conditions
3533 	 * so that a ETP error condition can be avoided.
3534 	 */
3535 
3536 	for (mpb = line = 0; line < scan_lines; line++, mpb = 0) {
3537 		/*
3538 		 * We get the old-AFSR before clearing the AFSR sticky bits
3539 		 * in {get_ecache_tag, check_ecache_line, get_ecache_dtag}
3540 		 * If CP bit is set in the old-AFSR, we log an Orphan CP event.
3541 		 */
3542 		ec_tag = get_ecache_tag(index, &nafsr, acc_afsr);
3543 		state = (uchar_t)((ec_tag & cpu_ec_state_mask) >>
3544 				cpu_ec_state_shift);
3545 
3546 		/*
3547 		 * ETP is set try to scrub the ecache tag.
3548 		 */
3549 		if (nafsr & P_AFSR_ETP) {
3550 			ecache_scrub_tag_err(nafsr, state, index);
3551 		} else if (state & cpu_ec_state_valid) {
3552 			/*
3553 			 * ETP is not set, E$ tag is valid.
3554 			 * Proceed with the E$ scrubbing.
3555 			 */
3556 			if (state & cpu_ec_state_dirty)
3557 				mpb |= ECACHE_STATE_MODIFIED;
3558 
3559 			tafsr = check_ecache_line(index, acc_afsr);
3560 
3561 			if (tafsr & P_AFSR_EDP) {
3562 				mpb |= ECACHE_STATE_PARITY;
3563 
3564 				if (ecache_scrub_verbose ||
3565 							ecache_scrub_panic) {
3566 					get_ecache_dtag(P2ALIGN(index, 64),
3567 						(uint64_t *)&ec_data[0],
3568 						&ec_tag, &oafsr, acc_afsr);
3569 				}
3570 			}
3571 
3572 			if (ssmp->ecache_busy)
3573 				mpb |= ECACHE_STATE_BUSY;
3574 
3575 			ec_knp = (kstat_named_t *)ec_ksp + mpb;
3576 			ec_knp->value.ul++;
3577 
3578 			paddr = ((ec_tag & cpu_ec_tag_mask) <<
3579 				cpu_ec_tag_shift) | (index % ec_set_size);
3580 
3581 			/*
3582 			 * We flush the E$ lines depending on the ec_flush,
3583 			 * we additionally flush clean_good_busy and
3584 			 * dirty_good_busy lines for mirrored E$.
3585 			 */
3586 			if (ec_action[mpb].ec_flush == ALWAYS_FLUSH) {
3587 				flushecacheline(paddr, ec_size);
3588 			} else if ((ec_mirror == ECACHE_CPU_MIRROR) &&
3589 				(ec_action[mpb].ec_flush == MIRROR_FLUSH)) {
3590 					flushecacheline(paddr, ec_size);
3591 			} else if (ec_action[mpb].ec_flush == NEVER_FLUSH) {
3592 				softcall(ecache_page_retire, (void *)paddr);
3593 			}
3594 
3595 			/*
3596 			 * Conditionally flush both the clean_good and
3597 			 * dirty_good lines when busy.
3598 			 */
3599 			if (CGB(mpb, ec_mirror) && (flush_clean_busy > 0)) {
3600 				flush_clean_busy--;
3601 				flushecacheline(paddr, ec_size);
3602 				ec_ksp->clean_good_busy_flush.value.ul++;
3603 			} else if (DGB(mpb, ec_mirror) &&
3604 						(flush_dirty_busy > 0)) {
3605 				flush_dirty_busy--;
3606 				flushecacheline(paddr, ec_size);
3607 				ec_ksp->dirty_good_busy_flush.value.ul++;
3608 			}
3609 
3610 			if (ec_action[mpb].ec_log && (ecache_scrub_verbose ||
3611 						ecache_scrub_panic)) {
3612 				ecache_scrub_log(ec_data, ec_tag, paddr, mpb,
3613 						tafsr);
3614 			}
3615 
3616 		} else {
3617 			ec_ksp->invalid_lines.value.ul++;
3618 		}
3619 
3620 		if ((index += ec_linesize) >= ec_size)
3621 			index = 0;
3622 
3623 	}
3624 
3625 	/*
3626 	 * set the ecache scrub index for the next time around
3627 	 */
3628 	ssmp->ecache_flush_index = index;
3629 
3630 	if (*acc_afsr & P_AFSR_CP) {
3631 		uint64_t ret_afsr;
3632 
3633 		ret_afsr = ecache_scrub_misc_err(CPU_ORPHAN_CP_ERR, *acc_afsr);
3634 		if ((ret_afsr & P_AFSR_CP) == 0)
3635 			*acc_afsr = 0;
3636 	}
3637 }
3638 
3639 /*
3640  * Handler for ecache_scrub_inum softint.  Call scrub_ecache_line until
3641  * we decrement the outstanding request count to zero.
3642  */
3643 
3644 /*ARGSUSED*/
3645 uint_t
3646 scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2)
3647 {
3648 	int i;
3649 	int outstanding;
3650 	spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(CPU, sfpr_scrub_misc);
3651 	uint32_t *countp = &ssmp->ec_scrub_outstanding;
3652 
3653 	do {
3654 		outstanding = *countp;
3655 		ASSERT(outstanding > 0);
3656 		for (i = 0; i < outstanding; i++)
3657 			scrub_ecache_line();
3658 	} while (atomic_add_32_nv(countp, -outstanding));
3659 
3660 	return (DDI_INTR_CLAIMED);
3661 }
3662 
3663 /*
3664  * force each cpu to perform an ecache scrub, called from a timeout
3665  */
3666 extern xcfunc_t ecache_scrubreq_tl1;
3667 
3668 void
3669 do_scrub_ecache_line(void)
3670 {
3671 	long delta;
3672 
3673 	if (ecache_calls_a_sec > hz)
3674 		ecache_calls_a_sec = hz;
3675 	else if (ecache_calls_a_sec <= 0)
3676 	    ecache_calls_a_sec = 1;
3677 
3678 	if (ecache_calls_a_sec_mirrored > hz)
3679 		ecache_calls_a_sec_mirrored = hz;
3680 	else if (ecache_calls_a_sec_mirrored <= 0)
3681 	    ecache_calls_a_sec_mirrored = 1;
3682 
3683 	if (ecache_scrub_enable) {
3684 		xt_all(ecache_scrubreq_tl1, ecache_scrub_inum, 0);
3685 		delta = hz / ec_timeout_calls;
3686 	} else {
3687 		delta = hz;
3688 	}
3689 
3690 	(void) realtime_timeout((void(*)(void *))do_scrub_ecache_line, 0,
3691 		delta);
3692 }
3693 
3694 /*
3695  * initialization for ecache scrubbing
3696  * This routine is called AFTER all cpus have had cpu_init_private called
3697  * to initialize their private data areas.
3698  */
3699 void
3700 cpu_init_cache_scrub(void)
3701 {
3702 	if (ecache_calls_a_sec > hz) {
3703 		cmn_err(CE_NOTE, "ecache_calls_a_sec set too high (%d); "
3704 		    "resetting to hz (%d)", ecache_calls_a_sec, hz);
3705 		ecache_calls_a_sec = hz;
3706 	}
3707 
3708 	/*
3709 	 * Register softint for ecache scrubbing.
3710 	 */
3711 	ecache_scrub_inum = add_softintr(ecache_scrub_pil,
3712 	    scrub_ecache_line_intr, NULL);
3713 
3714 	/*
3715 	 * kick off the scrubbing using realtime timeout
3716 	 */
3717 	(void) realtime_timeout((void(*)(void *))do_scrub_ecache_line, 0,
3718 	    hz / ecache_calls_a_sec);
3719 }
3720 
3721 /*
3722  * Unset the busy flag for this cpu.
3723  */
3724 void
3725 cpu_idle_ecache_scrub(struct cpu *cp)
3726 {
3727 	if (CPU_PRIVATE(cp) != NULL) {
3728 		spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp,
3729 							sfpr_scrub_misc);
3730 		ssmp->ecache_busy = ECACHE_CPU_IDLE;
3731 	}
3732 }
3733 
3734 /*
3735  * Set the busy flag for this cpu.
3736  */
3737 void
3738 cpu_busy_ecache_scrub(struct cpu *cp)
3739 {
3740 	if (CPU_PRIVATE(cp) != NULL) {
3741 		spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp,
3742 							sfpr_scrub_misc);
3743 		ssmp->ecache_busy = ECACHE_CPU_BUSY;
3744 	}
3745 }
3746 
3747 /*
3748  * initialize the ecache scrubber data structures
3749  * The global entry point cpu_init_private replaces this entry point.
3750  *
3751  */
3752 static void
3753 cpu_init_ecache_scrub_dr(struct cpu *cp)
3754 {
3755 	spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, sfpr_scrub_misc);
3756 	int cpuid = cp->cpu_id;
3757 
3758 	/*
3759 	 * intialize bookkeeping for cache scrubbing
3760 	 */
3761 	bzero(ssmp, sizeof (spitfire_scrub_misc_t));
3762 
3763 	ssmp->ecache_flush_index = 0;
3764 
3765 	ssmp->ecache_nlines =
3766 		cpunodes[cpuid].ecache_size / cpunodes[cpuid].ecache_linesize;
3767 
3768 	/*
3769 	 * Determine whether we are running on mirrored SRAM
3770 	 */
3771 
3772 	if (cpunodes[cpuid].msram == ECACHE_CPU_MIRROR)
3773 		ssmp->ecache_mirror = ECACHE_CPU_MIRROR;
3774 	else
3775 		ssmp->ecache_mirror = ECACHE_CPU_NON_MIRROR;
3776 
3777 	cpu_busy_ecache_scrub(cp);
3778 
3779 	/*
3780 	 * initialize the kstats
3781 	 */
3782 	ecache_kstat_init(cp);
3783 }
3784 
3785 /*
3786  * uninitialize the ecache scrubber data structures
3787  * The global entry point cpu_uninit_private replaces this entry point.
3788  */
3789 static void
3790 cpu_uninit_ecache_scrub_dr(struct cpu *cp)
3791 {
3792 	spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, sfpr_scrub_misc);
3793 
3794 	if (ssmp->ecache_ksp != NULL) {
3795 		kstat_delete(ssmp->ecache_ksp);
3796 		ssmp->ecache_ksp = NULL;
3797 	}
3798 
3799 	/*
3800 	 * un-initialize bookkeeping for cache scrubbing
3801 	 */
3802 	bzero(ssmp, sizeof (spitfire_scrub_misc_t));
3803 
3804 	cpu_idle_ecache_scrub(cp);
3805 }
3806 
3807 struct kmem_cache *sf_private_cache;
3808 
3809 /*
3810  * Cpu private initialization.  This includes allocating the cpu_private
3811  * data structure, initializing it, and initializing the scrubber for this
3812  * cpu.  This is called once for EVERY cpu, including CPU 0. This function
3813  * calls cpu_init_ecache_scrub_dr to init the scrubber.
3814  * We use kmem_cache_create for the spitfire private data structure because it
3815  * needs to be allocated on a S_ECACHE_MAX_LSIZE (64) byte boundary.
3816  */
3817 void
3818 cpu_init_private(struct cpu *cp)
3819 {
3820 	spitfire_private_t *sfprp;
3821 
3822 	ASSERT(CPU_PRIVATE(cp) == NULL);
3823 
3824 	/*
3825 	 * If the sf_private_cache has not been created, create it.
3826 	 */
3827 	if (sf_private_cache == NULL) {
3828 		sf_private_cache = kmem_cache_create("sf_private_cache",
3829 			sizeof (spitfire_private_t), S_ECACHE_MAX_LSIZE, NULL,
3830 			NULL, NULL, NULL, NULL, 0);
3831 		ASSERT(sf_private_cache);
3832 	}
3833 
3834 	sfprp = CPU_PRIVATE(cp) = kmem_cache_alloc(sf_private_cache, KM_SLEEP);
3835 
3836 	bzero(sfprp, sizeof (spitfire_private_t));
3837 
3838 	cpu_init_ecache_scrub_dr(cp);
3839 }
3840 
3841 /*
3842  * Cpu private unitialization.  Uninitialize the Ecache scrubber and
3843  * deallocate the scrubber data structures and cpu_private data structure.
3844  * For now, this function just calls cpu_unint_ecache_scrub_dr to uninit
3845  * the scrubber for the specified cpu.
3846  */
3847 void
3848 cpu_uninit_private(struct cpu *cp)
3849 {
3850 	ASSERT(CPU_PRIVATE(cp));
3851 
3852 	cpu_uninit_ecache_scrub_dr(cp);
3853 	kmem_cache_free(sf_private_cache, CPU_PRIVATE(cp));
3854 	CPU_PRIVATE(cp) = NULL;
3855 }
3856 
3857 /*
3858  * initialize the ecache kstats for each cpu
3859  */
3860 static void
3861 ecache_kstat_init(struct cpu *cp)
3862 {
3863 	struct kstat *ksp;
3864 	spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, sfpr_scrub_misc);
3865 
3866 	ASSERT(ssmp != NULL);
3867 
3868 	if ((ksp = kstat_create("unix", cp->cpu_id, "ecache_kstat", "misc",
3869 	    KSTAT_TYPE_NAMED,
3870 	    sizeof (ecache_kstat_t) / sizeof (kstat_named_t),
3871 	    KSTAT_FLAG_WRITABLE)) == NULL) {
3872 		ssmp->ecache_ksp = NULL;
3873 		cmn_err(CE_NOTE, "!ecache_kstat_init(%d) failed\n", cp->cpu_id);
3874 		return;
3875 	}
3876 
3877 	ssmp->ecache_ksp = ksp;
3878 	bcopy(&ec_kstat_template, ksp->ks_data, sizeof (ecache_kstat_t));
3879 	kstat_install(ksp);
3880 }
3881 
3882 /*
3883  * log the bad ecache information
3884  */
3885 static void
3886 ecache_scrub_log(ec_data_t *ec_data, uint64_t ec_tag, uint64_t paddr, int mpb,
3887 		uint64_t afsr)
3888 {
3889 	spitf_async_flt spf_flt;
3890 	struct async_flt *aflt;
3891 	int i;
3892 	char *class;
3893 
3894 	bzero(&spf_flt, sizeof (spitf_async_flt));
3895 	aflt = &spf_flt.cmn_asyncflt;
3896 
3897 	for (i = 0; i < 8; i++) {
3898 		spf_flt.flt_ec_data[i] = ec_data[i];
3899 	}
3900 
3901 	spf_flt.flt_ec_tag = ec_tag;
3902 
3903 	if (mpb < (sizeof (ec_action) / sizeof (ec_action[0]))) {
3904 		spf_flt.flt_type = ec_action[mpb].ec_log_type;
3905 	} else spf_flt.flt_type = (ushort_t)mpb;
3906 
3907 	aflt->flt_inst = CPU->cpu_id;
3908 	aflt->flt_class = CPU_FAULT;
3909 	aflt->flt_id = gethrtime_waitfree();
3910 	aflt->flt_addr = paddr;
3911 	aflt->flt_stat = afsr;
3912 	aflt->flt_panic = (uchar_t)ecache_scrub_panic;
3913 
3914 	switch (mpb) {
3915 	case CPU_ECACHE_TAG_ERR:
3916 	case CPU_ECACHE_ADDR_PAR_ERR:
3917 	case CPU_ECACHE_ETP_ETS_ERR:
3918 	case CPU_ECACHE_STATE_ERR:
3919 		class = FM_EREPORT_CPU_USII_ESCRUB_TAG;
3920 		break;
3921 	default:
3922 		class = FM_EREPORT_CPU_USII_ESCRUB_DATA;
3923 		break;
3924 	}
3925 
3926 	cpu_errorq_dispatch(class, (void *)&spf_flt, sizeof (spf_flt),
3927 	    ue_queue, aflt->flt_panic);
3928 
3929 	if (aflt->flt_panic)
3930 		cmn_err(CE_PANIC, "ecache_scrub_panic set and bad E$"
3931 					"line detected");
3932 }
3933 
3934 /*
3935  * Process an ecache error that occured during the E$ scrubbing.
3936  * We do the ecache scan to find the bad line, flush the bad line
3937  * and start the memscrubber to find any UE (in memory or in another cache)
3938  */
3939 static uint64_t
3940 ecache_scrub_misc_err(int type, uint64_t afsr)
3941 {
3942 	spitf_async_flt spf_flt;
3943 	struct async_flt *aflt;
3944 	uint64_t oafsr;
3945 
3946 	bzero(&spf_flt, sizeof (spitf_async_flt));
3947 	aflt = &spf_flt.cmn_asyncflt;
3948 
3949 	/*
3950 	 * Scan each line in the cache to look for the one
3951 	 * with bad parity
3952 	 */
3953 	aflt->flt_addr = AFLT_INV_ADDR;
3954 	scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0],
3955 		&spf_flt.flt_ec_tag, &spf_flt.flt_ec_lcnt, &oafsr);
3956 
3957 	if (oafsr & P_AFSR_CP) {
3958 		uint64_t *cp_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr);
3959 		*cp_afsr |= oafsr;
3960 	}
3961 
3962 	/*
3963 	 * If we found a bad PA, update the state to indicate if it is
3964 	 * memory or I/O space.
3965 	 */
3966 	if (aflt->flt_addr != AFLT_INV_ADDR) {
3967 		aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >>
3968 			MMU_PAGESHIFT)) ? 1 : 0;
3969 	}
3970 
3971 	spf_flt.flt_type = (ushort_t)type;
3972 
3973 	aflt->flt_inst = CPU->cpu_id;
3974 	aflt->flt_class = CPU_FAULT;
3975 	aflt->flt_id = gethrtime_waitfree();
3976 	aflt->flt_status = afsr;
3977 	aflt->flt_panic = (uchar_t)ecache_scrub_panic;
3978 
3979 	/*
3980 	 * We have the bad line, flush that line and start
3981 	 * the memscrubber.
3982 	 */
3983 	if (spf_flt.flt_ec_lcnt > 0) {
3984 		flushecacheline(P2ALIGN(aflt->flt_addr, 64),
3985 			cpunodes[CPU->cpu_id].ecache_size);
3986 		read_all_memscrub = 1;
3987 		memscrub_run();
3988 	}
3989 
3990 	cpu_errorq_dispatch((type == CPU_ORPHAN_CP_ERR) ?
3991 	    FM_EREPORT_CPU_USII_CP : FM_EREPORT_CPU_USII_UNKNOWN,
3992 	    (void *)&spf_flt, sizeof (spf_flt), ue_queue, aflt->flt_panic);
3993 
3994 	return (oafsr);
3995 }
3996 
3997 static void
3998 ecache_scrub_tag_err(uint64_t afsr, uchar_t state, uint32_t index)
3999 {
4000 	ushort_t afsr_ets = (afsr & P_AFSR_ETS) >> P_AFSR_ETS_SHIFT;
4001 	spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(CPU, sfpr_scrub_misc);
4002 	ecache_kstat_t *ec_ksp = (ecache_kstat_t *)ssmp->ecache_ksp->ks_data;
4003 	uint64_t ec_tag, paddr, oafsr;
4004 	ec_data_t ec_data[8];
4005 	int cpuid = CPU->cpu_id;
4006 	uint32_t ec_set_size = cpunodes[cpuid].ecache_size /
4007 						ecache_associativity;
4008 	uint64_t *cpu_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr);
4009 
4010 	get_ecache_dtag(P2ALIGN(index, 64), (uint64_t *)&ec_data[0], &ec_tag,
4011 			&oafsr, cpu_afsr);
4012 	paddr = ((ec_tag & cpu_ec_tag_mask) << cpu_ec_tag_shift) |
4013 						(index % ec_set_size);
4014 
4015 	/*
4016 	 * E$ tag state has good parity
4017 	 */
4018 	if ((afsr_ets & cpu_ec_state_parity) == 0) {
4019 		if (afsr_ets & cpu_ec_parity) {
4020 			/*
4021 			 * E$ tag state bits indicate the line is clean,
4022 			 * invalidate the E$ tag and continue.
4023 			 */
4024 			if (!(state & cpu_ec_state_dirty)) {
4025 				/*
4026 				 * Zero the tag and mark the state invalid
4027 				 * with good parity for the tag.
4028 				 */
4029 				if (isus2i || isus2e)
4030 					write_hb_ec_tag_parity(index);
4031 				else
4032 					write_ec_tag_parity(index);
4033 
4034 				/* Sync with the dual tag */
4035 				flushecacheline(0,
4036 					cpunodes[CPU->cpu_id].ecache_size);
4037 				ec_ksp->tags_cleared.value.ul++;
4038 				ecache_scrub_log(ec_data, ec_tag, paddr,
4039 					CPU_ECACHE_TAG_ERR, afsr);
4040 				return;
4041 			} else {
4042 				ecache_scrub_log(ec_data, ec_tag, paddr,
4043 					CPU_ECACHE_ADDR_PAR_ERR, afsr);
4044 				cmn_err(CE_PANIC, " E$ tag address has bad"
4045 							" parity");
4046 			}
4047 		} else if ((afsr_ets & cpu_ec_parity) == 0) {
4048 			/*
4049 			 * ETS is zero but ETP is set
4050 			 */
4051 			ecache_scrub_log(ec_data, ec_tag, paddr,
4052 				CPU_ECACHE_ETP_ETS_ERR, afsr);
4053 			cmn_err(CE_PANIC, "AFSR.ETP is set and"
4054 				" AFSR.ETS is zero");
4055 		}
4056 	} else {
4057 		/*
4058 		 * E$ tag state bit has a bad parity
4059 		 */
4060 		ecache_scrub_log(ec_data, ec_tag, paddr,
4061 				CPU_ECACHE_STATE_ERR, afsr);
4062 		cmn_err(CE_PANIC, "E$ tag state has bad parity");
4063 	}
4064 }
4065 
4066 static void
4067 ecache_page_retire(void *arg)
4068 {
4069 	uint64_t paddr = (uint64_t)arg;
4070 	(void) page_retire(paddr, PR_UE);
4071 }
4072 
4073 void
4074 sticksync_slave(void)
4075 {}
4076 
4077 void
4078 sticksync_master(void)
4079 {}
4080 
4081 /*ARGSUSED*/
4082 void
4083 cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t bpp)
4084 {}
4085 
4086 void
4087 cpu_run_bus_error_handlers(struct async_flt *aflt, int expected)
4088 {
4089 	int status;
4090 	ddi_fm_error_t de;
4091 
4092 	bzero(&de, sizeof (ddi_fm_error_t));
4093 
4094 	de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst,
4095 	    FM_ENA_FMT1);
4096 	de.fme_flag = expected;
4097 	de.fme_bus_specific = (void *)aflt->flt_addr;
4098 	status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de);
4099 
4100 	if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL))
4101 		aflt->flt_panic = 1;
4102 }
4103 
4104 /*ARGSUSED*/
4105 void
4106 cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz,
4107     errorq_t *eqp, uint_t flag)
4108 {
4109 	struct async_flt *aflt = (struct async_flt *)payload;
4110 
4111 	aflt->flt_erpt_class = error_class;
4112 	errorq_dispatch(eqp, payload, payload_sz, flag);
4113 }
4114 
4115 #define	MAX_SIMM	8
4116 
4117 struct ce_info {
4118 	char    name[UNUM_NAMLEN];
4119 	uint64_t intermittent_total;
4120 	uint64_t persistent_total;
4121 	uint64_t sticky_total;
4122 	unsigned short leaky_bucket_cnt;
4123 };
4124 
4125 /*
4126  * Separately-defined structure for use in reporting the ce_info
4127  * to SunVTS without exposing the internal layout and implementation
4128  * of struct ce_info.
4129  */
4130 static struct ecc_error_info ecc_error_info_data = {
4131 	{ "version", KSTAT_DATA_UINT32 },
4132 	{ "maxcount", KSTAT_DATA_UINT32 },
4133 	{ "count", KSTAT_DATA_UINT32 }
4134 };
4135 static const size_t ecc_error_info_ndata = sizeof (ecc_error_info_data) /
4136     sizeof (struct kstat_named);
4137 
4138 #if KSTAT_CE_UNUM_NAMLEN < UNUM_NAMLEN
4139 #error "Need to rev ecc_error_info version and update KSTAT_CE_UNUM_NAMLEN"
4140 #endif
4141 
4142 struct ce_info  *mem_ce_simm = NULL;
4143 size_t mem_ce_simm_size = 0;
4144 
4145 /*
4146  * Default values for the number of CE's allowed per interval.
4147  * Interval is defined in minutes
4148  * SOFTERR_MIN_TIMEOUT is defined in microseconds
4149  */
4150 #define	SOFTERR_LIMIT_DEFAULT		2
4151 #define	SOFTERR_INTERVAL_DEFAULT	1440		/* This is 24 hours */
4152 #define	SOFTERR_MIN_TIMEOUT		(60 * MICROSEC)	/* This is 1 minute */
4153 #define	TIMEOUT_NONE			((timeout_id_t)0)
4154 #define	TIMEOUT_SET			((timeout_id_t)1)
4155 
4156 /*
4157  * timeout identifer for leaky_bucket
4158  */
4159 static timeout_id_t leaky_bucket_timeout_id = TIMEOUT_NONE;
4160 
4161 /*
4162  * Tunables for maximum number of allowed CE's in a given time
4163  */
4164 int ecc_softerr_limit = SOFTERR_LIMIT_DEFAULT;
4165 int ecc_softerr_interval = SOFTERR_INTERVAL_DEFAULT;
4166 
4167 void
4168 cpu_mp_init(void)
4169 {
4170 	size_t size = cpu_aflt_size();
4171 	size_t i;
4172 	kstat_t *ksp;
4173 
4174 	/*
4175 	 * Initialize the CE error handling buffers.
4176 	 */
4177 	mem_ce_simm_size = MAX_SIMM * max_ncpus;
4178 	size = sizeof (struct ce_info) * mem_ce_simm_size;
4179 	mem_ce_simm = kmem_zalloc(size, KM_SLEEP);
4180 
4181 	ksp = kstat_create("unix", 0, "ecc-info", "misc",
4182 	    KSTAT_TYPE_NAMED, ecc_error_info_ndata, KSTAT_FLAG_VIRTUAL);
4183 	if (ksp != NULL) {
4184 		ksp->ks_data = (struct kstat_named *)&ecc_error_info_data;
4185 		ecc_error_info_data.version.value.ui32 = KSTAT_CE_INFO_VER;
4186 		ecc_error_info_data.maxcount.value.ui32 = mem_ce_simm_size;
4187 		ecc_error_info_data.count.value.ui32 = 0;
4188 		kstat_install(ksp);
4189 	}
4190 
4191 	for (i = 0; i < mem_ce_simm_size; i++) {
4192 		struct kstat_ecc_mm_info *kceip;
4193 
4194 		kceip = kmem_zalloc(sizeof (struct kstat_ecc_mm_info),
4195 		    KM_SLEEP);
4196 		ksp = kstat_create("mm", i, "ecc-info", "misc",
4197 		    KSTAT_TYPE_NAMED,
4198 		    sizeof (struct kstat_ecc_mm_info) / sizeof (kstat_named_t),
4199 		    KSTAT_FLAG_VIRTUAL);
4200 		if (ksp != NULL) {
4201 			/*
4202 			 * Re-declare ks_data_size to include room for the
4203 			 * UNUM name since we don't have KSTAT_FLAG_VAR_SIZE
4204 			 * set.
4205 			 */
4206 			ksp->ks_data_size = sizeof (struct kstat_ecc_mm_info) +
4207 			    KSTAT_CE_UNUM_NAMLEN;
4208 			ksp->ks_data = kceip;
4209 			kstat_named_init(&kceip->name,
4210 			    "name", KSTAT_DATA_STRING);
4211 			kstat_named_init(&kceip->intermittent_total,
4212 			    "intermittent_total", KSTAT_DATA_UINT64);
4213 			kstat_named_init(&kceip->persistent_total,
4214 			    "persistent_total", KSTAT_DATA_UINT64);
4215 			kstat_named_init(&kceip->sticky_total,
4216 			    "sticky_total", KSTAT_DATA_UINT64);
4217 			/*
4218 			 * Use the default snapshot routine as it knows how to
4219 			 * deal with named kstats with long strings.
4220 			 */
4221 			ksp->ks_update = ecc_kstat_update;
4222 			kstat_install(ksp);
4223 		} else {
4224 			kmem_free(kceip, sizeof (struct kstat_ecc_mm_info));
4225 		}
4226 	}
4227 }
4228 
4229 /*ARGSUSED*/
4230 static void
4231 leaky_bucket_timeout(void *arg)
4232 {
4233 	int i;
4234 	struct ce_info *psimm = mem_ce_simm;
4235 
4236 	for (i = 0; i < mem_ce_simm_size; i++) {
4237 		if (psimm[i].leaky_bucket_cnt > 0)
4238 			atomic_add_16(&psimm[i].leaky_bucket_cnt, -1);
4239 	}
4240 	add_leaky_bucket_timeout();
4241 }
4242 
4243 static void
4244 add_leaky_bucket_timeout(void)
4245 {
4246 	long timeout_in_microsecs;
4247 
4248 	/*
4249 	 * create timeout for next leak.
4250 	 *
4251 	 * The timeout interval is calculated as follows
4252 	 *
4253 	 * (ecc_softerr_interval * 60 * MICROSEC) / ecc_softerr_limit
4254 	 *
4255 	 * ecc_softerr_interval is in minutes, so multiply this by 60 (seconds
4256 	 * in a minute), then multiply this by MICROSEC to get the interval
4257 	 * in microseconds.  Divide this total by ecc_softerr_limit so that
4258 	 * the timeout interval is accurate to within a few microseconds.
4259 	 */
4260 
4261 	if (ecc_softerr_limit <= 0)
4262 		ecc_softerr_limit = SOFTERR_LIMIT_DEFAULT;
4263 	if (ecc_softerr_interval <= 0)
4264 		ecc_softerr_interval = SOFTERR_INTERVAL_DEFAULT;
4265 
4266 	timeout_in_microsecs = ((int64_t)ecc_softerr_interval * 60 * MICROSEC) /
4267 	    ecc_softerr_limit;
4268 
4269 	if (timeout_in_microsecs < SOFTERR_MIN_TIMEOUT)
4270 		timeout_in_microsecs = SOFTERR_MIN_TIMEOUT;
4271 
4272 	leaky_bucket_timeout_id = timeout(leaky_bucket_timeout,
4273 	    (void *)NULL, drv_usectohz((clock_t)timeout_in_microsecs));
4274 }
4275 
4276 /*
4277  * Legacy Correctable ECC Error Hash
4278  *
4279  * All of the code below this comment is used to implement a legacy array
4280  * which counted intermittent, persistent, and sticky CE errors by unum,
4281  * and then was later extended to publish the data as a kstat for SunVTS.
4282  * All of this code is replaced by FMA, and remains here until such time
4283  * that the UltraSPARC-I/II CPU code is converted to FMA, or is EOLed.
4284  *
4285  * Errors are saved in three buckets per-unum:
4286  * (1) sticky - scrub was unsuccessful, cannot be scrubbed
4287  *     This could represent a problem, and is immediately printed out.
4288  * (2) persistent - was successfully scrubbed
4289  *     These errors use the leaky bucket algorithm to determine
4290  *     if there is a serious problem.
4291  * (3) intermittent - may have originated from the cpu or upa/safari bus,
4292  *     and does not necessarily indicate any problem with the dimm itself,
4293  *     is critical information for debugging new hardware.
4294  *     Because we do not know if it came from the dimm, it would be
4295  *     inappropriate to include these in the leaky bucket counts.
4296  *
4297  * If the E$ line was modified before the scrub operation began, then the
4298  * displacement flush at the beginning of scrubphys() will cause the modified
4299  * line to be written out, which will clean up the CE.  Then, any subsequent
4300  * read will not cause an error, which will cause persistent errors to be
4301  * identified as intermittent.
4302  *
4303  * If a DIMM is going bad, it will produce true persistents as well as
4304  * false intermittents, so these intermittents can be safely ignored.
4305  *
4306  * If the error count is excessive for a DIMM, this function will return
4307  * PR_MCE, and the CPU module may then decide to remove that page from use.
4308  */
4309 static int
4310 ce_count_unum(int status, int len, char *unum)
4311 {
4312 	int i;
4313 	struct ce_info *psimm = mem_ce_simm;
4314 	int page_status = PR_OK;
4315 
4316 	ASSERT(psimm != NULL);
4317 
4318 	if (len <= 0 ||
4319 	    (status & (ECC_STICKY | ECC_PERSISTENT | ECC_INTERMITTENT)) == 0)
4320 		return (page_status);
4321 
4322 	/*
4323 	 * Initialize the leaky_bucket timeout
4324 	 */
4325 	if (casptr(&leaky_bucket_timeout_id,
4326 	    TIMEOUT_NONE, TIMEOUT_SET) == TIMEOUT_NONE)
4327 		add_leaky_bucket_timeout();
4328 
4329 	for (i = 0; i < mem_ce_simm_size; i++) {
4330 		if (psimm[i].name[0] == '\0') {
4331 			/*
4332 			 * Hit the end of the valid entries, add
4333 			 * a new one.
4334 			 */
4335 			(void) strncpy(psimm[i].name, unum, len);
4336 			if (status & ECC_STICKY) {
4337 				/*
4338 				 * Sticky - the leaky bucket is used to track
4339 				 * soft errors.  Since a sticky error is a
4340 				 * hard error and likely to be retired soon,
4341 				 * we do not count it in the leaky bucket.
4342 				 */
4343 				psimm[i].leaky_bucket_cnt = 0;
4344 				psimm[i].intermittent_total = 0;
4345 				psimm[i].persistent_total = 0;
4346 				psimm[i].sticky_total = 1;
4347 				cmn_err(CE_WARN,
4348 				    "[AFT0] Sticky Softerror encountered "
4349 				    "on Memory Module %s\n", unum);
4350 				page_status = PR_MCE;
4351 			} else if (status & ECC_PERSISTENT) {
4352 				psimm[i].leaky_bucket_cnt = 1;
4353 				psimm[i].intermittent_total = 0;
4354 				psimm[i].persistent_total = 1;
4355 				psimm[i].sticky_total = 0;
4356 			} else {
4357 				/*
4358 				 * Intermittent - Because the scrub operation
4359 				 * cannot find the error in the DIMM, we will
4360 				 * not count these in the leaky bucket
4361 				 */
4362 				psimm[i].leaky_bucket_cnt = 0;
4363 				psimm[i].intermittent_total = 1;
4364 				psimm[i].persistent_total = 0;
4365 				psimm[i].sticky_total = 0;
4366 			}
4367 			ecc_error_info_data.count.value.ui32++;
4368 			break;
4369 		} else if (strncmp(unum, psimm[i].name, len) == 0) {
4370 			/*
4371 			 * Found an existing entry for the current
4372 			 * memory module, adjust the counts.
4373 			 */
4374 			if (status & ECC_STICKY) {
4375 				psimm[i].sticky_total++;
4376 				cmn_err(CE_WARN,
4377 				    "[AFT0] Sticky Softerror encountered "
4378 				    "on Memory Module %s\n", unum);
4379 				page_status = PR_MCE;
4380 			} else if (status & ECC_PERSISTENT) {
4381 				int new_value;
4382 
4383 				new_value = atomic_add_16_nv(
4384 				    &psimm[i].leaky_bucket_cnt, 1);
4385 				psimm[i].persistent_total++;
4386 				if (new_value > ecc_softerr_limit) {
4387 					cmn_err(CE_WARN, "[AFT0] Most recent %d"
4388 					    " soft errors from Memory Module"
4389 					    " %s exceed threshold (N=%d,"
4390 					    " T=%dh:%02dm) triggering page"
4391 					    " retire", new_value, unum,
4392 					    ecc_softerr_limit,
4393 					    ecc_softerr_interval / 60,
4394 					    ecc_softerr_interval % 60);
4395 					atomic_add_16(
4396 					    &psimm[i].leaky_bucket_cnt, -1);
4397 					page_status = PR_MCE;
4398 				}
4399 			} else { /* Intermittent */
4400 				psimm[i].intermittent_total++;
4401 			}
4402 			break;
4403 		}
4404 	}
4405 
4406 	if (i >= mem_ce_simm_size)
4407 		cmn_err(CE_CONT, "[AFT0] Softerror: mem_ce_simm[] out of "
4408 		    "space.\n");
4409 
4410 	return (page_status);
4411 }
4412 
4413 /*
4414  * Function to support counting of IO detected CEs.
4415  */
4416 void
4417 cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum)
4418 {
4419 	int err;
4420 
4421 	err = ce_count_unum(ecc->flt_status, len, unum);
4422 	if (err != PR_OK && automatic_page_removal) {
4423 		(void) page_retire(ecc->flt_addr, err);
4424 	}
4425 }
4426 
4427 static int
4428 ecc_kstat_update(kstat_t *ksp, int rw)
4429 {
4430 	struct kstat_ecc_mm_info *kceip = ksp->ks_data;
4431 	struct ce_info *ceip = mem_ce_simm;
4432 	int i = ksp->ks_instance;
4433 
4434 	if (rw == KSTAT_WRITE)
4435 		return (EACCES);
4436 
4437 	ASSERT(ksp->ks_data != NULL);
4438 	ASSERT(i < mem_ce_simm_size && i >= 0);
4439 
4440 	/*
4441 	 * Since we're not using locks, make sure that we don't get partial
4442 	 * data. The name is always copied before the counters are incremented
4443 	 * so only do this update routine if at least one of the counters is
4444 	 * non-zero, which ensures that ce_count_unum() is done, and the
4445 	 * string is fully copied.
4446 	 */
4447 	if (ceip[i].intermittent_total == 0 &&
4448 	    ceip[i].persistent_total == 0 &&
4449 	    ceip[i].sticky_total == 0) {
4450 		/*
4451 		 * Uninitialized or partially initialized. Ignore.
4452 		 * The ks_data buffer was allocated via kmem_zalloc,
4453 		 * so no need to bzero it.
4454 		 */
4455 		return (0);
4456 	}
4457 
4458 	kstat_named_setstr(&kceip->name, ceip[i].name);
4459 	kceip->intermittent_total.value.ui64 = ceip[i].intermittent_total;
4460 	kceip->persistent_total.value.ui64 = ceip[i].persistent_total;
4461 	kceip->sticky_total.value.ui64 = ceip[i].sticky_total;
4462 
4463 	return (0);
4464 }
4465 
4466 #define	VIS_BLOCKSIZE		64
4467 
4468 int
4469 dtrace_blksuword32_err(uintptr_t addr, uint32_t *data)
4470 {
4471 	int ret, watched;
4472 
4473 	watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE);
4474 	ret = dtrace_blksuword32(addr, data, 0);
4475 	if (watched)
4476 		watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE);
4477 
4478 	return (ret);
4479 }
4480 
4481 /*ARGSUSED*/
4482 void
4483 cpu_faulted_enter(struct cpu *cp)
4484 {
4485 }
4486 
4487 /*ARGSUSED*/
4488 void
4489 cpu_faulted_exit(struct cpu *cp)
4490 {
4491 }
4492 
4493 static int mmu_disable_ism_large_pages = ((1 << TTE512K) |
4494 	(1 << TTE32M) | (1 << TTE256M));
4495 static int mmu_disable_large_pages = ((1 << TTE32M) | (1 << TTE256M));
4496 
4497 /*
4498  * The function returns the US_II mmu-specific values for the
4499  * hat's disable_large_pages and disable_ism_large_pages variables.
4500  */
4501 int
4502 mmu_large_pages_disabled(uint_t flag)
4503 {
4504 	int pages_disable = 0;
4505 
4506 	if (flag == HAT_LOAD) {
4507 		pages_disable = mmu_disable_large_pages;
4508 	} else if (flag == HAT_LOAD_SHARE) {
4509 		pages_disable = mmu_disable_ism_large_pages;
4510 	}
4511 	return (pages_disable);
4512 }
4513 
4514 /*ARGSUSED*/
4515 void
4516 mmu_init_kernel_pgsz(struct hat *hat)
4517 {
4518 }
4519 
4520 size_t
4521 mmu_get_kernel_lpsize(size_t lpsize)
4522 {
4523 	uint_t tte;
4524 
4525 	if (lpsize == 0) {
4526 		/* no setting for segkmem_lpsize in /etc/system: use default */
4527 		return (MMU_PAGESIZE4M);
4528 	}
4529 
4530 	for (tte = TTE8K; tte <= TTE4M; tte++) {
4531 		if (lpsize == TTEBYTES(tte))
4532 			return (lpsize);
4533 	}
4534 
4535 	return (TTEBYTES(TTE8K));
4536 }
4537