xref: /titanic_41/usr/src/uts/sun4u/cpu/us3_jalapeno.c (revision 7c2fbfb345896881c631598ee3852ce9ce33fb07)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/systm.h>
30 #include <sys/ddi.h>
31 #include <sys/sysmacros.h>
32 #include <sys/archsystm.h>
33 #include <sys/vmsystm.h>
34 #include <sys/machparam.h>
35 #include <sys/machsystm.h>
36 #include <sys/machthread.h>
37 #include <sys/cpu.h>
38 #include <sys/cmp.h>
39 #include <sys/elf_SPARC.h>
40 #include <vm/hat_sfmmu.h>
41 #include <vm/seg_kmem.h>
42 #include <sys/cpuvar.h>
43 #include <sys/cheetahregs.h>
44 #include <sys/us3_module.h>
45 #include <sys/async.h>
46 #include <sys/cmn_err.h>
47 #include <sys/debug.h>
48 #include <sys/dditypes.h>
49 #include <sys/prom_debug.h>
50 #include <sys/prom_plat.h>
51 #include <sys/cpu_module.h>
52 #include <sys/sysmacros.h>
53 #include <sys/intreg.h>
54 #include <sys/clock.h>
55 #include <sys/platform_module.h>
56 #include <sys/machtrap.h>
57 #include <sys/ontrap.h>
58 #include <sys/panic.h>
59 #include <sys/memlist.h>
60 #include <sys/bootconf.h>
61 #include <sys/ivintr.h>
62 #include <sys/atomic.h>
63 #include <sys/fm/protocol.h>
64 #include <sys/fm/cpu/UltraSPARC-III.h>
65 #include <sys/errclassify.h>
66 
67 #ifdef	CHEETAHPLUS_ERRATUM_25
68 #include <sys/cyclic.h>
69 #endif	/* CHEETAHPLUS_ERRATUM_25 */
70 
71 /* cpu estar private data */
72 typedef struct {
73 	uint8_t state : 7;
74 	uint8_t valid : 1;
75 } mcu_fsm_def_t;
76 mcu_fsm_def_t mcu_fsm_init_state[NCPU];
77 
78 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
79 /*
80  * jp_errata_85_enable can be set to 0 in /etc/system to disable
81  * JP Errata 85 workaround.
82  *
83  * jp_errata_85_allow_slow_scrub is usually set to !jp_errata_85_enable,
84  * but can be overridden in /etc/system.  If set, it allows the scrubber
85  * to run in 1/2 or 1/32 mode.  If a cpu is vulnerable to errata 85,
86  * this value should be zero.
87  *
88  * jp_errata_85_active is an internal variable and must not be
89  * set/changed via /etc/system or in any other way.
90  */
91 extern int	jp_errata_85_enable;	/* for /etc/system use */
92 extern int	jp_errata_85_allow_slow_scrub;	/* for /etc/system use */
93 
94 int	jp_errata_85_active = -1;	/* warn: modified in code ONLY */
95 uint64_t	jp_estar_tl0_data[8];
96 uint64_t	jp_estar_tl1_data[8];
97 #endif	/* JALAPENO && JALAPENO_ERRATA_85 */
98 
99 /*
100  * Setup trap handlers.
101  */
102 void
103 cpu_init_trap(void)
104 {
105 	CH_SET_TRAP(tt_pil15, ch_pil15_interrupt_instr);
106 
107 	CH_SET_TRAP(tt0_fecc, fecc_err_instr);
108 	CH_SET_TRAP(tt1_fecc, fecc_err_tl1_instr);
109 	CH_SET_TRAP(tt1_swtrap0, fecc_err_tl1_cont_instr);
110 
111 	CH_SET_TRAP(tt0_dperr, dcache_parity_instr);
112 	CH_SET_TRAP(tt1_dperr, dcache_parity_tl1_instr);
113 	CH_SET_TRAP(tt1_swtrap1, dcache_parity_tl1_cont_instr);
114 
115 	CH_SET_TRAP(tt0_iperr, icache_parity_instr);
116 	CH_SET_TRAP(tt1_iperr, icache_parity_tl1_instr);
117 	CH_SET_TRAP(tt1_swtrap2, icache_parity_tl1_cont_instr);
118 }
119 
120 
121 static int
122 getintprop(pnode_t node, char *name, int deflt)
123 {
124 	int	value;
125 
126 	switch (prom_getproplen(node, name)) {
127 	case sizeof (int):
128 		(void) prom_getprop(node, name, (caddr_t)&value);
129 		break;
130 
131 	default:
132 		value = deflt;
133 		break;
134 	}
135 
136 	return (value);
137 }
138 
139 /*
140  * Set the magic constants of the implementation.
141  */
142 /*ARGSUSED*/
143 void
144 cpu_fiximp(pnode_t dnode)
145 {
146 	int i, a;
147 	extern int vac_size, vac_shift;
148 	extern uint_t vac_mask;
149 
150 	static struct {
151 		char	*name;
152 		int	*var;
153 		int	defval;
154 	} prop[] = {
155 		"dcache-size", &dcache_size, CH_DCACHE_SIZE,
156 		"dcache-line-size", &dcache_linesize, CH_DCACHE_LSIZE,
157 		"icache-size", &icache_size, CH_ICACHE_SIZE,
158 		"icache-line-size", &icache_linesize, CH_ICACHE_LSIZE,
159 		"ecache-size", &ecache_size, JP_ECACHE_MAX_SIZE,
160 		"ecache-line-size", &ecache_alignsize, JP_ECACHE_MAX_LSIZE,
161 		"ecache-associativity", &ecache_associativity, JP_ECACHE_NWAY
162 	};
163 
164 	for (i = 0; i < sizeof (prop) / sizeof (prop[0]); i++)
165 		*prop[i].var = getintprop(dnode, prop[i].name, prop[i].defval);
166 
167 	ecache_setsize = ecache_size / ecache_associativity;
168 
169 	vac_size = CH_VAC_SIZE;
170 	vac_mask = MMU_PAGEMASK & (vac_size - 1);
171 	i = 0; a = vac_size;
172 	while (a >>= 1)
173 		++i;
174 	vac_shift = i;
175 	shm_alignment = vac_size;
176 	vac = 1;
177 }
178 
179 void
180 send_mondo_set(cpuset_t set)
181 {
182 	int lo, busy, nack, shipped = 0;
183 	uint16_t i, cpuids[IDSR_BN_SETS];
184 	uint64_t idsr, nackmask = 0, busymask, curnack, curbusy;
185 	uint64_t starttick, endtick, tick, lasttick;
186 #ifdef	CHEETAHPLUS_ERRATUM_25
187 	int recovered = 0;
188 	int cpuid;
189 #endif
190 
191 	ASSERT(!CPUSET_ISNULL(set));
192 	starttick = lasttick = gettick();
193 
194 	/*
195 	 * Lower 2 bits of the agent ID determine which BUSY/NACK pair
196 	 * will be used for dispatching interrupt. For now, assume
197 	 * there are no more than IDSR_BN_SETS CPUs, hence no aliasing
198 	 * issues with respect to BUSY/NACK pair usage.
199 	 */
200 	for (i = 0; i < NCPU; i++)
201 		if (CPU_IN_SET(set, i)) {
202 			shipit(i, shipped /* ignored */);
203 			nackmask |= IDSR_NACK_BIT(CPUID_TO_BN_PAIR(i));
204 			cpuids[CPUID_TO_BN_PAIR(i)] = i;
205 			shipped++;
206 			CPUSET_DEL(set, i);
207 			if (CPUSET_ISNULL(set))
208 				break;
209 		}
210 	CPU_STATS_ADDQ(CPU, sys, xcalls, shipped);
211 
212 	busymask = IDSR_NACK_TO_BUSY(nackmask);
213 	busy = nack = 0;
214 	endtick = starttick + xc_tick_limit;
215 	for (;;) {
216 		idsr = getidsr();
217 		if (idsr == 0)
218 			break;
219 		tick = gettick();
220 		/*
221 		 * If there is a big jump between the current tick
222 		 * count and lasttick, we have probably hit a break
223 		 * point.  Adjust endtick accordingly to avoid panic.
224 		 */
225 		if (tick > (lasttick + xc_tick_jump_limit))
226 			endtick += (tick - lasttick);
227 		lasttick = tick;
228 		if (tick > endtick) {
229 			if (panic_quiesce)
230 				return;
231 #ifdef	CHEETAHPLUS_ERRATUM_25
232 			cpuid = -1;
233 			for (i = 0; i < IDSR_BN_SETS; i++) {
234 				if (idsr & (IDSR_NACK_BIT(i) |
235 				    IDSR_BUSY_BIT(i))) {
236 					cpuid = cpuids[i];
237 					break;
238 				}
239 			}
240 			if (cheetah_sendmondo_recover && cpuid != -1 &&
241 			    recovered == 0) {
242 				if (mondo_recover(cpuid, i)) {
243 					/*
244 					 * We claimed the whole memory or
245 					 * full scan is disabled.
246 					 */
247 					recovered++;
248 				}
249 				tick = gettick();
250 				endtick = tick + xc_tick_limit;
251 				lasttick = tick;
252 				/*
253 				 * Recheck idsr
254 				 */
255 				continue;
256 			} else
257 #endif	/* CHEETAHPLUS_ERRATUM_25 */
258 			{
259 				cmn_err(CE_CONT, "send mondo timeout "
260 				    "[%d NACK %d BUSY]\nIDSR 0x%"
261 				    "" PRIx64 "  cpuids:", nack, busy, idsr);
262 				for (i = 0; i < IDSR_BN_SETS; i++) {
263 					if (idsr & (IDSR_NACK_BIT(i) |
264 					    IDSR_BUSY_BIT(i))) {
265 						cmn_err(CE_CONT, " 0x%x",
266 						    cpuids[i]);
267 					}
268 				}
269 				cmn_err(CE_CONT, "\n");
270 				cmn_err(CE_PANIC, "send_mondo_set: timeout");
271 			}
272 		}
273 		curnack = idsr & nackmask;
274 		curbusy = idsr & busymask;
275 		if (curbusy) {
276 			busy++;
277 			continue;
278 		}
279 
280 #ifdef SEND_MONDO_STATS
281 		{
282 			int n = gettick() - starttick;
283 			if (n < 8192)
284 				x_nack_stimes[n >> 7]++;
285 		}
286 #endif
287 		while (gettick() < (tick + sys_clock_mhz))
288 			;
289 		do {
290 			lo = lowbit(curnack) - 1;
291 			i = IDSR_NACK_IDX(lo);
292 			shipit(cpuids[i], i);
293 			curnack &= ~(1ull << lo);
294 		} while (curnack);
295 		nack++;
296 		busy = 0;
297 	}
298 #ifdef SEND_MONDO_STATS
299 	{
300 		int n = gettick() - starttick;
301 		if (n < 8192)
302 			x_set_stimes[n >> 7]++;
303 		else
304 			x_set_ltimes[(n >> 13) & 0xf]++;
305 	}
306 	x_set_cpus[shipped]++;
307 #endif
308 }
309 
310 /*
311  * Handles error logging for implementation specific error types
312  */
313 int
314 cpu_impl_async_log_err(void *flt, errorq_elem_t *eqep)
315 {
316 	ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt;
317 	struct async_flt *aflt = (struct async_flt *)flt;
318 	uint64_t errors;
319 
320 	switch (ch_flt->flt_type) {
321 
322 	case CPU_IC_PARITY:
323 		cpu_async_log_ic_parity_err(flt);
324 		return (CH_ASYNC_LOG_DONE);
325 
326 	case CPU_DC_PARITY:
327 		cpu_async_log_dc_parity_err(flt);
328 		return (CH_ASYNC_LOG_DONE);
329 
330 	case CPU_RCE:
331 		if (page_retire_check(aflt->flt_addr, &errors) == EINVAL) {
332 			CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
333 			    CE_XDIAG_SKIP_NOPP);
334 		} else if (errors != PR_OK) {
335 			CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
336 			    CE_XDIAG_SKIP_PAGEDET);
337 		} else if (ce_scrub_xdiag_recirc(aflt, ce_queue, eqep,
338 		    offsetof(ch_async_flt_t, cmn_asyncflt))) {
339 			return (CH_ASYNC_LOG_RECIRC);
340 		}
341 		/*FALLTHRU*/
342 	/*
343 	 * cases where we just want to report the error and continue.
344 	 */
345 	case CPU_BPAR:
346 	case CPU_UMS:
347 	case CPU_FRC:
348 	case CPU_FRU:
349 		cpu_log_err(aflt);
350 		return (CH_ASYNC_LOG_DONE);
351 
352 	/*
353 	 * Cases where we want to fall through to handle panicking.
354 	 */
355 	case CPU_RUE:
356 		cpu_log_err(aflt);
357 		return (CH_ASYNC_LOG_CONTINUE);
358 
359 	default:
360 		return (CH_ASYNC_LOG_UNKNOWN);
361 	}
362 }
363 
364 /*
365  * Figure out if Ecache is direct-mapped (Cheetah or Cheetah+ with Ecache
366  * control ECCR_ASSOC bit off or 2-way (Cheetah+ with ECCR_ASSOC on).
367  * We need to do this on the fly because we may have mixed Cheetah+'s with
368  * both direct and 2-way Ecaches.
369  */
370 int
371 cpu_ecache_nway(void)
372 {
373 	return (JP_ECACHE_NWAY);
374 }
375 
376 /*
377  * Note that these are entered into the table in the order:
378  * Fatal Errors first, orphaned UCU/UCC, AFAR Overwrite policy,
379  * FRC/FRU, and finally IVPE.
380  *
381  * Afar overwrite policy is:
382  * Jalapeno:
383  *   UCU,UCC > RUE,UE,EDU,WDU,CPU,WBP,BP > RCE,CE,EDC,WDC,CPC >
384  *   TO,BERR > UMS,OM
385  * Serrano:
386  *   UCU,UCC > RUE,UE,EDU,WDU,CPU,WBP,BP > RCE,CE,EDC,WDC,CPC,ETI,ETC >
387  *   TO,BERR > UMS,OM
388  */
389 ecc_type_to_info_t ecc_type_to_info[] = {
390 
391 	/* Fatal Errors */
392 	C_AFSR_JETO,	"JETO ",	ECC_ALL_TRAPS,	CPU_FATAL,
393 		"JETO Fatal",
394 		FM_EREPORT_PAYLOAD_SYSTEM1,
395 		FM_EREPORT_CPU_USIII_JETO,
396 	C_AFSR_SCE,	"SCE ",		ECC_ALL_TRAPS,	CPU_FATAL,
397 		"SCE Fatal",
398 		FM_EREPORT_PAYLOAD_SYSTEM1,
399 		FM_EREPORT_CPU_USIII_SCE,
400 	C_AFSR_JEIC,	"JEIC ",	ECC_ALL_TRAPS,	CPU_FATAL,
401 		"JEIC Fatal",
402 		FM_EREPORT_PAYLOAD_SYSTEM1,
403 		FM_EREPORT_CPU_USIII_JEIC,
404 	C_AFSR_JEIT,	"JEIT ",	ECC_ALL_TRAPS,	CPU_FATAL,
405 		"JEIT Fatal",
406 		FM_EREPORT_PAYLOAD_SYSTEM1,
407 		FM_EREPORT_CPU_USIII_JEIT,
408 	C_AFSR_JEIS,	"JEIS ",	ECC_ALL_TRAPS,	CPU_FATAL,
409 		"JEIS Fatal",
410 		FM_EREPORT_PAYLOAD_SYSTEM1,
411 		FM_EREPORT_CPU_USIII_JEIS,
412 #if defined(JALAPENO)
413 	C_AFSR_ETP,	"ETP ",		ECC_ALL_TRAPS,	CPU_FATAL,
414 		"ETP Fatal",
415 		FM_EREPORT_PAYLOAD_L2_TAG_PE,
416 		FM_EREPORT_CPU_USIII_ETP,
417 #elif defined(SERRANO)
418 	C_AFSR_ETS,	"ETS ",		ECC_ASYNC_TRAPS, CPU_FATAL,
419 		"ETS Fatal",
420 		FM_EREPORT_PAYLOAD_L2_TAG_ECC,
421 		FM_EREPORT_CPU_USIII_ETS,
422 	C_AFSR_ETU,	"ETU ",		ECC_ASYNC_TRAPS, CPU_FATAL,
423 		"ETU Fatal",
424 		FM_EREPORT_PAYLOAD_L2_TAG_ECC,
425 		FM_EREPORT_CPU_USIII_ETU,
426 #endif	/* SERRANO */
427 	C_AFSR_IERR,	"IERR ", 	ECC_ALL_TRAPS,	CPU_FATAL,
428 		"IERR Fatal",
429 		FM_EREPORT_PAYLOAD_SYSTEM2,
430 		FM_EREPORT_CPU_USIII_IERR,
431 	C_AFSR_ISAP,	"ISAP ",	ECC_ALL_TRAPS,	CPU_FATAL,
432 		"ISAP Fatal",
433 		FM_EREPORT_PAYLOAD_SYSTEM1,
434 		FM_EREPORT_CPU_USIII_ISAP,
435 
436 	/* Orphaned UCU/UCC Errors */
437 	C_AFSR_UCU,	"OUCU ",	ECC_ORPH_TRAPS, CPU_ORPH,
438 		"Orphaned UCU",
439 		FM_EREPORT_PAYLOAD_L2_DATA,
440 		FM_EREPORT_CPU_USIII_UCU,
441 	C_AFSR_UCC,	"OUCC ",	ECC_ORPH_TRAPS, CPU_ORPH,
442 		"Orphaned UCC",
443 		FM_EREPORT_PAYLOAD_L2_DATA,
444 		FM_EREPORT_CPU_USIII_UCC,
445 
446 	/* UCU, UCC */
447 	C_AFSR_UCU,	"UCU ",		ECC_F_TRAP,	CPU_UE_ECACHE,
448 		"UCU",
449 		FM_EREPORT_PAYLOAD_L2_DATA,
450 		FM_EREPORT_CPU_USIII_UCU,
451 	C_AFSR_UCC,	"UCC ",		ECC_F_TRAP,	CPU_CE_ECACHE,
452 		"UCC",
453 		FM_EREPORT_PAYLOAD_L2_DATA,
454 		FM_EREPORT_CPU_USIII_UCC,
455 
456 
457 	/* RUE, UE, EDU:ST, EDU:BLD, WDU, CPU, BP, WBP */
458 	C_AFSR_RUE,	"RUE ",		ECC_ASYNC_TRAPS, CPU_RUE,
459 		"Uncorrectable remote memory/cache (RUE)",
460 		FM_EREPORT_PAYLOAD_MEMORY,
461 		FM_EREPORT_CPU_USIII_RUE,
462 	C_AFSR_UE,	"UE ",		ECC_ASYNC_TRAPS, CPU_UE,
463 		"Uncorrectable memory (UE)",
464 		FM_EREPORT_PAYLOAD_MEMORY,
465 		FM_EREPORT_CPU_USIII_UE,
466 	C_AFSR_EDU,	"EDU ",		ECC_C_TRAP,	CPU_UE_ECACHE_RETIRE,
467 		"EDU:ST",
468 		FM_EREPORT_PAYLOAD_L2_DATA,
469 		FM_EREPORT_CPU_USIII_EDUST,
470 	C_AFSR_EDU,	"EDU ",		ECC_D_TRAP,	CPU_UE_ECACHE_RETIRE,
471 		"EDU:BLD",
472 		FM_EREPORT_PAYLOAD_L2_DATA,
473 		FM_EREPORT_CPU_USIII_EDUBL,
474 	C_AFSR_WDU,	"WDU ",		ECC_C_TRAP,	CPU_UE_ECACHE_RETIRE,
475 		"WDU",
476 		FM_EREPORT_PAYLOAD_L2_DATA,
477 		FM_EREPORT_CPU_USIII_WDU,
478 	C_AFSR_CPU,	"CPU ",		ECC_C_TRAP,	CPU_UE_ECACHE,
479 		"CPU",
480 		FM_EREPORT_PAYLOAD_L2_DATA,
481 		FM_EREPORT_CPU_USIII_CPU,
482 	C_AFSR_WBP,	"WBP ",		ECC_C_TRAP,	CPU_BPAR,
483 		"JBUS parity error on writeback or block store (WBP)",
484 		FM_EREPORT_PAYLOAD_SYSTEM3,
485 		FM_EREPORT_CPU_USIII_WBP,
486 	C_AFSR_BP,	"BP ",		ECC_ASYNC_TRAPS, CPU_BPAR,
487 		"JBUS parity error on returned read data (BP)",
488 		FM_EREPORT_PAYLOAD_SYSTEM3,
489 		FM_EREPORT_CPU_USIII_BP,
490 
491 	/* RCE, CE, EDC, WDC, CPC */
492 	C_AFSR_RCE,	"RCE ",		ECC_C_TRAP,	CPU_RCE,
493 		"Corrected remote memory/cache (RCE)",
494 		FM_EREPORT_PAYLOAD_MEMORY,
495 		FM_EREPORT_CPU_USIII_RCE,
496 	C_AFSR_CE,	"CE ",		ECC_C_TRAP,	CPU_CE,
497 		"Corrected memory (CE)",
498 		FM_EREPORT_PAYLOAD_MEMORY,
499 		FM_EREPORT_CPU_USIII_CE,
500 	C_AFSR_EDC,	"EDC ",		ECC_C_TRAP,	CPU_CE_ECACHE,
501 		"EDC",
502 		FM_EREPORT_PAYLOAD_L2_DATA,
503 		FM_EREPORT_CPU_USIII_EDC,
504 	C_AFSR_WDC,	"WDC ",		ECC_C_TRAP,	CPU_CE_ECACHE,
505 		"WDC",
506 		FM_EREPORT_PAYLOAD_L2_DATA,
507 		FM_EREPORT_CPU_USIII_WDC,
508 	C_AFSR_CPC,	"CPC ",		ECC_C_TRAP,	CPU_CE_ECACHE,
509 		"CPC",
510 		FM_EREPORT_PAYLOAD_L2_DATA,
511 		FM_EREPORT_CPU_USIII_CPC,
512 #if defined(SERRANO)
513 	/* ETI, ETC */
514 	C_AFSR_ETI,	"ETI",	ECC_F_TRAP | ECC_C_TRAP, CPU_CE_ECACHE,
515 		"ETI",
516 		FM_EREPORT_PAYLOAD_L2_TAG_ECC,
517 		FM_EREPORT_CPU_USIII_ETI,
518 	C_AFSR_ETC,	"ETC",	ECC_F_TRAP | ECC_C_TRAP, CPU_CE_ECACHE,
519 		"ETC",
520 		FM_EREPORT_PAYLOAD_L2_TAG_ECC,
521 		FM_EREPORT_CPU_USIII_ETC,
522 #endif	/* SERRANO */
523 
524 	/* TO, BERR */
525 	C_AFSR_TO,	"TO ",		ECC_ASYNC_TRAPS, CPU_TO,
526 		"Timeout (TO)",
527 		FM_EREPORT_PAYLOAD_IO,
528 		FM_EREPORT_CPU_USIII_TO,
529 	C_AFSR_BERR,	"BERR ",	ECC_ASYNC_TRAPS, CPU_BERR,
530 		"Bus Error (BERR)",
531 		FM_EREPORT_PAYLOAD_IO,
532 		FM_EREPORT_CPU_USIII_BERR,
533 
534 	/* UMS, OM */
535 	C_AFSR_UMS,	"UMS ",		ECC_C_TRAP,	 CPU_UMS,
536 		"Unsupported store (UMS)",
537 		FM_EREPORT_PAYLOAD_IO,
538 		FM_EREPORT_CPU_USIII_UMS,
539 	C_AFSR_OM,	"OM ",		ECC_ASYNC_TRAPS, CPU_BERR,
540 		"Out of range memory (OM)",
541 		FM_EREPORT_PAYLOAD_IO,
542 		FM_EREPORT_CPU_USIII_OM,
543 
544 	/* FRC, FRU */
545 	C_AFSR_FRC,	"FRC ",		ECC_C_TRAP,	CPU_FRC,
546 		"Corrected memory (FRC)",
547 		FM_EREPORT_PAYLOAD_MEMORY,
548 		FM_EREPORT_CPU_USIII_FRC,
549 	C_AFSR_FRU,	"FRU ",		ECC_C_TRAP,	 CPU_FRU,
550 		"Uncorrectable memory (FRU)",
551 		FM_EREPORT_PAYLOAD_MEMORY,
552 		FM_EREPORT_CPU_USIII_FRU,
553 
554 	/* IVPE */
555 	C_AFSR_IVPE,	"IVPE ",	ECC_C_TRAP,	CPU_IV,
556 		"IVPE",
557 		FM_EREPORT_PAYLOAD_SYSTEM1,
558 		FM_EREPORT_CPU_USIII_IVPE,
559 
560 	0,		NULL,		0,		0,
561 		NULL,
562 		FM_EREPORT_PAYLOAD_UNKNOWN,
563 		FM_EREPORT_CPU_USIII_UNKNOWN,
564 };
565 
566 /*
567  * J_REQ overwrite policy (see UltraSPARC-IIIi PRM)
568  *
569  *   Class 4:  RUE, BP, WBP
570  *   Class 3:  RCE
571  *   Class 2:  TO, BERR
572  *   Class 1:  UMS
573  */
574 uint64_t jreq_overwrite[] = {
575 	C_AFSR_RUE | C_AFSR_BP | C_AFSR_WBP,
576 	C_AFSR_RCE,
577 	C_AFSR_TO | C_AFSR_BERR,
578 	C_AFSR_UMS,
579 	0
580 };
581 
582 /*
583  * AGENT ID overwrite policy (see UltraSPARC-IIIi PRM)
584  *
585  *   Class 2:  CPU, FRU
586  *   Class 1:  CPC, FRC
587  */
588 uint64_t jbus_aid_overwrite[] = {
589 	C_AFSR_CPU | C_AFSR_FRU,
590 	C_AFSR_CPC | C_AFSR_FRC,
591 	0
592 };
593 
594 int
595 afsr_to_jaid_status(uint64_t afsr, uint64_t afsr_bit)
596 {
597 	return (afsr_to_overw_status(afsr, afsr_bit, jbus_aid_overwrite));
598 }
599 
600 /*
601  * See UltraSPARC-IIIi+ PRM
602  *   Class 5:  ETS, ETU, EFES
603  *   Class 4:  UCC, UCU
604  *   Class 3:  UE, RUE, BP, WBP, EDU, WDU, CPU
605  *   Class 2:  CE, RCE, EDC, WDC, CPC, ETI, ETC
606  *   Class 1:  TO, BERR
607  *   Class 0:  UMS, OM
608  *
609  * See UltraSPARC-IIIi PRM
610  *   Class 5:  ETP
611  *   Class 4:  UCC, UCU
612  *   Class 3:  UE, RUE, BP, WBP, EDU, WDU
613  *   Class 2:  CE, RCE, EDC, WDC
614  *   Class 1:  TO, BERR
615  *   Class 0:  UMS, OM
616  */
617 uint64_t afar_overwrite[] = {
618 #if defined(JALAPENO)
619 	C_AFSR_ETP,
620 #elif defined(SERRANO)
621 	C_AFSR_ETS | C_AFSR_ETU | C_AFSR_EFES,
622 #endif /* SERRANO */
623 	C_AFSR_UCC | C_AFSR_UCU,
624 	C_AFSR_UE | C_AFSR_RUE | C_AFSR_BP | C_AFSR_WBP | C_AFSR_EDU |
625 	    C_AFSR_WDU | C_AFSR_CPU,
626 #if defined(SERRANO)
627 	C_AFSR_ETI | C_AFSR_ETC |
628 #endif /* SERRANO */
629 	C_AFSR_CE | C_AFSR_RCE | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_CPC,
630 	C_AFSR_TO | C_AFSR_BERR,
631 	C_AFSR_UMS | C_AFSR_OM,
632 	0
633 };
634 
635 #if defined(SERRANO)
636 /*
637  * Serrano has a second AFAR that captures the physical address on
638  * FRC/FRU errors (which Jalapeno does not).  This register also
639  * captures the address for UE and CE errors.
640  *
641  * See UltraSPARC-IIIi+ PRM
642  *  Class 3: UE
643  *  Class 2: FRU
644  *  Class 1: CE
645  *  Class 0: FRC
646  */
647 uint64_t afar2_overwrite[] = {
648 	C_AFSR_UE,
649 	C_AFSR_FRU,
650 	C_AFSR_CE,
651 	C_AFSR_FRC,
652 	0
653 };
654 #endif  /* SERRANO */
655 
656 /*
657  * See UltraSPARC-IIIi PRM
658  *   Class 2:  UE, FRU, EDU, WDU, UCU, CPU
659  *   Class 1:  CE, FRC, EDC, WDC, UCC, CPC
660  */
661 uint64_t esynd_overwrite[] = {
662 #if defined(SERRANO)
663 	C_AFSR_ETS | C_AFSR_ETU |
664 #endif	/* SERRANO */
665 	C_AFSR_UE | C_AFSR_FRU | C_AFSR_EDU | C_AFSR_WDU | C_AFSR_UCU |
666 	    C_AFSR_CPU,
667 	C_AFSR_CE | C_AFSR_FRC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_UCC |
668 	    C_AFSR_CPC,
669 	0
670 };
671 
672 /*
673  * Prioritized list of Error bits for BSYND (referred to as
674  * MSYND to share code with CHEETAH & CHEETAH_PLUS) overwrite.
675  * See UltraSPARC-IIIi PRM
676  *   Class 3:  ISAP
677  *   Class 2:  BP
678  *   Class 1:  WBP, IVPE
679  */
680 uint64_t msynd_overwrite[] = {
681 	C_AFSR_ISAP,
682 	C_AFSR_BP,
683 	C_AFSR_WBP | C_AFSR_IVPE,
684 	0
685 };
686 
687 /*
688  * change cpu speed bits -- new speed will be normal-speed/divisor.
689  *
690  * The Jalapeno memory controllers are required to drain outstanding
691  * memory transactions within 32 JBus clocks in order to be ready
692  * to enter Estar mode.  In some corner cases however, that time
693  * fell short.
694  *
695  * A safe software solution is to force MCU to act like in Estar mode,
696  * then delay 1us (in ppm code) prior to assert J_CHNG_L signal.
697  * To reverse the effect, upon exiting Estar, software restores the
698  * MCU to its original state.
699  */
700 /* ARGSUSED1 */
701 void
702 cpu_change_speed(uint64_t divisor, uint64_t arg2)
703 {
704 	bus_config_eclk_t	*bceclk;
705 	uint64_t		reg;
706 	uint64_t		oldreg;
707 	uint64_t		mreg;
708 	uint64_t		val64;
709 	int			id = (CPU)->cpu_id;
710 	processor_info_t	*pi = &(CPU->cpu_type_info);
711 
712 #if defined(JALAPENO) && defined(JALAPENO_ERRATA_85)
713 	/*
714 	 * ASI Ecache flush in 1/2 or 1/32 speed mode can result
715 	 * in CPU fatal reset (JETO or IERR/TO on MP). A workaround
716 	 * is to force the CPU to full speed mode prior to using
717 	 * ASI Ecache flush opeartion to flush E$. Since we can't
718 	 * always use cross calls at the time of flushing E$, we
719 	 * cannot change other CPU speed. Hence, this workaround
720 	 * is applicable to uniprocessor configuration only and
721 	 * can't be used in multiprocessor configuration.
722 	 *
723 	 * Note that this workaround is activated only when the CPU
724 	 * has been fully initialized and its speed is lowered by the
725 	 * ppm for the first time. It can be disabled via /etc/system
726 	 * by setting jp_errata_85_enable to 0 and rebooting the
727 	 * system.
728 	 */
729 	if ((jp_errata_85_active == -1) &&
730 	    jp_errata_85_enable &&
731 	    (divisor != JBUS_CONFIG_ECLK_1_DIV)) {
732 		if (ncpus == 1)
733 			jp_errata_85_active = 1;
734 		else
735 			jp_errata_85_active = 0;
736 	}
737 	if ((!jp_errata_85_allow_slow_scrub) && (CPU_PRIVATE(CPU) != NULL)) {
738 		int i;
739 		ch_scrub_misc_t	*chpr_scrubp =
740 		    CPU_PRIVATE_PTR(CPU, chpr_scrub_misc);
741 
742 		/* We're only allowed to run the scrubbers at full speed */
743 
744 		for (i = 0; i < CACHE_SCRUBBER_COUNT; i++) {
745 			chpr_scrubp->chsm_enable[i] =
746 			    (divisor == JBUS_CONFIG_ECLK_1_DIV);
747 		}
748 	}
749 #endif	/* JALAPENO && JALAPENO_ERRATA_85 */
750 
751 	/*
752 	 * We're only interested in mcu_ctl_reg1 bit 26 and 25, of which
753 	 * the value will be stored in the lower half of a byte.  The
754 	 * top bit of this byte is designated as a valid bit - 0 means
755 	 * invalid, 1 means valid.
756 	 */
757 	if (!mcu_fsm_init_state[id].valid) {
758 		val64 = get_mcu_ctl_reg1() & JP_MCU_FSM_MASK;
759 		mcu_fsm_init_state[id].state = val64 >> JP_MCU_FSM_SHIFT;
760 		mcu_fsm_init_state[id].valid = 1;
761 	}
762 
763 	for (bceclk = bus_config_eclk; bceclk->divisor; bceclk++) {
764 		if (bceclk->divisor != divisor)
765 			continue;
766 		reg = get_jbus_config();
767 		oldreg = reg;
768 		reg &= ~JBUS_CONFIG_ECLK_MASK;
769 		reg |= bceclk->mask;
770 		set_jbus_config(reg);
771 		(void) get_jbus_config();
772 
773 		/*
774 		 * MCU workaround, refer to Jalapeno spec, EnergyStar section
775 		 * for detail.
776 		 */
777 
778 		/* Upon entering engery star mode, turn off extra MCU FSMs */
779 		if (((oldreg & JBUS_CONFIG_ECLK_MASK) == JBUS_CONFIG_ECLK_1) &&
780 		    ((divisor == JBUS_CONFIG_ECLK_2_DIV) ||
781 		    (divisor == JBUS_CONFIG_ECLK_32_DIV))) {
782 			mreg = get_mcu_ctl_reg1();
783 			if ((mreg & JP_MCU_FSM_MASK) != 0) {
784 				mreg &= ~JP_MCU_FSM_MASK;
785 				set_mcu_ctl_reg1(mreg);
786 				(void) get_mcu_ctl_reg1();
787 			}
788 		/* Upon exiting energy star mode, restore extra MCU FSMs */
789 		} else if (divisor == JBUS_CONFIG_ECLK_1_DIV) {
790 			mreg = get_mcu_ctl_reg1();
791 			val64 = mcu_fsm_init_state[id].state;
792 			mreg |= val64 << JP_MCU_FSM_SHIFT;
793 			set_mcu_ctl_reg1(mreg);
794 			(void) get_mcu_ctl_reg1();
795 		}
796 		CPU->cpu_m.divisor = (uchar_t)divisor;
797 		CPU->cpu_curr_clock =
798 		    (((uint64_t)pi->pi_clock * 1000000) / divisor);
799 		return;
800 	}
801 	/*
802 	 * We will reach here only if OBP and kernel don't agree on
803 	 * the speeds supported by the CPU.
804 	 */
805 	cmn_err(CE_WARN, "cpu_change_speed: bad divisor %" PRIu64, divisor);
806 }
807 
808 /*
809  * Cpu private initialization.  This includes allocating the cpu_private
810  * data structure, initializing it, and initializing the scrubber for this
811  * cpu.  This function calls cpu_init_ecache_scrub_dr to init the scrubber.
812  * We use kmem_cache_create for the cheetah private data structure because
813  * it needs to be allocated on a PAGESIZE (8192) byte boundary.
814  */
815 void
816 cpu_init_private(struct cpu *cp)
817 {
818 	cheetah_private_t *chprp;
819 	int i;
820 
821 	ASSERT(CPU_PRIVATE(cp) == NULL);
822 
823 	/* LINTED: E_TRUE_LOGICAL_EXPR */
824 	ASSERT((offsetof(cheetah_private_t, chpr_tl1_err_data) +
825 	    sizeof (ch_err_tl1_data_t) * CH_ERR_TL1_TLMAX) <= PAGESIZE);
826 
827 #if defined(SERRANO)
828 	if (!IS_SERRANO(cpunodes[cp->cpu_id].implementation)) {
829 		cmn_err(CE_PANIC, "CPU%d: implementation 0x%x not supported"
830 		    " on UltraSPARC-IIIi+ code\n", cp->cpu_id,
831 		    cpunodes[cp->cpu_id].implementation);
832 	}
833 #else /* SERRANO */
834 	if (!IS_JALAPENO(cpunodes[cp->cpu_id].implementation)) {
835 		cmn_err(CE_PANIC, "CPU%d: implementation 0x%x not supported"
836 		    " on UltraSPARC-IIIi code\n", cp->cpu_id,
837 		    cpunodes[cp->cpu_id].implementation);
838 	}
839 #endif /* SERRANO */
840 
841 	/*
842 	 * If the ch_private_cache has not been created, create it.
843 	 */
844 	if (ch_private_cache == NULL) {
845 		ch_private_cache = kmem_cache_create("ch_private_cache",
846 		    sizeof (cheetah_private_t), PAGESIZE, NULL, NULL,
847 		    NULL, NULL, static_arena, 0);
848 	}
849 
850 	chprp = CPU_PRIVATE(cp) = kmem_cache_alloc(ch_private_cache, KM_SLEEP);
851 
852 	bzero(chprp, sizeof (cheetah_private_t));
853 	chprp->chpr_fecctl0_logout.clo_data.chd_afar = LOGOUT_INVALID;
854 	chprp->chpr_cecc_logout.clo_data.chd_afar = LOGOUT_INVALID;
855 	chprp->chpr_async_logout.clo_data.chd_afar = LOGOUT_INVALID;
856 	for (i = 0; i < CH_ERR_TL1_TLMAX; i++)
857 		chprp->chpr_tl1_err_data[i].ch_err_tl1_logout.clo_data.chd_afar
858 		    = LOGOUT_INVALID;
859 
860 	chprp->chpr_icache_size = CH_ICACHE_SIZE;
861 	chprp->chpr_icache_linesize = CH_ICACHE_LSIZE;
862 
863 	cpu_init_ecache_scrub_dr(cp);
864 
865 	chprp->chpr_ec_set_size = cpunodes[cp->cpu_id].ecache_size /
866 	    cpu_ecache_nway();
867 
868 	adjust_hw_copy_limits(cpunodes[cp->cpu_id].ecache_size);
869 	ch_err_tl1_paddrs[cp->cpu_id] = va_to_pa(chprp);
870 	ASSERT(ch_err_tl1_paddrs[cp->cpu_id] != -1);
871 }
872 
873 /*
874  * Clear the error state registers for this CPU.
875  * For Jalapeno, just clear the AFSR
876  */
877 void
878 set_cpu_error_state(ch_cpu_errors_t *cpu_error_regs)
879 {
880 	set_asyncflt(cpu_error_regs->afsr & ~C_AFSR_FATAL_ERRS);
881 }
882 
883 /*
884  * Update cpu_offline_set so the scrubber knows which cpus are offline
885  */
886 /*ARGSUSED*/
887 int
888 cpu_scrub_cpu_setup(cpu_setup_t what, int cpuid, void *arg)
889 {
890 	switch (what) {
891 	case CPU_ON:
892 	case CPU_INIT:
893 		CPUSET_DEL(cpu_offline_set, cpuid);
894 		break;
895 	case CPU_OFF:
896 		CPUSET_ADD(cpu_offline_set, cpuid);
897 		break;
898 	default:
899 		break;
900 	}
901 	return (0);
902 }
903