xref: /illumos-gate/usr/src/uts/sun4u/cpu/us3_cheetahplus.c (revision 0e7515250c8395f368aa45fb9acae7c4f8f8b786)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/systm.h>
28 #include <sys/ddi.h>
29 #include <sys/sysmacros.h>
30 #include <sys/archsystm.h>
31 #include <sys/vmsystm.h>
32 #include <sys/machparam.h>
33 #include <sys/machsystm.h>
34 #include <sys/machthread.h>
35 #include <sys/cpu.h>
36 #include <sys/cmp.h>
37 #include <sys/elf_SPARC.h>
38 #include <vm/hat_sfmmu.h>
39 #include <vm/seg_kmem.h>
40 #include <sys/cpuvar.h>
41 #include <sys/cheetahregs.h>
42 #include <sys/us3_module.h>
43 #include <sys/async.h>
44 #include <sys/cmn_err.h>
45 #include <sys/debug.h>
46 #include <sys/dditypes.h>
47 #include <sys/prom_debug.h>
48 #include <sys/prom_plat.h>
49 #include <sys/cpu_module.h>
50 #include <sys/sysmacros.h>
51 #include <sys/intreg.h>
52 #include <sys/clock.h>
53 #include <sys/platform_module.h>
54 #include <sys/machtrap.h>
55 #include <sys/ontrap.h>
56 #include <sys/panic.h>
57 #include <sys/memlist.h>
58 #include <sys/bootconf.h>
59 #include <sys/ivintr.h>
60 #include <sys/atomic.h>
61 #include <sys/fm/protocol.h>
62 #include <sys/fm/cpu/UltraSPARC-III.h>
63 #include <sys/fm/util.h>
64 #include <sys/pghw.h>
65 
66 #ifdef	CHEETAHPLUS_ERRATUM_25
67 #include <sys/cyclic.h>
68 #endif	/* CHEETAHPLUS_ERRATUM_25 */
69 
70 /*
71  * See comment above cpu_scrub_cpu_setup() for description
72  */
73 #define	SCRUBBER_NEITHER_CORE_ONLINE	0x0
74 #define	SCRUBBER_CORE_0_ONLINE		0x1
75 #define	SCRUBBER_CORE_1_ONLINE		0x2
76 #define	SCRUBBER_BOTH_CORES_ONLINE	(SCRUBBER_CORE_0_ONLINE | \
77 					SCRUBBER_CORE_1_ONLINE)
78 
79 static int pn_matching_valid_l2_line(uint64_t faddr, ch_ec_data_t *clo_l2_data);
80 static void cpu_async_log_tlb_parity_err(void *flt);
81 static cpu_t *cpu_get_sibling_core(cpu_t *cpup);
82 
83 
84 /*
85  * Setup trap handlers.
86  */
87 void
cpu_init_trap(void)88 cpu_init_trap(void)
89 {
90 	CH_SET_TRAP(pil15_epilogue, ch_pil15_interrupt_instr);
91 
92 	CH_SET_TRAP(tt0_fecc, fecc_err_instr);
93 	CH_SET_TRAP(tt1_fecc, fecc_err_tl1_instr);
94 	CH_SET_TRAP(tt1_swtrap0, fecc_err_tl1_cont_instr);
95 
96 	CH_SET_TRAP(tt0_dperr, dcache_parity_instr);
97 	CH_SET_TRAP(tt1_dperr, dcache_parity_tl1_instr);
98 	CH_SET_TRAP(tt1_swtrap1, dcache_parity_tl1_cont_instr);
99 
100 	CH_SET_TRAP(tt0_iperr, icache_parity_instr);
101 	CH_SET_TRAP(tt1_iperr, icache_parity_tl1_instr);
102 	CH_SET_TRAP(tt1_swtrap2, icache_parity_tl1_cont_instr);
103 }
104 
105 /*
106  * Set the magic constants of the implementation.
107  */
108 /*ARGSUSED*/
109 void
cpu_fiximp(pnode_t dnode)110 cpu_fiximp(pnode_t dnode)
111 {
112 	int i, a;
113 	extern int vac_size, vac_shift;
114 	extern uint_t vac_mask;
115 
116 	dcache_size = CH_DCACHE_SIZE;
117 	dcache_linesize = CH_DCACHE_LSIZE;
118 
119 	icache_size = CHP_ICACHE_MAX_SIZE;
120 	icache_linesize = CHP_ICACHE_MIN_LSIZE;
121 
122 	ecache_size = CH_ECACHE_MAX_SIZE;
123 	ecache_alignsize = CH_ECACHE_MAX_LSIZE;
124 	ecache_associativity = CHP_ECACHE_MIN_NWAY;
125 
126 	/*
127 	 * ecache_setsize needs to maximum of all cpu ecache setsizes
128 	 */
129 	ecache_setsize = CHP_ECACHE_MAX_SETSIZE;
130 	ASSERT(ecache_setsize >= (ecache_size / ecache_associativity));
131 
132 	vac_size = CH_VAC_SIZE;
133 	vac_mask = MMU_PAGEMASK & (vac_size - 1);
134 	i = 0; a = vac_size;
135 	while (a >>= 1)
136 		++i;
137 	vac_shift = i;
138 	shm_alignment = vac_size;
139 	vac = 1;
140 }
141 
142 /*
143  * Use Panther values for Panther-only domains.
144  * See Panther PRM, 1.5.4 Cache Hierarchy
145  */
146 void
cpu_fix_allpanther(void)147 cpu_fix_allpanther(void)
148 {
149 	/* dcache same as Ch+ */
150 	icache_size = PN_ICACHE_SIZE;
151 	icache_linesize = PN_ICACHE_LSIZE;
152 	ecache_size = PN_L3_SIZE;
153 	ecache_alignsize = PN_L3_LINESIZE;
154 	ecache_associativity = PN_L3_NWAYS;
155 	ecache_setsize = PN_L3_SET_SIZE;
156 	ASSERT(ecache_setsize >= (ecache_size / ecache_associativity));
157 	/* vac same as Ch+ */
158 	/* fix hwcaps for USIV+-only domains */
159 	cpu_hwcap_flags |= AV_SPARC_POPC;
160 }
161 
162 void
send_mondo_set(cpuset_t set)163 send_mondo_set(cpuset_t set)
164 {
165 	int lo, busy, nack, shipped = 0;
166 	uint16_t i, cpuids[IDSR_BN_SETS];
167 	uint64_t idsr, nackmask = 0, busymask, curnack, curbusy;
168 	uint64_t starttick, endtick, tick, lasttick;
169 #if (NCPU > IDSR_BN_SETS)
170 	int index = 0;
171 	int ncpuids = 0;
172 #endif
173 #ifdef	CHEETAHPLUS_ERRATUM_25
174 	int recovered = 0;
175 	int cpuid;
176 #endif
177 
178 	ASSERT(!CPUSET_ISNULL(set));
179 	starttick = lasttick = gettick();
180 
181 #if (NCPU <= IDSR_BN_SETS)
182 	for (i = 0; i < NCPU; i++)
183 		if (CPU_IN_SET(set, i)) {
184 			shipit(i, shipped);
185 			nackmask |= IDSR_NACK_BIT(shipped);
186 			cpuids[shipped++] = i;
187 			CPUSET_DEL(set, i);
188 			if (CPUSET_ISNULL(set))
189 				break;
190 		}
191 	CPU_STATS_ADDQ(CPU, sys, xcalls, shipped);
192 #else
193 	for (i = 0; i < NCPU; i++)
194 		if (CPU_IN_SET(set, i)) {
195 			ncpuids++;
196 
197 			/*
198 			 * Ship only to the first (IDSR_BN_SETS) CPUs.  If we
199 			 * find we have shipped to more than (IDSR_BN_SETS)
200 			 * CPUs, set "index" to the highest numbered CPU in
201 			 * the set so we can ship to other CPUs a bit later on.
202 			 */
203 			if (shipped < IDSR_BN_SETS) {
204 				shipit(i, shipped);
205 				nackmask |= IDSR_NACK_BIT(shipped);
206 				cpuids[shipped++] = i;
207 				CPUSET_DEL(set, i);
208 				if (CPUSET_ISNULL(set))
209 					break;
210 			} else
211 				index = (int)i;
212 		}
213 
214 	CPU_STATS_ADDQ(CPU, sys, xcalls, ncpuids);
215 #endif
216 
217 	busymask = IDSR_NACK_TO_BUSY(nackmask);
218 	busy = nack = 0;
219 	endtick = starttick + xc_tick_limit;
220 	for (;;) {
221 		idsr = getidsr();
222 #if (NCPU <= IDSR_BN_SETS)
223 		if (idsr == 0)
224 			break;
225 #else
226 		if (idsr == 0 && shipped == ncpuids)
227 			break;
228 #endif
229 		tick = gettick();
230 		/*
231 		 * If there is a big jump between the current tick
232 		 * count and lasttick, we have probably hit a break
233 		 * point.  Adjust endtick accordingly to avoid panic.
234 		 */
235 		if (tick > (lasttick + xc_tick_jump_limit))
236 			endtick += (tick - lasttick);
237 		lasttick = tick;
238 		if (tick > endtick) {
239 			if (panic_quiesce)
240 				return;
241 #ifdef	CHEETAHPLUS_ERRATUM_25
242 			cpuid = -1;
243 			for (i = 0; i < IDSR_BN_SETS; i++) {
244 				if (idsr & (IDSR_NACK_BIT(i) |
245 				    IDSR_BUSY_BIT(i))) {
246 					cpuid = cpuids[i];
247 					break;
248 				}
249 			}
250 			if (cheetah_sendmondo_recover && cpuid != -1 &&
251 			    recovered == 0) {
252 				if (mondo_recover(cpuid, i)) {
253 					/*
254 					 * We claimed the whole memory or
255 					 * full scan is disabled.
256 					 */
257 					recovered++;
258 				}
259 				tick = gettick();
260 				endtick = tick + xc_tick_limit;
261 				lasttick = tick;
262 				/*
263 				 * Recheck idsr
264 				 */
265 				continue;
266 			} else
267 #endif	/* CHEETAHPLUS_ERRATUM_25 */
268 			{
269 				cmn_err(CE_CONT, "send mondo timeout "
270 				    "[%d NACK %d BUSY]\nIDSR 0x%"
271 				    "" PRIx64 "  cpuids:", nack, busy, idsr);
272 				for (i = 0; i < IDSR_BN_SETS; i++) {
273 					if (idsr & (IDSR_NACK_BIT(i) |
274 					    IDSR_BUSY_BIT(i))) {
275 						cmn_err(CE_CONT, " 0x%x",
276 						    cpuids[i]);
277 					}
278 				}
279 				cmn_err(CE_CONT, "\n");
280 				cmn_err(CE_PANIC, "send_mondo_set: timeout");
281 			}
282 		}
283 		curnack = idsr & nackmask;
284 		curbusy = idsr & busymask;
285 #if (NCPU > IDSR_BN_SETS)
286 		if (shipped < ncpuids) {
287 			uint64_t cpus_left;
288 			uint16_t next = (uint16_t)index;
289 
290 			cpus_left = ~(IDSR_NACK_TO_BUSY(curnack) | curbusy) &
291 			    busymask;
292 
293 			if (cpus_left) {
294 				do {
295 					/*
296 					 * Sequence through and ship to the
297 					 * remainder of the CPUs in the system
298 					 * (e.g. other than the first
299 					 * (IDSR_BN_SETS)) in reverse order.
300 					 */
301 					lo = lowbit(cpus_left) - 1;
302 					i = IDSR_BUSY_IDX(lo);
303 					shipit(next, i);
304 					shipped++;
305 					cpuids[i] = next;
306 
307 					/*
308 					 * If we've processed all the CPUs,
309 					 * exit the loop now and save
310 					 * instructions.
311 					 */
312 					if (shipped == ncpuids)
313 						break;
314 
315 					for ((index = ((int)next - 1));
316 					    index >= 0; index--)
317 						if (CPU_IN_SET(set, index)) {
318 							next = (uint16_t)index;
319 							break;
320 						}
321 
322 					cpus_left &= ~(1ull << lo);
323 				} while (cpus_left);
324 #ifdef	CHEETAHPLUS_ERRATUM_25
325 				/*
326 				 * Clear recovered because we are sending to
327 				 * a new set of targets.
328 				 */
329 				recovered = 0;
330 #endif
331 				continue;
332 			}
333 		}
334 #endif
335 		if (curbusy) {
336 			busy++;
337 			continue;
338 		}
339 
340 #ifdef SEND_MONDO_STATS
341 		{
342 			int n = gettick() - starttick;
343 			if (n < 8192)
344 				x_nack_stimes[n >> 7]++;
345 		}
346 #endif
347 		while (gettick() < (tick + sys_clock_mhz))
348 			;
349 		do {
350 			lo = lowbit(curnack) - 1;
351 			i = IDSR_NACK_IDX(lo);
352 			shipit(cpuids[i], i);
353 			curnack &= ~(1ull << lo);
354 		} while (curnack);
355 		nack++;
356 		busy = 0;
357 	}
358 #ifdef SEND_MONDO_STATS
359 	{
360 		int n = gettick() - starttick;
361 		if (n < 8192)
362 			x_set_stimes[n >> 7]++;
363 		else
364 			x_set_ltimes[(n >> 13) & 0xf]++;
365 	}
366 	x_set_cpus[shipped]++;
367 #endif
368 }
369 
370 /*
371  * Handles error logging for implementation specific error types
372  */
373 /*ARGSUSED1*/
374 int
cpu_impl_async_log_err(void * flt,errorq_elem_t * eqep)375 cpu_impl_async_log_err(void *flt, errorq_elem_t *eqep)
376 {
377 	ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt;
378 	struct async_flt *aflt = (struct async_flt *)flt;
379 
380 	switch (ch_flt->flt_type) {
381 
382 	case CPU_IC_PARITY:
383 		cpu_async_log_ic_parity_err(flt);
384 		return (CH_ASYNC_LOG_DONE);
385 
386 	case CPU_DC_PARITY:
387 		cpu_async_log_dc_parity_err(flt);
388 		return (CH_ASYNC_LOG_DONE);
389 
390 	case CPU_DUE:
391 		cpu_log_err(aflt);
392 		cpu_page_retire(ch_flt);
393 		return (CH_ASYNC_LOG_DONE);
394 
395 	case CPU_ITLB_PARITY:
396 	case CPU_DTLB_PARITY:
397 		cpu_async_log_tlb_parity_err(flt);
398 		return (CH_ASYNC_LOG_DONE);
399 
400 	/* report the error and continue */
401 	case CPU_L3_ADDR_PE:
402 		cpu_log_err(aflt);
403 		return (CH_ASYNC_LOG_DONE);
404 
405 	default:
406 		return (CH_ASYNC_LOG_UNKNOWN);
407 	}
408 }
409 
410 /*
411  * Figure out if Ecache is direct-mapped (Cheetah or Cheetah+ with Ecache
412  * control ECCR_ASSOC bit off or 2-way (Cheetah+ with ECCR_ASSOC on).
413  * We need to do this on the fly because we may have mixed Cheetah+'s with
414  * both direct and 2-way Ecaches. Panther only supports 4-way L3$.
415  */
416 int
cpu_ecache_nway(void)417 cpu_ecache_nway(void)
418 {
419 	if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation))
420 		return (PN_L3_NWAYS);
421 	return ((get_ecache_ctrl() & ECCR_ASSOC) ? 2 : 1);
422 }
423 
424 /*
425  * Note that these are entered into the table: Fatal Errors (PERR, IERR, ISAP,
426  * EMU, IMU) first, orphaned UCU/UCC, AFAR Overwrite policy, finally IVU, IVC.
427  * Afar overwrite policy is:
428  *   Class 4:
429  *      AFSR     -- UCC, UCU, TUE, TSCE, TUE_SH
430  *      AFSR_EXT -- L3_UCC, L3_UCU, L3_TUE, L3_TUE_SH
431  *   Class 3:
432  *      AFSR     -- UE, DUE, EDU, WDU, CPU
433  *      AFSR_EXT -- L3_EDU, L3_WDU, L3_CPU
434  *   Class 2:
435  *      AFSR     -- CE, EDC, EMC, WDC, CPC, THCE
436  *      AFSR_EXT -- L3_EDC, L3_WDC, L3_CPC, L3_THCE
437  *   Class 1:
438  *      AFSR     -- TO, DTO, BERR, DBERR
439  */
440 ecc_type_to_info_t ecc_type_to_info[] = {
441 
442 	/* Fatal Errors */
443 	C_AFSR_PERR,		"PERR ",	ECC_ALL_TRAPS,
444 		CPU_FATAL,	"PERR Fatal",
445 		FM_EREPORT_PAYLOAD_SYSTEM2,
446 		FM_EREPORT_CPU_USIII_PERR,
447 	C_AFSR_IERR,		"IERR ", 	ECC_ALL_TRAPS,
448 		CPU_FATAL,	"IERR Fatal",
449 		FM_EREPORT_PAYLOAD_SYSTEM2,
450 		FM_EREPORT_CPU_USIII_IERR,
451 	C_AFSR_ISAP,		"ISAP ",	ECC_ALL_TRAPS,
452 		CPU_FATAL,	"ISAP Fatal",
453 		FM_EREPORT_PAYLOAD_SYSTEM1,
454 		FM_EREPORT_CPU_USIII_ISAP,
455 	C_AFSR_L3_TUE_SH,	"L3_TUE_SH ", 	ECC_C_TRAP,
456 		CPU_FATAL,	"L3_TUE_SH Fatal",
457 		FM_EREPORT_PAYLOAD_L3_TAG_ECC,
458 		FM_EREPORT_CPU_USIII_L3_TUE_SH,
459 	C_AFSR_L3_TUE,		"L3_TUE ", 	ECC_C_TRAP,
460 		CPU_FATAL,	"L3_TUE Fatal",
461 		FM_EREPORT_PAYLOAD_L3_TAG_ECC,
462 		FM_EREPORT_CPU_USIII_L3_TUE,
463 	C_AFSR_TUE_SH,		"TUE_SH ", 	ECC_C_TRAP,
464 		CPU_FATAL,	"TUE_SH Fatal",
465 		FM_EREPORT_PAYLOAD_L2_TAG_ECC,
466 		FM_EREPORT_CPU_USIII_TUE_SH,
467 	C_AFSR_TUE,		"TUE ", 	ECC_ALL_TRAPS,
468 		CPU_FATAL,	"TUE Fatal",
469 		FM_EREPORT_PAYLOAD_L2_TAG_ECC,
470 		FM_EREPORT_CPU_USIII_TUE,
471 	C_AFSR_EMU,		"EMU ",		ECC_ASYNC_TRAPS,
472 		CPU_FATAL,	"EMU Fatal",
473 		FM_EREPORT_PAYLOAD_MEMORY,
474 		FM_EREPORT_CPU_USIII_EMU,
475 	C_AFSR_IMU,		"IMU ",		ECC_C_TRAP,
476 		CPU_FATAL,	"IMU Fatal",
477 		FM_EREPORT_PAYLOAD_SYSTEM1,
478 		FM_EREPORT_CPU_USIII_IMU,
479 
480 	/* L3$ Address parity errors are reported via the MECC bit */
481 	C_AFSR_L3_MECC,		"L3_MECC ",	ECC_MECC_TRAPS,
482 		CPU_L3_ADDR_PE,	"L3 Address Parity",
483 		FM_EREPORT_PAYLOAD_L3_DATA,
484 		FM_EREPORT_CPU_USIII_L3_MECC,
485 
486 	/* Orphaned UCC/UCU Errors */
487 	C_AFSR_L3_UCU,		"L3_OUCU ",	ECC_ORPH_TRAPS,
488 		CPU_ORPH,	"Orphaned L3_UCU",
489 		FM_EREPORT_PAYLOAD_L3_DATA,
490 		FM_EREPORT_CPU_USIII_L3_UCU,
491 	C_AFSR_L3_UCC,		"L3_OUCC ",	ECC_ORPH_TRAPS,
492 		CPU_ORPH,	"Orphaned L3_UCC",
493 		FM_EREPORT_PAYLOAD_L3_DATA,
494 		FM_EREPORT_CPU_USIII_L3_UCC,
495 	C_AFSR_UCU,		"OUCU ",	ECC_ORPH_TRAPS,
496 		CPU_ORPH,	"Orphaned UCU",
497 		FM_EREPORT_PAYLOAD_L2_DATA,
498 		FM_EREPORT_CPU_USIII_UCU,
499 	C_AFSR_UCC,		"OUCC ",	ECC_ORPH_TRAPS,
500 		CPU_ORPH,	"Orphaned UCC",
501 		FM_EREPORT_PAYLOAD_L2_DATA,
502 		FM_EREPORT_CPU_USIII_UCC,
503 
504 	/* UCU, UCC */
505 	C_AFSR_L3_UCU,		"L3_UCU ",	ECC_F_TRAP,
506 		CPU_UE_ECACHE,	"L3_UCU",
507 		FM_EREPORT_PAYLOAD_L3_DATA,
508 		FM_EREPORT_CPU_USIII_L3_UCU,
509 	C_AFSR_L3_UCC,		"L3_UCC ",	ECC_F_TRAP,
510 		CPU_CE_ECACHE,	"L3_UCC",
511 		FM_EREPORT_PAYLOAD_L3_DATA,
512 		FM_EREPORT_CPU_USIII_L3_UCC,
513 	C_AFSR_UCU,		"UCU ",		ECC_F_TRAP,
514 		CPU_UE_ECACHE,	"UCU",
515 		FM_EREPORT_PAYLOAD_L2_DATA,
516 		FM_EREPORT_CPU_USIII_UCU,
517 	C_AFSR_UCC,		"UCC ",		ECC_F_TRAP,
518 		CPU_CE_ECACHE,	"UCC",
519 		FM_EREPORT_PAYLOAD_L2_DATA,
520 		FM_EREPORT_CPU_USIII_UCC,
521 	C_AFSR_TSCE,		"TSCE ",	ECC_F_TRAP,
522 		CPU_CE_ECACHE,	"TSCE",
523 		FM_EREPORT_PAYLOAD_L2_TAG_ECC,
524 		FM_EREPORT_CPU_USIII_TSCE,
525 
526 	/* UE, EDU:ST, EDU:BLD, WDU, CPU */
527 	C_AFSR_UE,		"UE ",		ECC_ASYNC_TRAPS,
528 		CPU_UE,		"Uncorrectable system bus (UE)",
529 		FM_EREPORT_PAYLOAD_MEMORY,
530 		FM_EREPORT_CPU_USIII_UE,
531 	C_AFSR_L3_EDU,		"L3_EDU ",	ECC_C_TRAP,
532 		CPU_UE_ECACHE_RETIRE,	"L3_EDU:ST",
533 		FM_EREPORT_PAYLOAD_L3_DATA,
534 		FM_EREPORT_CPU_USIII_L3_EDUST,
535 	C_AFSR_L3_EDU,		"L3_EDU ",	ECC_D_TRAP,
536 		CPU_UE_ECACHE_RETIRE,	"L3_EDU:BLD",
537 		FM_EREPORT_PAYLOAD_L3_DATA,
538 		FM_EREPORT_CPU_USIII_L3_EDUBL,
539 	C_AFSR_L3_WDU,		"L3_WDU ",	ECC_C_TRAP,
540 		CPU_UE_ECACHE_RETIRE,	"L3_WDU",
541 		FM_EREPORT_PAYLOAD_L3_DATA,
542 		FM_EREPORT_CPU_USIII_L3_WDU,
543 	C_AFSR_L3_CPU,		"L3_CPU ",	ECC_C_TRAP,
544 		CPU_UE_ECACHE,	"L3_CPU",
545 		FM_EREPORT_PAYLOAD_L3_DATA,
546 		FM_EREPORT_CPU_USIII_L3_CPU,
547 	C_AFSR_EDU,		"EDU ",		ECC_C_TRAP,
548 		CPU_UE_ECACHE_RETIRE,	"EDU:ST",
549 		FM_EREPORT_PAYLOAD_L2_DATA,
550 		FM_EREPORT_CPU_USIII_EDUST,
551 	C_AFSR_EDU,		"EDU ",		ECC_D_TRAP,
552 		CPU_UE_ECACHE_RETIRE,	"EDU:BLD",
553 		FM_EREPORT_PAYLOAD_L2_DATA,
554 		FM_EREPORT_CPU_USIII_EDUBL,
555 	C_AFSR_WDU,		"WDU ",		ECC_C_TRAP,
556 		CPU_UE_ECACHE_RETIRE,	"WDU",
557 		FM_EREPORT_PAYLOAD_L2_DATA,
558 		FM_EREPORT_CPU_USIII_WDU,
559 	C_AFSR_CPU,		"CPU ",		ECC_C_TRAP,
560 		CPU_UE_ECACHE,	"CPU",
561 		FM_EREPORT_PAYLOAD_L2_DATA,
562 		FM_EREPORT_CPU_USIII_CPU,
563 	C_AFSR_DUE,		"DUE ",		ECC_C_TRAP,
564 		CPU_DUE,	"DUE",
565 		FM_EREPORT_PAYLOAD_MEMORY,
566 		FM_EREPORT_CPU_USIII_DUE,
567 
568 	/* CE, EDC, EMC, WDC, CPC */
569 	C_AFSR_CE,		"CE ",		ECC_C_TRAP,
570 		CPU_CE,		"Corrected system bus (CE)",
571 		FM_EREPORT_PAYLOAD_MEMORY,
572 		FM_EREPORT_CPU_USIII_CE,
573 	C_AFSR_L3_EDC,		"L3_EDC ",	ECC_C_TRAP,
574 		CPU_CE_ECACHE,	"L3_EDC",
575 		FM_EREPORT_PAYLOAD_L3_DATA,
576 		FM_EREPORT_CPU_USIII_L3_EDC,
577 	C_AFSR_EDC,		"EDC ",		ECC_C_TRAP,
578 		CPU_CE_ECACHE,	"EDC",
579 		FM_EREPORT_PAYLOAD_L2_DATA,
580 		FM_EREPORT_CPU_USIII_EDC,
581 	C_AFSR_EMC,		"EMC ",		ECC_C_TRAP,
582 		CPU_EMC,	"EMC",
583 		FM_EREPORT_PAYLOAD_MEMORY,
584 		FM_EREPORT_CPU_USIII_EMC,
585 	C_AFSR_L3_WDC,		"L3_WDC ",	ECC_C_TRAP,
586 		CPU_CE_ECACHE,	"L3_WDC",
587 		FM_EREPORT_PAYLOAD_L3_DATA,
588 		FM_EREPORT_CPU_USIII_L3_WDC,
589 	C_AFSR_L3_CPC,		"L3_CPC ",	ECC_C_TRAP,
590 		CPU_CE_ECACHE,	"L3_CPC",
591 		FM_EREPORT_PAYLOAD_L3_DATA,
592 		FM_EREPORT_CPU_USIII_L3_CPC,
593 	C_AFSR_L3_THCE,		"L3_THCE ",	ECC_C_TRAP,
594 		CPU_CE_ECACHE,	"L3_THCE",
595 		FM_EREPORT_PAYLOAD_L3_TAG_ECC,
596 		FM_EREPORT_CPU_USIII_L3_THCE,
597 	C_AFSR_WDC,		"WDC ",		ECC_C_TRAP,
598 		CPU_CE_ECACHE,	"WDC",
599 		FM_EREPORT_PAYLOAD_L2_DATA,
600 		FM_EREPORT_CPU_USIII_WDC,
601 	C_AFSR_CPC,		"CPC ",		ECC_C_TRAP,
602 		CPU_CE_ECACHE,	"CPC",
603 		FM_EREPORT_PAYLOAD_L2_DATA,
604 		FM_EREPORT_CPU_USIII_CPC,
605 	C_AFSR_THCE,		"THCE ",	ECC_C_TRAP,
606 		CPU_CE_ECACHE,	"THCE",
607 		FM_EREPORT_PAYLOAD_L2_TAG_ECC,
608 		FM_EREPORT_CPU_USIII_THCE,
609 
610 	/* TO, BERR */
611 	C_AFSR_TO,		"TO ",		ECC_ASYNC_TRAPS,
612 		CPU_TO,		"Timeout (TO)",
613 		FM_EREPORT_PAYLOAD_IO,
614 		FM_EREPORT_CPU_USIII_TO,
615 	C_AFSR_BERR,		"BERR ",	ECC_ASYNC_TRAPS,
616 		CPU_BERR,	"Bus Error (BERR)",
617 		FM_EREPORT_PAYLOAD_IO,
618 		FM_EREPORT_CPU_USIII_BERR,
619 	C_AFSR_DTO,		"DTO ",		ECC_C_TRAP,
620 		CPU_TO,		"Disrupting Timeout (DTO)",
621 		FM_EREPORT_PAYLOAD_IO,
622 		FM_EREPORT_CPU_USIII_DTO,
623 	C_AFSR_DBERR,		"DBERR ",	ECC_C_TRAP,
624 		CPU_BERR,	"Disrupting Bus Error (DBERR)",
625 		FM_EREPORT_PAYLOAD_IO,
626 		FM_EREPORT_CPU_USIII_DBERR,
627 
628 	/* IVU, IVC, IMC */
629 	C_AFSR_IVU,		"IVU ",		ECC_C_TRAP,
630 		CPU_IV,		"IVU",
631 		FM_EREPORT_PAYLOAD_SYSTEM1,
632 		FM_EREPORT_CPU_USIII_IVU,
633 	C_AFSR_IVC,		"IVC ",		ECC_C_TRAP,
634 		CPU_IV,		"IVC",
635 		FM_EREPORT_PAYLOAD_SYSTEM1,
636 		FM_EREPORT_CPU_USIII_IVC,
637 	C_AFSR_IMC,		"IMC ",		ECC_C_TRAP,
638 		CPU_IV,		"IMC",
639 		FM_EREPORT_PAYLOAD_SYSTEM1,
640 		FM_EREPORT_CPU_USIII_IMC,
641 
642 	0,			NULL,		0,
643 		0,		NULL,
644 		FM_EREPORT_PAYLOAD_UNKNOWN,
645 		FM_EREPORT_CPU_USIII_UNKNOWN,
646 };
647 
648 /*
649  * See Cheetah+ Delta PRM 10.9 and section P.6.1 of the Panther PRM
650  *   Class 4:
651  *      AFSR     -- UCC, UCU, TUE, TSCE, TUE_SH
652  *      AFSR_EXT -- L3_UCC, L3_UCU, L3_TUE, L3_TUE_SH
653  *   Class 3:
654  *      AFSR     -- UE, DUE, EDU, EMU, WDU, CPU
655  *      AFSR_EXT -- L3_EDU, L3_WDU, L3_CPU
656  *   Class 2:
657  *      AFSR     -- CE, EDC, EMC, WDC, CPC, THCE
658  *      AFSR_EXT -- L3_EDC, L3_WDC, L3_CPC, L3_THCE
659  *   Class 1:
660  *      AFSR     -- TO, DTO, BERR, DBERR
661  *      AFSR_EXT --
662  */
663 uint64_t afar_overwrite[] = {
664 	/* class 4: */
665 	C_AFSR_UCC | C_AFSR_UCU | C_AFSR_TUE | C_AFSR_TSCE | C_AFSR_TUE_SH |
666 	C_AFSR_L3_UCC | C_AFSR_L3_UCU | C_AFSR_L3_TUE | C_AFSR_L3_TUE_SH,
667 	/* class 3: */
668 	C_AFSR_UE | C_AFSR_DUE | C_AFSR_EDU | C_AFSR_EMU | C_AFSR_WDU |
669 	C_AFSR_CPU | C_AFSR_L3_EDU | C_AFSR_L3_WDU | C_AFSR_L3_CPU,
670 	/* class 2: */
671 	C_AFSR_CE | C_AFSR_EDC | C_AFSR_EMC | C_AFSR_WDC | C_AFSR_CPC |
672 	C_AFSR_THCE | C_AFSR_L3_EDC | C_AFSR_L3_WDC | C_AFSR_L3_CPC |
673 	C_AFSR_L3_THCE,
674 	/* class 1: */
675 	C_AFSR_TO | C_AFSR_DTO | C_AFSR_BERR | C_AFSR_DBERR,
676 
677 	0
678 };
679 
680 /*
681  * For Cheetah+, the E_SYND and M_SYND overwrite priorities are combined.
682  * See Cheetah+ Delta PRM 10.9 and Cheetah+ PRM 11.6.2
683  *   Class 2:  UE, DUE, IVU, EDU, EMU, WDU, UCU, CPU
684  *   Class 1:  CE, IVC, EDC, EMC, WDC, UCC, CPC
685  */
686 uint64_t esynd_overwrite[] = {
687 	/* class 2: */
688 	C_AFSR_UE | C_AFSR_DUE | C_AFSR_IVU | C_AFSR_EDU | C_AFSR_EMU |
689 	    C_AFSR_WDU | C_AFSR_UCU | C_AFSR_CPU,
690 	/* class 1: */
691 	C_AFSR_CE | C_AFSR_IVC | C_AFSR_EDC | C_AFSR_EMC | C_AFSR_WDC |
692 	    C_AFSR_UCC | C_AFSR_CPC,
693 	0
694 };
695 
696 /*
697  * In panther, the E_SYND overwrite policy changed a little bit
698  * by adding one more level.
699  * See Panther PRM P.6.2
700  *   class 3:
701  *      AFSR     -- UCU, UCC
702  *      AFSR_EXT -- L3_UCU, L3_UCC
703  *   Class 2:
704  *      AFSR     -- UE, DUE, IVU, EDU, WDU, CPU
705  *      AFSR_EXT -- L3_EDU, L3_WDU, L3_CPU
706  *   Class 1:
707  *      AFSR     -- CE, IVC, EDC, WDC, CPC
708  *      AFSR_EXT -- L3_EDC, L3_WDC, L3_CPC
709  */
710 uint64_t pn_esynd_overwrite[] = {
711 	/* class 3: */
712 	C_AFSR_UCU | C_AFSR_UCC |
713 	C_AFSR_L3_UCU | C_AFSR_L3_UCC,
714 	/* class 2: */
715 	C_AFSR_UE | C_AFSR_DUE | C_AFSR_IVU | C_AFSR_EDU | C_AFSR_WDU |
716 	    C_AFSR_CPU |
717 	C_AFSR_L3_EDU | C_AFSR_L3_WDU | C_AFSR_L3_CPU,
718 	/* class 1: */
719 	C_AFSR_CE | C_AFSR_IVC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_CPC |
720 	C_AFSR_L3_EDC | C_AFSR_L3_WDC | C_AFSR_L3_CPC,
721 
722 	0
723 };
724 
725 int
afsr_to_pn_esynd_status(uint64_t afsr,uint64_t afsr_bit)726 afsr_to_pn_esynd_status(uint64_t afsr, uint64_t afsr_bit)
727 {
728 	return (afsr_to_overw_status(afsr, afsr_bit, pn_esynd_overwrite));
729 }
730 
731 /*
732  * Prioritized list of Error bits for MSYND overwrite.
733  * See Panther PRM P.6.2 (For Cheetah+, see esynd_overwrite classes)
734  *   Class 2:  EMU, IMU
735  *   Class 1:  EMC, IMC
736  *
737  * Panther adds IMU and IMC.
738  */
739 uint64_t msynd_overwrite[] = {
740 	/* class 2: */
741 	C_AFSR_EMU | C_AFSR_IMU,
742 	/* class 1: */
743 	C_AFSR_EMC | C_AFSR_IMC,
744 
745 	0
746 };
747 
748 /*
749  * change cpu speed bits -- new speed will be normal-speed/divisor.
750  *
751  * The Jalapeno memory controllers are required to drain outstanding
752  * memory transactions within 32 JBus clocks in order to be ready
753  * to enter Estar mode.  In some corner cases however, that time
754  * fell short.
755  *
756  * A safe software solution is to force MCU to act like in Estar mode,
757  * then delay 1us (in ppm code) prior to assert J_CHNG_L signal.
758  * To reverse the effect, upon exiting Estar, software restores the
759  * MCU to its original state.
760  */
761 /* ARGSUSED1 */
762 void
cpu_change_speed(uint64_t divisor,uint64_t arg2)763 cpu_change_speed(uint64_t divisor, uint64_t arg2)
764 {
765 	bus_config_eclk_t	*bceclk;
766 	uint64_t		reg;
767 	processor_info_t	*pi = &(CPU->cpu_type_info);
768 
769 	for (bceclk = bus_config_eclk; bceclk->divisor; bceclk++) {
770 		if (bceclk->divisor != divisor)
771 			continue;
772 		reg = get_safari_config();
773 		reg &= ~SAFARI_CONFIG_ECLK_MASK;
774 		reg |= bceclk->mask;
775 		set_safari_config(reg);
776 		CPU->cpu_m.divisor = (uchar_t)divisor;
777 		cpu_set_curr_clock(((uint64_t)pi->pi_clock * 1000000) /
778 		    divisor);
779 		return;
780 	}
781 	/*
782 	 * We will reach here only if OBP and kernel don't agree on
783 	 * the speeds supported by the CPU.
784 	 */
785 	cmn_err(CE_WARN, "cpu_change_speed: bad divisor %" PRIu64, divisor);
786 }
787 
788 /*
789  * Cpu private initialization.  This includes allocating the cpu_private
790  * data structure, initializing it, and initializing the scrubber for this
791  * cpu.  This function calls cpu_init_ecache_scrub_dr to init the scrubber.
792  * We use kmem_cache_create for the cheetah private data structure because
793  * it needs to be allocated on a PAGESIZE (8192) byte boundary.
794  */
795 void
cpu_init_private(struct cpu * cp)796 cpu_init_private(struct cpu *cp)
797 {
798 	cheetah_private_t *chprp;
799 	int i;
800 
801 	ASSERT(CPU_PRIVATE(cp) == NULL);
802 
803 	/* LINTED: E_TRUE_LOGICAL_EXPR */
804 	ASSERT((offsetof(cheetah_private_t, chpr_tl1_err_data) +
805 	    sizeof (ch_err_tl1_data_t) * CH_ERR_TL1_TLMAX) <= PAGESIZE);
806 
807 	/*
808 	 * Running with Cheetah CPUs in a Cheetah+, Jaguar, Panther or
809 	 * mixed Cheetah+/Jaguar/Panther machine is not a supported
810 	 * configuration. Attempting to do so may result in unpredictable
811 	 * failures (e.g. running Cheetah+ CPUs with Cheetah E$ disp flush)
812 	 * so don't allow it.
813 	 *
814 	 * This is just defensive code since this configuration mismatch
815 	 * should have been caught prior to OS execution.
816 	 */
817 	if (!(IS_CHEETAH_PLUS(cpunodes[cp->cpu_id].implementation) ||
818 	    IS_JAGUAR(cpunodes[cp->cpu_id].implementation) ||
819 	    IS_PANTHER(cpunodes[cp->cpu_id].implementation))) {
820 		cmn_err(CE_PANIC, "CPU%d: UltraSPARC-III not supported"
821 		    " on UltraSPARC-III+/IV/IV+ code\n", cp->cpu_id);
822 	}
823 
824 	/*
825 	 * If the ch_private_cache has not been created, create it.
826 	 */
827 	if (ch_private_cache == NULL) {
828 		ch_private_cache = kmem_cache_create("ch_private_cache",
829 		    sizeof (cheetah_private_t), PAGESIZE, NULL, NULL,
830 		    NULL, NULL, static_arena, 0);
831 	}
832 
833 	chprp = CPU_PRIVATE(cp) = kmem_cache_alloc(ch_private_cache, KM_SLEEP);
834 
835 	bzero(chprp, sizeof (cheetah_private_t));
836 	chprp->chpr_fecctl0_logout.clo_data.chd_afar = LOGOUT_INVALID;
837 	chprp->chpr_cecc_logout.clo_data.chd_afar = LOGOUT_INVALID;
838 	chprp->chpr_async_logout.clo_data.chd_afar = LOGOUT_INVALID;
839 	chprp->chpr_tlb_logout.tlo_addr = LOGOUT_INVALID;
840 	for (i = 0; i < CH_ERR_TL1_TLMAX; i++)
841 		chprp->chpr_tl1_err_data[i].ch_err_tl1_logout.clo_data.chd_afar
842 		    = LOGOUT_INVALID;
843 
844 	/* Panther has a larger Icache compared to cheetahplus or Jaguar */
845 	if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) {
846 		chprp->chpr_icache_size = PN_ICACHE_SIZE;
847 		chprp->chpr_icache_linesize = PN_ICACHE_LSIZE;
848 	} else {
849 		chprp->chpr_icache_size = CH_ICACHE_SIZE;
850 		chprp->chpr_icache_linesize = CH_ICACHE_LSIZE;
851 	}
852 
853 	cpu_init_ecache_scrub_dr(cp);
854 
855 	/*
856 	 * Panther's L2$ and E$ are shared between cores, so the scrubber is
857 	 * only needed on one of the cores.  At this point, we assume all cores
858 	 * are online, and we only enable the scrubber on core 0.
859 	 */
860 	if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) {
861 		chprp->chpr_scrub_misc.chsm_core_state =
862 		    SCRUBBER_BOTH_CORES_ONLINE;
863 		if (cp->cpu_id != (processorid_t)cmp_cpu_to_chip(cp->cpu_id)) {
864 			chprp->chpr_scrub_misc.chsm_enable[
865 			    CACHE_SCRUBBER_INFO_E] = 0;
866 		}
867 	}
868 
869 	chprp->chpr_ec_set_size = cpunodes[cp->cpu_id].ecache_size /
870 	    cpu_ecache_nway();
871 
872 	adjust_hw_copy_limits(cpunodes[cp->cpu_id].ecache_size);
873 	ch_err_tl1_paddrs[cp->cpu_id] = va_to_pa(chprp);
874 	ASSERT(ch_err_tl1_paddrs[cp->cpu_id] != -1);
875 }
876 
877 /*
878  * Clear the error state registers for this CPU.
879  * For Cheetah+/Jaguar, just clear the AFSR but
880  * for Panther we also have to clear the AFSR_EXT.
881  */
882 void
set_cpu_error_state(ch_cpu_errors_t * cpu_error_regs)883 set_cpu_error_state(ch_cpu_errors_t *cpu_error_regs)
884 {
885 	set_asyncflt(cpu_error_regs->afsr & ~C_AFSR_FATAL_ERRS);
886 	if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
887 		set_afsr_ext(cpu_error_regs->afsr_ext & ~C_AFSR_EXT_FATAL_ERRS);
888 	}
889 }
890 
891 void
pn_cpu_log_diag_l2_info(ch_async_flt_t * ch_flt)892 pn_cpu_log_diag_l2_info(ch_async_flt_t *ch_flt) {
893 	struct async_flt *aflt = (struct async_flt *)ch_flt;
894 	ch_ec_data_t *l2_data = &ch_flt->flt_diag_data.chd_l2_data[0];
895 	uint64_t faddr = aflt->flt_addr;
896 	uint8_t log_way_mask = 0;
897 	int i;
898 
899 	/*
900 	 * Only Panther CPUs have the additional L2$ data that needs
901 	 * to be logged here
902 	 */
903 	if (!IS_PANTHER(cpunodes[aflt->flt_inst].implementation))
904 		return;
905 
906 	/*
907 	 * We'll use a simple bit mask to keep track of which way(s)
908 	 * of the stored cache line we want to log. The idea is to
909 	 * log the entry if it is a valid line and it matches our
910 	 * fault AFAR. If no match is found, we will simply log all
911 	 * the ways.
912 	 */
913 	for (i = 0; i < PN_L2_NWAYS; i++)
914 		if (pn_matching_valid_l2_line(faddr, &l2_data[i]))
915 			log_way_mask |= (1 << i);
916 
917 	/* If no matching valid lines were found, we log all ways */
918 	if (log_way_mask == 0)
919 		log_way_mask = (1 << PN_L2_NWAYS) - 1;
920 
921 	/* Log the cache lines */
922 	for (i = 0; i < PN_L2_NWAYS; i++)
923 		if (log_way_mask & (1 << i))
924 			l2_data[i].ec_logflag = EC_LOGFLAG_MAGIC;
925 }
926 
927 /*
928  * For this routine to return true, the L2 tag in question must be valid
929  * and the tag PA must match the fault address (faddr) assuming the correct
930  * index is being used.
931  */
932 static int
pn_matching_valid_l2_line(uint64_t faddr,ch_ec_data_t * clo_l2_data)933 pn_matching_valid_l2_line(uint64_t faddr, ch_ec_data_t *clo_l2_data) {
934 	if ((!PN_L2_LINE_INVALID(clo_l2_data->ec_tag)) &&
935 	((faddr & P2ALIGN(C_AFAR_PA, PN_L2_SET_SIZE)) ==
936 	    PN_L2TAG_TO_PA(clo_l2_data->ec_tag)))
937 		return (1);
938 	return (0);
939 }
940 
941 /*
942  * This array is used to convert the 3 digit PgSz encoding (as used in
943  * various MMU registers such as MMU_TAG_ACCESS_EXT) into the corresponding
944  * page size.
945  */
946 static uint64_t tlb_pgsz_to_size[] = {
947 	/* 000 = 8KB: */
948 	0x2000,
949 	/* 001 = 64KB: */
950 	0x10000,
951 	/* 010 = 512KB: */
952 	0x80000,
953 	/* 011 = 4MB: */
954 	0x400000,
955 	/* 100 = 32MB: */
956 	0x2000000,
957 	/* 101 = 256MB: */
958 	0x10000000,
959 	/* undefined for encodings 110 and 111: */
960 	0, 0
961 };
962 
963 /*
964  * The itlb_parity_trap and dtlb_parity_trap handlers transfer control here
965  * after collecting logout information related to the TLB parity error and
966  * flushing the offending TTE entries from the ITLB or DTLB.
967  *
968  * DTLB traps which occur at TL>0 are not recoverable because we will most
969  * likely be corrupting some other trap handler's alternate globals. As
970  * such, we simply panic here when that happens. ITLB parity errors are
971  * not expected to happen at TL>0.
972  */
973 void
cpu_tlb_parity_error(struct regs * rp,ulong_t trap_va,ulong_t tlb_info)974 cpu_tlb_parity_error(struct regs *rp, ulong_t trap_va, ulong_t tlb_info) {
975 	ch_async_flt_t ch_flt;
976 	struct async_flt *aflt;
977 	pn_tlb_logout_t *tlop = NULL;
978 	int immu_parity = (tlb_info & PN_TLO_INFO_IMMU) != 0;
979 	int tl1_trap = (tlb_info & PN_TLO_INFO_TL1) != 0;
980 	char *error_class;
981 
982 	bzero(&ch_flt, sizeof (ch_async_flt_t));
983 
984 	/*
985 	 * Get the CPU log out info. If we can't find our CPU private
986 	 * pointer, or if the logout information does not correspond to
987 	 * this error, then we will have to make due without detailed
988 	 * logout information.
989 	 */
990 	if (CPU_PRIVATE(CPU)) {
991 		tlop = CPU_PRIVATE_PTR(CPU, chpr_tlb_logout);
992 		if ((tlop->tlo_addr != trap_va) ||
993 		    (tlop->tlo_info != tlb_info))
994 			tlop = NULL;
995 	}
996 
997 	if (tlop) {
998 		ch_flt.tlb_diag_data = *tlop;
999 
1000 		/* Zero out + invalidate TLB logout. */
1001 		bzero(tlop, sizeof (pn_tlb_logout_t));
1002 		tlop->tlo_addr = LOGOUT_INVALID;
1003 	} else {
1004 		/*
1005 		 * Copy what logout information we have and mark
1006 		 * it incomplete.
1007 		 */
1008 		ch_flt.flt_data_incomplete = 1;
1009 		ch_flt.tlb_diag_data.tlo_info = tlb_info;
1010 		ch_flt.tlb_diag_data.tlo_addr = trap_va;
1011 	}
1012 
1013 	/*
1014 	 * Log the error.
1015 	 */
1016 	aflt = (struct async_flt *)&ch_flt;
1017 	aflt->flt_id = gethrtime_waitfree();
1018 	aflt->flt_bus_id = getprocessorid();
1019 	aflt->flt_inst = CPU->cpu_id;
1020 	aflt->flt_pc = (caddr_t)rp->r_pc;
1021 	aflt->flt_addr = trap_va;
1022 	aflt->flt_prot = AFLT_PROT_NONE;
1023 	aflt->flt_class = CPU_FAULT;
1024 	aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ?  1 : 0;
1025 	aflt->flt_tl = tl1_trap ? 1 : 0;
1026 	aflt->flt_panic = tl1_trap ? 1 : 0;
1027 
1028 	if (immu_parity) {
1029 		aflt->flt_status = ECC_ITLB_TRAP;
1030 		ch_flt.flt_type = CPU_ITLB_PARITY;
1031 		error_class = FM_EREPORT_CPU_USIII_ITLBPE;
1032 		aflt->flt_payload = FM_EREPORT_PAYLOAD_ITLB_PE;
1033 	} else {
1034 		aflt->flt_status = ECC_DTLB_TRAP;
1035 		ch_flt.flt_type = CPU_DTLB_PARITY;
1036 		error_class = FM_EREPORT_CPU_USIII_DTLBPE;
1037 		aflt->flt_payload = FM_EREPORT_PAYLOAD_DTLB_PE;
1038 	}
1039 
1040 	/*
1041 	 * The TLB entries have already been flushed by the TL1 trap
1042 	 * handler so at this point the only thing left to do is log
1043 	 * the error message.
1044 	 */
1045 	if (aflt->flt_panic) {
1046 		cpu_errorq_dispatch(error_class, (void *)&ch_flt,
1047 		    sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic);
1048 		/*
1049 		 * Panic here if aflt->flt_panic has been set.  Enqueued
1050 		 * errors will be logged as part of the panic flow.
1051 		 */
1052 		fm_panic("%sError(s)", immu_parity ? "ITLBPE " : "DTLBPE ");
1053 	} else {
1054 		cpu_errorq_dispatch(error_class, (void *)&ch_flt,
1055 		    sizeof (ch_async_flt_t), ce_queue, aflt->flt_panic);
1056 	}
1057 }
1058 
1059 /*
1060  * This routine is called when a TLB parity error event is 'ue_drain'ed
1061  * or 'ce_drain'ed from the errorq.
1062  */
1063 void
cpu_async_log_tlb_parity_err(void * flt)1064 cpu_async_log_tlb_parity_err(void *flt) {
1065 	ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt;
1066 	struct async_flt *aflt = (struct async_flt *)flt;
1067 #ifdef lint
1068 	aflt = aflt;
1069 #endif
1070 
1071 	/*
1072 	 * We only capture TLB information if we encountered
1073 	 * a TLB parity error and Panther is the only CPU which
1074 	 * can detect a TLB parity error.
1075 	 */
1076 	ASSERT(IS_PANTHER(cpunodes[aflt->flt_inst].implementation));
1077 	ASSERT((ch_flt->flt_type == CPU_ITLB_PARITY) ||
1078 	    (ch_flt->flt_type == CPU_DTLB_PARITY));
1079 
1080 	if (ch_flt->flt_data_incomplete == 0) {
1081 		if (ch_flt->flt_type == CPU_ITLB_PARITY)
1082 			ch_flt->tlb_diag_data.tlo_logflag = IT_LOGFLAG_MAGIC;
1083 		else /* parity error is in DTLB */
1084 			ch_flt->tlb_diag_data.tlo_logflag = DT_LOGFLAG_MAGIC;
1085 	}
1086 }
1087 
1088 /*
1089  * Add L1 Prefetch cache data to the ereport payload.
1090  */
1091 void
cpu_payload_add_pcache(struct async_flt * aflt,nvlist_t * nvl)1092 cpu_payload_add_pcache(struct async_flt *aflt, nvlist_t *nvl)
1093 {
1094 	ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
1095 	ch_pc_data_t *pcp;
1096 	ch_pc_data_t pcdata[CH_PCACHE_NWAY];
1097 	uint_t nelem;
1098 	int i, ways_logged = 0;
1099 
1100 	/*
1101 	 * We only capture P$ information if we encountered
1102 	 * a P$ parity error and Panther is the only CPU which
1103 	 * can detect a P$ parity error.
1104 	 */
1105 	ASSERT(IS_PANTHER(cpunodes[aflt->flt_inst].implementation));
1106 	for (i = 0; i < CH_PCACHE_NWAY; i++) {
1107 		pcp = &ch_flt->parity_data.dpe.cpl_pc[i];
1108 		if (pcp->pc_logflag == PC_LOGFLAG_MAGIC) {
1109 			bcopy(pcp, &pcdata[ways_logged],
1110 			    sizeof (ch_pc_data_t));
1111 			ways_logged++;
1112 		}
1113 	}
1114 
1115 	/*
1116 	 * Add the pcache data to the payload.
1117 	 */
1118 	fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1P_WAYS,
1119 	    DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL);
1120 	if (ways_logged != 0) {
1121 		nelem = sizeof (ch_pc_data_t) / sizeof (uint64_t) * ways_logged;
1122 		fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1P_DATA,
1123 		    DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)pcdata, NULL);
1124 	}
1125 }
1126 
1127 /*
1128  * Add TLB diagnostic data to the ereport payload.
1129  */
1130 void
cpu_payload_add_tlb(struct async_flt * aflt,nvlist_t * nvl)1131 cpu_payload_add_tlb(struct async_flt *aflt, nvlist_t *nvl)
1132 {
1133 	ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;
1134 	uint8_t num_entries, tlb_data_words;
1135 
1136 	/*
1137 	 * We only capture TLB information if we encountered
1138 	 * a TLB parity error and Panther is the only CPU which
1139 	 * can detect a TLB parity error.
1140 	 */
1141 	ASSERT(IS_PANTHER(cpunodes[aflt->flt_inst].implementation));
1142 	ASSERT((ch_flt->flt_type == CPU_ITLB_PARITY) ||
1143 	    (ch_flt->flt_type == CPU_DTLB_PARITY));
1144 
1145 	if (ch_flt->flt_type == CPU_ITLB_PARITY) {
1146 		num_entries = (uint8_t)(PN_ITLB_NWAYS * PN_NUM_512_ITLBS);
1147 		tlb_data_words = sizeof (ch_tte_entry_t) / sizeof (uint64_t) *
1148 		    num_entries;
1149 
1150 		/*
1151 		 * Add the TLB diagnostic data to the payload
1152 		 * if it was collected.
1153 		 */
1154 		if (ch_flt->tlb_diag_data.tlo_logflag == IT_LOGFLAG_MAGIC) {
1155 			fm_payload_set(nvl,
1156 			    FM_EREPORT_PAYLOAD_NAME_ITLB_ENTRIES,
1157 			    DATA_TYPE_UINT8, num_entries, NULL);
1158 			fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_ITLB_DATA,
1159 			    DATA_TYPE_UINT64_ARRAY, tlb_data_words,
1160 			    (uint64_t *)ch_flt->tlb_diag_data.tlo_itlb_tte,
1161 			    NULL);
1162 		}
1163 	} else {
1164 		num_entries = (uint8_t)(PN_DTLB_NWAYS * PN_NUM_512_DTLBS);
1165 		tlb_data_words = sizeof (ch_tte_entry_t) / sizeof (uint64_t) *
1166 		    num_entries;
1167 
1168 		fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_VA,
1169 		    DATA_TYPE_UINT64, ch_flt->tlb_diag_data.tlo_addr, NULL);
1170 
1171 		/*
1172 		 * Add the TLB diagnostic data to the payload
1173 		 * if it was collected.
1174 		 */
1175 		if (ch_flt->tlb_diag_data.tlo_logflag == DT_LOGFLAG_MAGIC) {
1176 			fm_payload_set(nvl,
1177 			    FM_EREPORT_PAYLOAD_NAME_DTLB_ENTRIES,
1178 			    DATA_TYPE_UINT8, num_entries, NULL);
1179 			fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_DTLB_DATA,
1180 			    DATA_TYPE_UINT64_ARRAY, tlb_data_words,
1181 			    (uint64_t *)ch_flt->tlb_diag_data.tlo_dtlb_tte,
1182 			    NULL);
1183 		}
1184 	}
1185 }
1186 
1187 /*
1188  * Panther Cache Scrubbing:
1189  *
1190  * In Jaguar, the E$ was split between cores, so the scrubber must run on both
1191  * cores.  For Panther, however, the L2$ and L3$ are shared across cores.
1192  * Therefore, the E$ scrubber only needs to run on one of the two cores.
1193  *
1194  * There are four possible states for the E$ scrubber:
1195  *
1196  * 0. If both cores are offline, add core 0 to cpu_offline_set so that
1197  *    the offline scrubber will run on it.
1198  * 1. If core 0 is online and core 1 off, we run the scrubber on core 0.
1199  * 2. If core 1 is online and core 0 off, we move the scrubber to run
1200  *    on core 1.
1201  * 3. If both cores are online, only run the scrubber on core 0.
1202  *
1203  * These states are enumerated by the SCRUBBER_[BOTH|CORE|NEITHER]_* defines
1204  * above.  One of those values is stored in
1205  * chpr_scrub_misc->chsm_core_state on each core.
1206  *
1207  * Also note that, for Panther, ecache_flush_line() will flush out the L2$
1208  * before the E$, so the L2$ will be scrubbed by the E$ scrubber.  No
1209  * additional code is necessary to scrub the L2$.
1210  *
1211  * For all cpu types, whenever a cpu or core is offlined, add it to
1212  * cpu_offline_set so the necessary scrubbers can still run.  This is still
1213  * necessary on Panther so the D$ scrubber can still run.
1214  */
1215 /*ARGSUSED*/
1216 int
cpu_scrub_cpu_setup(cpu_setup_t what,int cpuid,void * arg)1217 cpu_scrub_cpu_setup(cpu_setup_t what, int cpuid, void *arg)
1218 {
1219 	processorid_t core_0_id;
1220 	cpu_t *core_cpus[2];
1221 	ch_scrub_misc_t *core_scrub[2];
1222 	int old_state, i;
1223 	int new_state = SCRUBBER_NEITHER_CORE_ONLINE;
1224 
1225 	switch (what) {
1226 	case CPU_ON:
1227 	case CPU_INIT:
1228 		CPUSET_DEL(cpu_offline_set, cpuid);
1229 		break;
1230 	case CPU_OFF:
1231 		CPUSET_ADD(cpu_offline_set, cpuid);
1232 		break;
1233 	default:
1234 		return (0);
1235 	}
1236 
1237 	if (!IS_PANTHER(cpunodes[cpuid].implementation)) {
1238 		return (0);
1239 	}
1240 
1241 	/*
1242 	 * Update the chsm_enable[CACHE_SCRUBBER_INFO_E] value
1243 	 * if necessary
1244 	 */
1245 	core_0_id = cmp_cpu_to_chip(cpuid);
1246 	core_cpus[0] = cpu_get(core_0_id);
1247 	core_cpus[1] = cpu_get_sibling_core(core_cpus[0]);
1248 
1249 	for (i = 0; i < 2; i++) {
1250 		if (core_cpus[i] == NULL) {
1251 			/*
1252 			 * This may happen during DR - one core is offlined
1253 			 * and completely unconfigured before the second
1254 			 * core is offlined.  Give up and return quietly,
1255 			 * since the second core should quickly be removed
1256 			 * anyways.
1257 			 */
1258 			return (0);
1259 		}
1260 		core_scrub[i] = CPU_PRIVATE_PTR(core_cpus[i], chpr_scrub_misc);
1261 	}
1262 
1263 	if (cpuid == (processorid_t)cmp_cpu_to_chip(cpuid)) {
1264 		/* cpuid is core 0 */
1265 		if (cpu_is_active(core_cpus[1])) {
1266 			new_state |= SCRUBBER_CORE_1_ONLINE;
1267 		}
1268 		if (what != CPU_OFF) {
1269 			new_state |= SCRUBBER_CORE_0_ONLINE;
1270 		}
1271 	} else {
1272 		/* cpuid is core 1 */
1273 		if (cpu_is_active(core_cpus[0])) {
1274 			new_state |= SCRUBBER_CORE_0_ONLINE;
1275 		}
1276 		if (what != CPU_OFF) {
1277 			new_state |= SCRUBBER_CORE_1_ONLINE;
1278 		}
1279 	}
1280 
1281 	old_state = core_scrub[0]->chsm_core_state;
1282 
1283 	if (old_state == new_state) {
1284 		return (0);
1285 	}
1286 
1287 	if (old_state == SCRUBBER_CORE_1_ONLINE) {
1288 		/*
1289 		 * We need to move the scrubber state from core 1
1290 		 * back to core 0.  This data is not protected by
1291 		 * locks, but the worst that can happen is some
1292 		 * lines are scrubbed multiple times.  chsm_oustanding is
1293 		 * set to 0 to make sure an interrupt is scheduled the
1294 		 * first time through do_scrub().
1295 		 */
1296 		core_scrub[0]->chsm_flush_index[CACHE_SCRUBBER_INFO_E] =
1297 		    core_scrub[1]->chsm_flush_index[CACHE_SCRUBBER_INFO_E];
1298 		core_scrub[0]->chsm_outstanding[CACHE_SCRUBBER_INFO_E] = 0;
1299 	}
1300 
1301 	switch (new_state) {
1302 	case SCRUBBER_NEITHER_CORE_ONLINE:
1303 	case SCRUBBER_BOTH_CORES_ONLINE:
1304 	case SCRUBBER_CORE_0_ONLINE:
1305 		core_scrub[1]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 0;
1306 		core_scrub[0]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1;
1307 		break;
1308 
1309 	case SCRUBBER_CORE_1_ONLINE:
1310 	default:
1311 		/*
1312 		 * We need to move the scrubber state from core 0
1313 		 * to core 1.
1314 		 */
1315 		core_scrub[1]->chsm_flush_index[CACHE_SCRUBBER_INFO_E] =
1316 		    core_scrub[0]->chsm_flush_index[CACHE_SCRUBBER_INFO_E];
1317 		core_scrub[1]->chsm_outstanding[CACHE_SCRUBBER_INFO_E] = 0;
1318 
1319 		core_scrub[0]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 0;
1320 		core_scrub[1]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1;
1321 		break;
1322 	}
1323 
1324 	core_scrub[0]->chsm_core_state = new_state;
1325 	core_scrub[1]->chsm_core_state = new_state;
1326 	return (0);
1327 }
1328 
1329 /*
1330  * Returns a pointer to the cpu structure of the argument's sibling core.
1331  * If no sibling core can be found, return NULL.
1332  */
1333 static cpu_t *
cpu_get_sibling_core(cpu_t * cpup)1334 cpu_get_sibling_core(cpu_t *cpup)
1335 {
1336 	cpu_t		*nextp;
1337 	pg_t		*pg;
1338 	pg_cpu_itr_t	i;
1339 
1340 	if ((cpup == NULL) || (!cmp_cpu_is_cmp(cpup->cpu_id)))
1341 		return (NULL);
1342 	pg = (pg_t *)pghw_find_pg(cpup, PGHW_CHIP);
1343 	if (pg == NULL)
1344 		return (NULL);
1345 
1346 	/*
1347 	 * Iterate over the CPUs in the chip PG looking
1348 	 * for a CPU that isn't cpup
1349 	 */
1350 	PG_CPU_ITR_INIT(pg, i);
1351 	while ((nextp = pg_cpu_next(&i)) != NULL) {
1352 		if (nextp != cpup)
1353 			break;
1354 	}
1355 
1356 	if (nextp == NULL)
1357 		return (NULL);
1358 
1359 	return (nextp);
1360 }
1361