112 static void cpu_read_paddr(struct async_flt *aflt, short verbose, short ce_err);
116 static void log_ce_err(struct async_flt *aflt, char *unum);
117 static void log_ue_err(struct async_flt *aflt, char *unum);
120 static int check_ecc(struct async_flt *aflt);
926 log_ce_err(struct async_flt *aflt, char *unum)  in log_ce_err()  argument
930 	if ((aflt->flt_stat & P_AFSR_CE) && (ce_verbose_memory == 0)) {  in log_ce_err()
934 	spf_flt.cmn_asyncflt = *aflt;  in log_ce_err()
951 flt_to_error_type(struct async_flt *aflt)  in flt_to_error_type()  argument
953 	if (aflt->flt_status & ECC_INTERMITTENT)  in flt_to_error_type()
955 	if (aflt->flt_status & ECC_PERSISTENT)  in flt_to_error_type()
957 	if (aflt->flt_status & ECC_STICKY)  in flt_to_error_type()
972 	struct async_flt *aflt = (struct async_flt *)spf_flt;  in cpu_ce_log_status()  local
974 	status = aflt->flt_status;  in cpu_ce_log_status()
1172 cpu_ue_log_err(struct async_flt *aflt)  in cpu_ue_log_err()  argument
1175 	switch (aflt->flt_class) {  in cpu_ue_log_err()
1177 		cpu_async_log_err(aflt);  in cpu_ue_log_err()
1181 		bus_async_log_err(aflt);  in cpu_ue_log_err()
1186 		    "fault class (0x%x)", (void *)aflt, aflt->flt_class);  in cpu_ue_log_err()
1248 	struct async_flt *aflt;  in cpu_async_error()  local
1296 	aflt = (struct async_flt *)&spf_flt;  in cpu_async_error()
1297 	aflt->flt_id = gethrtime_waitfree();  in cpu_async_error()
1298 	aflt->flt_stat = t_afsr;  in cpu_async_error()
1299 	aflt->flt_addr = t_afar;  in cpu_async_error()
1300 	aflt->flt_bus_id = getprocessorid();  in cpu_async_error()
1301 	aflt->flt_inst = CPU->cpu_id;  in cpu_async_error()
1302 	aflt->flt_pc = (caddr_t)rp->r_pc;  in cpu_async_error()
1303 	aflt->flt_prot = AFLT_PROT_NONE;  in cpu_async_error()
1304 	aflt->flt_class = CPU_FAULT;  in cpu_async_error()
1305 	aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0;  in cpu_async_error()
1306 	aflt->flt_tl = (uchar_t)tl;  in cpu_async_error()
1307 	aflt->flt_panic = (tl != 0 || aft_testfatal != 0);  in cpu_async_error()
1308 	aflt->flt_core = (pflag & SDOCORE) ? 1 : 0;  in cpu_async_error()
1316 		aflt->flt_status = ECC_D_TRAP;  in cpu_async_error()
1319 		aflt->flt_status = ECC_I_TRAP;  in cpu_async_error()
1337 	if (aflt->flt_priv && tl == 0) {  in cpu_async_error()
1342 				aflt->flt_prot = AFLT_PROT_EC;  in cpu_async_error()
1351 				aflt->flt_prot = AFLT_PROT_ACCESS;  in cpu_async_error()
1368 			aflt->flt_prot = AFLT_PROT_COPY;  in cpu_async_error()
1397 	if (aflt->flt_prot == AFLT_PROT_NONE) {  in cpu_async_error()
1402 			if (aflt->flt_priv || aft_panic)  in cpu_async_error()
1403 				aflt->flt_panic = 1;  in cpu_async_error()
1407 			if (aflt->flt_priv)  in cpu_async_error()
1408 				aflt->flt_panic = 1;  in cpu_async_error()
1410 	} else if (aflt->flt_prot == AFLT_PROT_COPY && aft_panic) {  in cpu_async_error()
1411 		aflt->flt_panic = 1;  in cpu_async_error()
1419 		cpu_run_bus_error_handlers(aflt, expected);  in cpu_async_error()
1436 		aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >>  in cpu_async_error()
1444 		if (aflt->flt_in_memory) {  in cpu_async_error()
1448 			uint64_t faultpa = P2ALIGN(aflt->flt_addr, 64);  in cpu_async_error()
1461 				    (aflt->flt_addr % ec_set_size);  in cpu_async_error()
1472 				    ((uint64_t)aflt->flt_addr >>  in cpu_async_error()
1486 			    ((uint64_t)aflt->flt_addr >> cpu_ec_tag_shift)) {  in cpu_async_error()
1510 			flushecacheline(P2ALIGN(aflt->flt_addr, 64),  in cpu_async_error()
1532 			aflt->flt_panic = 1;  in cpu_async_error()
1535 			aflt->flt_synd = sdbh & P_DER_E_SYND;  in cpu_async_error()
1538 			    aflt->flt_panic);  in cpu_async_error()
1541 			aflt->flt_synd = sdbl & P_DER_E_SYND;  in cpu_async_error()
1542 			aflt->flt_synd |= UDBL_REG;	/* indicates UDBL */  in cpu_async_error()
1543 			if (!(aflt->flt_stat & P_AFSR_ME))  in cpu_async_error()
1544 				aflt->flt_addr |= 0x8;  in cpu_async_error()
1547 			    aflt->flt_panic);  in cpu_async_error()
1555 		if (aflt->flt_panic && aflt->flt_in_memory) {  in cpu_async_error()
1556 			panic_aflt = *aflt;  in cpu_async_error()
1578 		aflt->flt_addr = AFLT_INV_ADDR;  in cpu_async_error()
1579 		scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0],  in cpu_async_error()
1588 		if (aflt->flt_addr != AFLT_INV_ADDR) {  in cpu_async_error()
1589 			aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >>  in cpu_async_error()
1594 			aflt->flt_panic = 1;  in cpu_async_error()
1599 		    aflt->flt_panic);  in cpu_async_error()
1621 		if (aflt->flt_priv && aflt->flt_prot == AFLT_PROT_NONE) {  in cpu_async_error()
1625 			    aflt->flt_panic);  in cpu_async_error()
1643 			aflt->flt_panic = 1;  in cpu_async_error()
1650 		    aflt->flt_panic);  in cpu_async_error()
1662 			aflt->flt_panic = 1;  in cpu_async_error()
1666 			    aflt->flt_panic);  in cpu_async_error()
1683 			aflt->flt_addr = AFLT_INV_ADDR;  in cpu_async_error()
1684 			scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0],  in cpu_async_error()
1693 			if (aflt->flt_addr != AFLT_INV_ADDR) {  in cpu_async_error()
1694 				aflt->flt_in_memory =  in cpu_async_error()
1695 				    (pf_is_memory(aflt->flt_addr >>  in cpu_async_error()
1701 			    aflt->flt_panic);  in cpu_async_error()
1720 	    (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY)) {  in cpu_async_error()
1748 	ASSERT((aflt->flt_panic != 0) || (action != ACTION_NONE) ||  in cpu_async_error()
1773 	if ((aflt->flt_stat & P_AFSR_UE) && aflt->flt_addr == t_afar)  in cpu_async_error()
1774 		acc_afsr |= aflt->flt_stat & ~P_AFSR_UE;  in cpu_async_error()
1776 		acc_afsr |= aflt->flt_stat;  in cpu_async_error()
1788 			aflt->flt_panic = 1;  in cpu_async_error()
1791 		aflt->flt_stat = acc_afsr;  in cpu_async_error()
1794 		    aflt->flt_panic);  in cpu_async_error()
1803 	if (aflt->flt_panic) {  in cpu_async_error()
1824 	struct async_flt *aflt = (struct async_flt *)spf_flt;  in check_misc_err()  local
1832 	if (aflt->flt_stat & P_AFSR_ISAP)  in check_misc_err()
1834 	else if (aflt->flt_stat & P_AFSR_ETP)  in check_misc_err()
1836 	else if (aflt->flt_stat & P_AFSR_IVUE)  in check_misc_err()
1926 cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt,  in cpu_get_mem_unum_aflt()  argument
1929 	return (cpu_get_mem_unum(synd_status, SYND(aflt->flt_synd),  in cpu_get_mem_unum_aflt()
1930 	    aflt->flt_stat, aflt->flt_addr, aflt->flt_bus_id,  in cpu_get_mem_unum_aflt()
1931 	    aflt->flt_in_memory, aflt->flt_status, buf, buflen, lenp));  in cpu_get_mem_unum_aflt()
2008 log_ue_err(struct async_flt *aflt, char *unum)  in log_ue_err()  argument
2010 	spitf_async_flt *spf_flt = (spitf_async_flt *)aflt;  in log_ue_err()
2014 	int afsr_priv = (aflt->flt_stat & P_AFSR_PRIV) ? 1 : 0;  in log_ue_err()
2020 	if (afsr_priv != aflt->flt_priv) {  in log_ue_err()
2027 		    "TSTATE.PRIV=%d used", (aflt->flt_priv) ? 1 : 0);  in log_ue_err()
2030 		aflt->flt_stat &= ~P_AFSR_PRIV;  in log_ue_err()
2031 		if (aflt->flt_priv)  in log_ue_err()
2032 			aflt->flt_stat |= P_AFSR_PRIV;  in log_ue_err()
2036 	(void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, unum,  in log_ue_err()
2042 	if (SYND(aflt->flt_synd) == 0x3) {  in log_ue_err()
2048 	if (aflt->flt_in_memory)  in log_ue_err()
2063 	struct async_flt *aflt = (struct async_flt *)flt;  in cpu_async_log_err()  local
2091 		    (aflt->flt_stat & S_AFSR_ALL_ERRS) == P_AFSR_UE &&  in cpu_async_log_err()
2092 		    aflt->flt_prot == AFLT_PROT_EC) {  in cpu_async_log_err()
2093 			if (page_retire_check(aflt->flt_addr, NULL) == 0) {  in cpu_async_log_err()
2095 				softcall(ecc_page_zero, (void *)aflt->flt_addr);  in cpu_async_log_err()
2105 		log_ue_err(aflt, unum);  in cpu_async_log_err()
2106 		if (aflt->flt_in_memory)  in cpu_async_log_err()
2107 			cpu_check_allcpus(aflt);  in cpu_async_log_err()
2111 		if (aflt->flt_stat & P_AFSR_EDP)  in cpu_async_log_err()
2115 		if (aflt->flt_stat & P_AFSR_LDP)  in cpu_async_log_err()
2151 		if (aflt->flt_stat & P_AFSR_BERR) {  in cpu_async_log_err()
2152 			cpu_aflt_log(CE_WARN, aflt->flt_panic ? 1 : 2,  in cpu_async_log_err()
2155 			    aflt->flt_priv ? "privileged" : "user");  in cpu_async_log_err()
2158 		if (aflt->flt_stat & P_AFSR_TO) {  in cpu_async_log_err()
2159 			cpu_aflt_log(CE_WARN, aflt->flt_panic ? 1 : 2,  in cpu_async_log_err()
2162 			    aflt->flt_priv ? "privileged" : "user");  in cpu_async_log_err()
2174 		ASSERT(aflt->flt_id == panic_aflt.flt_id);  in cpu_async_log_err()
2187 		    aflt->flt_inst, space, (panic_aflt.flt_status & ECC_IOBUS) ?  in cpu_async_log_err()
2207 		    "on PCIBus)", aflt->flt_inst);  in cpu_async_log_err()
2300 	if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) {  in cpu_async_log_err()
2302 			(void) page_retire(aflt->flt_addr, PR_UE);  in cpu_async_log_err()
2309 			clearphys(P2ALIGN(aflt->flt_addr, 64),  in cpu_async_log_err()
2321 		if (!aflt->flt_priv) {  in cpu_async_log_err()
2326 		} else if (aflt->flt_prot == AFLT_PROT_COPY && aflt->flt_core) {  in cpu_async_log_err()
2330 		} else if (aflt->flt_prot == AFLT_PROT_COPY) {  in cpu_async_log_err()
2335 		} else if (aflt->flt_prot == AFLT_PROT_EC) {  in cpu_async_log_err()
2349 cpu_check_allcpus(struct async_flt *aflt)  in cpu_check_allcpus()  argument
2356 	cpflt->flt_id = aflt->flt_id;  in cpu_check_allcpus()
2357 	cpflt->flt_addr = aflt->flt_addr;  in cpu_check_allcpus()
2368 				if (aflt->flt_status & ECC_D_TRAP)  in cpu_check_allcpus()
2370 				else if (aflt->flt_status & ECC_I_TRAP)  in cpu_check_allcpus()
2379 				    (aflt->flt_status & ECC_IOBUS) ?  in cpu_check_allcpus()
2380 				    "IOBUS" : "CPU", aflt->flt_bus_id);  in cpu_check_allcpus()
2404 	struct async_flt *aflt = (struct async_flt *)arg;  in get_cpu_status()  local
2427 	aflt->flt_stat = afsr;  in get_cpu_status()
2451 		flt_addr_tag = aflt->flt_addr >> cpu_ec_tag_shift;  in get_cpu_status()
2456 		for (i = 0, ec_idx = (aflt->flt_addr % ec_set_size);  in get_cpu_status()
2495 	struct async_flt *aflt = (struct async_flt *)&cp;  in cpu_async_panic_callb()  local
2499 		aflt->flt_addr = panic_aflt.flt_addr;  in cpu_async_panic_callb()
2500 		(void) get_cpu_status((uint64_t)aflt);  in cpu_async_panic_callb()
2505 				aflt->flt_stat |= *scrub_afsr;  in cpu_async_panic_callb()
2509 		if (aflt->flt_stat & P_AFSR_CP) {  in cpu_async_panic_callb()
2510 			aflt->flt_id = panic_aflt.flt_id;  in cpu_async_panic_callb()
2511 			aflt->flt_panic = 1;  in cpu_async_panic_callb()
2512 			aflt->flt_inst = CPU->cpu_id;  in cpu_async_panic_callb()
2513 			aflt->flt_class = CPU_FAULT;  in cpu_async_panic_callb()
2517 			    aflt->flt_panic);  in cpu_async_panic_callb()
2708 			struct async_flt *aflt =  in check_ecc()  local
2713 			aflt->flt_stat = t_afsr;  in check_ecc()
2714 			aflt->flt_addr = t_afar;  in check_ecc()
2922 	struct async_flt *aflt = (struct async_flt *)spf_flt;  in clear_errors()  local
2932 		aflt->flt_stat = afsr;  in clear_errors()
2933 		get_asyncaddr(&aflt->flt_addr);  in clear_errors()
2934 		aflt->flt_addr &= SABRE_AFAR_PA;  in clear_errors()
3048 	struct async_flt *aflt = (struct async_flt *)spf_flt;  in cpu_log_ecmem_info()  local
3086 		    "E$parity 0x%02x %s", (uint32_t)(aflt->flt_addr >> 32),  in cpu_log_ecmem_info()
3087 		    (uint32_t)aflt->flt_addr, (uint32_t)(ecache_tag >> 32),  in cpu_log_ecmem_info()
3094 		    (uint32_t)(P2ALIGN(aflt->flt_addr, 64) >> 32),  in cpu_log_ecmem_info()
3095 		    (uint32_t)P2ALIGN(aflt->flt_addr, 64));  in cpu_log_ecmem_info()
3181 	struct async_flt *aflt = (struct async_flt *)spflt;  in cpu_aflt_log()  local
3186 	if ((aflt == NULL) || ((aflt->flt_class == CPU_FAULT) &&  in cpu_aflt_log()
3187 	    (aflt->flt_stat & P_AFSR_LEVEL1)) ||  in cpu_aflt_log()
3188 	    (aflt->flt_panic)) {  in cpu_aflt_log()
3191 		int verbose = ((aflt->flt_class == BUS_FAULT) ||  in cpu_aflt_log()
3192 		    (aflt->flt_stat & P_AFSR_CE)) ?  in cpu_aflt_log()
3211 		    (uint32_t)(aflt->flt_id >> 32), (uint32_t)aflt->flt_id);  in cpu_aflt_log()
3228 			    aflt->flt_inst);  in cpu_aflt_log()
3233 			if (aflt->flt_status & ECC_D_TRAP)  in cpu_aflt_log()
3236 			else if (aflt->flt_status & ECC_I_TRAP)  in cpu_aflt_log()
3244 			    aflt->flt_tl ? ">0" : "=0");  in cpu_aflt_log()
3251 			    (uint32_t)(aflt->flt_id >> 32),  in cpu_aflt_log()
3252 			    (uint32_t)aflt->flt_id);  in cpu_aflt_log()
3259 			    (uint32_t)(aflt->flt_stat >> 32), AFSR_FMTSTR0,  in cpu_aflt_log()
3260 			    (uint32_t)aflt->flt_stat, AFSR_FMTSTR1);  in cpu_aflt_log()
3266 			    (uint32_t)(aflt->flt_addr >> 32),  in cpu_aflt_log()
3267 			    (uint32_t)aflt->flt_addr);  in cpu_aflt_log()
3273 			    (aflt->flt_stat & P_AFSR_P_SYND);  in cpu_aflt_log()
3283 			    (uchar_t)((aflt->flt_stat & P_AFSR_ETS) >> 16));  in cpu_aflt_log()
3289 			    (void *)aflt->flt_pc);  in cpu_aflt_log()
3310 			ushort_t synd = SYND(aflt->flt_synd);  in cpu_aflt_log()
3314 			    UDBL(aflt->flt_synd) ? "UDBL" : "UDBH", synd);  in cpu_aflt_log()
3891 	struct async_flt *aflt;  in ecache_scrub_log()  local
3896 	aflt = &spf_flt.cmn_asyncflt;  in ecache_scrub_log()
3908 	aflt->flt_inst = CPU->cpu_id;  in ecache_scrub_log()
3909 	aflt->flt_class = CPU_FAULT;  in ecache_scrub_log()
3910 	aflt->flt_id = gethrtime_waitfree();  in ecache_scrub_log()
3911 	aflt->flt_addr = paddr;  in ecache_scrub_log()
3912 	aflt->flt_stat = afsr;  in ecache_scrub_log()
3913 	aflt->flt_panic = (uchar_t)ecache_scrub_panic;  in ecache_scrub_log()
3928 	    ue_queue, aflt->flt_panic);  in ecache_scrub_log()
3930 	if (aflt->flt_panic)  in ecache_scrub_log()
3944 	struct async_flt *aflt;  in ecache_scrub_misc_err()  local
3948 	aflt = &spf_flt.cmn_asyncflt;  in ecache_scrub_misc_err()
3954 	aflt->flt_addr = AFLT_INV_ADDR;  in ecache_scrub_misc_err()
3955 	scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0],  in ecache_scrub_misc_err()
3967 	if (aflt->flt_addr != AFLT_INV_ADDR) {  in ecache_scrub_misc_err()
3968 		aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >>  in ecache_scrub_misc_err()
3974 	aflt->flt_inst = CPU->cpu_id;  in ecache_scrub_misc_err()
3975 	aflt->flt_class = CPU_FAULT;  in ecache_scrub_misc_err()
3976 	aflt->flt_id = gethrtime_waitfree();  in ecache_scrub_misc_err()
3977 	aflt->flt_status = afsr;  in ecache_scrub_misc_err()
3978 	aflt->flt_panic = (uchar_t)ecache_scrub_panic;  in ecache_scrub_misc_err()
3985 		flushecacheline(P2ALIGN(aflt->flt_addr, 64),  in ecache_scrub_misc_err()
3993 	    (void *)&spf_flt, sizeof (spf_flt), ue_queue, aflt->flt_panic);  in ecache_scrub_misc_err()
4088 cpu_run_bus_error_handlers(struct async_flt *aflt, int expected)  in cpu_run_bus_error_handlers()  argument
4096 	de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst,  in cpu_run_bus_error_handlers()
4099 	de.fme_bus_specific = (void *)aflt->flt_addr;  in cpu_run_bus_error_handlers()
4102 	if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL))  in cpu_run_bus_error_handlers()
4103 		aflt->flt_panic = 1;  in cpu_run_bus_error_handlers()
4111 	struct async_flt *aflt = (struct async_flt *)payload;  in cpu_errorq_dispatch()  local
4113 	aflt->flt_erpt_class = error_class;  in cpu_errorq_dispatch()