/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include <sys/types.h> #include <sys/systm.h> #include <sys/ddi.h> #include <sys/sysmacros.h> #include <sys/archsystm.h> #include <sys/vmsystm.h> #include <sys/machparam.h> #include <sys/machsystm.h> #include <sys/machthread.h> #include <sys/cpu.h> #include <sys/cmp.h> #include <sys/elf_SPARC.h> #include <vm/vm_dep.h> #include <vm/hat_sfmmu.h> #include <vm/seg_kpm.h> #include <sys/cpuvar.h> #include <sys/cheetahregs.h> #include <sys/us3_module.h> #include <sys/async.h> #include <sys/cmn_err.h> #include <sys/debug.h> #include <sys/dditypes.h> #include <sys/prom_debug.h> #include <sys/prom_plat.h> #include <sys/cpu_module.h> #include <sys/sysmacros.h> #include <sys/intreg.h> #include <sys/clock.h> #include <sys/platform_module.h> #include <sys/machtrap.h> #include <sys/ontrap.h> #include <sys/panic.h> #include <sys/memlist.h> #include <sys/bootconf.h> #include <sys/ivintr.h> #include <sys/atomic.h> #include <sys/taskq.h> #include <sys/note.h> #include <sys/ndifm.h> #include <sys/ddifm.h> #include <sys/fm/protocol.h> #include <sys/fm/util.h> #include <sys/fm/cpu/UltraSPARC-III.h> #include <sys/fpras_impl.h> #include <sys/dtrace.h> #include <sys/watchpoint.h> #include <sys/plat_ecc_unum.h> #include <sys/cyclic.h> #include <sys/errorq.h> #include <sys/errclassify.h> #include <sys/pghw.h> #include <sys/clock_impl.h> #ifdef CHEETAHPLUS_ERRATUM_25 #include <sys/xc_impl.h> #endif /* CHEETAHPLUS_ERRATUM_25 */ ch_cpu_logout_t clop_before_flush; ch_cpu_logout_t clop_after_flush; uint_t flush_retries_done = 0; /* * Note that 'Cheetah PRM' refers to: * SPARC V9 JPS1 Implementation Supplement: Sun UltraSPARC-III */ /* * Per CPU pointers to physical address of TL>0 logout data areas. * These pointers have to be in the kernel nucleus to avoid MMU * misses. */ uint64_t ch_err_tl1_paddrs[NCPU]; /* * One statically allocated structure to use during startup/DR * to prevent unnecessary panics. */ ch_err_tl1_data_t ch_err_tl1_data; /* * Per CPU pending error at TL>0, used by level15 softint handler */ uchar_t ch_err_tl1_pending[NCPU]; /* * For deferred CE re-enable after trap. */ taskq_t *ch_check_ce_tq; /* * Internal functions. */ static int cpu_async_log_err(void *flt, errorq_elem_t *eqep); static void cpu_log_diag_info(ch_async_flt_t *ch_flt); static void cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, ecc_type_to_info_t *eccp, ch_diag_data_t *cdp); static int cpu_flt_in_memory_one_event(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit); static int clear_ecc(struct async_flt *ecc); #if defined(CPU_IMP_ECACHE_ASSOC) static int cpu_ecache_line_valid(ch_async_flt_t *ch_flt); #endif int cpu_ecache_set_size(struct cpu *cp); static int cpu_ectag_line_invalid(int cachesize, uint64_t tag); int cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr); uint64_t cpu_ectag_to_pa(int setsize, uint64_t tag); int cpu_ectag_pa_to_subblk_state(int cachesize, uint64_t subaddr, uint64_t tag); static void cpu_flush_ecache_line(ch_async_flt_t *ch_flt); static int afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit); static int afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit); static int afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit); static int afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit); static int synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit); static int cpu_get_mem_unum_synd(int synd_code, struct async_flt *, char *buf); static void cpu_uninit_ecache_scrub_dr(struct cpu *cp); static void cpu_scrubphys(struct async_flt *aflt); static void cpu_payload_add_aflt(struct async_flt *, nvlist_t *, nvlist_t *, int *, int *); static void cpu_payload_add_ecache(struct async_flt *, nvlist_t *); static void cpu_ereport_init(struct async_flt *aflt); static int cpu_check_secondary_errors(ch_async_flt_t *, uint64_t, uint64_t); static uint8_t cpu_flt_bit_to_plat_error(struct async_flt *aflt); static void cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, uint64_t nceen, ch_cpu_logout_t *clop); static int cpu_ce_delayed_ec_logout(uint64_t); static int cpu_matching_ecache_line(uint64_t, void *, int, int *); static int cpu_error_is_ecache_data(int, uint64_t); static void cpu_fmri_cpu_set(nvlist_t *, int); static int cpu_error_to_resource_type(struct async_flt *aflt); #ifdef CHEETAHPLUS_ERRATUM_25 static int mondo_recover_proc(uint16_t, int); static void cheetah_nudge_init(void); static void cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when); static void cheetah_nudge_buddy(void); #endif /* CHEETAHPLUS_ERRATUM_25 */ #if defined(CPU_IMP_L1_CACHE_PARITY) static void cpu_dcache_parity_info(ch_async_flt_t *ch_flt); static void cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index); static void cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word); static void cpu_icache_parity_info(ch_async_flt_t *ch_flt); static void cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index); static void cpu_pcache_parity_info(ch_async_flt_t *ch_flt); static void cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index); static void cpu_payload_add_dcache(struct async_flt *, nvlist_t *); static void cpu_payload_add_icache(struct async_flt *, nvlist_t *); #endif /* CPU_IMP_L1_CACHE_PARITY */ int (*p2get_mem_info)(int synd_code, uint64_t paddr, uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, int *segsp, int *banksp, int *mcidp); /* * This table is used to determine which bit(s) is(are) bad when an ECC * error occurs. The array is indexed by an 9-bit syndrome. The entries * of this array have the following semantics: * * 00-127 The number of the bad bit, when only one bit is bad. * 128 ECC bit C0 is bad. * 129 ECC bit C1 is bad. * 130 ECC bit C2 is bad. * 131 ECC bit C3 is bad. * 132 ECC bit C4 is bad. * 133 ECC bit C5 is bad. * 134 ECC bit C6 is bad. * 135 ECC bit C7 is bad. * 136 ECC bit C8 is bad. * 137-143 reserved for Mtag Data and ECC. * 144(M2) Two bits are bad within a nibble. * 145(M3) Three bits are bad within a nibble. * 146(M3) Four bits are bad within a nibble. * 147(M) Multiple bits (5 or more) are bad. * 148 NO bits are bad. * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-4,11-5. */ #define C0 128 #define C1 129 #define C2 130 #define C3 131 #define C4 132 #define C5 133 #define C6 134 #define C7 135 #define C8 136 #define MT0 137 /* Mtag Data bit 0 */ #define MT1 138 #define MT2 139 #define MTC0 140 /* Mtag Check bit 0 */ #define MTC1 141 #define MTC2 142 #define MTC3 143 #define M2 144 #define M3 145 #define M4 146 #define M 147 #define NA 148 #if defined(JALAPENO) || defined(SERRANO) #define S003 149 /* Syndrome 0x003 => likely from CPU/EDU:ST/FRU/BP */ #define S003MEM 150 /* Syndrome 0x003 => likely from WDU/WBP */ #define SLAST S003MEM /* last special syndrome */ #else /* JALAPENO || SERRANO */ #define S003 149 /* Syndrome 0x003 => likely from EDU:ST */ #define S071 150 /* Syndrome 0x071 => likely from WDU/CPU */ #define S11C 151 /* Syndrome 0x11c => likely from BERR/DBERR */ #define SLAST S11C /* last special syndrome */ #endif /* JALAPENO || SERRANO */ #if defined(JALAPENO) || defined(SERRANO) #define BPAR0 152 /* syndrom 152 through 167 for bus parity */ #define BPAR15 167 #endif /* JALAPENO || SERRANO */ static uint8_t ecc_syndrome_tab[] = { NA, C0, C1, S003, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M, C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16, C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10, M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M, C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6, M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4, M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4, #if defined(JALAPENO) || defined(SERRANO) 116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, #else /* JALAPENO || SERRANO */ 116, S071, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, #endif /* JALAPENO || SERRANO */ C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5, M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M, M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2, 103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3, M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M, 102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3, 98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M, M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M, C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4, #if defined(JALAPENO) || defined(SERRANO) M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M, #else /* JALAPENO || SERRANO */ M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, S11C, M, M3, M, #endif /* JALAPENO || SERRANO */ M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2, 94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M, M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4, 89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3, 86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3, M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2, M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4, 77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M, 74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3, M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M, 80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3, M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M, M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M, 111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M }; #define ESYND_TBL_SIZE (sizeof (ecc_syndrome_tab) / sizeof (uint8_t)) #if !(defined(JALAPENO) || defined(SERRANO)) /* * This table is used to determine which bit(s) is(are) bad when a Mtag * error occurs. The array is indexed by an 4-bit ECC syndrome. The entries * of this array have the following semantics: * * -1 Invalid mtag syndrome. * 137 Mtag Data 0 is bad. * 138 Mtag Data 1 is bad. * 139 Mtag Data 2 is bad. * 140 Mtag ECC 0 is bad. * 141 Mtag ECC 1 is bad. * 142 Mtag ECC 2 is bad. * 143 Mtag ECC 3 is bad. * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-6. */ short mtag_syndrome_tab[] = { NA, MTC0, MTC1, M2, MTC2, M2, M2, MT0, MTC3, M2, M2, MT1, M2, MT2, M2, M2 }; #define MSYND_TBL_SIZE (sizeof (mtag_syndrome_tab) / sizeof (short)) #else /* !(JALAPENO || SERRANO) */ #define BSYND_TBL_SIZE 16 #endif /* !(JALAPENO || SERRANO) */ /* * Virtual Address bit flag in the data cache. This is actually bit 2 in the * dcache data tag. */ #define VA13 INT64_C(0x0000000000000002) /* * Types returned from cpu_error_to_resource_type() */ #define ERRTYPE_UNKNOWN 0 #define ERRTYPE_CPU 1 #define ERRTYPE_MEMORY 2 #define ERRTYPE_ECACHE_DATA 3 /* * CE initial classification and subsequent action lookup table */ static ce_dispact_t ce_disp_table[CE_INITDISPTBL_SIZE]; static int ce_disp_inited; /* * Set to disable leaky and partner check for memory correctables */ int ce_xdiag_off; /* * The following are not incremented atomically so are indicative only */ static int ce_xdiag_drops; static int ce_xdiag_lkydrops; static int ce_xdiag_ptnrdrops; static int ce_xdiag_bad; /* * CE leaky check callback structure */ typedef struct { struct async_flt *lkycb_aflt; errorq_t *lkycb_eqp; errorq_elem_t *lkycb_eqep; } ce_lkychk_cb_t; /* * defines for various ecache_flush_flag's */ #define ECACHE_FLUSH_LINE 1 #define ECACHE_FLUSH_ALL 2 /* * STICK sync */ #define STICK_ITERATION 10 #define MAX_TSKEW 1 #define EV_A_START 0 #define EV_A_END 1 #define EV_B_START 2 #define EV_B_END 3 #define EVENTS 4 static int64_t stick_iter = STICK_ITERATION; static int64_t stick_tsk = MAX_TSKEW; typedef enum { EVENT_NULL = 0, SLAVE_START, SLAVE_CONT, MASTER_START } event_cmd_t; static volatile event_cmd_t stick_sync_cmd = EVENT_NULL; static int64_t timestamp[EVENTS]; static volatile int slave_done; #ifdef DEBUG #define DSYNC_ATTEMPTS 64 typedef struct { int64_t skew_val[DSYNC_ATTEMPTS]; } ss_t; ss_t stick_sync_stats[NCPU]; #endif /* DEBUG */ uint_t cpu_impl_dual_pgsz = 0; #if defined(CPU_IMP_DUAL_PAGESIZE) uint_t disable_dual_pgsz = 0; #endif /* CPU_IMP_DUAL_PAGESIZE */ /* * Save the cache bootup state for use when internal * caches are to be re-enabled after an error occurs. */ uint64_t cache_boot_state; /* * PA[22:0] represent Displacement in Safari configuration space. */ uint_t root_phys_addr_lo_mask = 0x7fffffu; bus_config_eclk_t bus_config_eclk[] = { #if defined(JALAPENO) || defined(SERRANO) {JBUS_CONFIG_ECLK_1_DIV, JBUS_CONFIG_ECLK_1}, {JBUS_CONFIG_ECLK_2_DIV, JBUS_CONFIG_ECLK_2}, {JBUS_CONFIG_ECLK_32_DIV, JBUS_CONFIG_ECLK_32}, #else /* JALAPENO || SERRANO */ {SAFARI_CONFIG_ECLK_1_DIV, SAFARI_CONFIG_ECLK_1}, {SAFARI_CONFIG_ECLK_2_DIV, SAFARI_CONFIG_ECLK_2}, {SAFARI_CONFIG_ECLK_32_DIV, SAFARI_CONFIG_ECLK_32}, #endif /* JALAPENO || SERRANO */ {0, 0} }; /* * Interval for deferred CEEN reenable */ int cpu_ceen_delay_secs = CPU_CEEN_DELAY_SECS; /* * set in /etc/system to control logging of user BERR/TO's */ int cpu_berr_to_verbose = 0; /* * set to 0 in /etc/system to defer CEEN reenable for all CEs */ uint64_t cpu_ce_not_deferred = CPU_CE_NOT_DEFERRED; uint64_t cpu_ce_not_deferred_ext = CPU_CE_NOT_DEFERRED_EXT; /* * Set of all offline cpus */ cpuset_t cpu_offline_set; static void cpu_delayed_check_ce_errors(void *); static void cpu_check_ce_errors(void *); void cpu_error_ecache_flush(ch_async_flt_t *); static int cpu_error_ecache_flush_required(ch_async_flt_t *); static void cpu_log_and_clear_ce(ch_async_flt_t *); void cpu_ce_detected(ch_cpu_errors_t *, int); /* * CE Leaky check timeout in microseconds. This is chosen to be twice the * memory refresh interval of current DIMMs (64ms). After initial fix that * gives at least one full refresh cycle in which the cell can leak * (whereafter further refreshes simply reinforce any incorrect bit value). */ clock_t cpu_ce_lkychk_timeout_usec = 128000; /* * CE partner check partner caching period in seconds */ int cpu_ce_ptnr_cachetime_sec = 60; /* * Sets trap table entry ttentry by overwriting eight instructions from ttlabel */ #define CH_SET_TRAP(ttentry, ttlabel) \ bcopy((const void *)&ttlabel, &ttentry, 32); \ flush_instr_mem((caddr_t)&ttentry, 32); static int min_ecache_size; static uint_t priv_hcl_1; static uint_t priv_hcl_2; static uint_t priv_hcl_4; static uint_t priv_hcl_8; void cpu_setup(void) { extern int at_flags; extern int cpc_has_overflow_intr; /* * Setup chip-specific trap handlers. */ cpu_init_trap(); cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT); at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1 | EF_SPARC_SUN_US3; /* * save the cache bootup state. */ cache_boot_state = get_dcu() & DCU_CACHE; /* * Due to the number of entries in the fully-associative tlb * this may have to be tuned lower than in spitfire. */ pp_slots = MIN(8, MAXPP_SLOTS); /* * Block stores do not invalidate all pages of the d$, pagecopy * et. al. need virtual translations with virtual coloring taken * into consideration. prefetch/ldd will pollute the d$ on the * load side. */ pp_consistent_coloring = PPAGE_STORE_VCOLORING | PPAGE_LOADS_POLLUTE; if (use_page_coloring) { do_pg_coloring = 1; } isa_list = "sparcv9+vis2 sparcv9+vis sparcv9 " "sparcv8plus+vis2 sparcv8plus+vis sparcv8plus " "sparcv8 sparcv8-fsmuld sparcv7 sparc"; /* * On Panther-based machines, this should * also include AV_SPARC_POPC too */ cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2; /* * On cheetah, there's no hole in the virtual address space */ hole_start = hole_end = 0; /* * The kpm mapping window. * kpm_size: * The size of a single kpm range. * The overall size will be: kpm_size * vac_colors. * kpm_vbase: * The virtual start address of the kpm range within the kernel * virtual address space. kpm_vbase has to be kpm_size aligned. */ kpm_size = (size_t)(8ull * 1024 * 1024 * 1024 * 1024); /* 8TB */ kpm_size_shift = 43; kpm_vbase = (caddr_t)0x8000000000000000ull; /* 8EB */ kpm_smallpages = 1; /* * The traptrace code uses either %tick or %stick for * timestamping. We have %stick so we can use it. */ traptrace_use_stick = 1; /* * Cheetah has a performance counter overflow interrupt */ cpc_has_overflow_intr = 1; #if defined(CPU_IMP_DUAL_PAGESIZE) /* * Use Cheetah+ and later dual page size support. */ if (!disable_dual_pgsz) { cpu_impl_dual_pgsz = 1; } #endif /* CPU_IMP_DUAL_PAGESIZE */ /* * Declare that this architecture/cpu combination does fpRAS. */ fpras_implemented = 1; /* * Setup CE lookup table */ CE_INITDISPTBL_POPULATE(ce_disp_table); ce_disp_inited = 1; } /* * Called by setcpudelay */ void cpu_init_tick_freq(void) { /* * For UltraSPARC III and beyond we want to use the * system clock rate as the basis for low level timing, * due to support of mixed speed CPUs and power managment. */ if (system_clock_freq == 0) cmn_err(CE_PANIC, "setcpudelay: invalid system_clock_freq"); sys_tick_freq = system_clock_freq; } #ifdef CHEETAHPLUS_ERRATUM_25 /* * Tunables */ int cheetah_bpe_off = 0; int cheetah_sendmondo_recover = 1; int cheetah_sendmondo_fullscan = 0; int cheetah_sendmondo_recover_delay = 5; #define CHEETAH_LIVELOCK_MIN_DELAY 1 /* * Recovery Statistics */ typedef struct cheetah_livelock_entry { int cpuid; /* fallen cpu */ int buddy; /* cpu that ran recovery */ clock_t lbolt; /* when recovery started */ hrtime_t recovery_time; /* time spent in recovery */ } cheetah_livelock_entry_t; #define CHEETAH_LIVELOCK_NENTRY 32 cheetah_livelock_entry_t cheetah_livelock_hist[CHEETAH_LIVELOCK_NENTRY]; int cheetah_livelock_entry_nxt; #define CHEETAH_LIVELOCK_ENTRY_NEXT(statp) { \ statp = cheetah_livelock_hist + cheetah_livelock_entry_nxt; \ if (++cheetah_livelock_entry_nxt >= CHEETAH_LIVELOCK_NENTRY) { \ cheetah_livelock_entry_nxt = 0; \ } \ } #define CHEETAH_LIVELOCK_ENTRY_SET(statp, item, val) statp->item = val struct { hrtime_t hrt; /* maximum recovery time */ int recovery; /* recovered */ int full_claimed; /* maximum pages claimed in full recovery */ int proc_entry; /* attempted to claim TSB */ int proc_tsb_scan; /* tsb scanned */ int proc_tsb_partscan; /* tsb partially scanned */ int proc_tsb_fullscan; /* whole tsb scanned */ int proc_claimed; /* maximum pages claimed in tsb scan */ int proc_user; /* user thread */ int proc_kernel; /* kernel thread */ int proc_onflt; /* bad stack */ int proc_cpu; /* null cpu */ int proc_thread; /* null thread */ int proc_proc; /* null proc */ int proc_as; /* null as */ int proc_hat; /* null hat */ int proc_hat_inval; /* hat contents don't make sense */ int proc_hat_busy; /* hat is changing TSBs */ int proc_tsb_reloc; /* TSB skipped because being relocated */ int proc_cnum_bad; /* cnum out of range */ int proc_cnum; /* last cnum processed */ tte_t proc_tte; /* last tte processed */ } cheetah_livelock_stat; #define CHEETAH_LIVELOCK_STAT(item) cheetah_livelock_stat.item++ #define CHEETAH_LIVELOCK_STATSET(item, value) \ cheetah_livelock_stat.item = value #define CHEETAH_LIVELOCK_MAXSTAT(item, value) { \ if (value > cheetah_livelock_stat.item) \ cheetah_livelock_stat.item = value; \ } /* * Attempt to recover a cpu by claiming every cache line as saved * in the TSB that the non-responsive cpu is using. Since we can't * grab any adaptive lock, this is at best an attempt to do so. Because * we don't grab any locks, we must operate under the protection of * on_fault(). * * Return 1 if cpuid could be recovered, 0 if failed. */ int mondo_recover_proc(uint16_t cpuid, int bn) { label_t ljb; cpu_t *cp; kthread_t *t; proc_t *p; struct as *as; struct hat *hat; uint_t cnum; struct tsb_info *tsbinfop; struct tsbe *tsbep; caddr_t tsbp; caddr_t end_tsbp; uint64_t paddr; uint64_t idsr; u_longlong_t pahi, palo; int pages_claimed = 0; tte_t tsbe_tte; int tried_kernel_tsb = 0; mmu_ctx_t *mmu_ctxp; CHEETAH_LIVELOCK_STAT(proc_entry); if (on_fault(&ljb)) { CHEETAH_LIVELOCK_STAT(proc_onflt); goto badstruct; } if ((cp = cpu[cpuid]) == NULL) { CHEETAH_LIVELOCK_STAT(proc_cpu); goto badstruct; } if ((t = cp->cpu_thread) == NULL) { CHEETAH_LIVELOCK_STAT(proc_thread); goto badstruct; } if ((p = ttoproc(t)) == NULL) { CHEETAH_LIVELOCK_STAT(proc_proc); goto badstruct; } if ((as = p->p_as) == NULL) { CHEETAH_LIVELOCK_STAT(proc_as); goto badstruct; } if ((hat = as->a_hat) == NULL) { CHEETAH_LIVELOCK_STAT(proc_hat); goto badstruct; } if (hat != ksfmmup) { CHEETAH_LIVELOCK_STAT(proc_user); if (hat->sfmmu_flags & (HAT_BUSY | HAT_SWAPPED | HAT_SWAPIN)) { CHEETAH_LIVELOCK_STAT(proc_hat_busy); goto badstruct; } tsbinfop = hat->sfmmu_tsb; if (tsbinfop == NULL) { CHEETAH_LIVELOCK_STAT(proc_hat_inval); goto badstruct; } tsbp = tsbinfop->tsb_va; end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); } else { CHEETAH_LIVELOCK_STAT(proc_kernel); tsbinfop = NULL; tsbp = ktsb_base; end_tsbp = tsbp + TSB_BYTES(ktsb_sz); } /* Verify as */ if (hat->sfmmu_as != as) { CHEETAH_LIVELOCK_STAT(proc_hat_inval); goto badstruct; } mmu_ctxp = CPU_MMU_CTXP(cp); ASSERT(mmu_ctxp); cnum = hat->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; CHEETAH_LIVELOCK_STATSET(proc_cnum, cnum); if ((cnum < 0) || (cnum == INVALID_CONTEXT) || (cnum >= mmu_ctxp->mmu_nctxs)) { CHEETAH_LIVELOCK_STAT(proc_cnum_bad); goto badstruct; } do { CHEETAH_LIVELOCK_STAT(proc_tsb_scan); /* * Skip TSBs being relocated. This is important because * we want to avoid the following deadlock scenario: * * 1) when we came in we set ourselves to "in recover" state. * 2) when we try to touch TSB being relocated the mapping * will be in the suspended state so we'll spin waiting * for it to be unlocked. * 3) when the CPU that holds the TSB mapping locked tries to * unlock it it will send a xtrap which will fail to xcall * us or the CPU we're trying to recover, and will in turn * enter the mondo code. * 4) since we are still spinning on the locked mapping * no further progress will be made and the system will * inevitably hard hang. * * A TSB not being relocated can't begin being relocated * while we're accessing it because we check * sendmondo_in_recover before relocating TSBs. */ if (hat != ksfmmup && (tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { CHEETAH_LIVELOCK_STAT(proc_tsb_reloc); goto next_tsbinfo; } for (tsbep = (struct tsbe *)tsbp; tsbep < (struct tsbe *)end_tsbp; tsbep++) { tsbe_tte = tsbep->tte_data; if (tsbe_tte.tte_val == 0) { /* * Invalid tte */ continue; } if (tsbe_tte.tte_se) { /* * Don't want device registers */ continue; } if (tsbe_tte.tte_cp == 0) { /* * Must be cached in E$ */ continue; } if (tsbep->tte_tag.tag_invalid != 0) { /* * Invalid tag, ingnore this entry. */ continue; } CHEETAH_LIVELOCK_STATSET(proc_tte, tsbe_tte); idsr = getidsr(); if ((idsr & (IDSR_NACK_BIT(bn) | IDSR_BUSY_BIT(bn))) == 0) { CHEETAH_LIVELOCK_STAT(proc_tsb_partscan); goto done; } pahi = tsbe_tte.tte_pahi; palo = tsbe_tte.tte_palo; paddr = (uint64_t)((pahi << 32) | (palo << MMU_PAGESHIFT)); claimlines(paddr, TTEBYTES(TTE_CSZ(&tsbe_tte)), CH_ECACHE_SUBBLK_SIZE); if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { shipit(cpuid, bn); } pages_claimed++; } next_tsbinfo: if (tsbinfop != NULL) tsbinfop = tsbinfop->tsb_next; if (tsbinfop != NULL) { tsbp = tsbinfop->tsb_va; end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); } else if (tsbp == ktsb_base) { tried_kernel_tsb = 1; } else if (!tried_kernel_tsb) { tsbp = ktsb_base; end_tsbp = tsbp + TSB_BYTES(ktsb_sz); hat = ksfmmup; tsbinfop = NULL; } } while (tsbinfop != NULL || ((tsbp == ktsb_base) && !tried_kernel_tsb)); CHEETAH_LIVELOCK_STAT(proc_tsb_fullscan); CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); no_fault(); idsr = getidsr(); if ((idsr & (IDSR_NACK_BIT(bn) | IDSR_BUSY_BIT(bn))) == 0) { return (1); } else { return (0); } done: no_fault(); CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); return (1); badstruct: no_fault(); return (0); } /* * Attempt to claim ownership, temporarily, of every cache line that a * non-responsive cpu might be using. This might kick that cpu out of * this state. * * The return value indicates to the caller if we have exhausted all recovery * techniques. If 1 is returned, it is useless to call this function again * even for a different target CPU. */ int mondo_recover(uint16_t cpuid, int bn) { struct memseg *seg; uint64_t begin_pa, end_pa, cur_pa; hrtime_t begin_hrt, end_hrt; int retval = 0; int pages_claimed = 0; cheetah_livelock_entry_t *histp; uint64_t idsr; if (atomic_cas_32(&sendmondo_in_recover, 0, 1) != 0) { /* * Wait while recovery takes place */ while (sendmondo_in_recover) { drv_usecwait(1); } /* * Assume we didn't claim the whole memory. If * the target of this caller is not recovered, * it will come back. */ return (retval); } CHEETAH_LIVELOCK_ENTRY_NEXT(histp); CHEETAH_LIVELOCK_ENTRY_SET(histp, lbolt, LBOLT_WAITFREE); CHEETAH_LIVELOCK_ENTRY_SET(histp, cpuid, cpuid); CHEETAH_LIVELOCK_ENTRY_SET(histp, buddy, CPU->cpu_id); begin_hrt = gethrtime_waitfree(); /* * First try to claim the lines in the TSB the target * may have been using. */ if (mondo_recover_proc(cpuid, bn) == 1) { /* * Didn't claim the whole memory */ goto done; } /* * We tried using the TSB. The target is still * not recovered. Check if complete memory scan is * enabled. */ if (cheetah_sendmondo_fullscan == 0) { /* * Full memory scan is disabled. */ retval = 1; goto done; } /* * Try claiming the whole memory. */ for (seg = memsegs; seg; seg = seg->next) { begin_pa = (uint64_t)(seg->pages_base) << MMU_PAGESHIFT; end_pa = (uint64_t)(seg->pages_end) << MMU_PAGESHIFT; for (cur_pa = begin_pa; cur_pa < end_pa; cur_pa += MMU_PAGESIZE) { idsr = getidsr(); if ((idsr & (IDSR_NACK_BIT(bn) | IDSR_BUSY_BIT(bn))) == 0) { /* * Didn't claim all memory */ goto done; } claimlines(cur_pa, MMU_PAGESIZE, CH_ECACHE_SUBBLK_SIZE); if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { shipit(cpuid, bn); } pages_claimed++; } } /* * We did all we could. */ retval = 1; done: /* * Update statistics */ end_hrt = gethrtime_waitfree(); CHEETAH_LIVELOCK_STAT(recovery); CHEETAH_LIVELOCK_MAXSTAT(hrt, (end_hrt - begin_hrt)); CHEETAH_LIVELOCK_MAXSTAT(full_claimed, pages_claimed); CHEETAH_LIVELOCK_ENTRY_SET(histp, recovery_time, \ (end_hrt - begin_hrt)); while (atomic_cas_32(&sendmondo_in_recover, 1, 0) != 1) ; return (retval); } /* * This is called by the cyclic framework when this CPU becomes online */ /*ARGSUSED*/ static void cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when) { hdlr->cyh_func = (cyc_func_t)cheetah_nudge_buddy; hdlr->cyh_level = CY_LOW_LEVEL; hdlr->cyh_arg = NULL; /* * Stagger the start time */ when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU); if (cheetah_sendmondo_recover_delay < CHEETAH_LIVELOCK_MIN_DELAY) { cheetah_sendmondo_recover_delay = CHEETAH_LIVELOCK_MIN_DELAY; } when->cyt_interval = cheetah_sendmondo_recover_delay * NANOSEC; } /* * Create a low level cyclic to send a xtrap to the next cpu online. * However, there's no need to have this running on a uniprocessor system. */ static void cheetah_nudge_init(void) { cyc_omni_handler_t hdlr; if (max_ncpus == 1) { return; } hdlr.cyo_online = cheetah_nudge_onln; hdlr.cyo_offline = NULL; hdlr.cyo_arg = NULL; mutex_enter(&cpu_lock); (void) cyclic_add_omni(&hdlr); mutex_exit(&cpu_lock); } /* * Cyclic handler to wake up buddy */ void cheetah_nudge_buddy(void) { /* * Disable kernel preemption to protect the cpu list */ kpreempt_disable(); if ((CPU->cpu_next_onln != CPU) && (sendmondo_in_recover == 0)) { xt_one(CPU->cpu_next_onln->cpu_id, (xcfunc_t *)xt_sync_tl1, 0, 0); } kpreempt_enable(); } #endif /* CHEETAHPLUS_ERRATUM_25 */ #ifdef SEND_MONDO_STATS uint32_t x_one_stimes[64]; uint32_t x_one_ltimes[16]; uint32_t x_set_stimes[64]; uint32_t x_set_ltimes[16]; uint32_t x_set_cpus[NCPU]; uint32_t x_nack_stimes[64]; #endif /* * Note: A version of this function is used by the debugger via the KDI, * and must be kept in sync with this version. Any changes made to this * function to support new chips or to accomodate errata must also be included * in the KDI-specific version. See us3_kdi.c. */ void send_one_mondo(int cpuid) { int busy, nack; uint64_t idsr, starttick, endtick, tick, lasttick; uint64_t busymask; #ifdef CHEETAHPLUS_ERRATUM_25 int recovered = 0; #endif CPU_STATS_ADDQ(CPU, sys, xcalls, 1); starttick = lasttick = gettick(); shipit(cpuid, 0); endtick = starttick + xc_tick_limit; busy = nack = 0; #if defined(JALAPENO) || defined(SERRANO) /* * Lower 2 bits of the agent ID determine which BUSY/NACK pair * will be used for dispatching interrupt. For now, assume * there are no more than IDSR_BN_SETS CPUs, hence no aliasing * issues with respect to BUSY/NACK pair usage. */ busymask = IDSR_BUSY_BIT(cpuid); #else /* JALAPENO || SERRANO */ busymask = IDSR_BUSY; #endif /* JALAPENO || SERRANO */ for (;;) { idsr = getidsr(); if (idsr == 0) break; tick = gettick(); /* * If there is a big jump between the current tick * count and lasttick, we have probably hit a break * point. Adjust endtick accordingly to avoid panic. */ if (tick > (lasttick + xc_tick_jump_limit)) endtick += (tick - lasttick); lasttick = tick; if (tick > endtick) { if (panic_quiesce) return; #ifdef CHEETAHPLUS_ERRATUM_25 if (cheetah_sendmondo_recover && recovered == 0) { if (mondo_recover(cpuid, 0)) { /* * We claimed the whole memory or * full scan is disabled. */ recovered++; } tick = gettick(); endtick = tick + xc_tick_limit; lasttick = tick; /* * Recheck idsr */ continue; } else #endif /* CHEETAHPLUS_ERRATUM_25 */ { cmn_err(CE_PANIC, "send mondo timeout " "(target 0x%x) [%d NACK %d BUSY]", cpuid, nack, busy); } } if (idsr & busymask) { busy++; continue; } drv_usecwait(1); shipit(cpuid, 0); nack++; busy = 0; } #ifdef SEND_MONDO_STATS { int n = gettick() - starttick; if (n < 8192) x_one_stimes[n >> 7]++; else x_one_ltimes[(n >> 13) & 0xf]++; } #endif } void syncfpu(void) { } /* * Return processor specific async error structure * size used. */ int cpu_aflt_size(void) { return (sizeof (ch_async_flt_t)); } /* * Tunable to disable the checking of other cpu logout areas during panic for * potential syndrome 71 generating errors. */ int enable_check_other_cpus_logout = 1; /* * Check other cpus logout area for potential synd 71 generating * errors. */ static void cpu_check_cpu_logout(int cpuid, caddr_t tpc, int tl, int ecc_type, ch_cpu_logout_t *clop) { struct async_flt *aflt; ch_async_flt_t ch_flt; uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; if (clop == NULL || clop->clo_data.chd_afar == LOGOUT_INVALID) { return; } bzero(&ch_flt, sizeof (ch_async_flt_t)); t_afar = clop->clo_data.chd_afar; t_afsr = clop->clo_data.chd_afsr; t_afsr_ext = clop->clo_data.chd_afsr_ext; #if defined(SERRANO) ch_flt.afar2 = clop->clo_data.chd_afar2; #endif /* SERRANO */ /* * In order to simplify code, we maintain this afsr_errs * variable which holds the aggregate of AFSR and AFSR_EXT * sticky bits. */ t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | (t_afsr & C_AFSR_ALL_ERRS); /* Setup the async fault structure */ aflt = (struct async_flt *)&ch_flt; aflt->flt_id = gethrtime_waitfree(); ch_flt.afsr_ext = t_afsr_ext; ch_flt.afsr_errs = t_afsr_errs; aflt->flt_stat = t_afsr; aflt->flt_addr = t_afar; aflt->flt_bus_id = cpuid; aflt->flt_inst = cpuid; aflt->flt_pc = tpc; aflt->flt_prot = AFLT_PROT_NONE; aflt->flt_class = CPU_FAULT; aflt->flt_priv = ((t_afsr & C_AFSR_PRIV) != 0); aflt->flt_tl = tl; aflt->flt_status = ecc_type; aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); /* * Queue events on the async event queue, one event per error bit. * If no events are queued, queue an event to complain. */ if (cpu_queue_events(&ch_flt, NULL, t_afsr_errs, clop) == 0) { ch_flt.flt_type = CPU_INV_AFSR; cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); } /* * Zero out + invalidate CPU logout. */ bzero(clop, sizeof (ch_cpu_logout_t)); clop->clo_data.chd_afar = LOGOUT_INVALID; } /* * Check the logout areas of all other cpus for unlogged errors. */ static void cpu_check_other_cpus_logout(void) { int i, j; processorid_t myid; struct cpu *cp; ch_err_tl1_data_t *cl1p; myid = CPU->cpu_id; for (i = 0; i < NCPU; i++) { cp = cpu[i]; if ((cp == NULL) || !(cp->cpu_flags & CPU_EXISTS) || (cp->cpu_id == myid) || (CPU_PRIVATE(cp) == NULL)) { continue; } /* * Check each of the tl>0 logout areas */ cl1p = CPU_PRIVATE_PTR(cp, chpr_tl1_err_data[0]); for (j = 0; j < CH_ERR_TL1_TLMAX; j++, cl1p++) { if (cl1p->ch_err_tl1_flags == 0) continue; cpu_check_cpu_logout(i, (caddr_t)cl1p->ch_err_tl1_tpc, 1, ECC_F_TRAP, &cl1p->ch_err_tl1_logout); } /* * Check each of the remaining logout areas */ cpu_check_cpu_logout(i, NULL, 0, ECC_F_TRAP, CPU_PRIVATE_PTR(cp, chpr_fecctl0_logout)); cpu_check_cpu_logout(i, NULL, 0, ECC_C_TRAP, CPU_PRIVATE_PTR(cp, chpr_cecc_logout)); cpu_check_cpu_logout(i, NULL, 0, ECC_D_TRAP, CPU_PRIVATE_PTR(cp, chpr_async_logout)); } } /* * The fast_ecc_err handler transfers control here for UCU, UCC events. * Note that we flush Ecache twice, once in the fast_ecc_err handler to * flush the error that caused the UCU/UCC, then again here at the end to * flush the TL=1 trap handler code out of the Ecache, so we can minimize * the probability of getting a TL>1 Fast ECC trap when we're fielding * another Fast ECC trap. * * Cheetah+ also handles: TSCE: No additional processing required. * Panther adds L3_UCU and L3_UCC which are reported in AFSR_EXT. * * Note that the p_clo_flags input is only valid in cases where the * cpu_private struct is not yet initialized (since that is the only * time that information cannot be obtained from the logout struct.) */ /*ARGSUSED*/ void cpu_fast_ecc_error(struct regs *rp, ulong_t p_clo_flags) { ch_cpu_logout_t *clop; uint64_t ceen, nceen; /* * Get the CPU log out info. If we can't find our CPU private * pointer, then we will have to make due without any detailed * logout information. */ if (CPU_PRIVATE(CPU) == NULL) { clop = NULL; ceen = p_clo_flags & EN_REG_CEEN; nceen = p_clo_flags & EN_REG_NCEEN; } else { clop = CPU_PRIVATE_PTR(CPU, chpr_fecctl0_logout); ceen = clop->clo_flags & EN_REG_CEEN; nceen = clop->clo_flags & EN_REG_NCEEN; } cpu_log_fast_ecc_error((caddr_t)rp->r_pc, (rp->r_tstate & TSTATE_PRIV) ? 1 : 0, 0, ceen, nceen, clop); } /* * Log fast ecc error, called from either Fast ECC at TL=0 or Fast * ECC at TL>0. Need to supply either a error register pointer or a * cpu logout structure pointer. */ static void cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, uint64_t nceen, ch_cpu_logout_t *clop) { struct async_flt *aflt; ch_async_flt_t ch_flt; uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; char pr_reason[MAX_REASON_STRING]; ch_cpu_errors_t cpu_error_regs; bzero(&ch_flt, sizeof (ch_async_flt_t)); /* * If no cpu logout data, then we will have to make due without * any detailed logout information. */ if (clop == NULL) { ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; get_cpu_error_state(&cpu_error_regs); set_cpu_error_state(&cpu_error_regs); t_afar = cpu_error_regs.afar; t_afsr = cpu_error_regs.afsr; t_afsr_ext = cpu_error_regs.afsr_ext; #if defined(SERRANO) ch_flt.afar2 = cpu_error_regs.afar2; #endif /* SERRANO */ } else { t_afar = clop->clo_data.chd_afar; t_afsr = clop->clo_data.chd_afsr; t_afsr_ext = clop->clo_data.chd_afsr_ext; #if defined(SERRANO) ch_flt.afar2 = clop->clo_data.chd_afar2; #endif /* SERRANO */ } /* * In order to simplify code, we maintain this afsr_errs * variable which holds the aggregate of AFSR and AFSR_EXT * sticky bits. */ t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | (t_afsr & C_AFSR_ALL_ERRS); pr_reason[0] = '\0'; /* Setup the async fault structure */ aflt = (struct async_flt *)&ch_flt; aflt->flt_id = gethrtime_waitfree(); ch_flt.afsr_ext = t_afsr_ext; ch_flt.afsr_errs = t_afsr_errs; aflt->flt_stat = t_afsr; aflt->flt_addr = t_afar; aflt->flt_bus_id = getprocessorid(); aflt->flt_inst = CPU->cpu_id; aflt->flt_pc = tpc; aflt->flt_prot = AFLT_PROT_NONE; aflt->flt_class = CPU_FAULT; aflt->flt_priv = priv; aflt->flt_tl = tl; aflt->flt_status = ECC_F_TRAP; aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); /* * XXXX - Phenomenal hack to get around Solaris not getting all the * cmn_err messages out to the console. The situation is a UCU (in * priv mode) which causes a WDU which causes a UE (on the retry). * The messages for the UCU and WDU are enqueued and then pulled off * the async queue via softint and syslogd starts to process them * but doesn't get them to the console. The UE causes a panic, but * since the UCU/WDU messages are already in transit, those aren't * on the async queue. The hack is to check if we have a matching * WDU event for the UCU, and if it matches, we're more than likely * going to panic with a UE, unless we're under protection. So, we * check to see if we got a matching WDU event and if we're under * protection. * * For Cheetah/Cheetah+/Jaguar/Jalapeno, the sequence we care about * looks like this: * UCU->WDU->UE * For Panther, it could look like either of these: * UCU---->WDU->L3_WDU->UE * L3_UCU->WDU->L3_WDU->UE */ if ((t_afsr_errs & (C_AFSR_UCU | C_AFSR_L3_UCU)) && aflt->flt_panic == 0 && aflt->flt_priv != 0 && curthread->t_ontrap == NULL && curthread->t_lofault == (uintptr_t)NULL) { get_cpu_error_state(&cpu_error_regs); if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { aflt->flt_panic |= ((cpu_error_regs.afsr & C_AFSR_WDU) && (cpu_error_regs.afsr_ext & C_AFSR_L3_WDU) && (cpu_error_regs.afar == t_afar)); aflt->flt_panic |= ((clop == NULL) && (t_afsr_errs & C_AFSR_WDU) && (t_afsr_errs & C_AFSR_L3_WDU)); } else { aflt->flt_panic |= ((cpu_error_regs.afsr & C_AFSR_WDU) && (cpu_error_regs.afar == t_afar)); aflt->flt_panic |= ((clop == NULL) && (t_afsr_errs & C_AFSR_WDU)); } } /* * Queue events on the async event queue, one event per error bit. * If no events are queued or no Fast ECC events are on in the AFSR, * queue an event to complain. */ if (cpu_queue_events(&ch_flt, pr_reason, t_afsr_errs, clop) == 0 || ((t_afsr_errs & (C_AFSR_FECC_ERRS | C_AFSR_EXT_FECC_ERRS)) == 0)) { ch_flt.flt_type = CPU_INV_AFSR; cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); } /* * Zero out + invalidate CPU logout. */ if (clop) { bzero(clop, sizeof (ch_cpu_logout_t)); clop->clo_data.chd_afar = LOGOUT_INVALID; } /* * We carefully re-enable NCEEN and CEEN and then check if any deferred * or disrupting errors have happened. We do this because if a * deferred or disrupting error had occurred with NCEEN/CEEN off, the * trap will not be taken when NCEEN/CEEN is re-enabled. Note that * CEEN works differently on Cheetah than on Spitfire. Also, we enable * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a * deferred or disrupting error happening between checking the AFSR and * enabling NCEEN/CEEN. * * Note: CEEN and NCEEN are only reenabled if they were on when trap * taken. */ set_error_enable(get_error_enable() | (nceen | ceen)); if (clear_errors(&ch_flt)) { aflt->flt_panic |= ((ch_flt.afsr_errs & (C_AFSR_EXT_ASYNC_ERRS | C_AFSR_ASYNC_ERRS)) != 0); (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, NULL); } /* * Panic here if aflt->flt_panic has been set. Enqueued errors will * be logged as part of the panic flow. */ if (aflt->flt_panic) fm_panic("%sError(s)", pr_reason); /* * Flushing the Ecache here gets the part of the trap handler that * is run at TL=1 out of the Ecache. */ cpu_flush_ecache(); } /* * This is called via sys_trap from pil15_interrupt code if the * corresponding entry in ch_err_tl1_pending is set. Checks the * various ch_err_tl1_data structures for valid entries based on the bit * settings in the ch_err_tl1_flags entry of the structure. */ /*ARGSUSED*/ void cpu_tl1_error(struct regs *rp, int panic) { ch_err_tl1_data_t *cl1p, cl1; int i, ncl1ps; uint64_t me_flags; uint64_t ceen, nceen; if (ch_err_tl1_paddrs[CPU->cpu_id] == 0) { cl1p = &ch_err_tl1_data; ncl1ps = 1; } else if (CPU_PRIVATE(CPU) != NULL) { cl1p = CPU_PRIVATE_PTR(CPU, chpr_tl1_err_data[0]); ncl1ps = CH_ERR_TL1_TLMAX; } else { ncl1ps = 0; } for (i = 0; i < ncl1ps; i++, cl1p++) { if (cl1p->ch_err_tl1_flags == 0) continue; /* * Grab a copy of the logout data and invalidate * the logout area. */ cl1 = *cl1p; bzero(cl1p, sizeof (ch_err_tl1_data_t)); cl1p->ch_err_tl1_logout.clo_data.chd_afar = LOGOUT_INVALID; me_flags = CH_ERR_ME_FLAGS(cl1.ch_err_tl1_flags); /* * Log "first error" in ch_err_tl1_data. */ if (cl1.ch_err_tl1_flags & CH_ERR_FECC) { ceen = get_error_enable() & EN_REG_CEEN; nceen = get_error_enable() & EN_REG_NCEEN; cpu_log_fast_ecc_error((caddr_t)cl1.ch_err_tl1_tpc, 1, 1, ceen, nceen, &cl1.ch_err_tl1_logout); } #if defined(CPU_IMP_L1_CACHE_PARITY) if (cl1.ch_err_tl1_flags & (CH_ERR_IPE | CH_ERR_DPE)) { cpu_parity_error(rp, cl1.ch_err_tl1_flags, (caddr_t)cl1.ch_err_tl1_tpc); } #endif /* CPU_IMP_L1_CACHE_PARITY */ /* * Log "multiple events" in ch_err_tl1_data. Note that * we don't read and clear the AFSR/AFAR in the TL>0 code * if the structure is busy, we just do the cache flushing * we have to do and then do the retry. So the AFSR/AFAR * at this point *should* have some relevant info. If there * are no valid errors in the AFSR, we'll assume they've * already been picked up and logged. For I$/D$ parity, * we just log an event with an "Unknown" (NULL) TPC. */ if (me_flags & CH_ERR_FECC) { ch_cpu_errors_t cpu_error_regs; uint64_t t_afsr_errs; /* * Get the error registers and see if there's * a pending error. If not, don't bother * generating an "Invalid AFSR" error event. */ get_cpu_error_state(&cpu_error_regs); t_afsr_errs = (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS) | (cpu_error_regs.afsr & C_AFSR_ALL_ERRS); if (t_afsr_errs != 0) { ceen = get_error_enable() & EN_REG_CEEN; nceen = get_error_enable() & EN_REG_NCEEN; cpu_log_fast_ecc_error((caddr_t)NULL, 1, 1, ceen, nceen, NULL); } } #if defined(CPU_IMP_L1_CACHE_PARITY) if (me_flags & (CH_ERR_IPE | CH_ERR_DPE)) { cpu_parity_error(rp, me_flags, (caddr_t)NULL); } #endif /* CPU_IMP_L1_CACHE_PARITY */ } } /* * Called from Fast ECC TL>0 handler in case of fatal error. * cpu_tl1_error should always find an associated ch_err_tl1_data structure, * but if we don't, we'll panic with something reasonable. */ /*ARGSUSED*/ void cpu_tl1_err_panic(struct regs *rp, ulong_t flags) { cpu_tl1_error(rp, 1); /* * Should never return, but just in case. */ fm_panic("Unsurvivable ECC Error at TL>0"); } /* * The ce_err/ce_err_tl1 handlers transfer control here for CE, EMC, EDU:ST, * EDC, WDU, WDC, CPU, CPC, IVU, IVC events. * Disrupting errors controlled by NCEEN: EDU:ST, WDU, CPU, IVU * Disrupting errors controlled by CEEN: CE, EMC, EDC, WDC, CPC, IVC * * Cheetah+ also handles (No additional processing required): * DUE, DTO, DBERR (NCEEN controlled) * THCE (CEEN and ET_ECC_en controlled) * TUE (ET_ECC_en controlled) * * Panther further adds: * IMU, L3_EDU, L3_WDU, L3_CPU (NCEEN controlled) * IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE (CEEN controlled) * TUE_SH, TUE (NCEEN and L2_tag_ECC_en controlled) * L3_TUE, L3_TUE_SH (NCEEN and ET_ECC_en controlled) * THCE (CEEN and L2_tag_ECC_en controlled) * L3_THCE (CEEN and ET_ECC_en controlled) * * Note that the p_clo_flags input is only valid in cases where the * cpu_private struct is not yet initialized (since that is the only * time that information cannot be obtained from the logout struct.) */ /*ARGSUSED*/ void cpu_disrupting_error(struct regs *rp, ulong_t p_clo_flags) { struct async_flt *aflt; ch_async_flt_t ch_flt; char pr_reason[MAX_REASON_STRING]; ch_cpu_logout_t *clop; uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; ch_cpu_errors_t cpu_error_regs; bzero(&ch_flt, sizeof (ch_async_flt_t)); /* * Get the CPU log out info. If we can't find our CPU private * pointer, then we will have to make due without any detailed * logout information. */ if (CPU_PRIVATE(CPU) == NULL) { clop = NULL; ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; get_cpu_error_state(&cpu_error_regs); set_cpu_error_state(&cpu_error_regs); t_afar = cpu_error_regs.afar; t_afsr = cpu_error_regs.afsr; t_afsr_ext = cpu_error_regs.afsr_ext; #if defined(SERRANO) ch_flt.afar2 = cpu_error_regs.afar2; #endif /* SERRANO */ } else { clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); t_afar = clop->clo_data.chd_afar; t_afsr = clop->clo_data.chd_afsr; t_afsr_ext = clop->clo_data.chd_afsr_ext; #if defined(SERRANO) ch_flt.afar2 = clop->clo_data.chd_afar2; #endif /* SERRANO */ } /* * In order to simplify code, we maintain this afsr_errs * variable which holds the aggregate of AFSR and AFSR_EXT * sticky bits. */ t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | (t_afsr & C_AFSR_ALL_ERRS); pr_reason[0] = '\0'; /* Setup the async fault structure */ aflt = (struct async_flt *)&ch_flt; ch_flt.afsr_ext = t_afsr_ext; ch_flt.afsr_errs = t_afsr_errs; aflt->flt_stat = t_afsr; aflt->flt_addr = t_afar; aflt->flt_pc = (caddr_t)rp->r_pc; aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; aflt->flt_tl = 0; aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); /* * If this trap is a result of one of the errors not masked * by cpu_ce_not_deferred, we don't reenable CEEN. Instead * indicate that a timeout is to be set later. */ if (!(t_afsr_errs & (cpu_ce_not_deferred | cpu_ce_not_deferred_ext)) && !aflt->flt_panic) ch_flt.flt_trapped_ce = CE_CEEN_DEFER | CE_CEEN_TRAPPED; else ch_flt.flt_trapped_ce = CE_CEEN_NODEFER | CE_CEEN_TRAPPED; /* * log the CE and clean up */ cpu_log_and_clear_ce(&ch_flt); /* * We re-enable CEEN (if required) and check if any disrupting errors * have happened. We do this because if a disrupting error had occurred * with CEEN off, the trap will not be taken when CEEN is re-enabled. * Note that CEEN works differently on Cheetah than on Spitfire. Also, * we enable CEEN *before* checking the AFSR to avoid the small window * of a error happening between checking the AFSR and enabling CEEN. */ if (ch_flt.flt_trapped_ce & CE_CEEN_NODEFER) set_error_enable(get_error_enable() | EN_REG_CEEN); if (clear_errors(&ch_flt)) { (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, NULL); } /* * Panic here if aflt->flt_panic has been set. Enqueued errors will * be logged as part of the panic flow. */ if (aflt->flt_panic) fm_panic("%sError(s)", pr_reason); } /* * The async_err handler transfers control here for UE, EMU, EDU:BLD, * L3_EDU:BLD, TO, and BERR events. * Deferred errors controlled by NCEEN: UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR * * Cheetah+: No additional errors handled. * * Note that the p_clo_flags input is only valid in cases where the * cpu_private struct is not yet initialized (since that is the only * time that information cannot be obtained from the logout struct.) */ /*ARGSUSED*/ void cpu_deferred_error(struct regs *rp, ulong_t p_clo_flags) { ushort_t ttype, tl; ch_async_flt_t ch_flt; struct async_flt *aflt; int trampolined = 0; char pr_reason[MAX_REASON_STRING]; ch_cpu_logout_t *clop; uint64_t ceen, clo_flags; uint64_t log_afsr; uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; ch_cpu_errors_t cpu_error_regs; int expected = DDI_FM_ERR_UNEXPECTED; ddi_acc_hdl_t *hp; /* * We need to look at p_flag to determine if the thread detected an * error while dumping core. We can't grab p_lock here, but it's ok * because we just need a consistent snapshot and we know that everyone * else will store a consistent set of bits while holding p_lock. We * don't have to worry about a race because SDOCORE is set once prior * to doing i/o from the process's address space and is never cleared. */ uint_t pflag = ttoproc(curthread)->p_flag; bzero(&ch_flt, sizeof (ch_async_flt_t)); /* * Get the CPU log out info. If we can't find our CPU private * pointer then we will have to make due without any detailed * logout information. */ if (CPU_PRIVATE(CPU) == NULL) { clop = NULL; ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; get_cpu_error_state(&cpu_error_regs); set_cpu_error_state(&cpu_error_regs); t_afar = cpu_error_regs.afar; t_afsr = cpu_error_regs.afsr; t_afsr_ext = cpu_error_regs.afsr_ext; #if defined(SERRANO) ch_flt.afar2 = cpu_error_regs.afar2; #endif /* SERRANO */ clo_flags = p_clo_flags; } else { clop = CPU_PRIVATE_PTR(CPU, chpr_async_logout); t_afar = clop->clo_data.chd_afar; t_afsr = clop->clo_data.chd_afsr; t_afsr_ext = clop->clo_data.chd_afsr_ext; #if defined(SERRANO) ch_flt.afar2 = clop->clo_data.chd_afar2; #endif /* SERRANO */ clo_flags = clop->clo_flags; } /* * In order to simplify code, we maintain this afsr_errs * variable which holds the aggregate of AFSR and AFSR_EXT * sticky bits. */ t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | (t_afsr & C_AFSR_ALL_ERRS); pr_reason[0] = '\0'; /* * Grab information encoded into our clo_flags field. */ ceen = clo_flags & EN_REG_CEEN; tl = (clo_flags & CLO_FLAGS_TL_MASK) >> CLO_FLAGS_TL_SHIFT; ttype = (clo_flags & CLO_FLAGS_TT_MASK) >> CLO_FLAGS_TT_SHIFT; /* * handle the specific error */ aflt = (struct async_flt *)&ch_flt; aflt->flt_id = gethrtime_waitfree(); aflt->flt_bus_id = getprocessorid(); aflt->flt_inst = CPU->cpu_id; ch_flt.afsr_ext = t_afsr_ext; ch_flt.afsr_errs = t_afsr_errs; aflt->flt_stat = t_afsr; aflt->flt_addr = t_afar; aflt->flt_pc = (caddr_t)rp->r_pc; aflt->flt_prot = AFLT_PROT_NONE; aflt->flt_class = CPU_FAULT; aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; aflt->flt_tl = (uchar_t)tl; aflt->flt_panic = ((tl != 0) || (aft_testfatal != 0) || C_AFSR_PANIC(t_afsr_errs)); aflt->flt_core = (pflag & SDOCORE) ? 1 : 0; aflt->flt_status = ((ttype == T_DATA_ERROR) ? ECC_D_TRAP : ECC_I_TRAP); /* * If the trap occurred in privileged mode at TL=0, we need to check to * see if we were executing in the kernel under on_trap() or t_lofault * protection. If so, modify the saved registers so that we return * from the trap to the appropriate trampoline routine. */ if (aflt->flt_priv && tl == 0) { if (curthread->t_ontrap != NULL) { on_trap_data_t *otp = curthread->t_ontrap; if (otp->ot_prot & OT_DATA_EC) { aflt->flt_prot = AFLT_PROT_EC; otp->ot_trap |= OT_DATA_EC; rp->r_pc = otp->ot_trampoline; rp->r_npc = rp->r_pc + 4; trampolined = 1; } if ((t_afsr & (C_AFSR_TO | C_AFSR_BERR)) && (otp->ot_prot & OT_DATA_ACCESS)) { aflt->flt_prot = AFLT_PROT_ACCESS; otp->ot_trap |= OT_DATA_ACCESS; rp->r_pc = otp->ot_trampoline; rp->r_npc = rp->r_pc + 4; trampolined = 1; /* * for peeks and caut_gets errors are expected */ hp = (ddi_acc_hdl_t *)otp->ot_handle; if (!hp) expected = DDI_FM_ERR_PEEK; else if (hp->ah_acc.devacc_attr_access == DDI_CAUTIOUS_ACC) expected = DDI_FM_ERR_EXPECTED; } } else if (curthread->t_lofault) { aflt->flt_prot = AFLT_PROT_COPY; rp->r_g1 = EFAULT; rp->r_pc = curthread->t_lofault; rp->r_npc = rp->r_pc + 4; trampolined = 1; } } /* * If we're in user mode or we're doing a protected copy, we either * want the ASTON code below to send a signal to the user process * or we want to panic if aft_panic is set. * * If we're in privileged mode and we're not doing a copy, then we * need to check if we've trampolined. If we haven't trampolined, * we should panic. */ if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { if (t_afsr_errs & ((C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS) & ~(C_AFSR_BERR | C_AFSR_TO))) aflt->flt_panic |= aft_panic; } else if (!trampolined) { aflt->flt_panic = 1; } /* * If we've trampolined due to a privileged TO or BERR, or if an * unprivileged TO or BERR occurred, we don't want to enqueue an * event for that TO or BERR. Queue all other events (if any) besides * the TO/BERR. Since we may not be enqueing any events, we need to * ignore the number of events queued. If we haven't trampolined due * to a TO or BERR, just enqueue events normally. */ log_afsr = t_afsr_errs; if (trampolined) { log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); } else if (!aflt->flt_priv) { /* * User mode, suppress messages if * cpu_berr_to_verbose is not set. */ if (!cpu_berr_to_verbose) log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); } /* * Log any errors that occurred */ if (((log_afsr & ((C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS) & ~C_AFSR_ME)) && cpu_queue_events(&ch_flt, pr_reason, log_afsr, clop) == 0) || (t_afsr_errs & (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) == 0) { ch_flt.flt_type = CPU_INV_AFSR; cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); } /* * Zero out + invalidate CPU logout. */ if (clop) { bzero(clop, sizeof (ch_cpu_logout_t)); clop->clo_data.chd_afar = LOGOUT_INVALID; } #if defined(JALAPENO) || defined(SERRANO) /* * UE/RUE/BERR/TO: Call our bus nexus friends to check for * IO errors that may have resulted in this trap. */ if (t_afsr & (C_AFSR_UE|C_AFSR_RUE|C_AFSR_TO|C_AFSR_BERR)) { cpu_run_bus_error_handlers(aflt, expected); } /* * UE/RUE: If UE or RUE is in memory, we need to flush the bad * line from the Ecache. We also need to query the bus nexus for * fatal errors. Attempts to do diagnostic read on caches may * introduce more errors (especially when the module is bad). */ if (t_afsr & (C_AFSR_UE|C_AFSR_RUE)) { /* * Ask our bus nexus friends if they have any fatal errors. If * so, they will log appropriate error messages. */ if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) aflt->flt_panic = 1; /* * We got a UE or RUE and are panicking, save the fault PA in * a known location so that the platform specific panic code * can check for copyback errors. */ if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { panic_aflt = *aflt; } } /* * Flush Ecache line or entire Ecache */ if (t_afsr & (C_AFSR_UE | C_AFSR_RUE | C_AFSR_EDU | C_AFSR_BERR)) cpu_error_ecache_flush(&ch_flt); #else /* JALAPENO || SERRANO */ /* * UE/BERR/TO: Call our bus nexus friends to check for * IO errors that may have resulted in this trap. */ if (t_afsr & (C_AFSR_UE|C_AFSR_TO|C_AFSR_BERR)) { cpu_run_bus_error_handlers(aflt, expected); } /* * UE: If the UE is in memory, we need to flush the bad * line from the Ecache. We also need to query the bus nexus for * fatal errors. Attempts to do diagnostic read on caches may * introduce more errors (especially when the module is bad). */ if (t_afsr & C_AFSR_UE) { /* * Ask our legacy bus nexus friends if they have any fatal * errors. If so, they will log appropriate error messages. */ if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) aflt->flt_panic = 1; /* * We got a UE and are panicking, save the fault PA in a known * location so that the platform specific panic code can check * for copyback errors. */ if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { panic_aflt = *aflt; } } /* * Flush Ecache line or entire Ecache */ if (t_afsr_errs & (C_AFSR_UE | C_AFSR_EDU | C_AFSR_BERR | C_AFSR_L3_EDU)) cpu_error_ecache_flush(&ch_flt); #endif /* JALAPENO || SERRANO */ /* * We carefully re-enable NCEEN and CEEN and then check if any deferred * or disrupting errors have happened. We do this because if a * deferred or disrupting error had occurred with NCEEN/CEEN off, the * trap will not be taken when NCEEN/CEEN is re-enabled. Note that * CEEN works differently on Cheetah than on Spitfire. Also, we enable * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a * deferred or disrupting error happening between checking the AFSR and * enabling NCEEN/CEEN. * * Note: CEEN reenabled only if it was on when trap taken. */ set_error_enable(get_error_enable() | (EN_REG_NCEEN | ceen)); if (clear_errors(&ch_flt)) { /* * Check for secondary errors, and avoid panicking if we * have them */ if (cpu_check_secondary_errors(&ch_flt, t_afsr_errs, t_afar) == 0) { aflt->flt_panic |= ((ch_flt.afsr_errs & (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) != 0); } (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, NULL); } /* * Panic here if aflt->flt_panic has been set. Enqueued errors will * be logged as part of the panic flow. */ if (aflt->flt_panic) fm_panic("%sError(s)", pr_reason); /* * If we queued an error and we are going to return from the trap and * the error was in user mode or inside of a copy routine, set AST flag * so the queue will be drained before returning to user mode. The * AST processing will also act on our failure policy. */ if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { int pcb_flag = 0; if (t_afsr_errs & (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS & ~(C_AFSR_BERR | C_AFSR_TO))) pcb_flag |= ASYNC_HWERR; if (t_afsr & C_AFSR_BERR) pcb_flag |= ASYNC_BERR; if (t_afsr & C_AFSR_TO) pcb_flag |= ASYNC_BTO; ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag; aston(curthread); } } #if defined(CPU_IMP_L1_CACHE_PARITY) /* * Handling of data and instruction parity errors (traps 0x71, 0x72). * * For Panther, P$ data parity errors during floating point load hits * are also detected (reported as TT 0x71) and handled by this trap * handler. * * AFSR/AFAR are not set for parity errors, only TPC (a virtual address) * is available. */ /*ARGSUSED*/ void cpu_parity_error(struct regs *rp, uint_t flags, caddr_t tpc) { ch_async_flt_t ch_flt; struct async_flt *aflt; uchar_t tl = ((flags & CH_ERR_TL) != 0); uchar_t iparity = ((flags & CH_ERR_IPE) != 0); uchar_t panic = ((flags & CH_ERR_PANIC) != 0); char *error_class; int index, way, word; ch_dc_data_t tmp_dcp; int dc_set_size = dcache_size / CH_DCACHE_NWAY; uint64_t parity_bits, pbits; /* The parity bit array corresponds to the result of summing two bits */ static int parity_bits_popc[] = { 0, 1, 1, 0 }; /* * Log the error. * For icache parity errors the fault address is the trap PC. * For dcache/pcache parity errors the instruction would have to * be decoded to determine the address and that isn't possible * at high PIL. */ bzero(&ch_flt, sizeof (ch_async_flt_t)); aflt = (struct async_flt *)&ch_flt; aflt->flt_id = gethrtime_waitfree(); aflt->flt_bus_id = getprocessorid(); aflt->flt_inst = CPU->cpu_id; aflt->flt_pc = tpc; aflt->flt_addr = iparity ? (uint64_t)tpc : AFLT_INV_ADDR; aflt->flt_prot = AFLT_PROT_NONE; aflt->flt_class = CPU_FAULT; aflt->flt_priv = (tl || (rp->r_tstate & TSTATE_PRIV)) ? 1 : 0; aflt->flt_tl = tl; aflt->flt_panic = panic; aflt->flt_status = iparity ? ECC_IP_TRAP : ECC_DP_TRAP; ch_flt.flt_type = iparity ? CPU_IC_PARITY : CPU_DC_PARITY; if (iparity) { cpu_icache_parity_info(&ch_flt); if (ch_flt.parity_data.ipe.cpl_off != -1) error_class = FM_EREPORT_CPU_USIII_IDSPE; else if (ch_flt.parity_data.ipe.cpl_way != -1) error_class = FM_EREPORT_CPU_USIII_ITSPE; else error_class = FM_EREPORT_CPU_USIII_IPE; aflt->flt_payload = FM_EREPORT_PAYLOAD_ICACHE_PE; } else { cpu_dcache_parity_info(&ch_flt); if (ch_flt.parity_data.dpe.cpl_off != -1) { /* * If not at TL 0 and running on a Jalapeno processor, * then process as a true ddspe. A true * ddspe error can only occur if the way == 0 */ way = ch_flt.parity_data.dpe.cpl_way; if ((tl == 0) && (way != 0) && IS_JALAPENO(cpunodes[CPU->cpu_id].implementation)) { for (index = 0; index < dc_set_size; index += dcache_linesize) { get_dcache_dtag(index + way * dc_set_size, (uint64_t *)&tmp_dcp); /* * Check data array for even parity. * The 8 parity bits are grouped into * 4 pairs each of which covers a 64-bit * word. The endianness is reversed * -- the low-order parity bits cover * the high-order data words. */ parity_bits = tmp_dcp.dc_utag >> 8; for (word = 0; word < 4; word++) { pbits = (parity_bits >> (6 - word * 2)) & 3; if (((popc64( tmp_dcp.dc_data[word]) + parity_bits_popc[pbits]) & 1) && (tmp_dcp.dc_tag & VA13)) { /* cleanup */ correct_dcache_parity( dcache_size, dcache_linesize); if (cache_boot_state & DCU_DC) { flush_dcache(); } set_dcu(get_dcu() | cache_boot_state); return; } } } } /* (tl == 0) && (way != 0) && IS JALAPENO */ error_class = FM_EREPORT_CPU_USIII_DDSPE; } else if (ch_flt.parity_data.dpe.cpl_way != -1) error_class = FM_EREPORT_CPU_USIII_DTSPE; else error_class = FM_EREPORT_CPU_USIII_DPE; aflt->flt_payload = FM_EREPORT_PAYLOAD_DCACHE_PE; /* * For panther we also need to check the P$ for parity errors. */ if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { cpu_pcache_parity_info(&ch_flt); if (ch_flt.parity_data.dpe.cpl_cache == CPU_PC_PARITY) { error_class = FM_EREPORT_CPU_USIII_PDSPE; aflt->flt_payload = FM_EREPORT_PAYLOAD_PCACHE_PE; } } } cpu_errorq_dispatch(error_class, (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); if (iparity) { /* * Invalidate entire I$. * This is required due to the use of diagnostic ASI * accesses that may result in a loss of I$ coherency. */ if (cache_boot_state & DCU_IC) { flush_icache(); } /* * According to section P.3.1 of the Panther PRM, we * need to do a little more for recovery on those * CPUs after encountering an I$ parity error. */ if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { flush_ipb(); correct_dcache_parity(dcache_size, dcache_linesize); flush_pcache(); } } else { /* * Since the valid bit is ignored when checking parity the * D$ data and tag must also be corrected. Set D$ data bits * to zero and set utag to 0, 1, 2, 3. */ correct_dcache_parity(dcache_size, dcache_linesize); /* * According to section P.3.3 of the Panther PRM, we * need to do a little more for recovery on those * CPUs after encountering a D$ or P$ parity error. * * As far as clearing P$ parity errors, it is enough to * simply invalidate all entries in the P$ since P$ parity * error traps are only generated for floating point load * hits. */ if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { flush_icache(); flush_ipb(); flush_pcache(); } } /* * Invalidate entire D$ if it was enabled. * This is done to avoid stale data in the D$ which might * occur with the D$ disabled and the trap handler doing * stores affecting lines already in the D$. */ if (cache_boot_state & DCU_DC) { flush_dcache(); } /* * Restore caches to their bootup state. */ set_dcu(get_dcu() | cache_boot_state); /* * Panic here if aflt->flt_panic has been set. Enqueued errors will * be logged as part of the panic flow. */ if (aflt->flt_panic) fm_panic("%sError(s)", iparity ? "IPE " : "DPE "); /* * If this error occurred at TL>0 then flush the E$ here to reduce * the chance of getting an unrecoverable Fast ECC error. This * flush will evict the part of the parity trap handler that is run * at TL>1. */ if (tl) { cpu_flush_ecache(); } } /* * On an I$ parity error, mark the appropriate entries in the ch_async_flt_t * to indicate which portions of the captured data should be in the ereport. */ void cpu_async_log_ic_parity_err(ch_async_flt_t *ch_flt) { int way = ch_flt->parity_data.ipe.cpl_way; int offset = ch_flt->parity_data.ipe.cpl_off; int tag_index; struct async_flt *aflt = (struct async_flt *)ch_flt; if ((offset != -1) || (way != -1)) { /* * Parity error in I$ tag or data */ tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) ch_flt->parity_data.ipe.cpl_ic[way].ic_way = PN_ICIDX_TO_WAY(tag_index); else ch_flt->parity_data.ipe.cpl_ic[way].ic_way = CH_ICIDX_TO_WAY(tag_index); ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = IC_LOGFLAG_MAGIC; } else { /* * Parity error was not identified. * Log tags and data for all ways. */ for (way = 0; way < CH_ICACHE_NWAY; way++) { tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) ch_flt->parity_data.ipe.cpl_ic[way].ic_way = PN_ICIDX_TO_WAY(tag_index); else ch_flt->parity_data.ipe.cpl_ic[way].ic_way = CH_ICIDX_TO_WAY(tag_index); ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = IC_LOGFLAG_MAGIC; } } } /* * On an D$ parity error, mark the appropriate entries in the ch_async_flt_t * to indicate which portions of the captured data should be in the ereport. */ void cpu_async_log_dc_parity_err(ch_async_flt_t *ch_flt) { int way = ch_flt->parity_data.dpe.cpl_way; int offset = ch_flt->parity_data.dpe.cpl_off; int tag_index; if (offset != -1) { /* * Parity error in D$ or P$ data array. * * First check to see whether the parity error is in D$ or P$ * since P$ data parity errors are reported in Panther using * the same trap. */ if (ch_flt->parity_data.dpe.cpl_cache == CPU_PC_PARITY) { tag_index = ch_flt->parity_data.dpe.cpl_pc[way].pc_idx; ch_flt->parity_data.dpe.cpl_pc[way].pc_way = CH_PCIDX_TO_WAY(tag_index); ch_flt->parity_data.dpe.cpl_pc[way].pc_logflag = PC_LOGFLAG_MAGIC; } else { tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; ch_flt->parity_data.dpe.cpl_dc[way].dc_way = CH_DCIDX_TO_WAY(tag_index); ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = DC_LOGFLAG_MAGIC; } } else if (way != -1) { /* * Parity error in D$ tag. */ tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; ch_flt->parity_data.dpe.cpl_dc[way].dc_way = CH_DCIDX_TO_WAY(tag_index); ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = DC_LOGFLAG_MAGIC; } } #endif /* CPU_IMP_L1_CACHE_PARITY */ /* * The cpu_async_log_err() function is called via the [uc]e_drain() function to * post-process CPU events that are dequeued. As such, it can be invoked * from softint context, from AST processing in the trap() flow, or from the * panic flow. We decode the CPU-specific data, and take appropriate actions. * Historically this entry point was used to log the actual cmn_err(9F) text; * now with FMA it is used to prepare 'flt' to be converted into an ereport. * With FMA this function now also returns a flag which indicates to the * caller whether the ereport should be posted (1) or suppressed (0). */ static int cpu_async_log_err(void *flt, errorq_elem_t *eqep) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt; struct async_flt *aflt = (struct async_flt *)flt; uint64_t errors; extern void memscrub_induced_error(void); switch (ch_flt->flt_type) { case CPU_INV_AFSR: /* * If it is a disrupting trap and the AFSR is zero, then * the event has probably already been noted. Do not post * an ereport. */ if ((aflt->flt_status & ECC_C_TRAP) && (!(aflt->flt_stat & C_AFSR_MASK))) return (0); else return (1); case CPU_TO: case CPU_BERR: case CPU_FATAL: case CPU_FPUERR: return (1); case CPU_UE_ECACHE_RETIRE: cpu_log_err(aflt); cpu_page_retire(ch_flt); return (1); /* * Cases where we may want to suppress logging or perform * extended diagnostics. */ case CPU_CE: case CPU_EMC: /* * We want to skip logging and further classification * only if ALL the following conditions are true: * * 1. There is only one error * 2. That error is a correctable memory error * 3. The error is caused by the memory scrubber (in * which case the error will have occurred under * on_trap protection) * 4. The error is on a retired page * * Note: AFLT_PROT_EC is used places other than the memory * scrubber. However, none of those errors should occur * on a retired page. */ if ((ch_flt->afsr_errs & (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_CE && aflt->flt_prot == AFLT_PROT_EC) { if (page_retire_check(aflt->flt_addr, NULL) == 0) { if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { /* * Since we're skipping logging, we'll need * to schedule the re-enabling of CEEN */ (void) timeout(cpu_delayed_check_ce_errors, (void *)(uintptr_t)aflt->flt_inst, drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); } /* * Inform memscrubber - scrubbing induced * CE on a retired page. */ memscrub_induced_error(); return (0); } } /* * Perform/schedule further classification actions, but * only if the page is healthy (we don't want bad * pages inducing too much diagnostic activity). If we could * not find a page pointer then we also skip this. If * ce_scrub_xdiag_recirc returns nonzero then it has chosen * to copy and recirculate the event (for further diagnostics) * and we should not proceed to log it here. * * This must be the last step here before the cpu_log_err() * below - if an event recirculates cpu_ce_log_err() will * not call the current function but just proceed directly * to cpu_ereport_post after the cpu_log_err() avoided below. * * Note: Check cpu_impl_async_log_err if changing this */ if (page_retire_check(aflt->flt_addr, &errors) == EINVAL) { CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOPP); } else { if (errors != PR_OK) { CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_PAGEDET); } else if (ce_scrub_xdiag_recirc(aflt, ce_queue, eqep, offsetof(ch_async_flt_t, cmn_asyncflt))) { return (0); } } /*FALLTHRU*/ /* * Cases where we just want to report the error and continue. */ case CPU_CE_ECACHE: case CPU_UE_ECACHE: case CPU_IV: case CPU_ORPH: cpu_log_err(aflt); return (1); /* * Cases where we want to fall through to handle panicking. */ case CPU_UE: /* * We want to skip logging in the same conditions as the * CE case. In addition, we want to make sure we're not * panicking. */ if (!panicstr && (ch_flt->afsr_errs & (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_UE && aflt->flt_prot == AFLT_PROT_EC) { if (page_retire_check(aflt->flt_addr, NULL) == 0) { /* Zero the address to clear the error */ softcall(ecc_page_zero, (void *)aflt->flt_addr); /* * Inform memscrubber - scrubbing induced * UE on a retired page. */ memscrub_induced_error(); return (0); } } cpu_log_err(aflt); break; default: /* * If the us3_common.c code doesn't know the flt_type, it may * be an implementation-specific code. Call into the impldep * backend to find out what to do: if it tells us to continue, * break and handle as if falling through from a UE; if not, * the impldep backend has handled the error and we're done. */ switch (cpu_impl_async_log_err(flt, eqep)) { case CH_ASYNC_LOG_DONE: return (1); case CH_ASYNC_LOG_RECIRC: return (0); case CH_ASYNC_LOG_CONTINUE: break; /* continue on to handle UE-like error */ default: cmn_err(CE_WARN, "discarding error 0x%p with " "invalid fault type (0x%x)", (void *)aflt, ch_flt->flt_type); return (0); } } /* ... fall through from the UE case */ if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) { if (!panicstr) { cpu_page_retire(ch_flt); } else { /* * Clear UEs on panic so that we don't * get haunted by them during panic or * after reboot */ cpu_clearphys(aflt); (void) clear_errors(NULL); } } return (1); } /* * Retire the bad page that may contain the flushed error. */ void cpu_page_retire(ch_async_flt_t *ch_flt) { struct async_flt *aflt = (struct async_flt *)ch_flt; (void) page_retire(aflt->flt_addr, PR_UE); } /* * Return true if the error specified in the AFSR indicates * an E$ data error (L2$ for Cheetah/Cheetah+/Jaguar, L3$ * for Panther, none for Jalapeno/Serrano). */ /* ARGSUSED */ static int cpu_error_is_ecache_data(int cpuid, uint64_t t_afsr) { #if defined(JALAPENO) || defined(SERRANO) return (0); #elif defined(CHEETAH_PLUS) if (IS_PANTHER(cpunodes[cpuid].implementation)) return ((t_afsr & C_AFSR_EXT_L3_DATA_ERRS) != 0); return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0); #else /* CHEETAH_PLUS */ return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0); #endif } /* * The cpu_log_err() function is called by cpu_async_log_err() to perform the * generic event post-processing for correctable and uncorrectable memory, * E$, and MTag errors. Historically this entry point was used to log bits of * common cmn_err(9F) text; now with FMA it is used to prepare 'flt' to be * converted into an ereport. In addition, it transmits the error to any * platform-specific service-processor FRU logging routines, if available. */ void cpu_log_err(struct async_flt *aflt) { char unum[UNUM_NAMLEN]; int synd_status, synd_code, afar_status; ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; if (cpu_error_is_ecache_data(aflt->flt_inst, ch_flt->flt_bit)) aflt->flt_status |= ECC_ECACHE; else aflt->flt_status &= ~ECC_ECACHE; /* * Determine syndrome status. */ synd_status = afsr_to_synd_status(aflt->flt_inst, ch_flt->afsr_errs, ch_flt->flt_bit); /* * Determine afar status. */ if (pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)) afar_status = afsr_to_afar_status(ch_flt->afsr_errs, ch_flt->flt_bit); else afar_status = AFLT_STAT_INVALID; synd_code = synd_to_synd_code(synd_status, aflt->flt_synd, ch_flt->flt_bit); /* * If afar status is not invalid do a unum lookup. */ if (afar_status != AFLT_STAT_INVALID) { (void) cpu_get_mem_unum_synd(synd_code, aflt, unum); } else { unum[0] = '\0'; } /* * Do not send the fruid message (plat_ecc_error_data_t) * to the SC if it can handle the enhanced error information * (plat_ecc_error2_data_t) or when the tunable * ecc_log_fruid_enable is set to 0. */ if (&plat_ecc_capability_sc_get && plat_ecc_capability_sc_get(PLAT_ECC_ERROR_MESSAGE)) { if (&plat_log_fruid_error) plat_log_fruid_error(synd_code, aflt, unum, ch_flt->flt_bit); } if (aflt->flt_func != NULL) aflt->flt_func(aflt, unum); if (afar_status != AFLT_STAT_INVALID) cpu_log_diag_info(ch_flt); /* * If we have a CEEN error , we do not reenable CEEN until after * we exit the trap handler. Otherwise, another error may * occur causing the handler to be entered recursively. * We set a timeout to trigger in cpu_ceen_delay_secs seconds, * to try and ensure that the CPU makes progress in the face * of a CE storm. */ if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { (void) timeout(cpu_delayed_check_ce_errors, (void *)(uintptr_t)aflt->flt_inst, drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); } } /* * Invoked by error_init() early in startup and therefore before * startup_errorq() is called to drain any error Q - * * startup() * startup_end() * error_init() * cpu_error_init() * errorq_init() * errorq_drain() * start_other_cpus() * * The purpose of this routine is to create error-related taskqs. Taskqs * are used for this purpose because cpu_lock can't be grabbed from interrupt * context. */ void cpu_error_init(int items) { /* * Create taskq(s) to reenable CE */ ch_check_ce_tq = taskq_create("cheetah_check_ce", 1, minclsyspri, items, items, TASKQ_PREPOPULATE); } void cpu_ce_log_err(struct async_flt *aflt, errorq_elem_t *eqep) { char unum[UNUM_NAMLEN]; int len; switch (aflt->flt_class) { case CPU_FAULT: cpu_ereport_init(aflt); if (cpu_async_log_err(aflt, eqep)) cpu_ereport_post(aflt); break; case BUS_FAULT: if (aflt->flt_func != NULL) { (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, unum, UNUM_NAMLEN, &len); aflt->flt_func(aflt, unum); } break; case RECIRC_CPU_FAULT: aflt->flt_class = CPU_FAULT; cpu_log_err(aflt); cpu_ereport_post(aflt); break; case RECIRC_BUS_FAULT: ASSERT(aflt->flt_class != RECIRC_BUS_FAULT); /*FALLTHRU*/ default: cmn_err(CE_WARN, "discarding CE error 0x%p with invalid " "fault class (0x%x)", (void *)aflt, aflt->flt_class); return; } } /* * Scrub and classify a CE. This function must not modify the * fault structure passed to it but instead should return the classification * information. */ static uchar_t cpu_ce_scrub_mem_err_common(struct async_flt *ecc, boolean_t logout_tried) { uchar_t disp = CE_XDIAG_EXTALG; on_trap_data_t otd; uint64_t orig_err; ch_cpu_logout_t *clop; /* * Clear CEEN. CPU CE TL > 0 trap handling will already have done * this, but our other callers have not. Disable preemption to * avoid CPU migration so that we restore CEEN on the correct * cpu later. * * CEEN is cleared so that further CEs that our instruction and * data footprint induce do not cause use to either creep down * kernel stack to the point of overflow, or do so much CE * notification as to make little real forward progress. * * NCEEN must not be cleared. However it is possible that * our accesses to the flt_addr may provoke a bus error or timeout * if the offending address has just been unconfigured as part of * a DR action. So we must operate under on_trap protection. */ kpreempt_disable(); orig_err = get_error_enable(); if (orig_err & EN_REG_CEEN) set_error_enable(orig_err & ~EN_REG_CEEN); /* * Our classification algorithm includes the line state before * the scrub; we'd like this captured after the detection and * before the algorithm below - the earlier the better. * * If we've come from a cpu CE trap then this info already exists * in the cpu logout area. * * For a CE detected by memscrub for which there was no trap * (running with CEEN off) cpu_log_and_clear_ce has called * cpu_ce_delayed_ec_logout to capture some cache data, and * marked the fault structure as incomplete as a flag to later * logging code. * * If called directly from an IO detected CE there has been * no line data capture. In this case we logout to the cpu logout * area - that's appropriate since it's the cpu cache data we need * for classification. We thus borrow the cpu logout area for a * short time, and cpu_ce_delayed_ec_logout will mark it as busy in * this time (we will invalidate it again below). * * If called from the partner check xcall handler then this cpu * (the partner) has not necessarily experienced a CE at this * address. But we want to capture line state before its scrub * attempt since we use that in our classification. */ if (logout_tried == B_FALSE) { if (!cpu_ce_delayed_ec_logout(ecc->flt_addr)) disp |= CE_XDIAG_NOLOGOUT; } /* * Scrub memory, then check AFSR for errors. The AFAR we scrub may * no longer be valid (if DR'd since the initial event) so we * perform this scrub under on_trap protection. If this access is * ok then further accesses below will also be ok - DR cannot * proceed while this thread is active (preemption is disabled); * to be safe we'll nonetheless use on_trap again below. */ if (!on_trap(&otd, OT_DATA_ACCESS)) { cpu_scrubphys(ecc); } else { no_trap(); if (orig_err & EN_REG_CEEN) set_error_enable(orig_err); kpreempt_enable(); return (disp); } no_trap(); /* * Did the casx read of the scrub log a CE that matches the AFAR? * Note that it's quite possible that the read sourced the data from * another cpu. */ if (clear_ecc(ecc)) disp |= CE_XDIAG_CE1; /* * Read the data again. This time the read is very likely to * come from memory since the scrub induced a writeback to memory. */ if (!on_trap(&otd, OT_DATA_ACCESS)) { (void) lddphys(P2ALIGN(ecc->flt_addr, 8)); } else { no_trap(); if (orig_err & EN_REG_CEEN) set_error_enable(orig_err); kpreempt_enable(); return (disp); } no_trap(); /* Did that read induce a CE that matches the AFAR? */ if (clear_ecc(ecc)) disp |= CE_XDIAG_CE2; /* * Look at the logout information and record whether we found the * line in l2/l3 cache. For Panther we are interested in whether * we found it in either cache (it won't reside in both but * it is possible to read it that way given the moving target). */ clop = CPU_PRIVATE(CPU) ? CPU_PRIVATE_PTR(CPU, chpr_cecc_logout) : NULL; if (!(disp & CE_XDIAG_NOLOGOUT) && clop && clop->clo_data.chd_afar != LOGOUT_INVALID) { int hit, level; int state; int totalsize; ch_ec_data_t *ecp; /* * If hit is nonzero then a match was found and hit will * be one greater than the index which hit. For Panther we * also need to pay attention to level to see which of l2$ or * l3$ it hit in. */ hit = cpu_matching_ecache_line(ecc->flt_addr, &clop->clo_data, 0, &level); if (hit) { --hit; disp |= CE_XDIAG_AFARMATCH; if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { if (level == 2) ecp = &clop->clo_data.chd_l2_data[hit]; else ecp = &clop->clo_data.chd_ec_data[hit]; } else { ASSERT(level == 2); ecp = &clop->clo_data.chd_ec_data[hit]; } totalsize = cpunodes[CPU->cpu_id].ecache_size; state = cpu_ectag_pa_to_subblk_state(totalsize, ecc->flt_addr, ecp->ec_tag); /* * Cheetah variants use different state encodings - * the CH_ECSTATE_* defines vary depending on the * module we're compiled for. Translate into our * one true version. Conflate Owner-Shared state * of SSM mode with Owner as victimisation of such * lines may cause a writeback. */ switch (state) { case CH_ECSTATE_MOD: disp |= EC_STATE_M; break; case CH_ECSTATE_OWN: case CH_ECSTATE_OWS: disp |= EC_STATE_O; break; case CH_ECSTATE_EXL: disp |= EC_STATE_E; break; case CH_ECSTATE_SHR: disp |= EC_STATE_S; break; default: disp |= EC_STATE_I; break; } } /* * If we initiated the delayed logout then we are responsible * for invalidating the logout area. */ if (logout_tried == B_FALSE) { bzero(clop, sizeof (ch_cpu_logout_t)); clop->clo_data.chd_afar = LOGOUT_INVALID; } } /* * Re-enable CEEN if we turned it off. */ if (orig_err & EN_REG_CEEN) set_error_enable(orig_err); kpreempt_enable(); return (disp); } /* * Scrub a correctable memory error and collect data for classification * of CE type. This function is called in the detection path, ie tl0 handling * of a correctable error trap (cpus) or interrupt (IO) at high PIL. */ void cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t logout_tried) { /* * Cheetah CE classification does not set any bits in flt_status. * Instead we will record classification datapoints in flt_disp. */ ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY); /* * To check if the error detected by IO is persistent, sticky or * intermittent. This is noticed by clear_ecc(). */ if (ecc->flt_status & ECC_IOBUS) ecc->flt_stat = C_AFSR_MEMORY; /* * Record information from this first part of the algorithm in * flt_disp. */ ecc->flt_disp = cpu_ce_scrub_mem_err_common(ecc, logout_tried); } /* * Select a partner to perform a further CE classification check from. * Must be called with kernel preemption disabled (to stop the cpu list * from changing). The detecting cpu we are partnering has cpuid * aflt->flt_inst; we might not be running on the detecting cpu. * * Restrict choice to active cpus in the same cpu partition as ourselves in * an effort to stop bad cpus in one partition causing other partitions to * perform excessive diagnostic activity. Actually since the errorq drain * is run from a softint most of the time and that is a global mechanism * this isolation is only partial. Return NULL if we fail to find a * suitable partner. * * We prefer a partner that is in a different latency group to ourselves as * we will share fewer datapaths. If such a partner is unavailable then * choose one in the same lgroup but prefer a different chip and only allow * a sibling core if flags includes PTNR_SIBLINGOK. If all else fails and * flags includes PTNR_SELFOK then permit selection of the original detector. * * We keep a cache of the last partner selected for a cpu, and we'll try to * use that previous partner if no more than cpu_ce_ptnr_cachetime_sec seconds * have passed since that selection was made. This provides the benefit * of the point-of-view of different partners over time but without * requiring frequent cpu list traversals. */ #define PTNR_SIBLINGOK 0x1 /* Allow selection of sibling core */ #define PTNR_SELFOK 0x2 /* Allow selection of cpu to "partner" itself */ static cpu_t * ce_ptnr_select(struct async_flt *aflt, int flags, int *typep) { cpu_t *sp, *dtcr, *ptnr, *locptnr, *sibptnr; hrtime_t lasttime, thistime; ASSERT(curthread->t_preempt > 0 || getpil() >= DISP_LEVEL); dtcr = cpu[aflt->flt_inst]; /* * Short-circuit for the following cases: * . the dtcr is not flagged active * . there is just one cpu present * . the detector has disappeared * . we were given a bad flt_inst cpuid; this should not happen * (eg PCI code now fills flt_inst) but if it does it is no * reason to panic. * . there is just one cpu left online in the cpu partition * * If we return NULL after this point then we do not update the * chpr_ceptnr_seltime which will cause us to perform a full lookup * again next time; this is the case where the only other cpu online * in the detector's partition is on the same chip as the detector * and since CEEN re-enable is throttled even that case should not * hurt performance. */ if (dtcr == NULL || !cpu_flagged_active(dtcr->cpu_flags)) { return (NULL); } if (ncpus == 1 || dtcr->cpu_part->cp_ncpus == 1) { if (flags & PTNR_SELFOK) { *typep = CE_XDIAG_PTNR_SELF; return (dtcr); } else { return (NULL); } } thistime = gethrtime(); lasttime = CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime); /* * Select a starting point. */ if (!lasttime) { /* * We've never selected a partner for this detector before. * Start the scan at the next online cpu in the same cpu * partition. */ sp = dtcr->cpu_next_part; } else if (thistime - lasttime < cpu_ce_ptnr_cachetime_sec * NANOSEC) { /* * Our last selection has not aged yet. If this partner: * . is still a valid cpu, * . is still in the same partition as the detector * . is still marked active * . satisfies the 'flags' argument criteria * then select it again without updating the timestamp. */ sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; if (sp == NULL || sp->cpu_part != dtcr->cpu_part || !cpu_flagged_active(sp->cpu_flags) || (sp == dtcr && !(flags & PTNR_SELFOK)) || (pg_plat_cpus_share(sp, dtcr, PGHW_CHIP) && !(flags & PTNR_SIBLINGOK))) { sp = dtcr->cpu_next_part; } else { if (sp->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { *typep = CE_XDIAG_PTNR_REMOTE; } else if (sp == dtcr) { *typep = CE_XDIAG_PTNR_SELF; } else if (pg_plat_cpus_share(sp, dtcr, PGHW_CHIP)) { *typep = CE_XDIAG_PTNR_SIBLING; } else { *typep = CE_XDIAG_PTNR_LOCAL; } return (sp); } } else { /* * Our last selection has aged. If it is nonetheless still a * valid cpu then start the scan at the next cpu in the * partition after our last partner. If the last selection * is no longer a valid cpu then go with our default. In * this way we slowly cycle through possible partners to * obtain multiple viewpoints over time. */ sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; if (sp == NULL) { sp = dtcr->cpu_next_part; } else { sp = sp->cpu_next_part; /* may be dtcr */ if (sp->cpu_part != dtcr->cpu_part) sp = dtcr; } } /* * We have a proposed starting point for our search, but if this * cpu is offline then its cpu_next_part will point to itself * so we can't use that to iterate over cpus in this partition in * the loop below. We still want to avoid iterating over cpus not * in our partition, so in the case that our starting point is offline * we will repoint it to be the detector itself; and if the detector * happens to be offline we'll return NULL from the following loop. */ if (!cpu_flagged_active(sp->cpu_flags)) { sp = dtcr; } ptnr = sp; locptnr = NULL; sibptnr = NULL; do { if (ptnr == dtcr || !cpu_flagged_active(ptnr->cpu_flags)) continue; if (ptnr->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = ptnr->cpu_id; CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; *typep = CE_XDIAG_PTNR_REMOTE; return (ptnr); } if (pg_plat_cpus_share(ptnr, dtcr, PGHW_CHIP)) { if (sibptnr == NULL) sibptnr = ptnr; continue; } if (locptnr == NULL) locptnr = ptnr; } while ((ptnr = ptnr->cpu_next_part) != sp); /* * A foreign partner has already been returned if one was available. * * If locptnr is not NULL it is a cpu in the same lgroup as the * detector, is active, and is not a sibling of the detector. * * If sibptnr is not NULL it is a sibling of the detector, and is * active. * * If we have to resort to using the detector itself we have already * checked that it is active. */ if (locptnr) { CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = locptnr->cpu_id; CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; *typep = CE_XDIAG_PTNR_LOCAL; return (locptnr); } else if (sibptnr && flags & PTNR_SIBLINGOK) { CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = sibptnr->cpu_id; CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; *typep = CE_XDIAG_PTNR_SIBLING; return (sibptnr); } else if (flags & PTNR_SELFOK) { CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = dtcr->cpu_id; CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; *typep = CE_XDIAG_PTNR_SELF; return (dtcr); } return (NULL); } /* * Cross call handler that is requested to run on the designated partner of * a cpu that experienced a possibly sticky or possibly persistnet CE. */ static void ce_ptnrchk_xc(struct async_flt *aflt, uchar_t *dispp) { *dispp = cpu_ce_scrub_mem_err_common(aflt, B_FALSE); } /* * The associated errorqs are never destroyed so we do not need to deal with * them disappearing before this timeout fires. If the affected memory * has been DR'd out since the original event the scrub algrithm will catch * any errors and return null disposition info. If the original detecting * cpu has been DR'd out then ereport detector info will not be able to * lookup CPU type; with a small timeout this is unlikely. */ static void ce_lkychk_cb(ce_lkychk_cb_t *cbarg) { struct async_flt *aflt = cbarg->lkycb_aflt; uchar_t disp; cpu_t *cp; int ptnrtype; kpreempt_disable(); if (cp = ce_ptnr_select(aflt, PTNR_SIBLINGOK | PTNR_SELFOK, &ptnrtype)) { xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, (uint64_t)aflt, (uint64_t)&disp); CE_XDIAG_SETLKYINFO(aflt->flt_disp, disp); CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); } else { ce_xdiag_lkydrops++; if (ncpus > 1) CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOPTNR); } kpreempt_enable(); errorq_commit(cbarg->lkycb_eqp, cbarg->lkycb_eqep, ERRORQ_ASYNC); kmem_free(cbarg, sizeof (ce_lkychk_cb_t)); } /* * Called from errorq drain code when processing a CE error, both from * CPU and PCI drain functions. Decide what further classification actions, * if any, we will perform. Perform immediate actions now, and schedule * delayed actions as required. Note that we are no longer necessarily running * on the detecting cpu, and that the async_flt structure will not persist on * return from this function. * * Calls to this function should aim to be self-throtlling in some way. With * the delayed re-enable of CEEN the absolute rate of calls should not * be excessive. Callers should also avoid performing in-depth classification * for events in pages that are already known to be suspect. * * We return nonzero to indicate that the event has been copied and * recirculated for further testing. The caller should not log the event * in this case - it will be logged when further test results are available. * * Our possible contexts are that of errorq_drain: below lock level or from * panic context. We can assume that the cpu we are running on is online. */ #ifdef DEBUG static int ce_xdiag_forceaction; #endif int ce_scrub_xdiag_recirc(struct async_flt *aflt, errorq_t *eqp, errorq_elem_t *eqep, size_t afltoffset) { ce_dispact_t dispact, action; cpu_t *cp; uchar_t dtcrinfo, disp; int ptnrtype; if (!ce_disp_inited || panicstr || ce_xdiag_off) { ce_xdiag_drops++; return (0); } else if (!aflt->flt_in_memory) { ce_xdiag_drops++; CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOTMEM); return (0); } dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); /* * Some correctable events are not scrubbed/classified, such as those * noticed at the tail of cpu_deferred_error. So if there is no * initial detector classification go no further. */ if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) { ce_xdiag_drops++; CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOSCRUB); return (0); } dispact = CE_DISPACT(ce_disp_table, CE_XDIAG_AFARMATCHED(dtcrinfo), CE_XDIAG_STATE(dtcrinfo), CE_XDIAG_CE1SEEN(dtcrinfo), CE_XDIAG_CE2SEEN(dtcrinfo)); action = CE_ACT(dispact); /* bad lookup caught below */ #ifdef DEBUG if (ce_xdiag_forceaction != 0) action = ce_xdiag_forceaction; #endif switch (action) { case CE_ACT_LKYCHK: { caddr_t ndata; errorq_elem_t *neqep; struct async_flt *ecc; ce_lkychk_cb_t *cbargp; if ((ndata = errorq_elem_dup(eqp, eqep, &neqep)) == NULL) { ce_xdiag_lkydrops++; CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_DUPFAIL); break; } ecc = (struct async_flt *)(ndata + afltoffset); ASSERT(ecc->flt_class == CPU_FAULT || ecc->flt_class == BUS_FAULT); ecc->flt_class = (ecc->flt_class == CPU_FAULT) ? RECIRC_CPU_FAULT : RECIRC_BUS_FAULT; cbargp = kmem_alloc(sizeof (ce_lkychk_cb_t), KM_SLEEP); cbargp->lkycb_aflt = ecc; cbargp->lkycb_eqp = eqp; cbargp->lkycb_eqep = neqep; (void) timeout((void (*)(void *))ce_lkychk_cb, (void *)cbargp, drv_usectohz(cpu_ce_lkychk_timeout_usec)); return (1); } case CE_ACT_PTNRCHK: kpreempt_disable(); /* stop cpu list changing */ if ((cp = ce_ptnr_select(aflt, 0, &ptnrtype)) != NULL) { xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, (uint64_t)aflt, (uint64_t)&disp); CE_XDIAG_SETPTNRINFO(aflt->flt_disp, disp); CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); } else if (ncpus > 1) { ce_xdiag_ptnrdrops++; CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOPTNR); } else { ce_xdiag_ptnrdrops++; CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_UNIPROC); } kpreempt_enable(); break; case CE_ACT_DONE: break; case CE_DISP_BAD: default: #ifdef DEBUG cmn_err(CE_PANIC, "ce_scrub_post: Bad action '%d'", action); #endif ce_xdiag_bad++; CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_ACTBAD); break; } return (0); } /* * We route all errors through a single switch statement. */ void cpu_ue_log_err(struct async_flt *aflt) { switch (aflt->flt_class) { case CPU_FAULT: cpu_ereport_init(aflt); if (cpu_async_log_err(aflt, NULL)) cpu_ereport_post(aflt); break; case BUS_FAULT: bus_async_log_err(aflt); break; default: cmn_err(CE_WARN, "discarding async error %p with invalid " "fault class (0x%x)", (void *)aflt, aflt->flt_class); return; } } /* * Routine for panic hook callback from panic_idle(). */ void cpu_async_panic_callb(void) { ch_async_flt_t ch_flt; struct async_flt *aflt; ch_cpu_errors_t cpu_error_regs; uint64_t afsr_errs; get_cpu_error_state(&cpu_error_regs); afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS); if (afsr_errs) { bzero(&ch_flt, sizeof (ch_async_flt_t)); aflt = (struct async_flt *)&ch_flt; aflt->flt_id = gethrtime_waitfree(); aflt->flt_bus_id = getprocessorid(); aflt->flt_inst = CPU->cpu_id; aflt->flt_stat = cpu_error_regs.afsr; aflt->flt_addr = cpu_error_regs.afar; aflt->flt_prot = AFLT_PROT_NONE; aflt->flt_class = CPU_FAULT; aflt->flt_priv = ((cpu_error_regs.afsr & C_AFSR_PRIV) != 0); aflt->flt_panic = 1; ch_flt.afsr_ext = cpu_error_regs.afsr_ext; ch_flt.afsr_errs = afsr_errs; #if defined(SERRANO) ch_flt.afar2 = cpu_error_regs.afar2; #endif /* SERRANO */ (void) cpu_queue_events(&ch_flt, NULL, afsr_errs, NULL); } } /* * Routine to convert a syndrome into a syndrome code. */ static int synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit) { if (synd_status == AFLT_STAT_INVALID) return (-1); /* * Use the syndrome to index the appropriate syndrome table, * to get the code indicating which bit(s) is(are) bad. */ if (afsr_bit & (C_AFSR_MSYND_ERRS | C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { if (afsr_bit & C_AFSR_MSYND_ERRS) { #if defined(JALAPENO) || defined(SERRANO) if ((synd == 0) || (synd >= BSYND_TBL_SIZE)) return (-1); else return (BPAR0 + synd); #else /* JALAPENO || SERRANO */ if ((synd == 0) || (synd >= MSYND_TBL_SIZE)) return (-1); else return (mtag_syndrome_tab[synd]); #endif /* JALAPENO || SERRANO */ } else { if ((synd == 0) || (synd >= ESYND_TBL_SIZE)) return (-1); else return (ecc_syndrome_tab[synd]); } } else { return (-1); } } int cpu_get_mem_sid(char *unum, char *buf, int buflen, int *lenp) { if (&plat_get_mem_sid) return (plat_get_mem_sid(unum, buf, buflen, lenp)); else return (ENOTSUP); } int cpu_get_mem_offset(uint64_t flt_addr, uint64_t *offp) { if (&plat_get_mem_offset) return (plat_get_mem_offset(flt_addr, offp)); else return (ENOTSUP); } int cpu_get_mem_addr(char *unum, char *sid, uint64_t offset, uint64_t *addrp) { if (&plat_get_mem_addr) return (plat_get_mem_addr(unum, sid, offset, addrp)); else return (ENOTSUP); } /* * Routine to return a string identifying the physical name * associated with a memory/cache error. */ int cpu_get_mem_unum(int synd_status, ushort_t flt_synd, uint64_t flt_stat, uint64_t flt_addr, int flt_bus_id, int flt_in_memory, ushort_t flt_status, char *buf, int buflen, int *lenp) { int synd_code; int ret; /* * An AFSR of -1 defaults to a memory syndrome. */ if (flt_stat == (uint64_t)-1) flt_stat = C_AFSR_CE; synd_code = synd_to_synd_code(synd_status, flt_synd, flt_stat); /* * Syndrome code must be either a single-bit error code * (0...143) or -1 for unum lookup. */ if (synd_code < 0 || synd_code >= M2) synd_code = -1; if (&plat_get_mem_unum) { if ((ret = plat_get_mem_unum(synd_code, flt_addr, flt_bus_id, flt_in_memory, flt_status, buf, buflen, lenp)) != 0) { buf[0] = '\0'; *lenp = 0; } return (ret); } return (ENOTSUP); } /* * Wrapper for cpu_get_mem_unum() routine that takes an * async_flt struct rather than explicit arguments. */ int cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt, char *buf, int buflen, int *lenp) { /* * If we come thru here for an IO bus error aflt->flt_stat will * not be the CPU AFSR, and we pass in a -1 to cpu_get_mem_unum() * so it will interpret this as a memory error. */ return (cpu_get_mem_unum(synd_status, aflt->flt_synd, (aflt->flt_class == BUS_FAULT) ? (uint64_t)-1 : ((ch_async_flt_t *)aflt)->flt_bit, aflt->flt_addr, aflt->flt_bus_id, aflt->flt_in_memory, aflt->flt_status, buf, buflen, lenp)); } /* * Return unum string given synd_code and async_flt into * the buf with size UNUM_NAMLEN */ static int cpu_get_mem_unum_synd(int synd_code, struct async_flt *aflt, char *buf) { int ret, len; /* * Syndrome code must be either a single-bit error code * (0...143) or -1 for unum lookup. */ if (synd_code < 0 || synd_code >= M2) synd_code = -1; if (&plat_get_mem_unum) { if ((ret = plat_get_mem_unum(synd_code, aflt->flt_addr, aflt->flt_bus_id, aflt->flt_in_memory, aflt->flt_status, buf, UNUM_NAMLEN, &len)) != 0) { buf[0] = '\0'; } return (ret); } buf[0] = '\0'; return (ENOTSUP); } /* * This routine is a more generic interface to cpu_get_mem_unum() * that may be used by other modules (e.g. the 'mm' driver, through * the 'MEM_NAME' ioctl, which is used by fmd to resolve unum's * for Jalapeno/Serrano FRC/RCE or FRU/RUE paired events). */ int cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar, char *buf, int buflen, int *lenp) { int synd_status, flt_in_memory, ret; ushort_t flt_status = 0; char unum[UNUM_NAMLEN]; uint64_t t_afsr_errs; /* * Check for an invalid address. */ if (afar == (uint64_t)-1) return (ENXIO); if (synd == (uint64_t)-1) synd_status = AFLT_STAT_INVALID; else synd_status = AFLT_STAT_VALID; flt_in_memory = (*afsr & C_AFSR_MEMORY) && pf_is_memory(afar >> MMU_PAGESHIFT); /* * Get aggregate AFSR for call to cpu_error_is_ecache_data. */ if (*afsr == (uint64_t)-1) t_afsr_errs = C_AFSR_CE; else { t_afsr_errs = (*afsr & C_AFSR_ALL_ERRS); #if defined(CHEETAH_PLUS) if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) t_afsr_errs |= (*(afsr + 1) & C_AFSR_EXT_ALL_ERRS); #endif /* CHEETAH_PLUS */ } /* * Turn on ECC_ECACHE if error type is E$ Data. */ if (cpu_error_is_ecache_data(CPU->cpu_id, t_afsr_errs)) flt_status |= ECC_ECACHE; ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, t_afsr_errs, afar, CPU->cpu_id, flt_in_memory, flt_status, unum, UNUM_NAMLEN, lenp); if (ret != 0) return (ret); if (*lenp >= buflen) return (ENAMETOOLONG); (void) strncpy(buf, unum, buflen); return (0); } /* * Routine to return memory information associated * with a physical address and syndrome. */ int cpu_get_mem_info(uint64_t synd, uint64_t afar, uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, int *segsp, int *banksp, int *mcidp) { int synd_status, synd_code; if (afar == (uint64_t)-1) return (ENXIO); if (synd == (uint64_t)-1) synd_status = AFLT_STAT_INVALID; else synd_status = AFLT_STAT_VALID; synd_code = synd_to_synd_code(synd_status, synd, C_AFSR_CE); if (p2get_mem_info != NULL) return ((p2get_mem_info)(synd_code, afar, mem_sizep, seg_sizep, bank_sizep, segsp, banksp, mcidp)); else return (ENOTSUP); } /* * Routine to return a string identifying the physical * name associated with a cpuid. */ int cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp) { int ret; char unum[UNUM_NAMLEN]; if (&plat_get_cpu_unum) { if ((ret = plat_get_cpu_unum(cpuid, unum, UNUM_NAMLEN, lenp)) != 0) return (ret); } else { return (ENOTSUP); } if (*lenp >= buflen) return (ENAMETOOLONG); (void) strncpy(buf, unum, buflen); return (0); } /* * This routine exports the name buffer size. */ size_t cpu_get_name_bufsize() { return (UNUM_NAMLEN); } /* * Historical function, apparantly not used. */ /* ARGSUSED */ void cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err) {} /* * Historical function only called for SBus errors in debugging. */ /*ARGSUSED*/ void read_ecc_data(struct async_flt *aflt, short verbose, short ce_err) {} /* * Clear the AFSR sticky bits. The routine returns a non-zero value if * any of the AFSR's sticky errors are detected. If a non-null pointer to * an async fault structure argument is passed in, the captured error state * (AFSR, AFAR) info will be returned in the structure. */ int clear_errors(ch_async_flt_t *ch_flt) { struct async_flt *aflt = (struct async_flt *)ch_flt; ch_cpu_errors_t cpu_error_regs; get_cpu_error_state(&cpu_error_regs); if (ch_flt != NULL) { aflt->flt_stat = cpu_error_regs.afsr & C_AFSR_MASK; aflt->flt_addr = cpu_error_regs.afar; ch_flt->afsr_ext = cpu_error_regs.afsr_ext; ch_flt->afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS); #if defined(SERRANO) ch_flt->afar2 = cpu_error_regs.afar2; #endif /* SERRANO */ } set_cpu_error_state(&cpu_error_regs); return (((cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS)) != 0); } /* * Clear any AFSR error bits, and check for persistence. * * It would be desirable to also insist that syndrome match. PCI handling * has already filled flt_synd. For errors trapped by CPU we only fill * flt_synd when we queue the event, so we do not have a valid flt_synd * during initial classification (it is valid if we're called as part of * subsequent low-pil additional classification attempts). We could try * to determine which syndrome to use: we know we're only called for * CE/RCE (Jalapeno & Serrano) and CE/EMC (others) so the syndrome to use * would be esynd/none and esynd/msynd, respectively. If that is * implemented then what do we do in the case that we do experience an * error on the same afar but with different syndrome? At the very least * we should count such occurences. Anyway, for now, we'll leave it as * it has been for ages. */ static int clear_ecc(struct async_flt *aflt) { ch_cpu_errors_t cpu_error_regs; /* * Snapshot the AFSR and AFAR and clear any errors */ get_cpu_error_state(&cpu_error_regs); set_cpu_error_state(&cpu_error_regs); /* * If any of the same memory access error bits are still on and * the AFAR matches, return that the error is persistent. */ return ((cpu_error_regs.afsr & (C_AFSR_MEMORY & aflt->flt_stat)) != 0 && cpu_error_regs.afar == aflt->flt_addr); } /* * Turn off all cpu error detection, normally only used for panics. */ void cpu_disable_errors(void) { xt_all(set_error_enable_tl1, EN_REG_DISABLE, EER_SET_ABSOLUTE); /* * With error detection now turned off, check the other cpus * logout areas for any unlogged errors. */ if (enable_check_other_cpus_logout) { cpu_check_other_cpus_logout(); /* * Make a second pass over the logout areas, in case * there is a failing CPU in an error-trap loop which * will write to the logout area once it is emptied. */ cpu_check_other_cpus_logout(); } } /* * Enable errors. */ void cpu_enable_errors(void) { xt_all(set_error_enable_tl1, EN_REG_ENABLE, EER_SET_ABSOLUTE); } /* * Flush the entire ecache using displacement flush by reading through a * physical address range twice as large as the Ecache. */ void cpu_flush_ecache(void) { flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size, cpunodes[CPU->cpu_id].ecache_linesize); } /* * Return CPU E$ set size - E$ size divided by the associativity. * We use this function in places where the CPU_PRIVATE ptr may not be * initialized yet. Note that for send_mondo and in the Ecache scrubber, * we're guaranteed that CPU_PRIVATE is initialized. Also, cpunodes is set * up before the kernel switches from OBP's to the kernel's trap table, so * we don't have to worry about cpunodes being unitialized. */ int cpu_ecache_set_size(struct cpu *cp) { if (CPU_PRIVATE(cp)) return (CPU_PRIVATE_VAL(cp, chpr_ec_set_size)); return (cpunodes[cp->cpu_id].ecache_size / cpu_ecache_nway()); } /* * Flush Ecache line. * Uses ASI_EC_DIAG for Cheetah+ and Jalapeno. * Uses normal displacement flush for Cheetah. */ static void cpu_flush_ecache_line(ch_async_flt_t *ch_flt) { struct async_flt *aflt = (struct async_flt *)ch_flt; int ec_set_size = cpu_ecache_set_size(CPU); ecache_flush_line(aflt->flt_addr, ec_set_size); } /* * Scrub physical address. * Scrub code is different depending upon whether this a Cheetah+ with 2-way * Ecache or direct-mapped Ecache. */ static void cpu_scrubphys(struct async_flt *aflt) { int ec_set_size = cpu_ecache_set_size(CPU); scrubphys(aflt->flt_addr, ec_set_size); } /* * Clear physical address. * Scrub code is different depending upon whether this a Cheetah+ with 2-way * Ecache or direct-mapped Ecache. */ void cpu_clearphys(struct async_flt *aflt) { int lsize = cpunodes[CPU->cpu_id].ecache_linesize; int ec_set_size = cpu_ecache_set_size(CPU); clearphys(aflt->flt_addr, ec_set_size, lsize); } #if defined(CPU_IMP_ECACHE_ASSOC) /* * Check for a matching valid line in all the sets. * If found, return set# + 1. Otherwise return 0. */ static int cpu_ecache_line_valid(ch_async_flt_t *ch_flt) { struct async_flt *aflt = (struct async_flt *)ch_flt; int totalsize = cpunodes[CPU->cpu_id].ecache_size; int ec_set_size = cpu_ecache_set_size(CPU); ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; int nway = cpu_ecache_nway(); int i; for (i = 0; i < nway; i++, ecp++) { if (!cpu_ectag_line_invalid(totalsize, ecp->ec_tag) && (aflt->flt_addr & P2ALIGN(C_AFAR_PA, ec_set_size)) == cpu_ectag_to_pa(ec_set_size, ecp->ec_tag)) return (i+1); } return (0); } #endif /* CPU_IMP_ECACHE_ASSOC */ /* * Check whether a line in the given logout info matches the specified * fault address. If reqval is set then the line must not be Invalid. * Returns 0 on failure; on success (way + 1) is returned an *level is * set to 2 for l2$ or 3 for l3$. */ static int cpu_matching_ecache_line(uint64_t faddr, void *data, int reqval, int *level) { ch_diag_data_t *cdp = data; ch_ec_data_t *ecp; int totalsize, ec_set_size; int i, ways; int match = 0; int tagvalid; uint64_t addr, tagpa; int ispanther = IS_PANTHER(cpunodes[CPU->cpu_id].implementation); /* * Check the l2$ logout data */ if (ispanther) { ecp = &cdp->chd_l2_data[0]; ec_set_size = PN_L2_SET_SIZE; ways = PN_L2_NWAYS; } else { ecp = &cdp->chd_ec_data[0]; ec_set_size = cpu_ecache_set_size(CPU); ways = cpu_ecache_nway(); totalsize = cpunodes[CPU->cpu_id].ecache_size; } /* remove low order PA bits from fault address not used in PA tag */ addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); for (i = 0; i < ways; i++, ecp++) { if (ispanther) { tagpa = PN_L2TAG_TO_PA(ecp->ec_tag); tagvalid = !PN_L2_LINE_INVALID(ecp->ec_tag); } else { tagpa = cpu_ectag_to_pa(ec_set_size, ecp->ec_tag); tagvalid = !cpu_ectag_line_invalid(totalsize, ecp->ec_tag); } if (tagpa == addr && (!reqval || tagvalid)) { match = i + 1; *level = 2; break; } } if (match || !ispanther) return (match); /* For Panther we also check the l3$ */ ecp = &cdp->chd_ec_data[0]; ec_set_size = PN_L3_SET_SIZE; ways = PN_L3_NWAYS; addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); for (i = 0; i < ways; i++, ecp++) { if (PN_L3TAG_TO_PA(ecp->ec_tag) == addr && (!reqval || !PN_L3_LINE_INVALID(ecp->ec_tag))) { match = i + 1; *level = 3; break; } } return (match); } #if defined(CPU_IMP_L1_CACHE_PARITY) /* * Record information related to the source of an Dcache Parity Error. */ static void cpu_dcache_parity_info(ch_async_flt_t *ch_flt) { int dc_set_size = dcache_size / CH_DCACHE_NWAY; int index; /* * Since instruction decode cannot be done at high PIL * just examine the entire Dcache to locate the error. */ if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { ch_flt->parity_data.dpe.cpl_way = -1; ch_flt->parity_data.dpe.cpl_off = -1; } for (index = 0; index < dc_set_size; index += dcache_linesize) cpu_dcache_parity_check(ch_flt, index); } /* * Check all ways of the Dcache at a specified index for good parity. */ static void cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index) { int dc_set_size = dcache_size / CH_DCACHE_NWAY; uint64_t parity_bits, pbits, data_word; static int parity_bits_popc[] = { 0, 1, 1, 0 }; int way, word, data_byte; ch_dc_data_t *dcp = &ch_flt->parity_data.dpe.cpl_dc[0]; ch_dc_data_t tmp_dcp; for (way = 0; way < CH_DCACHE_NWAY; way++, dcp++) { /* * Perform diagnostic read. */ get_dcache_dtag(index + way * dc_set_size, (uint64_t *)&tmp_dcp); /* * Check tag for even parity. * Sum of 1 bits (including parity bit) should be even. */ if (popc64(tmp_dcp.dc_tag & CHP_DCTAG_PARMASK) & 1) { /* * If this is the first error log detailed information * about it and check the snoop tag. Otherwise just * record the fact that we found another error. */ if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { ch_flt->parity_data.dpe.cpl_way = way; ch_flt->parity_data.dpe.cpl_cache = CPU_DC_PARITY; ch_flt->parity_data.dpe.cpl_tag |= CHP_DC_TAG; if (popc64(tmp_dcp.dc_sntag & CHP_DCSNTAG_PARMASK) & 1) { ch_flt->parity_data.dpe.cpl_tag |= CHP_DC_SNTAG; ch_flt->parity_data.dpe.cpl_lcnt++; } bcopy(&tmp_dcp, dcp, sizeof (ch_dc_data_t)); } ch_flt->parity_data.dpe.cpl_lcnt++; } if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { /* * Panther has more parity bits than the other * processors for covering dcache data and so each * byte of data in each word has its own parity bit. */ parity_bits = tmp_dcp.dc_pn_data_parity; for (word = 0; word < 4; word++) { data_word = tmp_dcp.dc_data[word]; pbits = parity_bits & PN_DC_DATA_PARITY_MASK; for (data_byte = 0; data_byte < 8; data_byte++) { if (((popc64(data_word & PN_DC_DATA_PARITY_MASK)) & 1) ^ (pbits & 1)) { cpu_record_dc_data_parity( ch_flt, dcp, &tmp_dcp, way, word); } pbits >>= 1; data_word >>= 8; } parity_bits >>= 8; } } else { /* * Check data array for even parity. * The 8 parity bits are grouped into 4 pairs each * of which covers a 64-bit word. The endianness is * reversed -- the low-order parity bits cover the * high-order data words. */ parity_bits = tmp_dcp.dc_utag >> 8; for (word = 0; word < 4; word++) { pbits = (parity_bits >> (6 - word * 2)) & 3; if ((popc64(tmp_dcp.dc_data[word]) + parity_bits_popc[pbits]) & 1) { cpu_record_dc_data_parity(ch_flt, dcp, &tmp_dcp, way, word); } } } } } static void cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word) { /* * If this is the first error log detailed information about it. * Otherwise just record the fact that we found another error. */ if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { ch_flt->parity_data.dpe.cpl_way = way; ch_flt->parity_data.dpe.cpl_cache = CPU_DC_PARITY; ch_flt->parity_data.dpe.cpl_off = word * 8; bcopy(src_dcp, dest_dcp, sizeof (ch_dc_data_t)); } ch_flt->parity_data.dpe.cpl_lcnt++; } /* * Record information related to the source of an Icache Parity Error. * * Called with the Icache disabled so any diagnostic accesses are safe. */ static void cpu_icache_parity_info(ch_async_flt_t *ch_flt) { int ic_set_size; int ic_linesize; int index; if (CPU_PRIVATE(CPU)) { ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / CH_ICACHE_NWAY; ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); } else { ic_set_size = icache_size / CH_ICACHE_NWAY; ic_linesize = icache_linesize; } ch_flt->parity_data.ipe.cpl_way = -1; ch_flt->parity_data.ipe.cpl_off = -1; for (index = 0; index < ic_set_size; index += ic_linesize) cpu_icache_parity_check(ch_flt, index); } /* * Check all ways of the Icache at a specified index for good parity. */ static void cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index) { uint64_t parmask, pn_inst_parity; int ic_set_size; int ic_linesize; int flt_index, way, instr, num_instr; struct async_flt *aflt = (struct async_flt *)ch_flt; ch_ic_data_t *icp = &ch_flt->parity_data.ipe.cpl_ic[0]; ch_ic_data_t tmp_icp; if (CPU_PRIVATE(CPU)) { ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / CH_ICACHE_NWAY; ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); } else { ic_set_size = icache_size / CH_ICACHE_NWAY; ic_linesize = icache_linesize; } /* * Panther has twice as many instructions per icache line and the * instruction parity bit is in a different location. */ if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { num_instr = PN_IC_DATA_REG_SIZE / sizeof (uint64_t); pn_inst_parity = PN_ICDATA_PARITY_BIT_MASK; } else { num_instr = CH_IC_DATA_REG_SIZE / sizeof (uint64_t); pn_inst_parity = 0; } /* * Index at which we expect to find the parity error. */ flt_index = P2ALIGN(aflt->flt_addr % ic_set_size, ic_linesize); for (way = 0; way < CH_ICACHE_NWAY; way++, icp++) { /* * Diagnostic reads expect address argument in ASI format. */ get_icache_dtag(2 * (index + way * ic_set_size), (uint64_t *)&tmp_icp); /* * If this is the index in which we expect to find the * error log detailed information about each of the ways. * This information will be displayed later if we can't * determine the exact way in which the error is located. */ if (flt_index == index) bcopy(&tmp_icp, icp, sizeof (ch_ic_data_t)); /* * Check tag for even parity. * Sum of 1 bits (including parity bit) should be even. */ if (popc64(tmp_icp.ic_patag & CHP_ICPATAG_PARMASK) & 1) { /* * If this way is the one in which we expected * to find the error record the way and check the * snoop tag. Otherwise just record the fact we * found another error. */ if (flt_index == index) { ch_flt->parity_data.ipe.cpl_way = way; ch_flt->parity_data.ipe.cpl_tag |= CHP_IC_TAG; if (popc64(tmp_icp.ic_sntag & CHP_ICSNTAG_PARMASK) & 1) { ch_flt->parity_data.ipe.cpl_tag |= CHP_IC_SNTAG; ch_flt->parity_data.ipe.cpl_lcnt++; } } ch_flt->parity_data.ipe.cpl_lcnt++; continue; } /* * Check instruction data for even parity. * Bits participating in parity differ for PC-relative * versus non-PC-relative instructions. */ for (instr = 0; instr < num_instr; instr++) { parmask = (tmp_icp.ic_data[instr] & CH_ICDATA_PRED_ISPCREL) ? (CHP_ICDATA_PCREL_PARMASK | pn_inst_parity) : (CHP_ICDATA_NPCREL_PARMASK | pn_inst_parity); if (popc64(tmp_icp.ic_data[instr] & parmask) & 1) { /* * If this way is the one in which we expected * to find the error record the way and offset. * Otherwise just log the fact we found another * error. */ if (flt_index == index) { ch_flt->parity_data.ipe.cpl_way = way; ch_flt->parity_data.ipe.cpl_off = instr * 4; } ch_flt->parity_data.ipe.cpl_lcnt++; continue; } } } } /* * Record information related to the source of an Pcache Parity Error. */ static void cpu_pcache_parity_info(ch_async_flt_t *ch_flt) { int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; int index; /* * Since instruction decode cannot be done at high PIL just * examine the entire Pcache to check for any parity errors. */ if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { ch_flt->parity_data.dpe.cpl_way = -1; ch_flt->parity_data.dpe.cpl_off = -1; } for (index = 0; index < pc_set_size; index += CH_PCACHE_LSIZE) cpu_pcache_parity_check(ch_flt, index); } /* * Check all ways of the Pcache at a specified index for good parity. */ static void cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index) { int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; int pc_data_words = CH_PC_DATA_REG_SIZE / sizeof (uint64_t); int way, word, pbit, parity_bits; ch_pc_data_t *pcp = &ch_flt->parity_data.dpe.cpl_pc[0]; ch_pc_data_t tmp_pcp; for (way = 0; way < CH_PCACHE_NWAY; way++, pcp++) { /* * Perform diagnostic read. */ get_pcache_dtag(index + way * pc_set_size, (uint64_t *)&tmp_pcp); /* * Check data array for odd parity. There are 8 parity * bits (bits 57:50 of ASI_PCACHE_STATUS_DATA) and each * of those bits covers exactly 8 bytes of the data * array: * * parity bit P$ data bytes covered * ---------- --------------------- * 50 63:56 * 51 55:48 * 52 47:40 * 53 39:32 * 54 31:24 * 55 23:16 * 56 15:8 * 57 7:0 */ parity_bits = PN_PC_PARITY_BITS(tmp_pcp.pc_status); for (word = 0; word < pc_data_words; word++) { pbit = (parity_bits >> (pc_data_words - word - 1)) & 1; if ((popc64(tmp_pcp.pc_data[word]) & 1) ^ pbit) { /* * If this is the first error log detailed * information about it. Otherwise just record * the fact that we found another error. */ if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { ch_flt->parity_data.dpe.cpl_way = way; ch_flt->parity_data.dpe.cpl_cache = CPU_PC_PARITY; ch_flt->parity_data.dpe.cpl_off = word * sizeof (uint64_t); bcopy(&tmp_pcp, pcp, sizeof (ch_pc_data_t)); } ch_flt->parity_data.dpe.cpl_lcnt++; } } } } /* * Add L1 Data cache data to the ereport payload. */ static void cpu_payload_add_dcache(struct async_flt *aflt, nvlist_t *nvl) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; ch_dc_data_t *dcp; ch_dc_data_t dcdata[CH_DCACHE_NWAY]; uint_t nelem; int i, ways_to_check, ways_logged = 0; /* * If this is an D$ fault then there may be multiple * ways captured in the ch_parity_log_t structure. * Otherwise, there will be at most one way captured * in the ch_diag_data_t struct. * Check each way to see if it should be encoded. */ if (ch_flt->flt_type == CPU_DC_PARITY) ways_to_check = CH_DCACHE_NWAY; else ways_to_check = 1; for (i = 0; i < ways_to_check; i++) { if (ch_flt->flt_type == CPU_DC_PARITY) dcp = &ch_flt->parity_data.dpe.cpl_dc[i]; else dcp = &ch_flt->flt_diag_data.chd_dc_data; if (dcp->dc_logflag == DC_LOGFLAG_MAGIC) { bcopy(dcp, &dcdata[ways_logged], sizeof (ch_dc_data_t)); ways_logged++; } } /* * Add the dcache data to the payload. */ fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_WAYS, DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); if (ways_logged != 0) { nelem = sizeof (ch_dc_data_t) / sizeof (uint64_t) * ways_logged; fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_DATA, DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)dcdata, NULL); } } /* * Add L1 Instruction cache data to the ereport payload. */ static void cpu_payload_add_icache(struct async_flt *aflt, nvlist_t *nvl) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; ch_ic_data_t *icp; ch_ic_data_t icdata[CH_ICACHE_NWAY]; uint_t nelem; int i, ways_to_check, ways_logged = 0; /* * If this is an I$ fault then there may be multiple * ways captured in the ch_parity_log_t structure. * Otherwise, there will be at most one way captured * in the ch_diag_data_t struct. * Check each way to see if it should be encoded. */ if (ch_flt->flt_type == CPU_IC_PARITY) ways_to_check = CH_ICACHE_NWAY; else ways_to_check = 1; for (i = 0; i < ways_to_check; i++) { if (ch_flt->flt_type == CPU_IC_PARITY) icp = &ch_flt->parity_data.ipe.cpl_ic[i]; else icp = &ch_flt->flt_diag_data.chd_ic_data; if (icp->ic_logflag == IC_LOGFLAG_MAGIC) { bcopy(icp, &icdata[ways_logged], sizeof (ch_ic_data_t)); ways_logged++; } } /* * Add the icache data to the payload. */ fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_WAYS, DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); if (ways_logged != 0) { nelem = sizeof (ch_ic_data_t) / sizeof (uint64_t) * ways_logged; fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_DATA, DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)icdata, NULL); } } #endif /* CPU_IMP_L1_CACHE_PARITY */ /* * Add ecache data to payload. */ static void cpu_payload_add_ecache(struct async_flt *aflt, nvlist_t *nvl) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; ch_ec_data_t *ecp; ch_ec_data_t ecdata[CHD_EC_DATA_SETS]; uint_t nelem; int i, ways_logged = 0; /* * Check each way to see if it should be encoded * and concatinate it into a temporary buffer. */ for (i = 0; i < CHD_EC_DATA_SETS; i++) { ecp = &ch_flt->flt_diag_data.chd_ec_data[i]; if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { bcopy(ecp, &ecdata[ways_logged], sizeof (ch_ec_data_t)); ways_logged++; } } /* * Panther CPUs have an additional level of cache and so * what we just collected was the L3 (ecache) and not the * L2 cache. */ if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { /* * Add the L3 (ecache) data to the payload. */ fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_WAYS, DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); if (ways_logged != 0) { nelem = sizeof (ch_ec_data_t) / sizeof (uint64_t) * ways_logged; fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_DATA, DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)ecdata, NULL); } /* * Now collect the L2 cache. */ ways_logged = 0; for (i = 0; i < PN_L2_NWAYS; i++) { ecp = &ch_flt->flt_diag_data.chd_l2_data[i]; if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { bcopy(ecp, &ecdata[ways_logged], sizeof (ch_ec_data_t)); ways_logged++; } } } /* * Add the L2 cache data to the payload. */ fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_WAYS, DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); if (ways_logged != 0) { nelem = sizeof (ch_ec_data_t) / sizeof (uint64_t) * ways_logged; fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_DATA, DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)ecdata, NULL); } } /* * Initialize cpu scheme for specified cpu. */ static void cpu_fmri_cpu_set(nvlist_t *cpu_fmri, int cpuid) { char sbuf[21]; /* sizeof (UINT64_MAX) + '\0' */ uint8_t mask; mask = cpunodes[cpuid].version; (void) snprintf(sbuf, sizeof (sbuf), "%llX", (u_longlong_t)cpunodes[cpuid].device_id); (void) fm_fmri_cpu_set(cpu_fmri, FM_CPU_SCHEME_VERSION, NULL, cpuid, &mask, (const char *)sbuf); } /* * Returns ereport resource type. */ static int cpu_error_to_resource_type(struct async_flt *aflt) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; switch (ch_flt->flt_type) { case CPU_CE_ECACHE: case CPU_UE_ECACHE: case CPU_UE_ECACHE_RETIRE: case CPU_ORPH: /* * If AFSR error bit indicates L2$ Data for Cheetah, * Cheetah+ or Jaguar, or L3$ Data for Panther, return * E$ Data type, otherwise, return CPU type. */ if (cpu_error_is_ecache_data(aflt->flt_inst, ch_flt->flt_bit)) return (ERRTYPE_ECACHE_DATA); return (ERRTYPE_CPU); case CPU_CE: case CPU_UE: case CPU_EMC: case CPU_DUE: case CPU_RCE: case CPU_RUE: case CPU_FRC: case CPU_FRU: return (ERRTYPE_MEMORY); case CPU_IC_PARITY: case CPU_DC_PARITY: case CPU_FPUERR: case CPU_PC_PARITY: case CPU_ITLB_PARITY: case CPU_DTLB_PARITY: return (ERRTYPE_CPU); } return (ERRTYPE_UNKNOWN); } /* * Encode the data saved in the ch_async_flt_t struct into * the FM ereport payload. */ static void cpu_payload_add_aflt(struct async_flt *aflt, nvlist_t *payload, nvlist_t *resource, int *afar_status, int *synd_status) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; *synd_status = AFLT_STAT_INVALID; *afar_status = AFLT_STAT_INVALID; if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR, DATA_TYPE_UINT64, aflt->flt_stat, NULL); } if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR_EXT) && IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR_EXT, DATA_TYPE_UINT64, ch_flt->afsr_ext, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR_STATUS) { *afar_status = afsr_to_afar_status(ch_flt->afsr_errs, ch_flt->flt_bit); fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR_STATUS, DATA_TYPE_UINT8, (uint8_t)*afar_status, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR, DATA_TYPE_UINT64, aflt->flt_addr, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PC) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PC, DATA_TYPE_UINT64, (uint64_t)aflt->flt_pc, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TL) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TL, DATA_TYPE_UINT8, (uint8_t)aflt->flt_tl, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TT) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TT, DATA_TYPE_UINT8, flt_to_trap_type(aflt), NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PRIV) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PRIV, DATA_TYPE_BOOLEAN_VALUE, (aflt->flt_priv ? B_TRUE : B_FALSE), NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ME) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ME, DATA_TYPE_BOOLEAN_VALUE, (aflt->flt_stat & C_AFSR_ME) ? B_TRUE : B_FALSE, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND_STATUS) { *synd_status = afsr_to_synd_status(aflt->flt_inst, ch_flt->afsr_errs, ch_flt->flt_bit); fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND_STATUS, DATA_TYPE_UINT8, (uint8_t)*synd_status, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND, DATA_TYPE_UINT16, (uint16_t)aflt->flt_synd, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_TYPE) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_TYPE, DATA_TYPE_STRING, flt_to_error_type(aflt), NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_DISP) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_DISP, DATA_TYPE_UINT64, aflt->flt_disp, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L2) cpu_payload_add_ecache(aflt, payload); if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_COPYFUNCTION) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_COPYFUNCTION, DATA_TYPE_UINT8, (uint8_t)aflt->flt_status & 0xff, NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_HOWDETECTED) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_HOWDETECTED, DATA_TYPE_UINT8, (uint8_t)(aflt->flt_status >> 8), NULL); } if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_INSTRBLOCK) { fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_INSTRBLOCK, DATA_TYPE_UINT32_ARRAY, 16, (uint32_t *)&ch_flt->flt_fpdata, NULL); } #if defined(CPU_IMP_L1_CACHE_PARITY) if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1D) cpu_payload_add_dcache(aflt, payload); if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1I) cpu_payload_add_icache(aflt, payload); #endif /* CPU_IMP_L1_CACHE_PARITY */ #if defined(CHEETAH_PLUS) if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1P) cpu_payload_add_pcache(aflt, payload); if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_TLB) cpu_payload_add_tlb(aflt, payload); #endif /* CHEETAH_PLUS */ /* * Create the FMRI that goes into the payload * and contains the unum info if necessary. */ if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_RESOURCE) { char unum[UNUM_NAMLEN] = ""; char sid[DIMM_SERIAL_ID_LEN] = ""; int len, ret, rtype, synd_code; uint64_t offset = (uint64_t)-1; rtype = cpu_error_to_resource_type(aflt); switch (rtype) { case ERRTYPE_MEMORY: case ERRTYPE_ECACHE_DATA: /* * Memory errors, do unum lookup */ if (*afar_status == AFLT_STAT_INVALID) break; if (rtype == ERRTYPE_ECACHE_DATA) aflt->flt_status |= ECC_ECACHE; else aflt->flt_status &= ~ECC_ECACHE; synd_code = synd_to_synd_code(*synd_status, aflt->flt_synd, ch_flt->flt_bit); if (cpu_get_mem_unum_synd(synd_code, aflt, unum) != 0) break; ret = cpu_get_mem_sid(unum, sid, DIMM_SERIAL_ID_LEN, &len); if (ret == 0) { (void) cpu_get_mem_offset(aflt->flt_addr, &offset); } fm_fmri_mem_set(resource, FM_MEM_SCHEME_VERSION, NULL, unum, (ret == 0) ? sid : NULL, offset); fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_RESOURCE, DATA_TYPE_NVLIST, resource, NULL); break; case ERRTYPE_CPU: /* * On-board processor array error, add cpu resource. */ cpu_fmri_cpu_set(resource, aflt->flt_inst); fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_RESOURCE, DATA_TYPE_NVLIST, resource, NULL); break; } } } /* * Initialize the way info if necessary. */ void cpu_ereport_init(struct async_flt *aflt) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; ch_ec_data_t *l2p = &ch_flt->flt_diag_data.chd_l2_data[0]; int i; /* * Initialize the info in the CPU logout structure. * The I$/D$ way information is not initialized here * since it is captured in the logout assembly code. */ for (i = 0; i < CHD_EC_DATA_SETS; i++) (ecp + i)->ec_way = i; for (i = 0; i < PN_L2_NWAYS; i++) (l2p + i)->ec_way = i; } /* * Returns whether fault address is valid for this error bit and * whether the address is "in memory" (i.e. pf_is_memory returns 1). */ int cpu_flt_in_memory(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit) { struct async_flt *aflt = (struct async_flt *)ch_flt; return ((t_afsr_bit & C_AFSR_MEMORY) && afsr_to_afar_status(ch_flt->afsr_errs, t_afsr_bit) == AFLT_STAT_VALID && pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)); } /* * Returns whether fault address is valid based on the error bit for the * one event being queued and whether the address is "in memory". */ static int cpu_flt_in_memory_one_event(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit) { struct async_flt *aflt = (struct async_flt *)ch_flt; int afar_status; uint64_t afsr_errs, afsr_ow, *ow_bits; if (!(t_afsr_bit & C_AFSR_MEMORY) || !pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)) return (0); afsr_errs = ch_flt->afsr_errs; afar_status = afsr_to_afar_status(afsr_errs, t_afsr_bit); switch (afar_status) { case AFLT_STAT_VALID: return (1); case AFLT_STAT_AMBIGUOUS: /* * Status is ambiguous since another error bit (or bits) * of equal priority to the specified bit on in the afsr, * so check those bits. Return 1 only if the bits on in the * same class as the t_afsr_bit are also C_AFSR_MEMORY bits. * Otherwise not all the equal priority bits are for memory * errors, so return 0. */ ow_bits = afar_overwrite; while ((afsr_ow = *ow_bits++) != 0) { /* * Get other bits that are on in t_afsr_bit's priority * class to check for Memory Error bits only. */ if (afsr_ow & t_afsr_bit) { if ((afsr_errs & afsr_ow) & ~C_AFSR_MEMORY) return (0); else return (1); } } /*FALLTHRU*/ default: return (0); } } static void cpu_log_diag_info(ch_async_flt_t *ch_flt) { struct async_flt *aflt = (struct async_flt *)ch_flt; ch_dc_data_t *dcp = &ch_flt->flt_diag_data.chd_dc_data; ch_ic_data_t *icp = &ch_flt->flt_diag_data.chd_ic_data; ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; #if defined(CPU_IMP_ECACHE_ASSOC) int i, nway; #endif /* CPU_IMP_ECACHE_ASSOC */ /* * Check if the CPU log out captured was valid. */ if (ch_flt->flt_diag_data.chd_afar == LOGOUT_INVALID || ch_flt->flt_data_incomplete) return; #if defined(CPU_IMP_ECACHE_ASSOC) nway = cpu_ecache_nway(); i = cpu_ecache_line_valid(ch_flt); if (i == 0 || i > nway) { for (i = 0; i < nway; i++) ecp[i].ec_logflag = EC_LOGFLAG_MAGIC; } else ecp[i - 1].ec_logflag = EC_LOGFLAG_MAGIC; #else /* CPU_IMP_ECACHE_ASSOC */ ecp->ec_logflag = EC_LOGFLAG_MAGIC; #endif /* CPU_IMP_ECACHE_ASSOC */ #if defined(CHEETAH_PLUS) pn_cpu_log_diag_l2_info(ch_flt); #endif /* CHEETAH_PLUS */ if (CH_DCTAG_MATCH(dcp->dc_tag, aflt->flt_addr)) { dcp->dc_way = CH_DCIDX_TO_WAY(dcp->dc_idx); dcp->dc_logflag = DC_LOGFLAG_MAGIC; } if (CH_ICTAG_MATCH(icp, aflt->flt_addr)) { if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) icp->ic_way = PN_ICIDX_TO_WAY(icp->ic_idx); else icp->ic_way = CH_ICIDX_TO_WAY(icp->ic_idx); icp->ic_logflag = IC_LOGFLAG_MAGIC; } } /* * Cheetah ECC calculation. * * We only need to do the calculation on the data bits and can ignore check * bit and Mtag bit terms in the calculation. */ static uint64_t ch_ecc_table[9][2] = { /* * low order 64-bits high-order 64-bits */ { 0x46bffffeccd1177f, 0x488800022100014c }, { 0x42fccc81331ff77f, 0x14424f1010249184 }, { 0x8898827c222f1ffe, 0x22c1222808184aaf }, { 0xf7632203e131ccf1, 0xe1241121848292b8 }, { 0x7f5511421b113809, 0x901c88d84288aafe }, { 0x1d49412184882487, 0x8f338c87c044c6ef }, { 0xf552181014448344, 0x7ff8f4443e411911 }, { 0x2189240808f24228, 0xfeeff8cc81333f42 }, { 0x3280008440001112, 0xfee88b337ffffd62 }, }; /* * 64-bit population count, use well-known popcnt trick. * We could use the UltraSPARC V9 POPC instruction, but some * CPUs including Cheetahplus and Jaguar do not support that * instruction. */ int popc64(uint64_t val) { int cnt; for (cnt = 0; val != 0; val &= val - 1) cnt++; return (cnt); } /* * Generate the 9 ECC bits for the 128-bit chunk based on the table above. * Note that xor'ing an odd number of 1 bits == 1 and xor'ing an even number * of 1 bits == 0, so we can just use the least significant bit of the popcnt * instead of doing all the xor's. */ uint32_t us3_gen_ecc(uint64_t data_low, uint64_t data_high) { int bitno, s; int synd = 0; for (bitno = 0; bitno < 9; bitno++) { s = (popc64(data_low & ch_ecc_table[bitno][0]) + popc64(data_high & ch_ecc_table[bitno][1])) & 1; synd |= (s << bitno); } return (synd); } /* * Queue one event based on ecc_type_to_info entry. If the event has an AFT1 * tag associated with it or is a fatal event (aflt_panic set), it is sent to * the UE event queue. Otherwise it is dispatched to the CE event queue. */ static void cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, ecc_type_to_info_t *eccp, ch_diag_data_t *cdp) { struct async_flt *aflt = (struct async_flt *)ch_flt; if (reason && strlen(reason) + strlen(eccp->ec_reason) < MAX_REASON_STRING) { (void) strcat(reason, eccp->ec_reason); } ch_flt->flt_bit = eccp->ec_afsr_bit; ch_flt->flt_type = eccp->ec_flt_type; if (cdp != NULL && cdp->chd_afar != LOGOUT_INVALID) ch_flt->flt_diag_data = *cdp; else ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; aflt->flt_in_memory = cpu_flt_in_memory_one_event(ch_flt, ch_flt->flt_bit); if (ch_flt->flt_bit & C_AFSR_MSYND_ERRS) aflt->flt_synd = GET_M_SYND(aflt->flt_stat); else if (ch_flt->flt_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) aflt->flt_synd = GET_E_SYND(aflt->flt_stat); else aflt->flt_synd = 0; aflt->flt_payload = eccp->ec_err_payload; if (aflt->flt_panic || (eccp->ec_afsr_bit & (C_AFSR_LEVEL1 | C_AFSR_EXT_LEVEL1))) cpu_errorq_dispatch(eccp->ec_err_class, (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); else cpu_errorq_dispatch(eccp->ec_err_class, (void *)ch_flt, sizeof (ch_async_flt_t), ce_queue, aflt->flt_panic); } /* * Queue events on async event queue one event per error bit. First we * queue the events that we "expect" for the given trap, then we queue events * that we may not expect. Return number of events queued. */ int cpu_queue_events(ch_async_flt_t *ch_flt, char *reason, uint64_t t_afsr_errs, ch_cpu_logout_t *clop) { struct async_flt *aflt = (struct async_flt *)ch_flt; ecc_type_to_info_t *eccp; int nevents = 0; uint64_t primary_afar = aflt->flt_addr, primary_afsr = aflt->flt_stat; #if defined(CHEETAH_PLUS) uint64_t orig_t_afsr_errs; #endif uint64_t primary_afsr_ext = ch_flt->afsr_ext; uint64_t primary_afsr_errs = ch_flt->afsr_errs; ch_diag_data_t *cdp = NULL; t_afsr_errs &= ((C_AFSR_ALL_ERRS & ~C_AFSR_ME) | C_AFSR_EXT_ALL_ERRS); #if defined(CHEETAH_PLUS) orig_t_afsr_errs = t_afsr_errs; /* * For Cheetah+, log the shadow AFSR/AFAR bits first. */ if (clop != NULL) { /* * Set the AFSR and AFAR fields to the shadow registers. The * flt_addr and flt_stat fields will be reset to the primaries * below, but the sdw_addr and sdw_stat will stay as the * secondaries. */ cdp = &clop->clo_sdw_data; aflt->flt_addr = ch_flt->flt_sdw_afar = cdp->chd_afar; aflt->flt_stat = ch_flt->flt_sdw_afsr = cdp->chd_afsr; ch_flt->afsr_ext = ch_flt->flt_sdw_afsr_ext = cdp->chd_afsr_ext; ch_flt->afsr_errs = (cdp->chd_afsr_ext & C_AFSR_EXT_ALL_ERRS) | (cdp->chd_afsr & C_AFSR_ALL_ERRS); /* * If the primary and shadow AFSR differ, tag the shadow as * the first fault. */ if ((primary_afar != cdp->chd_afar) || (primary_afsr_errs != ch_flt->afsr_errs)) { aflt->flt_stat |= (1ull << C_AFSR_FIRSTFLT_SHIFT); } /* * Check AFSR bits as well as AFSR_EXT bits in order of * the AFAR overwrite priority. Our stored AFSR_EXT value * is expected to be zero for those CPUs which do not have * an AFSR_EXT register. */ for (eccp = ecc_type_to_info; eccp->ec_desc != NULL; eccp++) { if ((eccp->ec_afsr_bit & (ch_flt->afsr_errs & t_afsr_errs)) && ((eccp->ec_flags & aflt->flt_status) != 0)) { cpu_queue_one_event(ch_flt, reason, eccp, cdp); cdp = NULL; t_afsr_errs &= ~eccp->ec_afsr_bit; nevents++; } } /* * If the ME bit is on in the primary AFSR turn all the * error bits on again that may set the ME bit to make * sure we see the ME AFSR error logs. */ if ((primary_afsr & C_AFSR_ME) != 0) t_afsr_errs = (orig_t_afsr_errs & C_AFSR_ALL_ME_ERRS); } #endif /* CHEETAH_PLUS */ if (clop != NULL) cdp = &clop->clo_data; /* * Queue expected errors, error bit and fault type must match * in the ecc_type_to_info table. */ for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; eccp++) { if ((eccp->ec_afsr_bit & t_afsr_errs) != 0 && (eccp->ec_flags & aflt->flt_status) != 0) { #if defined(SERRANO) /* * For FRC/FRU errors on Serrano the afar2 captures * the address and the associated data is * in the shadow logout area. */ if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { if (clop != NULL) cdp = &clop->clo_sdw_data; aflt->flt_addr = ch_flt->afar2; } else { if (clop != NULL) cdp = &clop->clo_data; aflt->flt_addr = primary_afar; } #else /* SERRANO */ aflt->flt_addr = primary_afar; #endif /* SERRANO */ aflt->flt_stat = primary_afsr; ch_flt->afsr_ext = primary_afsr_ext; ch_flt->afsr_errs = primary_afsr_errs; cpu_queue_one_event(ch_flt, reason, eccp, cdp); cdp = NULL; t_afsr_errs &= ~eccp->ec_afsr_bit; nevents++; } } /* * Queue unexpected errors, error bit only match. */ for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; eccp++) { if (eccp->ec_afsr_bit & t_afsr_errs) { #if defined(SERRANO) /* * For FRC/FRU errors on Serrano the afar2 captures * the address and the associated data is * in the shadow logout area. */ if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { if (clop != NULL) cdp = &clop->clo_sdw_data; aflt->flt_addr = ch_flt->afar2; } else { if (clop != NULL) cdp = &clop->clo_data; aflt->flt_addr = primary_afar; } #else /* SERRANO */ aflt->flt_addr = primary_afar; #endif /* SERRANO */ aflt->flt_stat = primary_afsr; ch_flt->afsr_ext = primary_afsr_ext; ch_flt->afsr_errs = primary_afsr_errs; cpu_queue_one_event(ch_flt, reason, eccp, cdp); cdp = NULL; t_afsr_errs &= ~eccp->ec_afsr_bit; nevents++; } } return (nevents); } /* * Return trap type number. */ uint8_t flt_to_trap_type(struct async_flt *aflt) { if (aflt->flt_status & ECC_I_TRAP) return (TRAP_TYPE_ECC_I); if (aflt->flt_status & ECC_D_TRAP) return (TRAP_TYPE_ECC_D); if (aflt->flt_status & ECC_F_TRAP) return (TRAP_TYPE_ECC_F); if (aflt->flt_status & ECC_C_TRAP) return (TRAP_TYPE_ECC_C); if (aflt->flt_status & ECC_DP_TRAP) return (TRAP_TYPE_ECC_DP); if (aflt->flt_status & ECC_IP_TRAP) return (TRAP_TYPE_ECC_IP); if (aflt->flt_status & ECC_ITLB_TRAP) return (TRAP_TYPE_ECC_ITLB); if (aflt->flt_status & ECC_DTLB_TRAP) return (TRAP_TYPE_ECC_DTLB); return (TRAP_TYPE_UNKNOWN); } /* * Decide an error type based on detector and leaky/partner tests. * The following array is used for quick translation - it must * stay in sync with ce_dispact_t. */ static char *cetypes[] = { CE_DISP_DESC_U, CE_DISP_DESC_I, CE_DISP_DESC_PP, CE_DISP_DESC_P, CE_DISP_DESC_L, CE_DISP_DESC_PS, CE_DISP_DESC_S }; char * flt_to_error_type(struct async_flt *aflt) { ce_dispact_t dispact, disp; uchar_t dtcrinfo, ptnrinfo, lkyinfo; /* * The memory payload bundle is shared by some events that do * not perform any classification. For those flt_disp will be * 0 and we will return "unknown". */ if (!ce_disp_inited || !aflt->flt_in_memory || aflt->flt_disp == 0) return (cetypes[CE_DISP_UNKNOWN]); dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); /* * It is also possible that no scrub/classification was performed * by the detector, for instance where a disrupting error logged * in the AFSR while CEEN was off in cpu_deferred_error. */ if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) return (cetypes[CE_DISP_UNKNOWN]); /* * Lookup type in initial classification/action table */ dispact = CE_DISPACT(ce_disp_table, CE_XDIAG_AFARMATCHED(dtcrinfo), CE_XDIAG_STATE(dtcrinfo), CE_XDIAG_CE1SEEN(dtcrinfo), CE_XDIAG_CE2SEEN(dtcrinfo)); /* * A bad lookup is not something to panic production systems for. */ ASSERT(dispact != CE_DISP_BAD); if (dispact == CE_DISP_BAD) return (cetypes[CE_DISP_UNKNOWN]); disp = CE_DISP(dispact); switch (disp) { case CE_DISP_UNKNOWN: case CE_DISP_INTERMITTENT: break; case CE_DISP_POSS_PERS: /* * "Possible persistent" errors to which we have applied a valid * leaky test can be separated into "persistent" or "leaky". */ lkyinfo = CE_XDIAG_LKYINFO(aflt->flt_disp); if (CE_XDIAG_TESTVALID(lkyinfo)) { if (CE_XDIAG_CE1SEEN(lkyinfo) || CE_XDIAG_CE2SEEN(lkyinfo)) disp = CE_DISP_LEAKY; else disp = CE_DISP_PERS; } break; case CE_DISP_POSS_STICKY: /* * Promote "possible sticky" results that have been * confirmed by a partner test to "sticky". Unconfirmed * "possible sticky" events are left at that status - we do not * guess at any bad reader/writer etc status here. */ ptnrinfo = CE_XDIAG_PTNRINFO(aflt->flt_disp); if (CE_XDIAG_TESTVALID(ptnrinfo) && CE_XDIAG_CE1SEEN(ptnrinfo) && CE_XDIAG_CE2SEEN(ptnrinfo)) disp = CE_DISP_STICKY; /* * Promote "possible sticky" results on a uniprocessor * to "sticky" */ if (disp == CE_DISP_POSS_STICKY && CE_XDIAG_SKIPCODE(disp) == CE_XDIAG_SKIP_UNIPROC) disp = CE_DISP_STICKY; break; default: disp = CE_DISP_UNKNOWN; break; } return (cetypes[disp]); } /* * Given the entire afsr, the specific bit to check and a prioritized list of * error bits, determine the validity of the various overwrite priority * features of the AFSR/AFAR: AFAR, ESYND and MSYND, each of which have * different overwrite priorities. * * Given a specific afsr error bit and the entire afsr, there are three cases: * INVALID: The specified bit is lower overwrite priority than some other * error bit which is on in the afsr (or IVU/IVC). * VALID: The specified bit is higher priority than all other error bits * which are on in the afsr. * AMBIGUOUS: Another error bit (or bits) of equal priority to the specified * bit is on in the afsr. */ int afsr_to_overw_status(uint64_t afsr, uint64_t afsr_bit, uint64_t *ow_bits) { uint64_t afsr_ow; while ((afsr_ow = *ow_bits++) != 0) { /* * If bit is in the priority class, check to see if another * bit in the same class is on => ambiguous. Otherwise, * the value is valid. If the bit is not on at this priority * class, but a higher priority bit is on, then the value is * invalid. */ if (afsr_ow & afsr_bit) { /* * If equal pri bit is on, ambiguous. */ if (afsr & (afsr_ow & ~afsr_bit)) return (AFLT_STAT_AMBIGUOUS); return (AFLT_STAT_VALID); } else if (afsr & afsr_ow) break; } /* * We didn't find a match or a higher priority bit was on. Not * finding a match handles the case of invalid AFAR for IVC, IVU. */ return (AFLT_STAT_INVALID); } static int afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit) { #if defined(SERRANO) if (afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) return (afsr_to_overw_status(afsr, afsr_bit, afar2_overwrite)); else #endif /* SERRANO */ return (afsr_to_overw_status(afsr, afsr_bit, afar_overwrite)); } static int afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit) { return (afsr_to_overw_status(afsr, afsr_bit, esynd_overwrite)); } static int afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit) { return (afsr_to_overw_status(afsr, afsr_bit, msynd_overwrite)); } static int afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit) { #ifdef lint cpuid = cpuid; #endif #if defined(CHEETAH_PLUS) /* * The M_SYND overwrite policy is combined with the E_SYND overwrite * policy for Cheetah+ and separate for Panther CPUs. */ if (afsr_bit & C_AFSR_MSYND_ERRS) { if (IS_PANTHER(cpunodes[cpuid].implementation)) return (afsr_to_msynd_status(afsr, afsr_bit)); else return (afsr_to_esynd_status(afsr, afsr_bit)); } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { if (IS_PANTHER(cpunodes[cpuid].implementation)) return (afsr_to_pn_esynd_status(afsr, afsr_bit)); else return (afsr_to_esynd_status(afsr, afsr_bit)); #else /* CHEETAH_PLUS */ if (afsr_bit & C_AFSR_MSYND_ERRS) { return (afsr_to_msynd_status(afsr, afsr_bit)); } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { return (afsr_to_esynd_status(afsr, afsr_bit)); #endif /* CHEETAH_PLUS */ } else { return (AFLT_STAT_INVALID); } } /* * Slave CPU stick synchronization. */ void sticksync_slave(void) { int i; int tries = 0; int64_t tskew; int64_t av_tskew; kpreempt_disable(); /* wait for the master side */ while (stick_sync_cmd != SLAVE_START) ; /* * Synchronization should only take a few tries at most. But in the * odd case where the cpu isn't cooperating we'll keep trying. A cpu * without it's stick synchronized wouldn't be a good citizen. */ while (slave_done == 0) { /* * Time skew calculation. */ av_tskew = tskew = 0; for (i = 0; i < stick_iter; i++) { /* make location hot */ timestamp[EV_A_START] = 0; stick_timestamp(×tamp[EV_A_START]); /* tell the master we're ready */ stick_sync_cmd = MASTER_START; /* and wait */ while (stick_sync_cmd != SLAVE_CONT) ; /* Event B end */ stick_timestamp(×tamp[EV_B_END]); /* calculate time skew */ tskew = ((timestamp[EV_B_END] - timestamp[EV_B_START]) - (timestamp[EV_A_END] - timestamp[EV_A_START])) / 2; /* keep running count */ av_tskew += tskew; } /* for */ /* * Adjust stick for time skew if not within the max allowed; * otherwise we're all done. */ if (stick_iter != 0) av_tskew = av_tskew/stick_iter; if (ABS(av_tskew) > stick_tsk) { /* * If the skew is 1 (the slave's STICK register * is 1 STICK ahead of the master's), stick_adj * could fail to adjust the slave's STICK register * if the STICK read on the slave happens to * align with the increment of the STICK. * Therefore, we increment the skew to 2. */ if (av_tskew == 1) av_tskew++; stick_adj(-av_tskew); } else slave_done = 1; #ifdef DEBUG if (tries < DSYNC_ATTEMPTS) stick_sync_stats[CPU->cpu_id].skew_val[tries] = av_tskew; ++tries; #endif /* DEBUG */ #ifdef lint tries = tries; #endif } /* while */ /* allow the master to finish */ stick_sync_cmd = EVENT_NULL; kpreempt_enable(); } /* * Master CPU side of stick synchronization. * - timestamp end of Event A * - timestamp beginning of Event B */ void sticksync_master(void) { int i; kpreempt_disable(); /* tell the slave we've started */ slave_done = 0; stick_sync_cmd = SLAVE_START; while (slave_done == 0) { for (i = 0; i < stick_iter; i++) { /* wait for the slave */ while (stick_sync_cmd != MASTER_START) ; /* Event A end */ stick_timestamp(×tamp[EV_A_END]); /* make location hot */ timestamp[EV_B_START] = 0; stick_timestamp(×tamp[EV_B_START]); /* tell the slave to continue */ stick_sync_cmd = SLAVE_CONT; } /* for */ /* wait while slave calculates time skew */ while (stick_sync_cmd == SLAVE_CONT) ; } /* while */ kpreempt_enable(); } /* * Cheetah/Cheetah+ have disrupting error for copyback's, so we don't need to * do Spitfire hack of xcall'ing all the cpus to ask to check for them. Also, * in cpu_async_panic_callb, each cpu checks for CPU events on its way to * panic idle. */ /*ARGSUSED*/ void cpu_check_allcpus(struct async_flt *aflt) {} struct kmem_cache *ch_private_cache; /* * Cpu private unitialization. Uninitialize the Ecache scrubber and * deallocate the scrubber data structures and cpu_private data structure. */ void cpu_uninit_private(struct cpu *cp) { cheetah_private_t *chprp = CPU_PRIVATE(cp); ASSERT(chprp); cpu_uninit_ecache_scrub_dr(cp); CPU_PRIVATE(cp) = NULL; ch_err_tl1_paddrs[cp->cpu_id] = 0; kmem_cache_free(ch_private_cache, chprp); cmp_delete_cpu(cp->cpu_id); } /* * Cheetah Cache Scrubbing * * The primary purpose of Cheetah cache scrubbing is to reduce the exposure * of E$ tags, D$ data, and I$ data to cosmic ray events since they are not * protected by either parity or ECC. * * We currently default the E$ and D$ scan rate to 100 (scan 10% of the * cache per second). Due to the the specifics of how the I$ control * logic works with respect to the ASI used to scrub I$ lines, the entire * I$ is scanned at once. */ /* * Tuneables to enable and disable the scrubbing of the caches, and to tune * scrubbing behavior. These may be changed via /etc/system or using mdb * on a running system. */ int dcache_scrub_enable = 1; /* D$ scrubbing is on by default */ /* * The following are the PIL levels that the softints/cross traps will fire at. */ uint_t ecache_scrub_pil = PIL_9; /* E$ scrub PIL for cross traps */ uint_t dcache_scrub_pil = PIL_9; /* D$ scrub PIL for cross traps */ uint_t icache_scrub_pil = PIL_9; /* I$ scrub PIL for cross traps */ #if defined(JALAPENO) /* * Due to several errata (82, 85, 86), we don't enable the L2$ scrubber * on Jalapeno. */ int ecache_scrub_enable = 0; #else /* JALAPENO */ /* * With all other cpu types, E$ scrubbing is on by default */ int ecache_scrub_enable = 1; #endif /* JALAPENO */ #if defined(CHEETAH_PLUS) || defined(JALAPENO) || defined(SERRANO) /* * The I$ scrubber tends to cause latency problems for real-time SW, so it * is disabled by default on non-Cheetah systems */ int icache_scrub_enable = 0; /* * Tuneables specifying the scrub calls per second and the scan rate * for each cache * * The cyclic times are set during boot based on the following values. * Changing these values in mdb after this time will have no effect. If * a different value is desired, it must be set in /etc/system before a * reboot. */ int ecache_calls_a_sec = 1; int dcache_calls_a_sec = 2; int icache_calls_a_sec = 2; int ecache_scan_rate_idle = 1; int ecache_scan_rate_busy = 1; int dcache_scan_rate_idle = 1; int dcache_scan_rate_busy = 1; int icache_scan_rate_idle = 1; int icache_scan_rate_busy = 1; #else /* CHEETAH_PLUS || JALAPENO || SERRANO */ int icache_scrub_enable = 1; /* I$ scrubbing is on by default */ int ecache_calls_a_sec = 100; /* E$ scrub calls per seconds */ int dcache_calls_a_sec = 100; /* D$ scrub calls per seconds */ int icache_calls_a_sec = 100; /* I$ scrub calls per seconds */ int ecache_scan_rate_idle = 100; /* E$ scan rate (in tenths of a %) */ int ecache_scan_rate_busy = 100; /* E$ scan rate (in tenths of a %) */ int dcache_scan_rate_idle = 100; /* D$ scan rate (in tenths of a %) */ int dcache_scan_rate_busy = 100; /* D$ scan rate (in tenths of a %) */ int icache_scan_rate_idle = 100; /* I$ scan rate (in tenths of a %) */ int icache_scan_rate_busy = 100; /* I$ scan rate (in tenths of a %) */ #endif /* CHEETAH_PLUS || JALAPENO || SERRANO */ /* * In order to scrub on offline cpus, a cross trap is sent. The handler will * increment the outstanding request counter and schedule a softint to run * the scrubber. */ extern xcfunc_t cache_scrubreq_tl1; /* * These are the softint functions for each cache scrubber */ static uint_t scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2); static uint_t scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2); static uint_t scrub_icache_line_intr(caddr_t arg1, caddr_t arg2); /* * The cache scrub info table contains cache specific information * and allows for some of the scrub code to be table driven, reducing * duplication of cache similar code. * * This table keeps a copy of the value in the calls per second variable * (?cache_calls_a_sec). This makes it much more difficult for someone * to cause us problems (for example, by setting ecache_calls_a_sec to 0 in * mdb in a misguided attempt to disable the scrubber). */ struct scrub_info { int *csi_enable; /* scrubber enable flag */ int csi_freq; /* scrubber calls per second */ int csi_index; /* index to chsm_outstanding[] */ uint64_t csi_inum; /* scrubber interrupt number */ cyclic_id_t csi_omni_cyc_id; /* omni cyclic ID */ cyclic_id_t csi_offline_cyc_id; /* offline cyclic ID */ char csi_name[3]; /* cache name for this scrub entry */ } cache_scrub_info[] = { { &ecache_scrub_enable, 0, CACHE_SCRUBBER_INFO_E, 0, 0, 0, "E$"}, { &dcache_scrub_enable, 0, CACHE_SCRUBBER_INFO_D, 0, 0, 0, "D$"}, { &icache_scrub_enable, 0, CACHE_SCRUBBER_INFO_I, 0, 0, 0, "I$"} }; /* * If scrubbing is enabled, increment the outstanding request counter. If it * is 1 (meaning there were no previous requests outstanding), call * setsoftint_tl1 through xt_one_unchecked, which eventually ends up doing * a self trap. */ static void do_scrub(struct scrub_info *csi) { ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); int index = csi->csi_index; uint32_t *outstanding = &csmp->chsm_outstanding[index]; if (*(csi->csi_enable) && (csmp->chsm_enable[index])) { if (atomic_inc_32_nv(outstanding) == 1) { xt_one_unchecked(CPU->cpu_id, setsoftint_tl1, csi->csi_inum, 0); } } } /* * Omni cyclics don't fire on offline cpus, so we use another cyclic to * cross-trap the offline cpus. */ static void do_scrub_offline(struct scrub_info *csi) { ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); if (CPUSET_ISNULL(cpu_offline_set)) { /* * No offline cpus - nothing to do */ return; } if (*(csi->csi_enable) && (csmp->chsm_enable[csi->csi_index])) { xt_some(cpu_offline_set, cache_scrubreq_tl1, csi->csi_inum, csi->csi_index); } } /* * This is the initial setup for the scrubber cyclics - it sets the * interrupt level, frequency, and function to call. */ /*ARGSUSED*/ static void cpu_scrub_cyclic_setup(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when) { struct scrub_info *csi = (struct scrub_info *)arg; ASSERT(csi != NULL); hdlr->cyh_func = (cyc_func_t)do_scrub; hdlr->cyh_level = CY_LOW_LEVEL; hdlr->cyh_arg = arg; when->cyt_when = 0; /* Start immediately */ when->cyt_interval = NANOSEC / csi->csi_freq; } /* * Initialization for cache scrubbing. * This routine is called AFTER all cpus have had cpu_init_private called * to initialize their private data areas. */ void cpu_init_cache_scrub(void) { int i; struct scrub_info *csi; cyc_omni_handler_t omni_hdlr; cyc_handler_t offline_hdlr; cyc_time_t when; /* * save away the maximum number of lines for the D$ */ dcache_nlines = dcache_size / dcache_linesize; /* * register the softints for the cache scrubbing */ cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_inum = add_softintr(ecache_scrub_pil, scrub_ecache_line_intr, (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_E], SOFTINT_MT); cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_freq = ecache_calls_a_sec; cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_inum = add_softintr(dcache_scrub_pil, scrub_dcache_line_intr, (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_D], SOFTINT_MT); cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_freq = dcache_calls_a_sec; cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_inum = add_softintr(icache_scrub_pil, scrub_icache_line_intr, (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_I], SOFTINT_MT); cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_freq = icache_calls_a_sec; /* * start the scrubbing for all the caches */ mutex_enter(&cpu_lock); for (i = 0; i < CACHE_SCRUBBER_COUNT; i++) { csi = &cache_scrub_info[i]; if (!(*csi->csi_enable)) continue; /* * force the following to be true: * 1 <= calls_a_sec <= hz */ if (csi->csi_freq > hz) { cmn_err(CE_NOTE, "%s scrub calls_a_sec set too high " "(%d); resetting to hz (%d)", csi->csi_name, csi->csi_freq, hz); csi->csi_freq = hz; } else if (csi->csi_freq < 1) { cmn_err(CE_NOTE, "%s scrub calls_a_sec set too low " "(%d); resetting to 1", csi->csi_name, csi->csi_freq); csi->csi_freq = 1; } omni_hdlr.cyo_online = cpu_scrub_cyclic_setup; omni_hdlr.cyo_offline = NULL; omni_hdlr.cyo_arg = (void *)csi; offline_hdlr.cyh_func = (cyc_func_t)do_scrub_offline; offline_hdlr.cyh_arg = (void *)csi; offline_hdlr.cyh_level = CY_LOW_LEVEL; when.cyt_when = 0; /* Start immediately */ when.cyt_interval = NANOSEC / csi->csi_freq; csi->csi_omni_cyc_id = cyclic_add_omni(&omni_hdlr); csi->csi_offline_cyc_id = cyclic_add(&offline_hdlr, &when); } register_cpu_setup_func(cpu_scrub_cpu_setup, NULL); mutex_exit(&cpu_lock); } /* * Indicate that the specified cpu is idle. */ void cpu_idle_ecache_scrub(struct cpu *cp) { if (CPU_PRIVATE(cp) != NULL) { ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); csmp->chsm_ecache_busy = ECACHE_CPU_IDLE; } } /* * Indicate that the specified cpu is busy. */ void cpu_busy_ecache_scrub(struct cpu *cp) { if (CPU_PRIVATE(cp) != NULL) { ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); csmp->chsm_ecache_busy = ECACHE_CPU_BUSY; } } /* * Initialization for cache scrubbing for the specified cpu. */ void cpu_init_ecache_scrub_dr(struct cpu *cp) { ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); int cpuid = cp->cpu_id; /* initialize the number of lines in the caches */ csmp->chsm_ecache_nlines = cpunodes[cpuid].ecache_size / cpunodes[cpuid].ecache_linesize; csmp->chsm_icache_nlines = CPU_PRIVATE_VAL(cp, chpr_icache_size) / CPU_PRIVATE_VAL(cp, chpr_icache_linesize); /* * do_scrub() and do_scrub_offline() check both the global * ?cache_scrub_enable and this per-cpu enable variable. All scrubbers * check this value before scrubbing. Currently, we use it to * disable the E$ scrubber on multi-core cpus or while running at * slowed speed. For now, just turn everything on and allow * cpu_init_private() to change it if necessary. */ csmp->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1; csmp->chsm_enable[CACHE_SCRUBBER_INFO_D] = 1; csmp->chsm_enable[CACHE_SCRUBBER_INFO_I] = 1; cpu_busy_ecache_scrub(cp); } /* * Un-initialization for cache scrubbing for the specified cpu. */ static void cpu_uninit_ecache_scrub_dr(struct cpu *cp) { ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); /* * un-initialize bookkeeping for cache scrubbing */ bzero(csmp, sizeof (ch_scrub_misc_t)); cpu_idle_ecache_scrub(cp); } /* * Called periodically on each CPU to scrub the D$. */ static void scrub_dcache(int how_many) { int i; ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D]; /* * scrub the desired number of lines */ for (i = 0; i < how_many; i++) { /* * scrub a D$ line */ dcache_inval_line(index); /* * calculate the next D$ line to scrub, assumes * that dcache_nlines is a power of 2 */ index = (index + 1) & (dcache_nlines - 1); } /* * set the scrub index for the next visit */ csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D] = index; } /* * Handler for D$ scrub inum softint. Call scrub_dcache until * we decrement the outstanding request count to zero. */ /*ARGSUSED*/ static uint_t scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2) { int i; int how_many; int outstanding; ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_D]; struct scrub_info *csi = (struct scrub_info *)arg1; int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? dcache_scan_rate_idle : dcache_scan_rate_busy; /* * The scan rates are expressed in units of tenths of a * percent. A scan rate of 1000 (100%) means the whole * cache is scanned every second. */ how_many = (dcache_nlines * scan_rate) / (1000 * csi->csi_freq); do { outstanding = *countp; for (i = 0; i < outstanding; i++) { scrub_dcache(how_many); } } while (atomic_add_32_nv(countp, -outstanding)); return (DDI_INTR_CLAIMED); } /* * Called periodically on each CPU to scrub the I$. The I$ is scrubbed * by invalidating lines. Due to the characteristics of the ASI which * is used to invalidate an I$ line, the entire I$ must be invalidated * vs. an individual I$ line. */ static void scrub_icache(int how_many) { int i; ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I]; int icache_nlines = csmp->chsm_icache_nlines; /* * scrub the desired number of lines */ for (i = 0; i < how_many; i++) { /* * since the entire I$ must be scrubbed at once, * wait until the index wraps to zero to invalidate * the entire I$ */ if (index == 0) { icache_inval_all(); } /* * calculate the next I$ line to scrub, assumes * that chsm_icache_nlines is a power of 2 */ index = (index + 1) & (icache_nlines - 1); } /* * set the scrub index for the next visit */ csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I] = index; } /* * Handler for I$ scrub inum softint. Call scrub_icache until * we decrement the outstanding request count to zero. */ /*ARGSUSED*/ static uint_t scrub_icache_line_intr(caddr_t arg1, caddr_t arg2) { int i; int how_many; int outstanding; ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_I]; struct scrub_info *csi = (struct scrub_info *)arg1; int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? icache_scan_rate_idle : icache_scan_rate_busy; int icache_nlines = csmp->chsm_icache_nlines; /* * The scan rates are expressed in units of tenths of a * percent. A scan rate of 1000 (100%) means the whole * cache is scanned every second. */ how_many = (icache_nlines * scan_rate) / (1000 * csi->csi_freq); do { outstanding = *countp; for (i = 0; i < outstanding; i++) { scrub_icache(how_many); } } while (atomic_add_32_nv(countp, -outstanding)); return (DDI_INTR_CLAIMED); } /* * Called periodically on each CPU to scrub the E$. */ static void scrub_ecache(int how_many) { ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); int i; int cpuid = CPU->cpu_id; int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E]; int nlines = csmp->chsm_ecache_nlines; int linesize = cpunodes[cpuid].ecache_linesize; int ec_set_size = cpu_ecache_set_size(CPU); /* * scrub the desired number of lines */ for (i = 0; i < how_many; i++) { /* * scrub the E$ line */ ecache_flush_line(ecache_flushaddr + (index * linesize), ec_set_size); /* * calculate the next E$ line to scrub based on twice * the number of E$ lines (to displace lines containing * flush area data), assumes that the number of lines * is a power of 2 */ index = (index + 1) & ((nlines << 1) - 1); } /* * set the ecache scrub index for the next visit */ csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E] = index; } /* * Handler for E$ scrub inum softint. Call the E$ scrubber until * we decrement the outstanding request count to zero. * * Due to interactions with cpu_scrub_cpu_setup(), the outstanding count may * become negative after the atomic_add_32_nv(). This is not a problem, as * the next trip around the loop won't scrub anything, and the next add will * reset the count back to zero. */ /*ARGSUSED*/ static uint_t scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2) { int i; int how_many; int outstanding; ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_E]; struct scrub_info *csi = (struct scrub_info *)arg1; int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? ecache_scan_rate_idle : ecache_scan_rate_busy; int ecache_nlines = csmp->chsm_ecache_nlines; /* * The scan rates are expressed in units of tenths of a * percent. A scan rate of 1000 (100%) means the whole * cache is scanned every second. */ how_many = (ecache_nlines * scan_rate) / (1000 * csi->csi_freq); do { outstanding = *countp; for (i = 0; i < outstanding; i++) { scrub_ecache(how_many); } } while (atomic_add_32_nv(countp, -outstanding)); return (DDI_INTR_CLAIMED); } /* * Timeout function to reenable CE */ static void cpu_delayed_check_ce_errors(void *arg) { if (taskq_dispatch(ch_check_ce_tq, cpu_check_ce_errors, arg, TQ_NOSLEEP) == TASKQID_INVALID) { (void) timeout(cpu_delayed_check_ce_errors, arg, drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); } } /* * CE Deferred Re-enable after trap. * * When the CPU gets a disrupting trap for any of the errors * controlled by the CEEN bit, CEEN is disabled in the trap handler * immediately. To eliminate the possibility of multiple CEs causing * recursive stack overflow in the trap handler, we cannot * reenable CEEN while still running in the trap handler. Instead, * after a CE is logged on a CPU, we schedule a timeout function, * cpu_check_ce_errors(), to trigger after cpu_ceen_delay_secs * seconds. This function will check whether any further CEs * have occurred on that CPU, and if none have, will reenable CEEN. * * If further CEs have occurred while CEEN is disabled, another * timeout will be scheduled. This is to ensure that the CPU can * make progress in the face of CE 'storms', and that it does not * spend all its time logging CE errors. */ static void cpu_check_ce_errors(void *arg) { int cpuid = (int)(uintptr_t)arg; cpu_t *cp; /* * We acquire cpu_lock. */ ASSERT(curthread->t_pil == 0); /* * verify that the cpu is still around, DR * could have got there first ... */ mutex_enter(&cpu_lock); cp = cpu_get(cpuid); if (cp == NULL) { mutex_exit(&cpu_lock); return; } /* * make sure we don't migrate across CPUs * while checking our CE status. */ kpreempt_disable(); /* * If we are running on the CPU that got the * CE, we can do the checks directly. */ if (cp->cpu_id == CPU->cpu_id) { mutex_exit(&cpu_lock); cpu_check_ce(TIMEOUT_CEEN_CHECK, 0, 0, 0); kpreempt_enable(); return; } kpreempt_enable(); /* * send an x-call to get the CPU that originally * got the CE to do the necessary checks. If we can't * send the x-call, reschedule the timeout, otherwise we * lose CEEN forever on that CPU. */ if (CPU_XCALL_READY(cp->cpu_id) && (!(cp->cpu_flags & CPU_QUIESCED))) { xc_one(cp->cpu_id, (xcfunc_t *)cpu_check_ce, TIMEOUT_CEEN_CHECK, 0); mutex_exit(&cpu_lock); } else { /* * When the CPU is not accepting xcalls, or * the processor is offlined, we don't want to * incur the extra overhead of trying to schedule the * CE timeout indefinitely. However, we don't want to lose * CE checking forever. * * Keep rescheduling the timeout, accepting the additional * overhead as the cost of correctness in the case where we get * a CE, disable CEEN, offline the CPU during the * the timeout interval, and then online it at some * point in the future. This is unlikely given the short * cpu_ceen_delay_secs. */ mutex_exit(&cpu_lock); (void) timeout(cpu_delayed_check_ce_errors, (void *)(uintptr_t)cp->cpu_id, drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); } } /* * This routine will check whether CEs have occurred while * CEEN is disabled. Any CEs detected will be logged and, if * possible, scrubbed. * * The memscrubber will also use this routine to clear any errors * caused by its scrubbing with CEEN disabled. * * flag == SCRUBBER_CEEN_CHECK * called from memscrubber, just check/scrub, no reset * paddr physical addr. for start of scrub pages * vaddr virtual addr. for scrub area * psz page size of area to be scrubbed * * flag == TIMEOUT_CEEN_CHECK * timeout function has triggered, reset timeout or CEEN * * Note: We must not migrate cpus during this function. This can be * achieved by one of: * - invoking as target of an x-call in which case we're at XCALL_PIL * The flag value must be first xcall argument. * - disabling kernel preemption. This should be done for very short * periods so is not suitable for SCRUBBER_CEEN_CHECK where we might * scrub an extended area with cpu_check_block. The call for * TIMEOUT_CEEN_CHECK uses this so cpu_check_ce must be kept * brief for this case. * - binding to a cpu, eg with thread_affinity_set(). This is used * in the SCRUBBER_CEEN_CHECK case, but is not practical for * the TIMEOUT_CEEN_CHECK because both need cpu_lock. */ void cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t psz) { ch_cpu_errors_t cpu_error_regs; uint64_t ec_err_enable; uint64_t page_offset; /* Read AFSR */ get_cpu_error_state(&cpu_error_regs); /* * If no CEEN errors have occurred during the timeout * interval, it is safe to re-enable CEEN and exit. */ if (((cpu_error_regs.afsr & C_AFSR_CECC_ERRS) | (cpu_error_regs.afsr_ext & C_AFSR_EXT_CECC_ERRS)) == 0) { if (flag == TIMEOUT_CEEN_CHECK && !((ec_err_enable = get_error_enable()) & EN_REG_CEEN)) set_error_enable(ec_err_enable | EN_REG_CEEN); return; } /* * Ensure that CEEN was not reenabled (maybe by DR) before * we log/clear the error. */ if ((ec_err_enable = get_error_enable()) & EN_REG_CEEN) set_error_enable(ec_err_enable & ~EN_REG_CEEN); /* * log/clear the CE. If CE_CEEN_DEFER is passed, the * timeout will be rescheduled when the error is logged. */ if (!((cpu_error_regs.afsr & cpu_ce_not_deferred) | (cpu_error_regs.afsr_ext & cpu_ce_not_deferred_ext))) cpu_ce_detected(&cpu_error_regs, CE_CEEN_DEFER | CE_CEEN_TIMEOUT); else cpu_ce_detected(&cpu_error_regs, CE_CEEN_TIMEOUT); /* * If the memory scrubber runs while CEEN is * disabled, (or if CEEN is disabled during the * scrub as a result of a CE being triggered by * it), the range being scrubbed will not be * completely cleaned. If there are multiple CEs * in the range at most two of these will be dealt * with, (one by the trap handler and one by the * timeout). It is also possible that none are dealt * with, (CEEN disabled and another CE occurs before * the timeout triggers). So to ensure that the * memory is actually scrubbed, we have to access each * memory location in the range and then check whether * that access causes a CE. */ if (flag == SCRUBBER_CEEN_CHECK && va) { if ((cpu_error_regs.afar >= pa) && (cpu_error_regs.afar < (pa + psz))) { /* * Force a load from physical memory for each * 64-byte block, then check AFSR to determine * whether this access caused an error. * * This is a slow way to do a scrub, but as it will * only be invoked when the memory scrubber actually * triggered a CE, it should not happen too * frequently. * * cut down what we need to check as the scrubber * has verified up to AFAR, so get it's offset * into the page and start there. */ page_offset = (uint64_t)(cpu_error_regs.afar & (psz - 1)); va = (caddr_t)(va + (P2ALIGN(page_offset, 64))); psz -= (uint_t)(P2ALIGN(page_offset, 64)); cpu_check_block((caddr_t)(P2ALIGN((uint64_t)va, 64)), psz); } } /* * Reset error enable if this CE is not masked. */ if ((flag == TIMEOUT_CEEN_CHECK) && (cpu_error_regs.afsr & cpu_ce_not_deferred)) set_error_enable(ec_err_enable | EN_REG_CEEN); } /* * Attempt a cpu logout for an error that we did not trap for, such * as a CE noticed with CEEN off. It is assumed that we are still running * on the cpu that took the error and that we cannot migrate. Returns * 0 on success, otherwise nonzero. */ static int cpu_ce_delayed_ec_logout(uint64_t afar) { ch_cpu_logout_t *clop; if (CPU_PRIVATE(CPU) == NULL) return (0); clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); if (atomic_cas_64(&clop->clo_data.chd_afar, LOGOUT_INVALID, afar) != LOGOUT_INVALID) return (0); cpu_delayed_logout(afar, clop); return (1); } /* * We got an error while CEEN was disabled. We * need to clean up after it and log whatever * information we have on the CE. */ void cpu_ce_detected(ch_cpu_errors_t *cpu_error_regs, int flag) { ch_async_flt_t ch_flt; struct async_flt *aflt; char pr_reason[MAX_REASON_STRING]; bzero(&ch_flt, sizeof (ch_async_flt_t)); ch_flt.flt_trapped_ce = flag; aflt = (struct async_flt *)&ch_flt; aflt->flt_stat = cpu_error_regs->afsr & C_AFSR_MASK; ch_flt.afsr_ext = cpu_error_regs->afsr_ext; ch_flt.afsr_errs = (cpu_error_regs->afsr_ext & C_AFSR_EXT_ALL_ERRS) | (cpu_error_regs->afsr & C_AFSR_ALL_ERRS); aflt->flt_addr = cpu_error_regs->afar; #if defined(SERRANO) ch_flt.afar2 = cpu_error_regs->afar2; #endif /* SERRANO */ aflt->flt_pc = NULL; aflt->flt_priv = ((cpu_error_regs->afsr & C_AFSR_PRIV) != 0); aflt->flt_tl = 0; aflt->flt_panic = 0; cpu_log_and_clear_ce(&ch_flt); /* * check if we caused any errors during cleanup */ if (clear_errors(&ch_flt)) { pr_reason[0] = '\0'; (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, NULL); } } /* * Log/clear CEEN-controlled disrupting errors */ static void cpu_log_and_clear_ce(ch_async_flt_t *ch_flt) { struct async_flt *aflt; uint64_t afsr, afsr_errs; ch_cpu_logout_t *clop; char pr_reason[MAX_REASON_STRING]; on_trap_data_t *otp = curthread->t_ontrap; aflt = (struct async_flt *)ch_flt; afsr = aflt->flt_stat; afsr_errs = ch_flt->afsr_errs; aflt->flt_id = gethrtime_waitfree(); aflt->flt_bus_id = getprocessorid(); aflt->flt_inst = CPU->cpu_id; aflt->flt_prot = AFLT_PROT_NONE; aflt->flt_class = CPU_FAULT; aflt->flt_status = ECC_C_TRAP; pr_reason[0] = '\0'; /* * Get the CPU log out info for Disrupting Trap. */ if (CPU_PRIVATE(CPU) == NULL) { clop = NULL; ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; } else { clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); } if (clop && ch_flt->flt_trapped_ce & CE_CEEN_TIMEOUT) { ch_cpu_errors_t cpu_error_regs; get_cpu_error_state(&cpu_error_regs); (void) cpu_ce_delayed_ec_logout(cpu_error_regs.afar); clop->clo_data.chd_afsr = cpu_error_regs.afsr; clop->clo_data.chd_afar = cpu_error_regs.afar; clop->clo_data.chd_afsr_ext = cpu_error_regs.afsr_ext; clop->clo_sdw_data.chd_afsr = cpu_error_regs.shadow_afsr; clop->clo_sdw_data.chd_afar = cpu_error_regs.shadow_afar; clop->clo_sdw_data.chd_afsr_ext = cpu_error_regs.shadow_afsr_ext; #if defined(SERRANO) clop->clo_data.chd_afar2 = cpu_error_regs.afar2; #endif /* SERRANO */ ch_flt->flt_data_incomplete = 1; /* * The logging/clear code expects AFSR/AFAR to be cleared. * The trap handler does it for CEEN enabled errors * so we need to do it here. */ set_cpu_error_state(&cpu_error_regs); } #if defined(JALAPENO) || defined(SERRANO) /* * FRC: Can't scrub memory as we don't have AFAR for Jalapeno. * For Serrano, even thou we do have the AFAR, we still do the * scrub on the RCE side since that's where the error type can * be properly classified as intermittent, persistent, etc. * * CE/RCE: If error is in memory and AFAR is valid, scrub the memory. * Must scrub memory before cpu_queue_events, as scrubbing memory sets * the flt_status bits. */ if ((afsr & (C_AFSR_CE|C_AFSR_RCE)) && (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_RCE)))) { cpu_ce_scrub_mem_err(aflt, B_TRUE); } #else /* JALAPENO || SERRANO */ /* * CE/EMC: If error is in memory and AFAR is valid, scrub the memory. * Must scrub memory before cpu_queue_events, as scrubbing memory sets * the flt_status bits. */ if (afsr & (C_AFSR_CE|C_AFSR_EMC)) { if (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_EMC))) { cpu_ce_scrub_mem_err(aflt, B_TRUE); } } #endif /* JALAPENO || SERRANO */ /* * Update flt_prot if this error occurred under on_trap protection. */ if (otp != NULL && (otp->ot_prot & OT_DATA_EC)) aflt->flt_prot = AFLT_PROT_EC; /* * Queue events on the async event queue, one event per error bit. */ if (cpu_queue_events(ch_flt, pr_reason, afsr_errs, clop) == 0 || (afsr_errs & (C_AFSR_CECC_ERRS | C_AFSR_EXT_CECC_ERRS)) == 0) { ch_flt->flt_type = CPU_INV_AFSR; cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); } /* * Zero out + invalidate CPU logout. */ if (clop) { bzero(clop, sizeof (ch_cpu_logout_t)); clop->clo_data.chd_afar = LOGOUT_INVALID; } /* * If either a CPC, WDC or EDC error has occurred while CEEN * was disabled, we need to flush either the entire * E$ or an E$ line. */ #if defined(JALAPENO) || defined(SERRANO) if (afsr & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_CPU | C_AFSR_WDC)) #else /* JALAPENO || SERRANO */ if (afsr_errs & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_WDC | C_AFSR_L3_EDC | C_AFSR_L3_CPC | C_AFSR_L3_WDC)) #endif /* JALAPENO || SERRANO */ cpu_error_ecache_flush(ch_flt); } /* * depending on the error type, we determine whether we * need to flush the entire ecache or just a line. */ static int cpu_error_ecache_flush_required(ch_async_flt_t *ch_flt) { struct async_flt *aflt; uint64_t afsr; uint64_t afsr_errs = ch_flt->afsr_errs; aflt = (struct async_flt *)ch_flt; afsr = aflt->flt_stat; /* * If we got multiple errors, no point in trying * the individual cases, just flush the whole cache */ if (afsr & C_AFSR_ME) { return (ECACHE_FLUSH_ALL); } /* * If either a CPC, WDC or EDC error has occurred while CEEN * was disabled, we need to flush entire E$. We can't just * flush the cache line affected as the ME bit * is not set when multiple correctable errors of the same * type occur, so we might have multiple CPC or EDC errors, * with only the first recorded. */ #if defined(JALAPENO) || defined(SERRANO) if (afsr & (C_AFSR_CPC | C_AFSR_CPU | C_AFSR_EDC | C_AFSR_WDC)) { #else /* JALAPENO || SERRANO */ if (afsr_errs & (C_AFSR_CPC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_L3_CPC | C_AFSR_L3_EDC | C_AFSR_L3_WDC)) { #endif /* JALAPENO || SERRANO */ return (ECACHE_FLUSH_ALL); } #if defined(JALAPENO) || defined(SERRANO) /* * If only UE or RUE is set, flush the Ecache line, otherwise * flush the entire Ecache. */ if (afsr & (C_AFSR_UE|C_AFSR_RUE)) { if ((afsr & C_AFSR_ALL_ERRS) == C_AFSR_UE || (afsr & C_AFSR_ALL_ERRS) == C_AFSR_RUE) { return (ECACHE_FLUSH_LINE); } else { return (ECACHE_FLUSH_ALL); } } #else /* JALAPENO || SERRANO */ /* * If UE only is set, flush the Ecache line, otherwise * flush the entire Ecache. */ if (afsr_errs & C_AFSR_UE) { if ((afsr_errs & (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_UE) { return (ECACHE_FLUSH_LINE); } else { return (ECACHE_FLUSH_ALL); } } #endif /* JALAPENO || SERRANO */ /* * EDU: If EDU only is set, flush the ecache line, otherwise * flush the entire Ecache. */ if (afsr_errs & (C_AFSR_EDU | C_AFSR_L3_EDU)) { if (((afsr_errs & ~C_AFSR_EDU) == 0) || ((afsr_errs & ~C_AFSR_L3_EDU) == 0)) { return (ECACHE_FLUSH_LINE); } else { return (ECACHE_FLUSH_ALL); } } /* * BERR: If BERR only is set, flush the Ecache line, otherwise * flush the entire Ecache. */ if (afsr_errs & C_AFSR_BERR) { if ((afsr_errs & ~C_AFSR_BERR) == 0) { return (ECACHE_FLUSH_LINE); } else { return (ECACHE_FLUSH_ALL); } } return (0); } void cpu_error_ecache_flush(ch_async_flt_t *ch_flt) { int ecache_flush_flag = cpu_error_ecache_flush_required(ch_flt); /* * Flush Ecache line or entire Ecache based on above checks. */ if (ecache_flush_flag == ECACHE_FLUSH_ALL) cpu_flush_ecache(); else if (ecache_flush_flag == ECACHE_FLUSH_LINE) { cpu_flush_ecache_line(ch_flt); } } /* * Extract the PA portion from the E$ tag. */ uint64_t cpu_ectag_to_pa(int setsize, uint64_t tag) { if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) return (JG_ECTAG_TO_PA(setsize, tag)); else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) return (PN_L3TAG_TO_PA(tag)); else return (CH_ECTAG_TO_PA(setsize, tag)); } /* * Convert the E$ tag PA into an E$ subblock index. */ int cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr) { if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) return (JG_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) /* Panther has only one subblock per line */ return (0); else return (CH_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); } /* * All subblocks in an E$ line must be invalid for * the line to be invalid. */ int cpu_ectag_line_invalid(int cachesize, uint64_t tag) { if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) return (JG_ECTAG_LINE_INVALID(cachesize, tag)); else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) return (PN_L3_LINE_INVALID(tag)); else return (CH_ECTAG_LINE_INVALID(cachesize, tag)); } /* * Extract state bits for a subblock given the tag. Note that for Panther * this works on both l2 and l3 tags. */ int cpu_ectag_pa_to_subblk_state(int cachesize, uint64_t subaddr, uint64_t tag) { if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) return (JG_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) return (tag & CH_ECSTATE_MASK); else return (CH_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); } /* * Cpu specific initialization. */ void cpu_mp_init(void) { #ifdef CHEETAHPLUS_ERRATUM_25 if (cheetah_sendmondo_recover) { cheetah_nudge_init(); } #endif } void cpu_ereport_post(struct async_flt *aflt) { char *cpu_type, buf[FM_MAX_CLASS]; nv_alloc_t *nva = NULL; nvlist_t *ereport, *detector, *resource; errorq_elem_t *eqep; ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; char unum[UNUM_NAMLEN]; int synd_code; uint8_t msg_type; plat_ecc_ch_async_flt_t plat_ecc_ch_flt; if (aflt->flt_panic || panicstr) { eqep = errorq_reserve(ereport_errorq); if (eqep == NULL) return; ereport = errorq_elem_nvl(ereport_errorq, eqep); nva = errorq_elem_nva(ereport_errorq, eqep); } else { ereport = fm_nvlist_create(nva); } /* * Create the scheme "cpu" FMRI. */ detector = fm_nvlist_create(nva); resource = fm_nvlist_create(nva); switch (cpunodes[aflt->flt_inst].implementation) { case CHEETAH_IMPL: cpu_type = FM_EREPORT_CPU_USIII; break; case CHEETAH_PLUS_IMPL: cpu_type = FM_EREPORT_CPU_USIIIplus; break; case JALAPENO_IMPL: cpu_type = FM_EREPORT_CPU_USIIIi; break; case SERRANO_IMPL: cpu_type = FM_EREPORT_CPU_USIIIiplus; break; case JAGUAR_IMPL: cpu_type = FM_EREPORT_CPU_USIV; break; case PANTHER_IMPL: cpu_type = FM_EREPORT_CPU_USIVplus; break; default: cpu_type = FM_EREPORT_CPU_UNSUPPORTED; break; } cpu_fmri_cpu_set(detector, aflt->flt_inst); /* * Encode all the common data into the ereport. */ (void) snprintf(buf, FM_MAX_CLASS, "%s.%s.%s", FM_ERROR_CPU, cpu_type, aflt->flt_erpt_class); fm_ereport_set(ereport, FM_EREPORT_VERSION, buf, fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, FM_ENA_FMT1), detector, NULL); /* * Encode the error specific data that was saved in * the async_flt structure into the ereport. */ cpu_payload_add_aflt(aflt, ereport, resource, &plat_ecc_ch_flt.ecaf_afar_status, &plat_ecc_ch_flt.ecaf_synd_status); if (aflt->flt_panic || panicstr) { errorq_commit(ereport_errorq, eqep, ERRORQ_SYNC); } else { (void) fm_ereport_post(ereport, EVCH_TRYHARD); fm_nvlist_destroy(ereport, FM_NVA_FREE); fm_nvlist_destroy(detector, FM_NVA_FREE); fm_nvlist_destroy(resource, FM_NVA_FREE); } /* * Send the enhanced error information (plat_ecc_error2_data_t) * to the SC olny if it can process it. */ if (&plat_ecc_capability_sc_get && plat_ecc_capability_sc_get(PLAT_ECC_ERROR2_MESSAGE)) { msg_type = cpu_flt_bit_to_plat_error(aflt); if (msg_type != PLAT_ECC_ERROR2_NONE) { /* * If afar status is not invalid do a unum lookup. */ if (plat_ecc_ch_flt.ecaf_afar_status != AFLT_STAT_INVALID) { synd_code = synd_to_synd_code( plat_ecc_ch_flt.ecaf_synd_status, aflt->flt_synd, ch_flt->flt_bit); (void) cpu_get_mem_unum_synd(synd_code, aflt, unum); } else { unum[0] = '\0'; } plat_ecc_ch_flt.ecaf_sdw_afar = ch_flt->flt_sdw_afar; plat_ecc_ch_flt.ecaf_sdw_afsr = ch_flt->flt_sdw_afsr; plat_ecc_ch_flt.ecaf_afsr_ext = ch_flt->afsr_ext; plat_ecc_ch_flt.ecaf_sdw_afsr_ext = ch_flt->flt_sdw_afsr_ext; if (&plat_log_fruid_error2) plat_log_fruid_error2(msg_type, unum, aflt, &plat_ecc_ch_flt); } } } void cpu_run_bus_error_handlers(struct async_flt *aflt, int expected) { int status; ddi_fm_error_t de; bzero(&de, sizeof (ddi_fm_error_t)); de.fme_version = DDI_FME_VERSION; de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, FM_ENA_FMT1); de.fme_flag = expected; de.fme_bus_specific = (void *)aflt->flt_addr; status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de); if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL)) aflt->flt_panic = 1; } void cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz, errorq_t *eqp, uint_t flag) { struct async_flt *aflt = (struct async_flt *)payload; aflt->flt_erpt_class = error_class; errorq_dispatch(eqp, payload, payload_sz, flag); } /* * This routine may be called by the IO module, but does not do * anything in this cpu module. The SERD algorithm is handled by * cpumem-diagnosis engine instead. */ /*ARGSUSED*/ void cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum) {} void adjust_hw_copy_limits(int ecache_size) { /* * Set hw copy limits. * * /etc/system will be parsed later and can override one or more * of these settings. * * At this time, ecache size seems only mildly relevant. * We seem to run into issues with the d-cache and stalls * we see on misses. * * Cycle measurement indicates that 2 byte aligned copies fare * little better than doing things with VIS at around 512 bytes. * 4 byte aligned shows promise until around 1024 bytes. 8 Byte * aligned is faster whenever the source and destination data * in cache and the total size is less than 2 Kbytes. The 2K * limit seems to be driven by the 2K write cache. * When more than 2K of copies are done in non-VIS mode, stores * backup in the write cache. In VIS mode, the write cache is * bypassed, allowing faster cache-line writes aligned on cache * boundaries. * * In addition, in non-VIS mode, there is no prefetching, so * for larger copies, the advantage of prefetching to avoid even * occasional cache misses is enough to justify using the VIS code. * * During testing, it was discovered that netbench ran 3% slower * when hw_copy_limit_8 was 2K or larger. Apparently for server * applications, data is only used once (copied to the output * buffer, then copied by the network device off the system). Using * the VIS copy saves more L2 cache state. Network copies are * around 1.3K to 1.5K in size for historical reasons. * * Therefore, a limit of 1K bytes will be used for the 8 byte * aligned copy even for large caches and 8 MB ecache. The * infrastructure to allow different limits for different sized * caches is kept to allow further tuning in later releases. */ if (min_ecache_size == 0 && use_hw_bcopy) { /* * First time through - should be before /etc/system * is read. * Could skip the checks for zero but this lets us * preserve any debugger rewrites. */ if (hw_copy_limit_1 == 0) { hw_copy_limit_1 = VIS_COPY_THRESHOLD; priv_hcl_1 = hw_copy_limit_1; } if (hw_copy_limit_2 == 0) { hw_copy_limit_2 = 2 * VIS_COPY_THRESHOLD; priv_hcl_2 = hw_copy_limit_2; } if (hw_copy_limit_4 == 0) { hw_copy_limit_4 = 4 * VIS_COPY_THRESHOLD; priv_hcl_4 = hw_copy_limit_4; } if (hw_copy_limit_8 == 0) { hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD; priv_hcl_8 = hw_copy_limit_8; } min_ecache_size = ecache_size; } else { /* * MP initialization. Called *after* /etc/system has * been parsed. One CPU has already been initialized. * Need to cater for /etc/system having scragged one * of our values. */ if (ecache_size == min_ecache_size) { /* * Same size ecache. We do nothing unless we * have a pessimistic ecache setting. In that * case we become more optimistic (if the cache is * large enough). */ if (hw_copy_limit_8 == 4 * VIS_COPY_THRESHOLD) { /* * Need to adjust hw_copy_limit* from our * pessimistic uniprocessor value to a more * optimistic UP value *iff* it hasn't been * reset. */ if ((ecache_size > 1048576) && (priv_hcl_8 == hw_copy_limit_8)) { if (ecache_size <= 2097152) hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD; else if (ecache_size <= 4194304) hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD; else hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD; priv_hcl_8 = hw_copy_limit_8; } } } else if (ecache_size < min_ecache_size) { /* * A different ecache size. Can this even happen? */ if (priv_hcl_8 == hw_copy_limit_8) { /* * The previous value that we set * is unchanged (i.e., it hasn't been * scragged by /etc/system). Rewrite it. */ if (ecache_size <= 1048576) hw_copy_limit_8 = 8 * VIS_COPY_THRESHOLD; else if (ecache_size <= 2097152) hw_copy_limit_8 = 8 * VIS_COPY_THRESHOLD; else if (ecache_size <= 4194304) hw_copy_limit_8 = 8 * VIS_COPY_THRESHOLD; else hw_copy_limit_8 = 10 * VIS_COPY_THRESHOLD; priv_hcl_8 = hw_copy_limit_8; min_ecache_size = ecache_size; } } } } /* * Called from illegal instruction trap handler to see if we can attribute * the trap to a fpras check. */ int fpras_chktrap(struct regs *rp) { int op; struct fpras_chkfngrp *cgp; uintptr_t tpc = (uintptr_t)rp->r_pc; if (fpras_chkfngrps == NULL) return (0); cgp = &fpras_chkfngrps[CPU->cpu_id]; for (op = 0; op < FPRAS_NCOPYOPS; ++op) { if (tpc >= (uintptr_t)&cgp->fpras_fn[op].fpras_blk0 && tpc < (uintptr_t)&cgp->fpras_fn[op].fpras_chkresult) break; } if (op == FPRAS_NCOPYOPS) return (0); /* * This is an fpRAS failure caught through an illegal * instruction - trampoline. */ rp->r_pc = (uintptr_t)&cgp->fpras_fn[op].fpras_trampoline; rp->r_npc = rp->r_pc + 4; return (1); } /* * fpras_failure is called when a fpras check detects a bad calculation * result or an illegal instruction trap is attributed to an fpras * check. In all cases we are still bound to CPU. */ int fpras_failure(int op, int how) { int use_hw_bcopy_orig, use_hw_bzero_orig; uint_t hcl1_orig, hcl2_orig, hcl4_orig, hcl8_orig; ch_async_flt_t ch_flt; struct async_flt *aflt = (struct async_flt *)&ch_flt; struct fpras_chkfn *sfp, *cfp; uint32_t *sip, *cip; int i; /* * We're running on a sick CPU. Avoid further FPU use at least for * the time in which we dispatch an ereport and (if applicable) panic. */ use_hw_bcopy_orig = use_hw_bcopy; use_hw_bzero_orig = use_hw_bzero; hcl1_orig = hw_copy_limit_1; hcl2_orig = hw_copy_limit_2; hcl4_orig = hw_copy_limit_4; hcl8_orig = hw_copy_limit_8; use_hw_bcopy = use_hw_bzero = 0; hw_copy_limit_1 = hw_copy_limit_2 = hw_copy_limit_4 = hw_copy_limit_8 = 0; bzero(&ch_flt, sizeof (ch_async_flt_t)); aflt->flt_id = gethrtime_waitfree(); aflt->flt_class = CPU_FAULT; aflt->flt_inst = CPU->cpu_id; aflt->flt_status = (how << 8) | op; aflt->flt_payload = FM_EREPORT_PAYLOAD_FPU_HWCOPY; ch_flt.flt_type = CPU_FPUERR; /* * We must panic if the copy operation had no lofault protection - * ie, don't panic for copyin, copyout, kcopy and bcopy called * under on_fault and do panic for unprotected bcopy and hwblkpagecopy. */ aflt->flt_panic = (curthread->t_lofault == (uintptr_t)NULL); /* * XOR the source instruction block with the copied instruction * block - this will show us which bit(s) are corrupted. */ sfp = (struct fpras_chkfn *)fpras_chkfn_type1; cfp = &fpras_chkfngrps[CPU->cpu_id].fpras_fn[op]; if (op == FPRAS_BCOPY || op == FPRAS_COPYOUT) { sip = &sfp->fpras_blk0[0]; cip = &cfp->fpras_blk0[0]; } else { sip = &sfp->fpras_blk1[0]; cip = &cfp->fpras_blk1[0]; } for (i = 0; i < 16; ++i, ++sip, ++cip) ch_flt.flt_fpdata[i] = *sip ^ *cip; cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_FPU_HWCOPY, (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); if (aflt->flt_panic) fm_panic("FPU failure on CPU %d", CPU->cpu_id); /* * We get here for copyin/copyout and kcopy or bcopy where the * caller has used on_fault. We will flag the error so that * the process may be killed The trap_async_hwerr mechanism will * take appropriate further action (such as a reboot, contract * notification etc). Since we may be continuing we will * restore the global hardware copy acceleration switches. * * When we return from this function to the copy function we want to * avoid potentially bad data being used, ie we want the affected * copy function to return an error. The caller should therefore * invoke its lofault handler (which always exists for these functions) * which will return the appropriate error. */ ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR; aston(curthread); use_hw_bcopy = use_hw_bcopy_orig; use_hw_bzero = use_hw_bzero_orig; hw_copy_limit_1 = hcl1_orig; hw_copy_limit_2 = hcl2_orig; hw_copy_limit_4 = hcl4_orig; hw_copy_limit_8 = hcl8_orig; return (1); } #define VIS_BLOCKSIZE 64 int dtrace_blksuword32_err(uintptr_t addr, uint32_t *data) { int ret, watched; watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); ret = dtrace_blksuword32(addr, data, 0); if (watched) watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); return (ret); } /* * Called when a cpu enters the CPU_FAULTED state (by the cpu placing the * faulted cpu into that state). Cross-trap to the faulted cpu to clear * CEEN from the EER to disable traps for further disrupting error types * on that cpu. We could cross-call instead, but that has a larger * instruction and data footprint than cross-trapping, and the cpu is known * to be faulted. */ void cpu_faulted_enter(struct cpu *cp) { xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_CLRBITS); } /* * Called when a cpu leaves the CPU_FAULTED state to return to one of * offline, spare, or online (by the cpu requesting this state change). * First we cross-call to clear the AFSR (and AFSR_EXT on Panther) of * disrupting error bits that have accumulated without trapping, then * we cross-trap to re-enable CEEN controlled traps. */ void cpu_faulted_exit(struct cpu *cp) { ch_cpu_errors_t cpu_error_regs; cpu_error_regs.afsr = C_AFSR_CECC_ERRS; if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) cpu_error_regs.afsr_ext &= C_AFSR_EXT_CECC_ERRS; xc_one(cp->cpu_id, (xcfunc_t *)set_cpu_error_state, (uint64_t)&cpu_error_regs, 0); xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_SETBITS); } /* * Return 1 if the errors in ch_flt's AFSR are secondary errors caused by * the errors in the original AFSR, 0 otherwise. * * For all procs if the initial error was a BERR or TO, then it is possible * that we may have caused a secondary BERR or TO in the process of logging the * inital error via cpu_run_bus_error_handlers(). If this is the case then * if the request was protected then a panic is still not necessary, if not * protected then aft_panic is already set - so either way there's no need * to set aft_panic for the secondary error. * * For Cheetah and Jalapeno, if the original error was a UE which occurred on * a store merge, then the error handling code will call cpu_deferred_error(). * When clear_errors() is called, it will determine that secondary errors have * occurred - in particular, the store merge also caused a EDU and WDU that * weren't discovered until this point. * * We do three checks to verify that we are in this case. If we pass all three * checks, we return 1 to indicate that we should not panic. If any unexpected * errors occur, we return 0. * * For Cheetah+ and derivative procs, the store merge causes a DUE, which is * handled in cpu_disrupting_errors(). Since this function is not even called * in the case we are interested in, we just return 0 for these processors. */ /*ARGSUSED*/ static int cpu_check_secondary_errors(ch_async_flt_t *ch_flt, uint64_t t_afsr_errs, uint64_t t_afar) { #if defined(CHEETAH_PLUS) #else /* CHEETAH_PLUS */ struct async_flt *aflt = (struct async_flt *)ch_flt; #endif /* CHEETAH_PLUS */ /* * Was the original error a BERR or TO and only a BERR or TO * (multiple errors are also OK) */ if ((t_afsr_errs & ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) { /* * Is the new error a BERR or TO and only a BERR or TO * (multiple errors are also OK) */ if ((ch_flt->afsr_errs & ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) return (1); } #if defined(CHEETAH_PLUS) return (0); #else /* CHEETAH_PLUS */ /* * Now look for secondary effects of a UE on cheetah/jalapeno * * Check the original error was a UE, and only a UE. Note that * the ME bit will cause us to fail this check. */ if (t_afsr_errs != C_AFSR_UE) return (0); /* * Check the secondary errors were exclusively an EDU and/or WDU. */ if ((ch_flt->afsr_errs & ~(C_AFSR_EDU|C_AFSR_WDU)) != 0) return (0); /* * Check the AFAR of the original error and secondary errors * match to the 64-byte boundary */ if (P2ALIGN(aflt->flt_addr, 64) != P2ALIGN(t_afar, 64)) return (0); /* * We've passed all the checks, so it's a secondary error! */ return (1); #endif /* CHEETAH_PLUS */ } /* * Translate the flt_bit or flt_type into an error type. First, flt_bit * is checked for any valid errors. If found, the error type is * returned. If not found, the flt_type is checked for L1$ parity errors. */ /*ARGSUSED*/ static uint8_t cpu_flt_bit_to_plat_error(struct async_flt *aflt) { #if defined(JALAPENO) /* * Currently, logging errors to the SC is not supported on Jalapeno */ return (PLAT_ECC_ERROR2_NONE); #else ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; switch (ch_flt->flt_bit) { case C_AFSR_CE: return (PLAT_ECC_ERROR2_CE); case C_AFSR_UCC: case C_AFSR_EDC: case C_AFSR_WDC: case C_AFSR_CPC: return (PLAT_ECC_ERROR2_L2_CE); case C_AFSR_EMC: return (PLAT_ECC_ERROR2_EMC); case C_AFSR_IVC: return (PLAT_ECC_ERROR2_IVC); case C_AFSR_UE: return (PLAT_ECC_ERROR2_UE); case C_AFSR_UCU: case C_AFSR_EDU: case C_AFSR_WDU: case C_AFSR_CPU: return (PLAT_ECC_ERROR2_L2_UE); case C_AFSR_IVU: return (PLAT_ECC_ERROR2_IVU); case C_AFSR_TO: return (PLAT_ECC_ERROR2_TO); case C_AFSR_BERR: return (PLAT_ECC_ERROR2_BERR); #if defined(CHEETAH_PLUS) case C_AFSR_L3_EDC: case C_AFSR_L3_UCC: case C_AFSR_L3_CPC: case C_AFSR_L3_WDC: return (PLAT_ECC_ERROR2_L3_CE); case C_AFSR_IMC: return (PLAT_ECC_ERROR2_IMC); case C_AFSR_TSCE: return (PLAT_ECC_ERROR2_L2_TSCE); case C_AFSR_THCE: return (PLAT_ECC_ERROR2_L2_THCE); case C_AFSR_L3_MECC: return (PLAT_ECC_ERROR2_L3_MECC); case C_AFSR_L3_THCE: return (PLAT_ECC_ERROR2_L3_THCE); case C_AFSR_L3_CPU: case C_AFSR_L3_EDU: case C_AFSR_L3_UCU: case C_AFSR_L3_WDU: return (PLAT_ECC_ERROR2_L3_UE); case C_AFSR_DUE: return (PLAT_ECC_ERROR2_DUE); case C_AFSR_DTO: return (PLAT_ECC_ERROR2_DTO); case C_AFSR_DBERR: return (PLAT_ECC_ERROR2_DBERR); #endif /* CHEETAH_PLUS */ default: switch (ch_flt->flt_type) { #if defined(CPU_IMP_L1_CACHE_PARITY) case CPU_IC_PARITY: return (PLAT_ECC_ERROR2_IPE); case CPU_DC_PARITY: if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { if (ch_flt->parity_data.dpe.cpl_cache == CPU_PC_PARITY) { return (PLAT_ECC_ERROR2_PCACHE); } } return (PLAT_ECC_ERROR2_DPE); #endif /* CPU_IMP_L1_CACHE_PARITY */ case CPU_ITLB_PARITY: return (PLAT_ECC_ERROR2_ITLB); case CPU_DTLB_PARITY: return (PLAT_ECC_ERROR2_DTLB); default: return (PLAT_ECC_ERROR2_NONE); } } #endif /* JALAPENO */ }