1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/systm.h> 28 #include <sys/archsystm.h> 29 #include <sys/machparam.h> 30 #include <sys/machsystm.h> 31 #include <sys/cpu.h> 32 #include <sys/elf_SPARC.h> 33 #include <vm/hat_sfmmu.h> 34 #include <vm/seg_kpm.h> 35 #include <vm/page.h> 36 #include <vm/vm_dep.h> 37 #include <sys/cpuvar.h> 38 #include <sys/spitregs.h> 39 #include <sys/async.h> 40 #include <sys/cmn_err.h> 41 #include <sys/debug.h> 42 #include <sys/dditypes.h> 43 #include <sys/sunddi.h> 44 #include <sys/cpu_module.h> 45 #include <sys/prom_debug.h> 46 #include <sys/vmsystm.h> 47 #include <sys/prom_plat.h> 48 #include <sys/sysmacros.h> 49 #include <sys/intreg.h> 50 #include <sys/machtrap.h> 51 #include <sys/ontrap.h> 52 #include <sys/ivintr.h> 53 #include <sys/atomic.h> 54 #include <sys/panic.h> 55 #include <sys/ndifm.h> 56 #include <sys/fm/protocol.h> 57 #include <sys/fm/util.h> 58 #include <sys/fm/cpu/UltraSPARC-II.h> 59 #include <sys/ddi.h> 60 #include <sys/ecc_kstat.h> 61 #include <sys/watchpoint.h> 62 #include <sys/dtrace.h> 63 #include <sys/errclassify.h> 64 65 uint_t cpu_impl_dual_pgsz = 0; 66 67 /* 68 * Structure for the 8 byte ecache data dump and the associated AFSR state. 69 * There will be 8 of these structures used to dump an ecache line (64 bytes). 70 */ 71 typedef struct sf_ec_data_elm { 72 uint64_t ec_d8; 73 uint64_t ec_afsr; 74 } ec_data_t; 75 76 /* 77 * Define spitfire (Ultra I/II) specific asynchronous error structure 78 */ 79 typedef struct spitfire_async_flt { 80 struct async_flt cmn_asyncflt; /* common - see sun4u/sys/async.h */ 81 ushort_t flt_type; /* types of faults - cpu specific */ 82 ec_data_t flt_ec_data[8]; /* for E$ or mem dump/state */ 83 uint64_t flt_ec_tag; /* E$ tag info */ 84 int flt_ec_lcnt; /* number of bad E$ lines */ 85 ushort_t flt_sdbh; /* UDBH reg */ 86 ushort_t flt_sdbl; /* UDBL reg */ 87 } spitf_async_flt; 88 89 /* 90 * Prototypes for support routines in spitfire_asm.s: 91 */ 92 extern void flush_ecache(uint64_t physaddr, size_t size, size_t linesize); 93 extern uint64_t get_lsu(void); 94 extern void set_lsu(uint64_t ncc); 95 extern void get_ecache_dtag(uint32_t ecache_idx, uint64_t *data, uint64_t *tag, 96 uint64_t *oafsr, uint64_t *acc_afsr); 97 extern uint64_t check_ecache_line(uint32_t id, uint64_t *acc_afsr); 98 extern uint64_t get_ecache_tag(uint32_t id, uint64_t *nafsr, 99 uint64_t *acc_afsr); 100 extern uint64_t read_and_clear_afsr(); 101 extern void write_ec_tag_parity(uint32_t id); 102 extern void write_hb_ec_tag_parity(uint32_t id); 103 104 /* 105 * Spitfire module routines: 106 */ 107 static void cpu_async_log_err(void *flt); 108 /*PRINTFLIKE6*/ 109 static void cpu_aflt_log(int ce_code, int tagnum, spitf_async_flt *spflt, 110 uint_t logflags, const char *endstr, const char *fmt, ...); 111 112 static void cpu_read_paddr(struct async_flt *aflt, short verbose, short ce_err); 113 static void cpu_ce_log_status(spitf_async_flt *spf_flt, char *unum); 114 static void cpu_log_ecmem_info(spitf_async_flt *spf_flt); 115 116 static void log_ce_err(struct async_flt *aflt, char *unum); 117 static void log_ue_err(struct async_flt *aflt, char *unum); 118 static void check_misc_err(spitf_async_flt *spf_flt); 119 static ushort_t ecc_gen(uint_t high_bytes, uint_t low_bytes); 120 static int check_ecc(struct async_flt *aflt); 121 static uint_t get_cpu_status(uint64_t arg); 122 static uint64_t clear_errors(spitf_async_flt *spf_flt, uint64_t *acc_afsr); 123 static void scan_ecache(uint64_t *afar, ec_data_t *data, uint64_t *tag, 124 int *m, uint64_t *afsr); 125 static void ecache_kstat_init(struct cpu *cp); 126 static void ecache_scrub_log(ec_data_t *ec_data, uint64_t ec_tag, 127 uint64_t paddr, int mpb, uint64_t); 128 static uint64_t ecache_scrub_misc_err(int, uint64_t); 129 static void ecache_scrub_tag_err(uint64_t, uchar_t, uint32_t); 130 static void ecache_page_retire(void *); 131 static int ecc_kstat_update(kstat_t *ksp, int rw); 132 static int ce_count_unum(int status, int len, char *unum); 133 static void add_leaky_bucket_timeout(void); 134 static int synd_to_synd_code(int synd_status, ushort_t synd); 135 136 extern uint_t read_all_memscrub; 137 extern void memscrub_run(void); 138 139 static uchar_t isus2i; /* set if sabre */ 140 static uchar_t isus2e; /* set if hummingbird */ 141 142 /* 143 * Default ecache mask and shift settings for Spitfire. If we detect a 144 * different CPU implementation, we will modify these values at boot time. 145 */ 146 static uint64_t cpu_ec_tag_mask = S_ECTAG_MASK; 147 static uint64_t cpu_ec_state_mask = S_ECSTATE_MASK; 148 static uint64_t cpu_ec_par_mask = S_ECPAR_MASK; 149 static int cpu_ec_par_shift = S_ECPAR_SHIFT; 150 static int cpu_ec_tag_shift = S_ECTAG_SHIFT; 151 static int cpu_ec_state_shift = S_ECSTATE_SHIFT; 152 static uchar_t cpu_ec_state_exl = S_ECSTATE_EXL; 153 static uchar_t cpu_ec_state_mod = S_ECSTATE_MOD; 154 static uchar_t cpu_ec_state_shr = S_ECSTATE_SHR; 155 static uchar_t cpu_ec_state_own = S_ECSTATE_OWN; 156 157 /* 158 * Default ecache state bits for Spitfire. These individual bits indicate if 159 * the given line is in any of the valid or modified states, respectively. 160 * Again, we modify these at boot if we detect a different CPU. 161 */ 162 static uchar_t cpu_ec_state_valid = S_ECSTATE_VALID; 163 static uchar_t cpu_ec_state_dirty = S_ECSTATE_DIRTY; 164 static uchar_t cpu_ec_parity = S_EC_PARITY; 165 static uchar_t cpu_ec_state_parity = S_ECSTATE_PARITY; 166 167 /* 168 * This table is used to determine which bit(s) is(are) bad when an ECC 169 * error occurrs. The array is indexed an 8-bit syndrome. The entries 170 * of this array have the following semantics: 171 * 172 * 00-63 The number of the bad bit, when only one bit is bad. 173 * 64 ECC bit C0 is bad. 174 * 65 ECC bit C1 is bad. 175 * 66 ECC bit C2 is bad. 176 * 67 ECC bit C3 is bad. 177 * 68 ECC bit C4 is bad. 178 * 69 ECC bit C5 is bad. 179 * 70 ECC bit C6 is bad. 180 * 71 ECC bit C7 is bad. 181 * 72 Two bits are bad. 182 * 73 Three bits are bad. 183 * 74 Four bits are bad. 184 * 75 More than Four bits are bad. 185 * 76 NO bits are bad. 186 * Based on "Galaxy Memory Subsystem SPECIFICATION" rev 0.6, pg. 28. 187 */ 188 189 #define C0 64 190 #define C1 65 191 #define C2 66 192 #define C3 67 193 #define C4 68 194 #define C5 69 195 #define C6 70 196 #define C7 71 197 #define M2 72 198 #define M3 73 199 #define M4 74 200 #define MX 75 201 #define NA 76 202 203 #define SYND_IS_SINGLE_BIT_DATA(synd_code) ((synd_code >= 0) && \ 204 (synd_code < C0)) 205 #define SYND_IS_SINGLE_BIT_CHK(synd_code) ((synd_code >= C0) && \ 206 (synd_code <= C7)) 207 208 static char ecc_syndrome_tab[] = 209 { 210 NA, C0, C1, M2, C2, M2, M2, M3, C3, M2, M2, M3, M2, M3, M3, M4, 211 C4, M2, M2, 32, M2, 57, MX, M2, M2, 37, 49, M2, 40, M2, M2, 44, 212 C5, M2, M2, 33, M2, 61, 4, M2, M2, MX, 53, M2, 45, M2, M2, 41, 213 M2, 0, 1, M2, 10, M2, M2, MX, 15, M2, M2, MX, M2, M3, M3, M2, 214 C6, M2, M2, 42, M2, 59, 39, M2, M2, MX, 51, M2, 34, M2, M2, 46, 215 M2, 25, 29, M2, 27, M4, M2, MX, 31, M2, M4, MX, M2, MX, MX, M2, 216 M2, MX, 36, M2, 7, M2, M2, 54, MX, M2, M2, 62, M2, 48, 56, M2, 217 M3, M2, M2, MX, M2, MX, 22, M2, M2, 18, MX, M2, M3, M2, M2, MX, 218 C7, M2, M2, 47, M2, 63, MX, M2, M2, 6, 55, M2, 35, M2, M2, 43, 219 M2, 5, MX, M2, MX, M2, M2, 50, 38, M2, M2, 58, M2, 52, 60, M2, 220 M2, 17, 21, M2, 19, M4, M2, MX, 23, M2, M4, MX, M2, MX, MX, M2, 221 M3, M2, M2, MX, M2, MX, 30, M2, M2, 26, MX, M2, M3, M2, M2, MX, 222 M2, 8, 13, M2, 2, M2, M2, M3, 3, M2, M2, M3, M2, MX, MX, M2, 223 M3, M2, M2, M3, M2, MX, 16, M2, M2, 20, MX, M2, MX, M2, M2, MX, 224 M3, M2, M2, M3, M2, MX, 24, M2, M2, 28, MX, M2, MX, M2, M2, MX, 225 M4, 12, 9, M2, 14, M2, M2, MX, 11, M2, M2, MX, M2, MX, MX, M4 226 }; 227 228 #define SYND_TBL_SIZE 256 229 230 /* 231 * Hack for determining UDBH/UDBL, for later cpu-specific error reporting. 232 * Cannot use bit 3 in afar, because it is a valid bit on a Sabre/Hummingbird. 233 */ 234 #define UDBL_REG 0x8000 235 #define UDBL(synd) ((synd & UDBL_REG) >> 15) 236 #define SYND(synd) (synd & 0x7FFF) 237 238 /* 239 * These error types are specific to Spitfire and are used internally for the 240 * spitfire fault structure flt_type field. 241 */ 242 #define CPU_UE_ERR 0 /* uncorrectable errors - UEs */ 243 #define CPU_EDP_LDP_ERR 1 /* LDP or EDP parity error */ 244 #define CPU_WP_ERR 2 /* WP parity error */ 245 #define CPU_BTO_BERR_ERR 3 /* bus timeout errors */ 246 #define CPU_PANIC_CP_ERR 4 /* cp error from panic polling */ 247 #define CPU_TRAPPING_CP_ERR 5 /* for sabre/hbird only, cp error */ 248 #define CPU_BADLINE_CI_ERR 6 /* E$ clean_bad line when idle */ 249 #define CPU_BADLINE_CB_ERR 7 /* E$ clean_bad line when busy */ 250 #define CPU_BADLINE_DI_ERR 8 /* E$ dirty_bad line when idle */ 251 #define CPU_BADLINE_DB_ERR 9 /* E$ dirty_bad line when busy */ 252 #define CPU_ORPHAN_CP_ERR 10 /* Orphan CP error */ 253 #define CPU_ECACHE_ADDR_PAR_ERR 11 /* Ecache Address parity error */ 254 #define CPU_ECACHE_STATE_ERR 12 /* Ecache state error */ 255 #define CPU_ECACHE_ETP_ETS_ERR 13 /* ETP set but ETS is zero */ 256 #define CPU_ECACHE_TAG_ERR 14 /* Scrub the E$ tag, if state clean */ 257 #define CPU_ADDITIONAL_ERR 15 /* Additional errors occurred */ 258 259 /* 260 * Macro to access the "Spitfire cpu private" data structure. 261 */ 262 #define CPU_PRIVATE_PTR(cp, x) (&(((spitfire_private_t *)CPU_PRIVATE(cp))->x)) 263 264 /* 265 * set to 0 to disable automatic retiring of pages on 266 * DIMMs that have excessive soft errors 267 */ 268 int automatic_page_removal = 1; 269 270 /* 271 * Heuristic for figuring out which module to replace. 272 * Relative likelihood that this P_SYND indicates that this module is bad. 273 * We call it a "score", though, not a relative likelihood. 274 * 275 * Step 1. 276 * Assign a score to each byte of P_SYND according to the following rules: 277 * If no bits on (0x00) or all bits on (0xFF), then give it a 5. 278 * If one bit on, give it a 95. 279 * If seven bits on, give it a 10. 280 * If two bits on: 281 * in different nybbles, a 90 282 * in same nybble, but unaligned, 85 283 * in same nybble and as an aligned pair, 80 284 * If six bits on, look at the bits that are off: 285 * in same nybble and as an aligned pair, 15 286 * in same nybble, but unaligned, 20 287 * in different nybbles, a 25 288 * If three bits on: 289 * in diferent nybbles, no aligned pairs, 75 290 * in diferent nybbles, one aligned pair, 70 291 * in the same nybble, 65 292 * If five bits on, look at the bits that are off: 293 * in the same nybble, 30 294 * in diferent nybbles, one aligned pair, 35 295 * in diferent nybbles, no aligned pairs, 40 296 * If four bits on: 297 * all in one nybble, 45 298 * as two aligned pairs, 50 299 * one aligned pair, 55 300 * no aligned pairs, 60 301 * 302 * Step 2: 303 * Take the higher of the two scores (one for each byte) as the score 304 * for the module. 305 * 306 * Print the score for each module, and field service should replace the 307 * module with the highest score. 308 */ 309 310 /* 311 * In the table below, the first row/column comment indicates the 312 * number of bits on in that nybble; the second row/column comment is 313 * the hex digit. 314 */ 315 316 static int 317 p_synd_score_table[256] = { 318 /* 0 1 1 2 1 2 2 3 1 2 2 3 2 3 3 4 */ 319 /* 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, A, B, C, D, E, F */ 320 /* 0 0 */ 5, 95, 95, 80, 95, 85, 85, 65, 95, 85, 85, 65, 80, 65, 65, 45, 321 /* 1 1 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30, 322 /* 1 2 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30, 323 /* 2 3 */ 80, 70, 70, 50, 70, 55, 55, 35, 70, 55, 55, 35, 50, 35, 35, 15, 324 /* 1 4 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30, 325 /* 2 5 */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20, 326 /* 2 6 */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20, 327 /* 3 7 */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10, 328 /* 1 8 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30, 329 /* 2 9 */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20, 330 /* 2 A */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20, 331 /* 3 B */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10, 332 /* 2 C */ 80, 70, 70, 50, 70, 55, 55, 35, 70, 55, 55, 35, 50, 35, 35, 15, 333 /* 3 D */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10, 334 /* 3 E */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10, 335 /* 4 F */ 45, 30, 30, 15, 30, 20, 20, 10, 30, 20, 20, 10, 15, 10, 10, 5, 336 }; 337 338 int 339 ecc_psynd_score(ushort_t p_synd) 340 { 341 int i, j, a, b; 342 343 i = p_synd & 0xFF; 344 j = (p_synd >> 8) & 0xFF; 345 346 a = p_synd_score_table[i]; 347 b = p_synd_score_table[j]; 348 349 return (a > b ? a : b); 350 } 351 352 /* 353 * Async Fault Logging 354 * 355 * To ease identifying, reading, and filtering async fault log messages, the 356 * label [AFT#] is now prepended to each async fault message. These messages 357 * and the logging rules are implemented by cpu_aflt_log(), below. 358 * 359 * [AFT0] - Tag for log messages that are associated with corrected ECC errors. 360 * This includes both corrected ECC memory and ecache faults. 361 * 362 * [AFT1] - Tag for log messages that are not ECC corrected (i.e. everything 363 * else except CE errors) with a priority of 1 (highest). This tag 364 * is also used for panic messages that result from an async fault. 365 * 366 * [AFT2] - These are lower priority diagnostic messages for uncorrected ECC 367 * [AFT3] or parity errors. For example, AFT2 is used for the actual dump 368 * of the E-$ data and tags. 369 * 370 * In a non-DEBUG kernel, AFT > 1 logs will be sent to the system log but not 371 * printed on the console. To send all AFT logs to both the log and the 372 * console, set aft_verbose = 1. 373 */ 374 375 #define CPU_FLTCPU 0x0001 /* print flt_inst as a CPU id */ 376 #define CPU_SPACE 0x0002 /* print flt_status (data or instr) */ 377 #define CPU_ERRID 0x0004 /* print flt_id */ 378 #define CPU_TL 0x0008 /* print flt_tl */ 379 #define CPU_ERRID_FIRST 0x0010 /* print flt_id first in message */ 380 #define CPU_AFSR 0x0020 /* print flt_stat as decoded %afsr */ 381 #define CPU_AFAR 0x0040 /* print flt_addr as %afar */ 382 #define CPU_AF_PSYND 0x0080 /* print flt_stat %afsr.PSYND */ 383 #define CPU_AF_ETS 0x0100 /* print flt_stat %afsr.ETS */ 384 #define CPU_UDBH 0x0200 /* print flt_sdbh and syndrome */ 385 #define CPU_UDBL 0x0400 /* print flt_sdbl and syndrome */ 386 #define CPU_FAULTPC 0x0800 /* print flt_pc */ 387 #define CPU_SYND 0x1000 /* print flt_synd and unum */ 388 389 #define CMN_LFLAGS (CPU_FLTCPU | CPU_SPACE | CPU_ERRID | CPU_TL | \ 390 CPU_AFSR | CPU_AFAR | CPU_AF_PSYND | \ 391 CPU_AF_ETS | CPU_UDBH | CPU_UDBL | \ 392 CPU_FAULTPC) 393 #define UE_LFLAGS (CMN_LFLAGS | CPU_SYND) 394 #define CE_LFLAGS (UE_LFLAGS & ~CPU_UDBH & ~CPU_UDBL & ~CPU_TL & \ 395 ~CPU_SPACE) 396 #define PARERR_LFLAGS (CMN_LFLAGS) 397 #define WP_LFLAGS (CMN_LFLAGS & ~CPU_SPACE & ~CPU_TL) 398 #define CP_LFLAGS (CMN_LFLAGS & ~CPU_SPACE & ~CPU_TL & \ 399 ~CPU_FLTCPU & ~CPU_FAULTPC) 400 #define BERRTO_LFLAGS (CMN_LFLAGS) 401 #define NO_LFLAGS (0) 402 403 #define AFSR_FMTSTR0 "\020\1ME" 404 #define AFSR_FMTSTR1 "\020\040PRIV\037ISAP\036ETP\035IVUE\034TO" \ 405 "\033BERR\032LDP\031CP\030WP\027EDP\026UE\025CE" 406 #define UDB_FMTSTR "\020\012UE\011CE" 407 408 /* 409 * Save the cache bootup state for use when internal 410 * caches are to be re-enabled after an error occurs. 411 */ 412 uint64_t cache_boot_state = 0; 413 414 /* 415 * PA[31:0] represent Displacement in UPA configuration space. 416 */ 417 uint_t root_phys_addr_lo_mask = 0xffffffff; 418 419 /* 420 * Spitfire legacy globals 421 */ 422 int itlb_entries; 423 int dtlb_entries; 424 425 void 426 cpu_setup(void) 427 { 428 extern int page_retire_messages; 429 extern int page_retire_first_ue; 430 extern int at_flags; 431 #if defined(SF_ERRATA_57) 432 extern caddr_t errata57_limit; 433 #endif 434 cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT); 435 436 at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1; 437 438 /* 439 * Spitfire isn't currently FMA-aware, so we have to enable the 440 * page retirement messages. We also change the default policy 441 * for UE retirement to allow clearing of transient errors. 442 */ 443 page_retire_messages = 1; 444 page_retire_first_ue = 0; 445 446 /* 447 * save the cache bootup state. 448 */ 449 cache_boot_state = get_lsu() & (LSU_IC | LSU_DC); 450 451 if (use_page_coloring) { 452 do_pg_coloring = 1; 453 } 454 455 /* 456 * Tune pp_slots to use up to 1/8th of the tlb entries. 457 */ 458 pp_slots = MIN(8, MAXPP_SLOTS); 459 460 /* 461 * Block stores invalidate all pages of the d$ so pagecopy 462 * et. al. do not need virtual translations with virtual 463 * coloring taken into consideration. 464 */ 465 pp_consistent_coloring = 0; 466 467 isa_list = 468 "sparcv9+vis sparcv9 " 469 "sparcv8plus+vis sparcv8plus " 470 "sparcv8 sparcv8-fsmuld sparcv7 sparc"; 471 472 cpu_hwcap_flags = AV_SPARC_VIS; 473 474 /* 475 * On Spitfire, there's a hole in the address space 476 * that we must never map (the hardware only support 44-bits of 477 * virtual address). Later CPUs are expected to have wider 478 * supported address ranges. 479 * 480 * See address map on p23 of the UltraSPARC 1 user's manual. 481 */ 482 hole_start = (caddr_t)0x80000000000ull; 483 hole_end = (caddr_t)0xfffff80000000000ull; 484 485 /* 486 * A spitfire call bug requires us to be a further 4Gbytes of 487 * firewall from the spec. 488 * 489 * See Spitfire Errata #21 490 */ 491 hole_start = (caddr_t)((uintptr_t)hole_start - (1ul << 32)); 492 hole_end = (caddr_t)((uintptr_t)hole_end + (1ul << 32)); 493 494 /* 495 * The kpm mapping window. 496 * kpm_size: 497 * The size of a single kpm range. 498 * The overall size will be: kpm_size * vac_colors. 499 * kpm_vbase: 500 * The virtual start address of the kpm range within the kernel 501 * virtual address space. kpm_vbase has to be kpm_size aligned. 502 */ 503 kpm_size = (size_t)(2ull * 1024 * 1024 * 1024 * 1024); /* 2TB */ 504 kpm_size_shift = 41; 505 kpm_vbase = (caddr_t)0xfffffa0000000000ull; /* 16EB - 6TB */ 506 507 /* 508 * All UltraSPARC platforms should use small kpm page as default, as 509 * the KPM large page VAC conflict code has no value to maintain. The 510 * new generation of SPARC no longer have VAC conflict issue. 511 */ 512 kpm_smallpages = 1; 513 514 #if defined(SF_ERRATA_57) 515 errata57_limit = (caddr_t)0x80000000ul; 516 #endif 517 518 /* 519 * Disable text by default. 520 * Note that the other defaults are set in sun4u/vm/mach_vm_dep.c. 521 */ 522 max_utext_lpsize = MMU_PAGESIZE; 523 } 524 525 static int 526 getintprop(pnode_t node, char *name, int deflt) 527 { 528 int value; 529 530 switch (prom_getproplen(node, name)) { 531 case 0: 532 value = 1; /* boolean properties */ 533 break; 534 535 case sizeof (int): 536 (void) prom_getprop(node, name, (caddr_t)&value); 537 break; 538 539 default: 540 value = deflt; 541 break; 542 } 543 544 return (value); 545 } 546 547 /* 548 * Set the magic constants of the implementation. 549 */ 550 void 551 cpu_fiximp(pnode_t dnode) 552 { 553 extern int vac_size, vac_shift; 554 extern uint_t vac_mask; 555 extern int dcache_line_mask; 556 int i, a; 557 static struct { 558 char *name; 559 int *var; 560 } prop[] = { 561 "dcache-size", &dcache_size, 562 "dcache-line-size", &dcache_linesize, 563 "icache-size", &icache_size, 564 "icache-line-size", &icache_linesize, 565 "ecache-size", &ecache_size, 566 "ecache-line-size", &ecache_alignsize, 567 "ecache-associativity", &ecache_associativity, 568 "#itlb-entries", &itlb_entries, 569 "#dtlb-entries", &dtlb_entries, 570 }; 571 572 for (i = 0; i < sizeof (prop) / sizeof (prop[0]); i++) { 573 if ((a = getintprop(dnode, prop[i].name, -1)) != -1) { 574 *prop[i].var = a; 575 } 576 } 577 578 ecache_setsize = ecache_size / ecache_associativity; 579 580 vac_size = S_VAC_SIZE; 581 vac_mask = MMU_PAGEMASK & (vac_size - 1); 582 i = 0; a = vac_size; 583 while (a >>= 1) 584 ++i; 585 vac_shift = i; 586 shm_alignment = vac_size; 587 vac = 1; 588 589 dcache_line_mask = (dcache_size - 1) & ~(dcache_linesize - 1); 590 591 /* 592 * UltraSPARC I & II have ecache sizes running 593 * as follows: .25 MB, .5 MB, 1 MB, 2 MB, 4 MB 594 * and 8 MB. Adjust the copyin/copyout limits 595 * according to the cache size. The magic number 596 * of VIS_COPY_THRESHOLD comes from the copyin/copyout code 597 * and its floor of VIS_COPY_THRESHOLD bytes before it will use 598 * VIS instructions. 599 * 600 * We assume that all CPUs on the system have the same size 601 * ecache. We're also called very early in the game. 602 * /etc/system will be parsed *after* we're called so 603 * these values can be overwritten. 604 */ 605 606 hw_copy_limit_1 = VIS_COPY_THRESHOLD; 607 if (ecache_size <= 524288) { 608 hw_copy_limit_2 = VIS_COPY_THRESHOLD; 609 hw_copy_limit_4 = VIS_COPY_THRESHOLD; 610 hw_copy_limit_8 = VIS_COPY_THRESHOLD; 611 } else if (ecache_size == 1048576) { 612 hw_copy_limit_2 = 1024; 613 hw_copy_limit_4 = 1280; 614 hw_copy_limit_8 = 1536; 615 } else if (ecache_size == 2097152) { 616 hw_copy_limit_2 = 1536; 617 hw_copy_limit_4 = 2048; 618 hw_copy_limit_8 = 2560; 619 } else if (ecache_size == 4194304) { 620 hw_copy_limit_2 = 2048; 621 hw_copy_limit_4 = 2560; 622 hw_copy_limit_8 = 3072; 623 } else { 624 hw_copy_limit_2 = 2560; 625 hw_copy_limit_4 = 3072; 626 hw_copy_limit_8 = 3584; 627 } 628 } 629 630 /* 631 * Called by setcpudelay 632 */ 633 void 634 cpu_init_tick_freq(void) 635 { 636 /* 637 * Determine the cpu frequency by calling 638 * tod_get_cpufrequency. Use an approximate freqency 639 * value computed by the prom if the tod module 640 * is not initialized and loaded yet. 641 */ 642 if (tod_ops.tod_get_cpufrequency != NULL) { 643 mutex_enter(&tod_lock); 644 sys_tick_freq = tod_ops.tod_get_cpufrequency(); 645 mutex_exit(&tod_lock); 646 } else { 647 #if defined(HUMMINGBIRD) 648 /* 649 * the hummingbird version of %stick is used as the basis for 650 * low level timing; this provides an independent constant-rate 651 * clock for general system use, and frees power mgmt to set 652 * various cpu clock speeds. 653 */ 654 if (system_clock_freq == 0) 655 cmn_err(CE_PANIC, "invalid system_clock_freq 0x%lx", 656 system_clock_freq); 657 sys_tick_freq = system_clock_freq; 658 #else /* SPITFIRE */ 659 sys_tick_freq = cpunodes[CPU->cpu_id].clock_freq; 660 #endif 661 } 662 } 663 664 665 void shipit(int upaid); 666 extern uint64_t xc_tick_limit; 667 extern uint64_t xc_tick_jump_limit; 668 669 #ifdef SEND_MONDO_STATS 670 uint64_t x_early[NCPU][64]; 671 #endif 672 673 /* 674 * Note: A version of this function is used by the debugger via the KDI, 675 * and must be kept in sync with this version. Any changes made to this 676 * function to support new chips or to accomodate errata must also be included 677 * in the KDI-specific version. See spitfire_kdi.c. 678 */ 679 void 680 send_one_mondo(int cpuid) 681 { 682 uint64_t idsr, starttick, endtick; 683 int upaid, busy, nack; 684 uint64_t tick, tick_prev; 685 ulong_t ticks; 686 687 CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 688 upaid = CPUID_TO_UPAID(cpuid); 689 tick = starttick = gettick(); 690 shipit(upaid); 691 endtick = starttick + xc_tick_limit; 692 busy = nack = 0; 693 for (;;) { 694 idsr = getidsr(); 695 if (idsr == 0) 696 break; 697 /* 698 * When we detect an irregular tick jump, we adjust 699 * the timer window to the current tick value. 700 */ 701 tick_prev = tick; 702 tick = gettick(); 703 ticks = tick - tick_prev; 704 if (ticks > xc_tick_jump_limit) { 705 endtick = tick + xc_tick_limit; 706 } else if (tick > endtick) { 707 if (panic_quiesce) 708 return; 709 cmn_err(CE_PANIC, 710 "send mondo timeout (target 0x%x) [%d NACK %d " 711 "BUSY]", upaid, nack, busy); 712 } 713 if (idsr & IDSR_BUSY) { 714 busy++; 715 continue; 716 } 717 drv_usecwait(1); 718 shipit(upaid); 719 nack++; 720 busy = 0; 721 } 722 #ifdef SEND_MONDO_STATS 723 x_early[getprocessorid()][highbit(gettick() - starttick) - 1]++; 724 #endif 725 } 726 727 void 728 send_mondo_set(cpuset_t set) 729 { 730 int i; 731 732 for (i = 0; i < NCPU; i++) 733 if (CPU_IN_SET(set, i)) { 734 send_one_mondo(i); 735 CPUSET_DEL(set, i); 736 if (CPUSET_ISNULL(set)) 737 break; 738 } 739 } 740 741 void 742 syncfpu(void) 743 { 744 } 745 746 /* 747 * Determine the size of the CPU module's error structure in bytes. This is 748 * called once during boot to initialize the error queues. 749 */ 750 int 751 cpu_aflt_size(void) 752 { 753 /* 754 * We need to determine whether this is a sabre, Hummingbird or a 755 * Spitfire/Blackbird impl and set the appropriate state variables for 756 * ecache tag manipulation. We can't do this in cpu_setup() as it is 757 * too early in the boot flow and the cpunodes are not initialized. 758 * This routine will be called once after cpunodes[] is ready, so do 759 * it here. 760 */ 761 if (cpunodes[CPU->cpu_id].implementation == SABRE_IMPL) { 762 isus2i = 1; 763 cpu_ec_tag_mask = SB_ECTAG_MASK; 764 cpu_ec_state_mask = SB_ECSTATE_MASK; 765 cpu_ec_par_mask = SB_ECPAR_MASK; 766 cpu_ec_par_shift = SB_ECPAR_SHIFT; 767 cpu_ec_tag_shift = SB_ECTAG_SHIFT; 768 cpu_ec_state_shift = SB_ECSTATE_SHIFT; 769 cpu_ec_state_exl = SB_ECSTATE_EXL; 770 cpu_ec_state_mod = SB_ECSTATE_MOD; 771 772 /* These states do not exist in sabre - set to 0xFF */ 773 cpu_ec_state_shr = 0xFF; 774 cpu_ec_state_own = 0xFF; 775 776 cpu_ec_state_valid = SB_ECSTATE_VALID; 777 cpu_ec_state_dirty = SB_ECSTATE_DIRTY; 778 cpu_ec_state_parity = SB_ECSTATE_PARITY; 779 cpu_ec_parity = SB_EC_PARITY; 780 } else if (cpunodes[CPU->cpu_id].implementation == HUMMBRD_IMPL) { 781 isus2e = 1; 782 cpu_ec_tag_mask = HB_ECTAG_MASK; 783 cpu_ec_state_mask = HB_ECSTATE_MASK; 784 cpu_ec_par_mask = HB_ECPAR_MASK; 785 cpu_ec_par_shift = HB_ECPAR_SHIFT; 786 cpu_ec_tag_shift = HB_ECTAG_SHIFT; 787 cpu_ec_state_shift = HB_ECSTATE_SHIFT; 788 cpu_ec_state_exl = HB_ECSTATE_EXL; 789 cpu_ec_state_mod = HB_ECSTATE_MOD; 790 791 /* These states do not exist in hummingbird - set to 0xFF */ 792 cpu_ec_state_shr = 0xFF; 793 cpu_ec_state_own = 0xFF; 794 795 cpu_ec_state_valid = HB_ECSTATE_VALID; 796 cpu_ec_state_dirty = HB_ECSTATE_DIRTY; 797 cpu_ec_state_parity = HB_ECSTATE_PARITY; 798 cpu_ec_parity = HB_EC_PARITY; 799 } 800 801 return (sizeof (spitf_async_flt)); 802 } 803 804 805 /* 806 * Correctable ecc error trap handler 807 */ 808 /*ARGSUSED*/ 809 void 810 cpu_ce_error(struct regs *rp, ulong_t p_afar, ulong_t p_afsr, 811 uint_t p_afsr_high, uint_t p_afar_high) 812 { 813 ushort_t sdbh, sdbl; 814 ushort_t e_syndh, e_syndl; 815 spitf_async_flt spf_flt; 816 struct async_flt *ecc; 817 int queue = 1; 818 819 uint64_t t_afar = p_afar; 820 uint64_t t_afsr = p_afsr; 821 822 /* 823 * Note: the Spitfire data buffer error registers 824 * (upper and lower halves) are or'ed into the upper 825 * word of the afsr by ce_err(). 826 */ 827 sdbh = (ushort_t)((t_afsr >> 33) & 0x3FF); 828 sdbl = (ushort_t)((t_afsr >> 43) & 0x3FF); 829 830 e_syndh = (uchar_t)(sdbh & (uint_t)P_DER_E_SYND); 831 e_syndl = (uchar_t)(sdbl & (uint_t)P_DER_E_SYND); 832 833 t_afsr &= S_AFSR_MASK; 834 t_afar &= SABRE_AFAR_PA; /* must use Sabre AFAR mask */ 835 836 /* Setup the async fault structure */ 837 bzero(&spf_flt, sizeof (spitf_async_flt)); 838 ecc = (struct async_flt *)&spf_flt; 839 ecc->flt_id = gethrtime_waitfree(); 840 ecc->flt_stat = t_afsr; 841 ecc->flt_addr = t_afar; 842 ecc->flt_status = ECC_C_TRAP; 843 ecc->flt_bus_id = getprocessorid(); 844 ecc->flt_inst = CPU->cpu_id; 845 ecc->flt_pc = (caddr_t)rp->r_pc; 846 ecc->flt_func = log_ce_err; 847 ecc->flt_in_memory = 848 (pf_is_memory(ecc->flt_addr >> MMU_PAGESHIFT)) ? 1: 0; 849 spf_flt.flt_sdbh = sdbh; 850 spf_flt.flt_sdbl = sdbl; 851 852 /* 853 * Check for fatal conditions. 854 */ 855 check_misc_err(&spf_flt); 856 857 /* 858 * Pananoid checks for valid AFSR and UDBs 859 */ 860 if ((t_afsr & P_AFSR_CE) == 0) { 861 cpu_aflt_log(CE_PANIC, 1, &spf_flt, CMN_LFLAGS, 862 "** Panic due to CE bit not set in the AFSR", 863 " Corrected Memory Error on"); 864 } 865 866 /* 867 * We want to skip logging only if ALL the following 868 * conditions are true: 869 * 870 * 1. There is only one error 871 * 2. That error is a correctable memory error 872 * 3. The error is caused by the memory scrubber (in which case 873 * the error will have occurred under on_trap protection) 874 * 4. The error is on a retired page 875 * 876 * Note: OT_DATA_EC is used places other than the memory scrubber. 877 * However, none of those errors should occur on a retired page. 878 */ 879 if ((ecc->flt_stat & (S_AFSR_ALL_ERRS & ~P_AFSR_ME)) == P_AFSR_CE && 880 curthread->t_ontrap != NULL) { 881 882 if (curthread->t_ontrap->ot_prot & OT_DATA_EC) { 883 if (page_retire_check(ecc->flt_addr, NULL) == 0) { 884 queue = 0; 885 } 886 } 887 } 888 889 if (((sdbh & P_DER_CE) == 0) && ((sdbl & P_DER_CE) == 0)) { 890 cpu_aflt_log(CE_PANIC, 1, &spf_flt, CMN_LFLAGS, 891 "** Panic due to CE bits not set in the UDBs", 892 " Corrected Memory Error on"); 893 } 894 895 if ((sdbh >> 8) & 1) { 896 ecc->flt_synd = e_syndh; 897 ce_scrub(ecc); 898 if (queue) { 899 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CE, ecc, 900 sizeof (*ecc), ce_queue, ERRORQ_ASYNC); 901 } 902 } 903 904 if ((sdbl >> 8) & 1) { 905 ecc->flt_addr = t_afar | 0x8; /* Sabres do not have a UDBL */ 906 ecc->flt_synd = e_syndl | UDBL_REG; 907 ce_scrub(ecc); 908 if (queue) { 909 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CE, ecc, 910 sizeof (*ecc), ce_queue, ERRORQ_ASYNC); 911 } 912 } 913 914 /* 915 * Re-enable all error trapping (CEEN currently cleared). 916 */ 917 clr_datapath(); 918 set_asyncflt(P_AFSR_CE); 919 set_error_enable(EER_ENABLE); 920 } 921 922 /* 923 * Cpu specific CE logging routine 924 */ 925 static void 926 log_ce_err(struct async_flt *aflt, char *unum) 927 { 928 spitf_async_flt spf_flt; 929 930 if ((aflt->flt_stat & P_AFSR_CE) && (ce_verbose_memory == 0)) { 931 return; 932 } 933 934 spf_flt.cmn_asyncflt = *aflt; 935 cpu_aflt_log(CE_CONT, 0, &spf_flt, CE_LFLAGS, unum, 936 " Corrected Memory Error detected by"); 937 } 938 939 /* 940 * Spitfire does not perform any further CE classification refinement 941 */ 942 /*ARGSUSED*/ 943 int 944 ce_scrub_xdiag_recirc(struct async_flt *ecc, errorq_t *eqp, errorq_elem_t *eqep, 945 size_t afltoffset) 946 { 947 return (0); 948 } 949 950 char * 951 flt_to_error_type(struct async_flt *aflt) 952 { 953 if (aflt->flt_status & ECC_INTERMITTENT) 954 return (ERR_TYPE_DESC_INTERMITTENT); 955 if (aflt->flt_status & ECC_PERSISTENT) 956 return (ERR_TYPE_DESC_PERSISTENT); 957 if (aflt->flt_status & ECC_STICKY) 958 return (ERR_TYPE_DESC_STICKY); 959 return (ERR_TYPE_DESC_UNKNOWN); 960 } 961 962 /* 963 * Called by correctable ecc error logging code to print out 964 * the stick/persistent/intermittent status of the error. 965 */ 966 static void 967 cpu_ce_log_status(spitf_async_flt *spf_flt, char *unum) 968 { 969 ushort_t status; 970 char *status1_str = "Memory"; 971 char *status2_str = "Intermittent"; 972 struct async_flt *aflt = (struct async_flt *)spf_flt; 973 974 status = aflt->flt_status; 975 976 if (status & ECC_ECACHE) 977 status1_str = "Ecache"; 978 979 if (status & ECC_STICKY) 980 status2_str = "Sticky"; 981 else if (status & ECC_PERSISTENT) 982 status2_str = "Persistent"; 983 984 cpu_aflt_log(CE_CONT, 0, spf_flt, CPU_ERRID_FIRST, 985 NULL, " Corrected %s Error on %s is %s", 986 status1_str, unum, status2_str); 987 } 988 989 /* 990 * check for a valid ce syndrome, then call the 991 * displacement flush scrubbing code, and then check the afsr to see if 992 * the error was persistent or intermittent. Reread the afar/afsr to see 993 * if the error was not scrubbed successfully, and is therefore sticky. 994 */ 995 /*ARGSUSED1*/ 996 void 997 cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t triedcpulogout) 998 { 999 uint64_t eer, afsr; 1000 ushort_t status; 1001 1002 ASSERT(getpil() > LOCK_LEVEL); 1003 1004 /* 1005 * It is possible that the flt_addr is not a valid 1006 * physical address. To deal with this, we disable 1007 * NCEEN while we scrub that address. If this causes 1008 * a TIMEOUT/BERR, we know this is an invalid 1009 * memory location. 1010 */ 1011 kpreempt_disable(); 1012 eer = get_error_enable(); 1013 if (eer & (EER_CEEN | EER_NCEEN)) 1014 set_error_enable(eer & ~(EER_CEEN | EER_NCEEN)); 1015 1016 /* 1017 * To check if the error detected by IO is persistent, sticky or 1018 * intermittent. 1019 */ 1020 if (ecc->flt_status & ECC_IOBUS) { 1021 ecc->flt_stat = P_AFSR_CE; 1022 } 1023 1024 scrubphys(P2ALIGN(ecc->flt_addr, 64), 1025 cpunodes[CPU->cpu_id].ecache_size); 1026 1027 get_asyncflt(&afsr); 1028 if (afsr & (P_AFSR_TO | P_AFSR_BERR)) { 1029 /* 1030 * Must ensure that we don't get the TIMEOUT/BERR 1031 * when we reenable NCEEN, so we clear the AFSR. 1032 */ 1033 set_asyncflt(afsr & (P_AFSR_TO | P_AFSR_BERR)); 1034 if (eer & (EER_CEEN | EER_NCEEN)) 1035 set_error_enable(eer); 1036 kpreempt_enable(); 1037 return; 1038 } 1039 1040 if (eer & EER_NCEEN) 1041 set_error_enable(eer & ~EER_CEEN); 1042 1043 /* 1044 * Check and clear any ECC errors from the scrub. If the scrub did 1045 * not trip over the error, mark it intermittent. If the scrub did 1046 * trip the error again and it did not scrub away, mark it sticky. 1047 * Otherwise mark it persistent. 1048 */ 1049 if (check_ecc(ecc) != 0) { 1050 cpu_read_paddr(ecc, 0, 1); 1051 1052 if (check_ecc(ecc) != 0) 1053 status = ECC_STICKY; 1054 else 1055 status = ECC_PERSISTENT; 1056 } else 1057 status = ECC_INTERMITTENT; 1058 1059 if (eer & (EER_CEEN | EER_NCEEN)) 1060 set_error_enable(eer); 1061 kpreempt_enable(); 1062 1063 ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY); 1064 ecc->flt_status |= status; 1065 } 1066 1067 /* 1068 * get the syndrome and unum, and then call the routines 1069 * to check the other cpus and iobuses, and then do the error logging. 1070 */ 1071 /*ARGSUSED1*/ 1072 void 1073 cpu_ce_log_err(struct async_flt *ecc, errorq_elem_t *eqep) 1074 { 1075 char unum[UNUM_NAMLEN]; 1076 int len = 0; 1077 int ce_verbose = 0; 1078 int err; 1079 1080 ASSERT(ecc->flt_func != NULL); 1081 1082 /* Get the unum string for logging purposes */ 1083 (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, ecc, unum, 1084 UNUM_NAMLEN, &len); 1085 1086 /* Call specific error logging routine */ 1087 (void) (*ecc->flt_func)(ecc, unum); 1088 1089 /* 1090 * Count errors per unum. 1091 * Non-memory errors are all counted via a special unum string. 1092 */ 1093 if ((err = ce_count_unum(ecc->flt_status, len, unum)) != PR_OK && 1094 automatic_page_removal) { 1095 (void) page_retire(ecc->flt_addr, err); 1096 } 1097 1098 if (ecc->flt_panic) { 1099 ce_verbose = 1; 1100 } else if ((ecc->flt_class == BUS_FAULT) || 1101 (ecc->flt_stat & P_AFSR_CE)) { 1102 ce_verbose = (ce_verbose_memory > 0); 1103 } else { 1104 ce_verbose = 1; 1105 } 1106 1107 if (ce_verbose) { 1108 spitf_async_flt sflt; 1109 int synd_code; 1110 1111 sflt.cmn_asyncflt = *ecc; /* for cpu_aflt_log() */ 1112 1113 cpu_ce_log_status(&sflt, unum); 1114 1115 synd_code = synd_to_synd_code(AFLT_STAT_VALID, 1116 SYND(ecc->flt_synd)); 1117 1118 if (SYND_IS_SINGLE_BIT_DATA(synd_code)) { 1119 cpu_aflt_log(CE_CONT, 0, &sflt, CPU_ERRID_FIRST, 1120 NULL, " ECC Data Bit %2d was in error " 1121 "and corrected", synd_code); 1122 } else if (SYND_IS_SINGLE_BIT_CHK(synd_code)) { 1123 cpu_aflt_log(CE_CONT, 0, &sflt, CPU_ERRID_FIRST, 1124 NULL, " ECC Check Bit %2d was in error " 1125 "and corrected", synd_code - C0); 1126 } else { 1127 /* 1128 * These are UE errors - we shouldn't be getting CE 1129 * traps for these; handle them in case of bad h/w. 1130 */ 1131 switch (synd_code) { 1132 case M2: 1133 cpu_aflt_log(CE_CONT, 0, &sflt, 1134 CPU_ERRID_FIRST, NULL, 1135 " Two ECC Bits were in error"); 1136 break; 1137 case M3: 1138 cpu_aflt_log(CE_CONT, 0, &sflt, 1139 CPU_ERRID_FIRST, NULL, 1140 " Three ECC Bits were in error"); 1141 break; 1142 case M4: 1143 cpu_aflt_log(CE_CONT, 0, &sflt, 1144 CPU_ERRID_FIRST, NULL, 1145 " Four ECC Bits were in error"); 1146 break; 1147 case MX: 1148 cpu_aflt_log(CE_CONT, 0, &sflt, 1149 CPU_ERRID_FIRST, NULL, 1150 " More than Four ECC bits were " 1151 "in error"); 1152 break; 1153 default: 1154 cpu_aflt_log(CE_CONT, 0, &sflt, 1155 CPU_ERRID_FIRST, NULL, 1156 " Unknown fault syndrome %d", 1157 synd_code); 1158 break; 1159 } 1160 } 1161 } 1162 1163 /* Display entire cache line, if valid address */ 1164 if (ce_show_data && ecc->flt_addr != AFLT_INV_ADDR) 1165 read_ecc_data(ecc, 1, 1); 1166 } 1167 1168 /* 1169 * We route all errors through a single switch statement. 1170 */ 1171 void 1172 cpu_ue_log_err(struct async_flt *aflt) 1173 { 1174 1175 switch (aflt->flt_class) { 1176 case CPU_FAULT: 1177 cpu_async_log_err(aflt); 1178 break; 1179 1180 case BUS_FAULT: 1181 bus_async_log_err(aflt); 1182 break; 1183 1184 default: 1185 cmn_err(CE_WARN, "discarding async error 0x%p with invalid " 1186 "fault class (0x%x)", (void *)aflt, aflt->flt_class); 1187 break; 1188 } 1189 } 1190 1191 /* Values for action variable in cpu_async_error() */ 1192 #define ACTION_NONE 0 1193 #define ACTION_TRAMPOLINE 1 1194 #define ACTION_AST_FLAGS 2 1195 1196 /* 1197 * Access error trap handler for asynchronous cpu errors. This routine is 1198 * called to handle a data or instruction access error. All fatal errors are 1199 * completely handled by this routine (by panicking). Non fatal error logging 1200 * is queued for later processing either via AST or softint at a lower PIL. 1201 * In case of panic, the error log queue will also be processed as part of the 1202 * panic flow to ensure all errors are logged. This routine is called with all 1203 * errors disabled at PIL15. The AFSR bits are cleared and the UDBL and UDBH 1204 * error bits are also cleared. The hardware has also disabled the I and 1205 * D-caches for us, so we must re-enable them before returning. 1206 * 1207 * A summary of the handling of tl=0 UE/LDP/EDP/TO/BERR/WP/CP: 1208 * 1209 * _______________________________________________________________ 1210 * | Privileged tl0 | Unprivileged | 1211 * | Protected | Unprotected | Protected | Unprotected | 1212 * |on_trap|lofault| | | | 1213 * -------------|-------|-------+---------------+---------------+-------------| 1214 * | | | | | | 1215 * UE/LDP/EDP | L,T,p | L,R,p | L,P | n/a | L,R,p | 1216 * | | | | | | 1217 * TO/BERR | T | S | L,P | n/a | S | 1218 * | | | | | | 1219 * WP | L,M,p | L,M,p | L,M,p | n/a | L,M,p | 1220 * | | | | | | 1221 * CP (IIi/IIe) | L,P | L,P | L,P | n/a | L,P | 1222 * ____________________________________________________________________________ 1223 * 1224 * 1225 * Action codes: 1226 * 1227 * L - log 1228 * M - kick off memscrubber if flt_in_memory 1229 * P - panic 1230 * p - panic if US-IIi or US-IIe (Sabre); overrides R and M 1231 * R - i) if aft_panic is set, panic 1232 * ii) otherwise, send hwerr event to contract and SIGKILL to process 1233 * S - send SIGBUS to process 1234 * T - trampoline 1235 * 1236 * Special cases: 1237 * 1238 * 1) if aft_testfatal is set, all faults result in a panic regardless 1239 * of type (even WP), protection (even on_trap), or privilege. 1240 */ 1241 /*ARGSUSED*/ 1242 void 1243 cpu_async_error(struct regs *rp, ulong_t p_afar, ulong_t p_afsr, 1244 uint_t p_afsr_high, uint_t p_afar_high) 1245 { 1246 ushort_t sdbh, sdbl, ttype, tl; 1247 spitf_async_flt spf_flt; 1248 struct async_flt *aflt; 1249 char pr_reason[28]; 1250 uint64_t oafsr; 1251 uint64_t acc_afsr = 0; /* accumulated afsr */ 1252 int action = ACTION_NONE; 1253 uint64_t t_afar = p_afar; 1254 uint64_t t_afsr = p_afsr; 1255 int expected = DDI_FM_ERR_UNEXPECTED; 1256 ddi_acc_hdl_t *hp; 1257 1258 /* 1259 * We need to look at p_flag to determine if the thread detected an 1260 * error while dumping core. We can't grab p_lock here, but it's ok 1261 * because we just need a consistent snapshot and we know that everyone 1262 * else will store a consistent set of bits while holding p_lock. We 1263 * don't have to worry about a race because SDOCORE is set once prior 1264 * to doing i/o from the process's address space and is never cleared. 1265 */ 1266 uint_t pflag = ttoproc(curthread)->p_flag; 1267 1268 pr_reason[0] = '\0'; 1269 1270 /* 1271 * Note: the Spitfire data buffer error registers 1272 * (upper and lower halves) are or'ed into the upper 1273 * word of the afsr by async_err() if P_AFSR_UE is set. 1274 */ 1275 sdbh = (ushort_t)((t_afsr >> 33) & 0x3FF); 1276 sdbl = (ushort_t)((t_afsr >> 43) & 0x3FF); 1277 1278 /* 1279 * Grab the ttype encoded in <63:53> of the saved 1280 * afsr passed from async_err() 1281 */ 1282 ttype = (ushort_t)((t_afsr >> 53) & 0x1FF); 1283 tl = (ushort_t)(t_afsr >> 62); 1284 1285 t_afsr &= S_AFSR_MASK; 1286 t_afar &= SABRE_AFAR_PA; /* must use Sabre AFAR mask */ 1287 1288 /* 1289 * Initialize most of the common and CPU-specific structure. We derive 1290 * aflt->flt_priv from %tstate, instead of from the AFSR.PRIV bit. The 1291 * initial setting of aflt->flt_panic is based on TL: we must panic if 1292 * the error occurred at TL > 0. We also set flt_panic if the test/demo 1293 * tuneable aft_testfatal is set (not the default). 1294 */ 1295 bzero(&spf_flt, sizeof (spitf_async_flt)); 1296 aflt = (struct async_flt *)&spf_flt; 1297 aflt->flt_id = gethrtime_waitfree(); 1298 aflt->flt_stat = t_afsr; 1299 aflt->flt_addr = t_afar; 1300 aflt->flt_bus_id = getprocessorid(); 1301 aflt->flt_inst = CPU->cpu_id; 1302 aflt->flt_pc = (caddr_t)rp->r_pc; 1303 aflt->flt_prot = AFLT_PROT_NONE; 1304 aflt->flt_class = CPU_FAULT; 1305 aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 1306 aflt->flt_tl = (uchar_t)tl; 1307 aflt->flt_panic = (tl != 0 || aft_testfatal != 0); 1308 aflt->flt_core = (pflag & SDOCORE) ? 1 : 0; 1309 1310 /* 1311 * Set flt_status based on the trap type. If we end up here as the 1312 * result of a UE detected by the CE handling code, leave status 0. 1313 */ 1314 switch (ttype) { 1315 case T_DATA_ERROR: 1316 aflt->flt_status = ECC_D_TRAP; 1317 break; 1318 case T_INSTR_ERROR: 1319 aflt->flt_status = ECC_I_TRAP; 1320 break; 1321 } 1322 1323 spf_flt.flt_sdbh = sdbh; 1324 spf_flt.flt_sdbl = sdbl; 1325 1326 /* 1327 * Check for fatal async errors. 1328 */ 1329 check_misc_err(&spf_flt); 1330 1331 /* 1332 * If the trap occurred in privileged mode at TL=0, we need to check to 1333 * see if we were executing in the kernel under on_trap() or t_lofault 1334 * protection. If so, modify the saved registers so that we return 1335 * from the trap to the appropriate trampoline routine. 1336 */ 1337 if (aflt->flt_priv && tl == 0) { 1338 if (curthread->t_ontrap != NULL) { 1339 on_trap_data_t *otp = curthread->t_ontrap; 1340 1341 if (otp->ot_prot & OT_DATA_EC) { 1342 aflt->flt_prot = AFLT_PROT_EC; 1343 otp->ot_trap |= OT_DATA_EC; 1344 rp->r_pc = otp->ot_trampoline; 1345 rp->r_npc = rp->r_pc + 4; 1346 action = ACTION_TRAMPOLINE; 1347 } 1348 1349 if ((t_afsr & (P_AFSR_TO | P_AFSR_BERR)) && 1350 (otp->ot_prot & OT_DATA_ACCESS)) { 1351 aflt->flt_prot = AFLT_PROT_ACCESS; 1352 otp->ot_trap |= OT_DATA_ACCESS; 1353 rp->r_pc = otp->ot_trampoline; 1354 rp->r_npc = rp->r_pc + 4; 1355 action = ACTION_TRAMPOLINE; 1356 /* 1357 * for peeks and caut_gets errors are expected 1358 */ 1359 hp = (ddi_acc_hdl_t *)otp->ot_handle; 1360 if (!hp) 1361 expected = DDI_FM_ERR_PEEK; 1362 else if (hp->ah_acc.devacc_attr_access == 1363 DDI_CAUTIOUS_ACC) 1364 expected = DDI_FM_ERR_EXPECTED; 1365 } 1366 1367 } else if (curthread->t_lofault) { 1368 aflt->flt_prot = AFLT_PROT_COPY; 1369 rp->r_g1 = EFAULT; 1370 rp->r_pc = curthread->t_lofault; 1371 rp->r_npc = rp->r_pc + 4; 1372 action = ACTION_TRAMPOLINE; 1373 } 1374 } 1375 1376 /* 1377 * Determine if this error needs to be treated as fatal. Note that 1378 * multiple errors detected upon entry to this trap handler does not 1379 * necessarily warrant a panic. We only want to panic if the trap 1380 * happened in privileged mode and not under t_ontrap or t_lofault 1381 * protection. The exception is WP: if we *only* get WP, it is not 1382 * fatal even if the trap occurred in privileged mode, except on Sabre. 1383 * 1384 * aft_panic, if set, effectively makes us treat usermode 1385 * UE/EDP/LDP faults as if they were privileged - so we we will 1386 * panic instead of sending a contract event. A lofault-protected 1387 * fault will normally follow the contract event; if aft_panic is 1388 * set this will be changed to a panic. 1389 * 1390 * For usermode BERR/BTO errors, eg from processes performing device 1391 * control through mapped device memory, we need only deliver 1392 * a SIGBUS to the offending process. 1393 * 1394 * Some additional flt_panic reasons (eg, WP on Sabre) will be 1395 * checked later; for now we implement the common reasons. 1396 */ 1397 if (aflt->flt_prot == AFLT_PROT_NONE) { 1398 /* 1399 * Beware - multiple bits may be set in AFSR 1400 */ 1401 if (t_afsr & (P_AFSR_UE | P_AFSR_LDP | P_AFSR_EDP)) { 1402 if (aflt->flt_priv || aft_panic) 1403 aflt->flt_panic = 1; 1404 } 1405 1406 if (t_afsr & (P_AFSR_TO | P_AFSR_BERR)) { 1407 if (aflt->flt_priv) 1408 aflt->flt_panic = 1; 1409 } 1410 } else if (aflt->flt_prot == AFLT_PROT_COPY && aft_panic) { 1411 aflt->flt_panic = 1; 1412 } 1413 1414 /* 1415 * UE/BERR/TO: Call our bus nexus friends to check for 1416 * IO errors that may have resulted in this trap. 1417 */ 1418 if (t_afsr & (P_AFSR_TO | P_AFSR_BERR | P_AFSR_UE)) { 1419 cpu_run_bus_error_handlers(aflt, expected); 1420 } 1421 1422 /* 1423 * Handle UE: If the UE is in memory, we need to flush the bad line from 1424 * the E-cache. We also need to query the bus nexus for fatal errors. 1425 * For sabre, we will panic on UEs. Attempts to do diagnostic read on 1426 * caches may introduce more parity errors (especially when the module 1427 * is bad) and in sabre there is no guarantee that such errors 1428 * (if introduced) are written back as poisoned data. 1429 */ 1430 if (t_afsr & P_AFSR_UE) { 1431 int i; 1432 1433 (void) strcat(pr_reason, "UE "); 1434 1435 spf_flt.flt_type = CPU_UE_ERR; 1436 aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >> 1437 MMU_PAGESHIFT)) ? 1: 0; 1438 1439 /* 1440 * With UE, we have the PA of the fault. 1441 * Let do a diagnostic read to get the ecache 1442 * data and tag info of the bad line for logging. 1443 */ 1444 if (aflt->flt_in_memory) { 1445 uint32_t ec_set_size; 1446 uchar_t state; 1447 uint32_t ecache_idx; 1448 uint64_t faultpa = P2ALIGN(aflt->flt_addr, 64); 1449 1450 /* touch the line to put it in ecache */ 1451 acc_afsr |= read_and_clear_afsr(); 1452 (void) lddphys(faultpa); 1453 acc_afsr |= (read_and_clear_afsr() & 1454 ~(P_AFSR_EDP | P_AFSR_UE)); 1455 1456 ec_set_size = cpunodes[CPU->cpu_id].ecache_size / 1457 ecache_associativity; 1458 1459 for (i = 0; i < ecache_associativity; i++) { 1460 ecache_idx = i * ec_set_size + 1461 (aflt->flt_addr % ec_set_size); 1462 get_ecache_dtag(P2ALIGN(ecache_idx, 64), 1463 (uint64_t *)&spf_flt.flt_ec_data[0], 1464 &spf_flt.flt_ec_tag, &oafsr, &acc_afsr); 1465 acc_afsr |= oafsr; 1466 1467 state = (uchar_t)((spf_flt.flt_ec_tag & 1468 cpu_ec_state_mask) >> cpu_ec_state_shift); 1469 1470 if ((state & cpu_ec_state_valid) && 1471 ((spf_flt.flt_ec_tag & cpu_ec_tag_mask) == 1472 ((uint64_t)aflt->flt_addr >> 1473 cpu_ec_tag_shift))) 1474 break; 1475 } 1476 1477 /* 1478 * Check to see if the ecache tag is valid for the 1479 * fault PA. In the very unlikely event where the 1480 * line could be victimized, no ecache info will be 1481 * available. If this is the case, capture the line 1482 * from memory instead. 1483 */ 1484 if ((state & cpu_ec_state_valid) == 0 || 1485 (spf_flt.flt_ec_tag & cpu_ec_tag_mask) != 1486 ((uint64_t)aflt->flt_addr >> cpu_ec_tag_shift)) { 1487 for (i = 0; i < 8; i++, faultpa += 8) { 1488 ec_data_t *ecdptr; 1489 1490 ecdptr = &spf_flt.flt_ec_data[i]; 1491 acc_afsr |= read_and_clear_afsr(); 1492 ecdptr->ec_d8 = lddphys(faultpa); 1493 acc_afsr |= (read_and_clear_afsr() & 1494 ~(P_AFSR_EDP | P_AFSR_UE)); 1495 ecdptr->ec_afsr = 0; 1496 /* null afsr value */ 1497 } 1498 1499 /* 1500 * Mark tag invalid to indicate mem dump 1501 * when we print out the info. 1502 */ 1503 spf_flt.flt_ec_tag = AFLT_INV_ADDR; 1504 } 1505 spf_flt.flt_ec_lcnt = 1; 1506 1507 /* 1508 * Flush out the bad line 1509 */ 1510 flushecacheline(P2ALIGN(aflt->flt_addr, 64), 1511 cpunodes[CPU->cpu_id].ecache_size); 1512 1513 acc_afsr |= clear_errors(NULL, NULL); 1514 } 1515 1516 /* 1517 * Ask our bus nexus friends if they have any fatal errors. If 1518 * so, they will log appropriate error messages and panic as a 1519 * result. We then queue an event for each UDB that reports a 1520 * UE. Each UE reported in a UDB will have its own log message. 1521 * 1522 * Note from kbn: In the case where there are multiple UEs 1523 * (ME bit is set) - the AFAR address is only accurate to 1524 * the 16-byte granularity. One cannot tell whether the AFAR 1525 * belongs to the UDBH or UDBL syndromes. In this case, we 1526 * always report the AFAR address to be 16-byte aligned. 1527 * 1528 * If we're on a Sabre, there is no SDBL, but it will always 1529 * read as zero, so the sdbl test below will safely fail. 1530 */ 1531 if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL || isus2i || isus2e) 1532 aflt->flt_panic = 1; 1533 1534 if (sdbh & P_DER_UE) { 1535 aflt->flt_synd = sdbh & P_DER_E_SYND; 1536 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_UE, 1537 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1538 aflt->flt_panic); 1539 } 1540 if (sdbl & P_DER_UE) { 1541 aflt->flt_synd = sdbl & P_DER_E_SYND; 1542 aflt->flt_synd |= UDBL_REG; /* indicates UDBL */ 1543 if (!(aflt->flt_stat & P_AFSR_ME)) 1544 aflt->flt_addr |= 0x8; 1545 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_UE, 1546 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1547 aflt->flt_panic); 1548 } 1549 1550 /* 1551 * We got a UE and are panicking, save the fault PA in a known 1552 * location so that the platform specific panic code can check 1553 * for copyback errors. 1554 */ 1555 if (aflt->flt_panic && aflt->flt_in_memory) { 1556 panic_aflt = *aflt; 1557 } 1558 } 1559 1560 /* 1561 * Handle EDP and LDP: Locate the line with bad parity and enqueue an 1562 * async error for logging. For Sabre, we panic on EDP or LDP. 1563 */ 1564 if (t_afsr & (P_AFSR_EDP | P_AFSR_LDP)) { 1565 spf_flt.flt_type = CPU_EDP_LDP_ERR; 1566 1567 if (t_afsr & P_AFSR_EDP) 1568 (void) strcat(pr_reason, "EDP "); 1569 1570 if (t_afsr & P_AFSR_LDP) 1571 (void) strcat(pr_reason, "LDP "); 1572 1573 /* 1574 * Here we have no PA to work with. 1575 * Scan each line in the ecache to look for 1576 * the one with bad parity. 1577 */ 1578 aflt->flt_addr = AFLT_INV_ADDR; 1579 scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0], 1580 &spf_flt.flt_ec_tag, &spf_flt.flt_ec_lcnt, &oafsr); 1581 acc_afsr |= (oafsr & ~P_AFSR_WP); 1582 1583 /* 1584 * If we found a bad PA, update the state to indicate if it is 1585 * memory or I/O space. This code will be important if we ever 1586 * support cacheable frame buffers. 1587 */ 1588 if (aflt->flt_addr != AFLT_INV_ADDR) { 1589 aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >> 1590 MMU_PAGESHIFT)) ? 1 : 0; 1591 } 1592 1593 if (isus2i || isus2e) 1594 aflt->flt_panic = 1; 1595 1596 cpu_errorq_dispatch((t_afsr & P_AFSR_EDP) ? 1597 FM_EREPORT_CPU_USII_EDP : FM_EREPORT_CPU_USII_LDP, 1598 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1599 aflt->flt_panic); 1600 } 1601 1602 /* 1603 * Timeout and bus error handling. There are two cases to consider: 1604 * 1605 * (1) If we are in the kernel protected by ddi_peek or ddi_poke,we 1606 * have already modified the saved registers so that we will return 1607 * from the trap to the appropriate trampoline routine; otherwise panic. 1608 * 1609 * (2) In user mode, we can simply use our AST mechanism to deliver 1610 * a SIGBUS. We do not log the occurence - processes performing 1611 * device control would generate lots of uninteresting messages. 1612 */ 1613 if (t_afsr & (P_AFSR_TO | P_AFSR_BERR)) { 1614 if (t_afsr & P_AFSR_TO) 1615 (void) strcat(pr_reason, "BTO "); 1616 1617 if (t_afsr & P_AFSR_BERR) 1618 (void) strcat(pr_reason, "BERR "); 1619 1620 spf_flt.flt_type = CPU_BTO_BERR_ERR; 1621 if (aflt->flt_priv && aflt->flt_prot == AFLT_PROT_NONE) { 1622 cpu_errorq_dispatch((t_afsr & P_AFSR_TO) ? 1623 FM_EREPORT_CPU_USII_TO : FM_EREPORT_CPU_USII_BERR, 1624 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1625 aflt->flt_panic); 1626 } 1627 } 1628 1629 /* 1630 * Handle WP: WP happens when the ecache is victimized and a parity 1631 * error was detected on a writeback. The data in question will be 1632 * poisoned as a UE will be written back. The PA is not logged and 1633 * it is possible that it doesn't belong to the trapped thread. The 1634 * WP trap is not fatal, but it could be fatal to someone that 1635 * subsequently accesses the toxic page. We set read_all_memscrub 1636 * to force the memscrubber to read all of memory when it awakens. 1637 * For Sabre/Hummingbird, WP is fatal because the HW doesn't write a 1638 * UE back to poison the data. 1639 */ 1640 if (t_afsr & P_AFSR_WP) { 1641 (void) strcat(pr_reason, "WP "); 1642 if (isus2i || isus2e) { 1643 aflt->flt_panic = 1; 1644 } else { 1645 read_all_memscrub = 1; 1646 } 1647 spf_flt.flt_type = CPU_WP_ERR; 1648 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_WP, 1649 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1650 aflt->flt_panic); 1651 } 1652 1653 /* 1654 * Handle trapping CP error: In Sabre/Hummingbird, parity error in 1655 * the ecache on a copyout due to a PCI DMA read is signaled as a CP. 1656 * This is fatal. 1657 */ 1658 1659 if (t_afsr & P_AFSR_CP) { 1660 if (isus2i || isus2e) { 1661 (void) strcat(pr_reason, "CP "); 1662 aflt->flt_panic = 1; 1663 spf_flt.flt_type = CPU_TRAPPING_CP_ERR; 1664 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CP, 1665 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1666 aflt->flt_panic); 1667 } else { 1668 /* 1669 * Orphan CP: Happens due to signal integrity problem 1670 * on a CPU, where a CP is reported, without reporting 1671 * its associated UE. This is handled by locating the 1672 * bad parity line and would kick off the memscrubber 1673 * to find the UE if in memory or in another's cache. 1674 */ 1675 spf_flt.flt_type = CPU_ORPHAN_CP_ERR; 1676 (void) strcat(pr_reason, "ORPHAN_CP "); 1677 1678 /* 1679 * Here we have no PA to work with. 1680 * Scan each line in the ecache to look for 1681 * the one with bad parity. 1682 */ 1683 aflt->flt_addr = AFLT_INV_ADDR; 1684 scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0], 1685 &spf_flt.flt_ec_tag, &spf_flt.flt_ec_lcnt, 1686 &oafsr); 1687 acc_afsr |= oafsr; 1688 1689 /* 1690 * If we found a bad PA, update the state to indicate 1691 * if it is memory or I/O space. 1692 */ 1693 if (aflt->flt_addr != AFLT_INV_ADDR) { 1694 aflt->flt_in_memory = 1695 (pf_is_memory(aflt->flt_addr >> 1696 MMU_PAGESHIFT)) ? 1 : 0; 1697 } 1698 read_all_memscrub = 1; 1699 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CP, 1700 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1701 aflt->flt_panic); 1702 1703 } 1704 } 1705 1706 /* 1707 * If we queued an error other than WP or CP and we are going to return 1708 * from the trap and the error was in user mode or inside of a 1709 * copy routine, set AST flag so the queue will be drained before 1710 * returning to user mode. 1711 * 1712 * For UE/LDP/EDP, the AST processing will SIGKILL the process 1713 * and send an event to its process contract. 1714 * 1715 * For BERR/BTO, the AST processing will SIGBUS the process. There 1716 * will have been no error queued in this case. 1717 */ 1718 if ((t_afsr & 1719 (P_AFSR_UE | P_AFSR_LDP | P_AFSR_EDP | P_AFSR_BERR | P_AFSR_TO)) && 1720 (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY)) { 1721 int pcb_flag = 0; 1722 1723 if (t_afsr & (P_AFSR_UE | P_AFSR_LDP | P_AFSR_EDP)) 1724 pcb_flag |= ASYNC_HWERR; 1725 1726 if (t_afsr & P_AFSR_BERR) 1727 pcb_flag |= ASYNC_BERR; 1728 1729 if (t_afsr & P_AFSR_TO) 1730 pcb_flag |= ASYNC_BTO; 1731 1732 ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag; 1733 aston(curthread); 1734 action = ACTION_AST_FLAGS; 1735 } 1736 1737 /* 1738 * In response to a deferred error, we must do one of three things: 1739 * (1) set the AST flags, (2) trampoline, or (3) panic. action is 1740 * set in cases (1) and (2) - check that either action is set or 1741 * (3) is true. 1742 * 1743 * On II, the WP writes poisoned data back to memory, which will 1744 * cause a UE and a panic or reboot when read. In this case, we 1745 * don't need to panic at this time. On IIi and IIe, 1746 * aflt->flt_panic is already set above. 1747 */ 1748 ASSERT((aflt->flt_panic != 0) || (action != ACTION_NONE) || 1749 (t_afsr & P_AFSR_WP)); 1750 1751 /* 1752 * Make a final sanity check to make sure we did not get any more async 1753 * errors and accumulate the afsr. 1754 */ 1755 flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size * 2, 1756 cpunodes[CPU->cpu_id].ecache_linesize); 1757 (void) clear_errors(&spf_flt, NULL); 1758 1759 /* 1760 * Take care of a special case: If there is a UE in the ecache flush 1761 * area, we'll see it in flush_ecache(). This will trigger the 1762 * CPU_ADDITIONAL_ERRORS case below. 1763 * 1764 * This could occur if the original error was a UE in the flush area, 1765 * or if the original error was an E$ error that was flushed out of 1766 * the E$ in scan_ecache(). 1767 * 1768 * If it's at the same address that we're already logging, then it's 1769 * probably one of these cases. Clear the bit so we don't trip over 1770 * it on the additional errors case, which could cause an unnecessary 1771 * panic. 1772 */ 1773 if ((aflt->flt_stat & P_AFSR_UE) && aflt->flt_addr == t_afar) 1774 acc_afsr |= aflt->flt_stat & ~P_AFSR_UE; 1775 else 1776 acc_afsr |= aflt->flt_stat; 1777 1778 /* 1779 * Check the acumulated afsr for the important bits. 1780 * Make sure the spf_flt.flt_type value is set, and 1781 * enque an error. 1782 */ 1783 if (acc_afsr & 1784 (P_AFSR_LEVEL1 | P_AFSR_IVUE | P_AFSR_ETP | P_AFSR_ISAP)) { 1785 if (acc_afsr & (P_AFSR_UE | P_AFSR_EDP | P_AFSR_LDP | 1786 P_AFSR_BERR | P_AFSR_TO | P_AFSR_IVUE | P_AFSR_ETP | 1787 P_AFSR_ISAP)) 1788 aflt->flt_panic = 1; 1789 1790 spf_flt.flt_type = CPU_ADDITIONAL_ERR; 1791 aflt->flt_stat = acc_afsr; 1792 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_UNKNOWN, 1793 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1794 aflt->flt_panic); 1795 } 1796 1797 /* 1798 * If aflt->flt_panic is set at this point, we need to panic as the 1799 * result of a trap at TL > 0, or an error we determined to be fatal. 1800 * We've already enqueued the error in one of the if-clauses above, 1801 * and it will be dequeued and logged as part of the panic flow. 1802 */ 1803 if (aflt->flt_panic) { 1804 cpu_aflt_log(CE_PANIC, 1, &spf_flt, CPU_ERRID_FIRST, 1805 "See previous message(s) for details", " %sError(s)", 1806 pr_reason); 1807 } 1808 1809 /* 1810 * Before returning, we must re-enable errors, and 1811 * reset the caches to their boot-up state. 1812 */ 1813 set_lsu(get_lsu() | cache_boot_state); 1814 set_error_enable(EER_ENABLE); 1815 } 1816 1817 /* 1818 * Check for miscellaneous fatal errors and call CE_PANIC if any are seen. 1819 * This routine is shared by the CE and UE handling code. 1820 */ 1821 static void 1822 check_misc_err(spitf_async_flt *spf_flt) 1823 { 1824 struct async_flt *aflt = (struct async_flt *)spf_flt; 1825 char *fatal_str = NULL; 1826 1827 /* 1828 * The ISAP and ETP errors are supposed to cause a POR 1829 * from the system, so in theory we never, ever see these messages. 1830 * ISAP, ETP and IVUE are considered to be fatal. 1831 */ 1832 if (aflt->flt_stat & P_AFSR_ISAP) 1833 fatal_str = " System Address Parity Error on"; 1834 else if (aflt->flt_stat & P_AFSR_ETP) 1835 fatal_str = " Ecache Tag Parity Error on"; 1836 else if (aflt->flt_stat & P_AFSR_IVUE) 1837 fatal_str = " Interrupt Vector Uncorrectable Error on"; 1838 if (fatal_str != NULL) { 1839 cpu_aflt_log(CE_PANIC, 1, spf_flt, CMN_LFLAGS, 1840 NULL, fatal_str); 1841 } 1842 } 1843 1844 /* 1845 * Routine to convert a syndrome into a syndrome code. 1846 */ 1847 static int 1848 synd_to_synd_code(int synd_status, ushort_t synd) 1849 { 1850 if (synd_status != AFLT_STAT_VALID) 1851 return (-1); 1852 1853 /* 1854 * Use the 8-bit syndrome to index the ecc_syndrome_tab 1855 * to get the code indicating which bit(s) is(are) bad. 1856 */ 1857 if ((synd == 0) || (synd >= SYND_TBL_SIZE)) 1858 return (-1); 1859 else 1860 return (ecc_syndrome_tab[synd]); 1861 } 1862 1863 /* ARGSUSED */ 1864 int 1865 cpu_get_mem_sid(char *unum, char *buf, int buflen, int *lenp) 1866 { 1867 return (ENOTSUP); 1868 } 1869 1870 /* ARGSUSED */ 1871 int 1872 cpu_get_mem_offset(uint64_t flt_addr, uint64_t *offp) 1873 { 1874 return (ENOTSUP); 1875 } 1876 1877 /* ARGSUSED */ 1878 int 1879 cpu_get_mem_addr(char *unum, char *sid, uint64_t offset, uint64_t *addrp) 1880 { 1881 return (ENOTSUP); 1882 } 1883 1884 /* 1885 * Routine to return a string identifying the physical name 1886 * associated with a memory/cache error. 1887 */ 1888 /* ARGSUSED */ 1889 int 1890 cpu_get_mem_unum(int synd_status, ushort_t synd, uint64_t afsr, 1891 uint64_t afar, int cpuid, int flt_in_memory, ushort_t flt_status, 1892 char *buf, int buflen, int *lenp) 1893 { 1894 short synd_code; 1895 int ret; 1896 1897 if (flt_in_memory) { 1898 synd_code = synd_to_synd_code(synd_status, synd); 1899 if (synd_code == -1) { 1900 ret = EINVAL; 1901 } else if (prom_get_unum(synd_code, P2ALIGN(afar, 8), 1902 buf, buflen, lenp) != 0) { 1903 ret = EIO; 1904 } else if (*lenp <= 1) { 1905 ret = EINVAL; 1906 } else { 1907 ret = 0; 1908 } 1909 } else { 1910 ret = ENOTSUP; 1911 } 1912 1913 if (ret != 0) { 1914 buf[0] = '\0'; 1915 *lenp = 0; 1916 } 1917 1918 return (ret); 1919 } 1920 1921 /* 1922 * Wrapper for cpu_get_mem_unum() routine that takes an 1923 * async_flt struct rather than explicit arguments. 1924 */ 1925 int 1926 cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt, 1927 char *buf, int buflen, int *lenp) 1928 { 1929 return (cpu_get_mem_unum(synd_status, SYND(aflt->flt_synd), 1930 aflt->flt_stat, aflt->flt_addr, aflt->flt_bus_id, 1931 aflt->flt_in_memory, aflt->flt_status, buf, buflen, lenp)); 1932 } 1933 1934 /* 1935 * This routine is a more generic interface to cpu_get_mem_unum(), 1936 * that may be used by other modules (e.g. mm). 1937 */ 1938 int 1939 cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar, 1940 char *buf, int buflen, int *lenp) 1941 { 1942 int synd_status, flt_in_memory, ret; 1943 char unum[UNUM_NAMLEN]; 1944 1945 /* 1946 * Check for an invalid address. 1947 */ 1948 if (afar == (uint64_t)-1) 1949 return (ENXIO); 1950 1951 if (synd == (uint64_t)-1) 1952 synd_status = AFLT_STAT_INVALID; 1953 else 1954 synd_status = AFLT_STAT_VALID; 1955 1956 flt_in_memory = (pf_is_memory(afar >> MMU_PAGESHIFT)) ? 1 : 0; 1957 1958 if ((ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, *afsr, afar, 1959 CPU->cpu_id, flt_in_memory, 0, unum, UNUM_NAMLEN, lenp)) 1960 != 0) 1961 return (ret); 1962 1963 if (*lenp >= buflen) 1964 return (ENAMETOOLONG); 1965 1966 (void) strncpy(buf, unum, buflen); 1967 1968 return (0); 1969 } 1970 1971 /* 1972 * Routine to return memory information associated 1973 * with a physical address and syndrome. 1974 */ 1975 /* ARGSUSED */ 1976 int 1977 cpu_get_mem_info(uint64_t synd, uint64_t afar, 1978 uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 1979 int *segsp, int *banksp, int *mcidp) 1980 { 1981 return (ENOTSUP); 1982 } 1983 1984 /* 1985 * Routine to return a string identifying the physical 1986 * name associated with a cpuid. 1987 */ 1988 /* ARGSUSED */ 1989 int 1990 cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp) 1991 { 1992 return (ENOTSUP); 1993 } 1994 1995 /* 1996 * This routine returns the size of the kernel's FRU name buffer. 1997 */ 1998 size_t 1999 cpu_get_name_bufsize() 2000 { 2001 return (UNUM_NAMLEN); 2002 } 2003 2004 /* 2005 * Cpu specific log func for UEs. 2006 */ 2007 static void 2008 log_ue_err(struct async_flt *aflt, char *unum) 2009 { 2010 spitf_async_flt *spf_flt = (spitf_async_flt *)aflt; 2011 int len = 0; 2012 2013 #ifdef DEBUG 2014 int afsr_priv = (aflt->flt_stat & P_AFSR_PRIV) ? 1 : 0; 2015 2016 /* 2017 * Paranoid Check for priv mismatch 2018 * Only applicable for UEs 2019 */ 2020 if (afsr_priv != aflt->flt_priv) { 2021 /* 2022 * The priv bits in %tstate and %afsr did not match; we expect 2023 * this to be very rare, so flag it with a message. 2024 */ 2025 cpu_aflt_log(CE_WARN, 2, spf_flt, CPU_ERRID_FIRST, NULL, 2026 ": PRIV bit in TSTATE and AFSR mismatched; " 2027 "TSTATE.PRIV=%d used", (aflt->flt_priv) ? 1 : 0); 2028 2029 /* update saved afsr to reflect the correct priv */ 2030 aflt->flt_stat &= ~P_AFSR_PRIV; 2031 if (aflt->flt_priv) 2032 aflt->flt_stat |= P_AFSR_PRIV; 2033 } 2034 #endif /* DEBUG */ 2035 2036 (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, unum, 2037 UNUM_NAMLEN, &len); 2038 2039 cpu_aflt_log(CE_WARN, 1, spf_flt, UE_LFLAGS, unum, 2040 " Uncorrectable Memory Error on"); 2041 2042 if (SYND(aflt->flt_synd) == 0x3) { 2043 cpu_aflt_log(CE_WARN, 1, spf_flt, CPU_ERRID_FIRST, NULL, 2044 " Syndrome 0x3 indicates that this may not be a " 2045 "memory module problem"); 2046 } 2047 2048 if (aflt->flt_in_memory) 2049 cpu_log_ecmem_info(spf_flt); 2050 } 2051 2052 2053 /* 2054 * The cpu_async_log_err() function is called via the ue_drain() function to 2055 * handle logging for CPU events that are dequeued. As such, it can be invoked 2056 * from softint context, from AST processing in the trap() flow, or from the 2057 * panic flow. We decode the CPU-specific data, and log appropriate messages. 2058 */ 2059 static void 2060 cpu_async_log_err(void *flt) 2061 { 2062 spitf_async_flt *spf_flt = (spitf_async_flt *)flt; 2063 struct async_flt *aflt = (struct async_flt *)flt; 2064 char unum[UNUM_NAMLEN]; 2065 char *space; 2066 char *ecache_scrub_logstr = NULL; 2067 2068 switch (spf_flt->flt_type) { 2069 case CPU_UE_ERR: 2070 /* 2071 * We want to skip logging only if ALL the following 2072 * conditions are true: 2073 * 2074 * 1. We are not panicking 2075 * 2. There is only one error 2076 * 3. That error is a memory error 2077 * 4. The error is caused by the memory scrubber (in 2078 * which case the error will have occurred under 2079 * on_trap protection) 2080 * 5. The error is on a retired page 2081 * 2082 * Note 1: AFLT_PROT_EC is used places other than the memory 2083 * scrubber. However, none of those errors should occur 2084 * on a retired page. 2085 * 2086 * Note 2: In the CE case, these errors are discarded before 2087 * the errorq. In the UE case, we must wait until now -- 2088 * softcall() grabs a mutex, which we can't do at a high PIL. 2089 */ 2090 if (!panicstr && 2091 (aflt->flt_stat & S_AFSR_ALL_ERRS) == P_AFSR_UE && 2092 aflt->flt_prot == AFLT_PROT_EC) { 2093 if (page_retire_check(aflt->flt_addr, NULL) == 0) { 2094 /* Zero the address to clear the error */ 2095 softcall(ecc_page_zero, (void *)aflt->flt_addr); 2096 return; 2097 } 2098 } 2099 2100 /* 2101 * Log the UE and check for causes of this UE error that 2102 * don't cause a trap (Copyback error). cpu_async_error() 2103 * has already checked the i/o buses for us. 2104 */ 2105 log_ue_err(aflt, unum); 2106 if (aflt->flt_in_memory) 2107 cpu_check_allcpus(aflt); 2108 break; 2109 2110 case CPU_EDP_LDP_ERR: 2111 if (aflt->flt_stat & P_AFSR_EDP) 2112 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, 2113 NULL, " EDP event on"); 2114 2115 if (aflt->flt_stat & P_AFSR_LDP) 2116 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, 2117 NULL, " LDP event on"); 2118 2119 /* Log ecache info if exist */ 2120 if (spf_flt->flt_ec_lcnt > 0) { 2121 cpu_log_ecmem_info(spf_flt); 2122 2123 cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, 2124 NULL, " AFAR was derived from E$Tag"); 2125 } else { 2126 cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, 2127 NULL, " No error found in ecache (No fault " 2128 "PA available)"); 2129 } 2130 break; 2131 2132 case CPU_WP_ERR: 2133 /* 2134 * If the memscrub thread hasn't yet read 2135 * all of memory, as we requested in the 2136 * trap handler, then give it a kick to 2137 * make sure it does. 2138 */ 2139 if (!isus2i && !isus2e && read_all_memscrub) 2140 memscrub_run(); 2141 2142 cpu_aflt_log(CE_WARN, 1, spf_flt, WP_LFLAGS, NULL, 2143 " WP event on"); 2144 return; 2145 2146 case CPU_BTO_BERR_ERR: 2147 /* 2148 * A bus timeout or error occurred that was in user mode or not 2149 * in a protected kernel code region. 2150 */ 2151 if (aflt->flt_stat & P_AFSR_BERR) { 2152 cpu_aflt_log(CE_WARN, aflt->flt_panic ? 1 : 2, 2153 spf_flt, BERRTO_LFLAGS, NULL, 2154 " Bus Error on System Bus in %s mode from", 2155 aflt->flt_priv ? "privileged" : "user"); 2156 } 2157 2158 if (aflt->flt_stat & P_AFSR_TO) { 2159 cpu_aflt_log(CE_WARN, aflt->flt_panic ? 1 : 2, 2160 spf_flt, BERRTO_LFLAGS, NULL, 2161 " Timeout on System Bus in %s mode from", 2162 aflt->flt_priv ? "privileged" : "user"); 2163 } 2164 2165 return; 2166 2167 case CPU_PANIC_CP_ERR: 2168 /* 2169 * Process the Copyback (CP) error info (if any) obtained from 2170 * polling all the cpus in the panic flow. This case is only 2171 * entered if we are panicking. 2172 */ 2173 ASSERT(panicstr != NULL); 2174 ASSERT(aflt->flt_id == panic_aflt.flt_id); 2175 2176 /* See which space - this info may not exist */ 2177 if (panic_aflt.flt_status & ECC_D_TRAP) 2178 space = "Data "; 2179 else if (panic_aflt.flt_status & ECC_I_TRAP) 2180 space = "Instruction "; 2181 else 2182 space = ""; 2183 2184 cpu_aflt_log(CE_WARN, 1, spf_flt, CP_LFLAGS, NULL, 2185 " AFAR was derived from UE report," 2186 " CP event on CPU%d (caused %saccess error on %s%d)", 2187 aflt->flt_inst, space, (panic_aflt.flt_status & ECC_IOBUS) ? 2188 "IOBUS" : "CPU", panic_aflt.flt_bus_id); 2189 2190 if (spf_flt->flt_ec_lcnt > 0) 2191 cpu_log_ecmem_info(spf_flt); 2192 else 2193 cpu_aflt_log(CE_WARN, 2, spf_flt, CPU_ERRID_FIRST, 2194 NULL, " No cache dump available"); 2195 2196 return; 2197 2198 case CPU_TRAPPING_CP_ERR: 2199 /* 2200 * For sabre only. This is a copyback ecache parity error due 2201 * to a PCI DMA read. We should be panicking if we get here. 2202 */ 2203 ASSERT(panicstr != NULL); 2204 cpu_aflt_log(CE_WARN, 1, spf_flt, CP_LFLAGS, NULL, 2205 " AFAR was derived from UE report," 2206 " CP event on CPU%d (caused Data access error " 2207 "on PCIBus)", aflt->flt_inst); 2208 return; 2209 2210 /* 2211 * We log the ecache lines of the following states, 2212 * clean_bad_idle, clean_bad_busy, dirty_bad_idle and 2213 * dirty_bad_busy if ecache_scrub_verbose is set and panic 2214 * in addition to logging if ecache_scrub_panic is set. 2215 */ 2216 case CPU_BADLINE_CI_ERR: 2217 ecache_scrub_logstr = "CBI"; 2218 /* FALLTHRU */ 2219 2220 case CPU_BADLINE_CB_ERR: 2221 if (ecache_scrub_logstr == NULL) 2222 ecache_scrub_logstr = "CBB"; 2223 /* FALLTHRU */ 2224 2225 case CPU_BADLINE_DI_ERR: 2226 if (ecache_scrub_logstr == NULL) 2227 ecache_scrub_logstr = "DBI"; 2228 /* FALLTHRU */ 2229 2230 case CPU_BADLINE_DB_ERR: 2231 if (ecache_scrub_logstr == NULL) 2232 ecache_scrub_logstr = "DBB"; 2233 2234 cpu_aflt_log(CE_NOTE, 2, spf_flt, 2235 (CPU_ERRID_FIRST | CPU_FLTCPU), NULL, 2236 " %s event on", ecache_scrub_logstr); 2237 cpu_log_ecmem_info(spf_flt); 2238 2239 return; 2240 2241 case CPU_ORPHAN_CP_ERR: 2242 /* 2243 * Orphan CPs, where the CP bit is set, but when a CPU 2244 * doesn't report a UE. 2245 */ 2246 if (read_all_memscrub) 2247 memscrub_run(); 2248 2249 cpu_aflt_log(CE_NOTE, 2, spf_flt, (CP_LFLAGS | CPU_FLTCPU), 2250 NULL, " Orphan CP event on"); 2251 2252 /* Log ecache info if exist */ 2253 if (spf_flt->flt_ec_lcnt > 0) 2254 cpu_log_ecmem_info(spf_flt); 2255 else 2256 cpu_aflt_log(CE_NOTE, 2, spf_flt, 2257 (CP_LFLAGS | CPU_FLTCPU), NULL, 2258 " No error found in ecache (No fault " 2259 "PA available"); 2260 return; 2261 2262 case CPU_ECACHE_ADDR_PAR_ERR: 2263 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL, 2264 " E$ Tag Address Parity error on"); 2265 cpu_log_ecmem_info(spf_flt); 2266 return; 2267 2268 case CPU_ECACHE_STATE_ERR: 2269 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL, 2270 " E$ Tag State Parity error on"); 2271 cpu_log_ecmem_info(spf_flt); 2272 return; 2273 2274 case CPU_ECACHE_TAG_ERR: 2275 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL, 2276 " E$ Tag scrub event on"); 2277 cpu_log_ecmem_info(spf_flt); 2278 return; 2279 2280 case CPU_ECACHE_ETP_ETS_ERR: 2281 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL, 2282 " AFSR.ETP is set and AFSR.ETS is zero on"); 2283 cpu_log_ecmem_info(spf_flt); 2284 return; 2285 2286 2287 case CPU_ADDITIONAL_ERR: 2288 cpu_aflt_log(CE_WARN, 1, spf_flt, CMN_LFLAGS & ~CPU_SPACE, NULL, 2289 " Additional errors detected during error processing on"); 2290 return; 2291 2292 default: 2293 cmn_err(CE_WARN, "cpu_async_log_err: fault %p has unknown " 2294 "fault type %x", (void *)spf_flt, spf_flt->flt_type); 2295 return; 2296 } 2297 2298 /* ... fall through from the UE, EDP, or LDP cases */ 2299 2300 if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) { 2301 if (!panicstr) { 2302 (void) page_retire(aflt->flt_addr, PR_UE); 2303 } else { 2304 /* 2305 * Clear UEs on panic so that we don't 2306 * get haunted by them during panic or 2307 * after reboot 2308 */ 2309 clearphys(P2ALIGN(aflt->flt_addr, 64), 2310 cpunodes[CPU->cpu_id].ecache_size, 2311 cpunodes[CPU->cpu_id].ecache_linesize); 2312 2313 (void) clear_errors(NULL, NULL); 2314 } 2315 } 2316 2317 /* 2318 * Log final recover message 2319 */ 2320 if (!panicstr) { 2321 if (!aflt->flt_priv) { 2322 cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST, 2323 NULL, " Above Error is in User Mode" 2324 "\n and is fatal: " 2325 "will SIGKILL process and notify contract"); 2326 } else if (aflt->flt_prot == AFLT_PROT_COPY && aflt->flt_core) { 2327 cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST, 2328 NULL, " Above Error detected while dumping core;" 2329 "\n core file will be truncated"); 2330 } else if (aflt->flt_prot == AFLT_PROT_COPY) { 2331 cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST, 2332 NULL, " Above Error is due to Kernel access" 2333 "\n to User space and is fatal: " 2334 "will SIGKILL process and notify contract"); 2335 } else if (aflt->flt_prot == AFLT_PROT_EC) { 2336 cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST, NULL, 2337 " Above Error detected by protected Kernel code" 2338 "\n that will try to clear error from system"); 2339 } 2340 } 2341 } 2342 2343 2344 /* 2345 * Check all cpus for non-trapping UE-causing errors 2346 * In Ultra I/II, we look for copyback errors (CPs) 2347 */ 2348 void 2349 cpu_check_allcpus(struct async_flt *aflt) 2350 { 2351 spitf_async_flt cp; 2352 spitf_async_flt *spf_cpflt = &cp; 2353 struct async_flt *cpflt = (struct async_flt *)&cp; 2354 int pix; 2355 2356 cpflt->flt_id = aflt->flt_id; 2357 cpflt->flt_addr = aflt->flt_addr; 2358 2359 for (pix = 0; pix < NCPU; pix++) { 2360 if (CPU_XCALL_READY(pix)) { 2361 xc_one(pix, (xcfunc_t *)get_cpu_status, 2362 (uint64_t)cpflt, 0); 2363 2364 if (cpflt->flt_stat & P_AFSR_CP) { 2365 char *space; 2366 2367 /* See which space - this info may not exist */ 2368 if (aflt->flt_status & ECC_D_TRAP) 2369 space = "Data "; 2370 else if (aflt->flt_status & ECC_I_TRAP) 2371 space = "Instruction "; 2372 else 2373 space = ""; 2374 2375 cpu_aflt_log(CE_WARN, 1, spf_cpflt, CP_LFLAGS, 2376 NULL, " AFAR was derived from UE report," 2377 " CP event on CPU%d (caused %saccess " 2378 "error on %s%d)", pix, space, 2379 (aflt->flt_status & ECC_IOBUS) ? 2380 "IOBUS" : "CPU", aflt->flt_bus_id); 2381 2382 if (spf_cpflt->flt_ec_lcnt > 0) 2383 cpu_log_ecmem_info(spf_cpflt); 2384 else 2385 cpu_aflt_log(CE_WARN, 2, spf_cpflt, 2386 CPU_ERRID_FIRST, NULL, 2387 " No cache dump available"); 2388 } 2389 } 2390 } 2391 } 2392 2393 #ifdef DEBUG 2394 int test_mp_cp = 0; 2395 #endif 2396 2397 /* 2398 * Cross-call callback routine to tell a CPU to read its own %afsr to check 2399 * for copyback errors and capture relevant information. 2400 */ 2401 static uint_t 2402 get_cpu_status(uint64_t arg) 2403 { 2404 struct async_flt *aflt = (struct async_flt *)arg; 2405 spitf_async_flt *spf_flt = (spitf_async_flt *)arg; 2406 uint64_t afsr; 2407 uint32_t ec_idx; 2408 uint64_t sdbh, sdbl; 2409 int i; 2410 uint32_t ec_set_size; 2411 uchar_t valid; 2412 ec_data_t ec_data[8]; 2413 uint64_t ec_tag, flt_addr_tag, oafsr; 2414 uint64_t *acc_afsr = NULL; 2415 2416 get_asyncflt(&afsr); 2417 if (CPU_PRIVATE(CPU) != NULL) { 2418 acc_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 2419 afsr |= *acc_afsr; 2420 *acc_afsr = 0; 2421 } 2422 2423 #ifdef DEBUG 2424 if (test_mp_cp) 2425 afsr |= P_AFSR_CP; 2426 #endif 2427 aflt->flt_stat = afsr; 2428 2429 if (afsr & P_AFSR_CP) { 2430 /* 2431 * Capture the UDBs 2432 */ 2433 get_udb_errors(&sdbh, &sdbl); 2434 spf_flt->flt_sdbh = (ushort_t)(sdbh & 0x3FF); 2435 spf_flt->flt_sdbl = (ushort_t)(sdbl & 0x3FF); 2436 2437 /* 2438 * Clear CP bit before capturing ecache data 2439 * and AFSR info. 2440 */ 2441 set_asyncflt(P_AFSR_CP); 2442 2443 /* 2444 * See if we can capture the ecache line for the 2445 * fault PA. 2446 * 2447 * Return a valid matching ecache line, if any. 2448 * Otherwise, return the first matching ecache 2449 * line marked invalid. 2450 */ 2451 flt_addr_tag = aflt->flt_addr >> cpu_ec_tag_shift; 2452 ec_set_size = cpunodes[CPU->cpu_id].ecache_size / 2453 ecache_associativity; 2454 spf_flt->flt_ec_lcnt = 0; 2455 2456 for (i = 0, ec_idx = (aflt->flt_addr % ec_set_size); 2457 i < ecache_associativity; i++, ec_idx += ec_set_size) { 2458 get_ecache_dtag(P2ALIGN(ec_idx, 64), 2459 (uint64_t *)&ec_data[0], &ec_tag, &oafsr, 2460 acc_afsr); 2461 2462 if ((ec_tag & cpu_ec_tag_mask) != flt_addr_tag) 2463 continue; 2464 2465 valid = cpu_ec_state_valid & 2466 (uchar_t)((ec_tag & cpu_ec_state_mask) >> 2467 cpu_ec_state_shift); 2468 2469 if (valid || spf_flt->flt_ec_lcnt == 0) { 2470 spf_flt->flt_ec_tag = ec_tag; 2471 bcopy(&ec_data, &spf_flt->flt_ec_data, 2472 sizeof (ec_data)); 2473 spf_flt->flt_ec_lcnt = 1; 2474 2475 if (valid) 2476 break; 2477 } 2478 } 2479 } 2480 return (0); 2481 } 2482 2483 /* 2484 * CPU-module callback for the non-panicking CPUs. This routine is invoked 2485 * from panic_idle() as part of the other CPUs stopping themselves when a 2486 * panic occurs. We need to be VERY careful what we do here, since panicstr 2487 * is NOT set yet and we cannot blow through locks. If panic_aflt is set 2488 * (panic_aflt.flt_id is non-zero), we need to read our %afsr to look for 2489 * CP error information. 2490 */ 2491 void 2492 cpu_async_panic_callb(void) 2493 { 2494 spitf_async_flt cp; 2495 struct async_flt *aflt = (struct async_flt *)&cp; 2496 uint64_t *scrub_afsr; 2497 2498 if (panic_aflt.flt_id != 0) { 2499 aflt->flt_addr = panic_aflt.flt_addr; 2500 (void) get_cpu_status((uint64_t)aflt); 2501 2502 if (CPU_PRIVATE(CPU) != NULL) { 2503 scrub_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 2504 if (*scrub_afsr & P_AFSR_CP) { 2505 aflt->flt_stat |= *scrub_afsr; 2506 *scrub_afsr = 0; 2507 } 2508 } 2509 if (aflt->flt_stat & P_AFSR_CP) { 2510 aflt->flt_id = panic_aflt.flt_id; 2511 aflt->flt_panic = 1; 2512 aflt->flt_inst = CPU->cpu_id; 2513 aflt->flt_class = CPU_FAULT; 2514 cp.flt_type = CPU_PANIC_CP_ERR; 2515 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CP, 2516 (void *)&cp, sizeof (cp), ue_queue, 2517 aflt->flt_panic); 2518 } 2519 } 2520 } 2521 2522 /* 2523 * Turn off all cpu error detection, normally only used for panics. 2524 */ 2525 void 2526 cpu_disable_errors(void) 2527 { 2528 xt_all(set_error_enable_tl1, EER_DISABLE, EER_SET_ABSOLUTE); 2529 } 2530 2531 /* 2532 * Enable errors. 2533 */ 2534 void 2535 cpu_enable_errors(void) 2536 { 2537 xt_all(set_error_enable_tl1, EER_ENABLE, EER_SET_ABSOLUTE); 2538 } 2539 2540 static void 2541 cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err) 2542 { 2543 uint64_t aligned_addr = P2ALIGN(ecc->flt_addr, 8); 2544 int i, loop = 1; 2545 ushort_t ecc_0; 2546 uint64_t paddr; 2547 uint64_t data; 2548 2549 if (verbose) 2550 loop = 8; 2551 for (i = 0; i < loop; i++) { 2552 paddr = aligned_addr + (i * 8); 2553 data = lddphys(paddr); 2554 if (verbose) { 2555 if (ce_err) { 2556 ecc_0 = ecc_gen((uint32_t)(data>>32), 2557 (uint32_t)data); 2558 cpu_aflt_log(CE_CONT, 0, NULL, NO_LFLAGS, 2559 NULL, " Paddr 0x%" PRIx64 ", " 2560 "Data 0x%08x.%08x, ECC 0x%x", paddr, 2561 (uint32_t)(data>>32), (uint32_t)data, 2562 ecc_0); 2563 } else { 2564 cpu_aflt_log(CE_CONT, 0, NULL, NO_LFLAGS, 2565 NULL, " Paddr 0x%" PRIx64 ", " 2566 "Data 0x%08x.%08x", paddr, 2567 (uint32_t)(data>>32), (uint32_t)data); 2568 } 2569 } 2570 } 2571 } 2572 2573 static struct { /* sec-ded-s4ed ecc code */ 2574 uint_t hi, lo; 2575 } ecc_code[8] = { 2576 { 0xee55de23U, 0x16161161U }, 2577 { 0x55eede93U, 0x61612212U }, 2578 { 0xbb557b8cU, 0x49494494U }, 2579 { 0x55bb7b6cU, 0x94948848U }, 2580 { 0x16161161U, 0xee55de23U }, 2581 { 0x61612212U, 0x55eede93U }, 2582 { 0x49494494U, 0xbb557b8cU }, 2583 { 0x94948848U, 0x55bb7b6cU } 2584 }; 2585 2586 static ushort_t 2587 ecc_gen(uint_t high_bytes, uint_t low_bytes) 2588 { 2589 int i, j; 2590 uchar_t checker, bit_mask; 2591 struct { 2592 uint_t hi, lo; 2593 } hex_data, masked_data[8]; 2594 2595 hex_data.hi = high_bytes; 2596 hex_data.lo = low_bytes; 2597 2598 /* mask out bits according to sec-ded-s4ed ecc code */ 2599 for (i = 0; i < 8; i++) { 2600 masked_data[i].hi = hex_data.hi & ecc_code[i].hi; 2601 masked_data[i].lo = hex_data.lo & ecc_code[i].lo; 2602 } 2603 2604 /* 2605 * xor all bits in masked_data[i] to get bit_i of checker, 2606 * where i = 0 to 7 2607 */ 2608 checker = 0; 2609 for (i = 0; i < 8; i++) { 2610 bit_mask = 1 << i; 2611 for (j = 0; j < 32; j++) { 2612 if (masked_data[i].lo & 1) checker ^= bit_mask; 2613 if (masked_data[i].hi & 1) checker ^= bit_mask; 2614 masked_data[i].hi >>= 1; 2615 masked_data[i].lo >>= 1; 2616 } 2617 } 2618 return (checker); 2619 } 2620 2621 /* 2622 * Flush the entire ecache using displacement flush by reading through a 2623 * physical address range as large as the ecache. 2624 */ 2625 void 2626 cpu_flush_ecache(void) 2627 { 2628 flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size * 2, 2629 cpunodes[CPU->cpu_id].ecache_linesize); 2630 } 2631 2632 /* 2633 * read and display the data in the cache line where the 2634 * original ce error occurred. 2635 * This routine is mainly used for debugging new hardware. 2636 */ 2637 void 2638 read_ecc_data(struct async_flt *ecc, short verbose, short ce_err) 2639 { 2640 kpreempt_disable(); 2641 /* disable ECC error traps */ 2642 set_error_enable(EER_ECC_DISABLE); 2643 2644 /* 2645 * flush the ecache 2646 * read the data 2647 * check to see if an ECC error occured 2648 */ 2649 flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size * 2, 2650 cpunodes[CPU->cpu_id].ecache_linesize); 2651 set_lsu(get_lsu() | cache_boot_state); 2652 cpu_read_paddr(ecc, verbose, ce_err); 2653 (void) check_ecc(ecc); 2654 2655 /* enable ECC error traps */ 2656 set_error_enable(EER_ENABLE); 2657 kpreempt_enable(); 2658 } 2659 2660 /* 2661 * Check the AFSR bits for UE/CE persistence. 2662 * If UE or CE errors are detected, the routine will 2663 * clears all the AFSR sticky bits (except CP for 2664 * spitfire/blackbird) and the UDBs. 2665 * if ce_debug or ue_debug is set, log any ue/ce errors detected. 2666 */ 2667 static int 2668 check_ecc(struct async_flt *ecc) 2669 { 2670 uint64_t t_afsr; 2671 uint64_t t_afar; 2672 uint64_t udbh; 2673 uint64_t udbl; 2674 ushort_t udb; 2675 int persistent = 0; 2676 2677 /* 2678 * Capture the AFSR, AFAR and UDBs info 2679 */ 2680 get_asyncflt(&t_afsr); 2681 get_asyncaddr(&t_afar); 2682 t_afar &= SABRE_AFAR_PA; 2683 get_udb_errors(&udbh, &udbl); 2684 2685 if ((t_afsr & P_AFSR_UE) || (t_afsr & P_AFSR_CE)) { 2686 /* 2687 * Clear the errors 2688 */ 2689 clr_datapath(); 2690 2691 if (isus2i || isus2e) 2692 set_asyncflt(t_afsr); 2693 else 2694 set_asyncflt(t_afsr & ~P_AFSR_CP); 2695 2696 /* 2697 * determine whether to check UDBH or UDBL for persistence 2698 */ 2699 if (ecc->flt_synd & UDBL_REG) { 2700 udb = (ushort_t)udbl; 2701 t_afar |= 0x8; 2702 } else { 2703 udb = (ushort_t)udbh; 2704 } 2705 2706 if (ce_debug || ue_debug) { 2707 spitf_async_flt spf_flt; /* for logging */ 2708 struct async_flt *aflt = 2709 (struct async_flt *)&spf_flt; 2710 2711 /* Package the info nicely in the spf_flt struct */ 2712 bzero(&spf_flt, sizeof (spitf_async_flt)); 2713 aflt->flt_stat = t_afsr; 2714 aflt->flt_addr = t_afar; 2715 spf_flt.flt_sdbh = (ushort_t)(udbh & 0x3FF); 2716 spf_flt.flt_sdbl = (ushort_t)(udbl & 0x3FF); 2717 2718 cpu_aflt_log(CE_CONT, 0, &spf_flt, (CPU_AFSR | 2719 CPU_AFAR | CPU_UDBH | CPU_UDBL), NULL, 2720 " check_ecc: Dumping captured error states ..."); 2721 } 2722 2723 /* 2724 * if the fault addresses don't match, not persistent 2725 */ 2726 if (t_afar != ecc->flt_addr) { 2727 return (persistent); 2728 } 2729 2730 /* 2731 * check for UE persistence 2732 * since all DIMMs in the bank are identified for a UE, 2733 * there's no reason to check the syndrome 2734 */ 2735 if ((ecc->flt_stat & P_AFSR_UE) && (t_afsr & P_AFSR_UE)) { 2736 persistent = 1; 2737 } 2738 2739 /* 2740 * check for CE persistence 2741 */ 2742 if ((ecc->flt_stat & P_AFSR_CE) && (t_afsr & P_AFSR_CE)) { 2743 if ((udb & P_DER_E_SYND) == 2744 (ecc->flt_synd & P_DER_E_SYND)) { 2745 persistent = 1; 2746 } 2747 } 2748 } 2749 return (persistent); 2750 } 2751 2752 #ifdef HUMMINGBIRD 2753 #define HB_FULL_DIV 1 2754 #define HB_HALF_DIV 2 2755 #define HB_LOWEST_DIV 8 2756 #define HB_ECLK_INVALID 0xdeadbad 2757 static uint64_t hb_eclk[HB_LOWEST_DIV + 1] = { 2758 HB_ECLK_INVALID, HB_ECLK_1, HB_ECLK_2, HB_ECLK_INVALID, 2759 HB_ECLK_4, HB_ECLK_INVALID, HB_ECLK_6, HB_ECLK_INVALID, 2760 HB_ECLK_8 }; 2761 2762 #define HB_SLOW_DOWN 0 2763 #define HB_SPEED_UP 1 2764 2765 #define SET_ESTAR_MODE(mode) \ 2766 stdphysio(HB_ESTAR_MODE, (mode)); \ 2767 /* \ 2768 * PLL logic requires minimum of 16 clock \ 2769 * cycles to lock to the new clock speed. \ 2770 * Wait 1 usec to satisfy this requirement. \ 2771 */ \ 2772 drv_usecwait(1); 2773 2774 #define CHANGE_REFRESH_COUNT(direction, cur_div, new_div) \ 2775 { \ 2776 volatile uint64_t data; \ 2777 uint64_t count, new_count; \ 2778 clock_t delay; \ 2779 data = lddphysio(HB_MEM_CNTRL0); \ 2780 count = (data & HB_REFRESH_COUNT_MASK) >> \ 2781 HB_REFRESH_COUNT_SHIFT; \ 2782 new_count = (HB_REFRESH_INTERVAL * \ 2783 cpunodes[CPU->cpu_id].clock_freq) / \ 2784 (HB_REFRESH_CLOCKS_PER_COUNT * (new_div) * NANOSEC);\ 2785 data = (data & ~HB_REFRESH_COUNT_MASK) | \ 2786 (new_count << HB_REFRESH_COUNT_SHIFT); \ 2787 stdphysio(HB_MEM_CNTRL0, data); \ 2788 data = lddphysio(HB_MEM_CNTRL0); \ 2789 /* \ 2790 * If we are slowing down the cpu and Memory \ 2791 * Self Refresh is not enabled, it is required \ 2792 * to wait for old refresh count to count-down and \ 2793 * new refresh count to go into effect (let new value \ 2794 * counts down once). \ 2795 */ \ 2796 if ((direction) == HB_SLOW_DOWN && \ 2797 (data & HB_SELF_REFRESH_MASK) == 0) { \ 2798 /* \ 2799 * Each count takes 64 cpu clock cycles \ 2800 * to decrement. Wait for current refresh \ 2801 * count plus new refresh count at current \ 2802 * cpu speed to count down to zero. Round \ 2803 * up the delay time. \ 2804 */ \ 2805 delay = ((HB_REFRESH_CLOCKS_PER_COUNT * \ 2806 (count + new_count) * MICROSEC * (cur_div)) /\ 2807 cpunodes[CPU->cpu_id].clock_freq) + 1; \ 2808 drv_usecwait(delay); \ 2809 } \ 2810 } 2811 2812 #define SET_SELF_REFRESH(bit) \ 2813 { \ 2814 volatile uint64_t data; \ 2815 data = lddphysio(HB_MEM_CNTRL0); \ 2816 data = (data & ~HB_SELF_REFRESH_MASK) | \ 2817 ((bit) << HB_SELF_REFRESH_SHIFT); \ 2818 stdphysio(HB_MEM_CNTRL0, data); \ 2819 data = lddphysio(HB_MEM_CNTRL0); \ 2820 } 2821 #endif /* HUMMINGBIRD */ 2822 2823 /* ARGSUSED */ 2824 void 2825 cpu_change_speed(uint64_t new_divisor, uint64_t arg2) 2826 { 2827 #ifdef HUMMINGBIRD 2828 uint64_t cur_mask, cur_divisor = 0; 2829 volatile uint64_t reg; 2830 processor_info_t *pi = &(CPU->cpu_type_info); 2831 int index; 2832 2833 if ((new_divisor < HB_FULL_DIV || new_divisor > HB_LOWEST_DIV) || 2834 (hb_eclk[new_divisor] == HB_ECLK_INVALID)) { 2835 cmn_err(CE_WARN, "cpu_change_speed: bad divisor 0x%lx", 2836 new_divisor); 2837 return; 2838 } 2839 2840 reg = lddphysio(HB_ESTAR_MODE); 2841 cur_mask = reg & HB_ECLK_MASK; 2842 for (index = HB_FULL_DIV; index <= HB_LOWEST_DIV; index++) { 2843 if (hb_eclk[index] == cur_mask) { 2844 cur_divisor = index; 2845 break; 2846 } 2847 } 2848 2849 if (cur_divisor == 0) 2850 cmn_err(CE_PANIC, "cpu_change_speed: current divisor " 2851 "can't be determined!"); 2852 2853 /* 2854 * If we are already at the requested divisor speed, just 2855 * return. 2856 */ 2857 if (cur_divisor == new_divisor) 2858 return; 2859 2860 if (cur_divisor == HB_FULL_DIV && new_divisor == HB_HALF_DIV) { 2861 CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, cur_divisor, new_divisor); 2862 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2863 SET_SELF_REFRESH(HB_SELF_REFRESH_ENABLE); 2864 2865 } else if (cur_divisor == HB_HALF_DIV && new_divisor == HB_FULL_DIV) { 2866 SET_SELF_REFRESH(HB_SELF_REFRESH_DISABLE); 2867 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2868 /* LINTED: E_FALSE_LOGICAL_EXPR */ 2869 CHANGE_REFRESH_COUNT(HB_SPEED_UP, cur_divisor, new_divisor); 2870 2871 } else if (cur_divisor == HB_FULL_DIV && new_divisor > HB_HALF_DIV) { 2872 /* 2873 * Transition to 1/2 speed first, then to 2874 * lower speed. 2875 */ 2876 CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, cur_divisor, HB_HALF_DIV); 2877 SET_ESTAR_MODE(hb_eclk[HB_HALF_DIV]); 2878 SET_SELF_REFRESH(HB_SELF_REFRESH_ENABLE); 2879 2880 CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, HB_HALF_DIV, new_divisor); 2881 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2882 2883 } else if (cur_divisor > HB_HALF_DIV && new_divisor == HB_FULL_DIV) { 2884 /* 2885 * Transition to 1/2 speed first, then to 2886 * full speed. 2887 */ 2888 SET_ESTAR_MODE(hb_eclk[HB_HALF_DIV]); 2889 /* LINTED: E_FALSE_LOGICAL_EXPR */ 2890 CHANGE_REFRESH_COUNT(HB_SPEED_UP, cur_divisor, HB_HALF_DIV); 2891 2892 SET_SELF_REFRESH(HB_SELF_REFRESH_DISABLE); 2893 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2894 /* LINTED: E_FALSE_LOGICAL_EXPR */ 2895 CHANGE_REFRESH_COUNT(HB_SPEED_UP, HB_HALF_DIV, new_divisor); 2896 2897 } else if (cur_divisor < new_divisor) { 2898 CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, cur_divisor, new_divisor); 2899 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2900 2901 } else if (cur_divisor > new_divisor) { 2902 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2903 /* LINTED: E_FALSE_LOGICAL_EXPR */ 2904 CHANGE_REFRESH_COUNT(HB_SPEED_UP, cur_divisor, new_divisor); 2905 } 2906 CPU->cpu_m.divisor = (uchar_t)new_divisor; 2907 cpu_set_curr_clock(((uint64_t)pi->pi_clock * 1000000) / new_divisor); 2908 #endif 2909 } 2910 2911 /* 2912 * Clear the AFSR sticky bits and the UDBs. For Sabre/Spitfire/Blackbird, 2913 * we clear all the sticky bits. If a non-null pointer to a async fault 2914 * structure argument is passed in, the captured error state (AFSR, AFAR, UDBs) 2915 * info will be returned in the structure. If a non-null pointer to a 2916 * uint64_t is passed in, this will be updated if the CP bit is set in the 2917 * AFSR. The afsr will be returned. 2918 */ 2919 static uint64_t 2920 clear_errors(spitf_async_flt *spf_flt, uint64_t *acc_afsr) 2921 { 2922 struct async_flt *aflt = (struct async_flt *)spf_flt; 2923 uint64_t afsr; 2924 uint64_t udbh, udbl; 2925 2926 get_asyncflt(&afsr); 2927 2928 if ((acc_afsr != NULL) && (afsr & P_AFSR_CP)) 2929 *acc_afsr |= afsr; 2930 2931 if (spf_flt != NULL) { 2932 aflt->flt_stat = afsr; 2933 get_asyncaddr(&aflt->flt_addr); 2934 aflt->flt_addr &= SABRE_AFAR_PA; 2935 2936 get_udb_errors(&udbh, &udbl); 2937 spf_flt->flt_sdbh = (ushort_t)(udbh & 0x3FF); 2938 spf_flt->flt_sdbl = (ushort_t)(udbl & 0x3FF); 2939 } 2940 2941 set_asyncflt(afsr); /* clear afsr */ 2942 clr_datapath(); /* clear udbs */ 2943 return (afsr); 2944 } 2945 2946 /* 2947 * Scan the ecache to look for bad lines. If found, the afsr, afar, e$ data 2948 * tag of the first bad line will be returned. We also return the old-afsr 2949 * (before clearing the sticky bits). The linecnt data will be updated to 2950 * indicate the number of bad lines detected. 2951 */ 2952 static void 2953 scan_ecache(uint64_t *t_afar, ec_data_t *ecache_data, 2954 uint64_t *ecache_tag, int *linecnt, uint64_t *t_afsr) 2955 { 2956 ec_data_t t_ecdata[8]; 2957 uint64_t t_etag, oafsr; 2958 uint64_t pa = AFLT_INV_ADDR; 2959 uint32_t i, j, ecache_sz; 2960 uint64_t acc_afsr = 0; 2961 uint64_t *cpu_afsr = NULL; 2962 2963 if (CPU_PRIVATE(CPU) != NULL) 2964 cpu_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 2965 2966 *linecnt = 0; 2967 ecache_sz = cpunodes[CPU->cpu_id].ecache_size; 2968 2969 for (i = 0; i < ecache_sz; i += 64) { 2970 get_ecache_dtag(i, (uint64_t *)&t_ecdata[0], &t_etag, &oafsr, 2971 cpu_afsr); 2972 acc_afsr |= oafsr; 2973 2974 /* 2975 * Scan through the whole 64 bytes line in 8 8-byte chunks 2976 * looking for the first occurrence of an EDP error. The AFSR 2977 * info is captured for each 8-byte chunk. Note that for 2978 * Spitfire/Blackbird, the AFSR.PSYND is captured by h/w in 2979 * 16-byte chunk granularity (i.e. the AFSR will be the same 2980 * for the high and low 8-byte words within the 16-byte chunk). 2981 * For Sabre/Hummingbird, the AFSR.PSYND is captured in 8-byte 2982 * granularity and only PSYND bits [7:0] are used. 2983 */ 2984 for (j = 0; j < 8; j++) { 2985 ec_data_t *ecdptr = &t_ecdata[j]; 2986 2987 if (ecdptr->ec_afsr & P_AFSR_EDP) { 2988 uint64_t errpa; 2989 ushort_t psynd; 2990 uint32_t ec_set_size = ecache_sz / 2991 ecache_associativity; 2992 2993 /* 2994 * For Spitfire/Blackbird, we need to look at 2995 * the PSYND to make sure that this 8-byte chunk 2996 * is the right one. PSYND bits [15:8] belong 2997 * to the upper 8-byte (even) chunk. Bits 2998 * [7:0] belong to the lower 8-byte chunk (odd). 2999 */ 3000 psynd = ecdptr->ec_afsr & P_AFSR_P_SYND; 3001 if (!isus2i && !isus2e) { 3002 if (j & 0x1) 3003 psynd = psynd & 0xFF; 3004 else 3005 psynd = psynd >> 8; 3006 3007 if (!psynd) 3008 continue; /* wrong chunk */ 3009 } 3010 3011 /* Construct the PA */ 3012 errpa = ((t_etag & cpu_ec_tag_mask) << 3013 cpu_ec_tag_shift) | ((i | (j << 3)) % 3014 ec_set_size); 3015 3016 /* clean up the cache line */ 3017 flushecacheline(P2ALIGN(errpa, 64), 3018 cpunodes[CPU->cpu_id].ecache_size); 3019 3020 oafsr = clear_errors(NULL, cpu_afsr); 3021 acc_afsr |= oafsr; 3022 3023 (*linecnt)++; 3024 3025 /* 3026 * Capture the PA for the first bad line found. 3027 * Return the ecache dump and tag info. 3028 */ 3029 if (pa == AFLT_INV_ADDR) { 3030 int k; 3031 3032 pa = errpa; 3033 for (k = 0; k < 8; k++) 3034 ecache_data[k] = t_ecdata[k]; 3035 *ecache_tag = t_etag; 3036 } 3037 break; 3038 } 3039 } 3040 } 3041 *t_afar = pa; 3042 *t_afsr = acc_afsr; 3043 } 3044 3045 static void 3046 cpu_log_ecmem_info(spitf_async_flt *spf_flt) 3047 { 3048 struct async_flt *aflt = (struct async_flt *)spf_flt; 3049 uint64_t ecache_tag = spf_flt->flt_ec_tag; 3050 char linestr[30]; 3051 char *state_str; 3052 int i; 3053 3054 /* 3055 * Check the ecache tag to make sure it 3056 * is valid. If invalid, a memory dump was 3057 * captured instead of a ecache dump. 3058 */ 3059 if (spf_flt->flt_ec_tag != AFLT_INV_ADDR) { 3060 uchar_t eparity = (uchar_t) 3061 ((ecache_tag & cpu_ec_par_mask) >> cpu_ec_par_shift); 3062 3063 uchar_t estate = (uchar_t) 3064 ((ecache_tag & cpu_ec_state_mask) >> cpu_ec_state_shift); 3065 3066 if (estate == cpu_ec_state_shr) 3067 state_str = "Shared"; 3068 else if (estate == cpu_ec_state_exl) 3069 state_str = "Exclusive"; 3070 else if (estate == cpu_ec_state_own) 3071 state_str = "Owner"; 3072 else if (estate == cpu_ec_state_mod) 3073 state_str = "Modified"; 3074 else 3075 state_str = "Invalid"; 3076 3077 if (spf_flt->flt_ec_lcnt > 1) { 3078 (void) snprintf(linestr, sizeof (linestr), 3079 "Badlines found=%d", spf_flt->flt_ec_lcnt); 3080 } else { 3081 linestr[0] = '\0'; 3082 } 3083 3084 cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, NULL, 3085 " PA=0x%08x.%08x\n E$tag 0x%08x.%08x E$State: %s " 3086 "E$parity 0x%02x %s", (uint32_t)(aflt->flt_addr >> 32), 3087 (uint32_t)aflt->flt_addr, (uint32_t)(ecache_tag >> 32), 3088 (uint32_t)ecache_tag, state_str, 3089 (uint32_t)eparity, linestr); 3090 } else { 3091 cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, NULL, 3092 " E$tag != PA from AFAR; E$line was victimized" 3093 "\n dumping memory from PA 0x%08x.%08x instead", 3094 (uint32_t)(P2ALIGN(aflt->flt_addr, 64) >> 32), 3095 (uint32_t)P2ALIGN(aflt->flt_addr, 64)); 3096 } 3097 3098 /* 3099 * Dump out all 8 8-byte ecache data captured 3100 * For each 8-byte data captured, we check the 3101 * captured afsr's parity syndrome to find out 3102 * which 8-byte chunk is bad. For memory dump, the 3103 * AFSR values were initialized to 0. 3104 */ 3105 for (i = 0; i < 8; i++) { 3106 ec_data_t *ecdptr; 3107 uint_t offset; 3108 ushort_t psynd; 3109 ushort_t bad; 3110 uint64_t edp; 3111 3112 offset = i << 3; /* multiply by 8 */ 3113 ecdptr = &spf_flt->flt_ec_data[i]; 3114 psynd = ecdptr->ec_afsr & P_AFSR_P_SYND; 3115 edp = ecdptr->ec_afsr & P_AFSR_EDP; 3116 3117 /* 3118 * For Sabre/Hummingbird, parity synd is captured only 3119 * in [7:0] of AFSR.PSYND for each 8-byte chunk. 3120 * For spitfire/blackbird, AFSR.PSYND is captured 3121 * in 16-byte granularity. [15:8] represent 3122 * the upper 8 byte and [7:0] the lower 8 byte. 3123 */ 3124 if (isus2i || isus2e || (i & 0x1)) 3125 bad = (psynd & 0xFF); /* check bits [7:0] */ 3126 else 3127 bad = (psynd & 0xFF00); /* check bits [15:8] */ 3128 3129 if (bad && edp) { 3130 cpu_aflt_log(CE_CONT, 2, spf_flt, NO_LFLAGS, NULL, 3131 " E$Data (0x%02x): 0x%08x.%08x " 3132 "*Bad* PSYND=0x%04x", offset, 3133 (uint32_t)(ecdptr->ec_d8 >> 32), 3134 (uint32_t)ecdptr->ec_d8, psynd); 3135 } else { 3136 cpu_aflt_log(CE_CONT, 2, spf_flt, NO_LFLAGS, NULL, 3137 " E$Data (0x%02x): 0x%08x.%08x", offset, 3138 (uint32_t)(ecdptr->ec_d8 >> 32), 3139 (uint32_t)ecdptr->ec_d8); 3140 } 3141 } 3142 } 3143 3144 /* 3145 * Common logging function for all cpu async errors. This function allows the 3146 * caller to generate a single cmn_err() call that logs the appropriate items 3147 * from the fault structure, and implements our rules for AFT logging levels. 3148 * 3149 * ce_code: cmn_err() code (e.g. CE_PANIC, CE_WARN, CE_CONT) 3150 * tagnum: 0, 1, 2, .. generate the [AFT#] tag 3151 * spflt: pointer to spitfire async fault structure 3152 * logflags: bitflags indicating what to output 3153 * endstr: a end string to appear at the end of this log 3154 * fmt: a format string to appear at the beginning of the log 3155 * 3156 * The logflags allows the construction of predetermined output from the spflt 3157 * structure. The individual data items always appear in a consistent order. 3158 * Note that either or both of the spflt structure pointer and logflags may be 3159 * NULL or zero respectively, indicating that the predetermined output 3160 * substrings are not requested in this log. The output looks like this: 3161 * 3162 * [AFT#] <CPU_ERRID_FIRST><fmt string><CPU_FLTCPU> 3163 * <CPU_SPACE><CPU_ERRID> 3164 * newline+4spaces<CPU_AFSR><CPU_AFAR> 3165 * newline+4spaces<CPU_AF_PSYND><CPU_AF_ETS><CPU_FAULTPC> 3166 * newline+4spaces<CPU_UDBH><CPU_UDBL> 3167 * newline+4spaces<CPU_SYND> 3168 * newline+4spaces<endstr> 3169 * 3170 * Note that <endstr> may not start on a newline if we are logging <CPU_PSYND>; 3171 * it is assumed that <endstr> will be the unum string in this case. The size 3172 * of our intermediate formatting buf[] is based on the worst case of all flags 3173 * being enabled. We pass the caller's varargs directly to vcmn_err() for 3174 * formatting so we don't need additional stack space to format them here. 3175 */ 3176 /*PRINTFLIKE6*/ 3177 static void 3178 cpu_aflt_log(int ce_code, int tagnum, spitf_async_flt *spflt, uint_t logflags, 3179 const char *endstr, const char *fmt, ...) 3180 { 3181 struct async_flt *aflt = (struct async_flt *)spflt; 3182 char buf[400], *p, *q; /* see comments about buf[] size above */ 3183 va_list ap; 3184 int console_log_flag; 3185 3186 if ((aflt == NULL) || ((aflt->flt_class == CPU_FAULT) && 3187 (aflt->flt_stat & P_AFSR_LEVEL1)) || 3188 (aflt->flt_panic)) { 3189 console_log_flag = (tagnum < 2) || aft_verbose; 3190 } else { 3191 int verbose = ((aflt->flt_class == BUS_FAULT) || 3192 (aflt->flt_stat & P_AFSR_CE)) ? 3193 ce_verbose_memory : ce_verbose_other; 3194 3195 if (!verbose) 3196 return; 3197 3198 console_log_flag = (verbose > 1); 3199 } 3200 3201 if (console_log_flag) 3202 (void) sprintf(buf, "[AFT%d]", tagnum); 3203 else 3204 (void) sprintf(buf, "![AFT%d]", tagnum); 3205 3206 p = buf + strlen(buf); /* current buffer position */ 3207 q = buf + sizeof (buf); /* pointer past end of buffer */ 3208 3209 if (spflt != NULL && (logflags & CPU_ERRID_FIRST)) { 3210 (void) snprintf(p, (size_t)(q - p), " errID 0x%08x.%08x", 3211 (uint32_t)(aflt->flt_id >> 32), (uint32_t)aflt->flt_id); 3212 p += strlen(p); 3213 } 3214 3215 /* 3216 * Copy the caller's format string verbatim into buf[]. It will be 3217 * formatted by the call to vcmn_err() at the end of this function. 3218 */ 3219 if (fmt != NULL && p < q) { 3220 (void) strncpy(p, fmt, (size_t)(q - p - 1)); 3221 buf[sizeof (buf) - 1] = '\0'; 3222 p += strlen(p); 3223 } 3224 3225 if (spflt != NULL) { 3226 if (logflags & CPU_FLTCPU) { 3227 (void) snprintf(p, (size_t)(q - p), " CPU%d", 3228 aflt->flt_inst); 3229 p += strlen(p); 3230 } 3231 3232 if (logflags & CPU_SPACE) { 3233 if (aflt->flt_status & ECC_D_TRAP) 3234 (void) snprintf(p, (size_t)(q - p), 3235 " Data access"); 3236 else if (aflt->flt_status & ECC_I_TRAP) 3237 (void) snprintf(p, (size_t)(q - p), 3238 " Instruction access"); 3239 p += strlen(p); 3240 } 3241 3242 if (logflags & CPU_TL) { 3243 (void) snprintf(p, (size_t)(q - p), " at TL%s", 3244 aflt->flt_tl ? ">0" : "=0"); 3245 p += strlen(p); 3246 } 3247 3248 if (logflags & CPU_ERRID) { 3249 (void) snprintf(p, (size_t)(q - p), 3250 ", errID 0x%08x.%08x", 3251 (uint32_t)(aflt->flt_id >> 32), 3252 (uint32_t)aflt->flt_id); 3253 p += strlen(p); 3254 } 3255 3256 if (logflags & CPU_AFSR) { 3257 (void) snprintf(p, (size_t)(q - p), 3258 "\n AFSR 0x%08b.%08b", 3259 (uint32_t)(aflt->flt_stat >> 32), AFSR_FMTSTR0, 3260 (uint32_t)aflt->flt_stat, AFSR_FMTSTR1); 3261 p += strlen(p); 3262 } 3263 3264 if (logflags & CPU_AFAR) { 3265 (void) snprintf(p, (size_t)(q - p), " AFAR 0x%08x.%08x", 3266 (uint32_t)(aflt->flt_addr >> 32), 3267 (uint32_t)aflt->flt_addr); 3268 p += strlen(p); 3269 } 3270 3271 if (logflags & CPU_AF_PSYND) { 3272 ushort_t psynd = (ushort_t) 3273 (aflt->flt_stat & P_AFSR_P_SYND); 3274 3275 (void) snprintf(p, (size_t)(q - p), 3276 "\n AFSR.PSYND 0x%04x(Score %02d)", 3277 psynd, ecc_psynd_score(psynd)); 3278 p += strlen(p); 3279 } 3280 3281 if (logflags & CPU_AF_ETS) { 3282 (void) snprintf(p, (size_t)(q - p), " AFSR.ETS 0x%02x", 3283 (uchar_t)((aflt->flt_stat & P_AFSR_ETS) >> 16)); 3284 p += strlen(p); 3285 } 3286 3287 if (logflags & CPU_FAULTPC) { 3288 (void) snprintf(p, (size_t)(q - p), " Fault_PC 0x%p", 3289 (void *)aflt->flt_pc); 3290 p += strlen(p); 3291 } 3292 3293 if (logflags & CPU_UDBH) { 3294 (void) snprintf(p, (size_t)(q - p), 3295 "\n UDBH 0x%04b UDBH.ESYND 0x%02x", 3296 spflt->flt_sdbh, UDB_FMTSTR, 3297 spflt->flt_sdbh & 0xFF); 3298 p += strlen(p); 3299 } 3300 3301 if (logflags & CPU_UDBL) { 3302 (void) snprintf(p, (size_t)(q - p), 3303 " UDBL 0x%04b UDBL.ESYND 0x%02x", 3304 spflt->flt_sdbl, UDB_FMTSTR, 3305 spflt->flt_sdbl & 0xFF); 3306 p += strlen(p); 3307 } 3308 3309 if (logflags & CPU_SYND) { 3310 ushort_t synd = SYND(aflt->flt_synd); 3311 3312 (void) snprintf(p, (size_t)(q - p), 3313 "\n %s Syndrome 0x%x Memory Module ", 3314 UDBL(aflt->flt_synd) ? "UDBL" : "UDBH", synd); 3315 p += strlen(p); 3316 } 3317 } 3318 3319 if (endstr != NULL) { 3320 if (!(logflags & CPU_SYND)) 3321 (void) snprintf(p, (size_t)(q - p), "\n %s", endstr); 3322 else 3323 (void) snprintf(p, (size_t)(q - p), "%s", endstr); 3324 p += strlen(p); 3325 } 3326 3327 if (ce_code == CE_CONT && (p < q - 1)) 3328 (void) strcpy(p, "\n"); /* add final \n if needed */ 3329 3330 va_start(ap, fmt); 3331 vcmn_err(ce_code, buf, ap); 3332 va_end(ap); 3333 } 3334 3335 /* 3336 * Ecache Scrubbing 3337 * 3338 * The basic idea is to prevent lines from sitting in the ecache long enough 3339 * to build up soft errors which can lead to ecache parity errors. 3340 * 3341 * The following rules are observed when flushing the ecache: 3342 * 3343 * 1. When the system is busy, flush bad clean lines 3344 * 2. When the system is idle, flush all clean lines 3345 * 3. When the system is idle, flush good dirty lines 3346 * 4. Never flush bad dirty lines. 3347 * 3348 * modify parity busy idle 3349 * ---------------------------- 3350 * clean good X 3351 * clean bad X X 3352 * dirty good X 3353 * dirty bad 3354 * 3355 * Bad or good refers to whether a line has an E$ parity error or not. 3356 * Clean or dirty refers to the state of the modified bit. We currently 3357 * default the scan rate to 100 (scan 10% of the cache per second). 3358 * 3359 * The following are E$ states and actions. 3360 * 3361 * We encode our state as a 3-bit number, consisting of: 3362 * ECACHE_STATE_MODIFIED (0=clean, 1=dirty) 3363 * ECACHE_STATE_PARITY (0=good, 1=bad) 3364 * ECACHE_STATE_BUSY (0=idle, 1=busy) 3365 * 3366 * We associate a flushing and a logging action with each state. 3367 * 3368 * E$ actions are different for Spitfire and Sabre/Hummingbird modules. 3369 * MIRROR_FLUSH indicates that an E$ line will be flushed for the mirrored 3370 * E$ only, in addition to value being set by ec_flush. 3371 */ 3372 3373 #define ALWAYS_FLUSH 0x1 /* flush E$ line on all E$ types */ 3374 #define NEVER_FLUSH 0x0 /* never the flush the E$ line */ 3375 #define MIRROR_FLUSH 0xF /* flush E$ line on mirrored E$ only */ 3376 3377 struct { 3378 char ec_flush; /* whether to flush or not */ 3379 char ec_log; /* ecache logging */ 3380 char ec_log_type; /* log type info */ 3381 } ec_action[] = { /* states of the E$ line in M P B */ 3382 { ALWAYS_FLUSH, 0, 0 }, /* 0 0 0 clean_good_idle */ 3383 { MIRROR_FLUSH, 0, 0 }, /* 0 0 1 clean_good_busy */ 3384 { ALWAYS_FLUSH, 1, CPU_BADLINE_CI_ERR }, /* 0 1 0 clean_bad_idle */ 3385 { ALWAYS_FLUSH, 1, CPU_BADLINE_CB_ERR }, /* 0 1 1 clean_bad_busy */ 3386 { ALWAYS_FLUSH, 0, 0 }, /* 1 0 0 dirty_good_idle */ 3387 { MIRROR_FLUSH, 0, 0 }, /* 1 0 1 dirty_good_busy */ 3388 { NEVER_FLUSH, 1, CPU_BADLINE_DI_ERR }, /* 1 1 0 dirty_bad_idle */ 3389 { NEVER_FLUSH, 1, CPU_BADLINE_DB_ERR } /* 1 1 1 dirty_bad_busy */ 3390 }; 3391 3392 /* 3393 * Offsets into the ec_action[] that determines clean_good_busy and 3394 * dirty_good_busy lines. 3395 */ 3396 #define ECACHE_CGB_LINE 1 /* E$ clean_good_busy line */ 3397 #define ECACHE_DGB_LINE 5 /* E$ dirty_good_busy line */ 3398 3399 /* 3400 * We are flushing lines which are Clean_Good_Busy and also the lines 3401 * Dirty_Good_Busy. And we only follow it for non-mirrored E$. 3402 */ 3403 #define CGB(x, m) (((x) == ECACHE_CGB_LINE) && (m != ECACHE_CPU_MIRROR)) 3404 #define DGB(x, m) (((x) == ECACHE_DGB_LINE) && (m != ECACHE_CPU_MIRROR)) 3405 3406 #define ECACHE_STATE_MODIFIED 0x4 3407 #define ECACHE_STATE_PARITY 0x2 3408 #define ECACHE_STATE_BUSY 0x1 3409 3410 /* 3411 * If ecache is mirrored ecache_calls_a_sec and ecache_scan_rate are reduced. 3412 */ 3413 int ecache_calls_a_sec_mirrored = 1; 3414 int ecache_lines_per_call_mirrored = 1; 3415 3416 int ecache_scrub_enable = 1; /* ecache scrubbing is on by default */ 3417 int ecache_scrub_verbose = 1; /* prints clean and dirty lines */ 3418 int ecache_scrub_panic = 0; /* panics on a clean and dirty line */ 3419 int ecache_calls_a_sec = 100; /* scrubber calls per sec */ 3420 int ecache_scan_rate = 100; /* scan rate (in tenths of a percent) */ 3421 int ecache_idle_factor = 1; /* increase the scan rate when idle */ 3422 int ecache_flush_clean_good_busy = 50; /* flush rate (in percent) */ 3423 int ecache_flush_dirty_good_busy = 100; /* flush rate (in percent) */ 3424 3425 volatile int ec_timeout_calls = 1; /* timeout calls */ 3426 3427 /* 3428 * Interrupt number and pil for ecache scrubber cross-trap calls. 3429 */ 3430 static uint64_t ecache_scrub_inum; 3431 uint_t ecache_scrub_pil = PIL_9; 3432 3433 /* 3434 * Kstats for the E$ scrubber. 3435 */ 3436 typedef struct ecache_kstat { 3437 kstat_named_t clean_good_idle; /* # of lines scrubbed */ 3438 kstat_named_t clean_good_busy; /* # of lines skipped */ 3439 kstat_named_t clean_bad_idle; /* # of lines scrubbed */ 3440 kstat_named_t clean_bad_busy; /* # of lines scrubbed */ 3441 kstat_named_t dirty_good_idle; /* # of lines scrubbed */ 3442 kstat_named_t dirty_good_busy; /* # of lines skipped */ 3443 kstat_named_t dirty_bad_idle; /* # of lines skipped */ 3444 kstat_named_t dirty_bad_busy; /* # of lines skipped */ 3445 kstat_named_t invalid_lines; /* # of invalid lines */ 3446 kstat_named_t clean_good_busy_flush; /* # of lines scrubbed */ 3447 kstat_named_t dirty_good_busy_flush; /* # of lines scrubbed */ 3448 kstat_named_t tags_cleared; /* # of E$ tags cleared */ 3449 } ecache_kstat_t; 3450 3451 static ecache_kstat_t ec_kstat_template = { 3452 { "clean_good_idle", KSTAT_DATA_ULONG }, 3453 { "clean_good_busy", KSTAT_DATA_ULONG }, 3454 { "clean_bad_idle", KSTAT_DATA_ULONG }, 3455 { "clean_bad_busy", KSTAT_DATA_ULONG }, 3456 { "dirty_good_idle", KSTAT_DATA_ULONG }, 3457 { "dirty_good_busy", KSTAT_DATA_ULONG }, 3458 { "dirty_bad_idle", KSTAT_DATA_ULONG }, 3459 { "dirty_bad_busy", KSTAT_DATA_ULONG }, 3460 { "invalid_lines", KSTAT_DATA_ULONG }, 3461 { "clean_good_busy_flush", KSTAT_DATA_ULONG }, 3462 { "dirty_good_busy_flush", KSTAT_DATA_ULONG }, 3463 { "ecache_tags_cleared", KSTAT_DATA_ULONG } 3464 }; 3465 3466 struct kmem_cache *sf_private_cache; 3467 3468 /* 3469 * Called periodically on each CPU to scan the ecache once a sec. 3470 * adjusting the ecache line index appropriately 3471 */ 3472 void 3473 scrub_ecache_line() 3474 { 3475 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(CPU, sfpr_scrub_misc); 3476 int cpuid = CPU->cpu_id; 3477 uint32_t index = ssmp->ecache_flush_index; 3478 uint64_t ec_size = cpunodes[cpuid].ecache_size; 3479 size_t ec_linesize = cpunodes[cpuid].ecache_linesize; 3480 int nlines = ssmp->ecache_nlines; 3481 uint32_t ec_set_size = ec_size / ecache_associativity; 3482 int ec_mirror = ssmp->ecache_mirror; 3483 ecache_kstat_t *ec_ksp = (ecache_kstat_t *)ssmp->ecache_ksp->ks_data; 3484 3485 int line, scan_lines, flush_clean_busy = 0, flush_dirty_busy = 0; 3486 int mpb; /* encode Modified, Parity, Busy for action */ 3487 uchar_t state; 3488 uint64_t ec_tag, paddr, oafsr, tafsr, nafsr; 3489 uint64_t *acc_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 3490 ec_data_t ec_data[8]; 3491 kstat_named_t *ec_knp; 3492 3493 switch (ec_mirror) { 3494 default: 3495 case ECACHE_CPU_NON_MIRROR: 3496 /* 3497 * The E$ scan rate is expressed in units of tenths of 3498 * a percent. ecache_scan_rate = 1000 (100%) means the 3499 * whole cache is scanned every second. 3500 */ 3501 scan_lines = (nlines * ecache_scan_rate) / 3502 (1000 * ecache_calls_a_sec); 3503 if (!(ssmp->ecache_busy)) { 3504 if (ecache_idle_factor > 0) { 3505 scan_lines *= ecache_idle_factor; 3506 } 3507 } else { 3508 flush_clean_busy = (scan_lines * 3509 ecache_flush_clean_good_busy) / 100; 3510 flush_dirty_busy = (scan_lines * 3511 ecache_flush_dirty_good_busy) / 100; 3512 } 3513 3514 ec_timeout_calls = (ecache_calls_a_sec ? 3515 ecache_calls_a_sec : 1); 3516 break; 3517 3518 case ECACHE_CPU_MIRROR: 3519 scan_lines = ecache_lines_per_call_mirrored; 3520 ec_timeout_calls = (ecache_calls_a_sec_mirrored ? 3521 ecache_calls_a_sec_mirrored : 1); 3522 break; 3523 } 3524 3525 /* 3526 * The ecache scrubber algorithm operates by reading and 3527 * decoding the E$ tag to determine whether the corresponding E$ line 3528 * can be scrubbed. There is a implicit assumption in the scrubber 3529 * logic that the E$ tag is valid. Unfortunately, this assertion is 3530 * flawed since the E$ tag may also be corrupted and have parity errors 3531 * The scrubber logic is enhanced to check the validity of the E$ tag 3532 * before scrubbing. When a parity error is detected in the E$ tag, 3533 * it is possible to recover and scrub the tag under certain conditions 3534 * so that a ETP error condition can be avoided. 3535 */ 3536 3537 for (mpb = line = 0; line < scan_lines; line++, mpb = 0) { 3538 /* 3539 * We get the old-AFSR before clearing the AFSR sticky bits 3540 * in {get_ecache_tag, check_ecache_line, get_ecache_dtag} 3541 * If CP bit is set in the old-AFSR, we log an Orphan CP event. 3542 */ 3543 ec_tag = get_ecache_tag(index, &nafsr, acc_afsr); 3544 state = (uchar_t)((ec_tag & cpu_ec_state_mask) >> 3545 cpu_ec_state_shift); 3546 3547 /* 3548 * ETP is set try to scrub the ecache tag. 3549 */ 3550 if (nafsr & P_AFSR_ETP) { 3551 ecache_scrub_tag_err(nafsr, state, index); 3552 } else if (state & cpu_ec_state_valid) { 3553 /* 3554 * ETP is not set, E$ tag is valid. 3555 * Proceed with the E$ scrubbing. 3556 */ 3557 if (state & cpu_ec_state_dirty) 3558 mpb |= ECACHE_STATE_MODIFIED; 3559 3560 tafsr = check_ecache_line(index, acc_afsr); 3561 3562 if (tafsr & P_AFSR_EDP) { 3563 mpb |= ECACHE_STATE_PARITY; 3564 3565 if (ecache_scrub_verbose || 3566 ecache_scrub_panic) { 3567 get_ecache_dtag(P2ALIGN(index, 64), 3568 (uint64_t *)&ec_data[0], 3569 &ec_tag, &oafsr, acc_afsr); 3570 } 3571 } 3572 3573 if (ssmp->ecache_busy) 3574 mpb |= ECACHE_STATE_BUSY; 3575 3576 ec_knp = (kstat_named_t *)ec_ksp + mpb; 3577 ec_knp->value.ul++; 3578 3579 paddr = ((ec_tag & cpu_ec_tag_mask) << 3580 cpu_ec_tag_shift) | (index % ec_set_size); 3581 3582 /* 3583 * We flush the E$ lines depending on the ec_flush, 3584 * we additionally flush clean_good_busy and 3585 * dirty_good_busy lines for mirrored E$. 3586 */ 3587 if (ec_action[mpb].ec_flush == ALWAYS_FLUSH) { 3588 flushecacheline(paddr, ec_size); 3589 } else if ((ec_mirror == ECACHE_CPU_MIRROR) && 3590 (ec_action[mpb].ec_flush == MIRROR_FLUSH)) { 3591 flushecacheline(paddr, ec_size); 3592 } else if (ec_action[mpb].ec_flush == NEVER_FLUSH) { 3593 softcall(ecache_page_retire, (void *)paddr); 3594 } 3595 3596 /* 3597 * Conditionally flush both the clean_good and 3598 * dirty_good lines when busy. 3599 */ 3600 if (CGB(mpb, ec_mirror) && (flush_clean_busy > 0)) { 3601 flush_clean_busy--; 3602 flushecacheline(paddr, ec_size); 3603 ec_ksp->clean_good_busy_flush.value.ul++; 3604 } else if (DGB(mpb, ec_mirror) && 3605 (flush_dirty_busy > 0)) { 3606 flush_dirty_busy--; 3607 flushecacheline(paddr, ec_size); 3608 ec_ksp->dirty_good_busy_flush.value.ul++; 3609 } 3610 3611 if (ec_action[mpb].ec_log && (ecache_scrub_verbose || 3612 ecache_scrub_panic)) { 3613 ecache_scrub_log(ec_data, ec_tag, paddr, mpb, 3614 tafsr); 3615 } 3616 3617 } else { 3618 ec_ksp->invalid_lines.value.ul++; 3619 } 3620 3621 if ((index += ec_linesize) >= ec_size) 3622 index = 0; 3623 3624 } 3625 3626 /* 3627 * set the ecache scrub index for the next time around 3628 */ 3629 ssmp->ecache_flush_index = index; 3630 3631 if (*acc_afsr & P_AFSR_CP) { 3632 uint64_t ret_afsr; 3633 3634 ret_afsr = ecache_scrub_misc_err(CPU_ORPHAN_CP_ERR, *acc_afsr); 3635 if ((ret_afsr & P_AFSR_CP) == 0) 3636 *acc_afsr = 0; 3637 } 3638 } 3639 3640 /* 3641 * Handler for ecache_scrub_inum softint. Call scrub_ecache_line until 3642 * we decrement the outstanding request count to zero. 3643 */ 3644 3645 /*ARGSUSED*/ 3646 uint_t 3647 scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2) 3648 { 3649 int i; 3650 int outstanding; 3651 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(CPU, sfpr_scrub_misc); 3652 uint32_t *countp = &ssmp->ec_scrub_outstanding; 3653 3654 do { 3655 outstanding = *countp; 3656 ASSERT(outstanding > 0); 3657 for (i = 0; i < outstanding; i++) 3658 scrub_ecache_line(); 3659 } while (atomic_add_32_nv(countp, -outstanding)); 3660 3661 return (DDI_INTR_CLAIMED); 3662 } 3663 3664 /* 3665 * force each cpu to perform an ecache scrub, called from a timeout 3666 */ 3667 extern xcfunc_t ecache_scrubreq_tl1; 3668 3669 void 3670 do_scrub_ecache_line(void) 3671 { 3672 long delta; 3673 3674 if (ecache_calls_a_sec > hz) 3675 ecache_calls_a_sec = hz; 3676 else if (ecache_calls_a_sec <= 0) 3677 ecache_calls_a_sec = 1; 3678 3679 if (ecache_calls_a_sec_mirrored > hz) 3680 ecache_calls_a_sec_mirrored = hz; 3681 else if (ecache_calls_a_sec_mirrored <= 0) 3682 ecache_calls_a_sec_mirrored = 1; 3683 3684 if (ecache_scrub_enable) { 3685 xt_all(ecache_scrubreq_tl1, ecache_scrub_inum, 0); 3686 delta = hz / ec_timeout_calls; 3687 } else { 3688 delta = hz; 3689 } 3690 3691 (void) realtime_timeout((void(*)(void *))do_scrub_ecache_line, 0, 3692 delta); 3693 } 3694 3695 /* 3696 * initialization for ecache scrubbing 3697 * This routine is called AFTER all cpus have had cpu_init_private called 3698 * to initialize their private data areas. 3699 */ 3700 void 3701 cpu_init_cache_scrub(void) 3702 { 3703 if (ecache_calls_a_sec > hz) { 3704 cmn_err(CE_NOTE, "ecache_calls_a_sec set too high (%d); " 3705 "resetting to hz (%d)", ecache_calls_a_sec, hz); 3706 ecache_calls_a_sec = hz; 3707 } 3708 3709 /* 3710 * Register softint for ecache scrubbing. 3711 */ 3712 ecache_scrub_inum = add_softintr(ecache_scrub_pil, 3713 scrub_ecache_line_intr, NULL, SOFTINT_MT); 3714 3715 /* 3716 * kick off the scrubbing using realtime timeout 3717 */ 3718 (void) realtime_timeout((void(*)(void *))do_scrub_ecache_line, 0, 3719 hz / ecache_calls_a_sec); 3720 } 3721 3722 /* 3723 * Unset the busy flag for this cpu. 3724 */ 3725 void 3726 cpu_idle_ecache_scrub(struct cpu *cp) 3727 { 3728 if (CPU_PRIVATE(cp) != NULL) { 3729 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, 3730 sfpr_scrub_misc); 3731 ssmp->ecache_busy = ECACHE_CPU_IDLE; 3732 } 3733 } 3734 3735 /* 3736 * Set the busy flag for this cpu. 3737 */ 3738 void 3739 cpu_busy_ecache_scrub(struct cpu *cp) 3740 { 3741 if (CPU_PRIVATE(cp) != NULL) { 3742 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, 3743 sfpr_scrub_misc); 3744 ssmp->ecache_busy = ECACHE_CPU_BUSY; 3745 } 3746 } 3747 3748 /* 3749 * initialize the ecache scrubber data structures 3750 * The global entry point cpu_init_private replaces this entry point. 3751 * 3752 */ 3753 static void 3754 cpu_init_ecache_scrub_dr(struct cpu *cp) 3755 { 3756 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, sfpr_scrub_misc); 3757 int cpuid = cp->cpu_id; 3758 3759 /* 3760 * intialize bookkeeping for cache scrubbing 3761 */ 3762 bzero(ssmp, sizeof (spitfire_scrub_misc_t)); 3763 3764 ssmp->ecache_flush_index = 0; 3765 3766 ssmp->ecache_nlines = 3767 cpunodes[cpuid].ecache_size / cpunodes[cpuid].ecache_linesize; 3768 3769 /* 3770 * Determine whether we are running on mirrored SRAM 3771 */ 3772 3773 if (cpunodes[cpuid].msram == ECACHE_CPU_MIRROR) 3774 ssmp->ecache_mirror = ECACHE_CPU_MIRROR; 3775 else 3776 ssmp->ecache_mirror = ECACHE_CPU_NON_MIRROR; 3777 3778 cpu_busy_ecache_scrub(cp); 3779 3780 /* 3781 * initialize the kstats 3782 */ 3783 ecache_kstat_init(cp); 3784 } 3785 3786 /* 3787 * uninitialize the ecache scrubber data structures 3788 * The global entry point cpu_uninit_private replaces this entry point. 3789 */ 3790 static void 3791 cpu_uninit_ecache_scrub_dr(struct cpu *cp) 3792 { 3793 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, sfpr_scrub_misc); 3794 3795 if (ssmp->ecache_ksp != NULL) { 3796 kstat_delete(ssmp->ecache_ksp); 3797 ssmp->ecache_ksp = NULL; 3798 } 3799 3800 /* 3801 * un-initialize bookkeeping for cache scrubbing 3802 */ 3803 bzero(ssmp, sizeof (spitfire_scrub_misc_t)); 3804 3805 cpu_idle_ecache_scrub(cp); 3806 } 3807 3808 struct kmem_cache *sf_private_cache; 3809 3810 /* 3811 * Cpu private initialization. This includes allocating the cpu_private 3812 * data structure, initializing it, and initializing the scrubber for this 3813 * cpu. This is called once for EVERY cpu, including CPU 0. This function 3814 * calls cpu_init_ecache_scrub_dr to init the scrubber. 3815 * We use kmem_cache_create for the spitfire private data structure because it 3816 * needs to be allocated on a S_ECACHE_MAX_LSIZE (64) byte boundary. 3817 */ 3818 void 3819 cpu_init_private(struct cpu *cp) 3820 { 3821 spitfire_private_t *sfprp; 3822 3823 ASSERT(CPU_PRIVATE(cp) == NULL); 3824 3825 /* 3826 * If the sf_private_cache has not been created, create it. 3827 */ 3828 if (sf_private_cache == NULL) { 3829 sf_private_cache = kmem_cache_create("sf_private_cache", 3830 sizeof (spitfire_private_t), S_ECACHE_MAX_LSIZE, NULL, 3831 NULL, NULL, NULL, NULL, 0); 3832 ASSERT(sf_private_cache); 3833 } 3834 3835 sfprp = CPU_PRIVATE(cp) = kmem_cache_alloc(sf_private_cache, KM_SLEEP); 3836 3837 bzero(sfprp, sizeof (spitfire_private_t)); 3838 3839 cpu_init_ecache_scrub_dr(cp); 3840 } 3841 3842 /* 3843 * Cpu private unitialization. Uninitialize the Ecache scrubber and 3844 * deallocate the scrubber data structures and cpu_private data structure. 3845 * For now, this function just calls cpu_unint_ecache_scrub_dr to uninit 3846 * the scrubber for the specified cpu. 3847 */ 3848 void 3849 cpu_uninit_private(struct cpu *cp) 3850 { 3851 ASSERT(CPU_PRIVATE(cp)); 3852 3853 cpu_uninit_ecache_scrub_dr(cp); 3854 kmem_cache_free(sf_private_cache, CPU_PRIVATE(cp)); 3855 CPU_PRIVATE(cp) = NULL; 3856 } 3857 3858 /* 3859 * initialize the ecache kstats for each cpu 3860 */ 3861 static void 3862 ecache_kstat_init(struct cpu *cp) 3863 { 3864 struct kstat *ksp; 3865 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, sfpr_scrub_misc); 3866 3867 ASSERT(ssmp != NULL); 3868 3869 if ((ksp = kstat_create("unix", cp->cpu_id, "ecache_kstat", "misc", 3870 KSTAT_TYPE_NAMED, 3871 sizeof (ecache_kstat_t) / sizeof (kstat_named_t), 3872 KSTAT_FLAG_WRITABLE)) == NULL) { 3873 ssmp->ecache_ksp = NULL; 3874 cmn_err(CE_NOTE, "!ecache_kstat_init(%d) failed\n", cp->cpu_id); 3875 return; 3876 } 3877 3878 ssmp->ecache_ksp = ksp; 3879 bcopy(&ec_kstat_template, ksp->ks_data, sizeof (ecache_kstat_t)); 3880 kstat_install(ksp); 3881 } 3882 3883 /* 3884 * log the bad ecache information 3885 */ 3886 static void 3887 ecache_scrub_log(ec_data_t *ec_data, uint64_t ec_tag, uint64_t paddr, int mpb, 3888 uint64_t afsr) 3889 { 3890 spitf_async_flt spf_flt; 3891 struct async_flt *aflt; 3892 int i; 3893 char *class; 3894 3895 bzero(&spf_flt, sizeof (spitf_async_flt)); 3896 aflt = &spf_flt.cmn_asyncflt; 3897 3898 for (i = 0; i < 8; i++) { 3899 spf_flt.flt_ec_data[i] = ec_data[i]; 3900 } 3901 3902 spf_flt.flt_ec_tag = ec_tag; 3903 3904 if (mpb < (sizeof (ec_action) / sizeof (ec_action[0]))) { 3905 spf_flt.flt_type = ec_action[mpb].ec_log_type; 3906 } else spf_flt.flt_type = (ushort_t)mpb; 3907 3908 aflt->flt_inst = CPU->cpu_id; 3909 aflt->flt_class = CPU_FAULT; 3910 aflt->flt_id = gethrtime_waitfree(); 3911 aflt->flt_addr = paddr; 3912 aflt->flt_stat = afsr; 3913 aflt->flt_panic = (uchar_t)ecache_scrub_panic; 3914 3915 switch (mpb) { 3916 case CPU_ECACHE_TAG_ERR: 3917 case CPU_ECACHE_ADDR_PAR_ERR: 3918 case CPU_ECACHE_ETP_ETS_ERR: 3919 case CPU_ECACHE_STATE_ERR: 3920 class = FM_EREPORT_CPU_USII_ESCRUB_TAG; 3921 break; 3922 default: 3923 class = FM_EREPORT_CPU_USII_ESCRUB_DATA; 3924 break; 3925 } 3926 3927 cpu_errorq_dispatch(class, (void *)&spf_flt, sizeof (spf_flt), 3928 ue_queue, aflt->flt_panic); 3929 3930 if (aflt->flt_panic) 3931 cmn_err(CE_PANIC, "ecache_scrub_panic set and bad E$" 3932 "line detected"); 3933 } 3934 3935 /* 3936 * Process an ecache error that occured during the E$ scrubbing. 3937 * We do the ecache scan to find the bad line, flush the bad line 3938 * and start the memscrubber to find any UE (in memory or in another cache) 3939 */ 3940 static uint64_t 3941 ecache_scrub_misc_err(int type, uint64_t afsr) 3942 { 3943 spitf_async_flt spf_flt; 3944 struct async_flt *aflt; 3945 uint64_t oafsr; 3946 3947 bzero(&spf_flt, sizeof (spitf_async_flt)); 3948 aflt = &spf_flt.cmn_asyncflt; 3949 3950 /* 3951 * Scan each line in the cache to look for the one 3952 * with bad parity 3953 */ 3954 aflt->flt_addr = AFLT_INV_ADDR; 3955 scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0], 3956 &spf_flt.flt_ec_tag, &spf_flt.flt_ec_lcnt, &oafsr); 3957 3958 if (oafsr & P_AFSR_CP) { 3959 uint64_t *cp_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 3960 *cp_afsr |= oafsr; 3961 } 3962 3963 /* 3964 * If we found a bad PA, update the state to indicate if it is 3965 * memory or I/O space. 3966 */ 3967 if (aflt->flt_addr != AFLT_INV_ADDR) { 3968 aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >> 3969 MMU_PAGESHIFT)) ? 1 : 0; 3970 } 3971 3972 spf_flt.flt_type = (ushort_t)type; 3973 3974 aflt->flt_inst = CPU->cpu_id; 3975 aflt->flt_class = CPU_FAULT; 3976 aflt->flt_id = gethrtime_waitfree(); 3977 aflt->flt_status = afsr; 3978 aflt->flt_panic = (uchar_t)ecache_scrub_panic; 3979 3980 /* 3981 * We have the bad line, flush that line and start 3982 * the memscrubber. 3983 */ 3984 if (spf_flt.flt_ec_lcnt > 0) { 3985 flushecacheline(P2ALIGN(aflt->flt_addr, 64), 3986 cpunodes[CPU->cpu_id].ecache_size); 3987 read_all_memscrub = 1; 3988 memscrub_run(); 3989 } 3990 3991 cpu_errorq_dispatch((type == CPU_ORPHAN_CP_ERR) ? 3992 FM_EREPORT_CPU_USII_CP : FM_EREPORT_CPU_USII_UNKNOWN, 3993 (void *)&spf_flt, sizeof (spf_flt), ue_queue, aflt->flt_panic); 3994 3995 return (oafsr); 3996 } 3997 3998 static void 3999 ecache_scrub_tag_err(uint64_t afsr, uchar_t state, uint32_t index) 4000 { 4001 ushort_t afsr_ets = (afsr & P_AFSR_ETS) >> P_AFSR_ETS_SHIFT; 4002 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(CPU, sfpr_scrub_misc); 4003 ecache_kstat_t *ec_ksp = (ecache_kstat_t *)ssmp->ecache_ksp->ks_data; 4004 uint64_t ec_tag, paddr, oafsr; 4005 ec_data_t ec_data[8]; 4006 int cpuid = CPU->cpu_id; 4007 uint32_t ec_set_size = cpunodes[cpuid].ecache_size / 4008 ecache_associativity; 4009 uint64_t *cpu_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 4010 4011 get_ecache_dtag(P2ALIGN(index, 64), (uint64_t *)&ec_data[0], &ec_tag, 4012 &oafsr, cpu_afsr); 4013 paddr = ((ec_tag & cpu_ec_tag_mask) << cpu_ec_tag_shift) | 4014 (index % ec_set_size); 4015 4016 /* 4017 * E$ tag state has good parity 4018 */ 4019 if ((afsr_ets & cpu_ec_state_parity) == 0) { 4020 if (afsr_ets & cpu_ec_parity) { 4021 /* 4022 * E$ tag state bits indicate the line is clean, 4023 * invalidate the E$ tag and continue. 4024 */ 4025 if (!(state & cpu_ec_state_dirty)) { 4026 /* 4027 * Zero the tag and mark the state invalid 4028 * with good parity for the tag. 4029 */ 4030 if (isus2i || isus2e) 4031 write_hb_ec_tag_parity(index); 4032 else 4033 write_ec_tag_parity(index); 4034 4035 /* Sync with the dual tag */ 4036 flushecacheline(0, 4037 cpunodes[CPU->cpu_id].ecache_size); 4038 ec_ksp->tags_cleared.value.ul++; 4039 ecache_scrub_log(ec_data, ec_tag, paddr, 4040 CPU_ECACHE_TAG_ERR, afsr); 4041 return; 4042 } else { 4043 ecache_scrub_log(ec_data, ec_tag, paddr, 4044 CPU_ECACHE_ADDR_PAR_ERR, afsr); 4045 cmn_err(CE_PANIC, " E$ tag address has bad" 4046 " parity"); 4047 } 4048 } else if ((afsr_ets & cpu_ec_parity) == 0) { 4049 /* 4050 * ETS is zero but ETP is set 4051 */ 4052 ecache_scrub_log(ec_data, ec_tag, paddr, 4053 CPU_ECACHE_ETP_ETS_ERR, afsr); 4054 cmn_err(CE_PANIC, "AFSR.ETP is set and" 4055 " AFSR.ETS is zero"); 4056 } 4057 } else { 4058 /* 4059 * E$ tag state bit has a bad parity 4060 */ 4061 ecache_scrub_log(ec_data, ec_tag, paddr, 4062 CPU_ECACHE_STATE_ERR, afsr); 4063 cmn_err(CE_PANIC, "E$ tag state has bad parity"); 4064 } 4065 } 4066 4067 static void 4068 ecache_page_retire(void *arg) 4069 { 4070 uint64_t paddr = (uint64_t)arg; 4071 (void) page_retire(paddr, PR_UE); 4072 } 4073 4074 void 4075 sticksync_slave(void) 4076 {} 4077 4078 void 4079 sticksync_master(void) 4080 {} 4081 4082 /*ARGSUSED*/ 4083 void 4084 cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t bpp) 4085 {} 4086 4087 void 4088 cpu_run_bus_error_handlers(struct async_flt *aflt, int expected) 4089 { 4090 int status; 4091 ddi_fm_error_t de; 4092 4093 bzero(&de, sizeof (ddi_fm_error_t)); 4094 4095 de.fme_version = DDI_FME_VERSION; 4096 de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, 4097 FM_ENA_FMT1); 4098 de.fme_flag = expected; 4099 de.fme_bus_specific = (void *)aflt->flt_addr; 4100 status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de); 4101 4102 if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL)) 4103 aflt->flt_panic = 1; 4104 } 4105 4106 /*ARGSUSED*/ 4107 void 4108 cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz, 4109 errorq_t *eqp, uint_t flag) 4110 { 4111 struct async_flt *aflt = (struct async_flt *)payload; 4112 4113 aflt->flt_erpt_class = error_class; 4114 errorq_dispatch(eqp, payload, payload_sz, flag); 4115 } 4116 4117 #define MAX_SIMM 8 4118 4119 struct ce_info { 4120 char name[UNUM_NAMLEN]; 4121 uint64_t intermittent_total; 4122 uint64_t persistent_total; 4123 uint64_t sticky_total; 4124 unsigned short leaky_bucket_cnt; 4125 }; 4126 4127 /* 4128 * Separately-defined structure for use in reporting the ce_info 4129 * to SunVTS without exposing the internal layout and implementation 4130 * of struct ce_info. 4131 */ 4132 static struct ecc_error_info ecc_error_info_data = { 4133 { "version", KSTAT_DATA_UINT32 }, 4134 { "maxcount", KSTAT_DATA_UINT32 }, 4135 { "count", KSTAT_DATA_UINT32 } 4136 }; 4137 static const size_t ecc_error_info_ndata = sizeof (ecc_error_info_data) / 4138 sizeof (struct kstat_named); 4139 4140 #if KSTAT_CE_UNUM_NAMLEN < UNUM_NAMLEN 4141 #error "Need to rev ecc_error_info version and update KSTAT_CE_UNUM_NAMLEN" 4142 #endif 4143 4144 struct ce_info *mem_ce_simm = NULL; 4145 size_t mem_ce_simm_size = 0; 4146 4147 /* 4148 * Default values for the number of CE's allowed per interval. 4149 * Interval is defined in minutes 4150 * SOFTERR_MIN_TIMEOUT is defined in microseconds 4151 */ 4152 #define SOFTERR_LIMIT_DEFAULT 2 4153 #define SOFTERR_INTERVAL_DEFAULT 1440 /* This is 24 hours */ 4154 #define SOFTERR_MIN_TIMEOUT (60 * MICROSEC) /* This is 1 minute */ 4155 #define TIMEOUT_NONE ((timeout_id_t)0) 4156 #define TIMEOUT_SET ((timeout_id_t)1) 4157 4158 /* 4159 * timeout identifer for leaky_bucket 4160 */ 4161 static timeout_id_t leaky_bucket_timeout_id = TIMEOUT_NONE; 4162 4163 /* 4164 * Tunables for maximum number of allowed CE's in a given time 4165 */ 4166 int ecc_softerr_limit = SOFTERR_LIMIT_DEFAULT; 4167 int ecc_softerr_interval = SOFTERR_INTERVAL_DEFAULT; 4168 4169 void 4170 cpu_mp_init(void) 4171 { 4172 size_t size = cpu_aflt_size(); 4173 size_t i; 4174 kstat_t *ksp; 4175 4176 /* 4177 * Initialize the CE error handling buffers. 4178 */ 4179 mem_ce_simm_size = MAX_SIMM * max_ncpus; 4180 size = sizeof (struct ce_info) * mem_ce_simm_size; 4181 mem_ce_simm = kmem_zalloc(size, KM_SLEEP); 4182 4183 ksp = kstat_create("unix", 0, "ecc-info", "misc", 4184 KSTAT_TYPE_NAMED, ecc_error_info_ndata, KSTAT_FLAG_VIRTUAL); 4185 if (ksp != NULL) { 4186 ksp->ks_data = (struct kstat_named *)&ecc_error_info_data; 4187 ecc_error_info_data.version.value.ui32 = KSTAT_CE_INFO_VER; 4188 ecc_error_info_data.maxcount.value.ui32 = mem_ce_simm_size; 4189 ecc_error_info_data.count.value.ui32 = 0; 4190 kstat_install(ksp); 4191 } 4192 4193 for (i = 0; i < mem_ce_simm_size; i++) { 4194 struct kstat_ecc_mm_info *kceip; 4195 4196 kceip = kmem_zalloc(sizeof (struct kstat_ecc_mm_info), 4197 KM_SLEEP); 4198 ksp = kstat_create("mm", i, "ecc-info", "misc", 4199 KSTAT_TYPE_NAMED, 4200 sizeof (struct kstat_ecc_mm_info) / sizeof (kstat_named_t), 4201 KSTAT_FLAG_VIRTUAL); 4202 if (ksp != NULL) { 4203 /* 4204 * Re-declare ks_data_size to include room for the 4205 * UNUM name since we don't have KSTAT_FLAG_VAR_SIZE 4206 * set. 4207 */ 4208 ksp->ks_data_size = sizeof (struct kstat_ecc_mm_info) + 4209 KSTAT_CE_UNUM_NAMLEN; 4210 ksp->ks_data = kceip; 4211 kstat_named_init(&kceip->name, 4212 "name", KSTAT_DATA_STRING); 4213 kstat_named_init(&kceip->intermittent_total, 4214 "intermittent_total", KSTAT_DATA_UINT64); 4215 kstat_named_init(&kceip->persistent_total, 4216 "persistent_total", KSTAT_DATA_UINT64); 4217 kstat_named_init(&kceip->sticky_total, 4218 "sticky_total", KSTAT_DATA_UINT64); 4219 /* 4220 * Use the default snapshot routine as it knows how to 4221 * deal with named kstats with long strings. 4222 */ 4223 ksp->ks_update = ecc_kstat_update; 4224 kstat_install(ksp); 4225 } else { 4226 kmem_free(kceip, sizeof (struct kstat_ecc_mm_info)); 4227 } 4228 } 4229 } 4230 4231 /*ARGSUSED*/ 4232 static void 4233 leaky_bucket_timeout(void *arg) 4234 { 4235 int i; 4236 struct ce_info *psimm = mem_ce_simm; 4237 4238 for (i = 0; i < mem_ce_simm_size; i++) { 4239 if (psimm[i].leaky_bucket_cnt > 0) 4240 atomic_dec_16(&psimm[i].leaky_bucket_cnt); 4241 } 4242 add_leaky_bucket_timeout(); 4243 } 4244 4245 static void 4246 add_leaky_bucket_timeout(void) 4247 { 4248 long timeout_in_microsecs; 4249 4250 /* 4251 * create timeout for next leak. 4252 * 4253 * The timeout interval is calculated as follows 4254 * 4255 * (ecc_softerr_interval * 60 * MICROSEC) / ecc_softerr_limit 4256 * 4257 * ecc_softerr_interval is in minutes, so multiply this by 60 (seconds 4258 * in a minute), then multiply this by MICROSEC to get the interval 4259 * in microseconds. Divide this total by ecc_softerr_limit so that 4260 * the timeout interval is accurate to within a few microseconds. 4261 */ 4262 4263 if (ecc_softerr_limit <= 0) 4264 ecc_softerr_limit = SOFTERR_LIMIT_DEFAULT; 4265 if (ecc_softerr_interval <= 0) 4266 ecc_softerr_interval = SOFTERR_INTERVAL_DEFAULT; 4267 4268 timeout_in_microsecs = ((int64_t)ecc_softerr_interval * 60 * MICROSEC) / 4269 ecc_softerr_limit; 4270 4271 if (timeout_in_microsecs < SOFTERR_MIN_TIMEOUT) 4272 timeout_in_microsecs = SOFTERR_MIN_TIMEOUT; 4273 4274 leaky_bucket_timeout_id = timeout(leaky_bucket_timeout, 4275 (void *)NULL, drv_usectohz((clock_t)timeout_in_microsecs)); 4276 } 4277 4278 /* 4279 * Legacy Correctable ECC Error Hash 4280 * 4281 * All of the code below this comment is used to implement a legacy array 4282 * which counted intermittent, persistent, and sticky CE errors by unum, 4283 * and then was later extended to publish the data as a kstat for SunVTS. 4284 * All of this code is replaced by FMA, and remains here until such time 4285 * that the UltraSPARC-I/II CPU code is converted to FMA, or is EOLed. 4286 * 4287 * Errors are saved in three buckets per-unum: 4288 * (1) sticky - scrub was unsuccessful, cannot be scrubbed 4289 * This could represent a problem, and is immediately printed out. 4290 * (2) persistent - was successfully scrubbed 4291 * These errors use the leaky bucket algorithm to determine 4292 * if there is a serious problem. 4293 * (3) intermittent - may have originated from the cpu or upa/safari bus, 4294 * and does not necessarily indicate any problem with the dimm itself, 4295 * is critical information for debugging new hardware. 4296 * Because we do not know if it came from the dimm, it would be 4297 * inappropriate to include these in the leaky bucket counts. 4298 * 4299 * If the E$ line was modified before the scrub operation began, then the 4300 * displacement flush at the beginning of scrubphys() will cause the modified 4301 * line to be written out, which will clean up the CE. Then, any subsequent 4302 * read will not cause an error, which will cause persistent errors to be 4303 * identified as intermittent. 4304 * 4305 * If a DIMM is going bad, it will produce true persistents as well as 4306 * false intermittents, so these intermittents can be safely ignored. 4307 * 4308 * If the error count is excessive for a DIMM, this function will return 4309 * PR_MCE, and the CPU module may then decide to remove that page from use. 4310 */ 4311 static int 4312 ce_count_unum(int status, int len, char *unum) 4313 { 4314 int i; 4315 struct ce_info *psimm = mem_ce_simm; 4316 int page_status = PR_OK; 4317 4318 ASSERT(psimm != NULL); 4319 4320 if (len <= 0 || 4321 (status & (ECC_STICKY | ECC_PERSISTENT | ECC_INTERMITTENT)) == 0) 4322 return (page_status); 4323 4324 /* 4325 * Initialize the leaky_bucket timeout 4326 */ 4327 if (atomic_cas_ptr(&leaky_bucket_timeout_id, 4328 TIMEOUT_NONE, TIMEOUT_SET) == TIMEOUT_NONE) 4329 add_leaky_bucket_timeout(); 4330 4331 for (i = 0; i < mem_ce_simm_size; i++) { 4332 if (psimm[i].name[0] == '\0') { 4333 /* 4334 * Hit the end of the valid entries, add 4335 * a new one. 4336 */ 4337 (void) strncpy(psimm[i].name, unum, len); 4338 if (status & ECC_STICKY) { 4339 /* 4340 * Sticky - the leaky bucket is used to track 4341 * soft errors. Since a sticky error is a 4342 * hard error and likely to be retired soon, 4343 * we do not count it in the leaky bucket. 4344 */ 4345 psimm[i].leaky_bucket_cnt = 0; 4346 psimm[i].intermittent_total = 0; 4347 psimm[i].persistent_total = 0; 4348 psimm[i].sticky_total = 1; 4349 cmn_err(CE_NOTE, 4350 "[AFT0] Sticky Softerror encountered " 4351 "on Memory Module %s\n", unum); 4352 page_status = PR_MCE; 4353 } else if (status & ECC_PERSISTENT) { 4354 psimm[i].leaky_bucket_cnt = 1; 4355 psimm[i].intermittent_total = 0; 4356 psimm[i].persistent_total = 1; 4357 psimm[i].sticky_total = 0; 4358 } else { 4359 /* 4360 * Intermittent - Because the scrub operation 4361 * cannot find the error in the DIMM, we will 4362 * not count these in the leaky bucket 4363 */ 4364 psimm[i].leaky_bucket_cnt = 0; 4365 psimm[i].intermittent_total = 1; 4366 psimm[i].persistent_total = 0; 4367 psimm[i].sticky_total = 0; 4368 } 4369 ecc_error_info_data.count.value.ui32++; 4370 break; 4371 } else if (strncmp(unum, psimm[i].name, len) == 0) { 4372 /* 4373 * Found an existing entry for the current 4374 * memory module, adjust the counts. 4375 */ 4376 if (status & ECC_STICKY) { 4377 psimm[i].sticky_total++; 4378 cmn_err(CE_NOTE, 4379 "[AFT0] Sticky Softerror encountered " 4380 "on Memory Module %s\n", unum); 4381 page_status = PR_MCE; 4382 } else if (status & ECC_PERSISTENT) { 4383 int new_value; 4384 4385 new_value = atomic_inc_16_nv( 4386 &psimm[i].leaky_bucket_cnt); 4387 psimm[i].persistent_total++; 4388 if (new_value > ecc_softerr_limit) { 4389 cmn_err(CE_NOTE, "[AFT0] Most recent %d" 4390 " soft errors from Memory Module" 4391 " %s exceed threshold (N=%d," 4392 " T=%dh:%02dm) triggering page" 4393 " retire", new_value, unum, 4394 ecc_softerr_limit, 4395 ecc_softerr_interval / 60, 4396 ecc_softerr_interval % 60); 4397 atomic_dec_16( 4398 &psimm[i].leaky_bucket_cnt); 4399 page_status = PR_MCE; 4400 } 4401 } else { /* Intermittent */ 4402 psimm[i].intermittent_total++; 4403 } 4404 break; 4405 } 4406 } 4407 4408 if (i >= mem_ce_simm_size) 4409 cmn_err(CE_CONT, "[AFT0] Softerror: mem_ce_simm[] out of " 4410 "space.\n"); 4411 4412 return (page_status); 4413 } 4414 4415 /* 4416 * Function to support counting of IO detected CEs. 4417 */ 4418 void 4419 cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum) 4420 { 4421 int err; 4422 4423 err = ce_count_unum(ecc->flt_status, len, unum); 4424 if (err != PR_OK && automatic_page_removal) { 4425 (void) page_retire(ecc->flt_addr, err); 4426 } 4427 } 4428 4429 static int 4430 ecc_kstat_update(kstat_t *ksp, int rw) 4431 { 4432 struct kstat_ecc_mm_info *kceip = ksp->ks_data; 4433 struct ce_info *ceip = mem_ce_simm; 4434 int i = ksp->ks_instance; 4435 4436 if (rw == KSTAT_WRITE) 4437 return (EACCES); 4438 4439 ASSERT(ksp->ks_data != NULL); 4440 ASSERT(i < mem_ce_simm_size && i >= 0); 4441 4442 /* 4443 * Since we're not using locks, make sure that we don't get partial 4444 * data. The name is always copied before the counters are incremented 4445 * so only do this update routine if at least one of the counters is 4446 * non-zero, which ensures that ce_count_unum() is done, and the 4447 * string is fully copied. 4448 */ 4449 if (ceip[i].intermittent_total == 0 && 4450 ceip[i].persistent_total == 0 && 4451 ceip[i].sticky_total == 0) { 4452 /* 4453 * Uninitialized or partially initialized. Ignore. 4454 * The ks_data buffer was allocated via kmem_zalloc, 4455 * so no need to bzero it. 4456 */ 4457 return (0); 4458 } 4459 4460 kstat_named_setstr(&kceip->name, ceip[i].name); 4461 kceip->intermittent_total.value.ui64 = ceip[i].intermittent_total; 4462 kceip->persistent_total.value.ui64 = ceip[i].persistent_total; 4463 kceip->sticky_total.value.ui64 = ceip[i].sticky_total; 4464 4465 return (0); 4466 } 4467 4468 #define VIS_BLOCKSIZE 64 4469 4470 int 4471 dtrace_blksuword32_err(uintptr_t addr, uint32_t *data) 4472 { 4473 int ret, watched; 4474 4475 watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 4476 ret = dtrace_blksuword32(addr, data, 0); 4477 if (watched) 4478 watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 4479 4480 return (ret); 4481 } 4482 4483 /*ARGSUSED*/ 4484 void 4485 cpu_faulted_enter(struct cpu *cp) 4486 { 4487 } 4488 4489 /*ARGSUSED*/ 4490 void 4491 cpu_faulted_exit(struct cpu *cp) 4492 { 4493 } 4494 4495 /*ARGSUSED*/ 4496 void 4497 mmu_init_kernel_pgsz(struct hat *hat) 4498 { 4499 } 4500 4501 size_t 4502 mmu_get_kernel_lpsize(size_t lpsize) 4503 { 4504 uint_t tte; 4505 4506 if (lpsize == 0) { 4507 /* no setting for segkmem_lpsize in /etc/system: use default */ 4508 return (MMU_PAGESIZE4M); 4509 } 4510 4511 for (tte = TTE8K; tte <= TTE4M; tte++) { 4512 if (lpsize == TTEBYTES(tte)) 4513 return (lpsize); 4514 } 4515 4516 return (TTEBYTES(TTE8K)); 4517 } 4518