1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/systm.h> 31 #include <sys/archsystm.h> 32 #include <sys/machparam.h> 33 #include <sys/machsystm.h> 34 #include <sys/cpu.h> 35 #include <sys/elf_SPARC.h> 36 #include <vm/hat_sfmmu.h> 37 #include <vm/page.h> 38 #include <sys/cpuvar.h> 39 #include <sys/spitregs.h> 40 #include <sys/async.h> 41 #include <sys/cmn_err.h> 42 #include <sys/debug.h> 43 #include <sys/dditypes.h> 44 #include <sys/sunddi.h> 45 #include <sys/cpu_module.h> 46 #include <sys/prom_debug.h> 47 #include <sys/vmsystm.h> 48 #include <sys/prom_plat.h> 49 #include <sys/sysmacros.h> 50 #include <sys/intreg.h> 51 #include <sys/machtrap.h> 52 #include <sys/ontrap.h> 53 #include <sys/ivintr.h> 54 #include <sys/atomic.h> 55 #include <sys/panic.h> 56 #include <sys/ndifm.h> 57 #include <sys/fm/protocol.h> 58 #include <sys/fm/util.h> 59 #include <sys/fm/cpu/UltraSPARC-II.h> 60 #include <sys/ddi.h> 61 #include <sys/ecc_kstat.h> 62 #include <sys/watchpoint.h> 63 #include <sys/dtrace.h> 64 #include <sys/errclassify.h> 65 66 uchar_t *ctx_pgsz_array = NULL; 67 68 /* 69 * Structure for the 8 byte ecache data dump and the associated AFSR state. 70 * There will be 8 of these structures used to dump an ecache line (64 bytes). 71 */ 72 typedef struct sf_ec_data_elm { 73 uint64_t ec_d8; 74 uint64_t ec_afsr; 75 } ec_data_t; 76 77 /* 78 * Define spitfire (Ultra I/II) specific asynchronous error structure 79 */ 80 typedef struct spitfire_async_flt { 81 struct async_flt cmn_asyncflt; /* common - see sun4u/sys/async.h */ 82 ushort_t flt_type; /* types of faults - cpu specific */ 83 ec_data_t flt_ec_data[8]; /* for E$ or mem dump/state */ 84 uint64_t flt_ec_tag; /* E$ tag info */ 85 int flt_ec_lcnt; /* number of bad E$ lines */ 86 ushort_t flt_sdbh; /* UDBH reg */ 87 ushort_t flt_sdbl; /* UDBL reg */ 88 } spitf_async_flt; 89 90 /* 91 * Prototypes for support routines in spitfire_asm.s: 92 */ 93 extern void flush_ecache(uint64_t physaddr, size_t size, size_t linesize); 94 extern uint64_t get_lsu(void); 95 extern void set_lsu(uint64_t ncc); 96 extern void get_ecache_dtag(uint32_t ecache_idx, uint64_t *data, uint64_t *tag, 97 uint64_t *oafsr, uint64_t *acc_afsr); 98 extern uint64_t check_ecache_line(uint32_t id, uint64_t *acc_afsr); 99 extern uint64_t get_ecache_tag(uint32_t id, uint64_t *nafsr, 100 uint64_t *acc_afsr); 101 extern uint64_t read_and_clear_afsr(); 102 extern void write_ec_tag_parity(uint32_t id); 103 extern void write_hb_ec_tag_parity(uint32_t id); 104 105 /* 106 * Spitfire module routines: 107 */ 108 static void cpu_async_log_err(void *flt); 109 /*PRINTFLIKE6*/ 110 static void cpu_aflt_log(int ce_code, int tagnum, spitf_async_flt *spflt, 111 uint_t logflags, const char *endstr, const char *fmt, ...); 112 113 static void cpu_read_paddr(struct async_flt *aflt, short verbose, short ce_err); 114 static void cpu_ce_log_status(spitf_async_flt *spf_flt, char *unum); 115 static void cpu_log_ecmem_info(spitf_async_flt *spf_flt); 116 117 static void log_ce_err(struct async_flt *aflt, char *unum); 118 static void log_ue_err(struct async_flt *aflt, char *unum); 119 static void check_misc_err(spitf_async_flt *spf_flt); 120 static ushort_t ecc_gen(uint_t high_bytes, uint_t low_bytes); 121 static int check_ecc(struct async_flt *aflt); 122 static uint_t get_cpu_status(uint64_t arg); 123 static uint64_t clear_errors(spitf_async_flt *spf_flt, uint64_t *acc_afsr); 124 static void scan_ecache(uint64_t *afar, ec_data_t *data, uint64_t *tag, 125 int *m, uint64_t *afsr); 126 static void ecache_kstat_init(struct cpu *cp); 127 static void ecache_scrub_log(ec_data_t *ec_data, uint64_t ec_tag, 128 uint64_t paddr, int mpb, uint64_t); 129 static uint64_t ecache_scrub_misc_err(int, uint64_t); 130 static void ecache_scrub_tag_err(uint64_t, uchar_t, uint32_t); 131 static void ecache_page_retire(void *); 132 static int ecc_kstat_update(kstat_t *ksp, int rw); 133 static int ce_count_unum(int status, int len, char *unum); 134 static void add_leaky_bucket_timeout(void); 135 static int synd_to_synd_code(int synd_status, ushort_t synd); 136 137 extern uint_t read_all_memscrub; 138 extern void memscrub_run(void); 139 140 static uchar_t isus2i; /* set if sabre */ 141 static uchar_t isus2e; /* set if hummingbird */ 142 143 /* 144 * Default ecache mask and shift settings for Spitfire. If we detect a 145 * different CPU implementation, we will modify these values at boot time. 146 */ 147 static uint64_t cpu_ec_tag_mask = S_ECTAG_MASK; 148 static uint64_t cpu_ec_state_mask = S_ECSTATE_MASK; 149 static uint64_t cpu_ec_par_mask = S_ECPAR_MASK; 150 static int cpu_ec_par_shift = S_ECPAR_SHIFT; 151 static int cpu_ec_tag_shift = S_ECTAG_SHIFT; 152 static int cpu_ec_state_shift = S_ECSTATE_SHIFT; 153 static uchar_t cpu_ec_state_exl = S_ECSTATE_EXL; 154 static uchar_t cpu_ec_state_mod = S_ECSTATE_MOD; 155 static uchar_t cpu_ec_state_shr = S_ECSTATE_SHR; 156 static uchar_t cpu_ec_state_own = S_ECSTATE_OWN; 157 158 /* 159 * Default ecache state bits for Spitfire. These individual bits indicate if 160 * the given line is in any of the valid or modified states, respectively. 161 * Again, we modify these at boot if we detect a different CPU. 162 */ 163 static uchar_t cpu_ec_state_valid = S_ECSTATE_VALID; 164 static uchar_t cpu_ec_state_dirty = S_ECSTATE_DIRTY; 165 static uchar_t cpu_ec_parity = S_EC_PARITY; 166 static uchar_t cpu_ec_state_parity = S_ECSTATE_PARITY; 167 168 /* 169 * This table is used to determine which bit(s) is(are) bad when an ECC 170 * error occurrs. The array is indexed an 8-bit syndrome. The entries 171 * of this array have the following semantics: 172 * 173 * 00-63 The number of the bad bit, when only one bit is bad. 174 * 64 ECC bit C0 is bad. 175 * 65 ECC bit C1 is bad. 176 * 66 ECC bit C2 is bad. 177 * 67 ECC bit C3 is bad. 178 * 68 ECC bit C4 is bad. 179 * 69 ECC bit C5 is bad. 180 * 70 ECC bit C6 is bad. 181 * 71 ECC bit C7 is bad. 182 * 72 Two bits are bad. 183 * 73 Three bits are bad. 184 * 74 Four bits are bad. 185 * 75 More than Four bits are bad. 186 * 76 NO bits are bad. 187 * Based on "Galaxy Memory Subsystem SPECIFICATION" rev 0.6, pg. 28. 188 */ 189 190 #define C0 64 191 #define C1 65 192 #define C2 66 193 #define C3 67 194 #define C4 68 195 #define C5 69 196 #define C6 70 197 #define C7 71 198 #define M2 72 199 #define M3 73 200 #define M4 74 201 #define MX 75 202 #define NA 76 203 204 #define SYND_IS_SINGLE_BIT_DATA(synd_code) ((synd_code >= 0) && \ 205 (synd_code < C0)) 206 #define SYND_IS_SINGLE_BIT_CHK(synd_code) ((synd_code >= C0) && \ 207 (synd_code <= C7)) 208 209 static char ecc_syndrome_tab[] = 210 { 211 NA, C0, C1, M2, C2, M2, M2, M3, C3, M2, M2, M3, M2, M3, M3, M4, 212 C4, M2, M2, 32, M2, 57, MX, M2, M2, 37, 49, M2, 40, M2, M2, 44, 213 C5, M2, M2, 33, M2, 61, 4, M2, M2, MX, 53, M2, 45, M2, M2, 41, 214 M2, 0, 1, M2, 10, M2, M2, MX, 15, M2, M2, MX, M2, M3, M3, M2, 215 C6, M2, M2, 42, M2, 59, 39, M2, M2, MX, 51, M2, 34, M2, M2, 46, 216 M2, 25, 29, M2, 27, M4, M2, MX, 31, M2, M4, MX, M2, MX, MX, M2, 217 M2, MX, 36, M2, 7, M2, M2, 54, MX, M2, M2, 62, M2, 48, 56, M2, 218 M3, M2, M2, MX, M2, MX, 22, M2, M2, 18, MX, M2, M3, M2, M2, MX, 219 C7, M2, M2, 47, M2, 63, MX, M2, M2, 6, 55, M2, 35, M2, M2, 43, 220 M2, 5, MX, M2, MX, M2, M2, 50, 38, M2, M2, 58, M2, 52, 60, M2, 221 M2, 17, 21, M2, 19, M4, M2, MX, 23, M2, M4, MX, M2, MX, MX, M2, 222 M3, M2, M2, MX, M2, MX, 30, M2, M2, 26, MX, M2, M3, M2, M2, MX, 223 M2, 8, 13, M2, 2, M2, M2, M3, 3, M2, M2, M3, M2, MX, MX, M2, 224 M3, M2, M2, M3, M2, MX, 16, M2, M2, 20, MX, M2, MX, M2, M2, MX, 225 M3, M2, M2, M3, M2, MX, 24, M2, M2, 28, MX, M2, MX, M2, M2, MX, 226 M4, 12, 9, M2, 14, M2, M2, MX, 11, M2, M2, MX, M2, MX, MX, M4 227 }; 228 229 #define SYND_TBL_SIZE 256 230 231 /* 232 * Hack for determining UDBH/UDBL, for later cpu-specific error reporting. 233 * Cannot use bit 3 in afar, because it is a valid bit on a Sabre/Hummingbird. 234 */ 235 #define UDBL_REG 0x8000 236 #define UDBL(synd) ((synd & UDBL_REG) >> 15) 237 #define SYND(synd) (synd & 0x7FFF) 238 239 /* 240 * These error types are specific to Spitfire and are used internally for the 241 * spitfire fault structure flt_type field. 242 */ 243 #define CPU_UE_ERR 0 /* uncorrectable errors - UEs */ 244 #define CPU_EDP_LDP_ERR 1 /* LDP or EDP parity error */ 245 #define CPU_WP_ERR 2 /* WP parity error */ 246 #define CPU_BTO_BERR_ERR 3 /* bus timeout errors */ 247 #define CPU_PANIC_CP_ERR 4 /* cp error from panic polling */ 248 #define CPU_TRAPPING_CP_ERR 5 /* for sabre/hbird only, cp error */ 249 #define CPU_BADLINE_CI_ERR 6 /* E$ clean_bad line when idle */ 250 #define CPU_BADLINE_CB_ERR 7 /* E$ clean_bad line when busy */ 251 #define CPU_BADLINE_DI_ERR 8 /* E$ dirty_bad line when idle */ 252 #define CPU_BADLINE_DB_ERR 9 /* E$ dirty_bad line when busy */ 253 #define CPU_ORPHAN_CP_ERR 10 /* Orphan CP error */ 254 #define CPU_ECACHE_ADDR_PAR_ERR 11 /* Ecache Address parity error */ 255 #define CPU_ECACHE_STATE_ERR 12 /* Ecache state error */ 256 #define CPU_ECACHE_ETP_ETS_ERR 13 /* ETP set but ETS is zero */ 257 #define CPU_ECACHE_TAG_ERR 14 /* Scrub the E$ tag, if state clean */ 258 #define CPU_ADDITIONAL_ERR 15 /* Additional errors occurred */ 259 260 /* 261 * Macro to access the "Spitfire cpu private" data structure. 262 */ 263 #define CPU_PRIVATE_PTR(cp, x) (&(((spitfire_private_t *)CPU_PRIVATE(cp))->x)) 264 265 /* 266 * set to 0 to disable automatic retiring of pages on 267 * DIMMs that have excessive soft errors 268 */ 269 int automatic_page_removal = 1; 270 271 /* 272 * Heuristic for figuring out which module to replace. 273 * Relative likelihood that this P_SYND indicates that this module is bad. 274 * We call it a "score", though, not a relative likelihood. 275 * 276 * Step 1. 277 * Assign a score to each byte of P_SYND according to the following rules: 278 * If no bits on (0x00) or all bits on (0xFF), then give it a 5. 279 * If one bit on, give it a 95. 280 * If seven bits on, give it a 10. 281 * If two bits on: 282 * in different nybbles, a 90 283 * in same nybble, but unaligned, 85 284 * in same nybble and as an aligned pair, 80 285 * If six bits on, look at the bits that are off: 286 * in same nybble and as an aligned pair, 15 287 * in same nybble, but unaligned, 20 288 * in different nybbles, a 25 289 * If three bits on: 290 * in diferent nybbles, no aligned pairs, 75 291 * in diferent nybbles, one aligned pair, 70 292 * in the same nybble, 65 293 * If five bits on, look at the bits that are off: 294 * in the same nybble, 30 295 * in diferent nybbles, one aligned pair, 35 296 * in diferent nybbles, no aligned pairs, 40 297 * If four bits on: 298 * all in one nybble, 45 299 * as two aligned pairs, 50 300 * one aligned pair, 55 301 * no aligned pairs, 60 302 * 303 * Step 2: 304 * Take the higher of the two scores (one for each byte) as the score 305 * for the module. 306 * 307 * Print the score for each module, and field service should replace the 308 * module with the highest score. 309 */ 310 311 /* 312 * In the table below, the first row/column comment indicates the 313 * number of bits on in that nybble; the second row/column comment is 314 * the hex digit. 315 */ 316 317 static int 318 p_synd_score_table[256] = { 319 /* 0 1 1 2 1 2 2 3 1 2 2 3 2 3 3 4 */ 320 /* 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, A, B, C, D, E, F */ 321 /* 0 0 */ 5, 95, 95, 80, 95, 85, 85, 65, 95, 85, 85, 65, 80, 65, 65, 45, 322 /* 1 1 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30, 323 /* 1 2 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30, 324 /* 2 3 */ 80, 70, 70, 50, 70, 55, 55, 35, 70, 55, 55, 35, 50, 35, 35, 15, 325 /* 1 4 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30, 326 /* 2 5 */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20, 327 /* 2 6 */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20, 328 /* 3 7 */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10, 329 /* 1 8 */ 95, 90, 90, 70, 90, 75, 75, 55, 90, 75, 75, 55, 70, 55, 55, 30, 330 /* 2 9 */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20, 331 /* 2 A */ 85, 75, 75, 55, 75, 60, 60, 40, 75, 60, 60, 40, 55, 40, 40, 20, 332 /* 3 B */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10, 333 /* 2 C */ 80, 70, 70, 50, 70, 55, 55, 35, 70, 55, 55, 35, 50, 35, 35, 15, 334 /* 3 D */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10, 335 /* 3 E */ 65, 55, 55, 35, 55, 40, 40, 25, 55, 40, 40, 25, 35, 25, 25, 10, 336 /* 4 F */ 45, 30, 30, 15, 30, 20, 20, 10, 30, 20, 20, 10, 15, 10, 10, 5, 337 }; 338 339 int 340 ecc_psynd_score(ushort_t p_synd) 341 { 342 int i, j, a, b; 343 344 i = p_synd & 0xFF; 345 j = (p_synd >> 8) & 0xFF; 346 347 a = p_synd_score_table[i]; 348 b = p_synd_score_table[j]; 349 350 return (a > b ? a : b); 351 } 352 353 /* 354 * Async Fault Logging 355 * 356 * To ease identifying, reading, and filtering async fault log messages, the 357 * label [AFT#] is now prepended to each async fault message. These messages 358 * and the logging rules are implemented by cpu_aflt_log(), below. 359 * 360 * [AFT0] - Tag for log messages that are associated with corrected ECC errors. 361 * This includes both corrected ECC memory and ecache faults. 362 * 363 * [AFT1] - Tag for log messages that are not ECC corrected (i.e. everything 364 * else except CE errors) with a priority of 1 (highest). This tag 365 * is also used for panic messages that result from an async fault. 366 * 367 * [AFT2] - These are lower priority diagnostic messages for uncorrected ECC 368 * [AFT3] or parity errors. For example, AFT2 is used for the actual dump 369 * of the E-$ data and tags. 370 * 371 * In a non-DEBUG kernel, AFT > 1 logs will be sent to the system log but not 372 * printed on the console. To send all AFT logs to both the log and the 373 * console, set aft_verbose = 1. 374 */ 375 376 #define CPU_FLTCPU 0x0001 /* print flt_inst as a CPU id */ 377 #define CPU_SPACE 0x0002 /* print flt_status (data or instr) */ 378 #define CPU_ERRID 0x0004 /* print flt_id */ 379 #define CPU_TL 0x0008 /* print flt_tl */ 380 #define CPU_ERRID_FIRST 0x0010 /* print flt_id first in message */ 381 #define CPU_AFSR 0x0020 /* print flt_stat as decoded %afsr */ 382 #define CPU_AFAR 0x0040 /* print flt_addr as %afar */ 383 #define CPU_AF_PSYND 0x0080 /* print flt_stat %afsr.PSYND */ 384 #define CPU_AF_ETS 0x0100 /* print flt_stat %afsr.ETS */ 385 #define CPU_UDBH 0x0200 /* print flt_sdbh and syndrome */ 386 #define CPU_UDBL 0x0400 /* print flt_sdbl and syndrome */ 387 #define CPU_FAULTPC 0x0800 /* print flt_pc */ 388 #define CPU_SYND 0x1000 /* print flt_synd and unum */ 389 390 #define CMN_LFLAGS (CPU_FLTCPU | CPU_SPACE | CPU_ERRID | CPU_TL | \ 391 CPU_AFSR | CPU_AFAR | CPU_AF_PSYND | \ 392 CPU_AF_ETS | CPU_UDBH | CPU_UDBL | \ 393 CPU_FAULTPC) 394 #define UE_LFLAGS (CMN_LFLAGS | CPU_SYND) 395 #define CE_LFLAGS (UE_LFLAGS & ~CPU_UDBH & ~CPU_UDBL & ~CPU_TL & \ 396 ~CPU_SPACE) 397 #define PARERR_LFLAGS (CMN_LFLAGS) 398 #define WP_LFLAGS (CMN_LFLAGS & ~CPU_SPACE & ~CPU_TL) 399 #define CP_LFLAGS (CMN_LFLAGS & ~CPU_SPACE & ~CPU_TL & \ 400 ~CPU_FLTCPU & ~CPU_FAULTPC) 401 #define BERRTO_LFLAGS (CMN_LFLAGS) 402 #define NO_LFLAGS (0) 403 404 #define AFSR_FMTSTR0 "\020\1ME" 405 #define AFSR_FMTSTR1 "\020\040PRIV\037ISAP\036ETP\035IVUE\034TO" \ 406 "\033BERR\032LDP\031CP\030WP\027EDP\026UE\025CE" 407 #define UDB_FMTSTR "\020\012UE\011CE" 408 409 /* 410 * Maximum number of contexts for Spitfire. 411 */ 412 #define MAX_NCTXS (1 << 13) 413 414 /* 415 * Save the cache bootup state for use when internal 416 * caches are to be re-enabled after an error occurs. 417 */ 418 uint64_t cache_boot_state = 0; 419 420 /* 421 * PA[31:0] represent Displacement in UPA configuration space. 422 */ 423 uint_t root_phys_addr_lo_mask = 0xffffffff; 424 425 /* 426 * Spitfire legacy globals 427 */ 428 int itlb_entries; 429 int dtlb_entries; 430 431 void 432 cpu_setup(void) 433 { 434 extern int page_retire_messages; 435 extern int page_retire_first_ue; 436 extern int at_flags; 437 #if defined(SF_ERRATA_57) 438 extern caddr_t errata57_limit; 439 #endif 440 extern int disable_text_largepages; 441 extern int disable_initdata_largepages; 442 443 cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT); 444 445 at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1; 446 447 /* 448 * Spitfire isn't currently FMA-aware, so we have to enable the 449 * page retirement messages. We also change the default policy 450 * for UE retirement to allow clearing of transient errors. 451 */ 452 page_retire_messages = 1; 453 page_retire_first_ue = 0; 454 455 /* 456 * save the cache bootup state. 457 */ 458 cache_boot_state = get_lsu() & (LSU_IC | LSU_DC); 459 460 /* 461 * Use the maximum number of contexts available for Spitfire unless 462 * it has been tuned for debugging. 463 * We are checking against 0 here since this value can be patched 464 * while booting. It can not be patched via /etc/system since it 465 * will be patched too late and thus cause the system to panic. 466 */ 467 if (nctxs == 0) 468 nctxs = MAX_NCTXS; 469 470 if (use_page_coloring) { 471 do_pg_coloring = 1; 472 if (use_virtual_coloring) 473 do_virtual_coloring = 1; 474 } 475 476 /* 477 * Tune pp_slots to use up to 1/8th of the tlb entries. 478 */ 479 pp_slots = MIN(8, MAXPP_SLOTS); 480 481 /* 482 * Block stores invalidate all pages of the d$ so pagecopy 483 * et. al. do not need virtual translations with virtual 484 * coloring taken into consideration. 485 */ 486 pp_consistent_coloring = 0; 487 488 isa_list = 489 "sparcv9+vis sparcv9 " 490 "sparcv8plus+vis sparcv8plus " 491 "sparcv8 sparcv8-fsmuld sparcv7 sparc"; 492 493 cpu_hwcap_flags = AV_SPARC_VIS; 494 495 /* 496 * On Spitfire, there's a hole in the address space 497 * that we must never map (the hardware only support 44-bits of 498 * virtual address). Later CPUs are expected to have wider 499 * supported address ranges. 500 * 501 * See address map on p23 of the UltraSPARC 1 user's manual. 502 */ 503 hole_start = (caddr_t)0x80000000000ull; 504 hole_end = (caddr_t)0xfffff80000000000ull; 505 506 /* 507 * A spitfire call bug requires us to be a further 4Gbytes of 508 * firewall from the spec. 509 * 510 * See Spitfire Errata #21 511 */ 512 hole_start = (caddr_t)((uintptr_t)hole_start - (1ul << 32)); 513 hole_end = (caddr_t)((uintptr_t)hole_end + (1ul << 32)); 514 515 /* 516 * The kpm mapping window. 517 * kpm_size: 518 * The size of a single kpm range. 519 * The overall size will be: kpm_size * vac_colors. 520 * kpm_vbase: 521 * The virtual start address of the kpm range within the kernel 522 * virtual address space. kpm_vbase has to be kpm_size aligned. 523 */ 524 kpm_size = (size_t)(2ull * 1024 * 1024 * 1024 * 1024); /* 2TB */ 525 kpm_size_shift = 41; 526 kpm_vbase = (caddr_t)0xfffffa0000000000ull; /* 16EB - 6TB */ 527 528 #if defined(SF_ERRATA_57) 529 errata57_limit = (caddr_t)0x80000000ul; 530 #endif 531 532 /* 533 * Allow only 8K, 64K and 4M pages for text by default. 534 * Allow only 8K and 64K page for initialized data segments by 535 * default. 536 */ 537 disable_text_largepages = (1 << TTE512K) | (1 << TTE32M) | 538 (1 << TTE256M); 539 disable_initdata_largepages = (1 << TTE512K) | (1 << TTE4M) | 540 (1 << TTE32M) | (1 << TTE256M); 541 } 542 543 static int 544 getintprop(pnode_t node, char *name, int deflt) 545 { 546 int value; 547 548 switch (prom_getproplen(node, name)) { 549 case 0: 550 value = 1; /* boolean properties */ 551 break; 552 553 case sizeof (int): 554 (void) prom_getprop(node, name, (caddr_t)&value); 555 break; 556 557 default: 558 value = deflt; 559 break; 560 } 561 562 return (value); 563 } 564 565 /* 566 * Set the magic constants of the implementation. 567 */ 568 void 569 cpu_fiximp(pnode_t dnode) 570 { 571 extern int vac_size, vac_shift; 572 extern uint_t vac_mask; 573 extern int dcache_line_mask; 574 int i, a; 575 static struct { 576 char *name; 577 int *var; 578 } prop[] = { 579 "dcache-size", &dcache_size, 580 "dcache-line-size", &dcache_linesize, 581 "icache-size", &icache_size, 582 "icache-line-size", &icache_linesize, 583 "ecache-size", &ecache_size, 584 "ecache-line-size", &ecache_alignsize, 585 "ecache-associativity", &ecache_associativity, 586 "#itlb-entries", &itlb_entries, 587 "#dtlb-entries", &dtlb_entries, 588 }; 589 590 for (i = 0; i < sizeof (prop) / sizeof (prop[0]); i++) { 591 if ((a = getintprop(dnode, prop[i].name, -1)) != -1) { 592 *prop[i].var = a; 593 } 594 } 595 596 ecache_setsize = ecache_size / ecache_associativity; 597 598 vac_size = S_VAC_SIZE; 599 vac_mask = MMU_PAGEMASK & (vac_size - 1); 600 i = 0; a = vac_size; 601 while (a >>= 1) 602 ++i; 603 vac_shift = i; 604 shm_alignment = vac_size; 605 vac = 1; 606 607 dcache_line_mask = (dcache_size - 1) & ~(dcache_linesize - 1); 608 609 /* 610 * UltraSPARC I & II have ecache sizes running 611 * as follows: .25 MB, .5 MB, 1 MB, 2 MB, 4 MB 612 * and 8 MB. Adjust the copyin/copyout limits 613 * according to the cache size. The magic number 614 * of VIS_COPY_THRESHOLD comes from the copyin/copyout code 615 * and its floor of VIS_COPY_THRESHOLD bytes before it will use 616 * VIS instructions. 617 * 618 * We assume that all CPUs on the system have the same size 619 * ecache. We're also called very early in the game. 620 * /etc/system will be parsed *after* we're called so 621 * these values can be overwritten. 622 */ 623 624 hw_copy_limit_1 = VIS_COPY_THRESHOLD; 625 if (ecache_size <= 524288) { 626 hw_copy_limit_2 = VIS_COPY_THRESHOLD; 627 hw_copy_limit_4 = VIS_COPY_THRESHOLD; 628 hw_copy_limit_8 = VIS_COPY_THRESHOLD; 629 } else if (ecache_size == 1048576) { 630 hw_copy_limit_2 = 1024; 631 hw_copy_limit_4 = 1280; 632 hw_copy_limit_8 = 1536; 633 } else if (ecache_size == 2097152) { 634 hw_copy_limit_2 = 1536; 635 hw_copy_limit_4 = 2048; 636 hw_copy_limit_8 = 2560; 637 } else if (ecache_size == 4194304) { 638 hw_copy_limit_2 = 2048; 639 hw_copy_limit_4 = 2560; 640 hw_copy_limit_8 = 3072; 641 } else { 642 hw_copy_limit_2 = 2560; 643 hw_copy_limit_4 = 3072; 644 hw_copy_limit_8 = 3584; 645 } 646 } 647 648 /* 649 * Called by setcpudelay 650 */ 651 void 652 cpu_init_tick_freq(void) 653 { 654 /* 655 * Determine the cpu frequency by calling 656 * tod_get_cpufrequency. Use an approximate freqency 657 * value computed by the prom if the tod module 658 * is not initialized and loaded yet. 659 */ 660 if (tod_ops.tod_get_cpufrequency != NULL) { 661 mutex_enter(&tod_lock); 662 sys_tick_freq = tod_ops.tod_get_cpufrequency(); 663 mutex_exit(&tod_lock); 664 } else { 665 #if defined(HUMMINGBIRD) 666 /* 667 * the hummingbird version of %stick is used as the basis for 668 * low level timing; this provides an independent constant-rate 669 * clock for general system use, and frees power mgmt to set 670 * various cpu clock speeds. 671 */ 672 if (system_clock_freq == 0) 673 cmn_err(CE_PANIC, "invalid system_clock_freq 0x%lx", 674 system_clock_freq); 675 sys_tick_freq = system_clock_freq; 676 #else /* SPITFIRE */ 677 sys_tick_freq = cpunodes[CPU->cpu_id].clock_freq; 678 #endif 679 } 680 } 681 682 683 void shipit(int upaid); 684 extern uint64_t xc_tick_limit; 685 extern uint64_t xc_tick_jump_limit; 686 687 #ifdef SEND_MONDO_STATS 688 uint64_t x_early[NCPU][64]; 689 #endif 690 691 /* 692 * Note: A version of this function is used by the debugger via the KDI, 693 * and must be kept in sync with this version. Any changes made to this 694 * function to support new chips or to accomodate errata must also be included 695 * in the KDI-specific version. See spitfire_kdi.c. 696 */ 697 void 698 send_one_mondo(int cpuid) 699 { 700 uint64_t idsr, starttick, endtick; 701 int upaid, busy, nack; 702 uint64_t tick, tick_prev; 703 ulong_t ticks; 704 705 CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 706 upaid = CPUID_TO_UPAID(cpuid); 707 tick = starttick = gettick(); 708 shipit(upaid); 709 endtick = starttick + xc_tick_limit; 710 busy = nack = 0; 711 for (;;) { 712 idsr = getidsr(); 713 if (idsr == 0) 714 break; 715 /* 716 * When we detect an irregular tick jump, we adjust 717 * the timer window to the current tick value. 718 */ 719 tick_prev = tick; 720 tick = gettick(); 721 ticks = tick - tick_prev; 722 if (ticks > xc_tick_jump_limit) { 723 endtick = tick + xc_tick_limit; 724 } else if (tick > endtick) { 725 if (panic_quiesce) 726 return; 727 cmn_err(CE_PANIC, 728 "send mondo timeout (target 0x%x) [%d NACK %d BUSY]", 729 upaid, nack, busy); 730 } 731 if (idsr & IDSR_BUSY) { 732 busy++; 733 continue; 734 } 735 drv_usecwait(1); 736 shipit(upaid); 737 nack++; 738 busy = 0; 739 } 740 #ifdef SEND_MONDO_STATS 741 x_early[getprocessorid()][highbit(gettick() - starttick) - 1]++; 742 #endif 743 } 744 745 void 746 send_mondo_set(cpuset_t set) 747 { 748 int i; 749 750 for (i = 0; i < NCPU; i++) 751 if (CPU_IN_SET(set, i)) { 752 send_one_mondo(i); 753 CPUSET_DEL(set, i); 754 if (CPUSET_ISNULL(set)) 755 break; 756 } 757 } 758 759 void 760 syncfpu(void) 761 { 762 } 763 764 /* 765 * Determine the size of the CPU module's error structure in bytes. This is 766 * called once during boot to initialize the error queues. 767 */ 768 int 769 cpu_aflt_size(void) 770 { 771 /* 772 * We need to determine whether this is a sabre, Hummingbird or a 773 * Spitfire/Blackbird impl and set the appropriate state variables for 774 * ecache tag manipulation. We can't do this in cpu_setup() as it is 775 * too early in the boot flow and the cpunodes are not initialized. 776 * This routine will be called once after cpunodes[] is ready, so do 777 * it here. 778 */ 779 if (cpunodes[CPU->cpu_id].implementation == SABRE_IMPL) { 780 isus2i = 1; 781 cpu_ec_tag_mask = SB_ECTAG_MASK; 782 cpu_ec_state_mask = SB_ECSTATE_MASK; 783 cpu_ec_par_mask = SB_ECPAR_MASK; 784 cpu_ec_par_shift = SB_ECPAR_SHIFT; 785 cpu_ec_tag_shift = SB_ECTAG_SHIFT; 786 cpu_ec_state_shift = SB_ECSTATE_SHIFT; 787 cpu_ec_state_exl = SB_ECSTATE_EXL; 788 cpu_ec_state_mod = SB_ECSTATE_MOD; 789 790 /* These states do not exist in sabre - set to 0xFF */ 791 cpu_ec_state_shr = 0xFF; 792 cpu_ec_state_own = 0xFF; 793 794 cpu_ec_state_valid = SB_ECSTATE_VALID; 795 cpu_ec_state_dirty = SB_ECSTATE_DIRTY; 796 cpu_ec_state_parity = SB_ECSTATE_PARITY; 797 cpu_ec_parity = SB_EC_PARITY; 798 } else if (cpunodes[CPU->cpu_id].implementation == HUMMBRD_IMPL) { 799 isus2e = 1; 800 cpu_ec_tag_mask = HB_ECTAG_MASK; 801 cpu_ec_state_mask = HB_ECSTATE_MASK; 802 cpu_ec_par_mask = HB_ECPAR_MASK; 803 cpu_ec_par_shift = HB_ECPAR_SHIFT; 804 cpu_ec_tag_shift = HB_ECTAG_SHIFT; 805 cpu_ec_state_shift = HB_ECSTATE_SHIFT; 806 cpu_ec_state_exl = HB_ECSTATE_EXL; 807 cpu_ec_state_mod = HB_ECSTATE_MOD; 808 809 /* These states do not exist in hummingbird - set to 0xFF */ 810 cpu_ec_state_shr = 0xFF; 811 cpu_ec_state_own = 0xFF; 812 813 cpu_ec_state_valid = HB_ECSTATE_VALID; 814 cpu_ec_state_dirty = HB_ECSTATE_DIRTY; 815 cpu_ec_state_parity = HB_ECSTATE_PARITY; 816 cpu_ec_parity = HB_EC_PARITY; 817 } 818 819 return (sizeof (spitf_async_flt)); 820 } 821 822 823 /* 824 * Correctable ecc error trap handler 825 */ 826 /*ARGSUSED*/ 827 void 828 cpu_ce_error(struct regs *rp, ulong_t p_afar, ulong_t p_afsr, 829 uint_t p_afsr_high, uint_t p_afar_high) 830 { 831 ushort_t sdbh, sdbl; 832 ushort_t e_syndh, e_syndl; 833 spitf_async_flt spf_flt; 834 struct async_flt *ecc; 835 int queue = 1; 836 837 uint64_t t_afar = p_afar; 838 uint64_t t_afsr = p_afsr; 839 840 /* 841 * Note: the Spitfire data buffer error registers 842 * (upper and lower halves) are or'ed into the upper 843 * word of the afsr by ce_err(). 844 */ 845 sdbh = (ushort_t)((t_afsr >> 33) & 0x3FF); 846 sdbl = (ushort_t)((t_afsr >> 43) & 0x3FF); 847 848 e_syndh = (uchar_t)(sdbh & (uint_t)P_DER_E_SYND); 849 e_syndl = (uchar_t)(sdbl & (uint_t)P_DER_E_SYND); 850 851 t_afsr &= S_AFSR_MASK; 852 t_afar &= SABRE_AFAR_PA; /* must use Sabre AFAR mask */ 853 854 /* Setup the async fault structure */ 855 bzero(&spf_flt, sizeof (spitf_async_flt)); 856 ecc = (struct async_flt *)&spf_flt; 857 ecc->flt_id = gethrtime_waitfree(); 858 ecc->flt_stat = t_afsr; 859 ecc->flt_addr = t_afar; 860 ecc->flt_status = ECC_C_TRAP; 861 ecc->flt_bus_id = getprocessorid(); 862 ecc->flt_inst = CPU->cpu_id; 863 ecc->flt_pc = (caddr_t)rp->r_pc; 864 ecc->flt_func = log_ce_err; 865 ecc->flt_in_memory = 866 (pf_is_memory(ecc->flt_addr >> MMU_PAGESHIFT)) ? 1: 0; 867 spf_flt.flt_sdbh = sdbh; 868 spf_flt.flt_sdbl = sdbl; 869 870 /* 871 * Check for fatal conditions. 872 */ 873 check_misc_err(&spf_flt); 874 875 /* 876 * Pananoid checks for valid AFSR and UDBs 877 */ 878 if ((t_afsr & P_AFSR_CE) == 0) { 879 cpu_aflt_log(CE_PANIC, 1, &spf_flt, CMN_LFLAGS, 880 "** Panic due to CE bit not set in the AFSR", 881 " Corrected Memory Error on"); 882 } 883 884 /* 885 * We want to skip logging only if ALL the following 886 * conditions are true: 887 * 888 * 1. There is only one error 889 * 2. That error is a correctable memory error 890 * 3. The error is caused by the memory scrubber (in which case 891 * the error will have occurred under on_trap protection) 892 * 4. The error is on a retired page 893 * 894 * Note: OT_DATA_EC is used places other than the memory scrubber. 895 * However, none of those errors should occur on a retired page. 896 */ 897 if ((ecc->flt_stat & (S_AFSR_ALL_ERRS & ~P_AFSR_ME)) == P_AFSR_CE && 898 curthread->t_ontrap != NULL) { 899 900 if (curthread->t_ontrap->ot_prot & OT_DATA_EC) { 901 if (page_retire_check(ecc->flt_addr, NULL) == 0) { 902 queue = 0; 903 } 904 } 905 } 906 907 if (((sdbh & P_DER_CE) == 0) && ((sdbl & P_DER_CE) == 0)) { 908 cpu_aflt_log(CE_PANIC, 1, &spf_flt, CMN_LFLAGS, 909 "** Panic due to CE bits not set in the UDBs", 910 " Corrected Memory Error on"); 911 } 912 913 if ((sdbh >> 8) & 1) { 914 ecc->flt_synd = e_syndh; 915 ce_scrub(ecc); 916 if (queue) { 917 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CE, ecc, 918 sizeof (*ecc), ce_queue, ERRORQ_ASYNC); 919 } 920 } 921 922 if ((sdbl >> 8) & 1) { 923 ecc->flt_addr = t_afar | 0x8; /* Sabres do not have a UDBL */ 924 ecc->flt_synd = e_syndl | UDBL_REG; 925 ce_scrub(ecc); 926 if (queue) { 927 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CE, ecc, 928 sizeof (*ecc), ce_queue, ERRORQ_ASYNC); 929 } 930 } 931 932 /* 933 * Re-enable all error trapping (CEEN currently cleared). 934 */ 935 clr_datapath(); 936 set_asyncflt(P_AFSR_CE); 937 set_error_enable(EER_ENABLE); 938 } 939 940 /* 941 * Cpu specific CE logging routine 942 */ 943 static void 944 log_ce_err(struct async_flt *aflt, char *unum) 945 { 946 spitf_async_flt spf_flt; 947 948 if ((aflt->flt_stat & P_AFSR_CE) && (ce_verbose_memory == 0)) { 949 return; 950 } 951 952 spf_flt.cmn_asyncflt = *aflt; 953 cpu_aflt_log(CE_CONT, 0, &spf_flt, CE_LFLAGS, unum, 954 " Corrected Memory Error detected by"); 955 } 956 957 /* 958 * Spitfire does not perform any further CE classification refinement 959 */ 960 /*ARGSUSED*/ 961 int 962 ce_scrub_xdiag_recirc(struct async_flt *ecc, errorq_t *eqp, errorq_elem_t *eqep, 963 size_t afltoffset) 964 { 965 return (0); 966 } 967 968 char * 969 flt_to_error_type(struct async_flt *aflt) 970 { 971 if (aflt->flt_status & ECC_INTERMITTENT) 972 return (ERR_TYPE_DESC_INTERMITTENT); 973 if (aflt->flt_status & ECC_PERSISTENT) 974 return (ERR_TYPE_DESC_PERSISTENT); 975 if (aflt->flt_status & ECC_STICKY) 976 return (ERR_TYPE_DESC_STICKY); 977 return (ERR_TYPE_DESC_UNKNOWN); 978 } 979 980 /* 981 * Called by correctable ecc error logging code to print out 982 * the stick/persistent/intermittent status of the error. 983 */ 984 static void 985 cpu_ce_log_status(spitf_async_flt *spf_flt, char *unum) 986 { 987 ushort_t status; 988 char *status1_str = "Memory"; 989 char *status2_str = "Intermittent"; 990 struct async_flt *aflt = (struct async_flt *)spf_flt; 991 992 status = aflt->flt_status; 993 994 if (status & ECC_ECACHE) 995 status1_str = "Ecache"; 996 997 if (status & ECC_STICKY) 998 status2_str = "Sticky"; 999 else if (status & ECC_PERSISTENT) 1000 status2_str = "Persistent"; 1001 1002 cpu_aflt_log(CE_CONT, 0, spf_flt, CPU_ERRID_FIRST, 1003 NULL, " Corrected %s Error on %s is %s", 1004 status1_str, unum, status2_str); 1005 } 1006 1007 /* 1008 * check for a valid ce syndrome, then call the 1009 * displacement flush scrubbing code, and then check the afsr to see if 1010 * the error was persistent or intermittent. Reread the afar/afsr to see 1011 * if the error was not scrubbed successfully, and is therefore sticky. 1012 */ 1013 /*ARGSUSED1*/ 1014 void 1015 cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t triedcpulogout) 1016 { 1017 uint64_t eer, afsr; 1018 ushort_t status; 1019 1020 ASSERT(getpil() > LOCK_LEVEL); 1021 1022 /* 1023 * It is possible that the flt_addr is not a valid 1024 * physical address. To deal with this, we disable 1025 * NCEEN while we scrub that address. If this causes 1026 * a TIMEOUT/BERR, we know this is an invalid 1027 * memory location. 1028 */ 1029 kpreempt_disable(); 1030 eer = get_error_enable(); 1031 if (eer & (EER_CEEN | EER_NCEEN)) 1032 set_error_enable(eer & ~(EER_CEEN | EER_NCEEN)); 1033 1034 /* 1035 * To check if the error detected by IO is persistent, sticky or 1036 * intermittent. 1037 */ 1038 if (ecc->flt_status & ECC_IOBUS) { 1039 ecc->flt_stat = P_AFSR_CE; 1040 } 1041 1042 scrubphys(P2ALIGN(ecc->flt_addr, 64), 1043 cpunodes[CPU->cpu_id].ecache_size); 1044 1045 get_asyncflt(&afsr); 1046 if (afsr & (P_AFSR_TO | P_AFSR_BERR)) { 1047 /* 1048 * Must ensure that we don't get the TIMEOUT/BERR 1049 * when we reenable NCEEN, so we clear the AFSR. 1050 */ 1051 set_asyncflt(afsr & (P_AFSR_TO | P_AFSR_BERR)); 1052 if (eer & (EER_CEEN | EER_NCEEN)) 1053 set_error_enable(eer); 1054 kpreempt_enable(); 1055 return; 1056 } 1057 1058 if (eer & EER_NCEEN) 1059 set_error_enable(eer & ~EER_CEEN); 1060 1061 /* 1062 * Check and clear any ECC errors from the scrub. If the scrub did 1063 * not trip over the error, mark it intermittent. If the scrub did 1064 * trip the error again and it did not scrub away, mark it sticky. 1065 * Otherwise mark it persistent. 1066 */ 1067 if (check_ecc(ecc) != 0) { 1068 cpu_read_paddr(ecc, 0, 1); 1069 1070 if (check_ecc(ecc) != 0) 1071 status = ECC_STICKY; 1072 else 1073 status = ECC_PERSISTENT; 1074 } else 1075 status = ECC_INTERMITTENT; 1076 1077 if (eer & (EER_CEEN | EER_NCEEN)) 1078 set_error_enable(eer); 1079 kpreempt_enable(); 1080 1081 ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY); 1082 ecc->flt_status |= status; 1083 } 1084 1085 /* 1086 * get the syndrome and unum, and then call the routines 1087 * to check the other cpus and iobuses, and then do the error logging. 1088 */ 1089 /*ARGSUSED1*/ 1090 void 1091 cpu_ce_log_err(struct async_flt *ecc, errorq_elem_t *eqep) 1092 { 1093 char unum[UNUM_NAMLEN]; 1094 int len = 0; 1095 int ce_verbose = 0; 1096 int err; 1097 1098 ASSERT(ecc->flt_func != NULL); 1099 1100 /* Get the unum string for logging purposes */ 1101 (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, ecc, unum, 1102 UNUM_NAMLEN, &len); 1103 1104 /* Call specific error logging routine */ 1105 (void) (*ecc->flt_func)(ecc, unum); 1106 1107 /* 1108 * Count errors per unum. 1109 * Non-memory errors are all counted via a special unum string. 1110 */ 1111 if ((err = ce_count_unum(ecc->flt_status, len, unum)) != PR_OK && 1112 automatic_page_removal) { 1113 (void) page_retire(ecc->flt_addr, err); 1114 } 1115 1116 if (ecc->flt_panic) { 1117 ce_verbose = 1; 1118 } else if ((ecc->flt_class == BUS_FAULT) || 1119 (ecc->flt_stat & P_AFSR_CE)) { 1120 ce_verbose = (ce_verbose_memory > 0); 1121 } else { 1122 ce_verbose = 1; 1123 } 1124 1125 if (ce_verbose) { 1126 spitf_async_flt sflt; 1127 int synd_code; 1128 1129 sflt.cmn_asyncflt = *ecc; /* for cpu_aflt_log() */ 1130 1131 cpu_ce_log_status(&sflt, unum); 1132 1133 synd_code = synd_to_synd_code(AFLT_STAT_VALID, 1134 SYND(ecc->flt_synd)); 1135 1136 if (SYND_IS_SINGLE_BIT_DATA(synd_code)) { 1137 cpu_aflt_log(CE_CONT, 0, &sflt, CPU_ERRID_FIRST, 1138 NULL, " ECC Data Bit %2d was in error " 1139 "and corrected", synd_code); 1140 } else if (SYND_IS_SINGLE_BIT_CHK(synd_code)) { 1141 cpu_aflt_log(CE_CONT, 0, &sflt, CPU_ERRID_FIRST, 1142 NULL, " ECC Check Bit %2d was in error " 1143 "and corrected", synd_code - C0); 1144 } else { 1145 /* 1146 * These are UE errors - we shouldn't be getting CE 1147 * traps for these; handle them in case of bad h/w. 1148 */ 1149 switch (synd_code) { 1150 case M2: 1151 cpu_aflt_log(CE_CONT, 0, &sflt, 1152 CPU_ERRID_FIRST, NULL, 1153 " Two ECC Bits were in error"); 1154 break; 1155 case M3: 1156 cpu_aflt_log(CE_CONT, 0, &sflt, 1157 CPU_ERRID_FIRST, NULL, 1158 " Three ECC Bits were in error"); 1159 break; 1160 case M4: 1161 cpu_aflt_log(CE_CONT, 0, &sflt, 1162 CPU_ERRID_FIRST, NULL, 1163 " Four ECC Bits were in error"); 1164 break; 1165 case MX: 1166 cpu_aflt_log(CE_CONT, 0, &sflt, 1167 CPU_ERRID_FIRST, NULL, 1168 " More than Four ECC bits were " 1169 "in error"); 1170 break; 1171 default: 1172 cpu_aflt_log(CE_CONT, 0, &sflt, 1173 CPU_ERRID_FIRST, NULL, 1174 " Unknown fault syndrome %d", 1175 synd_code); 1176 break; 1177 } 1178 } 1179 } 1180 1181 /* Display entire cache line, if valid address */ 1182 if (ce_show_data && ecc->flt_addr != AFLT_INV_ADDR) 1183 read_ecc_data(ecc, 1, 1); 1184 } 1185 1186 /* 1187 * We route all errors through a single switch statement. 1188 */ 1189 void 1190 cpu_ue_log_err(struct async_flt *aflt) 1191 { 1192 1193 switch (aflt->flt_class) { 1194 case CPU_FAULT: 1195 cpu_async_log_err(aflt); 1196 break; 1197 1198 case BUS_FAULT: 1199 bus_async_log_err(aflt); 1200 break; 1201 1202 default: 1203 cmn_err(CE_WARN, "discarding async error 0x%p with invalid " 1204 "fault class (0x%x)", (void *)aflt, aflt->flt_class); 1205 break; 1206 } 1207 } 1208 1209 /* Values for action variable in cpu_async_error() */ 1210 #define ACTION_NONE 0 1211 #define ACTION_TRAMPOLINE 1 1212 #define ACTION_AST_FLAGS 2 1213 1214 /* 1215 * Access error trap handler for asynchronous cpu errors. This routine is 1216 * called to handle a data or instruction access error. All fatal errors are 1217 * completely handled by this routine (by panicking). Non fatal error logging 1218 * is queued for later processing either via AST or softint at a lower PIL. 1219 * In case of panic, the error log queue will also be processed as part of the 1220 * panic flow to ensure all errors are logged. This routine is called with all 1221 * errors disabled at PIL15. The AFSR bits are cleared and the UDBL and UDBH 1222 * error bits are also cleared. The hardware has also disabled the I and 1223 * D-caches for us, so we must re-enable them before returning. 1224 * 1225 * A summary of the handling of tl=0 UE/LDP/EDP/TO/BERR/WP/CP: 1226 * 1227 * _______________________________________________________________ 1228 * | Privileged tl0 | Unprivileged | 1229 * | Protected | Unprotected | Protected | Unprotected | 1230 * |on_trap|lofault| | | | 1231 * -------------|-------|-------+---------------+---------------+-------------| 1232 * | | | | | | 1233 * UE/LDP/EDP | L,T,p | L,R,p | L,P | n/a | L,R,p | 1234 * | | | | | | 1235 * TO/BERR | T | S | L,P | n/a | S | 1236 * | | | | | | 1237 * WP | L,M,p | L,M,p | L,M,p | n/a | L,M,p | 1238 * | | | | | | 1239 * CP (IIi/IIe) | L,P | L,P | L,P | n/a | L,P | 1240 * ____________________________________________________________________________ 1241 * 1242 * 1243 * Action codes: 1244 * 1245 * L - log 1246 * M - kick off memscrubber if flt_in_memory 1247 * P - panic 1248 * p - panic if US-IIi or US-IIe (Sabre); overrides R and M 1249 * R - i) if aft_panic is set, panic 1250 * ii) otherwise, send hwerr event to contract and SIGKILL to process 1251 * S - send SIGBUS to process 1252 * T - trampoline 1253 * 1254 * Special cases: 1255 * 1256 * 1) if aft_testfatal is set, all faults result in a panic regardless 1257 * of type (even WP), protection (even on_trap), or privilege. 1258 */ 1259 /*ARGSUSED*/ 1260 void 1261 cpu_async_error(struct regs *rp, ulong_t p_afar, ulong_t p_afsr, 1262 uint_t p_afsr_high, uint_t p_afar_high) 1263 { 1264 ushort_t sdbh, sdbl, ttype, tl; 1265 spitf_async_flt spf_flt; 1266 struct async_flt *aflt; 1267 char pr_reason[28]; 1268 uint64_t oafsr; 1269 uint64_t acc_afsr = 0; /* accumulated afsr */ 1270 int action = ACTION_NONE; 1271 uint64_t t_afar = p_afar; 1272 uint64_t t_afsr = p_afsr; 1273 int expected = DDI_FM_ERR_UNEXPECTED; 1274 ddi_acc_hdl_t *hp; 1275 1276 /* 1277 * We need to look at p_flag to determine if the thread detected an 1278 * error while dumping core. We can't grab p_lock here, but it's ok 1279 * because we just need a consistent snapshot and we know that everyone 1280 * else will store a consistent set of bits while holding p_lock. We 1281 * don't have to worry about a race because SDOCORE is set once prior 1282 * to doing i/o from the process's address space and is never cleared. 1283 */ 1284 uint_t pflag = ttoproc(curthread)->p_flag; 1285 1286 pr_reason[0] = '\0'; 1287 1288 /* 1289 * Note: the Spitfire data buffer error registers 1290 * (upper and lower halves) are or'ed into the upper 1291 * word of the afsr by async_err() if P_AFSR_UE is set. 1292 */ 1293 sdbh = (ushort_t)((t_afsr >> 33) & 0x3FF); 1294 sdbl = (ushort_t)((t_afsr >> 43) & 0x3FF); 1295 1296 /* 1297 * Grab the ttype encoded in <63:53> of the saved 1298 * afsr passed from async_err() 1299 */ 1300 ttype = (ushort_t)((t_afsr >> 53) & 0x1FF); 1301 tl = (ushort_t)(t_afsr >> 62); 1302 1303 t_afsr &= S_AFSR_MASK; 1304 t_afar &= SABRE_AFAR_PA; /* must use Sabre AFAR mask */ 1305 1306 /* 1307 * Initialize most of the common and CPU-specific structure. We derive 1308 * aflt->flt_priv from %tstate, instead of from the AFSR.PRIV bit. The 1309 * initial setting of aflt->flt_panic is based on TL: we must panic if 1310 * the error occurred at TL > 0. We also set flt_panic if the test/demo 1311 * tuneable aft_testfatal is set (not the default). 1312 */ 1313 bzero(&spf_flt, sizeof (spitf_async_flt)); 1314 aflt = (struct async_flt *)&spf_flt; 1315 aflt->flt_id = gethrtime_waitfree(); 1316 aflt->flt_stat = t_afsr; 1317 aflt->flt_addr = t_afar; 1318 aflt->flt_bus_id = getprocessorid(); 1319 aflt->flt_inst = CPU->cpu_id; 1320 aflt->flt_pc = (caddr_t)rp->r_pc; 1321 aflt->flt_prot = AFLT_PROT_NONE; 1322 aflt->flt_class = CPU_FAULT; 1323 aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 1324 aflt->flt_tl = (uchar_t)tl; 1325 aflt->flt_panic = (tl != 0 || aft_testfatal != 0); 1326 aflt->flt_core = (pflag & SDOCORE) ? 1 : 0; 1327 1328 /* 1329 * Set flt_status based on the trap type. If we end up here as the 1330 * result of a UE detected by the CE handling code, leave status 0. 1331 */ 1332 switch (ttype) { 1333 case T_DATA_ERROR: 1334 aflt->flt_status = ECC_D_TRAP; 1335 break; 1336 case T_INSTR_ERROR: 1337 aflt->flt_status = ECC_I_TRAP; 1338 break; 1339 } 1340 1341 spf_flt.flt_sdbh = sdbh; 1342 spf_flt.flt_sdbl = sdbl; 1343 1344 /* 1345 * Check for fatal async errors. 1346 */ 1347 check_misc_err(&spf_flt); 1348 1349 /* 1350 * If the trap occurred in privileged mode at TL=0, we need to check to 1351 * see if we were executing in the kernel under on_trap() or t_lofault 1352 * protection. If so, modify the saved registers so that we return 1353 * from the trap to the appropriate trampoline routine. 1354 */ 1355 if (aflt->flt_priv && tl == 0) { 1356 if (curthread->t_ontrap != NULL) { 1357 on_trap_data_t *otp = curthread->t_ontrap; 1358 1359 if (otp->ot_prot & OT_DATA_EC) { 1360 aflt->flt_prot = AFLT_PROT_EC; 1361 otp->ot_trap |= OT_DATA_EC; 1362 rp->r_pc = otp->ot_trampoline; 1363 rp->r_npc = rp->r_pc + 4; 1364 action = ACTION_TRAMPOLINE; 1365 } 1366 1367 if ((t_afsr & (P_AFSR_TO | P_AFSR_BERR)) && 1368 (otp->ot_prot & OT_DATA_ACCESS)) { 1369 aflt->flt_prot = AFLT_PROT_ACCESS; 1370 otp->ot_trap |= OT_DATA_ACCESS; 1371 rp->r_pc = otp->ot_trampoline; 1372 rp->r_npc = rp->r_pc + 4; 1373 action = ACTION_TRAMPOLINE; 1374 /* 1375 * for peeks and caut_gets errors are expected 1376 */ 1377 hp = (ddi_acc_hdl_t *)otp->ot_handle; 1378 if (!hp) 1379 expected = DDI_FM_ERR_PEEK; 1380 else if (hp->ah_acc.devacc_attr_access == 1381 DDI_CAUTIOUS_ACC) 1382 expected = DDI_FM_ERR_EXPECTED; 1383 } 1384 1385 } else if (curthread->t_lofault) { 1386 aflt->flt_prot = AFLT_PROT_COPY; 1387 rp->r_g1 = EFAULT; 1388 rp->r_pc = curthread->t_lofault; 1389 rp->r_npc = rp->r_pc + 4; 1390 action = ACTION_TRAMPOLINE; 1391 } 1392 } 1393 1394 /* 1395 * Determine if this error needs to be treated as fatal. Note that 1396 * multiple errors detected upon entry to this trap handler does not 1397 * necessarily warrant a panic. We only want to panic if the trap 1398 * happened in privileged mode and not under t_ontrap or t_lofault 1399 * protection. The exception is WP: if we *only* get WP, it is not 1400 * fatal even if the trap occurred in privileged mode, except on Sabre. 1401 * 1402 * aft_panic, if set, effectively makes us treat usermode 1403 * UE/EDP/LDP faults as if they were privileged - so we we will 1404 * panic instead of sending a contract event. A lofault-protected 1405 * fault will normally follow the contract event; if aft_panic is 1406 * set this will be changed to a panic. 1407 * 1408 * For usermode BERR/BTO errors, eg from processes performing device 1409 * control through mapped device memory, we need only deliver 1410 * a SIGBUS to the offending process. 1411 * 1412 * Some additional flt_panic reasons (eg, WP on Sabre) will be 1413 * checked later; for now we implement the common reasons. 1414 */ 1415 if (aflt->flt_prot == AFLT_PROT_NONE) { 1416 /* 1417 * Beware - multiple bits may be set in AFSR 1418 */ 1419 if (t_afsr & (P_AFSR_UE | P_AFSR_LDP | P_AFSR_EDP)) { 1420 if (aflt->flt_priv || aft_panic) 1421 aflt->flt_panic = 1; 1422 } 1423 1424 if (t_afsr & (P_AFSR_TO | P_AFSR_BERR)) { 1425 if (aflt->flt_priv) 1426 aflt->flt_panic = 1; 1427 } 1428 } else if (aflt->flt_prot == AFLT_PROT_COPY && aft_panic) { 1429 aflt->flt_panic = 1; 1430 } 1431 1432 /* 1433 * UE/BERR/TO: Call our bus nexus friends to check for 1434 * IO errors that may have resulted in this trap. 1435 */ 1436 if (t_afsr & (P_AFSR_TO | P_AFSR_BERR | P_AFSR_UE)) { 1437 cpu_run_bus_error_handlers(aflt, expected); 1438 } 1439 1440 /* 1441 * Handle UE: If the UE is in memory, we need to flush the bad line from 1442 * the E-cache. We also need to query the bus nexus for fatal errors. 1443 * For sabre, we will panic on UEs. Attempts to do diagnostic read on 1444 * caches may introduce more parity errors (especially when the module 1445 * is bad) and in sabre there is no guarantee that such errors 1446 * (if introduced) are written back as poisoned data. 1447 */ 1448 if (t_afsr & P_AFSR_UE) { 1449 int i; 1450 1451 (void) strcat(pr_reason, "UE "); 1452 1453 spf_flt.flt_type = CPU_UE_ERR; 1454 aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >> 1455 MMU_PAGESHIFT)) ? 1: 0; 1456 1457 /* 1458 * With UE, we have the PA of the fault. 1459 * Let do a diagnostic read to get the ecache 1460 * data and tag info of the bad line for logging. 1461 */ 1462 if (aflt->flt_in_memory) { 1463 uint32_t ec_set_size; 1464 uchar_t state; 1465 uint32_t ecache_idx; 1466 uint64_t faultpa = P2ALIGN(aflt->flt_addr, 64); 1467 1468 /* touch the line to put it in ecache */ 1469 acc_afsr |= read_and_clear_afsr(); 1470 (void) lddphys(faultpa); 1471 acc_afsr |= (read_and_clear_afsr() & 1472 ~(P_AFSR_EDP | P_AFSR_UE)); 1473 1474 ec_set_size = cpunodes[CPU->cpu_id].ecache_size / 1475 ecache_associativity; 1476 1477 for (i = 0; i < ecache_associativity; i++) { 1478 ecache_idx = i * ec_set_size + 1479 (aflt->flt_addr % ec_set_size); 1480 get_ecache_dtag(P2ALIGN(ecache_idx, 64), 1481 (uint64_t *)&spf_flt.flt_ec_data[0], 1482 &spf_flt.flt_ec_tag, &oafsr, &acc_afsr); 1483 acc_afsr |= oafsr; 1484 1485 state = (uchar_t)((spf_flt.flt_ec_tag & 1486 cpu_ec_state_mask) >> cpu_ec_state_shift); 1487 1488 if ((state & cpu_ec_state_valid) && 1489 ((spf_flt.flt_ec_tag & cpu_ec_tag_mask) == 1490 ((uint64_t)aflt->flt_addr >> 1491 cpu_ec_tag_shift))) 1492 break; 1493 } 1494 1495 /* 1496 * Check to see if the ecache tag is valid for the 1497 * fault PA. In the very unlikely event where the 1498 * line could be victimized, no ecache info will be 1499 * available. If this is the case, capture the line 1500 * from memory instead. 1501 */ 1502 if ((state & cpu_ec_state_valid) == 0 || 1503 (spf_flt.flt_ec_tag & cpu_ec_tag_mask) != 1504 ((uint64_t)aflt->flt_addr >> cpu_ec_tag_shift)) { 1505 for (i = 0; i < 8; i++, faultpa += 8) { 1506 ec_data_t *ecdptr; 1507 1508 ecdptr = &spf_flt.flt_ec_data[i]; 1509 acc_afsr |= read_and_clear_afsr(); 1510 ecdptr->ec_d8 = lddphys(faultpa); 1511 acc_afsr |= (read_and_clear_afsr() & 1512 ~(P_AFSR_EDP | P_AFSR_UE)); 1513 ecdptr->ec_afsr = 0; 1514 /* null afsr value */ 1515 } 1516 1517 /* 1518 * Mark tag invalid to indicate mem dump 1519 * when we print out the info. 1520 */ 1521 spf_flt.flt_ec_tag = AFLT_INV_ADDR; 1522 } 1523 spf_flt.flt_ec_lcnt = 1; 1524 1525 /* 1526 * Flush out the bad line 1527 */ 1528 flushecacheline(P2ALIGN(aflt->flt_addr, 64), 1529 cpunodes[CPU->cpu_id].ecache_size); 1530 1531 acc_afsr |= clear_errors(NULL, NULL); 1532 } 1533 1534 /* 1535 * Ask our bus nexus friends if they have any fatal errors. If 1536 * so, they will log appropriate error messages and panic as a 1537 * result. We then queue an event for each UDB that reports a 1538 * UE. Each UE reported in a UDB will have its own log message. 1539 * 1540 * Note from kbn: In the case where there are multiple UEs 1541 * (ME bit is set) - the AFAR address is only accurate to 1542 * the 16-byte granularity. One cannot tell whether the AFAR 1543 * belongs to the UDBH or UDBL syndromes. In this case, we 1544 * always report the AFAR address to be 16-byte aligned. 1545 * 1546 * If we're on a Sabre, there is no SDBL, but it will always 1547 * read as zero, so the sdbl test below will safely fail. 1548 */ 1549 if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL || isus2i || isus2e) 1550 aflt->flt_panic = 1; 1551 1552 if (sdbh & P_DER_UE) { 1553 aflt->flt_synd = sdbh & P_DER_E_SYND; 1554 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_UE, 1555 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1556 aflt->flt_panic); 1557 } 1558 if (sdbl & P_DER_UE) { 1559 aflt->flt_synd = sdbl & P_DER_E_SYND; 1560 aflt->flt_synd |= UDBL_REG; /* indicates UDBL */ 1561 if (!(aflt->flt_stat & P_AFSR_ME)) 1562 aflt->flt_addr |= 0x8; 1563 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_UE, 1564 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1565 aflt->flt_panic); 1566 } 1567 1568 /* 1569 * We got a UE and are panicking, save the fault PA in a known 1570 * location so that the platform specific panic code can check 1571 * for copyback errors. 1572 */ 1573 if (aflt->flt_panic && aflt->flt_in_memory) { 1574 panic_aflt = *aflt; 1575 } 1576 } 1577 1578 /* 1579 * Handle EDP and LDP: Locate the line with bad parity and enqueue an 1580 * async error for logging. For Sabre, we panic on EDP or LDP. 1581 */ 1582 if (t_afsr & (P_AFSR_EDP | P_AFSR_LDP)) { 1583 spf_flt.flt_type = CPU_EDP_LDP_ERR; 1584 1585 if (t_afsr & P_AFSR_EDP) 1586 (void) strcat(pr_reason, "EDP "); 1587 1588 if (t_afsr & P_AFSR_LDP) 1589 (void) strcat(pr_reason, "LDP "); 1590 1591 /* 1592 * Here we have no PA to work with. 1593 * Scan each line in the ecache to look for 1594 * the one with bad parity. 1595 */ 1596 aflt->flt_addr = AFLT_INV_ADDR; 1597 scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0], 1598 &spf_flt.flt_ec_tag, &spf_flt.flt_ec_lcnt, &oafsr); 1599 acc_afsr |= (oafsr & ~P_AFSR_WP); 1600 1601 /* 1602 * If we found a bad PA, update the state to indicate if it is 1603 * memory or I/O space. This code will be important if we ever 1604 * support cacheable frame buffers. 1605 */ 1606 if (aflt->flt_addr != AFLT_INV_ADDR) { 1607 aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >> 1608 MMU_PAGESHIFT)) ? 1 : 0; 1609 } 1610 1611 if (isus2i || isus2e) 1612 aflt->flt_panic = 1; 1613 1614 cpu_errorq_dispatch((t_afsr & P_AFSR_EDP) ? 1615 FM_EREPORT_CPU_USII_EDP : FM_EREPORT_CPU_USII_LDP, 1616 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1617 aflt->flt_panic); 1618 } 1619 1620 /* 1621 * Timeout and bus error handling. There are two cases to consider: 1622 * 1623 * (1) If we are in the kernel protected by ddi_peek or ddi_poke,we 1624 * have already modified the saved registers so that we will return 1625 * from the trap to the appropriate trampoline routine; otherwise panic. 1626 * 1627 * (2) In user mode, we can simply use our AST mechanism to deliver 1628 * a SIGBUS. We do not log the occurence - processes performing 1629 * device control would generate lots of uninteresting messages. 1630 */ 1631 if (t_afsr & (P_AFSR_TO | P_AFSR_BERR)) { 1632 if (t_afsr & P_AFSR_TO) 1633 (void) strcat(pr_reason, "BTO "); 1634 1635 if (t_afsr & P_AFSR_BERR) 1636 (void) strcat(pr_reason, "BERR "); 1637 1638 spf_flt.flt_type = CPU_BTO_BERR_ERR; 1639 if (aflt->flt_priv && aflt->flt_prot == AFLT_PROT_NONE) { 1640 cpu_errorq_dispatch((t_afsr & P_AFSR_TO) ? 1641 FM_EREPORT_CPU_USII_TO : FM_EREPORT_CPU_USII_BERR, 1642 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1643 aflt->flt_panic); 1644 } 1645 } 1646 1647 /* 1648 * Handle WP: WP happens when the ecache is victimized and a parity 1649 * error was detected on a writeback. The data in question will be 1650 * poisoned as a UE will be written back. The PA is not logged and 1651 * it is possible that it doesn't belong to the trapped thread. The 1652 * WP trap is not fatal, but it could be fatal to someone that 1653 * subsequently accesses the toxic page. We set read_all_memscrub 1654 * to force the memscrubber to read all of memory when it awakens. 1655 * For Sabre/Hummingbird, WP is fatal because the HW doesn't write a 1656 * UE back to poison the data. 1657 */ 1658 if (t_afsr & P_AFSR_WP) { 1659 (void) strcat(pr_reason, "WP "); 1660 if (isus2i || isus2e) { 1661 aflt->flt_panic = 1; 1662 } else { 1663 read_all_memscrub = 1; 1664 } 1665 spf_flt.flt_type = CPU_WP_ERR; 1666 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_WP, 1667 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1668 aflt->flt_panic); 1669 } 1670 1671 /* 1672 * Handle trapping CP error: In Sabre/Hummingbird, parity error in 1673 * the ecache on a copyout due to a PCI DMA read is signaled as a CP. 1674 * This is fatal. 1675 */ 1676 1677 if (t_afsr & P_AFSR_CP) { 1678 if (isus2i || isus2e) { 1679 (void) strcat(pr_reason, "CP "); 1680 aflt->flt_panic = 1; 1681 spf_flt.flt_type = CPU_TRAPPING_CP_ERR; 1682 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CP, 1683 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1684 aflt->flt_panic); 1685 } else { 1686 /* 1687 * Orphan CP: Happens due to signal integrity problem 1688 * on a CPU, where a CP is reported, without reporting 1689 * its associated UE. This is handled by locating the 1690 * bad parity line and would kick off the memscrubber 1691 * to find the UE if in memory or in another's cache. 1692 */ 1693 spf_flt.flt_type = CPU_ORPHAN_CP_ERR; 1694 (void) strcat(pr_reason, "ORPHAN_CP "); 1695 1696 /* 1697 * Here we have no PA to work with. 1698 * Scan each line in the ecache to look for 1699 * the one with bad parity. 1700 */ 1701 aflt->flt_addr = AFLT_INV_ADDR; 1702 scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0], 1703 &spf_flt.flt_ec_tag, &spf_flt.flt_ec_lcnt, 1704 &oafsr); 1705 acc_afsr |= oafsr; 1706 1707 /* 1708 * If we found a bad PA, update the state to indicate 1709 * if it is memory or I/O space. 1710 */ 1711 if (aflt->flt_addr != AFLT_INV_ADDR) { 1712 aflt->flt_in_memory = 1713 (pf_is_memory(aflt->flt_addr >> 1714 MMU_PAGESHIFT)) ? 1 : 0; 1715 } 1716 read_all_memscrub = 1; 1717 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CP, 1718 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1719 aflt->flt_panic); 1720 1721 } 1722 } 1723 1724 /* 1725 * If we queued an error other than WP or CP and we are going to return 1726 * from the trap and the error was in user mode or inside of a 1727 * copy routine, set AST flag so the queue will be drained before 1728 * returning to user mode. 1729 * 1730 * For UE/LDP/EDP, the AST processing will SIGKILL the process 1731 * and send an event to its process contract. 1732 * 1733 * For BERR/BTO, the AST processing will SIGBUS the process. There 1734 * will have been no error queued in this case. 1735 */ 1736 if ((t_afsr & 1737 (P_AFSR_UE | P_AFSR_LDP | P_AFSR_EDP | P_AFSR_BERR | P_AFSR_TO)) && 1738 (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY)) { 1739 int pcb_flag = 0; 1740 1741 if (t_afsr & (P_AFSR_UE | P_AFSR_LDP | P_AFSR_EDP)) 1742 pcb_flag |= ASYNC_HWERR; 1743 1744 if (t_afsr & P_AFSR_BERR) 1745 pcb_flag |= ASYNC_BERR; 1746 1747 if (t_afsr & P_AFSR_TO) 1748 pcb_flag |= ASYNC_BTO; 1749 1750 ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag; 1751 aston(curthread); 1752 action = ACTION_AST_FLAGS; 1753 } 1754 1755 /* 1756 * In response to a deferred error, we must do one of three things: 1757 * (1) set the AST flags, (2) trampoline, or (3) panic. action is 1758 * set in cases (1) and (2) - check that either action is set or 1759 * (3) is true. 1760 * 1761 * On II, the WP writes poisoned data back to memory, which will 1762 * cause a UE and a panic or reboot when read. In this case, we 1763 * don't need to panic at this time. On IIi and IIe, 1764 * aflt->flt_panic is already set above. 1765 */ 1766 ASSERT((aflt->flt_panic != 0) || (action != ACTION_NONE) || 1767 (t_afsr & P_AFSR_WP)); 1768 1769 /* 1770 * Make a final sanity check to make sure we did not get any more async 1771 * errors and accumulate the afsr. 1772 */ 1773 flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size * 2, 1774 cpunodes[CPU->cpu_id].ecache_linesize); 1775 (void) clear_errors(&spf_flt, NULL); 1776 1777 /* 1778 * Take care of a special case: If there is a UE in the ecache flush 1779 * area, we'll see it in flush_ecache(). This will trigger the 1780 * CPU_ADDITIONAL_ERRORS case below. 1781 * 1782 * This could occur if the original error was a UE in the flush area, 1783 * or if the original error was an E$ error that was flushed out of 1784 * the E$ in scan_ecache(). 1785 * 1786 * If it's at the same address that we're already logging, then it's 1787 * probably one of these cases. Clear the bit so we don't trip over 1788 * it on the additional errors case, which could cause an unnecessary 1789 * panic. 1790 */ 1791 if ((aflt->flt_stat & P_AFSR_UE) && aflt->flt_addr == t_afar) 1792 acc_afsr |= aflt->flt_stat & ~P_AFSR_UE; 1793 else 1794 acc_afsr |= aflt->flt_stat; 1795 1796 /* 1797 * Check the acumulated afsr for the important bits. 1798 * Make sure the spf_flt.flt_type value is set, and 1799 * enque an error. 1800 */ 1801 if (acc_afsr & 1802 (P_AFSR_LEVEL1 | P_AFSR_IVUE | P_AFSR_ETP | P_AFSR_ISAP)) { 1803 if (acc_afsr & (P_AFSR_UE | P_AFSR_EDP | P_AFSR_LDP | 1804 P_AFSR_BERR | P_AFSR_TO | P_AFSR_IVUE | P_AFSR_ETP | 1805 P_AFSR_ISAP)) 1806 aflt->flt_panic = 1; 1807 1808 spf_flt.flt_type = CPU_ADDITIONAL_ERR; 1809 aflt->flt_stat = acc_afsr; 1810 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_UNKNOWN, 1811 (void *)&spf_flt, sizeof (spf_flt), ue_queue, 1812 aflt->flt_panic); 1813 } 1814 1815 /* 1816 * If aflt->flt_panic is set at this point, we need to panic as the 1817 * result of a trap at TL > 0, or an error we determined to be fatal. 1818 * We've already enqueued the error in one of the if-clauses above, 1819 * and it will be dequeued and logged as part of the panic flow. 1820 */ 1821 if (aflt->flt_panic) { 1822 cpu_aflt_log(CE_PANIC, 1, &spf_flt, CPU_ERRID_FIRST, 1823 "See previous message(s) for details", " %sError(s)", 1824 pr_reason); 1825 } 1826 1827 /* 1828 * Before returning, we must re-enable errors, and 1829 * reset the caches to their boot-up state. 1830 */ 1831 set_lsu(get_lsu() | cache_boot_state); 1832 set_error_enable(EER_ENABLE); 1833 } 1834 1835 /* 1836 * Check for miscellaneous fatal errors and call CE_PANIC if any are seen. 1837 * This routine is shared by the CE and UE handling code. 1838 */ 1839 static void 1840 check_misc_err(spitf_async_flt *spf_flt) 1841 { 1842 struct async_flt *aflt = (struct async_flt *)spf_flt; 1843 char *fatal_str = NULL; 1844 1845 /* 1846 * The ISAP and ETP errors are supposed to cause a POR 1847 * from the system, so in theory we never, ever see these messages. 1848 * ISAP, ETP and IVUE are considered to be fatal. 1849 */ 1850 if (aflt->flt_stat & P_AFSR_ISAP) 1851 fatal_str = " System Address Parity Error on"; 1852 else if (aflt->flt_stat & P_AFSR_ETP) 1853 fatal_str = " Ecache Tag Parity Error on"; 1854 else if (aflt->flt_stat & P_AFSR_IVUE) 1855 fatal_str = " Interrupt Vector Uncorrectable Error on"; 1856 if (fatal_str != NULL) { 1857 cpu_aflt_log(CE_PANIC, 1, spf_flt, CMN_LFLAGS, 1858 NULL, fatal_str); 1859 } 1860 } 1861 1862 /* 1863 * Routine to convert a syndrome into a syndrome code. 1864 */ 1865 static int 1866 synd_to_synd_code(int synd_status, ushort_t synd) 1867 { 1868 if (synd_status != AFLT_STAT_VALID) 1869 return (-1); 1870 1871 /* 1872 * Use the 8-bit syndrome to index the ecc_syndrome_tab 1873 * to get the code indicating which bit(s) is(are) bad. 1874 */ 1875 if ((synd == 0) || (synd >= SYND_TBL_SIZE)) 1876 return (-1); 1877 else 1878 return (ecc_syndrome_tab[synd]); 1879 } 1880 1881 /* 1882 * Routine to return a string identifying the physical name 1883 * associated with a memory/cache error. 1884 */ 1885 /* ARGSUSED */ 1886 int 1887 cpu_get_mem_unum(int synd_status, ushort_t synd, uint64_t afsr, 1888 uint64_t afar, int cpuid, int flt_in_memory, ushort_t flt_status, 1889 char *buf, int buflen, int *lenp) 1890 { 1891 short synd_code; 1892 int ret; 1893 1894 if (flt_in_memory) { 1895 synd_code = synd_to_synd_code(synd_status, synd); 1896 if (synd_code == -1) { 1897 ret = EINVAL; 1898 } else if (prom_get_unum(synd_code, P2ALIGN(afar, 8), 1899 buf, buflen, lenp) != 0) { 1900 ret = EIO; 1901 } else if (*lenp <= 1) { 1902 ret = EINVAL; 1903 } else { 1904 ret = 0; 1905 } 1906 } else { 1907 ret = ENOTSUP; 1908 } 1909 1910 if (ret != 0) { 1911 buf[0] = '\0'; 1912 *lenp = 0; 1913 } 1914 1915 return (ret); 1916 } 1917 1918 /* 1919 * Wrapper for cpu_get_mem_unum() routine that takes an 1920 * async_flt struct rather than explicit arguments. 1921 */ 1922 int 1923 cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt, 1924 char *buf, int buflen, int *lenp) 1925 { 1926 return (cpu_get_mem_unum(synd_status, SYND(aflt->flt_synd), 1927 aflt->flt_stat, aflt->flt_addr, aflt->flt_bus_id, 1928 aflt->flt_in_memory, aflt->flt_status, buf, buflen, lenp)); 1929 } 1930 1931 /* 1932 * This routine is a more generic interface to cpu_get_mem_unum(), 1933 * that may be used by other modules (e.g. mm). 1934 */ 1935 int 1936 cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar, 1937 char *buf, int buflen, int *lenp) 1938 { 1939 int synd_status, flt_in_memory, ret; 1940 char unum[UNUM_NAMLEN]; 1941 1942 /* 1943 * Check for an invalid address. 1944 */ 1945 if (afar == (uint64_t)-1) 1946 return (ENXIO); 1947 1948 if (synd == (uint64_t)-1) 1949 synd_status = AFLT_STAT_INVALID; 1950 else 1951 synd_status = AFLT_STAT_VALID; 1952 1953 flt_in_memory = (pf_is_memory(afar >> MMU_PAGESHIFT)) ? 1 : 0; 1954 1955 if ((ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, *afsr, afar, 1956 CPU->cpu_id, flt_in_memory, 0, unum, UNUM_NAMLEN, lenp)) 1957 != 0) 1958 return (ret); 1959 1960 if (*lenp >= buflen) 1961 return (ENAMETOOLONG); 1962 1963 (void) strncpy(buf, unum, buflen); 1964 1965 return (0); 1966 } 1967 1968 /* 1969 * Routine to return memory information associated 1970 * with a physical address and syndrome. 1971 */ 1972 /* ARGSUSED */ 1973 int 1974 cpu_get_mem_info(uint64_t synd, uint64_t afar, 1975 uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 1976 int *segsp, int *banksp, int *mcidp) 1977 { 1978 return (ENOTSUP); 1979 } 1980 1981 /* 1982 * Routine to return a string identifying the physical 1983 * name associated with a cpuid. 1984 */ 1985 /* ARGSUSED */ 1986 int 1987 cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp) 1988 { 1989 return (ENOTSUP); 1990 } 1991 1992 /* 1993 * This routine returns the size of the kernel's FRU name buffer. 1994 */ 1995 size_t 1996 cpu_get_name_bufsize() 1997 { 1998 return (UNUM_NAMLEN); 1999 } 2000 2001 /* 2002 * Cpu specific log func for UEs. 2003 */ 2004 static void 2005 log_ue_err(struct async_flt *aflt, char *unum) 2006 { 2007 spitf_async_flt *spf_flt = (spitf_async_flt *)aflt; 2008 int len = 0; 2009 2010 #ifdef DEBUG 2011 int afsr_priv = (aflt->flt_stat & P_AFSR_PRIV) ? 1 : 0; 2012 2013 /* 2014 * Paranoid Check for priv mismatch 2015 * Only applicable for UEs 2016 */ 2017 if (afsr_priv != aflt->flt_priv) { 2018 /* 2019 * The priv bits in %tstate and %afsr did not match; we expect 2020 * this to be very rare, so flag it with a message. 2021 */ 2022 cpu_aflt_log(CE_WARN, 2, spf_flt, CPU_ERRID_FIRST, NULL, 2023 ": PRIV bit in TSTATE and AFSR mismatched; " 2024 "TSTATE.PRIV=%d used", (aflt->flt_priv) ? 1 : 0); 2025 2026 /* update saved afsr to reflect the correct priv */ 2027 aflt->flt_stat &= ~P_AFSR_PRIV; 2028 if (aflt->flt_priv) 2029 aflt->flt_stat |= P_AFSR_PRIV; 2030 } 2031 #endif /* DEBUG */ 2032 2033 (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, unum, 2034 UNUM_NAMLEN, &len); 2035 2036 cpu_aflt_log(CE_WARN, 1, spf_flt, UE_LFLAGS, unum, 2037 " Uncorrectable Memory Error on"); 2038 2039 if (SYND(aflt->flt_synd) == 0x3) { 2040 cpu_aflt_log(CE_WARN, 1, spf_flt, CPU_ERRID_FIRST, NULL, 2041 " Syndrome 0x3 indicates that this may not be a " 2042 "memory module problem"); 2043 } 2044 2045 if (aflt->flt_in_memory) 2046 cpu_log_ecmem_info(spf_flt); 2047 } 2048 2049 2050 /* 2051 * The cpu_async_log_err() function is called via the ue_drain() function to 2052 * handle logging for CPU events that are dequeued. As such, it can be invoked 2053 * from softint context, from AST processing in the trap() flow, or from the 2054 * panic flow. We decode the CPU-specific data, and log appropriate messages. 2055 */ 2056 static void 2057 cpu_async_log_err(void *flt) 2058 { 2059 spitf_async_flt *spf_flt = (spitf_async_flt *)flt; 2060 struct async_flt *aflt = (struct async_flt *)flt; 2061 char unum[UNUM_NAMLEN]; 2062 char *space; 2063 char *ecache_scrub_logstr = NULL; 2064 2065 switch (spf_flt->flt_type) { 2066 case CPU_UE_ERR: 2067 /* 2068 * We want to skip logging only if ALL the following 2069 * conditions are true: 2070 * 2071 * 1. We are not panicking 2072 * 2. There is only one error 2073 * 3. That error is a memory error 2074 * 4. The error is caused by the memory scrubber (in 2075 * which case the error will have occurred under 2076 * on_trap protection) 2077 * 5. The error is on a retired page 2078 * 2079 * Note 1: AFLT_PROT_EC is used places other than the memory 2080 * scrubber. However, none of those errors should occur 2081 * on a retired page. 2082 * 2083 * Note 2: In the CE case, these errors are discarded before 2084 * the errorq. In the UE case, we must wait until now -- 2085 * softcall() grabs a mutex, which we can't do at a high PIL. 2086 */ 2087 if (!panicstr && 2088 (aflt->flt_stat & S_AFSR_ALL_ERRS) == P_AFSR_UE && 2089 aflt->flt_prot == AFLT_PROT_EC) { 2090 if (page_retire_check(aflt->flt_addr, NULL) == 0) { 2091 /* Zero the address to clear the error */ 2092 softcall(ecc_page_zero, (void *)aflt->flt_addr); 2093 return; 2094 } 2095 } 2096 2097 /* 2098 * Log the UE and check for causes of this UE error that 2099 * don't cause a trap (Copyback error). cpu_async_error() 2100 * has already checked the i/o buses for us. 2101 */ 2102 log_ue_err(aflt, unum); 2103 if (aflt->flt_in_memory) 2104 cpu_check_allcpus(aflt); 2105 break; 2106 2107 case CPU_EDP_LDP_ERR: 2108 if (aflt->flt_stat & P_AFSR_EDP) 2109 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, 2110 NULL, " EDP event on"); 2111 2112 if (aflt->flt_stat & P_AFSR_LDP) 2113 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, 2114 NULL, " LDP event on"); 2115 2116 /* Log ecache info if exist */ 2117 if (spf_flt->flt_ec_lcnt > 0) { 2118 cpu_log_ecmem_info(spf_flt); 2119 2120 cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, 2121 NULL, " AFAR was derived from E$Tag"); 2122 } else { 2123 cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, 2124 NULL, " No error found in ecache (No fault " 2125 "PA available)"); 2126 } 2127 break; 2128 2129 case CPU_WP_ERR: 2130 /* 2131 * If the memscrub thread hasn't yet read 2132 * all of memory, as we requested in the 2133 * trap handler, then give it a kick to 2134 * make sure it does. 2135 */ 2136 if (!isus2i && !isus2e && read_all_memscrub) 2137 memscrub_run(); 2138 2139 cpu_aflt_log(CE_WARN, 1, spf_flt, WP_LFLAGS, NULL, 2140 " WP event on"); 2141 return; 2142 2143 case CPU_BTO_BERR_ERR: 2144 /* 2145 * A bus timeout or error occurred that was in user mode or not 2146 * in a protected kernel code region. 2147 */ 2148 if (aflt->flt_stat & P_AFSR_BERR) { 2149 cpu_aflt_log(CE_WARN, aflt->flt_panic ? 1 : 2, 2150 spf_flt, BERRTO_LFLAGS, NULL, 2151 " Bus Error on System Bus in %s mode from", 2152 aflt->flt_priv ? "privileged" : "user"); 2153 } 2154 2155 if (aflt->flt_stat & P_AFSR_TO) { 2156 cpu_aflt_log(CE_WARN, aflt->flt_panic ? 1 : 2, 2157 spf_flt, BERRTO_LFLAGS, NULL, 2158 " Timeout on System Bus in %s mode from", 2159 aflt->flt_priv ? "privileged" : "user"); 2160 } 2161 2162 return; 2163 2164 case CPU_PANIC_CP_ERR: 2165 /* 2166 * Process the Copyback (CP) error info (if any) obtained from 2167 * polling all the cpus in the panic flow. This case is only 2168 * entered if we are panicking. 2169 */ 2170 ASSERT(panicstr != NULL); 2171 ASSERT(aflt->flt_id == panic_aflt.flt_id); 2172 2173 /* See which space - this info may not exist */ 2174 if (panic_aflt.flt_status & ECC_D_TRAP) 2175 space = "Data "; 2176 else if (panic_aflt.flt_status & ECC_I_TRAP) 2177 space = "Instruction "; 2178 else 2179 space = ""; 2180 2181 cpu_aflt_log(CE_WARN, 1, spf_flt, CP_LFLAGS, NULL, 2182 " AFAR was derived from UE report," 2183 " CP event on CPU%d (caused %saccess error on %s%d)", 2184 aflt->flt_inst, space, (panic_aflt.flt_status & ECC_IOBUS) ? 2185 "IOBUS" : "CPU", panic_aflt.flt_bus_id); 2186 2187 if (spf_flt->flt_ec_lcnt > 0) 2188 cpu_log_ecmem_info(spf_flt); 2189 else 2190 cpu_aflt_log(CE_WARN, 2, spf_flt, CPU_ERRID_FIRST, 2191 NULL, " No cache dump available"); 2192 2193 return; 2194 2195 case CPU_TRAPPING_CP_ERR: 2196 /* 2197 * For sabre only. This is a copyback ecache parity error due 2198 * to a PCI DMA read. We should be panicking if we get here. 2199 */ 2200 ASSERT(panicstr != NULL); 2201 cpu_aflt_log(CE_WARN, 1, spf_flt, CP_LFLAGS, NULL, 2202 " AFAR was derived from UE report," 2203 " CP event on CPU%d (caused Data access error " 2204 "on PCIBus)", aflt->flt_inst); 2205 return; 2206 2207 /* 2208 * We log the ecache lines of the following states, 2209 * clean_bad_idle, clean_bad_busy, dirty_bad_idle and 2210 * dirty_bad_busy if ecache_scrub_verbose is set and panic 2211 * in addition to logging if ecache_scrub_panic is set. 2212 */ 2213 case CPU_BADLINE_CI_ERR: 2214 ecache_scrub_logstr = "CBI"; 2215 /* FALLTHRU */ 2216 2217 case CPU_BADLINE_CB_ERR: 2218 if (ecache_scrub_logstr == NULL) 2219 ecache_scrub_logstr = "CBB"; 2220 /* FALLTHRU */ 2221 2222 case CPU_BADLINE_DI_ERR: 2223 if (ecache_scrub_logstr == NULL) 2224 ecache_scrub_logstr = "DBI"; 2225 /* FALLTHRU */ 2226 2227 case CPU_BADLINE_DB_ERR: 2228 if (ecache_scrub_logstr == NULL) 2229 ecache_scrub_logstr = "DBB"; 2230 2231 cpu_aflt_log(CE_NOTE, 2, spf_flt, 2232 (CPU_ERRID_FIRST | CPU_FLTCPU), NULL, 2233 " %s event on", ecache_scrub_logstr); 2234 cpu_log_ecmem_info(spf_flt); 2235 2236 return; 2237 2238 case CPU_ORPHAN_CP_ERR: 2239 /* 2240 * Orphan CPs, where the CP bit is set, but when a CPU 2241 * doesn't report a UE. 2242 */ 2243 if (read_all_memscrub) 2244 memscrub_run(); 2245 2246 cpu_aflt_log(CE_NOTE, 2, spf_flt, (CP_LFLAGS | CPU_FLTCPU), 2247 NULL, " Orphan CP event on"); 2248 2249 /* Log ecache info if exist */ 2250 if (spf_flt->flt_ec_lcnt > 0) 2251 cpu_log_ecmem_info(spf_flt); 2252 else 2253 cpu_aflt_log(CE_NOTE, 2, spf_flt, 2254 (CP_LFLAGS | CPU_FLTCPU), NULL, 2255 " No error found in ecache (No fault " 2256 "PA available"); 2257 return; 2258 2259 case CPU_ECACHE_ADDR_PAR_ERR: 2260 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL, 2261 " E$ Tag Address Parity error on"); 2262 cpu_log_ecmem_info(spf_flt); 2263 return; 2264 2265 case CPU_ECACHE_STATE_ERR: 2266 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL, 2267 " E$ Tag State Parity error on"); 2268 cpu_log_ecmem_info(spf_flt); 2269 return; 2270 2271 case CPU_ECACHE_TAG_ERR: 2272 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL, 2273 " E$ Tag scrub event on"); 2274 cpu_log_ecmem_info(spf_flt); 2275 return; 2276 2277 case CPU_ECACHE_ETP_ETS_ERR: 2278 cpu_aflt_log(CE_WARN, 1, spf_flt, PARERR_LFLAGS, NULL, 2279 " AFSR.ETP is set and AFSR.ETS is zero on"); 2280 cpu_log_ecmem_info(spf_flt); 2281 return; 2282 2283 2284 case CPU_ADDITIONAL_ERR: 2285 cpu_aflt_log(CE_WARN, 1, spf_flt, CMN_LFLAGS & ~CPU_SPACE, NULL, 2286 " Additional errors detected during error processing on"); 2287 return; 2288 2289 default: 2290 cmn_err(CE_WARN, "cpu_async_log_err: fault %p has unknown " 2291 "fault type %x", (void *)spf_flt, spf_flt->flt_type); 2292 return; 2293 } 2294 2295 /* ... fall through from the UE, EDP, or LDP cases */ 2296 2297 if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) { 2298 if (!panicstr) { 2299 (void) page_retire(aflt->flt_addr, PR_UE); 2300 } else { 2301 /* 2302 * Clear UEs on panic so that we don't 2303 * get haunted by them during panic or 2304 * after reboot 2305 */ 2306 clearphys(P2ALIGN(aflt->flt_addr, 64), 2307 cpunodes[CPU->cpu_id].ecache_size, 2308 cpunodes[CPU->cpu_id].ecache_linesize); 2309 2310 (void) clear_errors(NULL, NULL); 2311 } 2312 } 2313 2314 /* 2315 * Log final recover message 2316 */ 2317 if (!panicstr) { 2318 if (!aflt->flt_priv) { 2319 cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST, 2320 NULL, " Above Error is in User Mode" 2321 "\n and is fatal: " 2322 "will SIGKILL process and notify contract"); 2323 } else if (aflt->flt_prot == AFLT_PROT_COPY && aflt->flt_core) { 2324 cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST, 2325 NULL, " Above Error detected while dumping core;" 2326 "\n core file will be truncated"); 2327 } else if (aflt->flt_prot == AFLT_PROT_COPY) { 2328 cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST, 2329 NULL, " Above Error is due to Kernel access" 2330 "\n to User space and is fatal: " 2331 "will SIGKILL process and notify contract"); 2332 } else if (aflt->flt_prot == AFLT_PROT_EC) { 2333 cpu_aflt_log(CE_CONT, 3, spf_flt, CPU_ERRID_FIRST, NULL, 2334 " Above Error detected by protected Kernel code" 2335 "\n that will try to clear error from system"); 2336 } 2337 } 2338 } 2339 2340 2341 /* 2342 * Check all cpus for non-trapping UE-causing errors 2343 * In Ultra I/II, we look for copyback errors (CPs) 2344 */ 2345 void 2346 cpu_check_allcpus(struct async_flt *aflt) 2347 { 2348 spitf_async_flt cp; 2349 spitf_async_flt *spf_cpflt = &cp; 2350 struct async_flt *cpflt = (struct async_flt *)&cp; 2351 int pix; 2352 2353 cpflt->flt_id = aflt->flt_id; 2354 cpflt->flt_addr = aflt->flt_addr; 2355 2356 for (pix = 0; pix < NCPU; pix++) { 2357 if (CPU_XCALL_READY(pix)) { 2358 xc_one(pix, (xcfunc_t *)get_cpu_status, 2359 (uint64_t)cpflt, 0); 2360 2361 if (cpflt->flt_stat & P_AFSR_CP) { 2362 char *space; 2363 2364 /* See which space - this info may not exist */ 2365 if (aflt->flt_status & ECC_D_TRAP) 2366 space = "Data "; 2367 else if (aflt->flt_status & ECC_I_TRAP) 2368 space = "Instruction "; 2369 else 2370 space = ""; 2371 2372 cpu_aflt_log(CE_WARN, 1, spf_cpflt, CP_LFLAGS, 2373 NULL, " AFAR was derived from UE report," 2374 " CP event on CPU%d (caused %saccess " 2375 "error on %s%d)", pix, space, 2376 (aflt->flt_status & ECC_IOBUS) ? 2377 "IOBUS" : "CPU", aflt->flt_bus_id); 2378 2379 if (spf_cpflt->flt_ec_lcnt > 0) 2380 cpu_log_ecmem_info(spf_cpflt); 2381 else 2382 cpu_aflt_log(CE_WARN, 2, spf_cpflt, 2383 CPU_ERRID_FIRST, NULL, 2384 " No cache dump available"); 2385 } 2386 } 2387 } 2388 } 2389 2390 #ifdef DEBUG 2391 int test_mp_cp = 0; 2392 #endif 2393 2394 /* 2395 * Cross-call callback routine to tell a CPU to read its own %afsr to check 2396 * for copyback errors and capture relevant information. 2397 */ 2398 static uint_t 2399 get_cpu_status(uint64_t arg) 2400 { 2401 struct async_flt *aflt = (struct async_flt *)arg; 2402 spitf_async_flt *spf_flt = (spitf_async_flt *)arg; 2403 uint64_t afsr; 2404 uint32_t ec_idx; 2405 uint64_t sdbh, sdbl; 2406 int i; 2407 uint32_t ec_set_size; 2408 uchar_t valid; 2409 ec_data_t ec_data[8]; 2410 uint64_t ec_tag, flt_addr_tag, oafsr; 2411 uint64_t *acc_afsr = NULL; 2412 2413 get_asyncflt(&afsr); 2414 if (CPU_PRIVATE(CPU) != NULL) { 2415 acc_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 2416 afsr |= *acc_afsr; 2417 *acc_afsr = 0; 2418 } 2419 2420 #ifdef DEBUG 2421 if (test_mp_cp) 2422 afsr |= P_AFSR_CP; 2423 #endif 2424 aflt->flt_stat = afsr; 2425 2426 if (afsr & P_AFSR_CP) { 2427 /* 2428 * Capture the UDBs 2429 */ 2430 get_udb_errors(&sdbh, &sdbl); 2431 spf_flt->flt_sdbh = (ushort_t)(sdbh & 0x3FF); 2432 spf_flt->flt_sdbl = (ushort_t)(sdbl & 0x3FF); 2433 2434 /* 2435 * Clear CP bit before capturing ecache data 2436 * and AFSR info. 2437 */ 2438 set_asyncflt(P_AFSR_CP); 2439 2440 /* 2441 * See if we can capture the ecache line for the 2442 * fault PA. 2443 * 2444 * Return a valid matching ecache line, if any. 2445 * Otherwise, return the first matching ecache 2446 * line marked invalid. 2447 */ 2448 flt_addr_tag = aflt->flt_addr >> cpu_ec_tag_shift; 2449 ec_set_size = cpunodes[CPU->cpu_id].ecache_size / 2450 ecache_associativity; 2451 spf_flt->flt_ec_lcnt = 0; 2452 2453 for (i = 0, ec_idx = (aflt->flt_addr % ec_set_size); 2454 i < ecache_associativity; i++, ec_idx += ec_set_size) { 2455 get_ecache_dtag(P2ALIGN(ec_idx, 64), 2456 (uint64_t *)&ec_data[0], &ec_tag, &oafsr, 2457 acc_afsr); 2458 2459 if ((ec_tag & cpu_ec_tag_mask) != flt_addr_tag) 2460 continue; 2461 2462 valid = cpu_ec_state_valid & 2463 (uchar_t)((ec_tag & cpu_ec_state_mask) >> 2464 cpu_ec_state_shift); 2465 2466 if (valid || spf_flt->flt_ec_lcnt == 0) { 2467 spf_flt->flt_ec_tag = ec_tag; 2468 bcopy(&ec_data, &spf_flt->flt_ec_data, 2469 sizeof (ec_data)); 2470 spf_flt->flt_ec_lcnt = 1; 2471 2472 if (valid) 2473 break; 2474 } 2475 } 2476 } 2477 return (0); 2478 } 2479 2480 /* 2481 * CPU-module callback for the non-panicking CPUs. This routine is invoked 2482 * from panic_idle() as part of the other CPUs stopping themselves when a 2483 * panic occurs. We need to be VERY careful what we do here, since panicstr 2484 * is NOT set yet and we cannot blow through locks. If panic_aflt is set 2485 * (panic_aflt.flt_id is non-zero), we need to read our %afsr to look for 2486 * CP error information. 2487 */ 2488 void 2489 cpu_async_panic_callb(void) 2490 { 2491 spitf_async_flt cp; 2492 struct async_flt *aflt = (struct async_flt *)&cp; 2493 uint64_t *scrub_afsr; 2494 2495 if (panic_aflt.flt_id != 0) { 2496 aflt->flt_addr = panic_aflt.flt_addr; 2497 (void) get_cpu_status((uint64_t)aflt); 2498 2499 if (CPU_PRIVATE(CPU) != NULL) { 2500 scrub_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 2501 if (*scrub_afsr & P_AFSR_CP) { 2502 aflt->flt_stat |= *scrub_afsr; 2503 *scrub_afsr = 0; 2504 } 2505 } 2506 if (aflt->flt_stat & P_AFSR_CP) { 2507 aflt->flt_id = panic_aflt.flt_id; 2508 aflt->flt_panic = 1; 2509 aflt->flt_inst = CPU->cpu_id; 2510 aflt->flt_class = CPU_FAULT; 2511 cp.flt_type = CPU_PANIC_CP_ERR; 2512 cpu_errorq_dispatch(FM_EREPORT_CPU_USII_CP, 2513 (void *)&cp, sizeof (cp), ue_queue, 2514 aflt->flt_panic); 2515 } 2516 } 2517 } 2518 2519 /* 2520 * Turn off all cpu error detection, normally only used for panics. 2521 */ 2522 void 2523 cpu_disable_errors(void) 2524 { 2525 xt_all(set_error_enable_tl1, EER_DISABLE, EER_SET_ABSOLUTE); 2526 } 2527 2528 /* 2529 * Enable errors. 2530 */ 2531 void 2532 cpu_enable_errors(void) 2533 { 2534 xt_all(set_error_enable_tl1, EER_ENABLE, EER_SET_ABSOLUTE); 2535 } 2536 2537 static void 2538 cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err) 2539 { 2540 uint64_t aligned_addr = P2ALIGN(ecc->flt_addr, 8); 2541 int i, loop = 1; 2542 ushort_t ecc_0; 2543 uint64_t paddr; 2544 uint64_t data; 2545 2546 if (verbose) 2547 loop = 8; 2548 for (i = 0; i < loop; i++) { 2549 paddr = aligned_addr + (i * 8); 2550 data = lddphys(paddr); 2551 if (verbose) { 2552 if (ce_err) { 2553 ecc_0 = ecc_gen((uint32_t)(data>>32), 2554 (uint32_t)data); 2555 cpu_aflt_log(CE_CONT, 0, NULL, NO_LFLAGS, 2556 NULL, " Paddr 0x%" PRIx64 ", " 2557 "Data 0x%08x.%08x, ECC 0x%x", paddr, 2558 (uint32_t)(data>>32), (uint32_t)data, ecc_0); 2559 } else { 2560 cpu_aflt_log(CE_CONT, 0, NULL, NO_LFLAGS, 2561 NULL, " Paddr 0x%" PRIx64 ", " 2562 "Data 0x%08x.%08x", paddr, 2563 (uint32_t)(data>>32), (uint32_t)data); 2564 } 2565 } 2566 } 2567 } 2568 2569 static struct { /* sec-ded-s4ed ecc code */ 2570 uint_t hi, lo; 2571 } ecc_code[8] = { 2572 { 0xee55de23U, 0x16161161U }, 2573 { 0x55eede93U, 0x61612212U }, 2574 { 0xbb557b8cU, 0x49494494U }, 2575 { 0x55bb7b6cU, 0x94948848U }, 2576 { 0x16161161U, 0xee55de23U }, 2577 { 0x61612212U, 0x55eede93U }, 2578 { 0x49494494U, 0xbb557b8cU }, 2579 { 0x94948848U, 0x55bb7b6cU } 2580 }; 2581 2582 static ushort_t 2583 ecc_gen(uint_t high_bytes, uint_t low_bytes) 2584 { 2585 int i, j; 2586 uchar_t checker, bit_mask; 2587 struct { 2588 uint_t hi, lo; 2589 } hex_data, masked_data[8]; 2590 2591 hex_data.hi = high_bytes; 2592 hex_data.lo = low_bytes; 2593 2594 /* mask out bits according to sec-ded-s4ed ecc code */ 2595 for (i = 0; i < 8; i++) { 2596 masked_data[i].hi = hex_data.hi & ecc_code[i].hi; 2597 masked_data[i].lo = hex_data.lo & ecc_code[i].lo; 2598 } 2599 2600 /* 2601 * xor all bits in masked_data[i] to get bit_i of checker, 2602 * where i = 0 to 7 2603 */ 2604 checker = 0; 2605 for (i = 0; i < 8; i++) { 2606 bit_mask = 1 << i; 2607 for (j = 0; j < 32; j++) { 2608 if (masked_data[i].lo & 1) checker ^= bit_mask; 2609 if (masked_data[i].hi & 1) checker ^= bit_mask; 2610 masked_data[i].hi >>= 1; 2611 masked_data[i].lo >>= 1; 2612 } 2613 } 2614 return (checker); 2615 } 2616 2617 /* 2618 * Flush the entire ecache using displacement flush by reading through a 2619 * physical address range as large as the ecache. 2620 */ 2621 void 2622 cpu_flush_ecache(void) 2623 { 2624 flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size * 2, 2625 cpunodes[CPU->cpu_id].ecache_linesize); 2626 } 2627 2628 /* 2629 * read and display the data in the cache line where the 2630 * original ce error occurred. 2631 * This routine is mainly used for debugging new hardware. 2632 */ 2633 void 2634 read_ecc_data(struct async_flt *ecc, short verbose, short ce_err) 2635 { 2636 kpreempt_disable(); 2637 /* disable ECC error traps */ 2638 set_error_enable(EER_ECC_DISABLE); 2639 2640 /* 2641 * flush the ecache 2642 * read the data 2643 * check to see if an ECC error occured 2644 */ 2645 flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size * 2, 2646 cpunodes[CPU->cpu_id].ecache_linesize); 2647 set_lsu(get_lsu() | cache_boot_state); 2648 cpu_read_paddr(ecc, verbose, ce_err); 2649 (void) check_ecc(ecc); 2650 2651 /* enable ECC error traps */ 2652 set_error_enable(EER_ENABLE); 2653 kpreempt_enable(); 2654 } 2655 2656 /* 2657 * Check the AFSR bits for UE/CE persistence. 2658 * If UE or CE errors are detected, the routine will 2659 * clears all the AFSR sticky bits (except CP for 2660 * spitfire/blackbird) and the UDBs. 2661 * if ce_debug or ue_debug is set, log any ue/ce errors detected. 2662 */ 2663 static int 2664 check_ecc(struct async_flt *ecc) 2665 { 2666 uint64_t t_afsr; 2667 uint64_t t_afar; 2668 uint64_t udbh; 2669 uint64_t udbl; 2670 ushort_t udb; 2671 int persistent = 0; 2672 2673 /* 2674 * Capture the AFSR, AFAR and UDBs info 2675 */ 2676 get_asyncflt(&t_afsr); 2677 get_asyncaddr(&t_afar); 2678 t_afar &= SABRE_AFAR_PA; 2679 get_udb_errors(&udbh, &udbl); 2680 2681 if ((t_afsr & P_AFSR_UE) || (t_afsr & P_AFSR_CE)) { 2682 /* 2683 * Clear the errors 2684 */ 2685 clr_datapath(); 2686 2687 if (isus2i || isus2e) 2688 set_asyncflt(t_afsr); 2689 else 2690 set_asyncflt(t_afsr & ~P_AFSR_CP); 2691 2692 /* 2693 * determine whether to check UDBH or UDBL for persistence 2694 */ 2695 if (ecc->flt_synd & UDBL_REG) { 2696 udb = (ushort_t)udbl; 2697 t_afar |= 0x8; 2698 } else { 2699 udb = (ushort_t)udbh; 2700 } 2701 2702 if (ce_debug || ue_debug) { 2703 spitf_async_flt spf_flt; /* for logging */ 2704 struct async_flt *aflt = 2705 (struct async_flt *)&spf_flt; 2706 2707 /* Package the info nicely in the spf_flt struct */ 2708 bzero(&spf_flt, sizeof (spitf_async_flt)); 2709 aflt->flt_stat = t_afsr; 2710 aflt->flt_addr = t_afar; 2711 spf_flt.flt_sdbh = (ushort_t)(udbh & 0x3FF); 2712 spf_flt.flt_sdbl = (ushort_t)(udbl & 0x3FF); 2713 2714 cpu_aflt_log(CE_CONT, 0, &spf_flt, (CPU_AFSR | 2715 CPU_AFAR | CPU_UDBH | CPU_UDBL), NULL, 2716 " check_ecc: Dumping captured error states ..."); 2717 } 2718 2719 /* 2720 * if the fault addresses don't match, not persistent 2721 */ 2722 if (t_afar != ecc->flt_addr) { 2723 return (persistent); 2724 } 2725 2726 /* 2727 * check for UE persistence 2728 * since all DIMMs in the bank are identified for a UE, 2729 * there's no reason to check the syndrome 2730 */ 2731 if ((ecc->flt_stat & P_AFSR_UE) && (t_afsr & P_AFSR_UE)) { 2732 persistent = 1; 2733 } 2734 2735 /* 2736 * check for CE persistence 2737 */ 2738 if ((ecc->flt_stat & P_AFSR_CE) && (t_afsr & P_AFSR_CE)) { 2739 if ((udb & P_DER_E_SYND) == 2740 (ecc->flt_synd & P_DER_E_SYND)) { 2741 persistent = 1; 2742 } 2743 } 2744 } 2745 return (persistent); 2746 } 2747 2748 #ifdef HUMMINGBIRD 2749 #define HB_FULL_DIV 1 2750 #define HB_HALF_DIV 2 2751 #define HB_LOWEST_DIV 8 2752 #define HB_ECLK_INVALID 0xdeadbad 2753 static uint64_t hb_eclk[HB_LOWEST_DIV + 1] = { 2754 HB_ECLK_INVALID, HB_ECLK_1, HB_ECLK_2, HB_ECLK_INVALID, 2755 HB_ECLK_4, HB_ECLK_INVALID, HB_ECLK_6, HB_ECLK_INVALID, 2756 HB_ECLK_8 }; 2757 2758 #define HB_SLOW_DOWN 0 2759 #define HB_SPEED_UP 1 2760 2761 #define SET_ESTAR_MODE(mode) \ 2762 stdphysio(HB_ESTAR_MODE, (mode)); \ 2763 /* \ 2764 * PLL logic requires minimum of 16 clock \ 2765 * cycles to lock to the new clock speed. \ 2766 * Wait 1 usec to satisfy this requirement. \ 2767 */ \ 2768 drv_usecwait(1); 2769 2770 #define CHANGE_REFRESH_COUNT(direction, cur_div, new_div) \ 2771 { \ 2772 volatile uint64_t data; \ 2773 uint64_t count, new_count; \ 2774 clock_t delay; \ 2775 data = lddphysio(HB_MEM_CNTRL0); \ 2776 count = (data & HB_REFRESH_COUNT_MASK) >> \ 2777 HB_REFRESH_COUNT_SHIFT; \ 2778 new_count = (HB_REFRESH_INTERVAL * \ 2779 cpunodes[CPU->cpu_id].clock_freq) / \ 2780 (HB_REFRESH_CLOCKS_PER_COUNT * (new_div) * NANOSEC);\ 2781 data = (data & ~HB_REFRESH_COUNT_MASK) | \ 2782 (new_count << HB_REFRESH_COUNT_SHIFT); \ 2783 stdphysio(HB_MEM_CNTRL0, data); \ 2784 data = lddphysio(HB_MEM_CNTRL0); \ 2785 /* \ 2786 * If we are slowing down the cpu and Memory \ 2787 * Self Refresh is not enabled, it is required \ 2788 * to wait for old refresh count to count-down and \ 2789 * new refresh count to go into effect (let new value \ 2790 * counts down once). \ 2791 */ \ 2792 if ((direction) == HB_SLOW_DOWN && \ 2793 (data & HB_SELF_REFRESH_MASK) == 0) { \ 2794 /* \ 2795 * Each count takes 64 cpu clock cycles \ 2796 * to decrement. Wait for current refresh \ 2797 * count plus new refresh count at current \ 2798 * cpu speed to count down to zero. Round \ 2799 * up the delay time. \ 2800 */ \ 2801 delay = ((HB_REFRESH_CLOCKS_PER_COUNT * \ 2802 (count + new_count) * MICROSEC * (cur_div)) /\ 2803 cpunodes[CPU->cpu_id].clock_freq) + 1; \ 2804 drv_usecwait(delay); \ 2805 } \ 2806 } 2807 2808 #define SET_SELF_REFRESH(bit) \ 2809 { \ 2810 volatile uint64_t data; \ 2811 data = lddphysio(HB_MEM_CNTRL0); \ 2812 data = (data & ~HB_SELF_REFRESH_MASK) | \ 2813 ((bit) << HB_SELF_REFRESH_SHIFT); \ 2814 stdphysio(HB_MEM_CNTRL0, data); \ 2815 data = lddphysio(HB_MEM_CNTRL0); \ 2816 } 2817 #endif /* HUMMINGBIRD */ 2818 2819 /* ARGSUSED */ 2820 void 2821 cpu_change_speed(uint64_t new_divisor, uint64_t arg2) 2822 { 2823 #ifdef HUMMINGBIRD 2824 uint64_t cur_mask, cur_divisor = 0; 2825 volatile uint64_t reg; 2826 int index; 2827 2828 if ((new_divisor < HB_FULL_DIV || new_divisor > HB_LOWEST_DIV) || 2829 (hb_eclk[new_divisor] == HB_ECLK_INVALID)) { 2830 cmn_err(CE_WARN, "cpu_change_speed: bad divisor 0x%lx", 2831 new_divisor); 2832 return; 2833 } 2834 2835 reg = lddphysio(HB_ESTAR_MODE); 2836 cur_mask = reg & HB_ECLK_MASK; 2837 for (index = HB_FULL_DIV; index <= HB_LOWEST_DIV; index++) { 2838 if (hb_eclk[index] == cur_mask) { 2839 cur_divisor = index; 2840 break; 2841 } 2842 } 2843 2844 if (cur_divisor == 0) 2845 cmn_err(CE_PANIC, "cpu_change_speed: current divisor " 2846 "can't be determined!"); 2847 2848 /* 2849 * If we are already at the requested divisor speed, just 2850 * return. 2851 */ 2852 if (cur_divisor == new_divisor) 2853 return; 2854 2855 if (cur_divisor == HB_FULL_DIV && new_divisor == HB_HALF_DIV) { 2856 CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, cur_divisor, new_divisor); 2857 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2858 SET_SELF_REFRESH(HB_SELF_REFRESH_ENABLE); 2859 2860 } else if (cur_divisor == HB_HALF_DIV && new_divisor == HB_FULL_DIV) { 2861 SET_SELF_REFRESH(HB_SELF_REFRESH_DISABLE); 2862 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2863 /* LINTED: E_FALSE_LOGICAL_EXPR */ 2864 CHANGE_REFRESH_COUNT(HB_SPEED_UP, cur_divisor, new_divisor); 2865 2866 } else if (cur_divisor == HB_FULL_DIV && new_divisor > HB_HALF_DIV) { 2867 /* 2868 * Transition to 1/2 speed first, then to 2869 * lower speed. 2870 */ 2871 CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, cur_divisor, HB_HALF_DIV); 2872 SET_ESTAR_MODE(hb_eclk[HB_HALF_DIV]); 2873 SET_SELF_REFRESH(HB_SELF_REFRESH_ENABLE); 2874 2875 CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, HB_HALF_DIV, new_divisor); 2876 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2877 2878 } else if (cur_divisor > HB_HALF_DIV && new_divisor == HB_FULL_DIV) { 2879 /* 2880 * Transition to 1/2 speed first, then to 2881 * full speed. 2882 */ 2883 SET_ESTAR_MODE(hb_eclk[HB_HALF_DIV]); 2884 /* LINTED: E_FALSE_LOGICAL_EXPR */ 2885 CHANGE_REFRESH_COUNT(HB_SPEED_UP, cur_divisor, HB_HALF_DIV); 2886 2887 SET_SELF_REFRESH(HB_SELF_REFRESH_DISABLE); 2888 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2889 /* LINTED: E_FALSE_LOGICAL_EXPR */ 2890 CHANGE_REFRESH_COUNT(HB_SPEED_UP, HB_HALF_DIV, new_divisor); 2891 2892 } else if (cur_divisor < new_divisor) { 2893 CHANGE_REFRESH_COUNT(HB_SLOW_DOWN, cur_divisor, new_divisor); 2894 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2895 2896 } else if (cur_divisor > new_divisor) { 2897 SET_ESTAR_MODE(hb_eclk[new_divisor]); 2898 /* LINTED: E_FALSE_LOGICAL_EXPR */ 2899 CHANGE_REFRESH_COUNT(HB_SPEED_UP, cur_divisor, new_divisor); 2900 } 2901 CPU->cpu_m.divisor = (uchar_t)new_divisor; 2902 #endif 2903 } 2904 2905 /* 2906 * Clear the AFSR sticky bits and the UDBs. For Sabre/Spitfire/Blackbird, 2907 * we clear all the sticky bits. If a non-null pointer to a async fault 2908 * structure argument is passed in, the captured error state (AFSR, AFAR, UDBs) 2909 * info will be returned in the structure. If a non-null pointer to a 2910 * uint64_t is passed in, this will be updated if the CP bit is set in the 2911 * AFSR. The afsr will be returned. 2912 */ 2913 static uint64_t 2914 clear_errors(spitf_async_flt *spf_flt, uint64_t *acc_afsr) 2915 { 2916 struct async_flt *aflt = (struct async_flt *)spf_flt; 2917 uint64_t afsr; 2918 uint64_t udbh, udbl; 2919 2920 get_asyncflt(&afsr); 2921 2922 if ((acc_afsr != NULL) && (afsr & P_AFSR_CP)) 2923 *acc_afsr |= afsr; 2924 2925 if (spf_flt != NULL) { 2926 aflt->flt_stat = afsr; 2927 get_asyncaddr(&aflt->flt_addr); 2928 aflt->flt_addr &= SABRE_AFAR_PA; 2929 2930 get_udb_errors(&udbh, &udbl); 2931 spf_flt->flt_sdbh = (ushort_t)(udbh & 0x3FF); 2932 spf_flt->flt_sdbl = (ushort_t)(udbl & 0x3FF); 2933 } 2934 2935 set_asyncflt(afsr); /* clear afsr */ 2936 clr_datapath(); /* clear udbs */ 2937 return (afsr); 2938 } 2939 2940 /* 2941 * Scan the ecache to look for bad lines. If found, the afsr, afar, e$ data 2942 * tag of the first bad line will be returned. We also return the old-afsr 2943 * (before clearing the sticky bits). The linecnt data will be updated to 2944 * indicate the number of bad lines detected. 2945 */ 2946 static void 2947 scan_ecache(uint64_t *t_afar, ec_data_t *ecache_data, 2948 uint64_t *ecache_tag, int *linecnt, uint64_t *t_afsr) 2949 { 2950 ec_data_t t_ecdata[8]; 2951 uint64_t t_etag, oafsr; 2952 uint64_t pa = AFLT_INV_ADDR; 2953 uint32_t i, j, ecache_sz; 2954 uint64_t acc_afsr = 0; 2955 uint64_t *cpu_afsr = NULL; 2956 2957 if (CPU_PRIVATE(CPU) != NULL) 2958 cpu_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 2959 2960 *linecnt = 0; 2961 ecache_sz = cpunodes[CPU->cpu_id].ecache_size; 2962 2963 for (i = 0; i < ecache_sz; i += 64) { 2964 get_ecache_dtag(i, (uint64_t *)&t_ecdata[0], &t_etag, &oafsr, 2965 cpu_afsr); 2966 acc_afsr |= oafsr; 2967 2968 /* 2969 * Scan through the whole 64 bytes line in 8 8-byte chunks 2970 * looking for the first occurrence of an EDP error. The AFSR 2971 * info is captured for each 8-byte chunk. Note that for 2972 * Spitfire/Blackbird, the AFSR.PSYND is captured by h/w in 2973 * 16-byte chunk granularity (i.e. the AFSR will be the same 2974 * for the high and low 8-byte words within the 16-byte chunk). 2975 * For Sabre/Hummingbird, the AFSR.PSYND is captured in 8-byte 2976 * granularity and only PSYND bits [7:0] are used. 2977 */ 2978 for (j = 0; j < 8; j++) { 2979 ec_data_t *ecdptr = &t_ecdata[j]; 2980 2981 if (ecdptr->ec_afsr & P_AFSR_EDP) { 2982 uint64_t errpa; 2983 ushort_t psynd; 2984 uint32_t ec_set_size = ecache_sz / 2985 ecache_associativity; 2986 2987 /* 2988 * For Spitfire/Blackbird, we need to look at 2989 * the PSYND to make sure that this 8-byte chunk 2990 * is the right one. PSYND bits [15:8] belong 2991 * to the upper 8-byte (even) chunk. Bits 2992 * [7:0] belong to the lower 8-byte chunk (odd). 2993 */ 2994 psynd = ecdptr->ec_afsr & P_AFSR_P_SYND; 2995 if (!isus2i && !isus2e) { 2996 if (j & 0x1) 2997 psynd = psynd & 0xFF; 2998 else 2999 psynd = psynd >> 8; 3000 3001 if (!psynd) 3002 continue; /* wrong chunk */ 3003 } 3004 3005 /* Construct the PA */ 3006 errpa = ((t_etag & cpu_ec_tag_mask) << 3007 cpu_ec_tag_shift) | ((i | (j << 3)) % 3008 ec_set_size); 3009 3010 /* clean up the cache line */ 3011 flushecacheline(P2ALIGN(errpa, 64), 3012 cpunodes[CPU->cpu_id].ecache_size); 3013 3014 oafsr = clear_errors(NULL, cpu_afsr); 3015 acc_afsr |= oafsr; 3016 3017 (*linecnt)++; 3018 3019 /* 3020 * Capture the PA for the first bad line found. 3021 * Return the ecache dump and tag info. 3022 */ 3023 if (pa == AFLT_INV_ADDR) { 3024 int k; 3025 3026 pa = errpa; 3027 for (k = 0; k < 8; k++) 3028 ecache_data[k] = t_ecdata[k]; 3029 *ecache_tag = t_etag; 3030 } 3031 break; 3032 } 3033 } 3034 } 3035 *t_afar = pa; 3036 *t_afsr = acc_afsr; 3037 } 3038 3039 static void 3040 cpu_log_ecmem_info(spitf_async_flt *spf_flt) 3041 { 3042 struct async_flt *aflt = (struct async_flt *)spf_flt; 3043 uint64_t ecache_tag = spf_flt->flt_ec_tag; 3044 char linestr[30]; 3045 char *state_str; 3046 int i; 3047 3048 /* 3049 * Check the ecache tag to make sure it 3050 * is valid. If invalid, a memory dump was 3051 * captured instead of a ecache dump. 3052 */ 3053 if (spf_flt->flt_ec_tag != AFLT_INV_ADDR) { 3054 uchar_t eparity = (uchar_t) 3055 ((ecache_tag & cpu_ec_par_mask) >> cpu_ec_par_shift); 3056 3057 uchar_t estate = (uchar_t) 3058 ((ecache_tag & cpu_ec_state_mask) >> cpu_ec_state_shift); 3059 3060 if (estate == cpu_ec_state_shr) 3061 state_str = "Shared"; 3062 else if (estate == cpu_ec_state_exl) 3063 state_str = "Exclusive"; 3064 else if (estate == cpu_ec_state_own) 3065 state_str = "Owner"; 3066 else if (estate == cpu_ec_state_mod) 3067 state_str = "Modified"; 3068 else 3069 state_str = "Invalid"; 3070 3071 if (spf_flt->flt_ec_lcnt > 1) { 3072 (void) snprintf(linestr, sizeof (linestr), 3073 "Badlines found=%d", spf_flt->flt_ec_lcnt); 3074 } else { 3075 linestr[0] = '\0'; 3076 } 3077 3078 cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, NULL, 3079 " PA=0x%08x.%08x\n E$tag 0x%08x.%08x E$State: %s " 3080 "E$parity 0x%02x %s", (uint32_t)(aflt->flt_addr >> 32), 3081 (uint32_t)aflt->flt_addr, (uint32_t)(ecache_tag >> 32), 3082 (uint32_t)ecache_tag, state_str, 3083 (uint32_t)eparity, linestr); 3084 } else { 3085 cpu_aflt_log(CE_CONT, 2, spf_flt, CPU_ERRID_FIRST, NULL, 3086 " E$tag != PA from AFAR; E$line was victimized" 3087 "\n dumping memory from PA 0x%08x.%08x instead", 3088 (uint32_t)(P2ALIGN(aflt->flt_addr, 64) >> 32), 3089 (uint32_t)P2ALIGN(aflt->flt_addr, 64)); 3090 } 3091 3092 /* 3093 * Dump out all 8 8-byte ecache data captured 3094 * For each 8-byte data captured, we check the 3095 * captured afsr's parity syndrome to find out 3096 * which 8-byte chunk is bad. For memory dump, the 3097 * AFSR values were initialized to 0. 3098 */ 3099 for (i = 0; i < 8; i++) { 3100 ec_data_t *ecdptr; 3101 uint_t offset; 3102 ushort_t psynd; 3103 ushort_t bad; 3104 uint64_t edp; 3105 3106 offset = i << 3; /* multiply by 8 */ 3107 ecdptr = &spf_flt->flt_ec_data[i]; 3108 psynd = ecdptr->ec_afsr & P_AFSR_P_SYND; 3109 edp = ecdptr->ec_afsr & P_AFSR_EDP; 3110 3111 /* 3112 * For Sabre/Hummingbird, parity synd is captured only 3113 * in [7:0] of AFSR.PSYND for each 8-byte chunk. 3114 * For spitfire/blackbird, AFSR.PSYND is captured 3115 * in 16-byte granularity. [15:8] represent 3116 * the upper 8 byte and [7:0] the lower 8 byte. 3117 */ 3118 if (isus2i || isus2e || (i & 0x1)) 3119 bad = (psynd & 0xFF); /* check bits [7:0] */ 3120 else 3121 bad = (psynd & 0xFF00); /* check bits [15:8] */ 3122 3123 if (bad && edp) { 3124 cpu_aflt_log(CE_CONT, 2, spf_flt, NO_LFLAGS, NULL, 3125 " E$Data (0x%02x): 0x%08x.%08x " 3126 "*Bad* PSYND=0x%04x", offset, 3127 (uint32_t)(ecdptr->ec_d8 >> 32), 3128 (uint32_t)ecdptr->ec_d8, psynd); 3129 } else { 3130 cpu_aflt_log(CE_CONT, 2, spf_flt, NO_LFLAGS, NULL, 3131 " E$Data (0x%02x): 0x%08x.%08x", offset, 3132 (uint32_t)(ecdptr->ec_d8 >> 32), 3133 (uint32_t)ecdptr->ec_d8); 3134 } 3135 } 3136 } 3137 3138 /* 3139 * Common logging function for all cpu async errors. This function allows the 3140 * caller to generate a single cmn_err() call that logs the appropriate items 3141 * from the fault structure, and implements our rules for AFT logging levels. 3142 * 3143 * ce_code: cmn_err() code (e.g. CE_PANIC, CE_WARN, CE_CONT) 3144 * tagnum: 0, 1, 2, .. generate the [AFT#] tag 3145 * spflt: pointer to spitfire async fault structure 3146 * logflags: bitflags indicating what to output 3147 * endstr: a end string to appear at the end of this log 3148 * fmt: a format string to appear at the beginning of the log 3149 * 3150 * The logflags allows the construction of predetermined output from the spflt 3151 * structure. The individual data items always appear in a consistent order. 3152 * Note that either or both of the spflt structure pointer and logflags may be 3153 * NULL or zero respectively, indicating that the predetermined output 3154 * substrings are not requested in this log. The output looks like this: 3155 * 3156 * [AFT#] <CPU_ERRID_FIRST><fmt string><CPU_FLTCPU> 3157 * <CPU_SPACE><CPU_ERRID> 3158 * newline+4spaces<CPU_AFSR><CPU_AFAR> 3159 * newline+4spaces<CPU_AF_PSYND><CPU_AF_ETS><CPU_FAULTPC> 3160 * newline+4spaces<CPU_UDBH><CPU_UDBL> 3161 * newline+4spaces<CPU_SYND> 3162 * newline+4spaces<endstr> 3163 * 3164 * Note that <endstr> may not start on a newline if we are logging <CPU_PSYND>; 3165 * it is assumed that <endstr> will be the unum string in this case. The size 3166 * of our intermediate formatting buf[] is based on the worst case of all flags 3167 * being enabled. We pass the caller's varargs directly to vcmn_err() for 3168 * formatting so we don't need additional stack space to format them here. 3169 */ 3170 /*PRINTFLIKE6*/ 3171 static void 3172 cpu_aflt_log(int ce_code, int tagnum, spitf_async_flt *spflt, uint_t logflags, 3173 const char *endstr, const char *fmt, ...) 3174 { 3175 struct async_flt *aflt = (struct async_flt *)spflt; 3176 char buf[400], *p, *q; /* see comments about buf[] size above */ 3177 va_list ap; 3178 int console_log_flag; 3179 3180 if ((aflt == NULL) || ((aflt->flt_class == CPU_FAULT) && 3181 (aflt->flt_stat & P_AFSR_LEVEL1)) || 3182 (aflt->flt_panic)) { 3183 console_log_flag = (tagnum < 2) || aft_verbose; 3184 } else { 3185 int verbose = ((aflt->flt_class == BUS_FAULT) || 3186 (aflt->flt_stat & P_AFSR_CE)) ? 3187 ce_verbose_memory : ce_verbose_other; 3188 3189 if (!verbose) 3190 return; 3191 3192 console_log_flag = (verbose > 1); 3193 } 3194 3195 if (console_log_flag) 3196 (void) sprintf(buf, "[AFT%d]", tagnum); 3197 else 3198 (void) sprintf(buf, "![AFT%d]", tagnum); 3199 3200 p = buf + strlen(buf); /* current buffer position */ 3201 q = buf + sizeof (buf); /* pointer past end of buffer */ 3202 3203 if (spflt != NULL && (logflags & CPU_ERRID_FIRST)) { 3204 (void) snprintf(p, (size_t)(q - p), " errID 0x%08x.%08x", 3205 (uint32_t)(aflt->flt_id >> 32), (uint32_t)aflt->flt_id); 3206 p += strlen(p); 3207 } 3208 3209 /* 3210 * Copy the caller's format string verbatim into buf[]. It will be 3211 * formatted by the call to vcmn_err() at the end of this function. 3212 */ 3213 if (fmt != NULL && p < q) { 3214 (void) strncpy(p, fmt, (size_t)(q - p - 1)); 3215 buf[sizeof (buf) - 1] = '\0'; 3216 p += strlen(p); 3217 } 3218 3219 if (spflt != NULL) { 3220 if (logflags & CPU_FLTCPU) { 3221 (void) snprintf(p, (size_t)(q - p), " CPU%d", 3222 aflt->flt_inst); 3223 p += strlen(p); 3224 } 3225 3226 if (logflags & CPU_SPACE) { 3227 if (aflt->flt_status & ECC_D_TRAP) 3228 (void) snprintf(p, (size_t)(q - p), 3229 " Data access"); 3230 else if (aflt->flt_status & ECC_I_TRAP) 3231 (void) snprintf(p, (size_t)(q - p), 3232 " Instruction access"); 3233 p += strlen(p); 3234 } 3235 3236 if (logflags & CPU_TL) { 3237 (void) snprintf(p, (size_t)(q - p), " at TL%s", 3238 aflt->flt_tl ? ">0" : "=0"); 3239 p += strlen(p); 3240 } 3241 3242 if (logflags & CPU_ERRID) { 3243 (void) snprintf(p, (size_t)(q - p), 3244 ", errID 0x%08x.%08x", 3245 (uint32_t)(aflt->flt_id >> 32), 3246 (uint32_t)aflt->flt_id); 3247 p += strlen(p); 3248 } 3249 3250 if (logflags & CPU_AFSR) { 3251 (void) snprintf(p, (size_t)(q - p), 3252 "\n AFSR 0x%08b.%08b", 3253 (uint32_t)(aflt->flt_stat >> 32), AFSR_FMTSTR0, 3254 (uint32_t)aflt->flt_stat, AFSR_FMTSTR1); 3255 p += strlen(p); 3256 } 3257 3258 if (logflags & CPU_AFAR) { 3259 (void) snprintf(p, (size_t)(q - p), " AFAR 0x%08x.%08x", 3260 (uint32_t)(aflt->flt_addr >> 32), 3261 (uint32_t)aflt->flt_addr); 3262 p += strlen(p); 3263 } 3264 3265 if (logflags & CPU_AF_PSYND) { 3266 ushort_t psynd = (ushort_t) 3267 (aflt->flt_stat & P_AFSR_P_SYND); 3268 3269 (void) snprintf(p, (size_t)(q - p), 3270 "\n AFSR.PSYND 0x%04x(Score %02d)", 3271 psynd, ecc_psynd_score(psynd)); 3272 p += strlen(p); 3273 } 3274 3275 if (logflags & CPU_AF_ETS) { 3276 (void) snprintf(p, (size_t)(q - p), " AFSR.ETS 0x%02x", 3277 (uchar_t)((aflt->flt_stat & P_AFSR_ETS) >> 16)); 3278 p += strlen(p); 3279 } 3280 3281 if (logflags & CPU_FAULTPC) { 3282 (void) snprintf(p, (size_t)(q - p), " Fault_PC 0x%p", 3283 (void *)aflt->flt_pc); 3284 p += strlen(p); 3285 } 3286 3287 if (logflags & CPU_UDBH) { 3288 (void) snprintf(p, (size_t)(q - p), 3289 "\n UDBH 0x%04b UDBH.ESYND 0x%02x", 3290 spflt->flt_sdbh, UDB_FMTSTR, 3291 spflt->flt_sdbh & 0xFF); 3292 p += strlen(p); 3293 } 3294 3295 if (logflags & CPU_UDBL) { 3296 (void) snprintf(p, (size_t)(q - p), 3297 " UDBL 0x%04b UDBL.ESYND 0x%02x", 3298 spflt->flt_sdbl, UDB_FMTSTR, 3299 spflt->flt_sdbl & 0xFF); 3300 p += strlen(p); 3301 } 3302 3303 if (logflags & CPU_SYND) { 3304 ushort_t synd = SYND(aflt->flt_synd); 3305 3306 (void) snprintf(p, (size_t)(q - p), 3307 "\n %s Syndrome 0x%x Memory Module ", 3308 UDBL(aflt->flt_synd) ? "UDBL" : "UDBH", synd); 3309 p += strlen(p); 3310 } 3311 } 3312 3313 if (endstr != NULL) { 3314 if (!(logflags & CPU_SYND)) 3315 (void) snprintf(p, (size_t)(q - p), "\n %s", endstr); 3316 else 3317 (void) snprintf(p, (size_t)(q - p), "%s", endstr); 3318 p += strlen(p); 3319 } 3320 3321 if (ce_code == CE_CONT && (p < q - 1)) 3322 (void) strcpy(p, "\n"); /* add final \n if needed */ 3323 3324 va_start(ap, fmt); 3325 vcmn_err(ce_code, buf, ap); 3326 va_end(ap); 3327 } 3328 3329 /* 3330 * Ecache Scrubbing 3331 * 3332 * The basic idea is to prevent lines from sitting in the ecache long enough 3333 * to build up soft errors which can lead to ecache parity errors. 3334 * 3335 * The following rules are observed when flushing the ecache: 3336 * 3337 * 1. When the system is busy, flush bad clean lines 3338 * 2. When the system is idle, flush all clean lines 3339 * 3. When the system is idle, flush good dirty lines 3340 * 4. Never flush bad dirty lines. 3341 * 3342 * modify parity busy idle 3343 * ---------------------------- 3344 * clean good X 3345 * clean bad X X 3346 * dirty good X 3347 * dirty bad 3348 * 3349 * Bad or good refers to whether a line has an E$ parity error or not. 3350 * Clean or dirty refers to the state of the modified bit. We currently 3351 * default the scan rate to 100 (scan 10% of the cache per second). 3352 * 3353 * The following are E$ states and actions. 3354 * 3355 * We encode our state as a 3-bit number, consisting of: 3356 * ECACHE_STATE_MODIFIED (0=clean, 1=dirty) 3357 * ECACHE_STATE_PARITY (0=good, 1=bad) 3358 * ECACHE_STATE_BUSY (0=idle, 1=busy) 3359 * 3360 * We associate a flushing and a logging action with each state. 3361 * 3362 * E$ actions are different for Spitfire and Sabre/Hummingbird modules. 3363 * MIRROR_FLUSH indicates that an E$ line will be flushed for the mirrored 3364 * E$ only, in addition to value being set by ec_flush. 3365 */ 3366 3367 #define ALWAYS_FLUSH 0x1 /* flush E$ line on all E$ types */ 3368 #define NEVER_FLUSH 0x0 /* never the flush the E$ line */ 3369 #define MIRROR_FLUSH 0xF /* flush E$ line on mirrored E$ only */ 3370 3371 struct { 3372 char ec_flush; /* whether to flush or not */ 3373 char ec_log; /* ecache logging */ 3374 char ec_log_type; /* log type info */ 3375 } ec_action[] = { /* states of the E$ line in M P B */ 3376 { ALWAYS_FLUSH, 0, 0 }, /* 0 0 0 clean_good_idle */ 3377 { MIRROR_FLUSH, 0, 0 }, /* 0 0 1 clean_good_busy */ 3378 { ALWAYS_FLUSH, 1, CPU_BADLINE_CI_ERR }, /* 0 1 0 clean_bad_idle */ 3379 { ALWAYS_FLUSH, 1, CPU_BADLINE_CB_ERR }, /* 0 1 1 clean_bad_busy */ 3380 { ALWAYS_FLUSH, 0, 0 }, /* 1 0 0 dirty_good_idle */ 3381 { MIRROR_FLUSH, 0, 0 }, /* 1 0 1 dirty_good_busy */ 3382 { NEVER_FLUSH, 1, CPU_BADLINE_DI_ERR }, /* 1 1 0 dirty_bad_idle */ 3383 { NEVER_FLUSH, 1, CPU_BADLINE_DB_ERR } /* 1 1 1 dirty_bad_busy */ 3384 }; 3385 3386 /* 3387 * Offsets into the ec_action[] that determines clean_good_busy and 3388 * dirty_good_busy lines. 3389 */ 3390 #define ECACHE_CGB_LINE 1 /* E$ clean_good_busy line */ 3391 #define ECACHE_DGB_LINE 5 /* E$ dirty_good_busy line */ 3392 3393 /* 3394 * We are flushing lines which are Clean_Good_Busy and also the lines 3395 * Dirty_Good_Busy. And we only follow it for non-mirrored E$. 3396 */ 3397 #define CGB(x, m) (((x) == ECACHE_CGB_LINE) && (m != ECACHE_CPU_MIRROR)) 3398 #define DGB(x, m) (((x) == ECACHE_DGB_LINE) && (m != ECACHE_CPU_MIRROR)) 3399 3400 #define ECACHE_STATE_MODIFIED 0x4 3401 #define ECACHE_STATE_PARITY 0x2 3402 #define ECACHE_STATE_BUSY 0x1 3403 3404 /* 3405 * If ecache is mirrored ecache_calls_a_sec and ecache_scan_rate are reduced. 3406 */ 3407 int ecache_calls_a_sec_mirrored = 1; 3408 int ecache_lines_per_call_mirrored = 1; 3409 3410 int ecache_scrub_enable = 1; /* ecache scrubbing is on by default */ 3411 int ecache_scrub_verbose = 1; /* prints clean and dirty lines */ 3412 int ecache_scrub_panic = 0; /* panics on a clean and dirty line */ 3413 int ecache_calls_a_sec = 100; /* scrubber calls per sec */ 3414 int ecache_scan_rate = 100; /* scan rate (in tenths of a percent) */ 3415 int ecache_idle_factor = 1; /* increase the scan rate when idle */ 3416 int ecache_flush_clean_good_busy = 50; /* flush rate (in percent) */ 3417 int ecache_flush_dirty_good_busy = 100; /* flush rate (in percent) */ 3418 3419 volatile int ec_timeout_calls = 1; /* timeout calls */ 3420 3421 /* 3422 * Interrupt number and pil for ecache scrubber cross-trap calls. 3423 */ 3424 static uint_t ecache_scrub_inum; 3425 uint_t ecache_scrub_pil = PIL_9; 3426 3427 /* 3428 * Kstats for the E$ scrubber. 3429 */ 3430 typedef struct ecache_kstat { 3431 kstat_named_t clean_good_idle; /* # of lines scrubbed */ 3432 kstat_named_t clean_good_busy; /* # of lines skipped */ 3433 kstat_named_t clean_bad_idle; /* # of lines scrubbed */ 3434 kstat_named_t clean_bad_busy; /* # of lines scrubbed */ 3435 kstat_named_t dirty_good_idle; /* # of lines scrubbed */ 3436 kstat_named_t dirty_good_busy; /* # of lines skipped */ 3437 kstat_named_t dirty_bad_idle; /* # of lines skipped */ 3438 kstat_named_t dirty_bad_busy; /* # of lines skipped */ 3439 kstat_named_t invalid_lines; /* # of invalid lines */ 3440 kstat_named_t clean_good_busy_flush; /* # of lines scrubbed */ 3441 kstat_named_t dirty_good_busy_flush; /* # of lines scrubbed */ 3442 kstat_named_t tags_cleared; /* # of E$ tags cleared */ 3443 } ecache_kstat_t; 3444 3445 static ecache_kstat_t ec_kstat_template = { 3446 { "clean_good_idle", KSTAT_DATA_ULONG }, 3447 { "clean_good_busy", KSTAT_DATA_ULONG }, 3448 { "clean_bad_idle", KSTAT_DATA_ULONG }, 3449 { "clean_bad_busy", KSTAT_DATA_ULONG }, 3450 { "dirty_good_idle", KSTAT_DATA_ULONG }, 3451 { "dirty_good_busy", KSTAT_DATA_ULONG }, 3452 { "dirty_bad_idle", KSTAT_DATA_ULONG }, 3453 { "dirty_bad_busy", KSTAT_DATA_ULONG }, 3454 { "invalid_lines", KSTAT_DATA_ULONG }, 3455 { "clean_good_busy_flush", KSTAT_DATA_ULONG }, 3456 { "dirty_good_busy_flush", KSTAT_DATA_ULONG }, 3457 { "ecache_tags_cleared", KSTAT_DATA_ULONG } 3458 }; 3459 3460 struct kmem_cache *sf_private_cache; 3461 3462 /* 3463 * Called periodically on each CPU to scan the ecache once a sec. 3464 * adjusting the ecache line index appropriately 3465 */ 3466 void 3467 scrub_ecache_line() 3468 { 3469 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(CPU, sfpr_scrub_misc); 3470 int cpuid = CPU->cpu_id; 3471 uint32_t index = ssmp->ecache_flush_index; 3472 uint64_t ec_size = cpunodes[cpuid].ecache_size; 3473 size_t ec_linesize = cpunodes[cpuid].ecache_linesize; 3474 int nlines = ssmp->ecache_nlines; 3475 uint32_t ec_set_size = ec_size / ecache_associativity; 3476 int ec_mirror = ssmp->ecache_mirror; 3477 ecache_kstat_t *ec_ksp = (ecache_kstat_t *)ssmp->ecache_ksp->ks_data; 3478 3479 int line, scan_lines, flush_clean_busy = 0, flush_dirty_busy = 0; 3480 int mpb; /* encode Modified, Parity, Busy for action */ 3481 uchar_t state; 3482 uint64_t ec_tag, paddr, oafsr, tafsr, nafsr; 3483 uint64_t *acc_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 3484 ec_data_t ec_data[8]; 3485 kstat_named_t *ec_knp; 3486 3487 switch (ec_mirror) { 3488 default: 3489 case ECACHE_CPU_NON_MIRROR: 3490 /* 3491 * The E$ scan rate is expressed in units of tenths of 3492 * a percent. ecache_scan_rate = 1000 (100%) means the 3493 * whole cache is scanned every second. 3494 */ 3495 scan_lines = (nlines * ecache_scan_rate) / 3496 (1000 * ecache_calls_a_sec); 3497 if (!(ssmp->ecache_busy)) { 3498 if (ecache_idle_factor > 0) { 3499 scan_lines *= ecache_idle_factor; 3500 } 3501 } else { 3502 flush_clean_busy = (scan_lines * 3503 ecache_flush_clean_good_busy) / 100; 3504 flush_dirty_busy = (scan_lines * 3505 ecache_flush_dirty_good_busy) / 100; 3506 } 3507 3508 ec_timeout_calls = (ecache_calls_a_sec ? 3509 ecache_calls_a_sec : 1); 3510 break; 3511 3512 case ECACHE_CPU_MIRROR: 3513 scan_lines = ecache_lines_per_call_mirrored; 3514 ec_timeout_calls = (ecache_calls_a_sec_mirrored ? 3515 ecache_calls_a_sec_mirrored : 1); 3516 break; 3517 } 3518 3519 /* 3520 * The ecache scrubber algorithm operates by reading and 3521 * decoding the E$ tag to determine whether the corresponding E$ line 3522 * can be scrubbed. There is a implicit assumption in the scrubber 3523 * logic that the E$ tag is valid. Unfortunately, this assertion is 3524 * flawed since the E$ tag may also be corrupted and have parity errors 3525 * The scrubber logic is enhanced to check the validity of the E$ tag 3526 * before scrubbing. When a parity error is detected in the E$ tag, 3527 * it is possible to recover and scrub the tag under certain conditions 3528 * so that a ETP error condition can be avoided. 3529 */ 3530 3531 for (mpb = line = 0; line < scan_lines; line++, mpb = 0) { 3532 /* 3533 * We get the old-AFSR before clearing the AFSR sticky bits 3534 * in {get_ecache_tag, check_ecache_line, get_ecache_dtag} 3535 * If CP bit is set in the old-AFSR, we log an Orphan CP event. 3536 */ 3537 ec_tag = get_ecache_tag(index, &nafsr, acc_afsr); 3538 state = (uchar_t)((ec_tag & cpu_ec_state_mask) >> 3539 cpu_ec_state_shift); 3540 3541 /* 3542 * ETP is set try to scrub the ecache tag. 3543 */ 3544 if (nafsr & P_AFSR_ETP) { 3545 ecache_scrub_tag_err(nafsr, state, index); 3546 } else if (state & cpu_ec_state_valid) { 3547 /* 3548 * ETP is not set, E$ tag is valid. 3549 * Proceed with the E$ scrubbing. 3550 */ 3551 if (state & cpu_ec_state_dirty) 3552 mpb |= ECACHE_STATE_MODIFIED; 3553 3554 tafsr = check_ecache_line(index, acc_afsr); 3555 3556 if (tafsr & P_AFSR_EDP) { 3557 mpb |= ECACHE_STATE_PARITY; 3558 3559 if (ecache_scrub_verbose || 3560 ecache_scrub_panic) { 3561 get_ecache_dtag(P2ALIGN(index, 64), 3562 (uint64_t *)&ec_data[0], 3563 &ec_tag, &oafsr, acc_afsr); 3564 } 3565 } 3566 3567 if (ssmp->ecache_busy) 3568 mpb |= ECACHE_STATE_BUSY; 3569 3570 ec_knp = (kstat_named_t *)ec_ksp + mpb; 3571 ec_knp->value.ul++; 3572 3573 paddr = ((ec_tag & cpu_ec_tag_mask) << 3574 cpu_ec_tag_shift) | (index % ec_set_size); 3575 3576 /* 3577 * We flush the E$ lines depending on the ec_flush, 3578 * we additionally flush clean_good_busy and 3579 * dirty_good_busy lines for mirrored E$. 3580 */ 3581 if (ec_action[mpb].ec_flush == ALWAYS_FLUSH) { 3582 flushecacheline(paddr, ec_size); 3583 } else if ((ec_mirror == ECACHE_CPU_MIRROR) && 3584 (ec_action[mpb].ec_flush == MIRROR_FLUSH)) { 3585 flushecacheline(paddr, ec_size); 3586 } else if (ec_action[mpb].ec_flush == NEVER_FLUSH) { 3587 softcall(ecache_page_retire, (void *)paddr); 3588 } 3589 3590 /* 3591 * Conditionally flush both the clean_good and 3592 * dirty_good lines when busy. 3593 */ 3594 if (CGB(mpb, ec_mirror) && (flush_clean_busy > 0)) { 3595 flush_clean_busy--; 3596 flushecacheline(paddr, ec_size); 3597 ec_ksp->clean_good_busy_flush.value.ul++; 3598 } else if (DGB(mpb, ec_mirror) && 3599 (flush_dirty_busy > 0)) { 3600 flush_dirty_busy--; 3601 flushecacheline(paddr, ec_size); 3602 ec_ksp->dirty_good_busy_flush.value.ul++; 3603 } 3604 3605 if (ec_action[mpb].ec_log && (ecache_scrub_verbose || 3606 ecache_scrub_panic)) { 3607 ecache_scrub_log(ec_data, ec_tag, paddr, mpb, 3608 tafsr); 3609 } 3610 3611 } else { 3612 ec_ksp->invalid_lines.value.ul++; 3613 } 3614 3615 if ((index += ec_linesize) >= ec_size) 3616 index = 0; 3617 3618 } 3619 3620 /* 3621 * set the ecache scrub index for the next time around 3622 */ 3623 ssmp->ecache_flush_index = index; 3624 3625 if (*acc_afsr & P_AFSR_CP) { 3626 uint64_t ret_afsr; 3627 3628 ret_afsr = ecache_scrub_misc_err(CPU_ORPHAN_CP_ERR, *acc_afsr); 3629 if ((ret_afsr & P_AFSR_CP) == 0) 3630 *acc_afsr = 0; 3631 } 3632 } 3633 3634 /* 3635 * Handler for ecache_scrub_inum softint. Call scrub_ecache_line until 3636 * we decrement the outstanding request count to zero. 3637 */ 3638 3639 /*ARGSUSED*/ 3640 uint_t 3641 scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2) 3642 { 3643 int i; 3644 int outstanding; 3645 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(CPU, sfpr_scrub_misc); 3646 uint32_t *countp = &ssmp->ec_scrub_outstanding; 3647 3648 do { 3649 outstanding = *countp; 3650 ASSERT(outstanding > 0); 3651 for (i = 0; i < outstanding; i++) 3652 scrub_ecache_line(); 3653 } while (atomic_add_32_nv(countp, -outstanding)); 3654 3655 return (DDI_INTR_CLAIMED); 3656 } 3657 3658 /* 3659 * force each cpu to perform an ecache scrub, called from a timeout 3660 */ 3661 extern xcfunc_t ecache_scrubreq_tl1; 3662 3663 void 3664 do_scrub_ecache_line(void) 3665 { 3666 long delta; 3667 3668 if (ecache_calls_a_sec > hz) 3669 ecache_calls_a_sec = hz; 3670 else if (ecache_calls_a_sec <= 0) 3671 ecache_calls_a_sec = 1; 3672 3673 if (ecache_calls_a_sec_mirrored > hz) 3674 ecache_calls_a_sec_mirrored = hz; 3675 else if (ecache_calls_a_sec_mirrored <= 0) 3676 ecache_calls_a_sec_mirrored = 1; 3677 3678 if (ecache_scrub_enable) { 3679 xt_all(ecache_scrubreq_tl1, ecache_scrub_inum, 0); 3680 delta = hz / ec_timeout_calls; 3681 } else { 3682 delta = hz; 3683 } 3684 3685 (void) realtime_timeout((void(*)(void *))do_scrub_ecache_line, 0, 3686 delta); 3687 } 3688 3689 /* 3690 * initialization for ecache scrubbing 3691 * This routine is called AFTER all cpus have had cpu_init_private called 3692 * to initialize their private data areas. 3693 */ 3694 void 3695 cpu_init_cache_scrub(void) 3696 { 3697 if (ecache_calls_a_sec > hz) { 3698 cmn_err(CE_NOTE, "ecache_calls_a_sec set too high (%d); " 3699 "resetting to hz (%d)", ecache_calls_a_sec, hz); 3700 ecache_calls_a_sec = hz; 3701 } 3702 3703 /* 3704 * Register softint for ecache scrubbing. 3705 */ 3706 ecache_scrub_inum = add_softintr(ecache_scrub_pil, 3707 scrub_ecache_line_intr, NULL); 3708 3709 /* 3710 * kick off the scrubbing using realtime timeout 3711 */ 3712 (void) realtime_timeout((void(*)(void *))do_scrub_ecache_line, 0, 3713 hz / ecache_calls_a_sec); 3714 } 3715 3716 /* 3717 * Unset the busy flag for this cpu. 3718 */ 3719 void 3720 cpu_idle_ecache_scrub(struct cpu *cp) 3721 { 3722 if (CPU_PRIVATE(cp) != NULL) { 3723 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, 3724 sfpr_scrub_misc); 3725 ssmp->ecache_busy = ECACHE_CPU_IDLE; 3726 } 3727 } 3728 3729 /* 3730 * Set the busy flag for this cpu. 3731 */ 3732 void 3733 cpu_busy_ecache_scrub(struct cpu *cp) 3734 { 3735 if (CPU_PRIVATE(cp) != NULL) { 3736 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, 3737 sfpr_scrub_misc); 3738 ssmp->ecache_busy = ECACHE_CPU_BUSY; 3739 } 3740 } 3741 3742 /* 3743 * initialize the ecache scrubber data structures 3744 * The global entry point cpu_init_private replaces this entry point. 3745 * 3746 */ 3747 static void 3748 cpu_init_ecache_scrub_dr(struct cpu *cp) 3749 { 3750 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, sfpr_scrub_misc); 3751 int cpuid = cp->cpu_id; 3752 3753 /* 3754 * intialize bookkeeping for cache scrubbing 3755 */ 3756 bzero(ssmp, sizeof (spitfire_scrub_misc_t)); 3757 3758 ssmp->ecache_flush_index = 0; 3759 3760 ssmp->ecache_nlines = 3761 cpunodes[cpuid].ecache_size / cpunodes[cpuid].ecache_linesize; 3762 3763 /* 3764 * Determine whether we are running on mirrored SRAM 3765 */ 3766 3767 if (cpunodes[cpuid].msram == ECACHE_CPU_MIRROR) 3768 ssmp->ecache_mirror = ECACHE_CPU_MIRROR; 3769 else 3770 ssmp->ecache_mirror = ECACHE_CPU_NON_MIRROR; 3771 3772 cpu_busy_ecache_scrub(cp); 3773 3774 /* 3775 * initialize the kstats 3776 */ 3777 ecache_kstat_init(cp); 3778 } 3779 3780 /* 3781 * uninitialize the ecache scrubber data structures 3782 * The global entry point cpu_uninit_private replaces this entry point. 3783 */ 3784 static void 3785 cpu_uninit_ecache_scrub_dr(struct cpu *cp) 3786 { 3787 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, sfpr_scrub_misc); 3788 3789 if (ssmp->ecache_ksp != NULL) { 3790 kstat_delete(ssmp->ecache_ksp); 3791 ssmp->ecache_ksp = NULL; 3792 } 3793 3794 /* 3795 * un-initialize bookkeeping for cache scrubbing 3796 */ 3797 bzero(ssmp, sizeof (spitfire_scrub_misc_t)); 3798 3799 cpu_idle_ecache_scrub(cp); 3800 } 3801 3802 struct kmem_cache *sf_private_cache; 3803 3804 /* 3805 * Cpu private initialization. This includes allocating the cpu_private 3806 * data structure, initializing it, and initializing the scrubber for this 3807 * cpu. This is called once for EVERY cpu, including CPU 0. This function 3808 * calls cpu_init_ecache_scrub_dr to init the scrubber. 3809 * We use kmem_cache_create for the spitfire private data structure because it 3810 * needs to be allocated on a S_ECACHE_MAX_LSIZE (64) byte boundary. 3811 */ 3812 void 3813 cpu_init_private(struct cpu *cp) 3814 { 3815 spitfire_private_t *sfprp; 3816 3817 ASSERT(CPU_PRIVATE(cp) == NULL); 3818 3819 /* 3820 * If the sf_private_cache has not been created, create it. 3821 */ 3822 if (sf_private_cache == NULL) { 3823 sf_private_cache = kmem_cache_create("sf_private_cache", 3824 sizeof (spitfire_private_t), S_ECACHE_MAX_LSIZE, NULL, 3825 NULL, NULL, NULL, NULL, 0); 3826 ASSERT(sf_private_cache); 3827 } 3828 3829 sfprp = CPU_PRIVATE(cp) = kmem_cache_alloc(sf_private_cache, KM_SLEEP); 3830 3831 bzero(sfprp, sizeof (spitfire_private_t)); 3832 3833 cpu_init_ecache_scrub_dr(cp); 3834 } 3835 3836 /* 3837 * Cpu private unitialization. Uninitialize the Ecache scrubber and 3838 * deallocate the scrubber data structures and cpu_private data structure. 3839 * For now, this function just calls cpu_unint_ecache_scrub_dr to uninit 3840 * the scrubber for the specified cpu. 3841 */ 3842 void 3843 cpu_uninit_private(struct cpu *cp) 3844 { 3845 ASSERT(CPU_PRIVATE(cp)); 3846 3847 cpu_uninit_ecache_scrub_dr(cp); 3848 kmem_cache_free(sf_private_cache, CPU_PRIVATE(cp)); 3849 CPU_PRIVATE(cp) = NULL; 3850 } 3851 3852 /* 3853 * initialize the ecache kstats for each cpu 3854 */ 3855 static void 3856 ecache_kstat_init(struct cpu *cp) 3857 { 3858 struct kstat *ksp; 3859 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(cp, sfpr_scrub_misc); 3860 3861 ASSERT(ssmp != NULL); 3862 3863 if ((ksp = kstat_create("unix", cp->cpu_id, "ecache_kstat", "misc", 3864 KSTAT_TYPE_NAMED, 3865 sizeof (ecache_kstat_t) / sizeof (kstat_named_t), 3866 KSTAT_FLAG_WRITABLE)) == NULL) { 3867 ssmp->ecache_ksp = NULL; 3868 cmn_err(CE_NOTE, "!ecache_kstat_init(%d) failed\n", cp->cpu_id); 3869 return; 3870 } 3871 3872 ssmp->ecache_ksp = ksp; 3873 bcopy(&ec_kstat_template, ksp->ks_data, sizeof (ecache_kstat_t)); 3874 kstat_install(ksp); 3875 } 3876 3877 /* 3878 * log the bad ecache information 3879 */ 3880 static void 3881 ecache_scrub_log(ec_data_t *ec_data, uint64_t ec_tag, uint64_t paddr, int mpb, 3882 uint64_t afsr) 3883 { 3884 spitf_async_flt spf_flt; 3885 struct async_flt *aflt; 3886 int i; 3887 char *class; 3888 3889 bzero(&spf_flt, sizeof (spitf_async_flt)); 3890 aflt = &spf_flt.cmn_asyncflt; 3891 3892 for (i = 0; i < 8; i++) { 3893 spf_flt.flt_ec_data[i] = ec_data[i]; 3894 } 3895 3896 spf_flt.flt_ec_tag = ec_tag; 3897 3898 if (mpb < (sizeof (ec_action) / sizeof (ec_action[0]))) { 3899 spf_flt.flt_type = ec_action[mpb].ec_log_type; 3900 } else spf_flt.flt_type = (ushort_t)mpb; 3901 3902 aflt->flt_inst = CPU->cpu_id; 3903 aflt->flt_class = CPU_FAULT; 3904 aflt->flt_id = gethrtime_waitfree(); 3905 aflt->flt_addr = paddr; 3906 aflt->flt_stat = afsr; 3907 aflt->flt_panic = (uchar_t)ecache_scrub_panic; 3908 3909 switch (mpb) { 3910 case CPU_ECACHE_TAG_ERR: 3911 case CPU_ECACHE_ADDR_PAR_ERR: 3912 case CPU_ECACHE_ETP_ETS_ERR: 3913 case CPU_ECACHE_STATE_ERR: 3914 class = FM_EREPORT_CPU_USII_ESCRUB_TAG; 3915 break; 3916 default: 3917 class = FM_EREPORT_CPU_USII_ESCRUB_DATA; 3918 break; 3919 } 3920 3921 cpu_errorq_dispatch(class, (void *)&spf_flt, sizeof (spf_flt), 3922 ue_queue, aflt->flt_panic); 3923 3924 if (aflt->flt_panic) 3925 cmn_err(CE_PANIC, "ecache_scrub_panic set and bad E$" 3926 "line detected"); 3927 } 3928 3929 /* 3930 * Process an ecache error that occured during the E$ scrubbing. 3931 * We do the ecache scan to find the bad line, flush the bad line 3932 * and start the memscrubber to find any UE (in memory or in another cache) 3933 */ 3934 static uint64_t 3935 ecache_scrub_misc_err(int type, uint64_t afsr) 3936 { 3937 spitf_async_flt spf_flt; 3938 struct async_flt *aflt; 3939 uint64_t oafsr; 3940 3941 bzero(&spf_flt, sizeof (spitf_async_flt)); 3942 aflt = &spf_flt.cmn_asyncflt; 3943 3944 /* 3945 * Scan each line in the cache to look for the one 3946 * with bad parity 3947 */ 3948 aflt->flt_addr = AFLT_INV_ADDR; 3949 scan_ecache(&aflt->flt_addr, &spf_flt.flt_ec_data[0], 3950 &spf_flt.flt_ec_tag, &spf_flt.flt_ec_lcnt, &oafsr); 3951 3952 if (oafsr & P_AFSR_CP) { 3953 uint64_t *cp_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 3954 *cp_afsr |= oafsr; 3955 } 3956 3957 /* 3958 * If we found a bad PA, update the state to indicate if it is 3959 * memory or I/O space. 3960 */ 3961 if (aflt->flt_addr != AFLT_INV_ADDR) { 3962 aflt->flt_in_memory = (pf_is_memory(aflt->flt_addr >> 3963 MMU_PAGESHIFT)) ? 1 : 0; 3964 } 3965 3966 spf_flt.flt_type = (ushort_t)type; 3967 3968 aflt->flt_inst = CPU->cpu_id; 3969 aflt->flt_class = CPU_FAULT; 3970 aflt->flt_id = gethrtime_waitfree(); 3971 aflt->flt_status = afsr; 3972 aflt->flt_panic = (uchar_t)ecache_scrub_panic; 3973 3974 /* 3975 * We have the bad line, flush that line and start 3976 * the memscrubber. 3977 */ 3978 if (spf_flt.flt_ec_lcnt > 0) { 3979 flushecacheline(P2ALIGN(aflt->flt_addr, 64), 3980 cpunodes[CPU->cpu_id].ecache_size); 3981 read_all_memscrub = 1; 3982 memscrub_run(); 3983 } 3984 3985 cpu_errorq_dispatch((type == CPU_ORPHAN_CP_ERR) ? 3986 FM_EREPORT_CPU_USII_CP : FM_EREPORT_CPU_USII_UNKNOWN, 3987 (void *)&spf_flt, sizeof (spf_flt), ue_queue, aflt->flt_panic); 3988 3989 return (oafsr); 3990 } 3991 3992 static void 3993 ecache_scrub_tag_err(uint64_t afsr, uchar_t state, uint32_t index) 3994 { 3995 ushort_t afsr_ets = (afsr & P_AFSR_ETS) >> P_AFSR_ETS_SHIFT; 3996 spitfire_scrub_misc_t *ssmp = CPU_PRIVATE_PTR(CPU, sfpr_scrub_misc); 3997 ecache_kstat_t *ec_ksp = (ecache_kstat_t *)ssmp->ecache_ksp->ks_data; 3998 uint64_t ec_tag, paddr, oafsr; 3999 ec_data_t ec_data[8]; 4000 int cpuid = CPU->cpu_id; 4001 uint32_t ec_set_size = cpunodes[cpuid].ecache_size / 4002 ecache_associativity; 4003 uint64_t *cpu_afsr = CPU_PRIVATE_PTR(CPU, sfpr_scrub_afsr); 4004 4005 get_ecache_dtag(P2ALIGN(index, 64), (uint64_t *)&ec_data[0], &ec_tag, 4006 &oafsr, cpu_afsr); 4007 paddr = ((ec_tag & cpu_ec_tag_mask) << cpu_ec_tag_shift) | 4008 (index % ec_set_size); 4009 4010 /* 4011 * E$ tag state has good parity 4012 */ 4013 if ((afsr_ets & cpu_ec_state_parity) == 0) { 4014 if (afsr_ets & cpu_ec_parity) { 4015 /* 4016 * E$ tag state bits indicate the line is clean, 4017 * invalidate the E$ tag and continue. 4018 */ 4019 if (!(state & cpu_ec_state_dirty)) { 4020 /* 4021 * Zero the tag and mark the state invalid 4022 * with good parity for the tag. 4023 */ 4024 if (isus2i || isus2e) 4025 write_hb_ec_tag_parity(index); 4026 else 4027 write_ec_tag_parity(index); 4028 4029 /* Sync with the dual tag */ 4030 flushecacheline(0, 4031 cpunodes[CPU->cpu_id].ecache_size); 4032 ec_ksp->tags_cleared.value.ul++; 4033 ecache_scrub_log(ec_data, ec_tag, paddr, 4034 CPU_ECACHE_TAG_ERR, afsr); 4035 return; 4036 } else { 4037 ecache_scrub_log(ec_data, ec_tag, paddr, 4038 CPU_ECACHE_ADDR_PAR_ERR, afsr); 4039 cmn_err(CE_PANIC, " E$ tag address has bad" 4040 " parity"); 4041 } 4042 } else if ((afsr_ets & cpu_ec_parity) == 0) { 4043 /* 4044 * ETS is zero but ETP is set 4045 */ 4046 ecache_scrub_log(ec_data, ec_tag, paddr, 4047 CPU_ECACHE_ETP_ETS_ERR, afsr); 4048 cmn_err(CE_PANIC, "AFSR.ETP is set and" 4049 " AFSR.ETS is zero"); 4050 } 4051 } else { 4052 /* 4053 * E$ tag state bit has a bad parity 4054 */ 4055 ecache_scrub_log(ec_data, ec_tag, paddr, 4056 CPU_ECACHE_STATE_ERR, afsr); 4057 cmn_err(CE_PANIC, "E$ tag state has bad parity"); 4058 } 4059 } 4060 4061 static void 4062 ecache_page_retire(void *arg) 4063 { 4064 uint64_t paddr = (uint64_t)arg; 4065 (void) page_retire(paddr, PR_UE); 4066 } 4067 4068 void 4069 sticksync_slave(void) 4070 {} 4071 4072 void 4073 sticksync_master(void) 4074 {} 4075 4076 /*ARGSUSED*/ 4077 void 4078 cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t bpp) 4079 {} 4080 4081 void 4082 cpu_run_bus_error_handlers(struct async_flt *aflt, int expected) 4083 { 4084 int status; 4085 ddi_fm_error_t de; 4086 4087 bzero(&de, sizeof (ddi_fm_error_t)); 4088 4089 de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, 4090 FM_ENA_FMT1); 4091 de.fme_flag = expected; 4092 de.fme_bus_specific = (void *)aflt->flt_addr; 4093 status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de); 4094 4095 if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL)) 4096 aflt->flt_panic = 1; 4097 } 4098 4099 /*ARGSUSED*/ 4100 void 4101 cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz, 4102 errorq_t *eqp, uint_t flag) 4103 { 4104 struct async_flt *aflt = (struct async_flt *)payload; 4105 4106 aflt->flt_erpt_class = error_class; 4107 errorq_dispatch(eqp, payload, payload_sz, flag); 4108 } 4109 4110 #define MAX_SIMM 8 4111 4112 struct ce_info { 4113 char name[UNUM_NAMLEN]; 4114 uint64_t intermittent_total; 4115 uint64_t persistent_total; 4116 uint64_t sticky_total; 4117 unsigned short leaky_bucket_cnt; 4118 }; 4119 4120 /* 4121 * Separately-defined structure for use in reporting the ce_info 4122 * to SunVTS without exposing the internal layout and implementation 4123 * of struct ce_info. 4124 */ 4125 static struct ecc_error_info ecc_error_info_data = { 4126 { "version", KSTAT_DATA_UINT32 }, 4127 { "maxcount", KSTAT_DATA_UINT32 }, 4128 { "count", KSTAT_DATA_UINT32 } 4129 }; 4130 static const size_t ecc_error_info_ndata = sizeof (ecc_error_info_data) / 4131 sizeof (struct kstat_named); 4132 4133 #if KSTAT_CE_UNUM_NAMLEN < UNUM_NAMLEN 4134 #error "Need to rev ecc_error_info version and update KSTAT_CE_UNUM_NAMLEN" 4135 #endif 4136 4137 struct ce_info *mem_ce_simm = NULL; 4138 size_t mem_ce_simm_size = 0; 4139 4140 /* 4141 * Default values for the number of CE's allowed per interval. 4142 * Interval is defined in minutes 4143 * SOFTERR_MIN_TIMEOUT is defined in microseconds 4144 */ 4145 #define SOFTERR_LIMIT_DEFAULT 2 4146 #define SOFTERR_INTERVAL_DEFAULT 1440 /* This is 24 hours */ 4147 #define SOFTERR_MIN_TIMEOUT (60 * MICROSEC) /* This is 1 minute */ 4148 #define TIMEOUT_NONE ((timeout_id_t)0) 4149 #define TIMEOUT_SET ((timeout_id_t)1) 4150 4151 /* 4152 * timeout identifer for leaky_bucket 4153 */ 4154 static timeout_id_t leaky_bucket_timeout_id = TIMEOUT_NONE; 4155 4156 /* 4157 * Tunables for maximum number of allowed CE's in a given time 4158 */ 4159 int ecc_softerr_limit = SOFTERR_LIMIT_DEFAULT; 4160 int ecc_softerr_interval = SOFTERR_INTERVAL_DEFAULT; 4161 4162 void 4163 cpu_mp_init(void) 4164 { 4165 size_t size = cpu_aflt_size(); 4166 size_t i; 4167 kstat_t *ksp; 4168 4169 /* 4170 * Initialize the CE error handling buffers. 4171 */ 4172 mem_ce_simm_size = MAX_SIMM * max_ncpus; 4173 size = sizeof (struct ce_info) * mem_ce_simm_size; 4174 mem_ce_simm = kmem_zalloc(size, KM_SLEEP); 4175 4176 ksp = kstat_create("unix", 0, "ecc-info", "misc", 4177 KSTAT_TYPE_NAMED, ecc_error_info_ndata, KSTAT_FLAG_VIRTUAL); 4178 if (ksp != NULL) { 4179 ksp->ks_data = (struct kstat_named *)&ecc_error_info_data; 4180 ecc_error_info_data.version.value.ui32 = KSTAT_CE_INFO_VER; 4181 ecc_error_info_data.maxcount.value.ui32 = mem_ce_simm_size; 4182 ecc_error_info_data.count.value.ui32 = 0; 4183 kstat_install(ksp); 4184 } 4185 4186 for (i = 0; i < mem_ce_simm_size; i++) { 4187 struct kstat_ecc_mm_info *kceip; 4188 4189 kceip = kmem_zalloc(sizeof (struct kstat_ecc_mm_info), 4190 KM_SLEEP); 4191 ksp = kstat_create("mm", i, "ecc-info", "misc", 4192 KSTAT_TYPE_NAMED, 4193 sizeof (struct kstat_ecc_mm_info) / sizeof (kstat_named_t), 4194 KSTAT_FLAG_VIRTUAL); 4195 if (ksp != NULL) { 4196 /* 4197 * Re-declare ks_data_size to include room for the 4198 * UNUM name since we don't have KSTAT_FLAG_VAR_SIZE 4199 * set. 4200 */ 4201 ksp->ks_data_size = sizeof (struct kstat_ecc_mm_info) + 4202 KSTAT_CE_UNUM_NAMLEN; 4203 ksp->ks_data = kceip; 4204 kstat_named_init(&kceip->name, 4205 "name", KSTAT_DATA_STRING); 4206 kstat_named_init(&kceip->intermittent_total, 4207 "intermittent_total", KSTAT_DATA_UINT64); 4208 kstat_named_init(&kceip->persistent_total, 4209 "persistent_total", KSTAT_DATA_UINT64); 4210 kstat_named_init(&kceip->sticky_total, 4211 "sticky_total", KSTAT_DATA_UINT64); 4212 /* 4213 * Use the default snapshot routine as it knows how to 4214 * deal with named kstats with long strings. 4215 */ 4216 ksp->ks_update = ecc_kstat_update; 4217 kstat_install(ksp); 4218 } else { 4219 kmem_free(kceip, sizeof (struct kstat_ecc_mm_info)); 4220 } 4221 } 4222 } 4223 4224 /*ARGSUSED*/ 4225 static void 4226 leaky_bucket_timeout(void *arg) 4227 { 4228 int i; 4229 struct ce_info *psimm = mem_ce_simm; 4230 4231 for (i = 0; i < mem_ce_simm_size; i++) { 4232 if (psimm[i].leaky_bucket_cnt > 0) 4233 atomic_add_16(&psimm[i].leaky_bucket_cnt, -1); 4234 } 4235 add_leaky_bucket_timeout(); 4236 } 4237 4238 static void 4239 add_leaky_bucket_timeout(void) 4240 { 4241 long timeout_in_microsecs; 4242 4243 /* 4244 * create timeout for next leak. 4245 * 4246 * The timeout interval is calculated as follows 4247 * 4248 * (ecc_softerr_interval * 60 * MICROSEC) / ecc_softerr_limit 4249 * 4250 * ecc_softerr_interval is in minutes, so multiply this by 60 (seconds 4251 * in a minute), then multiply this by MICROSEC to get the interval 4252 * in microseconds. Divide this total by ecc_softerr_limit so that 4253 * the timeout interval is accurate to within a few microseconds. 4254 */ 4255 4256 if (ecc_softerr_limit <= 0) 4257 ecc_softerr_limit = SOFTERR_LIMIT_DEFAULT; 4258 if (ecc_softerr_interval <= 0) 4259 ecc_softerr_interval = SOFTERR_INTERVAL_DEFAULT; 4260 4261 timeout_in_microsecs = ((int64_t)ecc_softerr_interval * 60 * MICROSEC) / 4262 ecc_softerr_limit; 4263 4264 if (timeout_in_microsecs < SOFTERR_MIN_TIMEOUT) 4265 timeout_in_microsecs = SOFTERR_MIN_TIMEOUT; 4266 4267 leaky_bucket_timeout_id = timeout(leaky_bucket_timeout, 4268 (void *)NULL, drv_usectohz((clock_t)timeout_in_microsecs)); 4269 } 4270 4271 /* 4272 * Legacy Correctable ECC Error Hash 4273 * 4274 * All of the code below this comment is used to implement a legacy array 4275 * which counted intermittent, persistent, and sticky CE errors by unum, 4276 * and then was later extended to publish the data as a kstat for SunVTS. 4277 * All of this code is replaced by FMA, and remains here until such time 4278 * that the UltraSPARC-I/II CPU code is converted to FMA, or is EOLed. 4279 * 4280 * Errors are saved in three buckets per-unum: 4281 * (1) sticky - scrub was unsuccessful, cannot be scrubbed 4282 * This could represent a problem, and is immediately printed out. 4283 * (2) persistent - was successfully scrubbed 4284 * These errors use the leaky bucket algorithm to determine 4285 * if there is a serious problem. 4286 * (3) intermittent - may have originated from the cpu or upa/safari bus, 4287 * and does not necessarily indicate any problem with the dimm itself, 4288 * is critical information for debugging new hardware. 4289 * Because we do not know if it came from the dimm, it would be 4290 * inappropriate to include these in the leaky bucket counts. 4291 * 4292 * If the E$ line was modified before the scrub operation began, then the 4293 * displacement flush at the beginning of scrubphys() will cause the modified 4294 * line to be written out, which will clean up the CE. Then, any subsequent 4295 * read will not cause an error, which will cause persistent errors to be 4296 * identified as intermittent. 4297 * 4298 * If a DIMM is going bad, it will produce true persistents as well as 4299 * false intermittents, so these intermittents can be safely ignored. 4300 * 4301 * If the error count is excessive for a DIMM, this function will return 4302 * PR_MCE, and the CPU module may then decide to remove that page from use. 4303 */ 4304 static int 4305 ce_count_unum(int status, int len, char *unum) 4306 { 4307 int i; 4308 struct ce_info *psimm = mem_ce_simm; 4309 int page_status = PR_OK; 4310 4311 ASSERT(psimm != NULL); 4312 4313 if (len <= 0 || 4314 (status & (ECC_STICKY | ECC_PERSISTENT | ECC_INTERMITTENT)) == 0) 4315 return (page_status); 4316 4317 /* 4318 * Initialize the leaky_bucket timeout 4319 */ 4320 if (casptr(&leaky_bucket_timeout_id, 4321 TIMEOUT_NONE, TIMEOUT_SET) == TIMEOUT_NONE) 4322 add_leaky_bucket_timeout(); 4323 4324 for (i = 0; i < mem_ce_simm_size; i++) { 4325 if (psimm[i].name[0] == '\0') { 4326 /* 4327 * Hit the end of the valid entries, add 4328 * a new one. 4329 */ 4330 (void) strncpy(psimm[i].name, unum, len); 4331 if (status & ECC_STICKY) { 4332 /* 4333 * Sticky - the leaky bucket is used to track 4334 * soft errors. Since a sticky error is a 4335 * hard error and likely to be retired soon, 4336 * we do not count it in the leaky bucket. 4337 */ 4338 psimm[i].leaky_bucket_cnt = 0; 4339 psimm[i].intermittent_total = 0; 4340 psimm[i].persistent_total = 0; 4341 psimm[i].sticky_total = 1; 4342 cmn_err(CE_WARN, 4343 "[AFT0] Sticky Softerror encountered " 4344 "on Memory Module %s\n", unum); 4345 page_status = PR_MCE; 4346 } else if (status & ECC_PERSISTENT) { 4347 psimm[i].leaky_bucket_cnt = 1; 4348 psimm[i].intermittent_total = 0; 4349 psimm[i].persistent_total = 1; 4350 psimm[i].sticky_total = 0; 4351 } else { 4352 /* 4353 * Intermittent - Because the scrub operation 4354 * cannot find the error in the DIMM, we will 4355 * not count these in the leaky bucket 4356 */ 4357 psimm[i].leaky_bucket_cnt = 0; 4358 psimm[i].intermittent_total = 1; 4359 psimm[i].persistent_total = 0; 4360 psimm[i].sticky_total = 0; 4361 } 4362 ecc_error_info_data.count.value.ui32++; 4363 break; 4364 } else if (strncmp(unum, psimm[i].name, len) == 0) { 4365 /* 4366 * Found an existing entry for the current 4367 * memory module, adjust the counts. 4368 */ 4369 if (status & ECC_STICKY) { 4370 psimm[i].sticky_total++; 4371 cmn_err(CE_WARN, 4372 "[AFT0] Sticky Softerror encountered " 4373 "on Memory Module %s\n", unum); 4374 page_status = PR_MCE; 4375 } else if (status & ECC_PERSISTENT) { 4376 int new_value; 4377 4378 new_value = atomic_add_16_nv( 4379 &psimm[i].leaky_bucket_cnt, 1); 4380 psimm[i].persistent_total++; 4381 if (new_value > ecc_softerr_limit) { 4382 cmn_err(CE_WARN, "[AFT0] Most recent %d" 4383 " soft errors from Memory Module" 4384 " %s exceed threshold (N=%d," 4385 " T=%dh:%02dm) triggering page" 4386 " retire", new_value, unum, 4387 ecc_softerr_limit, 4388 ecc_softerr_interval / 60, 4389 ecc_softerr_interval % 60); 4390 atomic_add_16( 4391 &psimm[i].leaky_bucket_cnt, -1); 4392 page_status = PR_MCE; 4393 } 4394 } else { /* Intermittent */ 4395 psimm[i].intermittent_total++; 4396 } 4397 break; 4398 } 4399 } 4400 4401 if (i >= mem_ce_simm_size) 4402 cmn_err(CE_CONT, "[AFT0] Softerror: mem_ce_simm[] out of " 4403 "space.\n"); 4404 4405 return (page_status); 4406 } 4407 4408 /* 4409 * Function to support counting of IO detected CEs. 4410 */ 4411 void 4412 cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum) 4413 { 4414 int err; 4415 4416 err = ce_count_unum(ecc->flt_status, len, unum); 4417 if (err != PR_OK && automatic_page_removal) { 4418 (void) page_retire(ecc->flt_addr, err); 4419 } 4420 } 4421 4422 static int 4423 ecc_kstat_update(kstat_t *ksp, int rw) 4424 { 4425 struct kstat_ecc_mm_info *kceip = ksp->ks_data; 4426 struct ce_info *ceip = mem_ce_simm; 4427 int i = ksp->ks_instance; 4428 4429 if (rw == KSTAT_WRITE) 4430 return (EACCES); 4431 4432 ASSERT(ksp->ks_data != NULL); 4433 ASSERT(i < mem_ce_simm_size && i >= 0); 4434 4435 /* 4436 * Since we're not using locks, make sure that we don't get partial 4437 * data. The name is always copied before the counters are incremented 4438 * so only do this update routine if at least one of the counters is 4439 * non-zero, which ensures that ce_count_unum() is done, and the 4440 * string is fully copied. 4441 */ 4442 if (ceip[i].intermittent_total == 0 && 4443 ceip[i].persistent_total == 0 && 4444 ceip[i].sticky_total == 0) { 4445 /* 4446 * Uninitialized or partially initialized. Ignore. 4447 * The ks_data buffer was allocated via kmem_zalloc, 4448 * so no need to bzero it. 4449 */ 4450 return (0); 4451 } 4452 4453 kstat_named_setstr(&kceip->name, ceip[i].name); 4454 kceip->intermittent_total.value.ui64 = ceip[i].intermittent_total; 4455 kceip->persistent_total.value.ui64 = ceip[i].persistent_total; 4456 kceip->sticky_total.value.ui64 = ceip[i].sticky_total; 4457 4458 return (0); 4459 } 4460 4461 #define VIS_BLOCKSIZE 64 4462 4463 int 4464 dtrace_blksuword32_err(uintptr_t addr, uint32_t *data) 4465 { 4466 int ret, watched; 4467 4468 watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 4469 ret = dtrace_blksuword32(addr, data, 0); 4470 if (watched) 4471 watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 4472 4473 return (ret); 4474 } 4475 4476 /*ARGSUSED*/ 4477 void 4478 cpu_faulted_enter(struct cpu *cp) 4479 { 4480 } 4481 4482 /*ARGSUSED*/ 4483 void 4484 cpu_faulted_exit(struct cpu *cp) 4485 { 4486 } 4487 4488 static int mmu_disable_ism_large_pages = ((1 << TTE512K) | 4489 (1 << TTE32M) | (1 << TTE256M)); 4490 static int mmu_disable_large_pages = ((1 << TTE32M) | (1 << TTE256M)); 4491 4492 /* 4493 * The function returns the US_II mmu-specific values for the 4494 * hat's disable_large_pages and disable_ism_large_pages variables. 4495 */ 4496 int 4497 mmu_large_pages_disabled(uint_t flag) 4498 { 4499 int pages_disable = 0; 4500 4501 if (flag == HAT_LOAD) { 4502 pages_disable = mmu_disable_large_pages; 4503 } else if (flag == HAT_LOAD_SHARE) { 4504 pages_disable = mmu_disable_ism_large_pages; 4505 } 4506 return (pages_disable); 4507 } 4508 4509 /*ARGSUSED*/ 4510 void 4511 mmu_init_kernel_pgsz(struct hat *hat) 4512 { 4513 } 4514 4515 size_t 4516 mmu_get_kernel_lpsize(size_t lpsize) 4517 { 4518 uint_t tte; 4519 4520 if (lpsize == 0) { 4521 /* no setting for segkmem_lpsize in /etc/system: use default */ 4522 return (MMU_PAGESIZE4M); 4523 } 4524 4525 for (tte = TTE8K; tte <= TTE4M; tte++) { 4526 if (lpsize == TTEBYTES(tte)) 4527 return (lpsize); 4528 } 4529 4530 return (TTEBYTES(TTE8K)); 4531 } 4532