1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/systm.h> 31 #include <sys/ddi.h> 32 #include <sys/sysmacros.h> 33 #include <sys/archsystm.h> 34 #include <sys/vmsystm.h> 35 #include <sys/machparam.h> 36 #include <sys/machsystm.h> 37 #include <sys/machthread.h> 38 #include <sys/cpu.h> 39 #include <sys/cmp.h> 40 #include <sys/elf_SPARC.h> 41 #include <vm/vm_dep.h> 42 #include <vm/hat_sfmmu.h> 43 #include <vm/seg_kpm.h> 44 #include <sys/cpuvar.h> 45 #include <sys/cheetahregs.h> 46 #include <sys/us3_module.h> 47 #include <sys/async.h> 48 #include <sys/cmn_err.h> 49 #include <sys/debug.h> 50 #include <sys/dditypes.h> 51 #include <sys/prom_debug.h> 52 #include <sys/prom_plat.h> 53 #include <sys/cpu_module.h> 54 #include <sys/sysmacros.h> 55 #include <sys/intreg.h> 56 #include <sys/clock.h> 57 #include <sys/platform_module.h> 58 #include <sys/machtrap.h> 59 #include <sys/ontrap.h> 60 #include <sys/panic.h> 61 #include <sys/memlist.h> 62 #include <sys/bootconf.h> 63 #include <sys/ivintr.h> 64 #include <sys/atomic.h> 65 #include <sys/taskq.h> 66 #include <sys/note.h> 67 #include <sys/ndifm.h> 68 #include <sys/ddifm.h> 69 #include <sys/fm/protocol.h> 70 #include <sys/fm/util.h> 71 #include <sys/fm/cpu/UltraSPARC-III.h> 72 #include <sys/fpras_impl.h> 73 #include <sys/dtrace.h> 74 #include <sys/watchpoint.h> 75 #include <sys/plat_ecc_unum.h> 76 #include <sys/cyclic.h> 77 #include <sys/errorq.h> 78 #include <sys/errclassify.h> 79 80 #ifdef CHEETAHPLUS_ERRATUM_25 81 #include <sys/xc_impl.h> 82 #endif /* CHEETAHPLUS_ERRATUM_25 */ 83 84 /* 85 * Note that 'Cheetah PRM' refers to: 86 * SPARC V9 JPS1 Implementation Supplement: Sun UltraSPARC-III 87 */ 88 89 /* 90 * Per CPU pointers to physical address of TL>0 logout data areas. 91 * These pointers have to be in the kernel nucleus to avoid MMU 92 * misses. 93 */ 94 uint64_t ch_err_tl1_paddrs[NCPU]; 95 96 /* 97 * One statically allocated structure to use during startup/DR 98 * to prevent unnecessary panics. 99 */ 100 ch_err_tl1_data_t ch_err_tl1_data; 101 102 /* 103 * Per CPU pending error at TL>0, used by level15 softint handler 104 */ 105 uchar_t ch_err_tl1_pending[NCPU]; 106 107 /* 108 * For deferred CE re-enable after trap. 109 */ 110 taskq_t *ch_check_ce_tq; 111 112 /* 113 * Internal functions. 114 */ 115 static int cpu_async_log_err(void *flt, errorq_elem_t *eqep); 116 static void cpu_log_diag_info(ch_async_flt_t *ch_flt); 117 static void cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 118 ecc_type_to_info_t *eccp, ch_diag_data_t *cdp); 119 static int clear_ecc(struct async_flt *ecc); 120 #if defined(CPU_IMP_ECACHE_ASSOC) 121 static int cpu_ecache_line_valid(ch_async_flt_t *ch_flt); 122 #endif 123 static int cpu_ecache_set_size(struct cpu *cp); 124 static int cpu_ectag_line_invalid(int cachesize, uint64_t tag); 125 static int cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr); 126 static uint64_t cpu_ectag_to_pa(int setsize, uint64_t tag); 127 static int cpu_ectag_pa_to_subblk_state(int cachesize, 128 uint64_t subaddr, uint64_t tag); 129 static void cpu_flush_ecache_line(ch_async_flt_t *ch_flt); 130 static int afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit); 131 static int afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit); 132 static int afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit); 133 static int afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit); 134 static int synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit); 135 static void cpu_uninit_ecache_scrub_dr(struct cpu *cp); 136 static void cpu_scrubphys(struct async_flt *aflt); 137 static void cpu_payload_add_aflt(struct async_flt *, nvlist_t *, nvlist_t *, 138 int *, int *); 139 static void cpu_payload_add_ecache(struct async_flt *, nvlist_t *); 140 static void cpu_ereport_init(struct async_flt *aflt); 141 static int cpu_check_secondary_errors(ch_async_flt_t *, uint64_t, uint64_t); 142 static uint8_t cpu_flt_bit_to_plat_error(struct async_flt *aflt); 143 static void cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 144 uint64_t nceen, ch_cpu_logout_t *clop); 145 static int cpu_ce_delayed_ec_logout(uint64_t); 146 static int cpu_matching_ecache_line(uint64_t, void *, int, int *); 147 148 #ifdef CHEETAHPLUS_ERRATUM_25 149 static int mondo_recover_proc(uint16_t, int); 150 static void cheetah_nudge_init(void); 151 static void cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 152 cyc_time_t *when); 153 static void cheetah_nudge_buddy(void); 154 #endif /* CHEETAHPLUS_ERRATUM_25 */ 155 156 #if defined(CPU_IMP_L1_CACHE_PARITY) 157 static void cpu_dcache_parity_info(ch_async_flt_t *ch_flt); 158 static void cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index); 159 static void cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 160 ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word); 161 static void cpu_icache_parity_info(ch_async_flt_t *ch_flt); 162 static void cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index); 163 static void cpu_pcache_parity_info(ch_async_flt_t *ch_flt); 164 static void cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index); 165 static void cpu_payload_add_dcache(struct async_flt *, nvlist_t *); 166 static void cpu_payload_add_icache(struct async_flt *, nvlist_t *); 167 #endif /* CPU_IMP_L1_CACHE_PARITY */ 168 169 int (*p2get_mem_info)(int synd_code, uint64_t paddr, 170 uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 171 int *segsp, int *banksp, int *mcidp); 172 173 /* 174 * This table is used to determine which bit(s) is(are) bad when an ECC 175 * error occurs. The array is indexed by an 9-bit syndrome. The entries 176 * of this array have the following semantics: 177 * 178 * 00-127 The number of the bad bit, when only one bit is bad. 179 * 128 ECC bit C0 is bad. 180 * 129 ECC bit C1 is bad. 181 * 130 ECC bit C2 is bad. 182 * 131 ECC bit C3 is bad. 183 * 132 ECC bit C4 is bad. 184 * 133 ECC bit C5 is bad. 185 * 134 ECC bit C6 is bad. 186 * 135 ECC bit C7 is bad. 187 * 136 ECC bit C8 is bad. 188 * 137-143 reserved for Mtag Data and ECC. 189 * 144(M2) Two bits are bad within a nibble. 190 * 145(M3) Three bits are bad within a nibble. 191 * 146(M3) Four bits are bad within a nibble. 192 * 147(M) Multiple bits (5 or more) are bad. 193 * 148 NO bits are bad. 194 * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-4,11-5. 195 */ 196 197 #define C0 128 198 #define C1 129 199 #define C2 130 200 #define C3 131 201 #define C4 132 202 #define C5 133 203 #define C6 134 204 #define C7 135 205 #define C8 136 206 #define MT0 137 /* Mtag Data bit 0 */ 207 #define MT1 138 208 #define MT2 139 209 #define MTC0 140 /* Mtag Check bit 0 */ 210 #define MTC1 141 211 #define MTC2 142 212 #define MTC3 143 213 #define M2 144 214 #define M3 145 215 #define M4 146 216 #define M 147 217 #define NA 148 218 #if defined(JALAPENO) || defined(SERRANO) 219 #define S003 149 /* Syndrome 0x003 => likely from CPU/EDU:ST/FRU/BP */ 220 #define S003MEM 150 /* Syndrome 0x003 => likely from WDU/WBP */ 221 #define SLAST S003MEM /* last special syndrome */ 222 #else /* JALAPENO || SERRANO */ 223 #define S003 149 /* Syndrome 0x003 => likely from EDU:ST */ 224 #define S071 150 /* Syndrome 0x071 => likely from WDU/CPU */ 225 #define S11C 151 /* Syndrome 0x11c => likely from BERR/DBERR */ 226 #define SLAST S11C /* last special syndrome */ 227 #endif /* JALAPENO || SERRANO */ 228 #if defined(JALAPENO) || defined(SERRANO) 229 #define BPAR0 152 /* syndrom 152 through 167 for bus parity */ 230 #define BPAR15 167 231 #endif /* JALAPENO || SERRANO */ 232 233 static uint8_t ecc_syndrome_tab[] = 234 { 235 NA, C0, C1, S003, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M, 236 C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16, 237 C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10, 238 M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M, 239 C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6, 240 M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4, 241 M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4, 242 #if defined(JALAPENO) || defined(SERRANO) 243 116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 244 #else /* JALAPENO || SERRANO */ 245 116, S071, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 246 #endif /* JALAPENO || SERRANO */ 247 C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5, 248 M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M, 249 M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2, 250 103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3, 251 M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M, 252 102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3, 253 98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M, 254 M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M, 255 C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4, 256 #if defined(JALAPENO) || defined(SERRANO) 257 M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M, 258 #else /* JALAPENO || SERRANO */ 259 M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, S11C, M, M3, M, 260 #endif /* JALAPENO || SERRANO */ 261 M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2, 262 94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M, 263 M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4, 264 89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3, 265 86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3, 266 M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2, 267 M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4, 268 77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M, 269 74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3, 270 M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M, 271 80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3, 272 M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M, 273 M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M, 274 111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M 275 }; 276 277 #define ESYND_TBL_SIZE (sizeof (ecc_syndrome_tab) / sizeof (uint8_t)) 278 279 #if !(defined(JALAPENO) || defined(SERRANO)) 280 /* 281 * This table is used to determine which bit(s) is(are) bad when a Mtag 282 * error occurs. The array is indexed by an 4-bit ECC syndrome. The entries 283 * of this array have the following semantics: 284 * 285 * -1 Invalid mtag syndrome. 286 * 137 Mtag Data 0 is bad. 287 * 138 Mtag Data 1 is bad. 288 * 139 Mtag Data 2 is bad. 289 * 140 Mtag ECC 0 is bad. 290 * 141 Mtag ECC 1 is bad. 291 * 142 Mtag ECC 2 is bad. 292 * 143 Mtag ECC 3 is bad. 293 * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-6. 294 */ 295 short mtag_syndrome_tab[] = 296 { 297 NA, MTC0, MTC1, M2, MTC2, M2, M2, MT0, MTC3, M2, M2, MT1, M2, MT2, M2, M2 298 }; 299 300 #define MSYND_TBL_SIZE (sizeof (mtag_syndrome_tab) / sizeof (short)) 301 302 #else /* !(JALAPENO || SERRANO) */ 303 304 #define BSYND_TBL_SIZE 16 305 306 #endif /* !(JALAPENO || SERRANO) */ 307 308 /* 309 * CE initial classification and subsequent action lookup table 310 */ 311 static ce_dispact_t ce_disp_table[CE_INITDISPTBL_SIZE]; 312 static int ce_disp_inited; 313 314 /* 315 * Set to disable leaky and partner check for memory correctables 316 */ 317 int ce_xdiag_off; 318 319 /* 320 * The following are not incremented atomically so are indicative only 321 */ 322 static int ce_xdiag_drops; 323 static int ce_xdiag_lkydrops; 324 static int ce_xdiag_ptnrdrops; 325 static int ce_xdiag_bad; 326 327 /* 328 * CE leaky check callback structure 329 */ 330 typedef struct { 331 struct async_flt *lkycb_aflt; 332 errorq_t *lkycb_eqp; 333 errorq_elem_t *lkycb_eqep; 334 } ce_lkychk_cb_t; 335 336 /* 337 * defines for various ecache_flush_flag's 338 */ 339 #define ECACHE_FLUSH_LINE 1 340 #define ECACHE_FLUSH_ALL 2 341 342 /* 343 * STICK sync 344 */ 345 #define STICK_ITERATION 10 346 #define MAX_TSKEW 1 347 #define EV_A_START 0 348 #define EV_A_END 1 349 #define EV_B_START 2 350 #define EV_B_END 3 351 #define EVENTS 4 352 353 static int64_t stick_iter = STICK_ITERATION; 354 static int64_t stick_tsk = MAX_TSKEW; 355 356 typedef enum { 357 EVENT_NULL = 0, 358 SLAVE_START, 359 SLAVE_CONT, 360 MASTER_START 361 } event_cmd_t; 362 363 static volatile event_cmd_t stick_sync_cmd = EVENT_NULL; 364 static int64_t timestamp[EVENTS]; 365 static volatile int slave_done; 366 367 #ifdef DEBUG 368 #define DSYNC_ATTEMPTS 64 369 typedef struct { 370 int64_t skew_val[DSYNC_ATTEMPTS]; 371 } ss_t; 372 373 ss_t stick_sync_stats[NCPU]; 374 #endif /* DEBUG */ 375 376 /* 377 * Maximum number of contexts for Cheetah. 378 */ 379 #define MAX_NCTXS (1 << 13) 380 381 /* Will be set !NULL for Cheetah+ and derivatives. */ 382 uchar_t *ctx_pgsz_array = NULL; 383 #if defined(CPU_IMP_DUAL_PAGESIZE) 384 static uchar_t ctx_pgsz_arr[MAX_NCTXS]; 385 uint_t disable_dual_pgsz = 0; 386 #endif /* CPU_IMP_DUAL_PAGESIZE */ 387 388 /* 389 * Save the cache bootup state for use when internal 390 * caches are to be re-enabled after an error occurs. 391 */ 392 uint64_t cache_boot_state; 393 394 /* 395 * PA[22:0] represent Displacement in Safari configuration space. 396 */ 397 uint_t root_phys_addr_lo_mask = 0x7fffffu; 398 399 bus_config_eclk_t bus_config_eclk[] = { 400 #if defined(JALAPENO) || defined(SERRANO) 401 {JBUS_CONFIG_ECLK_1_DIV, JBUS_CONFIG_ECLK_1}, 402 {JBUS_CONFIG_ECLK_2_DIV, JBUS_CONFIG_ECLK_2}, 403 {JBUS_CONFIG_ECLK_32_DIV, JBUS_CONFIG_ECLK_32}, 404 #else /* JALAPENO || SERRANO */ 405 {SAFARI_CONFIG_ECLK_1_DIV, SAFARI_CONFIG_ECLK_1}, 406 {SAFARI_CONFIG_ECLK_2_DIV, SAFARI_CONFIG_ECLK_2}, 407 {SAFARI_CONFIG_ECLK_32_DIV, SAFARI_CONFIG_ECLK_32}, 408 #endif /* JALAPENO || SERRANO */ 409 {0, 0} 410 }; 411 412 /* 413 * Interval for deferred CEEN reenable 414 */ 415 int cpu_ceen_delay_secs = CPU_CEEN_DELAY_SECS; 416 417 /* 418 * set in /etc/system to control logging of user BERR/TO's 419 */ 420 int cpu_berr_to_verbose = 0; 421 422 /* 423 * set to 0 in /etc/system to defer CEEN reenable for all CEs 424 */ 425 uint64_t cpu_ce_not_deferred = CPU_CE_NOT_DEFERRED; 426 uint64_t cpu_ce_not_deferred_ext = CPU_CE_NOT_DEFERRED_EXT; 427 428 /* 429 * Set of all offline cpus 430 */ 431 cpuset_t cpu_offline_set; 432 433 static void cpu_delayed_check_ce_errors(void *); 434 static void cpu_check_ce_errors(void *); 435 void cpu_error_ecache_flush(ch_async_flt_t *); 436 static int cpu_error_ecache_flush_required(ch_async_flt_t *); 437 static void cpu_log_and_clear_ce(ch_async_flt_t *); 438 void cpu_ce_detected(ch_cpu_errors_t *, int); 439 440 /* 441 * CE Leaky check timeout in microseconds. This is chosen to be twice the 442 * memory refresh interval of current DIMMs (64ms). After initial fix that 443 * gives at least one full refresh cycle in which the cell can leak 444 * (whereafter further refreshes simply reinforce any incorrect bit value). 445 */ 446 clock_t cpu_ce_lkychk_timeout_usec = 128000; 447 448 /* 449 * CE partner check partner caching period in seconds 450 */ 451 int cpu_ce_ptnr_cachetime_sec = 60; 452 453 /* 454 * Sets trap table entry ttentry by overwriting eight instructions from ttlabel 455 */ 456 #define CH_SET_TRAP(ttentry, ttlabel) \ 457 bcopy((const void *)&ttlabel, &ttentry, 32); \ 458 flush_instr_mem((caddr_t)&ttentry, 32); 459 460 static int min_ecache_size; 461 static uint_t priv_hcl_1; 462 static uint_t priv_hcl_2; 463 static uint_t priv_hcl_4; 464 static uint_t priv_hcl_8; 465 466 void 467 cpu_setup(void) 468 { 469 extern int at_flags; 470 extern int disable_delay_tlb_flush, delay_tlb_flush; 471 extern int cpc_has_overflow_intr; 472 extern int disable_text_largepages; 473 extern int use_text_pgsz4m; 474 475 /* 476 * Setup chip-specific trap handlers. 477 */ 478 cpu_init_trap(); 479 480 cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT); 481 482 at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1 | EF_SPARC_SUN_US3; 483 484 /* 485 * save the cache bootup state. 486 */ 487 cache_boot_state = get_dcu() & DCU_CACHE; 488 489 /* 490 * Use the maximum number of contexts available for Cheetah 491 * unless it has been tuned for debugging. 492 * We are checking against 0 here since this value can be patched 493 * while booting. It can not be patched via /etc/system since it 494 * will be patched too late and thus cause the system to panic. 495 */ 496 if (nctxs == 0) 497 nctxs = MAX_NCTXS; 498 499 /* 500 * Due to the number of entries in the fully-associative tlb 501 * this may have to be tuned lower than in spitfire. 502 */ 503 pp_slots = MIN(8, MAXPP_SLOTS); 504 505 /* 506 * Block stores do not invalidate all pages of the d$, pagecopy 507 * et. al. need virtual translations with virtual coloring taken 508 * into consideration. prefetch/ldd will pollute the d$ on the 509 * load side. 510 */ 511 pp_consistent_coloring = PPAGE_STORE_VCOLORING | PPAGE_LOADS_POLLUTE; 512 513 if (use_page_coloring) { 514 do_pg_coloring = 1; 515 if (use_virtual_coloring) 516 do_virtual_coloring = 1; 517 } 518 519 isa_list = 520 "sparcv9+vis2 sparcv9+vis sparcv9 " 521 "sparcv8plus+vis2 sparcv8plus+vis sparcv8plus " 522 "sparcv8 sparcv8-fsmuld sparcv7 sparc"; 523 524 /* 525 * On Panther-based machines, this should 526 * also include AV_SPARC_POPC too 527 */ 528 cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2; 529 530 /* 531 * On cheetah, there's no hole in the virtual address space 532 */ 533 hole_start = hole_end = 0; 534 535 /* 536 * The kpm mapping window. 537 * kpm_size: 538 * The size of a single kpm range. 539 * The overall size will be: kpm_size * vac_colors. 540 * kpm_vbase: 541 * The virtual start address of the kpm range within the kernel 542 * virtual address space. kpm_vbase has to be kpm_size aligned. 543 */ 544 kpm_size = (size_t)(8ull * 1024 * 1024 * 1024 * 1024); /* 8TB */ 545 kpm_size_shift = 43; 546 kpm_vbase = (caddr_t)0x8000000000000000ull; /* 8EB */ 547 kpm_smallpages = 1; 548 549 /* 550 * The traptrace code uses either %tick or %stick for 551 * timestamping. We have %stick so we can use it. 552 */ 553 traptrace_use_stick = 1; 554 555 /* 556 * Cheetah has a performance counter overflow interrupt 557 */ 558 cpc_has_overflow_intr = 1; 559 560 /* 561 * Use cheetah flush-all support 562 */ 563 if (!disable_delay_tlb_flush) 564 delay_tlb_flush = 1; 565 566 #if defined(CPU_IMP_DUAL_PAGESIZE) 567 /* 568 * Use Cheetah+ and later dual page size support. 569 */ 570 if (!disable_dual_pgsz) { 571 ctx_pgsz_array = ctx_pgsz_arr; 572 } 573 #endif /* CPU_IMP_DUAL_PAGESIZE */ 574 575 /* 576 * Declare that this architecture/cpu combination does fpRAS. 577 */ 578 fpras_implemented = 1; 579 580 /* 581 * Enable 4M pages to be used for mapping user text by default. Don't 582 * use large pages for initialized data segments since we may not know 583 * at exec() time what should be the preferred large page size for DTLB 584 * programming. 585 */ 586 use_text_pgsz4m = 1; 587 disable_text_largepages = (1 << TTE64K) | (1 << TTE512K) | 588 (1 << TTE32M) | (1 << TTE256M); 589 590 /* 591 * Setup CE lookup table 592 */ 593 CE_INITDISPTBL_POPULATE(ce_disp_table); 594 ce_disp_inited = 1; 595 } 596 597 /* 598 * Called by setcpudelay 599 */ 600 void 601 cpu_init_tick_freq(void) 602 { 603 /* 604 * For UltraSPARC III and beyond we want to use the 605 * system clock rate as the basis for low level timing, 606 * due to support of mixed speed CPUs and power managment. 607 */ 608 if (system_clock_freq == 0) 609 cmn_err(CE_PANIC, "setcpudelay: invalid system_clock_freq"); 610 611 sys_tick_freq = system_clock_freq; 612 } 613 614 #ifdef CHEETAHPLUS_ERRATUM_25 615 /* 616 * Tunables 617 */ 618 int cheetah_bpe_off = 0; 619 int cheetah_sendmondo_recover = 1; 620 int cheetah_sendmondo_fullscan = 0; 621 int cheetah_sendmondo_recover_delay = 5; 622 623 #define CHEETAH_LIVELOCK_MIN_DELAY 1 624 625 /* 626 * Recovery Statistics 627 */ 628 typedef struct cheetah_livelock_entry { 629 int cpuid; /* fallen cpu */ 630 int buddy; /* cpu that ran recovery */ 631 clock_t lbolt; /* when recovery started */ 632 hrtime_t recovery_time; /* time spent in recovery */ 633 } cheetah_livelock_entry_t; 634 635 #define CHEETAH_LIVELOCK_NENTRY 32 636 637 cheetah_livelock_entry_t cheetah_livelock_hist[CHEETAH_LIVELOCK_NENTRY]; 638 int cheetah_livelock_entry_nxt; 639 640 #define CHEETAH_LIVELOCK_ENTRY_NEXT(statp) { \ 641 statp = cheetah_livelock_hist + cheetah_livelock_entry_nxt; \ 642 if (++cheetah_livelock_entry_nxt >= CHEETAH_LIVELOCK_NENTRY) { \ 643 cheetah_livelock_entry_nxt = 0; \ 644 } \ 645 } 646 647 #define CHEETAH_LIVELOCK_ENTRY_SET(statp, item, val) statp->item = val 648 649 struct { 650 hrtime_t hrt; /* maximum recovery time */ 651 int recovery; /* recovered */ 652 int full_claimed; /* maximum pages claimed in full recovery */ 653 int proc_entry; /* attempted to claim TSB */ 654 int proc_tsb_scan; /* tsb scanned */ 655 int proc_tsb_partscan; /* tsb partially scanned */ 656 int proc_tsb_fullscan; /* whole tsb scanned */ 657 int proc_claimed; /* maximum pages claimed in tsb scan */ 658 int proc_user; /* user thread */ 659 int proc_kernel; /* kernel thread */ 660 int proc_onflt; /* bad stack */ 661 int proc_cpu; /* null cpu */ 662 int proc_thread; /* null thread */ 663 int proc_proc; /* null proc */ 664 int proc_as; /* null as */ 665 int proc_hat; /* null hat */ 666 int proc_hat_inval; /* hat contents don't make sense */ 667 int proc_hat_busy; /* hat is changing TSBs */ 668 int proc_tsb_reloc; /* TSB skipped because being relocated */ 669 int proc_cnum_bad; /* cnum out of range */ 670 int proc_cnum; /* last cnum processed */ 671 tte_t proc_tte; /* last tte processed */ 672 } cheetah_livelock_stat; 673 674 #define CHEETAH_LIVELOCK_STAT(item) cheetah_livelock_stat.item++ 675 676 #define CHEETAH_LIVELOCK_STATSET(item, value) \ 677 cheetah_livelock_stat.item = value 678 679 #define CHEETAH_LIVELOCK_MAXSTAT(item, value) { \ 680 if (value > cheetah_livelock_stat.item) \ 681 cheetah_livelock_stat.item = value; \ 682 } 683 684 /* 685 * Attempt to recover a cpu by claiming every cache line as saved 686 * in the TSB that the non-responsive cpu is using. Since we can't 687 * grab any adaptive lock, this is at best an attempt to do so. Because 688 * we don't grab any locks, we must operate under the protection of 689 * on_fault(). 690 * 691 * Return 1 if cpuid could be recovered, 0 if failed. 692 */ 693 int 694 mondo_recover_proc(uint16_t cpuid, int bn) 695 { 696 label_t ljb; 697 cpu_t *cp; 698 kthread_t *t; 699 proc_t *p; 700 struct as *as; 701 struct hat *hat; 702 short cnum; 703 struct tsb_info *tsbinfop; 704 struct tsbe *tsbep; 705 caddr_t tsbp; 706 caddr_t end_tsbp; 707 uint64_t paddr; 708 uint64_t idsr; 709 u_longlong_t pahi, palo; 710 int pages_claimed = 0; 711 tte_t tsbe_tte; 712 int tried_kernel_tsb = 0; 713 714 CHEETAH_LIVELOCK_STAT(proc_entry); 715 716 if (on_fault(&ljb)) { 717 CHEETAH_LIVELOCK_STAT(proc_onflt); 718 goto badstruct; 719 } 720 721 if ((cp = cpu[cpuid]) == NULL) { 722 CHEETAH_LIVELOCK_STAT(proc_cpu); 723 goto badstruct; 724 } 725 726 if ((t = cp->cpu_thread) == NULL) { 727 CHEETAH_LIVELOCK_STAT(proc_thread); 728 goto badstruct; 729 } 730 731 if ((p = ttoproc(t)) == NULL) { 732 CHEETAH_LIVELOCK_STAT(proc_proc); 733 goto badstruct; 734 } 735 736 if ((as = p->p_as) == NULL) { 737 CHEETAH_LIVELOCK_STAT(proc_as); 738 goto badstruct; 739 } 740 741 if ((hat = as->a_hat) == NULL) { 742 CHEETAH_LIVELOCK_STAT(proc_hat); 743 goto badstruct; 744 } 745 746 if (hat != ksfmmup) { 747 CHEETAH_LIVELOCK_STAT(proc_user); 748 if (hat->sfmmu_flags & (HAT_BUSY | HAT_SWAPPED | HAT_SWAPIN)) { 749 CHEETAH_LIVELOCK_STAT(proc_hat_busy); 750 goto badstruct; 751 } 752 tsbinfop = hat->sfmmu_tsb; 753 if (tsbinfop == NULL) { 754 CHEETAH_LIVELOCK_STAT(proc_hat_inval); 755 goto badstruct; 756 } 757 tsbp = tsbinfop->tsb_va; 758 end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 759 } else { 760 CHEETAH_LIVELOCK_STAT(proc_kernel); 761 tsbinfop = NULL; 762 tsbp = ktsb_base; 763 end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 764 } 765 766 /* Verify as */ 767 if (hat->sfmmu_as != as) { 768 CHEETAH_LIVELOCK_STAT(proc_hat_inval); 769 goto badstruct; 770 } 771 772 cnum = hat->sfmmu_cnum; 773 CHEETAH_LIVELOCK_STATSET(proc_cnum, cnum); 774 775 if ((cnum < 0) || (cnum == INVALID_CONTEXT) || (cnum >= nctxs)) { 776 CHEETAH_LIVELOCK_STAT(proc_cnum_bad); 777 goto badstruct; 778 } 779 780 do { 781 CHEETAH_LIVELOCK_STAT(proc_tsb_scan); 782 783 /* 784 * Skip TSBs being relocated. This is important because 785 * we want to avoid the following deadlock scenario: 786 * 787 * 1) when we came in we set ourselves to "in recover" state. 788 * 2) when we try to touch TSB being relocated the mapping 789 * will be in the suspended state so we'll spin waiting 790 * for it to be unlocked. 791 * 3) when the CPU that holds the TSB mapping locked tries to 792 * unlock it it will send a xtrap which will fail to xcall 793 * us or the CPU we're trying to recover, and will in turn 794 * enter the mondo code. 795 * 4) since we are still spinning on the locked mapping 796 * no further progress will be made and the system will 797 * inevitably hard hang. 798 * 799 * A TSB not being relocated can't begin being relocated 800 * while we're accessing it because we check 801 * sendmondo_in_recover before relocating TSBs. 802 */ 803 if (hat != ksfmmup && 804 (tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 805 CHEETAH_LIVELOCK_STAT(proc_tsb_reloc); 806 goto next_tsbinfo; 807 } 808 809 for (tsbep = (struct tsbe *)tsbp; 810 tsbep < (struct tsbe *)end_tsbp; tsbep++) { 811 tsbe_tte = tsbep->tte_data; 812 813 if (tsbe_tte.tte_val == 0) { 814 /* 815 * Invalid tte 816 */ 817 continue; 818 } 819 if (tsbe_tte.tte_se) { 820 /* 821 * Don't want device registers 822 */ 823 continue; 824 } 825 if (tsbe_tte.tte_cp == 0) { 826 /* 827 * Must be cached in E$ 828 */ 829 continue; 830 } 831 CHEETAH_LIVELOCK_STATSET(proc_tte, tsbe_tte); 832 idsr = getidsr(); 833 if ((idsr & (IDSR_NACK_BIT(bn) | 834 IDSR_BUSY_BIT(bn))) == 0) { 835 CHEETAH_LIVELOCK_STAT(proc_tsb_partscan); 836 goto done; 837 } 838 pahi = tsbe_tte.tte_pahi; 839 palo = tsbe_tte.tte_palo; 840 paddr = (uint64_t)((pahi << 32) | 841 (palo << MMU_PAGESHIFT)); 842 claimlines(paddr, TTEBYTES(TTE_CSZ(&tsbe_tte)), 843 CH_ECACHE_SUBBLK_SIZE); 844 if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 845 shipit(cpuid, bn); 846 } 847 pages_claimed++; 848 } 849 next_tsbinfo: 850 if (tsbinfop != NULL) 851 tsbinfop = tsbinfop->tsb_next; 852 if (tsbinfop != NULL) { 853 tsbp = tsbinfop->tsb_va; 854 end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 855 } else if (tsbp == ktsb_base) { 856 tried_kernel_tsb = 1; 857 } else if (!tried_kernel_tsb) { 858 tsbp = ktsb_base; 859 end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 860 hat = ksfmmup; 861 tsbinfop = NULL; 862 } 863 } while (tsbinfop != NULL || 864 ((tsbp == ktsb_base) && !tried_kernel_tsb)); 865 866 CHEETAH_LIVELOCK_STAT(proc_tsb_fullscan); 867 CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 868 no_fault(); 869 idsr = getidsr(); 870 if ((idsr & (IDSR_NACK_BIT(bn) | 871 IDSR_BUSY_BIT(bn))) == 0) { 872 return (1); 873 } else { 874 return (0); 875 } 876 877 done: 878 no_fault(); 879 CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 880 return (1); 881 882 badstruct: 883 no_fault(); 884 return (0); 885 } 886 887 /* 888 * Attempt to claim ownership, temporarily, of every cache line that a 889 * non-responsive cpu might be using. This might kick that cpu out of 890 * this state. 891 * 892 * The return value indicates to the caller if we have exhausted all recovery 893 * techniques. If 1 is returned, it is useless to call this function again 894 * even for a different target CPU. 895 */ 896 int 897 mondo_recover(uint16_t cpuid, int bn) 898 { 899 struct memseg *seg; 900 uint64_t begin_pa, end_pa, cur_pa; 901 hrtime_t begin_hrt, end_hrt; 902 int retval = 0; 903 int pages_claimed = 0; 904 cheetah_livelock_entry_t *histp; 905 uint64_t idsr; 906 907 if (cas32(&sendmondo_in_recover, 0, 1) != 0) { 908 /* 909 * Wait while recovery takes place 910 */ 911 while (sendmondo_in_recover) { 912 drv_usecwait(1); 913 } 914 /* 915 * Assume we didn't claim the whole memory. If 916 * the target of this caller is not recovered, 917 * it will come back. 918 */ 919 return (retval); 920 } 921 922 CHEETAH_LIVELOCK_ENTRY_NEXT(histp) 923 CHEETAH_LIVELOCK_ENTRY_SET(histp, lbolt, lbolt); 924 CHEETAH_LIVELOCK_ENTRY_SET(histp, cpuid, cpuid); 925 CHEETAH_LIVELOCK_ENTRY_SET(histp, buddy, CPU->cpu_id); 926 927 begin_hrt = gethrtime_waitfree(); 928 /* 929 * First try to claim the lines in the TSB the target 930 * may have been using. 931 */ 932 if (mondo_recover_proc(cpuid, bn) == 1) { 933 /* 934 * Didn't claim the whole memory 935 */ 936 goto done; 937 } 938 939 /* 940 * We tried using the TSB. The target is still 941 * not recovered. Check if complete memory scan is 942 * enabled. 943 */ 944 if (cheetah_sendmondo_fullscan == 0) { 945 /* 946 * Full memory scan is disabled. 947 */ 948 retval = 1; 949 goto done; 950 } 951 952 /* 953 * Try claiming the whole memory. 954 */ 955 for (seg = memsegs; seg; seg = seg->next) { 956 begin_pa = (uint64_t)(seg->pages_base) << MMU_PAGESHIFT; 957 end_pa = (uint64_t)(seg->pages_end) << MMU_PAGESHIFT; 958 for (cur_pa = begin_pa; cur_pa < end_pa; 959 cur_pa += MMU_PAGESIZE) { 960 idsr = getidsr(); 961 if ((idsr & (IDSR_NACK_BIT(bn) | 962 IDSR_BUSY_BIT(bn))) == 0) { 963 /* 964 * Didn't claim all memory 965 */ 966 goto done; 967 } 968 claimlines(cur_pa, MMU_PAGESIZE, 969 CH_ECACHE_SUBBLK_SIZE); 970 if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 971 shipit(cpuid, bn); 972 } 973 pages_claimed++; 974 } 975 } 976 977 /* 978 * We did all we could. 979 */ 980 retval = 1; 981 982 done: 983 /* 984 * Update statistics 985 */ 986 end_hrt = gethrtime_waitfree(); 987 CHEETAH_LIVELOCK_STAT(recovery); 988 CHEETAH_LIVELOCK_MAXSTAT(hrt, (end_hrt - begin_hrt)); 989 CHEETAH_LIVELOCK_MAXSTAT(full_claimed, pages_claimed); 990 CHEETAH_LIVELOCK_ENTRY_SET(histp, recovery_time, \ 991 (end_hrt - begin_hrt)); 992 993 while (cas32(&sendmondo_in_recover, 1, 0) != 1); 994 995 return (retval); 996 } 997 998 /* 999 * This is called by the cyclic framework when this CPU becomes online 1000 */ 1001 /*ARGSUSED*/ 1002 static void 1003 cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when) 1004 { 1005 1006 hdlr->cyh_func = (cyc_func_t)cheetah_nudge_buddy; 1007 hdlr->cyh_level = CY_LOW_LEVEL; 1008 hdlr->cyh_arg = NULL; 1009 1010 /* 1011 * Stagger the start time 1012 */ 1013 when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU); 1014 if (cheetah_sendmondo_recover_delay < CHEETAH_LIVELOCK_MIN_DELAY) { 1015 cheetah_sendmondo_recover_delay = CHEETAH_LIVELOCK_MIN_DELAY; 1016 } 1017 when->cyt_interval = cheetah_sendmondo_recover_delay * NANOSEC; 1018 } 1019 1020 /* 1021 * Create a low level cyclic to send a xtrap to the next cpu online. 1022 * However, there's no need to have this running on a uniprocessor system. 1023 */ 1024 static void 1025 cheetah_nudge_init(void) 1026 { 1027 cyc_omni_handler_t hdlr; 1028 1029 if (max_ncpus == 1) { 1030 return; 1031 } 1032 1033 hdlr.cyo_online = cheetah_nudge_onln; 1034 hdlr.cyo_offline = NULL; 1035 hdlr.cyo_arg = NULL; 1036 1037 mutex_enter(&cpu_lock); 1038 (void) cyclic_add_omni(&hdlr); 1039 mutex_exit(&cpu_lock); 1040 } 1041 1042 /* 1043 * Cyclic handler to wake up buddy 1044 */ 1045 void 1046 cheetah_nudge_buddy(void) 1047 { 1048 /* 1049 * Disable kernel preemption to protect the cpu list 1050 */ 1051 kpreempt_disable(); 1052 if ((CPU->cpu_next_onln != CPU) && (sendmondo_in_recover == 0)) { 1053 xt_one(CPU->cpu_next_onln->cpu_id, (xcfunc_t *)xt_sync_tl1, 1054 0, 0); 1055 } 1056 kpreempt_enable(); 1057 } 1058 1059 #endif /* CHEETAHPLUS_ERRATUM_25 */ 1060 1061 #ifdef SEND_MONDO_STATS 1062 uint32_t x_one_stimes[64]; 1063 uint32_t x_one_ltimes[16]; 1064 uint32_t x_set_stimes[64]; 1065 uint32_t x_set_ltimes[16]; 1066 uint32_t x_set_cpus[NCPU]; 1067 uint32_t x_nack_stimes[64]; 1068 #endif 1069 1070 /* 1071 * Note: A version of this function is used by the debugger via the KDI, 1072 * and must be kept in sync with this version. Any changes made to this 1073 * function to support new chips or to accomodate errata must also be included 1074 * in the KDI-specific version. See us3_kdi.c. 1075 */ 1076 void 1077 send_one_mondo(int cpuid) 1078 { 1079 int busy, nack; 1080 uint64_t idsr, starttick, endtick, tick, lasttick; 1081 uint64_t busymask; 1082 #ifdef CHEETAHPLUS_ERRATUM_25 1083 int recovered = 0; 1084 #endif 1085 1086 CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 1087 starttick = lasttick = gettick(); 1088 shipit(cpuid, 0); 1089 endtick = starttick + xc_tick_limit; 1090 busy = nack = 0; 1091 #if defined(JALAPENO) || defined(SERRANO) 1092 /* 1093 * Lower 2 bits of the agent ID determine which BUSY/NACK pair 1094 * will be used for dispatching interrupt. For now, assume 1095 * there are no more than IDSR_BN_SETS CPUs, hence no aliasing 1096 * issues with respect to BUSY/NACK pair usage. 1097 */ 1098 busymask = IDSR_BUSY_BIT(cpuid); 1099 #else /* JALAPENO || SERRANO */ 1100 busymask = IDSR_BUSY; 1101 #endif /* JALAPENO || SERRANO */ 1102 for (;;) { 1103 idsr = getidsr(); 1104 if (idsr == 0) 1105 break; 1106 1107 tick = gettick(); 1108 /* 1109 * If there is a big jump between the current tick 1110 * count and lasttick, we have probably hit a break 1111 * point. Adjust endtick accordingly to avoid panic. 1112 */ 1113 if (tick > (lasttick + xc_tick_jump_limit)) 1114 endtick += (tick - lasttick); 1115 lasttick = tick; 1116 if (tick > endtick) { 1117 if (panic_quiesce) 1118 return; 1119 #ifdef CHEETAHPLUS_ERRATUM_25 1120 if (cheetah_sendmondo_recover && recovered == 0) { 1121 if (mondo_recover(cpuid, 0)) { 1122 /* 1123 * We claimed the whole memory or 1124 * full scan is disabled. 1125 */ 1126 recovered++; 1127 } 1128 tick = gettick(); 1129 endtick = tick + xc_tick_limit; 1130 lasttick = tick; 1131 /* 1132 * Recheck idsr 1133 */ 1134 continue; 1135 } else 1136 #endif /* CHEETAHPLUS_ERRATUM_25 */ 1137 { 1138 cmn_err(CE_PANIC, "send mondo timeout " 1139 "(target 0x%x) [%d NACK %d BUSY]", 1140 cpuid, nack, busy); 1141 } 1142 } 1143 1144 if (idsr & busymask) { 1145 busy++; 1146 continue; 1147 } 1148 drv_usecwait(1); 1149 shipit(cpuid, 0); 1150 nack++; 1151 busy = 0; 1152 } 1153 #ifdef SEND_MONDO_STATS 1154 { 1155 int n = gettick() - starttick; 1156 if (n < 8192) 1157 x_one_stimes[n >> 7]++; 1158 else 1159 x_one_ltimes[(n >> 13) & 0xf]++; 1160 } 1161 #endif 1162 } 1163 1164 void 1165 syncfpu(void) 1166 { 1167 } 1168 1169 /* 1170 * Return processor specific async error structure 1171 * size used. 1172 */ 1173 int 1174 cpu_aflt_size(void) 1175 { 1176 return (sizeof (ch_async_flt_t)); 1177 } 1178 1179 /* 1180 * Tunable to disable the checking of other cpu logout areas during panic for 1181 * potential syndrome 71 generating errors. 1182 */ 1183 int enable_check_other_cpus_logout = 1; 1184 1185 /* 1186 * Check other cpus logout area for potential synd 71 generating 1187 * errors. 1188 */ 1189 static void 1190 cpu_check_cpu_logout(int cpuid, caddr_t tpc, int tl, int ecc_type, 1191 ch_cpu_logout_t *clop) 1192 { 1193 struct async_flt *aflt; 1194 ch_async_flt_t ch_flt; 1195 uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1196 1197 if (clop == NULL || clop->clo_data.chd_afar == LOGOUT_INVALID) { 1198 return; 1199 } 1200 1201 bzero(&ch_flt, sizeof (ch_async_flt_t)); 1202 1203 t_afar = clop->clo_data.chd_afar; 1204 t_afsr = clop->clo_data.chd_afsr; 1205 t_afsr_ext = clop->clo_data.chd_afsr_ext; 1206 #if defined(SERRANO) 1207 ch_flt.afar2 = clop->clo_data.chd_afar2; 1208 #endif /* SERRANO */ 1209 1210 /* 1211 * In order to simplify code, we maintain this afsr_errs 1212 * variable which holds the aggregate of AFSR and AFSR_EXT 1213 * sticky bits. 1214 */ 1215 t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1216 (t_afsr & C_AFSR_ALL_ERRS); 1217 1218 /* Setup the async fault structure */ 1219 aflt = (struct async_flt *)&ch_flt; 1220 aflt->flt_id = gethrtime_waitfree(); 1221 ch_flt.afsr_ext = t_afsr_ext; 1222 ch_flt.afsr_errs = t_afsr_errs; 1223 aflt->flt_stat = t_afsr; 1224 aflt->flt_addr = t_afar; 1225 aflt->flt_bus_id = cpuid; 1226 aflt->flt_inst = cpuid; 1227 aflt->flt_pc = tpc; 1228 aflt->flt_prot = AFLT_PROT_NONE; 1229 aflt->flt_class = CPU_FAULT; 1230 aflt->flt_priv = ((t_afsr & C_AFSR_PRIV) != 0); 1231 aflt->flt_tl = tl; 1232 aflt->flt_status = ecc_type; 1233 aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 1234 1235 /* 1236 * Queue events on the async event queue, one event per error bit. 1237 * If no events are queued, queue an event to complain. 1238 */ 1239 if (cpu_queue_events(&ch_flt, NULL, t_afsr_errs, clop) == 0) { 1240 ch_flt.flt_type = CPU_INV_AFSR; 1241 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 1242 (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 1243 aflt->flt_panic); 1244 } 1245 1246 /* 1247 * Zero out + invalidate CPU logout. 1248 */ 1249 bzero(clop, sizeof (ch_cpu_logout_t)); 1250 clop->clo_data.chd_afar = LOGOUT_INVALID; 1251 } 1252 1253 /* 1254 * Check the logout areas of all other cpus for unlogged errors. 1255 */ 1256 static void 1257 cpu_check_other_cpus_logout(void) 1258 { 1259 int i, j; 1260 processorid_t myid; 1261 struct cpu *cp; 1262 ch_err_tl1_data_t *cl1p; 1263 1264 myid = CPU->cpu_id; 1265 for (i = 0; i < NCPU; i++) { 1266 cp = cpu[i]; 1267 1268 if ((cp == NULL) || !(cp->cpu_flags & CPU_EXISTS) || 1269 (cp->cpu_id == myid) || (CPU_PRIVATE(cp) == NULL)) { 1270 continue; 1271 } 1272 1273 /* 1274 * Check each of the tl>0 logout areas 1275 */ 1276 cl1p = CPU_PRIVATE_PTR(cp, chpr_tl1_err_data[0]); 1277 for (j = 0; j < CH_ERR_TL1_TLMAX; j++, cl1p++) { 1278 if (cl1p->ch_err_tl1_flags == 0) 1279 continue; 1280 1281 cpu_check_cpu_logout(i, (caddr_t)cl1p->ch_err_tl1_tpc, 1282 1, ECC_F_TRAP, &cl1p->ch_err_tl1_logout); 1283 } 1284 1285 /* 1286 * Check each of the remaining logout areas 1287 */ 1288 cpu_check_cpu_logout(i, NULL, 0, ECC_F_TRAP, 1289 CPU_PRIVATE_PTR(cp, chpr_fecctl0_logout)); 1290 cpu_check_cpu_logout(i, NULL, 0, ECC_C_TRAP, 1291 CPU_PRIVATE_PTR(cp, chpr_cecc_logout)); 1292 cpu_check_cpu_logout(i, NULL, 0, ECC_D_TRAP, 1293 CPU_PRIVATE_PTR(cp, chpr_async_logout)); 1294 } 1295 } 1296 1297 /* 1298 * The fast_ecc_err handler transfers control here for UCU, UCC events. 1299 * Note that we flush Ecache twice, once in the fast_ecc_err handler to 1300 * flush the error that caused the UCU/UCC, then again here at the end to 1301 * flush the TL=1 trap handler code out of the Ecache, so we can minimize 1302 * the probability of getting a TL>1 Fast ECC trap when we're fielding 1303 * another Fast ECC trap. 1304 * 1305 * Cheetah+ also handles: TSCE: No additional processing required. 1306 * Panther adds L3_UCU and L3_UCC which are reported in AFSR_EXT. 1307 * 1308 * Note that the p_clo_flags input is only valid in cases where the 1309 * cpu_private struct is not yet initialized (since that is the only 1310 * time that information cannot be obtained from the logout struct.) 1311 */ 1312 /*ARGSUSED*/ 1313 void 1314 cpu_fast_ecc_error(struct regs *rp, ulong_t p_clo_flags) 1315 { 1316 ch_cpu_logout_t *clop; 1317 uint64_t ceen, nceen; 1318 1319 /* 1320 * Get the CPU log out info. If we can't find our CPU private 1321 * pointer, then we will have to make due without any detailed 1322 * logout information. 1323 */ 1324 if (CPU_PRIVATE(CPU) == NULL) { 1325 clop = NULL; 1326 ceen = p_clo_flags & EN_REG_CEEN; 1327 nceen = p_clo_flags & EN_REG_NCEEN; 1328 } else { 1329 clop = CPU_PRIVATE_PTR(CPU, chpr_fecctl0_logout); 1330 ceen = clop->clo_flags & EN_REG_CEEN; 1331 nceen = clop->clo_flags & EN_REG_NCEEN; 1332 } 1333 1334 cpu_log_fast_ecc_error((caddr_t)rp->r_pc, 1335 (rp->r_tstate & TSTATE_PRIV) ? 1 : 0, 0, ceen, nceen, clop); 1336 } 1337 1338 /* 1339 * Log fast ecc error, called from either Fast ECC at TL=0 or Fast 1340 * ECC at TL>0. Need to supply either a error register pointer or a 1341 * cpu logout structure pointer. 1342 */ 1343 static void 1344 cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 1345 uint64_t nceen, ch_cpu_logout_t *clop) 1346 { 1347 struct async_flt *aflt; 1348 ch_async_flt_t ch_flt; 1349 uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1350 char pr_reason[MAX_REASON_STRING]; 1351 ch_cpu_errors_t cpu_error_regs; 1352 1353 bzero(&ch_flt, sizeof (ch_async_flt_t)); 1354 /* 1355 * If no cpu logout data, then we will have to make due without 1356 * any detailed logout information. 1357 */ 1358 if (clop == NULL) { 1359 ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 1360 get_cpu_error_state(&cpu_error_regs); 1361 set_cpu_error_state(&cpu_error_regs); 1362 t_afar = cpu_error_regs.afar; 1363 t_afsr = cpu_error_regs.afsr; 1364 t_afsr_ext = cpu_error_regs.afsr_ext; 1365 #if defined(SERRANO) 1366 ch_flt.afar2 = cpu_error_regs.afar2; 1367 #endif /* SERRANO */ 1368 } else { 1369 t_afar = clop->clo_data.chd_afar; 1370 t_afsr = clop->clo_data.chd_afsr; 1371 t_afsr_ext = clop->clo_data.chd_afsr_ext; 1372 #if defined(SERRANO) 1373 ch_flt.afar2 = clop->clo_data.chd_afar2; 1374 #endif /* SERRANO */ 1375 } 1376 1377 /* 1378 * In order to simplify code, we maintain this afsr_errs 1379 * variable which holds the aggregate of AFSR and AFSR_EXT 1380 * sticky bits. 1381 */ 1382 t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1383 (t_afsr & C_AFSR_ALL_ERRS); 1384 pr_reason[0] = '\0'; 1385 1386 /* Setup the async fault structure */ 1387 aflt = (struct async_flt *)&ch_flt; 1388 aflt->flt_id = gethrtime_waitfree(); 1389 ch_flt.afsr_ext = t_afsr_ext; 1390 ch_flt.afsr_errs = t_afsr_errs; 1391 aflt->flt_stat = t_afsr; 1392 aflt->flt_addr = t_afar; 1393 aflt->flt_bus_id = getprocessorid(); 1394 aflt->flt_inst = CPU->cpu_id; 1395 aflt->flt_pc = tpc; 1396 aflt->flt_prot = AFLT_PROT_NONE; 1397 aflt->flt_class = CPU_FAULT; 1398 aflt->flt_priv = priv; 1399 aflt->flt_tl = tl; 1400 aflt->flt_status = ECC_F_TRAP; 1401 aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 1402 1403 /* 1404 * XXXX - Phenomenal hack to get around Solaris not getting all the 1405 * cmn_err messages out to the console. The situation is a UCU (in 1406 * priv mode) which causes a WDU which causes a UE (on the retry). 1407 * The messages for the UCU and WDU are enqueued and then pulled off 1408 * the async queue via softint and syslogd starts to process them 1409 * but doesn't get them to the console. The UE causes a panic, but 1410 * since the UCU/WDU messages are already in transit, those aren't 1411 * on the async queue. The hack is to check if we have a matching 1412 * WDU event for the UCU, and if it matches, we're more than likely 1413 * going to panic with a UE, unless we're under protection. So, we 1414 * check to see if we got a matching WDU event and if we're under 1415 * protection. 1416 * 1417 * For Cheetah/Cheetah+/Jaguar/Jalapeno, the sequence we care about 1418 * looks like this: 1419 * UCU->WDU->UE 1420 * For Panther, it could look like either of these: 1421 * UCU---->WDU->L3_WDU->UE 1422 * L3_UCU->WDU->L3_WDU->UE 1423 */ 1424 if ((t_afsr_errs & (C_AFSR_UCU | C_AFSR_L3_UCU)) && 1425 aflt->flt_panic == 0 && aflt->flt_priv != 0 && 1426 curthread->t_ontrap == NULL && curthread->t_lofault == NULL) { 1427 get_cpu_error_state(&cpu_error_regs); 1428 aflt->flt_panic |= ((cpu_error_regs.afsr & C_AFSR_WDU) && 1429 (cpu_error_regs.afar == t_afar)); 1430 aflt->flt_panic |= ((clop == NULL) && 1431 (t_afsr_errs & C_AFSR_WDU)); 1432 } 1433 1434 /* 1435 * Queue events on the async event queue, one event per error bit. 1436 * If no events are queued or no Fast ECC events are on in the AFSR, 1437 * queue an event to complain. 1438 */ 1439 if (cpu_queue_events(&ch_flt, pr_reason, t_afsr_errs, clop) == 0 || 1440 ((t_afsr_errs & (C_AFSR_FECC_ERRS | C_AFSR_EXT_FECC_ERRS)) == 0)) { 1441 ch_flt.flt_type = CPU_INV_AFSR; 1442 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 1443 (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 1444 aflt->flt_panic); 1445 } 1446 1447 /* 1448 * Zero out + invalidate CPU logout. 1449 */ 1450 if (clop) { 1451 bzero(clop, sizeof (ch_cpu_logout_t)); 1452 clop->clo_data.chd_afar = LOGOUT_INVALID; 1453 } 1454 1455 /* 1456 * We carefully re-enable NCEEN and CEEN and then check if any deferred 1457 * or disrupting errors have happened. We do this because if a 1458 * deferred or disrupting error had occurred with NCEEN/CEEN off, the 1459 * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 1460 * CEEN works differently on Cheetah than on Spitfire. Also, we enable 1461 * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 1462 * deferred or disrupting error happening between checking the AFSR and 1463 * enabling NCEEN/CEEN. 1464 * 1465 * Note: CEEN and NCEEN are only reenabled if they were on when trap 1466 * taken. 1467 */ 1468 set_error_enable(get_error_enable() | (nceen | ceen)); 1469 if (clear_errors(&ch_flt)) { 1470 aflt->flt_panic |= ((ch_flt.afsr_errs & 1471 (C_AFSR_EXT_ASYNC_ERRS | C_AFSR_ASYNC_ERRS)) != 0); 1472 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 1473 NULL); 1474 } 1475 1476 /* 1477 * Panic here if aflt->flt_panic has been set. Enqueued errors will 1478 * be logged as part of the panic flow. 1479 */ 1480 if (aflt->flt_panic) 1481 fm_panic("%sError(s)", pr_reason); 1482 1483 /* 1484 * Flushing the Ecache here gets the part of the trap handler that 1485 * is run at TL=1 out of the Ecache. 1486 */ 1487 cpu_flush_ecache(); 1488 } 1489 1490 /* 1491 * This is called via sys_trap from pil15_interrupt code if the 1492 * corresponding entry in ch_err_tl1_pending is set. Checks the 1493 * various ch_err_tl1_data structures for valid entries based on the bit 1494 * settings in the ch_err_tl1_flags entry of the structure. 1495 */ 1496 /*ARGSUSED*/ 1497 void 1498 cpu_tl1_error(struct regs *rp, int panic) 1499 { 1500 ch_err_tl1_data_t *cl1p, cl1; 1501 int i, ncl1ps; 1502 uint64_t me_flags; 1503 uint64_t ceen, nceen; 1504 1505 if (ch_err_tl1_paddrs[CPU->cpu_id] == 0) { 1506 cl1p = &ch_err_tl1_data; 1507 ncl1ps = 1; 1508 } else if (CPU_PRIVATE(CPU) != NULL) { 1509 cl1p = CPU_PRIVATE_PTR(CPU, chpr_tl1_err_data[0]); 1510 ncl1ps = CH_ERR_TL1_TLMAX; 1511 } else { 1512 ncl1ps = 0; 1513 } 1514 1515 for (i = 0; i < ncl1ps; i++, cl1p++) { 1516 if (cl1p->ch_err_tl1_flags == 0) 1517 continue; 1518 1519 /* 1520 * Grab a copy of the logout data and invalidate 1521 * the logout area. 1522 */ 1523 cl1 = *cl1p; 1524 bzero(cl1p, sizeof (ch_err_tl1_data_t)); 1525 cl1p->ch_err_tl1_logout.clo_data.chd_afar = LOGOUT_INVALID; 1526 me_flags = CH_ERR_ME_FLAGS(cl1.ch_err_tl1_flags); 1527 1528 /* 1529 * Log "first error" in ch_err_tl1_data. 1530 */ 1531 if (cl1.ch_err_tl1_flags & CH_ERR_FECC) { 1532 ceen = get_error_enable() & EN_REG_CEEN; 1533 nceen = get_error_enable() & EN_REG_NCEEN; 1534 cpu_log_fast_ecc_error((caddr_t)cl1.ch_err_tl1_tpc, 1, 1535 1, ceen, nceen, &cl1.ch_err_tl1_logout); 1536 } 1537 #if defined(CPU_IMP_L1_CACHE_PARITY) 1538 if (cl1.ch_err_tl1_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 1539 cpu_parity_error(rp, cl1.ch_err_tl1_flags, 1540 (caddr_t)cl1.ch_err_tl1_tpc); 1541 } 1542 #endif /* CPU_IMP_L1_CACHE_PARITY */ 1543 1544 /* 1545 * Log "multiple events" in ch_err_tl1_data. Note that 1546 * we don't read and clear the AFSR/AFAR in the TL>0 code 1547 * if the structure is busy, we just do the cache flushing 1548 * we have to do and then do the retry. So the AFSR/AFAR 1549 * at this point *should* have some relevant info. If there 1550 * are no valid errors in the AFSR, we'll assume they've 1551 * already been picked up and logged. For I$/D$ parity, 1552 * we just log an event with an "Unknown" (NULL) TPC. 1553 */ 1554 if (me_flags & CH_ERR_FECC) { 1555 ch_cpu_errors_t cpu_error_regs; 1556 uint64_t t_afsr_errs; 1557 1558 /* 1559 * Get the error registers and see if there's 1560 * a pending error. If not, don't bother 1561 * generating an "Invalid AFSR" error event. 1562 */ 1563 get_cpu_error_state(&cpu_error_regs); 1564 t_afsr_errs = (cpu_error_regs.afsr_ext & 1565 C_AFSR_EXT_ALL_ERRS) | 1566 (cpu_error_regs.afsr & C_AFSR_ALL_ERRS); 1567 if (t_afsr_errs != 0) { 1568 ceen = get_error_enable() & EN_REG_CEEN; 1569 nceen = get_error_enable() & EN_REG_NCEEN; 1570 cpu_log_fast_ecc_error((caddr_t)NULL, 1, 1571 1, ceen, nceen, NULL); 1572 } 1573 } 1574 #if defined(CPU_IMP_L1_CACHE_PARITY) 1575 if (me_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 1576 cpu_parity_error(rp, me_flags, (caddr_t)NULL); 1577 } 1578 #endif /* CPU_IMP_L1_CACHE_PARITY */ 1579 } 1580 } 1581 1582 /* 1583 * Called from Fast ECC TL>0 handler in case of fatal error. 1584 * cpu_tl1_error should always find an associated ch_err_tl1_data structure, 1585 * but if we don't, we'll panic with something reasonable. 1586 */ 1587 /*ARGSUSED*/ 1588 void 1589 cpu_tl1_err_panic(struct regs *rp, ulong_t flags) 1590 { 1591 cpu_tl1_error(rp, 1); 1592 /* 1593 * Should never return, but just in case. 1594 */ 1595 fm_panic("Unsurvivable ECC Error at TL>0"); 1596 } 1597 1598 /* 1599 * The ce_err/ce_err_tl1 handlers transfer control here for CE, EMC, EDU:ST, 1600 * EDC, WDU, WDC, CPU, CPC, IVU, IVC events. 1601 * Disrupting errors controlled by NCEEN: EDU:ST, WDU, CPU, IVU 1602 * Disrupting errors controlled by CEEN: CE, EMC, EDC, WDC, CPC, IVC 1603 * 1604 * Cheetah+ also handles (No additional processing required): 1605 * DUE, DTO, DBERR (NCEEN controlled) 1606 * THCE (CEEN and ET_ECC_en controlled) 1607 * TUE (ET_ECC_en controlled) 1608 * 1609 * Panther further adds: 1610 * IMU, L3_EDU, L3_WDU, L3_CPU (NCEEN controlled) 1611 * IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE (CEEN controlled) 1612 * TUE_SH, TUE (NCEEN and L2_tag_ECC_en controlled) 1613 * L3_TUE, L3_TUE_SH (NCEEN and ET_ECC_en controlled) 1614 * THCE (CEEN and L2_tag_ECC_en controlled) 1615 * L3_THCE (CEEN and ET_ECC_en controlled) 1616 * 1617 * Note that the p_clo_flags input is only valid in cases where the 1618 * cpu_private struct is not yet initialized (since that is the only 1619 * time that information cannot be obtained from the logout struct.) 1620 */ 1621 /*ARGSUSED*/ 1622 void 1623 cpu_disrupting_error(struct regs *rp, ulong_t p_clo_flags) 1624 { 1625 struct async_flt *aflt; 1626 ch_async_flt_t ch_flt; 1627 char pr_reason[MAX_REASON_STRING]; 1628 ch_cpu_logout_t *clop; 1629 uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1630 ch_cpu_errors_t cpu_error_regs; 1631 1632 bzero(&ch_flt, sizeof (ch_async_flt_t)); 1633 /* 1634 * Get the CPU log out info. If we can't find our CPU private 1635 * pointer, then we will have to make due without any detailed 1636 * logout information. 1637 */ 1638 if (CPU_PRIVATE(CPU) == NULL) { 1639 clop = NULL; 1640 ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 1641 get_cpu_error_state(&cpu_error_regs); 1642 set_cpu_error_state(&cpu_error_regs); 1643 t_afar = cpu_error_regs.afar; 1644 t_afsr = cpu_error_regs.afsr; 1645 t_afsr_ext = cpu_error_regs.afsr_ext; 1646 #if defined(SERRANO) 1647 ch_flt.afar2 = cpu_error_regs.afar2; 1648 #endif /* SERRANO */ 1649 } else { 1650 clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 1651 t_afar = clop->clo_data.chd_afar; 1652 t_afsr = clop->clo_data.chd_afsr; 1653 t_afsr_ext = clop->clo_data.chd_afsr_ext; 1654 #if defined(SERRANO) 1655 ch_flt.afar2 = clop->clo_data.chd_afar2; 1656 #endif /* SERRANO */ 1657 } 1658 1659 /* 1660 * In order to simplify code, we maintain this afsr_errs 1661 * variable which holds the aggregate of AFSR and AFSR_EXT 1662 * sticky bits. 1663 */ 1664 t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1665 (t_afsr & C_AFSR_ALL_ERRS); 1666 1667 pr_reason[0] = '\0'; 1668 /* Setup the async fault structure */ 1669 aflt = (struct async_flt *)&ch_flt; 1670 ch_flt.afsr_ext = t_afsr_ext; 1671 ch_flt.afsr_errs = t_afsr_errs; 1672 aflt->flt_stat = t_afsr; 1673 aflt->flt_addr = t_afar; 1674 aflt->flt_pc = (caddr_t)rp->r_pc; 1675 aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 1676 aflt->flt_tl = 0; 1677 aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 1678 1679 /* 1680 * If this trap is a result of one of the errors not masked 1681 * by cpu_ce_not_deferred, we don't reenable CEEN. Instead 1682 * indicate that a timeout is to be set later. 1683 */ 1684 if (!(t_afsr_errs & (cpu_ce_not_deferred | cpu_ce_not_deferred_ext)) && 1685 !aflt->flt_panic) 1686 ch_flt.flt_trapped_ce = CE_CEEN_DEFER | CE_CEEN_TRAPPED; 1687 else 1688 ch_flt.flt_trapped_ce = CE_CEEN_NODEFER | CE_CEEN_TRAPPED; 1689 1690 /* 1691 * log the CE and clean up 1692 */ 1693 cpu_log_and_clear_ce(&ch_flt); 1694 1695 /* 1696 * We re-enable CEEN (if required) and check if any disrupting errors 1697 * have happened. We do this because if a disrupting error had occurred 1698 * with CEEN off, the trap will not be taken when CEEN is re-enabled. 1699 * Note that CEEN works differently on Cheetah than on Spitfire. Also, 1700 * we enable CEEN *before* checking the AFSR to avoid the small window 1701 * of a error happening between checking the AFSR and enabling CEEN. 1702 */ 1703 if (ch_flt.flt_trapped_ce & CE_CEEN_NODEFER) 1704 set_error_enable(get_error_enable() | EN_REG_CEEN); 1705 if (clear_errors(&ch_flt)) { 1706 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 1707 NULL); 1708 } 1709 1710 /* 1711 * Panic here if aflt->flt_panic has been set. Enqueued errors will 1712 * be logged as part of the panic flow. 1713 */ 1714 if (aflt->flt_panic) 1715 fm_panic("%sError(s)", pr_reason); 1716 } 1717 1718 /* 1719 * The async_err handler transfers control here for UE, EMU, EDU:BLD, 1720 * L3_EDU:BLD, TO, and BERR events. 1721 * Deferred errors controlled by NCEEN: UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR 1722 * 1723 * Cheetah+: No additional errors handled. 1724 * 1725 * Note that the p_clo_flags input is only valid in cases where the 1726 * cpu_private struct is not yet initialized (since that is the only 1727 * time that information cannot be obtained from the logout struct.) 1728 */ 1729 /*ARGSUSED*/ 1730 void 1731 cpu_deferred_error(struct regs *rp, ulong_t p_clo_flags) 1732 { 1733 ushort_t ttype, tl; 1734 ch_async_flt_t ch_flt; 1735 struct async_flt *aflt; 1736 int trampolined = 0; 1737 char pr_reason[MAX_REASON_STRING]; 1738 ch_cpu_logout_t *clop; 1739 uint64_t ceen, clo_flags; 1740 uint64_t log_afsr; 1741 uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1742 ch_cpu_errors_t cpu_error_regs; 1743 int expected = DDI_FM_ERR_UNEXPECTED; 1744 ddi_acc_hdl_t *hp; 1745 1746 /* 1747 * We need to look at p_flag to determine if the thread detected an 1748 * error while dumping core. We can't grab p_lock here, but it's ok 1749 * because we just need a consistent snapshot and we know that everyone 1750 * else will store a consistent set of bits while holding p_lock. We 1751 * don't have to worry about a race because SDOCORE is set once prior 1752 * to doing i/o from the process's address space and is never cleared. 1753 */ 1754 uint_t pflag = ttoproc(curthread)->p_flag; 1755 1756 bzero(&ch_flt, sizeof (ch_async_flt_t)); 1757 /* 1758 * Get the CPU log out info. If we can't find our CPU private 1759 * pointer then we will have to make due without any detailed 1760 * logout information. 1761 */ 1762 if (CPU_PRIVATE(CPU) == NULL) { 1763 clop = NULL; 1764 ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 1765 get_cpu_error_state(&cpu_error_regs); 1766 set_cpu_error_state(&cpu_error_regs); 1767 t_afar = cpu_error_regs.afar; 1768 t_afsr = cpu_error_regs.afsr; 1769 t_afsr_ext = cpu_error_regs.afsr_ext; 1770 #if defined(SERRANO) 1771 ch_flt.afar2 = cpu_error_regs.afar2; 1772 #endif /* SERRANO */ 1773 clo_flags = p_clo_flags; 1774 } else { 1775 clop = CPU_PRIVATE_PTR(CPU, chpr_async_logout); 1776 t_afar = clop->clo_data.chd_afar; 1777 t_afsr = clop->clo_data.chd_afsr; 1778 t_afsr_ext = clop->clo_data.chd_afsr_ext; 1779 #if defined(SERRANO) 1780 ch_flt.afar2 = clop->clo_data.chd_afar2; 1781 #endif /* SERRANO */ 1782 clo_flags = clop->clo_flags; 1783 } 1784 1785 /* 1786 * In order to simplify code, we maintain this afsr_errs 1787 * variable which holds the aggregate of AFSR and AFSR_EXT 1788 * sticky bits. 1789 */ 1790 t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1791 (t_afsr & C_AFSR_ALL_ERRS); 1792 pr_reason[0] = '\0'; 1793 1794 /* 1795 * Grab information encoded into our clo_flags field. 1796 */ 1797 ceen = clo_flags & EN_REG_CEEN; 1798 tl = (clo_flags & CLO_FLAGS_TL_MASK) >> CLO_FLAGS_TL_SHIFT; 1799 ttype = (clo_flags & CLO_FLAGS_TT_MASK) >> CLO_FLAGS_TT_SHIFT; 1800 1801 /* 1802 * handle the specific error 1803 */ 1804 aflt = (struct async_flt *)&ch_flt; 1805 aflt->flt_id = gethrtime_waitfree(); 1806 aflt->flt_bus_id = getprocessorid(); 1807 aflt->flt_inst = CPU->cpu_id; 1808 ch_flt.afsr_ext = t_afsr_ext; 1809 ch_flt.afsr_errs = t_afsr_errs; 1810 aflt->flt_stat = t_afsr; 1811 aflt->flt_addr = t_afar; 1812 aflt->flt_pc = (caddr_t)rp->r_pc; 1813 aflt->flt_prot = AFLT_PROT_NONE; 1814 aflt->flt_class = CPU_FAULT; 1815 aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 1816 aflt->flt_tl = (uchar_t)tl; 1817 aflt->flt_panic = ((tl != 0) || (aft_testfatal != 0) || 1818 C_AFSR_PANIC(t_afsr_errs)); 1819 aflt->flt_core = (pflag & SDOCORE) ? 1 : 0; 1820 aflt->flt_status = ((ttype == T_DATA_ERROR) ? ECC_D_TRAP : ECC_I_TRAP); 1821 1822 /* 1823 * If the trap occurred in privileged mode at TL=0, we need to check to 1824 * see if we were executing in the kernel under on_trap() or t_lofault 1825 * protection. If so, modify the saved registers so that we return 1826 * from the trap to the appropriate trampoline routine. 1827 */ 1828 if (aflt->flt_priv && tl == 0) { 1829 if (curthread->t_ontrap != NULL) { 1830 on_trap_data_t *otp = curthread->t_ontrap; 1831 1832 if (otp->ot_prot & OT_DATA_EC) { 1833 aflt->flt_prot = AFLT_PROT_EC; 1834 otp->ot_trap |= OT_DATA_EC; 1835 rp->r_pc = otp->ot_trampoline; 1836 rp->r_npc = rp->r_pc + 4; 1837 trampolined = 1; 1838 } 1839 1840 if ((t_afsr & (C_AFSR_TO | C_AFSR_BERR)) && 1841 (otp->ot_prot & OT_DATA_ACCESS)) { 1842 aflt->flt_prot = AFLT_PROT_ACCESS; 1843 otp->ot_trap |= OT_DATA_ACCESS; 1844 rp->r_pc = otp->ot_trampoline; 1845 rp->r_npc = rp->r_pc + 4; 1846 trampolined = 1; 1847 /* 1848 * for peeks and caut_gets errors are expected 1849 */ 1850 hp = (ddi_acc_hdl_t *)otp->ot_handle; 1851 if (!hp) 1852 expected = DDI_FM_ERR_PEEK; 1853 else if (hp->ah_acc.devacc_attr_access == 1854 DDI_CAUTIOUS_ACC) 1855 expected = DDI_FM_ERR_EXPECTED; 1856 } 1857 1858 } else if (curthread->t_lofault) { 1859 aflt->flt_prot = AFLT_PROT_COPY; 1860 rp->r_g1 = EFAULT; 1861 rp->r_pc = curthread->t_lofault; 1862 rp->r_npc = rp->r_pc + 4; 1863 trampolined = 1; 1864 } 1865 } 1866 1867 /* 1868 * If we're in user mode or we're doing a protected copy, we either 1869 * want the ASTON code below to send a signal to the user process 1870 * or we want to panic if aft_panic is set. 1871 * 1872 * If we're in privileged mode and we're not doing a copy, then we 1873 * need to check if we've trampolined. If we haven't trampolined, 1874 * we should panic. 1875 */ 1876 if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 1877 if (t_afsr_errs & 1878 ((C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS) & 1879 ~(C_AFSR_BERR | C_AFSR_TO))) 1880 aflt->flt_panic |= aft_panic; 1881 } else if (!trampolined) { 1882 aflt->flt_panic = 1; 1883 } 1884 1885 /* 1886 * If we've trampolined due to a privileged TO or BERR, or if an 1887 * unprivileged TO or BERR occurred, we don't want to enqueue an 1888 * event for that TO or BERR. Queue all other events (if any) besides 1889 * the TO/BERR. Since we may not be enqueing any events, we need to 1890 * ignore the number of events queued. If we haven't trampolined due 1891 * to a TO or BERR, just enqueue events normally. 1892 */ 1893 log_afsr = t_afsr_errs; 1894 if (trampolined) { 1895 log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 1896 } else if (!aflt->flt_priv) { 1897 /* 1898 * User mode, suppress messages if 1899 * cpu_berr_to_verbose is not set. 1900 */ 1901 if (!cpu_berr_to_verbose) 1902 log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 1903 } 1904 1905 /* 1906 * Log any errors that occurred 1907 */ 1908 if (((log_afsr & 1909 ((C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS) & ~C_AFSR_ME)) && 1910 cpu_queue_events(&ch_flt, pr_reason, log_afsr, clop) == 0) || 1911 (t_afsr_errs & 1912 (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) == 0) { 1913 ch_flt.flt_type = CPU_INV_AFSR; 1914 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 1915 (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 1916 aflt->flt_panic); 1917 } 1918 1919 /* 1920 * Zero out + invalidate CPU logout. 1921 */ 1922 if (clop) { 1923 bzero(clop, sizeof (ch_cpu_logout_t)); 1924 clop->clo_data.chd_afar = LOGOUT_INVALID; 1925 } 1926 1927 #if defined(JALAPENO) || defined(SERRANO) 1928 /* 1929 * UE/RUE/BERR/TO: Call our bus nexus friends to check for 1930 * IO errors that may have resulted in this trap. 1931 */ 1932 if (t_afsr & (C_AFSR_UE|C_AFSR_RUE|C_AFSR_TO|C_AFSR_BERR)) { 1933 cpu_run_bus_error_handlers(aflt, expected); 1934 } 1935 1936 /* 1937 * UE/RUE: If UE or RUE is in memory, we need to flush the bad 1938 * line from the Ecache. We also need to query the bus nexus for 1939 * fatal errors. Attempts to do diagnostic read on caches may 1940 * introduce more errors (especially when the module is bad). 1941 */ 1942 if (t_afsr & (C_AFSR_UE|C_AFSR_RUE)) { 1943 /* 1944 * Ask our bus nexus friends if they have any fatal errors. If 1945 * so, they will log appropriate error messages. 1946 */ 1947 if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 1948 aflt->flt_panic = 1; 1949 1950 /* 1951 * We got a UE or RUE and are panicking, save the fault PA in 1952 * a known location so that the platform specific panic code 1953 * can check for copyback errors. 1954 */ 1955 if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 1956 panic_aflt = *aflt; 1957 } 1958 } 1959 1960 /* 1961 * Flush Ecache line or entire Ecache 1962 */ 1963 if (t_afsr & (C_AFSR_UE | C_AFSR_RUE | C_AFSR_EDU | C_AFSR_BERR)) 1964 cpu_error_ecache_flush(&ch_flt); 1965 #else /* JALAPENO || SERRANO */ 1966 /* 1967 * UE/BERR/TO: Call our bus nexus friends to check for 1968 * IO errors that may have resulted in this trap. 1969 */ 1970 if (t_afsr & (C_AFSR_UE|C_AFSR_TO|C_AFSR_BERR)) { 1971 cpu_run_bus_error_handlers(aflt, expected); 1972 } 1973 1974 /* 1975 * UE: If the UE is in memory, we need to flush the bad 1976 * line from the Ecache. We also need to query the bus nexus for 1977 * fatal errors. Attempts to do diagnostic read on caches may 1978 * introduce more errors (especially when the module is bad). 1979 */ 1980 if (t_afsr & C_AFSR_UE) { 1981 /* 1982 * Ask our legacy bus nexus friends if they have any fatal 1983 * errors. If so, they will log appropriate error messages. 1984 */ 1985 if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 1986 aflt->flt_panic = 1; 1987 1988 /* 1989 * We got a UE and are panicking, save the fault PA in a known 1990 * location so that the platform specific panic code can check 1991 * for copyback errors. 1992 */ 1993 if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 1994 panic_aflt = *aflt; 1995 } 1996 } 1997 1998 /* 1999 * Flush Ecache line or entire Ecache 2000 */ 2001 if (t_afsr_errs & 2002 (C_AFSR_UE | C_AFSR_EDU | C_AFSR_BERR | C_AFSR_L3_EDU)) 2003 cpu_error_ecache_flush(&ch_flt); 2004 #endif /* JALAPENO || SERRANO */ 2005 2006 /* 2007 * We carefully re-enable NCEEN and CEEN and then check if any deferred 2008 * or disrupting errors have happened. We do this because if a 2009 * deferred or disrupting error had occurred with NCEEN/CEEN off, the 2010 * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 2011 * CEEN works differently on Cheetah than on Spitfire. Also, we enable 2012 * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 2013 * deferred or disrupting error happening between checking the AFSR and 2014 * enabling NCEEN/CEEN. 2015 * 2016 * Note: CEEN reenabled only if it was on when trap taken. 2017 */ 2018 set_error_enable(get_error_enable() | (EN_REG_NCEEN | ceen)); 2019 if (clear_errors(&ch_flt)) { 2020 /* 2021 * Check for secondary errors, and avoid panicking if we 2022 * have them 2023 */ 2024 if (cpu_check_secondary_errors(&ch_flt, t_afsr_errs, 2025 t_afar) == 0) { 2026 aflt->flt_panic |= ((ch_flt.afsr_errs & 2027 (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) != 0); 2028 } 2029 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 2030 NULL); 2031 } 2032 2033 /* 2034 * Panic here if aflt->flt_panic has been set. Enqueued errors will 2035 * be logged as part of the panic flow. 2036 */ 2037 if (aflt->flt_panic) 2038 fm_panic("%sError(s)", pr_reason); 2039 2040 /* 2041 * If we queued an error and we are going to return from the trap and 2042 * the error was in user mode or inside of a copy routine, set AST flag 2043 * so the queue will be drained before returning to user mode. The 2044 * AST processing will also act on our failure policy. 2045 */ 2046 if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 2047 int pcb_flag = 0; 2048 2049 if (t_afsr_errs & 2050 (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS & 2051 ~(C_AFSR_BERR | C_AFSR_TO))) 2052 pcb_flag |= ASYNC_HWERR; 2053 2054 if (t_afsr & C_AFSR_BERR) 2055 pcb_flag |= ASYNC_BERR; 2056 2057 if (t_afsr & C_AFSR_TO) 2058 pcb_flag |= ASYNC_BTO; 2059 2060 ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag; 2061 aston(curthread); 2062 } 2063 } 2064 2065 #if defined(CPU_IMP_L1_CACHE_PARITY) 2066 /* 2067 * Handling of data and instruction parity errors (traps 0x71, 0x72). 2068 * 2069 * For Panther, P$ data parity errors during floating point load hits 2070 * are also detected (reported as TT 0x71) and handled by this trap 2071 * handler. 2072 * 2073 * AFSR/AFAR are not set for parity errors, only TPC (a virtual address) 2074 * is available. 2075 */ 2076 /*ARGSUSED*/ 2077 void 2078 cpu_parity_error(struct regs *rp, uint_t flags, caddr_t tpc) 2079 { 2080 ch_async_flt_t ch_flt; 2081 struct async_flt *aflt; 2082 uchar_t tl = ((flags & CH_ERR_TL) != 0); 2083 uchar_t iparity = ((flags & CH_ERR_IPE) != 0); 2084 uchar_t panic = ((flags & CH_ERR_PANIC) != 0); 2085 char *error_class; 2086 2087 /* 2088 * Log the error. 2089 * For icache parity errors the fault address is the trap PC. 2090 * For dcache/pcache parity errors the instruction would have to 2091 * be decoded to determine the address and that isn't possible 2092 * at high PIL. 2093 */ 2094 bzero(&ch_flt, sizeof (ch_async_flt_t)); 2095 aflt = (struct async_flt *)&ch_flt; 2096 aflt->flt_id = gethrtime_waitfree(); 2097 aflt->flt_bus_id = getprocessorid(); 2098 aflt->flt_inst = CPU->cpu_id; 2099 aflt->flt_pc = tpc; 2100 aflt->flt_addr = iparity ? (uint64_t)tpc : AFLT_INV_ADDR; 2101 aflt->flt_prot = AFLT_PROT_NONE; 2102 aflt->flt_class = CPU_FAULT; 2103 aflt->flt_priv = (tl || (rp->r_tstate & TSTATE_PRIV)) ? 1 : 0; 2104 aflt->flt_tl = tl; 2105 aflt->flt_panic = panic; 2106 aflt->flt_status = iparity ? ECC_IP_TRAP : ECC_DP_TRAP; 2107 ch_flt.flt_type = iparity ? CPU_IC_PARITY : CPU_DC_PARITY; 2108 2109 if (iparity) { 2110 cpu_icache_parity_info(&ch_flt); 2111 if (ch_flt.parity_data.ipe.cpl_off != -1) 2112 error_class = FM_EREPORT_CPU_USIII_IDSPE; 2113 else if (ch_flt.parity_data.ipe.cpl_way != -1) 2114 error_class = FM_EREPORT_CPU_USIII_ITSPE; 2115 else 2116 error_class = FM_EREPORT_CPU_USIII_IPE; 2117 aflt->flt_payload = FM_EREPORT_PAYLOAD_ICACHE_PE; 2118 } else { 2119 cpu_dcache_parity_info(&ch_flt); 2120 if (ch_flt.parity_data.dpe.cpl_off != -1) 2121 error_class = FM_EREPORT_CPU_USIII_DDSPE; 2122 else if (ch_flt.parity_data.dpe.cpl_way != -1) 2123 error_class = FM_EREPORT_CPU_USIII_DTSPE; 2124 else 2125 error_class = FM_EREPORT_CPU_USIII_DPE; 2126 aflt->flt_payload = FM_EREPORT_PAYLOAD_DCACHE_PE; 2127 /* 2128 * For panther we also need to check the P$ for parity errors. 2129 */ 2130 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 2131 cpu_pcache_parity_info(&ch_flt); 2132 if (ch_flt.parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 2133 error_class = FM_EREPORT_CPU_USIII_PDSPE; 2134 aflt->flt_payload = 2135 FM_EREPORT_PAYLOAD_PCACHE_PE; 2136 } 2137 } 2138 } 2139 2140 cpu_errorq_dispatch(error_class, (void *)&ch_flt, 2141 sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 2142 2143 if (iparity) { 2144 /* 2145 * Invalidate entire I$. 2146 * This is required due to the use of diagnostic ASI 2147 * accesses that may result in a loss of I$ coherency. 2148 */ 2149 if (cache_boot_state & DCU_IC) { 2150 flush_icache(); 2151 } 2152 /* 2153 * According to section P.3.1 of the Panther PRM, we 2154 * need to do a little more for recovery on those 2155 * CPUs after encountering an I$ parity error. 2156 */ 2157 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 2158 flush_ipb(); 2159 correct_dcache_parity(dcache_size, 2160 dcache_linesize); 2161 flush_pcache(); 2162 } 2163 } else { 2164 /* 2165 * Since the valid bit is ignored when checking parity the 2166 * D$ data and tag must also be corrected. Set D$ data bits 2167 * to zero and set utag to 0, 1, 2, 3. 2168 */ 2169 correct_dcache_parity(dcache_size, dcache_linesize); 2170 2171 /* 2172 * According to section P.3.3 of the Panther PRM, we 2173 * need to do a little more for recovery on those 2174 * CPUs after encountering a D$ or P$ parity error. 2175 * 2176 * As far as clearing P$ parity errors, it is enough to 2177 * simply invalidate all entries in the P$ since P$ parity 2178 * error traps are only generated for floating point load 2179 * hits. 2180 */ 2181 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 2182 flush_icache(); 2183 flush_ipb(); 2184 flush_pcache(); 2185 } 2186 } 2187 2188 /* 2189 * Invalidate entire D$ if it was enabled. 2190 * This is done to avoid stale data in the D$ which might 2191 * occur with the D$ disabled and the trap handler doing 2192 * stores affecting lines already in the D$. 2193 */ 2194 if (cache_boot_state & DCU_DC) { 2195 flush_dcache(); 2196 } 2197 2198 /* 2199 * Restore caches to their bootup state. 2200 */ 2201 set_dcu(get_dcu() | cache_boot_state); 2202 2203 /* 2204 * Panic here if aflt->flt_panic has been set. Enqueued errors will 2205 * be logged as part of the panic flow. 2206 */ 2207 if (aflt->flt_panic) 2208 fm_panic("%sError(s)", iparity ? "IPE " : "DPE "); 2209 2210 /* 2211 * If this error occurred at TL>0 then flush the E$ here to reduce 2212 * the chance of getting an unrecoverable Fast ECC error. This 2213 * flush will evict the part of the parity trap handler that is run 2214 * at TL>1. 2215 */ 2216 if (tl) { 2217 cpu_flush_ecache(); 2218 } 2219 } 2220 2221 /* 2222 * On an I$ parity error, mark the appropriate entries in the ch_async_flt_t 2223 * to indicate which portions of the captured data should be in the ereport. 2224 */ 2225 void 2226 cpu_async_log_ic_parity_err(ch_async_flt_t *ch_flt) 2227 { 2228 int way = ch_flt->parity_data.ipe.cpl_way; 2229 int offset = ch_flt->parity_data.ipe.cpl_off; 2230 int tag_index; 2231 struct async_flt *aflt = (struct async_flt *)ch_flt; 2232 2233 2234 if ((offset != -1) || (way != -1)) { 2235 /* 2236 * Parity error in I$ tag or data 2237 */ 2238 tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 2239 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 2240 ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 2241 PN_ICIDX_TO_WAY(tag_index); 2242 else 2243 ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 2244 CH_ICIDX_TO_WAY(tag_index); 2245 ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 2246 IC_LOGFLAG_MAGIC; 2247 } else { 2248 /* 2249 * Parity error was not identified. 2250 * Log tags and data for all ways. 2251 */ 2252 for (way = 0; way < CH_ICACHE_NWAY; way++) { 2253 tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 2254 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 2255 ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 2256 PN_ICIDX_TO_WAY(tag_index); 2257 else 2258 ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 2259 CH_ICIDX_TO_WAY(tag_index); 2260 ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 2261 IC_LOGFLAG_MAGIC; 2262 } 2263 } 2264 } 2265 2266 /* 2267 * On an D$ parity error, mark the appropriate entries in the ch_async_flt_t 2268 * to indicate which portions of the captured data should be in the ereport. 2269 */ 2270 void 2271 cpu_async_log_dc_parity_err(ch_async_flt_t *ch_flt) 2272 { 2273 int way = ch_flt->parity_data.dpe.cpl_way; 2274 int offset = ch_flt->parity_data.dpe.cpl_off; 2275 int tag_index; 2276 2277 if (offset != -1) { 2278 /* 2279 * Parity error in D$ or P$ data array. 2280 * 2281 * First check to see whether the parity error is in D$ or P$ 2282 * since P$ data parity errors are reported in Panther using 2283 * the same trap. 2284 */ 2285 if (ch_flt->parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 2286 tag_index = ch_flt->parity_data.dpe.cpl_pc[way].pc_idx; 2287 ch_flt->parity_data.dpe.cpl_pc[way].pc_way = 2288 CH_PCIDX_TO_WAY(tag_index); 2289 ch_flt->parity_data.dpe.cpl_pc[way].pc_logflag = 2290 PC_LOGFLAG_MAGIC; 2291 } else { 2292 tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 2293 ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 2294 CH_DCIDX_TO_WAY(tag_index); 2295 ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 2296 DC_LOGFLAG_MAGIC; 2297 } 2298 } else if (way != -1) { 2299 /* 2300 * Parity error in D$ tag. 2301 */ 2302 tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 2303 ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 2304 CH_DCIDX_TO_WAY(tag_index); 2305 ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 2306 DC_LOGFLAG_MAGIC; 2307 } 2308 } 2309 #endif /* CPU_IMP_L1_CACHE_PARITY */ 2310 2311 /* 2312 * The cpu_async_log_err() function is called via the [uc]e_drain() function to 2313 * post-process CPU events that are dequeued. As such, it can be invoked 2314 * from softint context, from AST processing in the trap() flow, or from the 2315 * panic flow. We decode the CPU-specific data, and take appropriate actions. 2316 * Historically this entry point was used to log the actual cmn_err(9F) text; 2317 * now with FMA it is used to prepare 'flt' to be converted into an ereport. 2318 * With FMA this function now also returns a flag which indicates to the 2319 * caller whether the ereport should be posted (1) or suppressed (0). 2320 */ 2321 static int 2322 cpu_async_log_err(void *flt, errorq_elem_t *eqep) 2323 { 2324 ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt; 2325 struct async_flt *aflt = (struct async_flt *)flt; 2326 uint64_t errors; 2327 2328 switch (ch_flt->flt_type) { 2329 case CPU_INV_AFSR: 2330 /* 2331 * If it is a disrupting trap and the AFSR is zero, then 2332 * the event has probably already been noted. Do not post 2333 * an ereport. 2334 */ 2335 if ((aflt->flt_status & ECC_C_TRAP) && 2336 (!(aflt->flt_stat & C_AFSR_MASK))) 2337 return (0); 2338 else 2339 return (1); 2340 case CPU_TO: 2341 case CPU_BERR: 2342 case CPU_FATAL: 2343 case CPU_FPUERR: 2344 return (1); 2345 2346 case CPU_UE_ECACHE_RETIRE: 2347 cpu_log_err(aflt); 2348 cpu_page_retire(ch_flt); 2349 return (1); 2350 2351 /* 2352 * Cases where we may want to suppress logging or perform 2353 * extended diagnostics. 2354 */ 2355 case CPU_CE: 2356 case CPU_EMC: 2357 /* 2358 * We want to skip logging and further classification 2359 * only if ALL the following conditions are true: 2360 * 2361 * 1. There is only one error 2362 * 2. That error is a correctable memory error 2363 * 3. The error is caused by the memory scrubber (in 2364 * which case the error will have occurred under 2365 * on_trap protection) 2366 * 4. The error is on a retired page 2367 * 2368 * Note: AFLT_PROT_EC is used places other than the memory 2369 * scrubber. However, none of those errors should occur 2370 * on a retired page. 2371 */ 2372 if ((ch_flt->afsr_errs & 2373 (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_CE && 2374 aflt->flt_prot == AFLT_PROT_EC) { 2375 2376 if (page_retire_check(aflt->flt_addr, NULL) == 0) { 2377 if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 2378 2379 /* 2380 * Since we're skipping logging, we'll need 2381 * to schedule the re-enabling of CEEN 2382 */ 2383 (void) timeout(cpu_delayed_check_ce_errors, 2384 (void *)(uintptr_t)aflt->flt_inst, 2385 drv_usectohz((clock_t)cpu_ceen_delay_secs 2386 * MICROSEC)); 2387 } 2388 return (0); 2389 } 2390 } 2391 2392 /* 2393 * Perform/schedule further classification actions, but 2394 * only if the page is healthy (we don't want bad 2395 * pages inducing too much diagnostic activity). If we could 2396 * not find a page pointer then we also skip this. If 2397 * ce_scrub_xdiag_recirc returns nonzero then it has chosen 2398 * to copy and recirculate the event (for further diagnostics) 2399 * and we should not proceed to log it here. 2400 * 2401 * This must be the last step here before the cpu_log_err() 2402 * below - if an event recirculates cpu_ce_log_err() will 2403 * not call the current function but just proceed directly 2404 * to cpu_ereport_post after the cpu_log_err() avoided below. 2405 * 2406 * Note: Check cpu_impl_async_log_err if changing this 2407 */ 2408 if (page_retire_check(aflt->flt_addr, &errors) == EINVAL) { 2409 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 2410 CE_XDIAG_SKIP_NOPP); 2411 } else { 2412 if (errors != PR_OK) { 2413 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 2414 CE_XDIAG_SKIP_PAGEDET); 2415 } else if (ce_scrub_xdiag_recirc(aflt, ce_queue, eqep, 2416 offsetof(ch_async_flt_t, cmn_asyncflt))) { 2417 return (0); 2418 } 2419 } 2420 /*FALLTHRU*/ 2421 2422 /* 2423 * Cases where we just want to report the error and continue. 2424 */ 2425 case CPU_CE_ECACHE: 2426 case CPU_UE_ECACHE: 2427 case CPU_IV: 2428 case CPU_ORPH: 2429 cpu_log_err(aflt); 2430 return (1); 2431 2432 /* 2433 * Cases where we want to fall through to handle panicking. 2434 */ 2435 case CPU_UE: 2436 /* 2437 * We want to skip logging in the same conditions as the 2438 * CE case. In addition, we want to make sure we're not 2439 * panicking. 2440 */ 2441 if (!panicstr && (ch_flt->afsr_errs & 2442 (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_UE && 2443 aflt->flt_prot == AFLT_PROT_EC) { 2444 if (page_retire_check(aflt->flt_addr, NULL) == 0) { 2445 /* Zero the address to clear the error */ 2446 softcall(ecc_page_zero, (void *)aflt->flt_addr); 2447 return (0); 2448 } 2449 } 2450 cpu_log_err(aflt); 2451 break; 2452 2453 default: 2454 /* 2455 * If the us3_common.c code doesn't know the flt_type, it may 2456 * be an implementation-specific code. Call into the impldep 2457 * backend to find out what to do: if it tells us to continue, 2458 * break and handle as if falling through from a UE; if not, 2459 * the impldep backend has handled the error and we're done. 2460 */ 2461 switch (cpu_impl_async_log_err(flt, eqep)) { 2462 case CH_ASYNC_LOG_DONE: 2463 return (1); 2464 case CH_ASYNC_LOG_RECIRC: 2465 return (0); 2466 case CH_ASYNC_LOG_CONTINUE: 2467 break; /* continue on to handle UE-like error */ 2468 default: 2469 cmn_err(CE_WARN, "discarding error 0x%p with " 2470 "invalid fault type (0x%x)", 2471 (void *)aflt, ch_flt->flt_type); 2472 return (0); 2473 } 2474 } 2475 2476 /* ... fall through from the UE case */ 2477 2478 if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) { 2479 if (!panicstr) { 2480 cpu_page_retire(ch_flt); 2481 } else { 2482 /* 2483 * Clear UEs on panic so that we don't 2484 * get haunted by them during panic or 2485 * after reboot 2486 */ 2487 cpu_clearphys(aflt); 2488 (void) clear_errors(NULL); 2489 } 2490 } 2491 2492 return (1); 2493 } 2494 2495 /* 2496 * Retire the bad page that may contain the flushed error. 2497 */ 2498 void 2499 cpu_page_retire(ch_async_flt_t *ch_flt) 2500 { 2501 struct async_flt *aflt = (struct async_flt *)ch_flt; 2502 (void) page_retire(aflt->flt_addr, PR_UE); 2503 } 2504 2505 /* 2506 * The cpu_log_err() function is called by cpu_async_log_err() to perform the 2507 * generic event post-processing for correctable and uncorrectable memory, 2508 * E$, and MTag errors. Historically this entry point was used to log bits of 2509 * common cmn_err(9F) text; now with FMA it is used to prepare 'flt' to be 2510 * converted into an ereport. In addition, it transmits the error to any 2511 * platform-specific service-processor FRU logging routines, if available. 2512 */ 2513 void 2514 cpu_log_err(struct async_flt *aflt) 2515 { 2516 char unum[UNUM_NAMLEN]; 2517 int len = 0; 2518 int synd_status, synd_code, afar_status; 2519 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 2520 2521 /* 2522 * Need to turn on ECC_ECACHE for plat_get_mem_unum(). 2523 * For Panther, L2$ is not external, so we don't want to 2524 * generate an E$ unum for those errors. 2525 */ 2526 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 2527 if (ch_flt->flt_bit & C_AFSR_EXT_L3_ERRS) 2528 aflt->flt_status |= ECC_ECACHE; 2529 } else { 2530 if (ch_flt->flt_bit & C_AFSR_ECACHE) 2531 aflt->flt_status |= ECC_ECACHE; 2532 } 2533 2534 /* 2535 * Determine syndrome status. 2536 */ 2537 synd_status = afsr_to_synd_status(aflt->flt_inst, 2538 ch_flt->afsr_errs, ch_flt->flt_bit); 2539 2540 /* 2541 * Determine afar status. 2542 */ 2543 if (pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)) 2544 afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 2545 ch_flt->flt_bit); 2546 else 2547 afar_status = AFLT_STAT_INVALID; 2548 2549 /* 2550 * If afar status is not invalid do a unum lookup. 2551 */ 2552 if (afar_status != AFLT_STAT_INVALID) { 2553 (void) cpu_get_mem_unum_aflt(synd_status, aflt, unum, 2554 UNUM_NAMLEN, &len); 2555 } else { 2556 unum[0] = '\0'; 2557 } 2558 2559 synd_code = synd_to_synd_code(synd_status, 2560 aflt->flt_synd, ch_flt->flt_bit); 2561 2562 /* 2563 * Do not send the fruid message (plat_ecc_error_data_t) 2564 * to the SC if it can handle the enhanced error information 2565 * (plat_ecc_error2_data_t) or when the tunable 2566 * ecc_log_fruid_enable is set to 0. 2567 */ 2568 2569 if (&plat_ecc_capability_sc_get && 2570 plat_ecc_capability_sc_get(PLAT_ECC_ERROR_MESSAGE)) { 2571 if (&plat_log_fruid_error) 2572 plat_log_fruid_error(synd_code, aflt, unum, 2573 ch_flt->flt_bit); 2574 } 2575 2576 if (aflt->flt_func != NULL) 2577 aflt->flt_func(aflt, unum); 2578 2579 if (afar_status != AFLT_STAT_INVALID) 2580 cpu_log_diag_info(ch_flt); 2581 2582 /* 2583 * If we have a CEEN error , we do not reenable CEEN until after 2584 * we exit the trap handler. Otherwise, another error may 2585 * occur causing the handler to be entered recursively. 2586 * We set a timeout to trigger in cpu_ceen_delay_secs seconds, 2587 * to try and ensure that the CPU makes progress in the face 2588 * of a CE storm. 2589 */ 2590 if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 2591 (void) timeout(cpu_delayed_check_ce_errors, 2592 (void *)(uintptr_t)aflt->flt_inst, 2593 drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 2594 } 2595 } 2596 2597 /* 2598 * Invoked by error_init() early in startup and therefore before 2599 * startup_errorq() is called to drain any error Q - 2600 * 2601 * startup() 2602 * startup_end() 2603 * error_init() 2604 * cpu_error_init() 2605 * errorq_init() 2606 * errorq_drain() 2607 * start_other_cpus() 2608 * 2609 * The purpose of this routine is to create error-related taskqs. Taskqs 2610 * are used for this purpose because cpu_lock can't be grabbed from interrupt 2611 * context. 2612 */ 2613 void 2614 cpu_error_init(int items) 2615 { 2616 /* 2617 * Create taskq(s) to reenable CE 2618 */ 2619 ch_check_ce_tq = taskq_create("cheetah_check_ce", 1, minclsyspri, 2620 items, items, TASKQ_PREPOPULATE); 2621 } 2622 2623 void 2624 cpu_ce_log_err(struct async_flt *aflt, errorq_elem_t *eqep) 2625 { 2626 char unum[UNUM_NAMLEN]; 2627 int len; 2628 2629 switch (aflt->flt_class) { 2630 case CPU_FAULT: 2631 cpu_ereport_init(aflt); 2632 if (cpu_async_log_err(aflt, eqep)) 2633 cpu_ereport_post(aflt); 2634 break; 2635 2636 case BUS_FAULT: 2637 if (aflt->flt_func != NULL) { 2638 (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, 2639 unum, UNUM_NAMLEN, &len); 2640 aflt->flt_func(aflt, unum); 2641 } 2642 break; 2643 2644 case RECIRC_CPU_FAULT: 2645 aflt->flt_class = CPU_FAULT; 2646 cpu_log_err(aflt); 2647 cpu_ereport_post(aflt); 2648 break; 2649 2650 case RECIRC_BUS_FAULT: 2651 ASSERT(aflt->flt_class != RECIRC_BUS_FAULT); 2652 /*FALLTHRU*/ 2653 default: 2654 cmn_err(CE_WARN, "discarding CE error 0x%p with invalid " 2655 "fault class (0x%x)", (void *)aflt, aflt->flt_class); 2656 return; 2657 } 2658 } 2659 2660 /* 2661 * Scrub and classify a CE. This function must not modify the 2662 * fault structure passed to it but instead should return the classification 2663 * information. 2664 */ 2665 2666 static uchar_t 2667 cpu_ce_scrub_mem_err_common(struct async_flt *ecc, boolean_t logout_tried) 2668 { 2669 uchar_t disp = CE_XDIAG_EXTALG; 2670 on_trap_data_t otd; 2671 uint64_t orig_err; 2672 ch_cpu_logout_t *clop; 2673 2674 /* 2675 * Clear CEEN. CPU CE TL > 0 trap handling will already have done 2676 * this, but our other callers have not. Disable preemption to 2677 * avoid CPU migration so that we restore CEEN on the correct 2678 * cpu later. 2679 * 2680 * CEEN is cleared so that further CEs that our instruction and 2681 * data footprint induce do not cause use to either creep down 2682 * kernel stack to the point of overflow, or do so much CE 2683 * notification as to make little real forward progress. 2684 * 2685 * NCEEN must not be cleared. However it is possible that 2686 * our accesses to the flt_addr may provoke a bus error or timeout 2687 * if the offending address has just been unconfigured as part of 2688 * a DR action. So we must operate under on_trap protection. 2689 */ 2690 kpreempt_disable(); 2691 orig_err = get_error_enable(); 2692 if (orig_err & EN_REG_CEEN) 2693 set_error_enable(orig_err & ~EN_REG_CEEN); 2694 2695 /* 2696 * Our classification algorithm includes the line state before 2697 * the scrub; we'd like this captured after the detection and 2698 * before the algorithm below - the earlier the better. 2699 * 2700 * If we've come from a cpu CE trap then this info already exists 2701 * in the cpu logout area. 2702 * 2703 * For a CE detected by memscrub for which there was no trap 2704 * (running with CEEN off) cpu_log_and_clear_ce has called 2705 * cpu_ce_delayed_ec_logout to capture some cache data, and 2706 * marked the fault structure as incomplete as a flag to later 2707 * logging code. 2708 * 2709 * If called directly from an IO detected CE there has been 2710 * no line data capture. In this case we logout to the cpu logout 2711 * area - that's appropriate since it's the cpu cache data we need 2712 * for classification. We thus borrow the cpu logout area for a 2713 * short time, and cpu_ce_delayed_ec_logout will mark it as busy in 2714 * this time (we will invalidate it again below). 2715 * 2716 * If called from the partner check xcall handler then this cpu 2717 * (the partner) has not necessarily experienced a CE at this 2718 * address. But we want to capture line state before its scrub 2719 * attempt since we use that in our classification. 2720 */ 2721 if (logout_tried == B_FALSE) { 2722 if (!cpu_ce_delayed_ec_logout(ecc->flt_addr)) 2723 disp |= CE_XDIAG_NOLOGOUT; 2724 } 2725 2726 /* 2727 * Scrub memory, then check AFSR for errors. The AFAR we scrub may 2728 * no longer be valid (if DR'd since the initial event) so we 2729 * perform this scrub under on_trap protection. If this access is 2730 * ok then further accesses below will also be ok - DR cannot 2731 * proceed while this thread is active (preemption is disabled); 2732 * to be safe we'll nonetheless use on_trap again below. 2733 */ 2734 if (!on_trap(&otd, OT_DATA_ACCESS)) { 2735 cpu_scrubphys(ecc); 2736 } else { 2737 no_trap(); 2738 if (orig_err & EN_REG_CEEN) 2739 set_error_enable(orig_err); 2740 kpreempt_enable(); 2741 return (disp); 2742 } 2743 no_trap(); 2744 2745 /* 2746 * Did the casx read of the scrub log a CE that matches the AFAR? 2747 * Note that it's quite possible that the read sourced the data from 2748 * another cpu. 2749 */ 2750 if (clear_ecc(ecc)) 2751 disp |= CE_XDIAG_CE1; 2752 2753 /* 2754 * Read the data again. This time the read is very likely to 2755 * come from memory since the scrub induced a writeback to memory. 2756 */ 2757 if (!on_trap(&otd, OT_DATA_ACCESS)) { 2758 (void) lddphys(P2ALIGN(ecc->flt_addr, 8)); 2759 } else { 2760 no_trap(); 2761 if (orig_err & EN_REG_CEEN) 2762 set_error_enable(orig_err); 2763 kpreempt_enable(); 2764 return (disp); 2765 } 2766 no_trap(); 2767 2768 /* Did that read induce a CE that matches the AFAR? */ 2769 if (clear_ecc(ecc)) 2770 disp |= CE_XDIAG_CE2; 2771 2772 /* 2773 * Look at the logout information and record whether we found the 2774 * line in l2/l3 cache. For Panther we are interested in whether 2775 * we found it in either cache (it won't reside in both but 2776 * it is possible to read it that way given the moving target). 2777 */ 2778 clop = CPU_PRIVATE(CPU) ? CPU_PRIVATE_PTR(CPU, chpr_cecc_logout) : NULL; 2779 if (!(disp & CE_XDIAG_NOLOGOUT) && clop && 2780 clop->clo_data.chd_afar != LOGOUT_INVALID) { 2781 int hit, level; 2782 int state; 2783 int totalsize; 2784 ch_ec_data_t *ecp; 2785 2786 /* 2787 * If hit is nonzero then a match was found and hit will 2788 * be one greater than the index which hit. For Panther we 2789 * also need to pay attention to level to see which of l2$ or 2790 * l3$ it hit in. 2791 */ 2792 hit = cpu_matching_ecache_line(ecc->flt_addr, &clop->clo_data, 2793 0, &level); 2794 2795 if (hit) { 2796 --hit; 2797 disp |= CE_XDIAG_AFARMATCH; 2798 2799 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 2800 if (level == 2) 2801 ecp = &clop->clo_data.chd_l2_data[hit]; 2802 else 2803 ecp = &clop->clo_data.chd_ec_data[hit]; 2804 } else { 2805 ASSERT(level == 2); 2806 ecp = &clop->clo_data.chd_ec_data[hit]; 2807 } 2808 totalsize = cpunodes[CPU->cpu_id].ecache_size; 2809 state = cpu_ectag_pa_to_subblk_state(totalsize, 2810 ecc->flt_addr, ecp->ec_tag); 2811 2812 /* 2813 * Cheetah variants use different state encodings - 2814 * the CH_ECSTATE_* defines vary depending on the 2815 * module we're compiled for. Translate into our 2816 * one true version. Conflate Owner-Shared state 2817 * of SSM mode with Owner as victimisation of such 2818 * lines may cause a writeback. 2819 */ 2820 switch (state) { 2821 case CH_ECSTATE_MOD: 2822 disp |= EC_STATE_M; 2823 break; 2824 2825 case CH_ECSTATE_OWN: 2826 case CH_ECSTATE_OWS: 2827 disp |= EC_STATE_O; 2828 break; 2829 2830 case CH_ECSTATE_EXL: 2831 disp |= EC_STATE_E; 2832 break; 2833 2834 case CH_ECSTATE_SHR: 2835 disp |= EC_STATE_S; 2836 break; 2837 2838 default: 2839 disp |= EC_STATE_I; 2840 break; 2841 } 2842 } 2843 2844 /* 2845 * If we initiated the delayed logout then we are responsible 2846 * for invalidating the logout area. 2847 */ 2848 if (logout_tried == B_FALSE) { 2849 bzero(clop, sizeof (ch_cpu_logout_t)); 2850 clop->clo_data.chd_afar = LOGOUT_INVALID; 2851 } 2852 } 2853 2854 /* 2855 * Re-enable CEEN if we turned it off. 2856 */ 2857 if (orig_err & EN_REG_CEEN) 2858 set_error_enable(orig_err); 2859 kpreempt_enable(); 2860 2861 return (disp); 2862 } 2863 2864 /* 2865 * Scrub a correctable memory error and collect data for classification 2866 * of CE type. This function is called in the detection path, ie tl0 handling 2867 * of a correctable error trap (cpus) or interrupt (IO) at high PIL. 2868 */ 2869 void 2870 cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t logout_tried) 2871 { 2872 /* 2873 * Cheetah CE classification does not set any bits in flt_status. 2874 * Instead we will record classification datapoints in flt_disp. 2875 */ 2876 ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY); 2877 2878 /* 2879 * To check if the error detected by IO is persistent, sticky or 2880 * intermittent. This is noticed by clear_ecc(). 2881 */ 2882 if (ecc->flt_status & ECC_IOBUS) 2883 ecc->flt_stat = C_AFSR_MEMORY; 2884 2885 /* 2886 * Record information from this first part of the algorithm in 2887 * flt_disp. 2888 */ 2889 ecc->flt_disp = cpu_ce_scrub_mem_err_common(ecc, logout_tried); 2890 } 2891 2892 /* 2893 * Select a partner to perform a further CE classification check from. 2894 * Must be called with kernel preemption disabled (to stop the cpu list 2895 * from changing). The detecting cpu we are partnering has cpuid 2896 * aflt->flt_inst; we might not be running on the detecting cpu. 2897 * 2898 * Restrict choice to active cpus in the same cpu partition as ourselves in 2899 * an effort to stop bad cpus in one partition causing other partitions to 2900 * perform excessive diagnostic activity. Actually since the errorq drain 2901 * is run from a softint most of the time and that is a global mechanism 2902 * this isolation is only partial. Return NULL if we fail to find a 2903 * suitable partner. 2904 * 2905 * We prefer a partner that is in a different latency group to ourselves as 2906 * we will share fewer datapaths. If such a partner is unavailable then 2907 * choose one in the same lgroup but prefer a different chip and only allow 2908 * a sibling core if flags includes PTNR_SIBLINGOK. If all else fails and 2909 * flags includes PTNR_SELFOK then permit selection of the original detector. 2910 * 2911 * We keep a cache of the last partner selected for a cpu, and we'll try to 2912 * use that previous partner if no more than cpu_ce_ptnr_cachetime_sec seconds 2913 * have passed since that selection was made. This provides the benefit 2914 * of the point-of-view of different partners over time but without 2915 * requiring frequent cpu list traversals. 2916 */ 2917 2918 #define PTNR_SIBLINGOK 0x1 /* Allow selection of sibling core */ 2919 #define PTNR_SELFOK 0x2 /* Allow selection of cpu to "partner" itself */ 2920 2921 static cpu_t * 2922 ce_ptnr_select(struct async_flt *aflt, int flags, int *typep) 2923 { 2924 cpu_t *sp, *dtcr, *ptnr, *locptnr, *sibptnr; 2925 hrtime_t lasttime, thistime; 2926 2927 ASSERT(curthread->t_preempt > 0 || getpil() >= DISP_LEVEL); 2928 2929 dtcr = cpu[aflt->flt_inst]; 2930 2931 /* 2932 * Short-circuit for the following cases: 2933 * . the dtcr is not flagged active 2934 * . there is just one cpu present 2935 * . the detector has disappeared 2936 * . we were given a bad flt_inst cpuid; this should not happen 2937 * (eg PCI code now fills flt_inst) but if it does it is no 2938 * reason to panic. 2939 * . there is just one cpu left online in the cpu partition 2940 * 2941 * If we return NULL after this point then we do not update the 2942 * chpr_ceptnr_seltime which will cause us to perform a full lookup 2943 * again next time; this is the case where the only other cpu online 2944 * in the detector's partition is on the same chip as the detector 2945 * and since CEEN re-enable is throttled even that case should not 2946 * hurt performance. 2947 */ 2948 if (dtcr == NULL || !cpu_flagged_active(dtcr->cpu_flags)) { 2949 return (NULL); 2950 } 2951 if (ncpus == 1 || dtcr->cpu_part->cp_ncpus == 1) { 2952 if (flags & PTNR_SELFOK) { 2953 *typep = CE_XDIAG_PTNR_SELF; 2954 return (dtcr); 2955 } else { 2956 return (NULL); 2957 } 2958 } 2959 2960 thistime = gethrtime(); 2961 lasttime = CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime); 2962 2963 /* 2964 * Select a starting point. 2965 */ 2966 if (!lasttime) { 2967 /* 2968 * We've never selected a partner for this detector before. 2969 * Start the scan at the next online cpu in the same cpu 2970 * partition. 2971 */ 2972 sp = dtcr->cpu_next_part; 2973 } else if (thistime - lasttime < cpu_ce_ptnr_cachetime_sec * NANOSEC) { 2974 /* 2975 * Our last selection has not aged yet. If this partner: 2976 * . is still a valid cpu, 2977 * . is still in the same partition as the detector 2978 * . is still marked active 2979 * . satisfies the 'flags' argument criteria 2980 * then select it again without updating the timestamp. 2981 */ 2982 sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 2983 if (sp == NULL || sp->cpu_part != dtcr->cpu_part || 2984 !cpu_flagged_active(sp->cpu_flags) || 2985 (sp == dtcr && !(flags & PTNR_SELFOK)) || 2986 (sp->cpu_chip->chip_id == dtcr->cpu_chip->chip_id && 2987 !(flags & PTNR_SIBLINGOK))) { 2988 sp = dtcr->cpu_next_part; 2989 } else { 2990 if (sp->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 2991 *typep = CE_XDIAG_PTNR_REMOTE; 2992 } else if (sp == dtcr) { 2993 *typep = CE_XDIAG_PTNR_SELF; 2994 } else if (sp->cpu_chip->chip_id == 2995 dtcr->cpu_chip->chip_id) { 2996 *typep = CE_XDIAG_PTNR_SIBLING; 2997 } else { 2998 *typep = CE_XDIAG_PTNR_LOCAL; 2999 } 3000 return (sp); 3001 } 3002 } else { 3003 /* 3004 * Our last selection has aged. If it is nonetheless still a 3005 * valid cpu then start the scan at the next cpu in the 3006 * partition after our last partner. If the last selection 3007 * is no longer a valid cpu then go with our default. In 3008 * this way we slowly cycle through possible partners to 3009 * obtain multiple viewpoints over time. 3010 */ 3011 sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 3012 if (sp == NULL) { 3013 sp = dtcr->cpu_next_part; 3014 } else { 3015 sp = sp->cpu_next_part; /* may be dtcr */ 3016 if (sp->cpu_part != dtcr->cpu_part) 3017 sp = dtcr; 3018 } 3019 } 3020 3021 /* 3022 * We have a proposed starting point for our search, but if this 3023 * cpu is offline then its cpu_next_part will point to itself 3024 * so we can't use that to iterate over cpus in this partition in 3025 * the loop below. We still want to avoid iterating over cpus not 3026 * in our partition, so in the case that our starting point is offline 3027 * we will repoint it to be the detector itself; and if the detector 3028 * happens to be offline we'll return NULL from the following loop. 3029 */ 3030 if (!cpu_flagged_active(sp->cpu_flags)) { 3031 sp = dtcr; 3032 } 3033 3034 ptnr = sp; 3035 locptnr = NULL; 3036 sibptnr = NULL; 3037 do { 3038 if (ptnr == dtcr || !cpu_flagged_active(ptnr->cpu_flags)) 3039 continue; 3040 if (ptnr->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 3041 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = ptnr->cpu_id; 3042 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 3043 *typep = CE_XDIAG_PTNR_REMOTE; 3044 return (ptnr); 3045 } 3046 if (ptnr->cpu_chip->chip_id == dtcr->cpu_chip->chip_id) { 3047 if (sibptnr == NULL) 3048 sibptnr = ptnr; 3049 continue; 3050 } 3051 if (locptnr == NULL) 3052 locptnr = ptnr; 3053 } while ((ptnr = ptnr->cpu_next_part) != sp); 3054 3055 /* 3056 * A foreign partner has already been returned if one was available. 3057 * 3058 * If locptnr is not NULL it is a cpu in the same lgroup as the 3059 * detector, is active, and is not a sibling of the detector. 3060 * 3061 * If sibptnr is not NULL it is a sibling of the detector, and is 3062 * active. 3063 * 3064 * If we have to resort to using the detector itself we have already 3065 * checked that it is active. 3066 */ 3067 if (locptnr) { 3068 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = locptnr->cpu_id; 3069 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 3070 *typep = CE_XDIAG_PTNR_LOCAL; 3071 return (locptnr); 3072 } else if (sibptnr && flags & PTNR_SIBLINGOK) { 3073 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = sibptnr->cpu_id; 3074 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 3075 *typep = CE_XDIAG_PTNR_SIBLING; 3076 return (sibptnr); 3077 } else if (flags & PTNR_SELFOK) { 3078 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = dtcr->cpu_id; 3079 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 3080 *typep = CE_XDIAG_PTNR_SELF; 3081 return (dtcr); 3082 } 3083 3084 return (NULL); 3085 } 3086 3087 /* 3088 * Cross call handler that is requested to run on the designated partner of 3089 * a cpu that experienced a possibly sticky or possibly persistnet CE. 3090 */ 3091 static void 3092 ce_ptnrchk_xc(struct async_flt *aflt, uchar_t *dispp) 3093 { 3094 *dispp = cpu_ce_scrub_mem_err_common(aflt, B_FALSE); 3095 } 3096 3097 /* 3098 * The associated errorqs are never destroyed so we do not need to deal with 3099 * them disappearing before this timeout fires. If the affected memory 3100 * has been DR'd out since the original event the scrub algrithm will catch 3101 * any errors and return null disposition info. If the original detecting 3102 * cpu has been DR'd out then ereport detector info will not be able to 3103 * lookup CPU type; with a small timeout this is unlikely. 3104 */ 3105 static void 3106 ce_lkychk_cb(ce_lkychk_cb_t *cbarg) 3107 { 3108 struct async_flt *aflt = cbarg->lkycb_aflt; 3109 uchar_t disp; 3110 cpu_t *cp; 3111 int ptnrtype; 3112 3113 kpreempt_disable(); 3114 if (cp = ce_ptnr_select(aflt, PTNR_SIBLINGOK | PTNR_SELFOK, 3115 &ptnrtype)) { 3116 xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, (uint64_t)aflt, 3117 (uint64_t)&disp); 3118 CE_XDIAG_SETLKYINFO(aflt->flt_disp, disp); 3119 CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 3120 CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 3121 } else { 3122 ce_xdiag_lkydrops++; 3123 if (ncpus > 1) 3124 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 3125 CE_XDIAG_SKIP_NOPTNR); 3126 } 3127 kpreempt_enable(); 3128 3129 errorq_commit(cbarg->lkycb_eqp, cbarg->lkycb_eqep, ERRORQ_ASYNC); 3130 kmem_free(cbarg, sizeof (ce_lkychk_cb_t)); 3131 } 3132 3133 /* 3134 * Called from errorq drain code when processing a CE error, both from 3135 * CPU and PCI drain functions. Decide what further classification actions, 3136 * if any, we will perform. Perform immediate actions now, and schedule 3137 * delayed actions as required. Note that we are no longer necessarily running 3138 * on the detecting cpu, and that the async_flt structure will not persist on 3139 * return from this function. 3140 * 3141 * Calls to this function should aim to be self-throtlling in some way. With 3142 * the delayed re-enable of CEEN the absolute rate of calls should not 3143 * be excessive. Callers should also avoid performing in-depth classification 3144 * for events in pages that are already known to be suspect. 3145 * 3146 * We return nonzero to indicate that the event has been copied and 3147 * recirculated for further testing. The caller should not log the event 3148 * in this case - it will be logged when further test results are available. 3149 * 3150 * Our possible contexts are that of errorq_drain: below lock level or from 3151 * panic context. We can assume that the cpu we are running on is online. 3152 */ 3153 3154 3155 #ifdef DEBUG 3156 static int ce_xdiag_forceaction; 3157 #endif 3158 3159 int 3160 ce_scrub_xdiag_recirc(struct async_flt *aflt, errorq_t *eqp, 3161 errorq_elem_t *eqep, size_t afltoffset) 3162 { 3163 ce_dispact_t dispact, action; 3164 cpu_t *cp; 3165 uchar_t dtcrinfo, disp; 3166 int ptnrtype; 3167 3168 if (!ce_disp_inited || panicstr || ce_xdiag_off) { 3169 ce_xdiag_drops++; 3170 return (0); 3171 } else if (!aflt->flt_in_memory) { 3172 ce_xdiag_drops++; 3173 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOTMEM); 3174 return (0); 3175 } 3176 3177 dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 3178 3179 /* 3180 * Some correctable events are not scrubbed/classified, such as those 3181 * noticed at the tail of cpu_deferred_error. So if there is no 3182 * initial detector classification go no further. 3183 */ 3184 if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) { 3185 ce_xdiag_drops++; 3186 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOSCRUB); 3187 return (0); 3188 } 3189 3190 dispact = CE_DISPACT(ce_disp_table, 3191 CE_XDIAG_AFARMATCHED(dtcrinfo), 3192 CE_XDIAG_STATE(dtcrinfo), 3193 CE_XDIAG_CE1SEEN(dtcrinfo), 3194 CE_XDIAG_CE2SEEN(dtcrinfo)); 3195 3196 3197 action = CE_ACT(dispact); /* bad lookup caught below */ 3198 #ifdef DEBUG 3199 if (ce_xdiag_forceaction != 0) 3200 action = ce_xdiag_forceaction; 3201 #endif 3202 3203 switch (action) { 3204 case CE_ACT_LKYCHK: { 3205 caddr_t ndata; 3206 errorq_elem_t *neqep; 3207 struct async_flt *ecc; 3208 ce_lkychk_cb_t *cbargp; 3209 3210 if ((ndata = errorq_elem_dup(eqp, eqep, &neqep)) == NULL) { 3211 ce_xdiag_lkydrops++; 3212 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 3213 CE_XDIAG_SKIP_DUPFAIL); 3214 break; 3215 } 3216 ecc = (struct async_flt *)(ndata + afltoffset); 3217 3218 ASSERT(ecc->flt_class == CPU_FAULT || 3219 ecc->flt_class == BUS_FAULT); 3220 ecc->flt_class = (ecc->flt_class == CPU_FAULT) ? 3221 RECIRC_CPU_FAULT : RECIRC_BUS_FAULT; 3222 3223 cbargp = kmem_alloc(sizeof (ce_lkychk_cb_t), KM_SLEEP); 3224 cbargp->lkycb_aflt = ecc; 3225 cbargp->lkycb_eqp = eqp; 3226 cbargp->lkycb_eqep = neqep; 3227 3228 (void) timeout((void (*)(void *))ce_lkychk_cb, 3229 (void *)cbargp, drv_usectohz(cpu_ce_lkychk_timeout_usec)); 3230 return (1); 3231 } 3232 3233 case CE_ACT_PTNRCHK: 3234 kpreempt_disable(); /* stop cpu list changing */ 3235 if ((cp = ce_ptnr_select(aflt, 0, &ptnrtype)) != NULL) { 3236 xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, 3237 (uint64_t)aflt, (uint64_t)&disp); 3238 CE_XDIAG_SETPTNRINFO(aflt->flt_disp, disp); 3239 CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 3240 CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 3241 } else if (ncpus > 1) { 3242 ce_xdiag_ptnrdrops++; 3243 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 3244 CE_XDIAG_SKIP_NOPTNR); 3245 } else { 3246 ce_xdiag_ptnrdrops++; 3247 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 3248 CE_XDIAG_SKIP_UNIPROC); 3249 } 3250 kpreempt_enable(); 3251 break; 3252 3253 case CE_ACT_DONE: 3254 break; 3255 3256 case CE_ACT(CE_DISP_BAD): 3257 default: 3258 #ifdef DEBUG 3259 cmn_err(CE_PANIC, "ce_scrub_post: Bad action '%d'", action); 3260 #endif 3261 ce_xdiag_bad++; 3262 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_ACTBAD); 3263 break; 3264 } 3265 3266 return (0); 3267 } 3268 3269 /* 3270 * We route all errors through a single switch statement. 3271 */ 3272 void 3273 cpu_ue_log_err(struct async_flt *aflt) 3274 { 3275 switch (aflt->flt_class) { 3276 case CPU_FAULT: 3277 cpu_ereport_init(aflt); 3278 if (cpu_async_log_err(aflt, NULL)) 3279 cpu_ereport_post(aflt); 3280 break; 3281 3282 case BUS_FAULT: 3283 bus_async_log_err(aflt); 3284 break; 3285 3286 default: 3287 cmn_err(CE_WARN, "discarding async error %p with invalid " 3288 "fault class (0x%x)", (void *)aflt, aflt->flt_class); 3289 return; 3290 } 3291 } 3292 3293 /* 3294 * Routine for panic hook callback from panic_idle(). 3295 */ 3296 void 3297 cpu_async_panic_callb(void) 3298 { 3299 ch_async_flt_t ch_flt; 3300 struct async_flt *aflt; 3301 ch_cpu_errors_t cpu_error_regs; 3302 uint64_t afsr_errs; 3303 3304 get_cpu_error_state(&cpu_error_regs); 3305 3306 afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 3307 (cpu_error_regs.afsr_ext & C_AFSR_EXT_L3_ERRS); 3308 3309 if (afsr_errs) { 3310 3311 bzero(&ch_flt, sizeof (ch_async_flt_t)); 3312 aflt = (struct async_flt *)&ch_flt; 3313 aflt->flt_id = gethrtime_waitfree(); 3314 aflt->flt_bus_id = getprocessorid(); 3315 aflt->flt_inst = CPU->cpu_id; 3316 aflt->flt_stat = cpu_error_regs.afsr; 3317 aflt->flt_addr = cpu_error_regs.afar; 3318 aflt->flt_prot = AFLT_PROT_NONE; 3319 aflt->flt_class = CPU_FAULT; 3320 aflt->flt_priv = ((cpu_error_regs.afsr & C_AFSR_PRIV) != 0); 3321 aflt->flt_panic = 1; 3322 ch_flt.afsr_ext = cpu_error_regs.afsr_ext; 3323 ch_flt.afsr_errs = afsr_errs; 3324 #if defined(SERRANO) 3325 ch_flt.afar2 = cpu_error_regs.afar2; 3326 #endif /* SERRANO */ 3327 (void) cpu_queue_events(&ch_flt, NULL, afsr_errs, NULL); 3328 } 3329 } 3330 3331 /* 3332 * Routine to convert a syndrome into a syndrome code. 3333 */ 3334 static int 3335 synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit) 3336 { 3337 if (synd_status == AFLT_STAT_INVALID) 3338 return (-1); 3339 3340 /* 3341 * Use the syndrome to index the appropriate syndrome table, 3342 * to get the code indicating which bit(s) is(are) bad. 3343 */ 3344 if (afsr_bit & 3345 (C_AFSR_MSYND_ERRS | C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 3346 if (afsr_bit & C_AFSR_MSYND_ERRS) { 3347 #if defined(JALAPENO) || defined(SERRANO) 3348 if ((synd == 0) || (synd >= BSYND_TBL_SIZE)) 3349 return (-1); 3350 else 3351 return (BPAR0 + synd); 3352 #else /* JALAPENO || SERRANO */ 3353 if ((synd == 0) || (synd >= MSYND_TBL_SIZE)) 3354 return (-1); 3355 else 3356 return (mtag_syndrome_tab[synd]); 3357 #endif /* JALAPENO || SERRANO */ 3358 } else { 3359 if ((synd == 0) || (synd >= ESYND_TBL_SIZE)) 3360 return (-1); 3361 else 3362 return (ecc_syndrome_tab[synd]); 3363 } 3364 } else { 3365 return (-1); 3366 } 3367 } 3368 3369 /* 3370 * Routine to return a string identifying the physical name 3371 * associated with a memory/cache error. 3372 */ 3373 int 3374 cpu_get_mem_unum(int synd_status, ushort_t flt_synd, uint64_t flt_stat, 3375 uint64_t flt_addr, int flt_bus_id, int flt_in_memory, 3376 ushort_t flt_status, char *buf, int buflen, int *lenp) 3377 { 3378 int synd_code; 3379 int ret; 3380 3381 /* 3382 * An AFSR of -1 defaults to a memory syndrome. 3383 */ 3384 if (flt_stat == (uint64_t)-1) 3385 flt_stat = C_AFSR_CE; 3386 3387 synd_code = synd_to_synd_code(synd_status, flt_synd, flt_stat); 3388 3389 /* 3390 * Syndrome code must be either a single-bit error code 3391 * (0...143) or -1 for unum lookup. 3392 */ 3393 if (synd_code < 0 || synd_code >= M2) 3394 synd_code = -1; 3395 if (&plat_get_mem_unum) { 3396 if ((ret = plat_get_mem_unum(synd_code, flt_addr, flt_bus_id, 3397 flt_in_memory, flt_status, buf, buflen, lenp)) != 0) { 3398 buf[0] = '\0'; 3399 *lenp = 0; 3400 } 3401 3402 return (ret); 3403 } 3404 3405 return (ENOTSUP); 3406 } 3407 3408 /* 3409 * Wrapper for cpu_get_mem_unum() routine that takes an 3410 * async_flt struct rather than explicit arguments. 3411 */ 3412 int 3413 cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt, 3414 char *buf, int buflen, int *lenp) 3415 { 3416 /* 3417 * If we come thru here for an IO bus error aflt->flt_stat will 3418 * not be the CPU AFSR, and we pass in a -1 to cpu_get_mem_unum() 3419 * so it will interpret this as a memory error. 3420 */ 3421 return (cpu_get_mem_unum(synd_status, aflt->flt_synd, 3422 (aflt->flt_class == BUS_FAULT) ? 3423 (uint64_t)-1 : ((ch_async_flt_t *)(aflt))->afsr_errs, 3424 aflt->flt_addr, aflt->flt_bus_id, aflt->flt_in_memory, 3425 aflt->flt_status, buf, buflen, lenp)); 3426 } 3427 3428 /* 3429 * This routine is a more generic interface to cpu_get_mem_unum() 3430 * that may be used by other modules (e.g. mm). 3431 */ 3432 int 3433 cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar, 3434 char *buf, int buflen, int *lenp) 3435 { 3436 int synd_status, flt_in_memory, ret; 3437 ushort_t flt_status = 0; 3438 char unum[UNUM_NAMLEN]; 3439 3440 /* 3441 * Check for an invalid address. 3442 */ 3443 if (afar == (uint64_t)-1) 3444 return (ENXIO); 3445 3446 if (synd == (uint64_t)-1) 3447 synd_status = AFLT_STAT_INVALID; 3448 else 3449 synd_status = AFLT_STAT_VALID; 3450 3451 flt_in_memory = (*afsr & C_AFSR_MEMORY) && 3452 pf_is_memory(afar >> MMU_PAGESHIFT); 3453 3454 /* 3455 * Need to turn on ECC_ECACHE for plat_get_mem_unum(). 3456 * For Panther, L2$ is not external, so we don't want to 3457 * generate an E$ unum for those errors. 3458 */ 3459 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 3460 if (*(afsr + 1) & C_AFSR_EXT_L3_ERRS) 3461 flt_status |= ECC_ECACHE; 3462 } else { 3463 if (*afsr & C_AFSR_ECACHE) 3464 flt_status |= ECC_ECACHE; 3465 } 3466 3467 ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, *afsr, afar, 3468 CPU->cpu_id, flt_in_memory, flt_status, unum, UNUM_NAMLEN, lenp); 3469 if (ret != 0) 3470 return (ret); 3471 3472 if (*lenp >= buflen) 3473 return (ENAMETOOLONG); 3474 3475 (void) strncpy(buf, unum, buflen); 3476 3477 return (0); 3478 } 3479 3480 /* 3481 * Routine to return memory information associated 3482 * with a physical address and syndrome. 3483 */ 3484 int 3485 cpu_get_mem_info(uint64_t synd, uint64_t afar, 3486 uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 3487 int *segsp, int *banksp, int *mcidp) 3488 { 3489 int synd_status, synd_code; 3490 3491 if (afar == (uint64_t)-1) 3492 return (ENXIO); 3493 3494 if (synd == (uint64_t)-1) 3495 synd_status = AFLT_STAT_INVALID; 3496 else 3497 synd_status = AFLT_STAT_VALID; 3498 3499 synd_code = synd_to_synd_code(synd_status, synd, C_AFSR_CE); 3500 3501 if (p2get_mem_info != NULL) 3502 return ((p2get_mem_info)(synd_code, afar, 3503 mem_sizep, seg_sizep, bank_sizep, 3504 segsp, banksp, mcidp)); 3505 else 3506 return (ENOTSUP); 3507 } 3508 3509 /* 3510 * Routine to return a string identifying the physical 3511 * name associated with a cpuid. 3512 */ 3513 int 3514 cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp) 3515 { 3516 int ret; 3517 char unum[UNUM_NAMLEN]; 3518 3519 if (&plat_get_cpu_unum) { 3520 if ((ret = plat_get_cpu_unum(cpuid, unum, UNUM_NAMLEN, lenp)) 3521 != 0) 3522 return (ret); 3523 } else { 3524 return (ENOTSUP); 3525 } 3526 3527 if (*lenp >= buflen) 3528 return (ENAMETOOLONG); 3529 3530 (void) strncpy(buf, unum, buflen); 3531 3532 return (0); 3533 } 3534 3535 /* 3536 * This routine exports the name buffer size. 3537 */ 3538 size_t 3539 cpu_get_name_bufsize() 3540 { 3541 return (UNUM_NAMLEN); 3542 } 3543 3544 /* 3545 * Historical function, apparantly not used. 3546 */ 3547 /* ARGSUSED */ 3548 void 3549 cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err) 3550 {} 3551 3552 /* 3553 * Historical function only called for SBus errors in debugging. 3554 */ 3555 /*ARGSUSED*/ 3556 void 3557 read_ecc_data(struct async_flt *aflt, short verbose, short ce_err) 3558 {} 3559 3560 /* 3561 * Clear the AFSR sticky bits. The routine returns a non-zero value if 3562 * any of the AFSR's sticky errors are detected. If a non-null pointer to 3563 * an async fault structure argument is passed in, the captured error state 3564 * (AFSR, AFAR) info will be returned in the structure. 3565 */ 3566 int 3567 clear_errors(ch_async_flt_t *ch_flt) 3568 { 3569 struct async_flt *aflt = (struct async_flt *)ch_flt; 3570 ch_cpu_errors_t cpu_error_regs; 3571 3572 get_cpu_error_state(&cpu_error_regs); 3573 3574 if (ch_flt != NULL) { 3575 aflt->flt_stat = cpu_error_regs.afsr & C_AFSR_MASK; 3576 aflt->flt_addr = cpu_error_regs.afar; 3577 ch_flt->afsr_ext = cpu_error_regs.afsr_ext; 3578 ch_flt->afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 3579 (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS); 3580 #if defined(SERRANO) 3581 ch_flt->afar2 = cpu_error_regs.afar2; 3582 #endif /* SERRANO */ 3583 } 3584 3585 set_cpu_error_state(&cpu_error_regs); 3586 3587 return (((cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 3588 (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS)) != 0); 3589 } 3590 3591 /* 3592 * Clear any AFSR error bits, and check for persistence. 3593 * 3594 * It would be desirable to also insist that syndrome match. PCI handling 3595 * has already filled flt_synd. For errors trapped by CPU we only fill 3596 * flt_synd when we queue the event, so we do not have a valid flt_synd 3597 * during initial classification (it is valid if we're called as part of 3598 * subsequent low-pil additional classification attempts). We could try 3599 * to determine which syndrome to use: we know we're only called for 3600 * CE/RCE (Jalapeno & Serrano) and CE/EMC (others) so the syndrome to use 3601 * would be esynd/none and esynd/msynd, respectively. If that is 3602 * implemented then what do we do in the case that we do experience an 3603 * error on the same afar but with different syndrome? At the very least 3604 * we should count such occurences. Anyway, for now, we'll leave it as 3605 * it has been for ages. 3606 */ 3607 static int 3608 clear_ecc(struct async_flt *aflt) 3609 { 3610 ch_cpu_errors_t cpu_error_regs; 3611 3612 /* 3613 * Snapshot the AFSR and AFAR and clear any errors 3614 */ 3615 get_cpu_error_state(&cpu_error_regs); 3616 set_cpu_error_state(&cpu_error_regs); 3617 3618 /* 3619 * If any of the same memory access error bits are still on and 3620 * the AFAR matches, return that the error is persistent. 3621 */ 3622 return ((cpu_error_regs.afsr & (C_AFSR_MEMORY & aflt->flt_stat)) != 0 && 3623 cpu_error_regs.afar == aflt->flt_addr); 3624 } 3625 3626 /* 3627 * Turn off all cpu error detection, normally only used for panics. 3628 */ 3629 void 3630 cpu_disable_errors(void) 3631 { 3632 xt_all(set_error_enable_tl1, EN_REG_DISABLE, EER_SET_ABSOLUTE); 3633 3634 /* 3635 * With error detection now turned off, check the other cpus 3636 * logout areas for any unlogged errors. 3637 */ 3638 if (enable_check_other_cpus_logout) { 3639 cpu_check_other_cpus_logout(); 3640 /* 3641 * Make a second pass over the logout areas, in case 3642 * there is a failing CPU in an error-trap loop which 3643 * will write to the logout area once it is emptied. 3644 */ 3645 cpu_check_other_cpus_logout(); 3646 } 3647 } 3648 3649 /* 3650 * Enable errors. 3651 */ 3652 void 3653 cpu_enable_errors(void) 3654 { 3655 xt_all(set_error_enable_tl1, EN_REG_ENABLE, EER_SET_ABSOLUTE); 3656 } 3657 3658 /* 3659 * Flush the entire ecache using displacement flush by reading through a 3660 * physical address range twice as large as the Ecache. 3661 */ 3662 void 3663 cpu_flush_ecache(void) 3664 { 3665 flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size, 3666 cpunodes[CPU->cpu_id].ecache_linesize); 3667 } 3668 3669 /* 3670 * Return CPU E$ set size - E$ size divided by the associativity. 3671 * We use this function in places where the CPU_PRIVATE ptr may not be 3672 * initialized yet. Note that for send_mondo and in the Ecache scrubber, 3673 * we're guaranteed that CPU_PRIVATE is initialized. Also, cpunodes is set 3674 * up before the kernel switches from OBP's to the kernel's trap table, so 3675 * we don't have to worry about cpunodes being unitialized. 3676 */ 3677 int 3678 cpu_ecache_set_size(struct cpu *cp) 3679 { 3680 if (CPU_PRIVATE(cp)) 3681 return (CPU_PRIVATE_VAL(cp, chpr_ec_set_size)); 3682 3683 return (cpunodes[cp->cpu_id].ecache_size / cpu_ecache_nway()); 3684 } 3685 3686 /* 3687 * Flush Ecache line. 3688 * Uses ASI_EC_DIAG for Cheetah+ and Jalapeno. 3689 * Uses normal displacement flush for Cheetah. 3690 */ 3691 static void 3692 cpu_flush_ecache_line(ch_async_flt_t *ch_flt) 3693 { 3694 struct async_flt *aflt = (struct async_flt *)ch_flt; 3695 int ec_set_size = cpu_ecache_set_size(CPU); 3696 3697 ecache_flush_line(aflt->flt_addr, ec_set_size); 3698 } 3699 3700 /* 3701 * Scrub physical address. 3702 * Scrub code is different depending upon whether this a Cheetah+ with 2-way 3703 * Ecache or direct-mapped Ecache. 3704 */ 3705 static void 3706 cpu_scrubphys(struct async_flt *aflt) 3707 { 3708 int ec_set_size = cpu_ecache_set_size(CPU); 3709 3710 scrubphys(aflt->flt_addr, ec_set_size); 3711 } 3712 3713 /* 3714 * Clear physical address. 3715 * Scrub code is different depending upon whether this a Cheetah+ with 2-way 3716 * Ecache or direct-mapped Ecache. 3717 */ 3718 void 3719 cpu_clearphys(struct async_flt *aflt) 3720 { 3721 int lsize = cpunodes[CPU->cpu_id].ecache_linesize; 3722 int ec_set_size = cpu_ecache_set_size(CPU); 3723 3724 3725 clearphys(P2ALIGN(aflt->flt_addr, lsize), ec_set_size, lsize); 3726 } 3727 3728 #if defined(CPU_IMP_ECACHE_ASSOC) 3729 /* 3730 * Check for a matching valid line in all the sets. 3731 * If found, return set# + 1. Otherwise return 0. 3732 */ 3733 static int 3734 cpu_ecache_line_valid(ch_async_flt_t *ch_flt) 3735 { 3736 struct async_flt *aflt = (struct async_flt *)ch_flt; 3737 int totalsize = cpunodes[CPU->cpu_id].ecache_size; 3738 int ec_set_size = cpu_ecache_set_size(CPU); 3739 ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 3740 int nway = cpu_ecache_nway(); 3741 int i; 3742 3743 for (i = 0; i < nway; i++, ecp++) { 3744 if (!cpu_ectag_line_invalid(totalsize, ecp->ec_tag) && 3745 (aflt->flt_addr & P2ALIGN(C_AFAR_PA, ec_set_size)) == 3746 cpu_ectag_to_pa(ec_set_size, ecp->ec_tag)) 3747 return (i+1); 3748 } 3749 return (0); 3750 } 3751 #endif /* CPU_IMP_ECACHE_ASSOC */ 3752 3753 /* 3754 * Check whether a line in the given logout info matches the specified 3755 * fault address. If reqval is set then the line must not be Invalid. 3756 * Returns 0 on failure; on success (way + 1) is returned an *level is 3757 * set to 2 for l2$ or 3 for l3$. 3758 */ 3759 static int 3760 cpu_matching_ecache_line(uint64_t faddr, void *data, int reqval, int *level) 3761 { 3762 ch_diag_data_t *cdp = data; 3763 ch_ec_data_t *ecp; 3764 int totalsize, ec_set_size; 3765 int i, ways; 3766 int match = 0; 3767 int tagvalid; 3768 uint64_t addr, tagpa; 3769 int ispanther = IS_PANTHER(cpunodes[CPU->cpu_id].implementation); 3770 3771 /* 3772 * Check the l2$ logout data 3773 */ 3774 if (ispanther) { 3775 ecp = &cdp->chd_l2_data[0]; 3776 ec_set_size = PN_L2_SET_SIZE; 3777 ways = PN_L2_NWAYS; 3778 } else { 3779 ecp = &cdp->chd_ec_data[0]; 3780 ec_set_size = cpu_ecache_set_size(CPU); 3781 ways = cpu_ecache_nway(); 3782 totalsize = cpunodes[CPU->cpu_id].ecache_size; 3783 } 3784 /* remove low order PA bits from fault address not used in PA tag */ 3785 addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 3786 for (i = 0; i < ways; i++, ecp++) { 3787 if (ispanther) { 3788 tagpa = PN_L2TAG_TO_PA(ecp->ec_tag); 3789 tagvalid = !PN_L2_LINE_INVALID(ecp->ec_tag); 3790 } else { 3791 tagpa = cpu_ectag_to_pa(ec_set_size, ecp->ec_tag); 3792 tagvalid = !cpu_ectag_line_invalid(totalsize, 3793 ecp->ec_tag); 3794 } 3795 if (tagpa == addr && (!reqval || tagvalid)) { 3796 match = i + 1; 3797 *level = 2; 3798 break; 3799 } 3800 } 3801 3802 if (match || !ispanther) 3803 return (match); 3804 3805 /* For Panther we also check the l3$ */ 3806 ecp = &cdp->chd_ec_data[0]; 3807 ec_set_size = PN_L3_SET_SIZE; 3808 ways = PN_L3_NWAYS; 3809 addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 3810 3811 for (i = 0; i < ways; i++, ecp++) { 3812 if (PN_L3TAG_TO_PA(ecp->ec_tag) == addr && (!reqval || 3813 !PN_L3_LINE_INVALID(ecp->ec_tag))) { 3814 match = i + 1; 3815 *level = 3; 3816 break; 3817 } 3818 } 3819 3820 return (match); 3821 } 3822 3823 #if defined(CPU_IMP_L1_CACHE_PARITY) 3824 /* 3825 * Record information related to the source of an Dcache Parity Error. 3826 */ 3827 static void 3828 cpu_dcache_parity_info(ch_async_flt_t *ch_flt) 3829 { 3830 int dc_set_size = dcache_size / CH_DCACHE_NWAY; 3831 int index; 3832 3833 /* 3834 * Since instruction decode cannot be done at high PIL 3835 * just examine the entire Dcache to locate the error. 3836 */ 3837 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 3838 ch_flt->parity_data.dpe.cpl_way = -1; 3839 ch_flt->parity_data.dpe.cpl_off = -1; 3840 } 3841 for (index = 0; index < dc_set_size; index += dcache_linesize) 3842 cpu_dcache_parity_check(ch_flt, index); 3843 } 3844 3845 /* 3846 * Check all ways of the Dcache at a specified index for good parity. 3847 */ 3848 static void 3849 cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index) 3850 { 3851 int dc_set_size = dcache_size / CH_DCACHE_NWAY; 3852 uint64_t parity_bits, pbits, data_word; 3853 static int parity_bits_popc[] = { 0, 1, 1, 0 }; 3854 int way, word, data_byte; 3855 ch_dc_data_t *dcp = &ch_flt->parity_data.dpe.cpl_dc[0]; 3856 ch_dc_data_t tmp_dcp; 3857 3858 for (way = 0; way < CH_DCACHE_NWAY; way++, dcp++) { 3859 /* 3860 * Perform diagnostic read. 3861 */ 3862 get_dcache_dtag(index + way * dc_set_size, 3863 (uint64_t *)&tmp_dcp); 3864 3865 /* 3866 * Check tag for even parity. 3867 * Sum of 1 bits (including parity bit) should be even. 3868 */ 3869 if (popc64(tmp_dcp.dc_tag & CHP_DCTAG_PARMASK) & 1) { 3870 /* 3871 * If this is the first error log detailed information 3872 * about it and check the snoop tag. Otherwise just 3873 * record the fact that we found another error. 3874 */ 3875 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 3876 ch_flt->parity_data.dpe.cpl_way = way; 3877 ch_flt->parity_data.dpe.cpl_cache = 3878 CPU_DC_PARITY; 3879 ch_flt->parity_data.dpe.cpl_tag |= CHP_DC_TAG; 3880 3881 if (popc64(tmp_dcp.dc_sntag & 3882 CHP_DCSNTAG_PARMASK) & 1) { 3883 ch_flt->parity_data.dpe.cpl_tag |= 3884 CHP_DC_SNTAG; 3885 ch_flt->parity_data.dpe.cpl_lcnt++; 3886 } 3887 3888 bcopy(&tmp_dcp, dcp, sizeof (ch_dc_data_t)); 3889 } 3890 3891 ch_flt->parity_data.dpe.cpl_lcnt++; 3892 } 3893 3894 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 3895 /* 3896 * Panther has more parity bits than the other 3897 * processors for covering dcache data and so each 3898 * byte of data in each word has its own parity bit. 3899 */ 3900 parity_bits = tmp_dcp.dc_pn_data_parity; 3901 for (word = 0; word < 4; word++) { 3902 data_word = tmp_dcp.dc_data[word]; 3903 pbits = parity_bits & PN_DC_DATA_PARITY_MASK; 3904 for (data_byte = 0; data_byte < 8; 3905 data_byte++) { 3906 if (((popc64(data_word & 3907 PN_DC_DATA_PARITY_MASK)) & 1) ^ 3908 (pbits & 1)) { 3909 cpu_record_dc_data_parity( 3910 ch_flt, dcp, &tmp_dcp, way, 3911 word); 3912 } 3913 pbits >>= 1; 3914 data_word >>= 8; 3915 } 3916 parity_bits >>= 8; 3917 } 3918 } else { 3919 /* 3920 * Check data array for even parity. 3921 * The 8 parity bits are grouped into 4 pairs each 3922 * of which covers a 64-bit word. The endianness is 3923 * reversed -- the low-order parity bits cover the 3924 * high-order data words. 3925 */ 3926 parity_bits = tmp_dcp.dc_utag >> 8; 3927 for (word = 0; word < 4; word++) { 3928 pbits = (parity_bits >> (6 - word * 2)) & 3; 3929 if ((popc64(tmp_dcp.dc_data[word]) + 3930 parity_bits_popc[pbits]) & 1) { 3931 cpu_record_dc_data_parity(ch_flt, dcp, 3932 &tmp_dcp, way, word); 3933 } 3934 } 3935 } 3936 } 3937 } 3938 3939 static void 3940 cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 3941 ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word) 3942 { 3943 /* 3944 * If this is the first error log detailed information about it. 3945 * Otherwise just record the fact that we found another error. 3946 */ 3947 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 3948 ch_flt->parity_data.dpe.cpl_way = way; 3949 ch_flt->parity_data.dpe.cpl_cache = CPU_DC_PARITY; 3950 ch_flt->parity_data.dpe.cpl_off = word * 8; 3951 bcopy(src_dcp, dest_dcp, sizeof (ch_dc_data_t)); 3952 } 3953 ch_flt->parity_data.dpe.cpl_lcnt++; 3954 } 3955 3956 /* 3957 * Record information related to the source of an Icache Parity Error. 3958 * 3959 * Called with the Icache disabled so any diagnostic accesses are safe. 3960 */ 3961 static void 3962 cpu_icache_parity_info(ch_async_flt_t *ch_flt) 3963 { 3964 int ic_set_size; 3965 int ic_linesize; 3966 int index; 3967 3968 if (CPU_PRIVATE(CPU)) { 3969 ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 3970 CH_ICACHE_NWAY; 3971 ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 3972 } else { 3973 ic_set_size = icache_size / CH_ICACHE_NWAY; 3974 ic_linesize = icache_linesize; 3975 } 3976 3977 ch_flt->parity_data.ipe.cpl_way = -1; 3978 ch_flt->parity_data.ipe.cpl_off = -1; 3979 3980 for (index = 0; index < ic_set_size; index += ic_linesize) 3981 cpu_icache_parity_check(ch_flt, index); 3982 } 3983 3984 /* 3985 * Check all ways of the Icache at a specified index for good parity. 3986 */ 3987 static void 3988 cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index) 3989 { 3990 uint64_t parmask, pn_inst_parity; 3991 int ic_set_size; 3992 int ic_linesize; 3993 int flt_index, way, instr, num_instr; 3994 struct async_flt *aflt = (struct async_flt *)ch_flt; 3995 ch_ic_data_t *icp = &ch_flt->parity_data.ipe.cpl_ic[0]; 3996 ch_ic_data_t tmp_icp; 3997 3998 if (CPU_PRIVATE(CPU)) { 3999 ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 4000 CH_ICACHE_NWAY; 4001 ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 4002 } else { 4003 ic_set_size = icache_size / CH_ICACHE_NWAY; 4004 ic_linesize = icache_linesize; 4005 } 4006 4007 /* 4008 * Panther has twice as many instructions per icache line and the 4009 * instruction parity bit is in a different location. 4010 */ 4011 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 4012 num_instr = PN_IC_DATA_REG_SIZE / sizeof (uint64_t); 4013 pn_inst_parity = PN_ICDATA_PARITY_BIT_MASK; 4014 } else { 4015 num_instr = CH_IC_DATA_REG_SIZE / sizeof (uint64_t); 4016 pn_inst_parity = 0; 4017 } 4018 4019 /* 4020 * Index at which we expect to find the parity error. 4021 */ 4022 flt_index = P2ALIGN(aflt->flt_addr % ic_set_size, ic_linesize); 4023 4024 for (way = 0; way < CH_ICACHE_NWAY; way++, icp++) { 4025 /* 4026 * Diagnostic reads expect address argument in ASI format. 4027 */ 4028 get_icache_dtag(2 * (index + way * ic_set_size), 4029 (uint64_t *)&tmp_icp); 4030 4031 /* 4032 * If this is the index in which we expect to find the 4033 * error log detailed information about each of the ways. 4034 * This information will be displayed later if we can't 4035 * determine the exact way in which the error is located. 4036 */ 4037 if (flt_index == index) 4038 bcopy(&tmp_icp, icp, sizeof (ch_ic_data_t)); 4039 4040 /* 4041 * Check tag for even parity. 4042 * Sum of 1 bits (including parity bit) should be even. 4043 */ 4044 if (popc64(tmp_icp.ic_patag & CHP_ICPATAG_PARMASK) & 1) { 4045 /* 4046 * If this way is the one in which we expected 4047 * to find the error record the way and check the 4048 * snoop tag. Otherwise just record the fact we 4049 * found another error. 4050 */ 4051 if (flt_index == index) { 4052 ch_flt->parity_data.ipe.cpl_way = way; 4053 ch_flt->parity_data.ipe.cpl_tag |= CHP_IC_TAG; 4054 4055 if (popc64(tmp_icp.ic_sntag & 4056 CHP_ICSNTAG_PARMASK) & 1) { 4057 ch_flt->parity_data.ipe.cpl_tag |= 4058 CHP_IC_SNTAG; 4059 ch_flt->parity_data.ipe.cpl_lcnt++; 4060 } 4061 4062 } 4063 ch_flt->parity_data.ipe.cpl_lcnt++; 4064 continue; 4065 } 4066 4067 /* 4068 * Check instruction data for even parity. 4069 * Bits participating in parity differ for PC-relative 4070 * versus non-PC-relative instructions. 4071 */ 4072 for (instr = 0; instr < num_instr; instr++) { 4073 parmask = (tmp_icp.ic_data[instr] & 4074 CH_ICDATA_PRED_ISPCREL) ? 4075 (CHP_ICDATA_PCREL_PARMASK | pn_inst_parity) : 4076 (CHP_ICDATA_NPCREL_PARMASK | pn_inst_parity); 4077 if (popc64(tmp_icp.ic_data[instr] & parmask) & 1) { 4078 /* 4079 * If this way is the one in which we expected 4080 * to find the error record the way and offset. 4081 * Otherwise just log the fact we found another 4082 * error. 4083 */ 4084 if (flt_index == index) { 4085 ch_flt->parity_data.ipe.cpl_way = way; 4086 ch_flt->parity_data.ipe.cpl_off = 4087 instr * 4; 4088 } 4089 ch_flt->parity_data.ipe.cpl_lcnt++; 4090 continue; 4091 } 4092 } 4093 } 4094 } 4095 4096 /* 4097 * Record information related to the source of an Pcache Parity Error. 4098 */ 4099 static void 4100 cpu_pcache_parity_info(ch_async_flt_t *ch_flt) 4101 { 4102 int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 4103 int index; 4104 4105 /* 4106 * Since instruction decode cannot be done at high PIL just 4107 * examine the entire Pcache to check for any parity errors. 4108 */ 4109 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 4110 ch_flt->parity_data.dpe.cpl_way = -1; 4111 ch_flt->parity_data.dpe.cpl_off = -1; 4112 } 4113 for (index = 0; index < pc_set_size; index += CH_PCACHE_LSIZE) 4114 cpu_pcache_parity_check(ch_flt, index); 4115 } 4116 4117 /* 4118 * Check all ways of the Pcache at a specified index for good parity. 4119 */ 4120 static void 4121 cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index) 4122 { 4123 int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 4124 int pc_data_words = CH_PC_DATA_REG_SIZE / sizeof (uint64_t); 4125 int way, word, pbit, parity_bits; 4126 ch_pc_data_t *pcp = &ch_flt->parity_data.dpe.cpl_pc[0]; 4127 ch_pc_data_t tmp_pcp; 4128 4129 for (way = 0; way < CH_PCACHE_NWAY; way++, pcp++) { 4130 /* 4131 * Perform diagnostic read. 4132 */ 4133 get_pcache_dtag(index + way * pc_set_size, 4134 (uint64_t *)&tmp_pcp); 4135 /* 4136 * Check data array for odd parity. There are 8 parity 4137 * bits (bits 57:50 of ASI_PCACHE_STATUS_DATA) and each 4138 * of those bits covers exactly 8 bytes of the data 4139 * array: 4140 * 4141 * parity bit P$ data bytes covered 4142 * ---------- --------------------- 4143 * 50 63:56 4144 * 51 55:48 4145 * 52 47:40 4146 * 53 39:32 4147 * 54 31:24 4148 * 55 23:16 4149 * 56 15:8 4150 * 57 7:0 4151 */ 4152 parity_bits = PN_PC_PARITY_BITS(tmp_pcp.pc_status); 4153 for (word = 0; word < pc_data_words; word++) { 4154 pbit = (parity_bits >> (pc_data_words - word - 1)) & 1; 4155 if ((popc64(tmp_pcp.pc_data[word]) & 1) ^ pbit) { 4156 /* 4157 * If this is the first error log detailed 4158 * information about it. Otherwise just record 4159 * the fact that we found another error. 4160 */ 4161 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 4162 ch_flt->parity_data.dpe.cpl_way = way; 4163 ch_flt->parity_data.dpe.cpl_cache = 4164 CPU_PC_PARITY; 4165 ch_flt->parity_data.dpe.cpl_off = 4166 word * sizeof (uint64_t); 4167 bcopy(&tmp_pcp, pcp, 4168 sizeof (ch_pc_data_t)); 4169 } 4170 ch_flt->parity_data.dpe.cpl_lcnt++; 4171 } 4172 } 4173 } 4174 } 4175 4176 4177 /* 4178 * Add L1 Data cache data to the ereport payload. 4179 */ 4180 static void 4181 cpu_payload_add_dcache(struct async_flt *aflt, nvlist_t *nvl) 4182 { 4183 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4184 ch_dc_data_t *dcp; 4185 ch_dc_data_t dcdata[CH_DCACHE_NWAY]; 4186 uint_t nelem; 4187 int i, ways_to_check, ways_logged = 0; 4188 4189 /* 4190 * If this is an D$ fault then there may be multiple 4191 * ways captured in the ch_parity_log_t structure. 4192 * Otherwise, there will be at most one way captured 4193 * in the ch_diag_data_t struct. 4194 * Check each way to see if it should be encoded. 4195 */ 4196 if (ch_flt->flt_type == CPU_DC_PARITY) 4197 ways_to_check = CH_DCACHE_NWAY; 4198 else 4199 ways_to_check = 1; 4200 for (i = 0; i < ways_to_check; i++) { 4201 if (ch_flt->flt_type == CPU_DC_PARITY) 4202 dcp = &ch_flt->parity_data.dpe.cpl_dc[i]; 4203 else 4204 dcp = &ch_flt->flt_diag_data.chd_dc_data; 4205 if (dcp->dc_logflag == DC_LOGFLAG_MAGIC) { 4206 bcopy(dcp, &dcdata[ways_logged], 4207 sizeof (ch_dc_data_t)); 4208 ways_logged++; 4209 } 4210 } 4211 4212 /* 4213 * Add the dcache data to the payload. 4214 */ 4215 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_WAYS, 4216 DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 4217 if (ways_logged != 0) { 4218 nelem = sizeof (ch_dc_data_t) / sizeof (uint64_t) * ways_logged; 4219 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_DATA, 4220 DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)dcdata, NULL); 4221 } 4222 } 4223 4224 /* 4225 * Add L1 Instruction cache data to the ereport payload. 4226 */ 4227 static void 4228 cpu_payload_add_icache(struct async_flt *aflt, nvlist_t *nvl) 4229 { 4230 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4231 ch_ic_data_t *icp; 4232 ch_ic_data_t icdata[CH_ICACHE_NWAY]; 4233 uint_t nelem; 4234 int i, ways_to_check, ways_logged = 0; 4235 4236 /* 4237 * If this is an I$ fault then there may be multiple 4238 * ways captured in the ch_parity_log_t structure. 4239 * Otherwise, there will be at most one way captured 4240 * in the ch_diag_data_t struct. 4241 * Check each way to see if it should be encoded. 4242 */ 4243 if (ch_flt->flt_type == CPU_IC_PARITY) 4244 ways_to_check = CH_ICACHE_NWAY; 4245 else 4246 ways_to_check = 1; 4247 for (i = 0; i < ways_to_check; i++) { 4248 if (ch_flt->flt_type == CPU_IC_PARITY) 4249 icp = &ch_flt->parity_data.ipe.cpl_ic[i]; 4250 else 4251 icp = &ch_flt->flt_diag_data.chd_ic_data; 4252 if (icp->ic_logflag == IC_LOGFLAG_MAGIC) { 4253 bcopy(icp, &icdata[ways_logged], 4254 sizeof (ch_ic_data_t)); 4255 ways_logged++; 4256 } 4257 } 4258 4259 /* 4260 * Add the icache data to the payload. 4261 */ 4262 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_WAYS, 4263 DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 4264 if (ways_logged != 0) { 4265 nelem = sizeof (ch_ic_data_t) / sizeof (uint64_t) * ways_logged; 4266 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_DATA, 4267 DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)icdata, NULL); 4268 } 4269 } 4270 4271 #endif /* CPU_IMP_L1_CACHE_PARITY */ 4272 4273 /* 4274 * Add ecache data to payload. 4275 */ 4276 static void 4277 cpu_payload_add_ecache(struct async_flt *aflt, nvlist_t *nvl) 4278 { 4279 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4280 ch_ec_data_t *ecp; 4281 ch_ec_data_t ecdata[CHD_EC_DATA_SETS]; 4282 uint_t nelem; 4283 int i, ways_logged = 0; 4284 4285 /* 4286 * Check each way to see if it should be encoded 4287 * and concatinate it into a temporary buffer. 4288 */ 4289 for (i = 0; i < CHD_EC_DATA_SETS; i++) { 4290 ecp = &ch_flt->flt_diag_data.chd_ec_data[i]; 4291 if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 4292 bcopy(ecp, &ecdata[ways_logged], 4293 sizeof (ch_ec_data_t)); 4294 ways_logged++; 4295 } 4296 } 4297 4298 /* 4299 * Panther CPUs have an additional level of cache and so 4300 * what we just collected was the L3 (ecache) and not the 4301 * L2 cache. 4302 */ 4303 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 4304 /* 4305 * Add the L3 (ecache) data to the payload. 4306 */ 4307 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_WAYS, 4308 DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 4309 if (ways_logged != 0) { 4310 nelem = sizeof (ch_ec_data_t) / 4311 sizeof (uint64_t) * ways_logged; 4312 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_DATA, 4313 DATA_TYPE_UINT64_ARRAY, nelem, 4314 (uint64_t *)ecdata, NULL); 4315 } 4316 4317 /* 4318 * Now collect the L2 cache. 4319 */ 4320 ways_logged = 0; 4321 for (i = 0; i < PN_L2_NWAYS; i++) { 4322 ecp = &ch_flt->flt_diag_data.chd_l2_data[i]; 4323 if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 4324 bcopy(ecp, &ecdata[ways_logged], 4325 sizeof (ch_ec_data_t)); 4326 ways_logged++; 4327 } 4328 } 4329 } 4330 4331 /* 4332 * Add the L2 cache data to the payload. 4333 */ 4334 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_WAYS, 4335 DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 4336 if (ways_logged != 0) { 4337 nelem = sizeof (ch_ec_data_t) / 4338 sizeof (uint64_t) * ways_logged; 4339 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_DATA, 4340 DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)ecdata, NULL); 4341 } 4342 } 4343 4344 /* 4345 * Encode the data saved in the ch_async_flt_t struct into 4346 * the FM ereport payload. 4347 */ 4348 static void 4349 cpu_payload_add_aflt(struct async_flt *aflt, nvlist_t *payload, 4350 nvlist_t *resource, int *afar_status, int *synd_status) 4351 { 4352 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4353 *synd_status = AFLT_STAT_INVALID; 4354 *afar_status = AFLT_STAT_INVALID; 4355 4356 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR) { 4357 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR, 4358 DATA_TYPE_UINT64, aflt->flt_stat, NULL); 4359 } 4360 4361 if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR_EXT) && 4362 IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 4363 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR_EXT, 4364 DATA_TYPE_UINT64, ch_flt->afsr_ext, NULL); 4365 } 4366 4367 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR_STATUS) { 4368 *afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 4369 ch_flt->flt_bit); 4370 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR_STATUS, 4371 DATA_TYPE_UINT8, (uint8_t)*afar_status, NULL); 4372 } 4373 4374 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR) { 4375 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR, 4376 DATA_TYPE_UINT64, aflt->flt_addr, NULL); 4377 } 4378 4379 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PC) { 4380 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PC, 4381 DATA_TYPE_UINT64, (uint64_t)aflt->flt_pc, NULL); 4382 } 4383 4384 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TL) { 4385 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TL, 4386 DATA_TYPE_UINT8, (uint8_t)aflt->flt_tl, NULL); 4387 } 4388 4389 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TT) { 4390 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TT, 4391 DATA_TYPE_UINT8, flt_to_trap_type(aflt), NULL); 4392 } 4393 4394 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PRIV) { 4395 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PRIV, 4396 DATA_TYPE_BOOLEAN_VALUE, 4397 (aflt->flt_priv ? B_TRUE : B_FALSE), NULL); 4398 } 4399 4400 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ME) { 4401 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ME, 4402 DATA_TYPE_BOOLEAN_VALUE, 4403 (aflt->flt_stat & C_AFSR_ME) ? B_TRUE : B_FALSE, NULL); 4404 } 4405 4406 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND_STATUS) { 4407 *synd_status = afsr_to_synd_status(aflt->flt_inst, 4408 ch_flt->afsr_errs, ch_flt->flt_bit); 4409 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND_STATUS, 4410 DATA_TYPE_UINT8, (uint8_t)*synd_status, NULL); 4411 } 4412 4413 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND) { 4414 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND, 4415 DATA_TYPE_UINT16, (uint16_t)aflt->flt_synd, NULL); 4416 } 4417 4418 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_TYPE) { 4419 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_TYPE, 4420 DATA_TYPE_STRING, flt_to_error_type(aflt), NULL); 4421 } 4422 4423 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_DISP) { 4424 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_DISP, 4425 DATA_TYPE_UINT64, aflt->flt_disp, NULL); 4426 } 4427 4428 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L2) 4429 cpu_payload_add_ecache(aflt, payload); 4430 4431 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_COPYFUNCTION) { 4432 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_COPYFUNCTION, 4433 DATA_TYPE_UINT8, (uint8_t)aflt->flt_status & 0xff, NULL); 4434 } 4435 4436 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_HOWDETECTED) { 4437 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_HOWDETECTED, 4438 DATA_TYPE_UINT8, (uint8_t)(aflt->flt_status >> 8), NULL); 4439 } 4440 4441 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_INSTRBLOCK) { 4442 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_INSTRBLOCK, 4443 DATA_TYPE_UINT32_ARRAY, 16, 4444 (uint32_t *)&ch_flt->flt_fpdata, NULL); 4445 } 4446 4447 #if defined(CPU_IMP_L1_CACHE_PARITY) 4448 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1D) 4449 cpu_payload_add_dcache(aflt, payload); 4450 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1I) 4451 cpu_payload_add_icache(aflt, payload); 4452 #endif /* CPU_IMP_L1_CACHE_PARITY */ 4453 4454 #if defined(CHEETAH_PLUS) 4455 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1P) 4456 cpu_payload_add_pcache(aflt, payload); 4457 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_TLB) 4458 cpu_payload_add_tlb(aflt, payload); 4459 #endif /* CHEETAH_PLUS */ 4460 /* 4461 * Create the FMRI that goes into the payload 4462 * and contains the unum info if necessary. 4463 */ 4464 if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_RESOURCE) && 4465 (*afar_status == AFLT_STAT_VALID)) { 4466 char unum[UNUM_NAMLEN]; 4467 int len; 4468 4469 if (cpu_get_mem_unum_aflt(*synd_status, aflt, unum, 4470 UNUM_NAMLEN, &len) == 0) { 4471 fm_fmri_mem_set(resource, FM_MEM_SCHEME_VERSION, 4472 NULL, unum, NULL); 4473 fm_payload_set(payload, 4474 FM_EREPORT_PAYLOAD_NAME_RESOURCE, 4475 DATA_TYPE_NVLIST, resource, NULL); 4476 } 4477 } 4478 } 4479 4480 /* 4481 * Initialize the way info if necessary. 4482 */ 4483 void 4484 cpu_ereport_init(struct async_flt *aflt) 4485 { 4486 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4487 ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 4488 ch_ec_data_t *l2p = &ch_flt->flt_diag_data.chd_l2_data[0]; 4489 int i; 4490 4491 /* 4492 * Initialize the info in the CPU logout structure. 4493 * The I$/D$ way information is not initialized here 4494 * since it is captured in the logout assembly code. 4495 */ 4496 for (i = 0; i < CHD_EC_DATA_SETS; i++) 4497 (ecp + i)->ec_way = i; 4498 4499 for (i = 0; i < PN_L2_NWAYS; i++) 4500 (l2p + i)->ec_way = i; 4501 } 4502 4503 /* 4504 * Returns whether fault address is valid for this error bit and 4505 * whether the address is "in memory" (i.e. pf_is_memory returns 1). 4506 */ 4507 int 4508 cpu_flt_in_memory(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit) 4509 { 4510 struct async_flt *aflt = (struct async_flt *)ch_flt; 4511 4512 return ((aflt->flt_stat & C_AFSR_MEMORY) && 4513 afsr_to_afar_status(ch_flt->afsr_errs, t_afsr_bit) == 4514 AFLT_STAT_VALID && 4515 pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)); 4516 } 4517 4518 static void 4519 cpu_log_diag_info(ch_async_flt_t *ch_flt) 4520 { 4521 struct async_flt *aflt = (struct async_flt *)ch_flt; 4522 ch_dc_data_t *dcp = &ch_flt->flt_diag_data.chd_dc_data; 4523 ch_ic_data_t *icp = &ch_flt->flt_diag_data.chd_ic_data; 4524 ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 4525 #if defined(CPU_IMP_ECACHE_ASSOC) 4526 int i, nway; 4527 #endif /* CPU_IMP_ECACHE_ASSOC */ 4528 4529 /* 4530 * Check if the CPU log out captured was valid. 4531 */ 4532 if (ch_flt->flt_diag_data.chd_afar == LOGOUT_INVALID || 4533 ch_flt->flt_data_incomplete) 4534 return; 4535 4536 #if defined(CPU_IMP_ECACHE_ASSOC) 4537 nway = cpu_ecache_nway(); 4538 i = cpu_ecache_line_valid(ch_flt); 4539 if (i == 0 || i > nway) { 4540 for (i = 0; i < nway; i++) 4541 ecp[i].ec_logflag = EC_LOGFLAG_MAGIC; 4542 } else 4543 ecp[i - 1].ec_logflag = EC_LOGFLAG_MAGIC; 4544 #else /* CPU_IMP_ECACHE_ASSOC */ 4545 ecp->ec_logflag = EC_LOGFLAG_MAGIC; 4546 #endif /* CPU_IMP_ECACHE_ASSOC */ 4547 4548 #if defined(CHEETAH_PLUS) 4549 pn_cpu_log_diag_l2_info(ch_flt); 4550 #endif /* CHEETAH_PLUS */ 4551 4552 if (CH_DCTAG_MATCH(dcp->dc_tag, aflt->flt_addr)) { 4553 dcp->dc_way = CH_DCIDX_TO_WAY(dcp->dc_idx); 4554 dcp->dc_logflag = DC_LOGFLAG_MAGIC; 4555 } 4556 4557 if (CH_ICTAG_MATCH(icp, aflt->flt_addr)) { 4558 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 4559 icp->ic_way = PN_ICIDX_TO_WAY(icp->ic_idx); 4560 else 4561 icp->ic_way = CH_ICIDX_TO_WAY(icp->ic_idx); 4562 icp->ic_logflag = IC_LOGFLAG_MAGIC; 4563 } 4564 } 4565 4566 /* 4567 * Cheetah ECC calculation. 4568 * 4569 * We only need to do the calculation on the data bits and can ignore check 4570 * bit and Mtag bit terms in the calculation. 4571 */ 4572 static uint64_t ch_ecc_table[9][2] = { 4573 /* 4574 * low order 64-bits high-order 64-bits 4575 */ 4576 { 0x46bffffeccd1177f, 0x488800022100014c }, 4577 { 0x42fccc81331ff77f, 0x14424f1010249184 }, 4578 { 0x8898827c222f1ffe, 0x22c1222808184aaf }, 4579 { 0xf7632203e131ccf1, 0xe1241121848292b8 }, 4580 { 0x7f5511421b113809, 0x901c88d84288aafe }, 4581 { 0x1d49412184882487, 0x8f338c87c044c6ef }, 4582 { 0xf552181014448344, 0x7ff8f4443e411911 }, 4583 { 0x2189240808f24228, 0xfeeff8cc81333f42 }, 4584 { 0x3280008440001112, 0xfee88b337ffffd62 }, 4585 }; 4586 4587 /* 4588 * 64-bit population count, use well-known popcnt trick. 4589 * We could use the UltraSPARC V9 POPC instruction, but some 4590 * CPUs including Cheetahplus and Jaguar do not support that 4591 * instruction. 4592 */ 4593 int 4594 popc64(uint64_t val) 4595 { 4596 int cnt; 4597 4598 for (cnt = 0; val != 0; val &= val - 1) 4599 cnt++; 4600 return (cnt); 4601 } 4602 4603 /* 4604 * Generate the 9 ECC bits for the 128-bit chunk based on the table above. 4605 * Note that xor'ing an odd number of 1 bits == 1 and xor'ing an even number 4606 * of 1 bits == 0, so we can just use the least significant bit of the popcnt 4607 * instead of doing all the xor's. 4608 */ 4609 uint32_t 4610 us3_gen_ecc(uint64_t data_low, uint64_t data_high) 4611 { 4612 int bitno, s; 4613 int synd = 0; 4614 4615 for (bitno = 0; bitno < 9; bitno++) { 4616 s = (popc64(data_low & ch_ecc_table[bitno][0]) + 4617 popc64(data_high & ch_ecc_table[bitno][1])) & 1; 4618 synd |= (s << bitno); 4619 } 4620 return (synd); 4621 4622 } 4623 4624 /* 4625 * Queue one event based on ecc_type_to_info entry. If the event has an AFT1 4626 * tag associated with it or is a fatal event (aflt_panic set), it is sent to 4627 * the UE event queue. Otherwise it is dispatched to the CE event queue. 4628 */ 4629 static void 4630 cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 4631 ecc_type_to_info_t *eccp, ch_diag_data_t *cdp) 4632 { 4633 struct async_flt *aflt = (struct async_flt *)ch_flt; 4634 4635 if (reason && 4636 strlen(reason) + strlen(eccp->ec_reason) < MAX_REASON_STRING) { 4637 (void) strcat(reason, eccp->ec_reason); 4638 } 4639 4640 ch_flt->flt_bit = eccp->ec_afsr_bit; 4641 ch_flt->flt_type = eccp->ec_flt_type; 4642 if (cdp != NULL && cdp->chd_afar != LOGOUT_INVALID) 4643 ch_flt->flt_diag_data = *cdp; 4644 else 4645 ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 4646 aflt->flt_in_memory = cpu_flt_in_memory(ch_flt, ch_flt->flt_bit); 4647 4648 if (ch_flt->flt_bit & C_AFSR_MSYND_ERRS) 4649 aflt->flt_synd = GET_M_SYND(aflt->flt_stat); 4650 else if (ch_flt->flt_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) 4651 aflt->flt_synd = GET_E_SYND(aflt->flt_stat); 4652 else 4653 aflt->flt_synd = 0; 4654 4655 aflt->flt_payload = eccp->ec_err_payload; 4656 4657 if (aflt->flt_panic || (eccp->ec_afsr_bit & 4658 (C_AFSR_LEVEL1 | C_AFSR_EXT_LEVEL1))) 4659 cpu_errorq_dispatch(eccp->ec_err_class, 4660 (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 4661 aflt->flt_panic); 4662 else 4663 cpu_errorq_dispatch(eccp->ec_err_class, 4664 (void *)ch_flt, sizeof (ch_async_flt_t), ce_queue, 4665 aflt->flt_panic); 4666 } 4667 4668 /* 4669 * Queue events on async event queue one event per error bit. First we 4670 * queue the events that we "expect" for the given trap, then we queue events 4671 * that we may not expect. Return number of events queued. 4672 */ 4673 int 4674 cpu_queue_events(ch_async_flt_t *ch_flt, char *reason, uint64_t t_afsr_errs, 4675 ch_cpu_logout_t *clop) 4676 { 4677 struct async_flt *aflt = (struct async_flt *)ch_flt; 4678 ecc_type_to_info_t *eccp; 4679 int nevents = 0; 4680 uint64_t primary_afar = aflt->flt_addr, primary_afsr = aflt->flt_stat; 4681 #if defined(CHEETAH_PLUS) 4682 uint64_t orig_t_afsr_errs; 4683 #endif 4684 uint64_t primary_afsr_ext = ch_flt->afsr_ext; 4685 uint64_t primary_afsr_errs = ch_flt->afsr_errs; 4686 ch_diag_data_t *cdp = NULL; 4687 4688 t_afsr_errs &= ((C_AFSR_ALL_ERRS & ~C_AFSR_ME) | C_AFSR_EXT_ALL_ERRS); 4689 4690 #if defined(CHEETAH_PLUS) 4691 orig_t_afsr_errs = t_afsr_errs; 4692 4693 /* 4694 * For Cheetah+, log the shadow AFSR/AFAR bits first. 4695 */ 4696 if (clop != NULL) { 4697 /* 4698 * Set the AFSR and AFAR fields to the shadow registers. The 4699 * flt_addr and flt_stat fields will be reset to the primaries 4700 * below, but the sdw_addr and sdw_stat will stay as the 4701 * secondaries. 4702 */ 4703 cdp = &clop->clo_sdw_data; 4704 aflt->flt_addr = ch_flt->flt_sdw_afar = cdp->chd_afar; 4705 aflt->flt_stat = ch_flt->flt_sdw_afsr = cdp->chd_afsr; 4706 ch_flt->afsr_ext = ch_flt->flt_sdw_afsr_ext = cdp->chd_afsr_ext; 4707 ch_flt->afsr_errs = (cdp->chd_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 4708 (cdp->chd_afsr & C_AFSR_ALL_ERRS); 4709 4710 /* 4711 * If the primary and shadow AFSR differ, tag the shadow as 4712 * the first fault. 4713 */ 4714 if ((primary_afar != cdp->chd_afar) || 4715 (primary_afsr_errs != ch_flt->afsr_errs)) { 4716 aflt->flt_stat |= (1ull << C_AFSR_FIRSTFLT_SHIFT); 4717 } 4718 4719 /* 4720 * Check AFSR bits as well as AFSR_EXT bits in order of 4721 * the AFAR overwrite priority. Our stored AFSR_EXT value 4722 * is expected to be zero for those CPUs which do not have 4723 * an AFSR_EXT register. 4724 */ 4725 for (eccp = ecc_type_to_info; eccp->ec_desc != NULL; eccp++) { 4726 if ((eccp->ec_afsr_bit & 4727 (ch_flt->afsr_errs & t_afsr_errs)) && 4728 ((eccp->ec_flags & aflt->flt_status) != 0)) { 4729 cpu_queue_one_event(ch_flt, reason, eccp, cdp); 4730 cdp = NULL; 4731 t_afsr_errs &= ~eccp->ec_afsr_bit; 4732 nevents++; 4733 } 4734 } 4735 4736 /* 4737 * If the ME bit is on in the primary AFSR turn all the 4738 * error bits on again that may set the ME bit to make 4739 * sure we see the ME AFSR error logs. 4740 */ 4741 if ((primary_afsr & C_AFSR_ME) != 0) 4742 t_afsr_errs = (orig_t_afsr_errs & C_AFSR_ALL_ME_ERRS); 4743 } 4744 #endif /* CHEETAH_PLUS */ 4745 4746 if (clop != NULL) 4747 cdp = &clop->clo_data; 4748 4749 /* 4750 * Queue expected errors, error bit and fault type must match 4751 * in the ecc_type_to_info table. 4752 */ 4753 for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 4754 eccp++) { 4755 if ((eccp->ec_afsr_bit & t_afsr_errs) != 0 && 4756 (eccp->ec_flags & aflt->flt_status) != 0) { 4757 #if defined(SERRANO) 4758 /* 4759 * For FRC/FRU errors on Serrano the afar2 captures 4760 * the address and the associated data is 4761 * in the shadow logout area. 4762 */ 4763 if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 4764 if (clop != NULL) 4765 cdp = &clop->clo_sdw_data; 4766 aflt->flt_addr = ch_flt->afar2; 4767 } else { 4768 if (clop != NULL) 4769 cdp = &clop->clo_data; 4770 aflt->flt_addr = primary_afar; 4771 } 4772 #else /* SERRANO */ 4773 aflt->flt_addr = primary_afar; 4774 #endif /* SERRANO */ 4775 aflt->flt_stat = primary_afsr; 4776 ch_flt->afsr_ext = primary_afsr_ext; 4777 ch_flt->afsr_errs = primary_afsr_errs; 4778 cpu_queue_one_event(ch_flt, reason, eccp, cdp); 4779 cdp = NULL; 4780 t_afsr_errs &= ~eccp->ec_afsr_bit; 4781 nevents++; 4782 } 4783 } 4784 4785 /* 4786 * Queue unexpected errors, error bit only match. 4787 */ 4788 for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 4789 eccp++) { 4790 if (eccp->ec_afsr_bit & t_afsr_errs) { 4791 #if defined(SERRANO) 4792 /* 4793 * For FRC/FRU errors on Serrano the afar2 captures 4794 * the address and the associated data is 4795 * in the shadow logout area. 4796 */ 4797 if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 4798 if (clop != NULL) 4799 cdp = &clop->clo_sdw_data; 4800 aflt->flt_addr = ch_flt->afar2; 4801 } else { 4802 if (clop != NULL) 4803 cdp = &clop->clo_data; 4804 aflt->flt_addr = primary_afar; 4805 } 4806 #else /* SERRANO */ 4807 aflt->flt_addr = primary_afar; 4808 #endif /* SERRANO */ 4809 aflt->flt_stat = primary_afsr; 4810 ch_flt->afsr_ext = primary_afsr_ext; 4811 ch_flt->afsr_errs = primary_afsr_errs; 4812 cpu_queue_one_event(ch_flt, reason, eccp, cdp); 4813 cdp = NULL; 4814 t_afsr_errs &= ~eccp->ec_afsr_bit; 4815 nevents++; 4816 } 4817 } 4818 return (nevents); 4819 } 4820 4821 /* 4822 * Return trap type number. 4823 */ 4824 uint8_t 4825 flt_to_trap_type(struct async_flt *aflt) 4826 { 4827 if (aflt->flt_status & ECC_I_TRAP) 4828 return (TRAP_TYPE_ECC_I); 4829 if (aflt->flt_status & ECC_D_TRAP) 4830 return (TRAP_TYPE_ECC_D); 4831 if (aflt->flt_status & ECC_F_TRAP) 4832 return (TRAP_TYPE_ECC_F); 4833 if (aflt->flt_status & ECC_C_TRAP) 4834 return (TRAP_TYPE_ECC_C); 4835 if (aflt->flt_status & ECC_DP_TRAP) 4836 return (TRAP_TYPE_ECC_DP); 4837 if (aflt->flt_status & ECC_IP_TRAP) 4838 return (TRAP_TYPE_ECC_IP); 4839 if (aflt->flt_status & ECC_ITLB_TRAP) 4840 return (TRAP_TYPE_ECC_ITLB); 4841 if (aflt->flt_status & ECC_DTLB_TRAP) 4842 return (TRAP_TYPE_ECC_DTLB); 4843 return (TRAP_TYPE_UNKNOWN); 4844 } 4845 4846 /* 4847 * Decide an error type based on detector and leaky/partner tests. 4848 * The following array is used for quick translation - it must 4849 * stay in sync with ce_dispact_t. 4850 */ 4851 4852 static char *cetypes[] = { 4853 CE_DISP_DESC_U, 4854 CE_DISP_DESC_I, 4855 CE_DISP_DESC_PP, 4856 CE_DISP_DESC_P, 4857 CE_DISP_DESC_L, 4858 CE_DISP_DESC_PS, 4859 CE_DISP_DESC_S 4860 }; 4861 4862 char * 4863 flt_to_error_type(struct async_flt *aflt) 4864 { 4865 ce_dispact_t dispact, disp; 4866 uchar_t dtcrinfo, ptnrinfo, lkyinfo; 4867 4868 /* 4869 * The memory payload bundle is shared by some events that do 4870 * not perform any classification. For those flt_disp will be 4871 * 0 and we will return "unknown". 4872 */ 4873 if (!ce_disp_inited || !aflt->flt_in_memory || aflt->flt_disp == 0) 4874 return (cetypes[CE_DISP_UNKNOWN]); 4875 4876 dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 4877 4878 /* 4879 * It is also possible that no scrub/classification was performed 4880 * by the detector, for instance where a disrupting error logged 4881 * in the AFSR while CEEN was off in cpu_deferred_error. 4882 */ 4883 if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) 4884 return (cetypes[CE_DISP_UNKNOWN]); 4885 4886 /* 4887 * Lookup type in initial classification/action table 4888 */ 4889 dispact = CE_DISPACT(ce_disp_table, 4890 CE_XDIAG_AFARMATCHED(dtcrinfo), 4891 CE_XDIAG_STATE(dtcrinfo), 4892 CE_XDIAG_CE1SEEN(dtcrinfo), 4893 CE_XDIAG_CE2SEEN(dtcrinfo)); 4894 4895 /* 4896 * A bad lookup is not something to panic production systems for. 4897 */ 4898 ASSERT(dispact != CE_DISP_BAD); 4899 if (dispact == CE_DISP_BAD) 4900 return (cetypes[CE_DISP_UNKNOWN]); 4901 4902 disp = CE_DISP(dispact); 4903 4904 switch (disp) { 4905 case CE_DISP_UNKNOWN: 4906 case CE_DISP_INTERMITTENT: 4907 break; 4908 4909 case CE_DISP_POSS_PERS: 4910 /* 4911 * "Possible persistent" errors to which we have applied a valid 4912 * leaky test can be separated into "persistent" or "leaky". 4913 */ 4914 lkyinfo = CE_XDIAG_LKYINFO(aflt->flt_disp); 4915 if (CE_XDIAG_TESTVALID(lkyinfo)) { 4916 if (CE_XDIAG_CE1SEEN(lkyinfo) || 4917 CE_XDIAG_CE2SEEN(lkyinfo)) 4918 disp = CE_DISP_LEAKY; 4919 else 4920 disp = CE_DISP_PERS; 4921 } 4922 break; 4923 4924 case CE_DISP_POSS_STICKY: 4925 /* 4926 * Promote "possible sticky" results that have been 4927 * confirmed by a partner test to "sticky". Unconfirmed 4928 * "possible sticky" events are left at that status - we do not 4929 * guess at any bad reader/writer etc status here. 4930 */ 4931 ptnrinfo = CE_XDIAG_PTNRINFO(aflt->flt_disp); 4932 if (CE_XDIAG_TESTVALID(ptnrinfo) && 4933 CE_XDIAG_CE1SEEN(ptnrinfo) && CE_XDIAG_CE2SEEN(ptnrinfo)) 4934 disp = CE_DISP_STICKY; 4935 4936 /* 4937 * Promote "possible sticky" results on a uniprocessor 4938 * to "sticky" 4939 */ 4940 if (disp == CE_DISP_POSS_STICKY && 4941 CE_XDIAG_SKIPCODE(disp) == CE_XDIAG_SKIP_UNIPROC) 4942 disp = CE_DISP_STICKY; 4943 break; 4944 4945 default: 4946 disp = CE_DISP_UNKNOWN; 4947 break; 4948 } 4949 4950 return (cetypes[disp]); 4951 } 4952 4953 /* 4954 * Given the entire afsr, the specific bit to check and a prioritized list of 4955 * error bits, determine the validity of the various overwrite priority 4956 * features of the AFSR/AFAR: AFAR, ESYND and MSYND, each of which have 4957 * different overwrite priorities. 4958 * 4959 * Given a specific afsr error bit and the entire afsr, there are three cases: 4960 * INVALID: The specified bit is lower overwrite priority than some other 4961 * error bit which is on in the afsr (or IVU/IVC). 4962 * VALID: The specified bit is higher priority than all other error bits 4963 * which are on in the afsr. 4964 * AMBIGUOUS: Another error bit (or bits) of equal priority to the specified 4965 * bit is on in the afsr. 4966 */ 4967 int 4968 afsr_to_overw_status(uint64_t afsr, uint64_t afsr_bit, uint64_t *ow_bits) 4969 { 4970 uint64_t afsr_ow; 4971 4972 while ((afsr_ow = *ow_bits++) != 0) { 4973 /* 4974 * If bit is in the priority class, check to see if another 4975 * bit in the same class is on => ambiguous. Otherwise, 4976 * the value is valid. If the bit is not on at this priority 4977 * class, but a higher priority bit is on, then the value is 4978 * invalid. 4979 */ 4980 if (afsr_ow & afsr_bit) { 4981 /* 4982 * If equal pri bit is on, ambiguous. 4983 */ 4984 if (afsr & (afsr_ow & ~afsr_bit)) 4985 return (AFLT_STAT_AMBIGUOUS); 4986 return (AFLT_STAT_VALID); 4987 } else if (afsr & afsr_ow) 4988 break; 4989 } 4990 4991 /* 4992 * We didn't find a match or a higher priority bit was on. Not 4993 * finding a match handles the case of invalid AFAR for IVC, IVU. 4994 */ 4995 return (AFLT_STAT_INVALID); 4996 } 4997 4998 static int 4999 afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit) 5000 { 5001 #if defined(SERRANO) 5002 if (afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) 5003 return (afsr_to_overw_status(afsr, afsr_bit, afar2_overwrite)); 5004 else 5005 #endif /* SERRANO */ 5006 return (afsr_to_overw_status(afsr, afsr_bit, afar_overwrite)); 5007 } 5008 5009 static int 5010 afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit) 5011 { 5012 return (afsr_to_overw_status(afsr, afsr_bit, esynd_overwrite)); 5013 } 5014 5015 static int 5016 afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit) 5017 { 5018 return (afsr_to_overw_status(afsr, afsr_bit, msynd_overwrite)); 5019 } 5020 5021 static int 5022 afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit) 5023 { 5024 #ifdef lint 5025 cpuid = cpuid; 5026 #endif 5027 if (afsr_bit & C_AFSR_MSYND_ERRS) { 5028 return (afsr_to_msynd_status(afsr, afsr_bit)); 5029 } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 5030 #if defined(CHEETAH_PLUS) 5031 /* 5032 * The E_SYND overwrite policy is slightly different 5033 * for Panther CPUs. 5034 */ 5035 if (IS_PANTHER(cpunodes[cpuid].implementation)) 5036 return (afsr_to_pn_esynd_status(afsr, afsr_bit)); 5037 else 5038 return (afsr_to_esynd_status(afsr, afsr_bit)); 5039 #else /* CHEETAH_PLUS */ 5040 return (afsr_to_esynd_status(afsr, afsr_bit)); 5041 #endif /* CHEETAH_PLUS */ 5042 } else { 5043 return (AFLT_STAT_INVALID); 5044 } 5045 } 5046 5047 /* 5048 * Slave CPU stick synchronization. 5049 */ 5050 void 5051 sticksync_slave(void) 5052 { 5053 int i; 5054 int tries = 0; 5055 int64_t tskew; 5056 int64_t av_tskew; 5057 5058 kpreempt_disable(); 5059 /* wait for the master side */ 5060 while (stick_sync_cmd != SLAVE_START) 5061 ; 5062 /* 5063 * Synchronization should only take a few tries at most. But in the 5064 * odd case where the cpu isn't cooperating we'll keep trying. A cpu 5065 * without it's stick synchronized wouldn't be a good citizen. 5066 */ 5067 while (slave_done == 0) { 5068 /* 5069 * Time skew calculation. 5070 */ 5071 av_tskew = tskew = 0; 5072 5073 for (i = 0; i < stick_iter; i++) { 5074 /* make location hot */ 5075 timestamp[EV_A_START] = 0; 5076 stick_timestamp(×tamp[EV_A_START]); 5077 5078 /* tell the master we're ready */ 5079 stick_sync_cmd = MASTER_START; 5080 5081 /* and wait */ 5082 while (stick_sync_cmd != SLAVE_CONT) 5083 ; 5084 /* Event B end */ 5085 stick_timestamp(×tamp[EV_B_END]); 5086 5087 /* calculate time skew */ 5088 tskew = ((timestamp[EV_B_END] - timestamp[EV_B_START]) 5089 - (timestamp[EV_A_END] - 5090 timestamp[EV_A_START])) / 2; 5091 5092 /* keep running count */ 5093 av_tskew += tskew; 5094 } /* for */ 5095 5096 /* 5097 * Adjust stick for time skew if not within the max allowed; 5098 * otherwise we're all done. 5099 */ 5100 if (stick_iter != 0) 5101 av_tskew = av_tskew/stick_iter; 5102 if (ABS(av_tskew) > stick_tsk) { 5103 /* 5104 * If the skew is 1 (the slave's STICK register 5105 * is 1 STICK ahead of the master's), stick_adj 5106 * could fail to adjust the slave's STICK register 5107 * if the STICK read on the slave happens to 5108 * align with the increment of the STICK. 5109 * Therefore, we increment the skew to 2. 5110 */ 5111 if (av_tskew == 1) 5112 av_tskew++; 5113 stick_adj(-av_tskew); 5114 } else 5115 slave_done = 1; 5116 #ifdef DEBUG 5117 if (tries < DSYNC_ATTEMPTS) 5118 stick_sync_stats[CPU->cpu_id].skew_val[tries] = 5119 av_tskew; 5120 ++tries; 5121 #endif /* DEBUG */ 5122 #ifdef lint 5123 tries = tries; 5124 #endif 5125 5126 } /* while */ 5127 5128 /* allow the master to finish */ 5129 stick_sync_cmd = EVENT_NULL; 5130 kpreempt_enable(); 5131 } 5132 5133 /* 5134 * Master CPU side of stick synchronization. 5135 * - timestamp end of Event A 5136 * - timestamp beginning of Event B 5137 */ 5138 void 5139 sticksync_master(void) 5140 { 5141 int i; 5142 5143 kpreempt_disable(); 5144 /* tell the slave we've started */ 5145 slave_done = 0; 5146 stick_sync_cmd = SLAVE_START; 5147 5148 while (slave_done == 0) { 5149 for (i = 0; i < stick_iter; i++) { 5150 /* wait for the slave */ 5151 while (stick_sync_cmd != MASTER_START) 5152 ; 5153 /* Event A end */ 5154 stick_timestamp(×tamp[EV_A_END]); 5155 5156 /* make location hot */ 5157 timestamp[EV_B_START] = 0; 5158 stick_timestamp(×tamp[EV_B_START]); 5159 5160 /* tell the slave to continue */ 5161 stick_sync_cmd = SLAVE_CONT; 5162 } /* for */ 5163 5164 /* wait while slave calculates time skew */ 5165 while (stick_sync_cmd == SLAVE_CONT) 5166 ; 5167 } /* while */ 5168 kpreempt_enable(); 5169 } 5170 5171 /* 5172 * Cheetah/Cheetah+ have disrupting error for copyback's, so we don't need to 5173 * do Spitfire hack of xcall'ing all the cpus to ask to check for them. Also, 5174 * in cpu_async_panic_callb, each cpu checks for CPU events on its way to 5175 * panic idle. 5176 */ 5177 /*ARGSUSED*/ 5178 void 5179 cpu_check_allcpus(struct async_flt *aflt) 5180 {} 5181 5182 struct kmem_cache *ch_private_cache; 5183 5184 /* 5185 * Cpu private unitialization. Uninitialize the Ecache scrubber and 5186 * deallocate the scrubber data structures and cpu_private data structure. 5187 */ 5188 void 5189 cpu_uninit_private(struct cpu *cp) 5190 { 5191 cheetah_private_t *chprp = CPU_PRIVATE(cp); 5192 5193 ASSERT(chprp); 5194 cpu_uninit_ecache_scrub_dr(cp); 5195 CPU_PRIVATE(cp) = NULL; 5196 ch_err_tl1_paddrs[cp->cpu_id] = NULL; 5197 kmem_cache_free(ch_private_cache, chprp); 5198 cmp_delete_cpu(cp->cpu_id); 5199 5200 } 5201 5202 /* 5203 * Cheetah Cache Scrubbing 5204 * 5205 * The primary purpose of Cheetah cache scrubbing is to reduce the exposure 5206 * of E$ tags, D$ data, and I$ data to cosmic ray events since they are not 5207 * protected by either parity or ECC. 5208 * 5209 * We currently default the E$ and D$ scan rate to 100 (scan 10% of the 5210 * cache per second). Due to the the specifics of how the I$ control 5211 * logic works with respect to the ASI used to scrub I$ lines, the entire 5212 * I$ is scanned at once. 5213 */ 5214 5215 /* 5216 * Tuneables to enable and disable the scrubbing of the caches, and to tune 5217 * scrubbing behavior. These may be changed via /etc/system or using mdb 5218 * on a running system. 5219 */ 5220 int dcache_scrub_enable = 1; /* D$ scrubbing is on by default */ 5221 5222 /* 5223 * The following are the PIL levels that the softints/cross traps will fire at. 5224 */ 5225 uint_t ecache_scrub_pil = PIL_9; /* E$ scrub PIL for cross traps */ 5226 uint_t dcache_scrub_pil = PIL_9; /* D$ scrub PIL for cross traps */ 5227 uint_t icache_scrub_pil = PIL_9; /* I$ scrub PIL for cross traps */ 5228 5229 #if defined(JALAPENO) 5230 5231 /* 5232 * Due to several errata (82, 85, 86), we don't enable the L2$ scrubber 5233 * on Jalapeno. 5234 */ 5235 int ecache_scrub_enable = 0; 5236 5237 #else /* JALAPENO */ 5238 5239 /* 5240 * With all other cpu types, E$ scrubbing is on by default 5241 */ 5242 int ecache_scrub_enable = 1; 5243 5244 #endif /* JALAPENO */ 5245 5246 5247 #if defined(CHEETAH_PLUS) || defined(JALAPENO) || defined(SERRANO) 5248 5249 /* 5250 * The I$ scrubber tends to cause latency problems for real-time SW, so it 5251 * is disabled by default on non-Cheetah systems 5252 */ 5253 int icache_scrub_enable = 0; 5254 5255 /* 5256 * Tuneables specifying the scrub calls per second and the scan rate 5257 * for each cache 5258 * 5259 * The cyclic times are set during boot based on the following values. 5260 * Changing these values in mdb after this time will have no effect. If 5261 * a different value is desired, it must be set in /etc/system before a 5262 * reboot. 5263 */ 5264 int ecache_calls_a_sec = 1; 5265 int dcache_calls_a_sec = 2; 5266 int icache_calls_a_sec = 2; 5267 5268 int ecache_scan_rate_idle = 1; 5269 int ecache_scan_rate_busy = 1; 5270 int dcache_scan_rate_idle = 1; 5271 int dcache_scan_rate_busy = 1; 5272 int icache_scan_rate_idle = 1; 5273 int icache_scan_rate_busy = 1; 5274 5275 #else /* CHEETAH_PLUS || JALAPENO || SERRANO */ 5276 5277 int icache_scrub_enable = 1; /* I$ scrubbing is on by default */ 5278 5279 int ecache_calls_a_sec = 100; /* E$ scrub calls per seconds */ 5280 int dcache_calls_a_sec = 100; /* D$ scrub calls per seconds */ 5281 int icache_calls_a_sec = 100; /* I$ scrub calls per seconds */ 5282 5283 int ecache_scan_rate_idle = 100; /* E$ scan rate (in tenths of a %) */ 5284 int ecache_scan_rate_busy = 100; /* E$ scan rate (in tenths of a %) */ 5285 int dcache_scan_rate_idle = 100; /* D$ scan rate (in tenths of a %) */ 5286 int dcache_scan_rate_busy = 100; /* D$ scan rate (in tenths of a %) */ 5287 int icache_scan_rate_idle = 100; /* I$ scan rate (in tenths of a %) */ 5288 int icache_scan_rate_busy = 100; /* I$ scan rate (in tenths of a %) */ 5289 5290 #endif /* CHEETAH_PLUS || JALAPENO || SERRANO */ 5291 5292 /* 5293 * In order to scrub on offline cpus, a cross trap is sent. The handler will 5294 * increment the outstanding request counter and schedule a softint to run 5295 * the scrubber. 5296 */ 5297 extern xcfunc_t cache_scrubreq_tl1; 5298 5299 /* 5300 * These are the softint functions for each cache scrubber 5301 */ 5302 static uint_t scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2); 5303 static uint_t scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2); 5304 static uint_t scrub_icache_line_intr(caddr_t arg1, caddr_t arg2); 5305 5306 /* 5307 * The cache scrub info table contains cache specific information 5308 * and allows for some of the scrub code to be table driven, reducing 5309 * duplication of cache similar code. 5310 * 5311 * This table keeps a copy of the value in the calls per second variable 5312 * (?cache_calls_a_sec). This makes it much more difficult for someone 5313 * to cause us problems (for example, by setting ecache_calls_a_sec to 0 in 5314 * mdb in a misguided attempt to disable the scrubber). 5315 */ 5316 struct scrub_info { 5317 int *csi_enable; /* scrubber enable flag */ 5318 int csi_freq; /* scrubber calls per second */ 5319 int csi_index; /* index to chsm_outstanding[] */ 5320 uint_t csi_inum; /* scrubber interrupt number */ 5321 cyclic_id_t csi_omni_cyc_id; /* omni cyclic ID */ 5322 cyclic_id_t csi_offline_cyc_id; /* offline cyclic ID */ 5323 char csi_name[3]; /* cache name for this scrub entry */ 5324 } cache_scrub_info[] = { 5325 { &ecache_scrub_enable, 0, CACHE_SCRUBBER_INFO_E, 0, 0, 0, "E$"}, 5326 { &dcache_scrub_enable, 0, CACHE_SCRUBBER_INFO_D, 0, 0, 0, "D$"}, 5327 { &icache_scrub_enable, 0, CACHE_SCRUBBER_INFO_I, 0, 0, 0, "I$"} 5328 }; 5329 5330 /* 5331 * If scrubbing is enabled, increment the outstanding request counter. If it 5332 * is 1 (meaning there were no previous requests outstanding), call 5333 * setsoftint_tl1 through xt_one_unchecked, which eventually ends up doing 5334 * a self trap. 5335 */ 5336 static void 5337 do_scrub(struct scrub_info *csi) 5338 { 5339 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5340 int index = csi->csi_index; 5341 uint32_t *outstanding = &csmp->chsm_outstanding[index]; 5342 5343 if (*(csi->csi_enable) && (csmp->chsm_enable[index])) { 5344 if (atomic_add_32_nv(outstanding, 1) == 1) { 5345 xt_one_unchecked(CPU->cpu_id, setsoftint_tl1, 5346 csi->csi_inum, 0); 5347 } 5348 } 5349 } 5350 5351 /* 5352 * Omni cyclics don't fire on offline cpus, so we use another cyclic to 5353 * cross-trap the offline cpus. 5354 */ 5355 static void 5356 do_scrub_offline(struct scrub_info *csi) 5357 { 5358 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5359 5360 if (CPUSET_ISNULL(cpu_offline_set)) { 5361 /* 5362 * No offline cpus - nothing to do 5363 */ 5364 return; 5365 } 5366 5367 if (*(csi->csi_enable) && (csmp->chsm_enable[csi->csi_index])) { 5368 xt_some(cpu_offline_set, cache_scrubreq_tl1, csi->csi_inum, 5369 csi->csi_index); 5370 } 5371 } 5372 5373 /* 5374 * This is the initial setup for the scrubber cyclics - it sets the 5375 * interrupt level, frequency, and function to call. 5376 */ 5377 /*ARGSUSED*/ 5378 static void 5379 cpu_scrub_cyclic_setup(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 5380 cyc_time_t *when) 5381 { 5382 struct scrub_info *csi = (struct scrub_info *)arg; 5383 5384 ASSERT(csi != NULL); 5385 hdlr->cyh_func = (cyc_func_t)do_scrub; 5386 hdlr->cyh_level = CY_LOW_LEVEL; 5387 hdlr->cyh_arg = arg; 5388 5389 when->cyt_when = 0; /* Start immediately */ 5390 when->cyt_interval = NANOSEC / csi->csi_freq; 5391 } 5392 5393 /* 5394 * Initialization for cache scrubbing. 5395 * This routine is called AFTER all cpus have had cpu_init_private called 5396 * to initialize their private data areas. 5397 */ 5398 void 5399 cpu_init_cache_scrub(void) 5400 { 5401 int i; 5402 struct scrub_info *csi; 5403 cyc_omni_handler_t omni_hdlr; 5404 cyc_handler_t offline_hdlr; 5405 cyc_time_t when; 5406 5407 /* 5408 * save away the maximum number of lines for the D$ 5409 */ 5410 dcache_nlines = dcache_size / dcache_linesize; 5411 5412 /* 5413 * register the softints for the cache scrubbing 5414 */ 5415 cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_inum = 5416 add_softintr(ecache_scrub_pil, scrub_ecache_line_intr, 5417 (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_E]); 5418 cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_freq = ecache_calls_a_sec; 5419 5420 cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_inum = 5421 add_softintr(dcache_scrub_pil, scrub_dcache_line_intr, 5422 (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_D]); 5423 cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_freq = dcache_calls_a_sec; 5424 5425 cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_inum = 5426 add_softintr(icache_scrub_pil, scrub_icache_line_intr, 5427 (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_I]); 5428 cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_freq = icache_calls_a_sec; 5429 5430 /* 5431 * start the scrubbing for all the caches 5432 */ 5433 mutex_enter(&cpu_lock); 5434 for (i = 0; i < CACHE_SCRUBBER_COUNT; i++) { 5435 5436 csi = &cache_scrub_info[i]; 5437 5438 if (!(*csi->csi_enable)) 5439 continue; 5440 5441 /* 5442 * force the following to be true: 5443 * 1 <= calls_a_sec <= hz 5444 */ 5445 if (csi->csi_freq > hz) { 5446 cmn_err(CE_NOTE, "%s scrub calls_a_sec set too high " 5447 "(%d); resetting to hz (%d)", csi->csi_name, 5448 csi->csi_freq, hz); 5449 csi->csi_freq = hz; 5450 } else if (csi->csi_freq < 1) { 5451 cmn_err(CE_NOTE, "%s scrub calls_a_sec set too low " 5452 "(%d); resetting to 1", csi->csi_name, 5453 csi->csi_freq); 5454 csi->csi_freq = 1; 5455 } 5456 5457 omni_hdlr.cyo_online = cpu_scrub_cyclic_setup; 5458 omni_hdlr.cyo_offline = NULL; 5459 omni_hdlr.cyo_arg = (void *)csi; 5460 5461 offline_hdlr.cyh_func = (cyc_func_t)do_scrub_offline; 5462 offline_hdlr.cyh_arg = (void *)csi; 5463 offline_hdlr.cyh_level = CY_LOW_LEVEL; 5464 5465 when.cyt_when = 0; /* Start immediately */ 5466 when.cyt_interval = NANOSEC / csi->csi_freq; 5467 5468 csi->csi_omni_cyc_id = cyclic_add_omni(&omni_hdlr); 5469 csi->csi_offline_cyc_id = cyclic_add(&offline_hdlr, &when); 5470 } 5471 register_cpu_setup_func(cpu_scrub_cpu_setup, NULL); 5472 mutex_exit(&cpu_lock); 5473 } 5474 5475 /* 5476 * Indicate that the specified cpu is idle. 5477 */ 5478 void 5479 cpu_idle_ecache_scrub(struct cpu *cp) 5480 { 5481 if (CPU_PRIVATE(cp) != NULL) { 5482 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 5483 csmp->chsm_ecache_busy = ECACHE_CPU_IDLE; 5484 } 5485 } 5486 5487 /* 5488 * Indicate that the specified cpu is busy. 5489 */ 5490 void 5491 cpu_busy_ecache_scrub(struct cpu *cp) 5492 { 5493 if (CPU_PRIVATE(cp) != NULL) { 5494 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 5495 csmp->chsm_ecache_busy = ECACHE_CPU_BUSY; 5496 } 5497 } 5498 5499 /* 5500 * Initialization for cache scrubbing for the specified cpu. 5501 */ 5502 void 5503 cpu_init_ecache_scrub_dr(struct cpu *cp) 5504 { 5505 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 5506 int cpuid = cp->cpu_id; 5507 5508 /* initialize the number of lines in the caches */ 5509 csmp->chsm_ecache_nlines = cpunodes[cpuid].ecache_size / 5510 cpunodes[cpuid].ecache_linesize; 5511 csmp->chsm_icache_nlines = CPU_PRIVATE_VAL(cp, chpr_icache_size) / 5512 CPU_PRIVATE_VAL(cp, chpr_icache_linesize); 5513 5514 /* 5515 * do_scrub() and do_scrub_offline() check both the global 5516 * ?cache_scrub_enable and this per-cpu enable variable. All scrubbers 5517 * check this value before scrubbing. Currently, we use it to 5518 * disable the E$ scrubber on multi-core cpus or while running at 5519 * slowed speed. For now, just turn everything on and allow 5520 * cpu_init_private() to change it if necessary. 5521 */ 5522 csmp->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1; 5523 csmp->chsm_enable[CACHE_SCRUBBER_INFO_D] = 1; 5524 csmp->chsm_enable[CACHE_SCRUBBER_INFO_I] = 1; 5525 5526 cpu_busy_ecache_scrub(cp); 5527 } 5528 5529 /* 5530 * Un-initialization for cache scrubbing for the specified cpu. 5531 */ 5532 static void 5533 cpu_uninit_ecache_scrub_dr(struct cpu *cp) 5534 { 5535 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 5536 5537 /* 5538 * un-initialize bookkeeping for cache scrubbing 5539 */ 5540 bzero(csmp, sizeof (ch_scrub_misc_t)); 5541 5542 cpu_idle_ecache_scrub(cp); 5543 } 5544 5545 /* 5546 * Called periodically on each CPU to scrub the D$. 5547 */ 5548 static void 5549 scrub_dcache(int how_many) 5550 { 5551 int i; 5552 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5553 int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D]; 5554 5555 /* 5556 * scrub the desired number of lines 5557 */ 5558 for (i = 0; i < how_many; i++) { 5559 /* 5560 * scrub a D$ line 5561 */ 5562 dcache_inval_line(index); 5563 5564 /* 5565 * calculate the next D$ line to scrub, assumes 5566 * that dcache_nlines is a power of 2 5567 */ 5568 index = (index + 1) & (dcache_nlines - 1); 5569 } 5570 5571 /* 5572 * set the scrub index for the next visit 5573 */ 5574 csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D] = index; 5575 } 5576 5577 /* 5578 * Handler for D$ scrub inum softint. Call scrub_dcache until 5579 * we decrement the outstanding request count to zero. 5580 */ 5581 /*ARGSUSED*/ 5582 static uint_t 5583 scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2) 5584 { 5585 int i; 5586 int how_many; 5587 int outstanding; 5588 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5589 uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_D]; 5590 struct scrub_info *csi = (struct scrub_info *)arg1; 5591 int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 5592 dcache_scan_rate_idle : dcache_scan_rate_busy; 5593 5594 /* 5595 * The scan rates are expressed in units of tenths of a 5596 * percent. A scan rate of 1000 (100%) means the whole 5597 * cache is scanned every second. 5598 */ 5599 how_many = (dcache_nlines * scan_rate) / (1000 * csi->csi_freq); 5600 5601 do { 5602 outstanding = *countp; 5603 for (i = 0; i < outstanding; i++) { 5604 scrub_dcache(how_many); 5605 } 5606 } while (atomic_add_32_nv(countp, -outstanding)); 5607 5608 return (DDI_INTR_CLAIMED); 5609 } 5610 5611 /* 5612 * Called periodically on each CPU to scrub the I$. The I$ is scrubbed 5613 * by invalidating lines. Due to the characteristics of the ASI which 5614 * is used to invalidate an I$ line, the entire I$ must be invalidated 5615 * vs. an individual I$ line. 5616 */ 5617 static void 5618 scrub_icache(int how_many) 5619 { 5620 int i; 5621 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5622 int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I]; 5623 int icache_nlines = csmp->chsm_icache_nlines; 5624 5625 /* 5626 * scrub the desired number of lines 5627 */ 5628 for (i = 0; i < how_many; i++) { 5629 /* 5630 * since the entire I$ must be scrubbed at once, 5631 * wait until the index wraps to zero to invalidate 5632 * the entire I$ 5633 */ 5634 if (index == 0) { 5635 icache_inval_all(); 5636 } 5637 5638 /* 5639 * calculate the next I$ line to scrub, assumes 5640 * that chsm_icache_nlines is a power of 2 5641 */ 5642 index = (index + 1) & (icache_nlines - 1); 5643 } 5644 5645 /* 5646 * set the scrub index for the next visit 5647 */ 5648 csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I] = index; 5649 } 5650 5651 /* 5652 * Handler for I$ scrub inum softint. Call scrub_icache until 5653 * we decrement the outstanding request count to zero. 5654 */ 5655 /*ARGSUSED*/ 5656 static uint_t 5657 scrub_icache_line_intr(caddr_t arg1, caddr_t arg2) 5658 { 5659 int i; 5660 int how_many; 5661 int outstanding; 5662 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5663 uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_I]; 5664 struct scrub_info *csi = (struct scrub_info *)arg1; 5665 int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 5666 icache_scan_rate_idle : icache_scan_rate_busy; 5667 int icache_nlines = csmp->chsm_icache_nlines; 5668 5669 /* 5670 * The scan rates are expressed in units of tenths of a 5671 * percent. A scan rate of 1000 (100%) means the whole 5672 * cache is scanned every second. 5673 */ 5674 how_many = (icache_nlines * scan_rate) / (1000 * csi->csi_freq); 5675 5676 do { 5677 outstanding = *countp; 5678 for (i = 0; i < outstanding; i++) { 5679 scrub_icache(how_many); 5680 } 5681 } while (atomic_add_32_nv(countp, -outstanding)); 5682 5683 return (DDI_INTR_CLAIMED); 5684 } 5685 5686 /* 5687 * Called periodically on each CPU to scrub the E$. 5688 */ 5689 static void 5690 scrub_ecache(int how_many) 5691 { 5692 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5693 int i; 5694 int cpuid = CPU->cpu_id; 5695 int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E]; 5696 int nlines = csmp->chsm_ecache_nlines; 5697 int linesize = cpunodes[cpuid].ecache_linesize; 5698 int ec_set_size = cpu_ecache_set_size(CPU); 5699 5700 /* 5701 * scrub the desired number of lines 5702 */ 5703 for (i = 0; i < how_many; i++) { 5704 /* 5705 * scrub the E$ line 5706 */ 5707 ecache_flush_line(ecache_flushaddr + (index * linesize), 5708 ec_set_size); 5709 5710 /* 5711 * calculate the next E$ line to scrub based on twice 5712 * the number of E$ lines (to displace lines containing 5713 * flush area data), assumes that the number of lines 5714 * is a power of 2 5715 */ 5716 index = (index + 1) & ((nlines << 1) - 1); 5717 } 5718 5719 /* 5720 * set the ecache scrub index for the next visit 5721 */ 5722 csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E] = index; 5723 } 5724 5725 /* 5726 * Handler for E$ scrub inum softint. Call the E$ scrubber until 5727 * we decrement the outstanding request count to zero. 5728 * 5729 * Due to interactions with cpu_scrub_cpu_setup(), the outstanding count may 5730 * become negative after the atomic_add_32_nv(). This is not a problem, as 5731 * the next trip around the loop won't scrub anything, and the next add will 5732 * reset the count back to zero. 5733 */ 5734 /*ARGSUSED*/ 5735 static uint_t 5736 scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2) 5737 { 5738 int i; 5739 int how_many; 5740 int outstanding; 5741 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5742 uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_E]; 5743 struct scrub_info *csi = (struct scrub_info *)arg1; 5744 int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 5745 ecache_scan_rate_idle : ecache_scan_rate_busy; 5746 int ecache_nlines = csmp->chsm_ecache_nlines; 5747 5748 /* 5749 * The scan rates are expressed in units of tenths of a 5750 * percent. A scan rate of 1000 (100%) means the whole 5751 * cache is scanned every second. 5752 */ 5753 how_many = (ecache_nlines * scan_rate) / (1000 * csi->csi_freq); 5754 5755 do { 5756 outstanding = *countp; 5757 for (i = 0; i < outstanding; i++) { 5758 scrub_ecache(how_many); 5759 } 5760 } while (atomic_add_32_nv(countp, -outstanding)); 5761 5762 return (DDI_INTR_CLAIMED); 5763 } 5764 5765 /* 5766 * Timeout function to reenable CE 5767 */ 5768 static void 5769 cpu_delayed_check_ce_errors(void *arg) 5770 { 5771 if (!taskq_dispatch(ch_check_ce_tq, cpu_check_ce_errors, arg, 5772 TQ_NOSLEEP)) { 5773 (void) timeout(cpu_delayed_check_ce_errors, arg, 5774 drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 5775 } 5776 } 5777 5778 /* 5779 * CE Deferred Re-enable after trap. 5780 * 5781 * When the CPU gets a disrupting trap for any of the errors 5782 * controlled by the CEEN bit, CEEN is disabled in the trap handler 5783 * immediately. To eliminate the possibility of multiple CEs causing 5784 * recursive stack overflow in the trap handler, we cannot 5785 * reenable CEEN while still running in the trap handler. Instead, 5786 * after a CE is logged on a CPU, we schedule a timeout function, 5787 * cpu_check_ce_errors(), to trigger after cpu_ceen_delay_secs 5788 * seconds. This function will check whether any further CEs 5789 * have occurred on that CPU, and if none have, will reenable CEEN. 5790 * 5791 * If further CEs have occurred while CEEN is disabled, another 5792 * timeout will be scheduled. This is to ensure that the CPU can 5793 * make progress in the face of CE 'storms', and that it does not 5794 * spend all its time logging CE errors. 5795 */ 5796 static void 5797 cpu_check_ce_errors(void *arg) 5798 { 5799 int cpuid = (int)(uintptr_t)arg; 5800 cpu_t *cp; 5801 5802 /* 5803 * We acquire cpu_lock. 5804 */ 5805 ASSERT(curthread->t_pil == 0); 5806 5807 /* 5808 * verify that the cpu is still around, DR 5809 * could have got there first ... 5810 */ 5811 mutex_enter(&cpu_lock); 5812 cp = cpu_get(cpuid); 5813 if (cp == NULL) { 5814 mutex_exit(&cpu_lock); 5815 return; 5816 } 5817 /* 5818 * make sure we don't migrate across CPUs 5819 * while checking our CE status. 5820 */ 5821 kpreempt_disable(); 5822 5823 /* 5824 * If we are running on the CPU that got the 5825 * CE, we can do the checks directly. 5826 */ 5827 if (cp->cpu_id == CPU->cpu_id) { 5828 mutex_exit(&cpu_lock); 5829 cpu_check_ce(TIMEOUT_CEEN_CHECK, 0, 0, 0); 5830 kpreempt_enable(); 5831 return; 5832 } 5833 kpreempt_enable(); 5834 5835 /* 5836 * send an x-call to get the CPU that originally 5837 * got the CE to do the necessary checks. If we can't 5838 * send the x-call, reschedule the timeout, otherwise we 5839 * lose CEEN forever on that CPU. 5840 */ 5841 if (CPU_XCALL_READY(cp->cpu_id) && (!(cp->cpu_flags & CPU_QUIESCED))) { 5842 xc_one(cp->cpu_id, (xcfunc_t *)cpu_check_ce, 5843 TIMEOUT_CEEN_CHECK, 0); 5844 mutex_exit(&cpu_lock); 5845 } else { 5846 /* 5847 * When the CPU is not accepting xcalls, or 5848 * the processor is offlined, we don't want to 5849 * incur the extra overhead of trying to schedule the 5850 * CE timeout indefinitely. However, we don't want to lose 5851 * CE checking forever. 5852 * 5853 * Keep rescheduling the timeout, accepting the additional 5854 * overhead as the cost of correctness in the case where we get 5855 * a CE, disable CEEN, offline the CPU during the 5856 * the timeout interval, and then online it at some 5857 * point in the future. This is unlikely given the short 5858 * cpu_ceen_delay_secs. 5859 */ 5860 mutex_exit(&cpu_lock); 5861 (void) timeout(cpu_delayed_check_ce_errors, 5862 (void *)(uintptr_t)cp->cpu_id, 5863 drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 5864 } 5865 } 5866 5867 /* 5868 * This routine will check whether CEs have occurred while 5869 * CEEN is disabled. Any CEs detected will be logged and, if 5870 * possible, scrubbed. 5871 * 5872 * The memscrubber will also use this routine to clear any errors 5873 * caused by its scrubbing with CEEN disabled. 5874 * 5875 * flag == SCRUBBER_CEEN_CHECK 5876 * called from memscrubber, just check/scrub, no reset 5877 * paddr physical addr. for start of scrub pages 5878 * vaddr virtual addr. for scrub area 5879 * psz page size of area to be scrubbed 5880 * 5881 * flag == TIMEOUT_CEEN_CHECK 5882 * timeout function has triggered, reset timeout or CEEN 5883 * 5884 * Note: We must not migrate cpus during this function. This can be 5885 * achieved by one of: 5886 * - invoking as target of an x-call in which case we're at XCALL_PIL 5887 * The flag value must be first xcall argument. 5888 * - disabling kernel preemption. This should be done for very short 5889 * periods so is not suitable for SCRUBBER_CEEN_CHECK where we might 5890 * scrub an extended area with cpu_check_block. The call for 5891 * TIMEOUT_CEEN_CHECK uses this so cpu_check_ce must be kept 5892 * brief for this case. 5893 * - binding to a cpu, eg with thread_affinity_set(). This is used 5894 * in the SCRUBBER_CEEN_CHECK case, but is not practical for 5895 * the TIMEOUT_CEEN_CHECK because both need cpu_lock. 5896 */ 5897 void 5898 cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t psz) 5899 { 5900 ch_cpu_errors_t cpu_error_regs; 5901 uint64_t ec_err_enable; 5902 uint64_t page_offset; 5903 5904 /* Read AFSR */ 5905 get_cpu_error_state(&cpu_error_regs); 5906 5907 /* 5908 * If no CEEN errors have occurred during the timeout 5909 * interval, it is safe to re-enable CEEN and exit. 5910 */ 5911 if ((cpu_error_regs.afsr & C_AFSR_CECC_ERRS) == 0) { 5912 if (flag == TIMEOUT_CEEN_CHECK && 5913 !((ec_err_enable = get_error_enable()) & EN_REG_CEEN)) 5914 set_error_enable(ec_err_enable | EN_REG_CEEN); 5915 return; 5916 } 5917 5918 /* 5919 * Ensure that CEEN was not reenabled (maybe by DR) before 5920 * we log/clear the error. 5921 */ 5922 if ((ec_err_enable = get_error_enable()) & EN_REG_CEEN) 5923 set_error_enable(ec_err_enable & ~EN_REG_CEEN); 5924 5925 /* 5926 * log/clear the CE. If CE_CEEN_DEFER is passed, the 5927 * timeout will be rescheduled when the error is logged. 5928 */ 5929 if (!(cpu_error_regs.afsr & cpu_ce_not_deferred)) 5930 cpu_ce_detected(&cpu_error_regs, 5931 CE_CEEN_DEFER | CE_CEEN_TIMEOUT); 5932 else 5933 cpu_ce_detected(&cpu_error_regs, CE_CEEN_TIMEOUT); 5934 5935 /* 5936 * If the memory scrubber runs while CEEN is 5937 * disabled, (or if CEEN is disabled during the 5938 * scrub as a result of a CE being triggered by 5939 * it), the range being scrubbed will not be 5940 * completely cleaned. If there are multiple CEs 5941 * in the range at most two of these will be dealt 5942 * with, (one by the trap handler and one by the 5943 * timeout). It is also possible that none are dealt 5944 * with, (CEEN disabled and another CE occurs before 5945 * the timeout triggers). So to ensure that the 5946 * memory is actually scrubbed, we have to access each 5947 * memory location in the range and then check whether 5948 * that access causes a CE. 5949 */ 5950 if (flag == SCRUBBER_CEEN_CHECK && va) { 5951 if ((cpu_error_regs.afar >= pa) && 5952 (cpu_error_regs.afar < (pa + psz))) { 5953 /* 5954 * Force a load from physical memory for each 5955 * 64-byte block, then check AFSR to determine 5956 * whether this access caused an error. 5957 * 5958 * This is a slow way to do a scrub, but as it will 5959 * only be invoked when the memory scrubber actually 5960 * triggered a CE, it should not happen too 5961 * frequently. 5962 * 5963 * cut down what we need to check as the scrubber 5964 * has verified up to AFAR, so get it's offset 5965 * into the page and start there. 5966 */ 5967 page_offset = (uint64_t)(cpu_error_regs.afar & 5968 (psz - 1)); 5969 va = (caddr_t)(va + (P2ALIGN(page_offset, 64))); 5970 psz -= (uint_t)(P2ALIGN(page_offset, 64)); 5971 cpu_check_block((caddr_t)(P2ALIGN((uint64_t)va, 64)), 5972 psz); 5973 } 5974 } 5975 5976 /* 5977 * Reset error enable if this CE is not masked. 5978 */ 5979 if ((flag == TIMEOUT_CEEN_CHECK) && 5980 (cpu_error_regs.afsr & cpu_ce_not_deferred)) 5981 set_error_enable(ec_err_enable | EN_REG_CEEN); 5982 5983 } 5984 5985 /* 5986 * Attempt a cpu logout for an error that we did not trap for, such 5987 * as a CE noticed with CEEN off. It is assumed that we are still running 5988 * on the cpu that took the error and that we cannot migrate. Returns 5989 * 0 on success, otherwise nonzero. 5990 */ 5991 static int 5992 cpu_ce_delayed_ec_logout(uint64_t afar) 5993 { 5994 ch_cpu_logout_t *clop; 5995 5996 if (CPU_PRIVATE(CPU) == NULL) 5997 return (0); 5998 5999 clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 6000 if (cas64(&clop->clo_data.chd_afar, LOGOUT_INVALID, afar) != 6001 LOGOUT_INVALID) 6002 return (0); 6003 6004 cpu_delayed_logout(afar, clop); 6005 return (1); 6006 } 6007 6008 /* 6009 * We got an error while CEEN was disabled. We 6010 * need to clean up after it and log whatever 6011 * information we have on the CE. 6012 */ 6013 void 6014 cpu_ce_detected(ch_cpu_errors_t *cpu_error_regs, int flag) 6015 { 6016 ch_async_flt_t ch_flt; 6017 struct async_flt *aflt; 6018 char pr_reason[MAX_REASON_STRING]; 6019 6020 bzero(&ch_flt, sizeof (ch_async_flt_t)); 6021 ch_flt.flt_trapped_ce = flag; 6022 aflt = (struct async_flt *)&ch_flt; 6023 aflt->flt_stat = cpu_error_regs->afsr & C_AFSR_MASK; 6024 ch_flt.afsr_ext = cpu_error_regs->afsr_ext; 6025 ch_flt.afsr_errs = (cpu_error_regs->afsr_ext & C_AFSR_EXT_ALL_ERRS) | 6026 (cpu_error_regs->afsr & C_AFSR_ALL_ERRS); 6027 aflt->flt_addr = cpu_error_regs->afar; 6028 #if defined(SERRANO) 6029 ch_flt.afar2 = cpu_error_regs->afar2; 6030 #endif /* SERRANO */ 6031 aflt->flt_pc = NULL; 6032 aflt->flt_priv = ((cpu_error_regs->afsr & C_AFSR_PRIV) != 0); 6033 aflt->flt_tl = 0; 6034 aflt->flt_panic = 0; 6035 cpu_log_and_clear_ce(&ch_flt); 6036 6037 /* 6038 * check if we caused any errors during cleanup 6039 */ 6040 if (clear_errors(&ch_flt)) { 6041 pr_reason[0] = '\0'; 6042 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 6043 NULL); 6044 } 6045 } 6046 6047 /* 6048 * Log/clear CEEN-controlled disrupting errors 6049 */ 6050 static void 6051 cpu_log_and_clear_ce(ch_async_flt_t *ch_flt) 6052 { 6053 struct async_flt *aflt; 6054 uint64_t afsr, afsr_errs; 6055 ch_cpu_logout_t *clop; 6056 char pr_reason[MAX_REASON_STRING]; 6057 on_trap_data_t *otp = curthread->t_ontrap; 6058 6059 aflt = (struct async_flt *)ch_flt; 6060 afsr = aflt->flt_stat; 6061 afsr_errs = ch_flt->afsr_errs; 6062 aflt->flt_id = gethrtime_waitfree(); 6063 aflt->flt_bus_id = getprocessorid(); 6064 aflt->flt_inst = CPU->cpu_id; 6065 aflt->flt_prot = AFLT_PROT_NONE; 6066 aflt->flt_class = CPU_FAULT; 6067 aflt->flt_status = ECC_C_TRAP; 6068 6069 pr_reason[0] = '\0'; 6070 /* 6071 * Get the CPU log out info for Disrupting Trap. 6072 */ 6073 if (CPU_PRIVATE(CPU) == NULL) { 6074 clop = NULL; 6075 ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 6076 } else { 6077 clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 6078 } 6079 6080 if (clop && ch_flt->flt_trapped_ce & CE_CEEN_TIMEOUT) { 6081 ch_cpu_errors_t cpu_error_regs; 6082 6083 get_cpu_error_state(&cpu_error_regs); 6084 (void) cpu_ce_delayed_ec_logout(cpu_error_regs.afar); 6085 clop->clo_data.chd_afsr = cpu_error_regs.afsr; 6086 clop->clo_data.chd_afar = cpu_error_regs.afar; 6087 clop->clo_data.chd_afsr_ext = cpu_error_regs.afsr_ext; 6088 clop->clo_sdw_data.chd_afsr = cpu_error_regs.shadow_afsr; 6089 clop->clo_sdw_data.chd_afar = cpu_error_regs.shadow_afar; 6090 clop->clo_sdw_data.chd_afsr_ext = 6091 cpu_error_regs.shadow_afsr_ext; 6092 #if defined(SERRANO) 6093 clop->clo_data.chd_afar2 = cpu_error_regs.afar2; 6094 #endif /* SERRANO */ 6095 ch_flt->flt_data_incomplete = 1; 6096 6097 /* 6098 * The logging/clear code expects AFSR/AFAR to be cleared. 6099 * The trap handler does it for CEEN enabled errors 6100 * so we need to do it here. 6101 */ 6102 set_cpu_error_state(&cpu_error_regs); 6103 } 6104 6105 #if defined(JALAPENO) || defined(SERRANO) 6106 /* 6107 * FRC: Can't scrub memory as we don't have AFAR for Jalapeno. 6108 * For Serrano, even thou we do have the AFAR, we still do the 6109 * scrub on the RCE side since that's where the error type can 6110 * be properly classified as intermittent, persistent, etc. 6111 * 6112 * CE/RCE: If error is in memory and AFAR is valid, scrub the memory. 6113 * Must scrub memory before cpu_queue_events, as scrubbing memory sets 6114 * the flt_status bits. 6115 */ 6116 if ((afsr & (C_AFSR_CE|C_AFSR_RCE)) && 6117 (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 6118 cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_RCE)))) { 6119 cpu_ce_scrub_mem_err(aflt, B_TRUE); 6120 } 6121 #else /* JALAPENO || SERRANO */ 6122 /* 6123 * CE/EMC: If error is in memory and AFAR is valid, scrub the memory. 6124 * Must scrub memory before cpu_queue_events, as scrubbing memory sets 6125 * the flt_status bits. 6126 */ 6127 if (afsr & (C_AFSR_CE|C_AFSR_EMC)) { 6128 if (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 6129 cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_EMC))) { 6130 cpu_ce_scrub_mem_err(aflt, B_TRUE); 6131 } 6132 } 6133 6134 #endif /* JALAPENO || SERRANO */ 6135 6136 /* 6137 * Update flt_prot if this error occurred under on_trap protection. 6138 */ 6139 if (otp != NULL && (otp->ot_prot & OT_DATA_EC)) 6140 aflt->flt_prot = AFLT_PROT_EC; 6141 6142 /* 6143 * Queue events on the async event queue, one event per error bit. 6144 */ 6145 if (cpu_queue_events(ch_flt, pr_reason, afsr_errs, clop) == 0 || 6146 (afsr_errs & (C_AFSR_CECC_ERRS | C_AFSR_EXT_CECC_ERRS)) == 0) { 6147 ch_flt->flt_type = CPU_INV_AFSR; 6148 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 6149 (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 6150 aflt->flt_panic); 6151 } 6152 6153 /* 6154 * Zero out + invalidate CPU logout. 6155 */ 6156 if (clop) { 6157 bzero(clop, sizeof (ch_cpu_logout_t)); 6158 clop->clo_data.chd_afar = LOGOUT_INVALID; 6159 } 6160 6161 /* 6162 * If either a CPC, WDC or EDC error has occurred while CEEN 6163 * was disabled, we need to flush either the entire 6164 * E$ or an E$ line. 6165 */ 6166 #if defined(JALAPENO) || defined(SERRANO) 6167 if (afsr & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_CPU | C_AFSR_WDC)) 6168 #else /* JALAPENO || SERRANO */ 6169 if (afsr_errs & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_WDC | C_AFSR_L3_EDC | 6170 C_AFSR_L3_CPC | C_AFSR_L3_WDC)) 6171 #endif /* JALAPENO || SERRANO */ 6172 cpu_error_ecache_flush(ch_flt); 6173 6174 } 6175 6176 /* 6177 * depending on the error type, we determine whether we 6178 * need to flush the entire ecache or just a line. 6179 */ 6180 static int 6181 cpu_error_ecache_flush_required(ch_async_flt_t *ch_flt) 6182 { 6183 struct async_flt *aflt; 6184 uint64_t afsr; 6185 uint64_t afsr_errs = ch_flt->afsr_errs; 6186 6187 aflt = (struct async_flt *)ch_flt; 6188 afsr = aflt->flt_stat; 6189 6190 /* 6191 * If we got multiple errors, no point in trying 6192 * the individual cases, just flush the whole cache 6193 */ 6194 if (afsr & C_AFSR_ME) { 6195 return (ECACHE_FLUSH_ALL); 6196 } 6197 6198 /* 6199 * If either a CPC, WDC or EDC error has occurred while CEEN 6200 * was disabled, we need to flush entire E$. We can't just 6201 * flush the cache line affected as the ME bit 6202 * is not set when multiple correctable errors of the same 6203 * type occur, so we might have multiple CPC or EDC errors, 6204 * with only the first recorded. 6205 */ 6206 #if defined(JALAPENO) || defined(SERRANO) 6207 if (afsr & (C_AFSR_CPC | C_AFSR_CPU | C_AFSR_EDC | C_AFSR_WDC)) { 6208 #else /* JALAPENO || SERRANO */ 6209 if (afsr_errs & (C_AFSR_CPC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_L3_CPC | 6210 C_AFSR_L3_EDC | C_AFSR_L3_WDC)) { 6211 #endif /* JALAPENO || SERRANO */ 6212 return (ECACHE_FLUSH_ALL); 6213 } 6214 6215 #if defined(JALAPENO) || defined(SERRANO) 6216 /* 6217 * If only UE or RUE is set, flush the Ecache line, otherwise 6218 * flush the entire Ecache. 6219 */ 6220 if (afsr & (C_AFSR_UE|C_AFSR_RUE)) { 6221 if ((afsr & C_AFSR_ALL_ERRS) == C_AFSR_UE || 6222 (afsr & C_AFSR_ALL_ERRS) == C_AFSR_RUE) { 6223 return (ECACHE_FLUSH_LINE); 6224 } else { 6225 return (ECACHE_FLUSH_ALL); 6226 } 6227 } 6228 #else /* JALAPENO || SERRANO */ 6229 /* 6230 * If UE only is set, flush the Ecache line, otherwise 6231 * flush the entire Ecache. 6232 */ 6233 if (afsr_errs & C_AFSR_UE) { 6234 if ((afsr_errs & (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == 6235 C_AFSR_UE) { 6236 return (ECACHE_FLUSH_LINE); 6237 } else { 6238 return (ECACHE_FLUSH_ALL); 6239 } 6240 } 6241 #endif /* JALAPENO || SERRANO */ 6242 6243 /* 6244 * EDU: If EDU only is set, flush the ecache line, otherwise 6245 * flush the entire Ecache. 6246 */ 6247 if (afsr_errs & (C_AFSR_EDU | C_AFSR_L3_EDU)) { 6248 if (((afsr_errs & ~C_AFSR_EDU) == 0) || 6249 ((afsr_errs & ~C_AFSR_L3_EDU) == 0)) { 6250 return (ECACHE_FLUSH_LINE); 6251 } else { 6252 return (ECACHE_FLUSH_ALL); 6253 } 6254 } 6255 6256 /* 6257 * BERR: If BERR only is set, flush the Ecache line, otherwise 6258 * flush the entire Ecache. 6259 */ 6260 if (afsr_errs & C_AFSR_BERR) { 6261 if ((afsr_errs & ~C_AFSR_BERR) == 0) { 6262 return (ECACHE_FLUSH_LINE); 6263 } else { 6264 return (ECACHE_FLUSH_ALL); 6265 } 6266 } 6267 6268 return (0); 6269 } 6270 6271 void 6272 cpu_error_ecache_flush(ch_async_flt_t *ch_flt) 6273 { 6274 int ecache_flush_flag = 6275 cpu_error_ecache_flush_required(ch_flt); 6276 6277 /* 6278 * Flush Ecache line or entire Ecache based on above checks. 6279 */ 6280 if (ecache_flush_flag == ECACHE_FLUSH_ALL) 6281 cpu_flush_ecache(); 6282 else if (ecache_flush_flag == ECACHE_FLUSH_LINE) { 6283 cpu_flush_ecache_line(ch_flt); 6284 } 6285 6286 } 6287 6288 /* 6289 * Extract the PA portion from the E$ tag. 6290 */ 6291 uint64_t 6292 cpu_ectag_to_pa(int setsize, uint64_t tag) 6293 { 6294 if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 6295 return (JG_ECTAG_TO_PA(setsize, tag)); 6296 else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 6297 return (PN_L3TAG_TO_PA(tag)); 6298 else 6299 return (CH_ECTAG_TO_PA(setsize, tag)); 6300 } 6301 6302 /* 6303 * Convert the E$ tag PA into an E$ subblock index. 6304 */ 6305 static int 6306 cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr) 6307 { 6308 if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 6309 return (JG_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 6310 else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 6311 /* Panther has only one subblock per line */ 6312 return (0); 6313 else 6314 return (CH_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 6315 } 6316 6317 /* 6318 * All subblocks in an E$ line must be invalid for 6319 * the line to be invalid. 6320 */ 6321 int 6322 cpu_ectag_line_invalid(int cachesize, uint64_t tag) 6323 { 6324 if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 6325 return (JG_ECTAG_LINE_INVALID(cachesize, tag)); 6326 else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 6327 return (PN_L3_LINE_INVALID(tag)); 6328 else 6329 return (CH_ECTAG_LINE_INVALID(cachesize, tag)); 6330 } 6331 6332 /* 6333 * Extract state bits for a subblock given the tag. Note that for Panther 6334 * this works on both l2 and l3 tags. 6335 */ 6336 static int 6337 cpu_ectag_pa_to_subblk_state(int cachesize, uint64_t subaddr, uint64_t tag) 6338 { 6339 if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 6340 return (JG_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 6341 else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 6342 return (tag & CH_ECSTATE_MASK); 6343 else 6344 return (CH_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 6345 } 6346 6347 /* 6348 * Cpu specific initialization. 6349 */ 6350 void 6351 cpu_mp_init(void) 6352 { 6353 #ifdef CHEETAHPLUS_ERRATUM_25 6354 if (cheetah_sendmondo_recover) { 6355 cheetah_nudge_init(); 6356 } 6357 #endif 6358 } 6359 6360 void 6361 cpu_ereport_post(struct async_flt *aflt) 6362 { 6363 char *cpu_type, buf[FM_MAX_CLASS]; 6364 nv_alloc_t *nva = NULL; 6365 nvlist_t *ereport, *detector, *resource; 6366 errorq_elem_t *eqep; 6367 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 6368 char unum[UNUM_NAMLEN]; 6369 int len = 0; 6370 uint8_t msg_type; 6371 plat_ecc_ch_async_flt_t plat_ecc_ch_flt; 6372 6373 if (aflt->flt_panic || panicstr) { 6374 eqep = errorq_reserve(ereport_errorq); 6375 if (eqep == NULL) 6376 return; 6377 ereport = errorq_elem_nvl(ereport_errorq, eqep); 6378 nva = errorq_elem_nva(ereport_errorq, eqep); 6379 } else { 6380 ereport = fm_nvlist_create(nva); 6381 } 6382 6383 /* 6384 * Create the scheme "cpu" FMRI. 6385 */ 6386 detector = fm_nvlist_create(nva); 6387 resource = fm_nvlist_create(nva); 6388 switch (cpunodes[aflt->flt_inst].implementation) { 6389 case CHEETAH_IMPL: 6390 cpu_type = FM_EREPORT_CPU_USIII; 6391 break; 6392 case CHEETAH_PLUS_IMPL: 6393 cpu_type = FM_EREPORT_CPU_USIIIplus; 6394 break; 6395 case JALAPENO_IMPL: 6396 cpu_type = FM_EREPORT_CPU_USIIIi; 6397 break; 6398 case SERRANO_IMPL: 6399 cpu_type = FM_EREPORT_CPU_USIIIiplus; 6400 break; 6401 case JAGUAR_IMPL: 6402 cpu_type = FM_EREPORT_CPU_USIV; 6403 break; 6404 case PANTHER_IMPL: 6405 cpu_type = FM_EREPORT_CPU_USIVplus; 6406 break; 6407 default: 6408 cpu_type = FM_EREPORT_CPU_UNSUPPORTED; 6409 break; 6410 } 6411 (void) fm_fmri_cpu_set(detector, FM_CPU_SCHEME_VERSION, NULL, 6412 aflt->flt_inst, (uint8_t)cpunodes[aflt->flt_inst].version, 6413 cpunodes[aflt->flt_inst].device_id); 6414 6415 /* 6416 * Encode all the common data into the ereport. 6417 */ 6418 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s.%s", 6419 FM_ERROR_CPU, cpu_type, aflt->flt_erpt_class); 6420 6421 fm_ereport_set(ereport, FM_EREPORT_VERSION, buf, 6422 fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, FM_ENA_FMT1), 6423 detector, NULL); 6424 6425 /* 6426 * Encode the error specific data that was saved in 6427 * the async_flt structure into the ereport. 6428 */ 6429 cpu_payload_add_aflt(aflt, ereport, resource, 6430 &plat_ecc_ch_flt.ecaf_afar_status, 6431 &plat_ecc_ch_flt.ecaf_synd_status); 6432 6433 if (aflt->flt_panic || panicstr) { 6434 errorq_commit(ereport_errorq, eqep, ERRORQ_SYNC); 6435 } else { 6436 (void) fm_ereport_post(ereport, EVCH_TRYHARD); 6437 fm_nvlist_destroy(ereport, FM_NVA_FREE); 6438 fm_nvlist_destroy(detector, FM_NVA_FREE); 6439 fm_nvlist_destroy(resource, FM_NVA_FREE); 6440 } 6441 /* 6442 * Send the enhanced error information (plat_ecc_error2_data_t) 6443 * to the SC olny if it can process it. 6444 */ 6445 6446 if (&plat_ecc_capability_sc_get && 6447 plat_ecc_capability_sc_get(PLAT_ECC_ERROR2_MESSAGE)) { 6448 msg_type = cpu_flt_bit_to_plat_error(aflt); 6449 if (msg_type != PLAT_ECC_ERROR2_NONE) { 6450 /* 6451 * If afar status is not invalid do a unum lookup. 6452 */ 6453 if (plat_ecc_ch_flt.ecaf_afar_status != 6454 AFLT_STAT_INVALID) { 6455 (void) cpu_get_mem_unum_aflt( 6456 plat_ecc_ch_flt.ecaf_synd_status, aflt, 6457 unum, UNUM_NAMLEN, &len); 6458 } else { 6459 unum[0] = '\0'; 6460 } 6461 plat_ecc_ch_flt.ecaf_sdw_afar = ch_flt->flt_sdw_afar; 6462 plat_ecc_ch_flt.ecaf_sdw_afsr = ch_flt->flt_sdw_afsr; 6463 plat_ecc_ch_flt.ecaf_afsr_ext = ch_flt->afsr_ext; 6464 plat_ecc_ch_flt.ecaf_sdw_afsr_ext = 6465 ch_flt->flt_sdw_afsr_ext; 6466 6467 if (&plat_log_fruid_error2) 6468 plat_log_fruid_error2(msg_type, unum, aflt, 6469 &plat_ecc_ch_flt); 6470 } 6471 } 6472 } 6473 6474 void 6475 cpu_run_bus_error_handlers(struct async_flt *aflt, int expected) 6476 { 6477 int status; 6478 ddi_fm_error_t de; 6479 6480 bzero(&de, sizeof (ddi_fm_error_t)); 6481 6482 de.fme_version = DDI_FME_VERSION; 6483 de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, 6484 FM_ENA_FMT1); 6485 de.fme_flag = expected; 6486 de.fme_bus_specific = (void *)aflt->flt_addr; 6487 status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de); 6488 if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL)) 6489 aflt->flt_panic = 1; 6490 } 6491 6492 void 6493 cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz, 6494 errorq_t *eqp, uint_t flag) 6495 { 6496 struct async_flt *aflt = (struct async_flt *)payload; 6497 6498 aflt->flt_erpt_class = error_class; 6499 errorq_dispatch(eqp, payload, payload_sz, flag); 6500 } 6501 6502 /* 6503 * This routine may be called by the IO module, but does not do 6504 * anything in this cpu module. The SERD algorithm is handled by 6505 * cpumem-diagnosis engine instead. 6506 */ 6507 /*ARGSUSED*/ 6508 void 6509 cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum) 6510 {} 6511 6512 void 6513 adjust_hw_copy_limits(int ecache_size) 6514 { 6515 /* 6516 * Set hw copy limits. 6517 * 6518 * /etc/system will be parsed later and can override one or more 6519 * of these settings. 6520 * 6521 * At this time, ecache size seems only mildly relevant. 6522 * We seem to run into issues with the d-cache and stalls 6523 * we see on misses. 6524 * 6525 * Cycle measurement indicates that 2 byte aligned copies fare 6526 * little better than doing things with VIS at around 512 bytes. 6527 * 4 byte aligned shows promise until around 1024 bytes. 8 Byte 6528 * aligned is faster whenever the source and destination data 6529 * in cache and the total size is less than 2 Kbytes. The 2K 6530 * limit seems to be driven by the 2K write cache. 6531 * When more than 2K of copies are done in non-VIS mode, stores 6532 * backup in the write cache. In VIS mode, the write cache is 6533 * bypassed, allowing faster cache-line writes aligned on cache 6534 * boundaries. 6535 * 6536 * In addition, in non-VIS mode, there is no prefetching, so 6537 * for larger copies, the advantage of prefetching to avoid even 6538 * occasional cache misses is enough to justify using the VIS code. 6539 * 6540 * During testing, it was discovered that netbench ran 3% slower 6541 * when hw_copy_limit_8 was 2K or larger. Apparently for server 6542 * applications, data is only used once (copied to the output 6543 * buffer, then copied by the network device off the system). Using 6544 * the VIS copy saves more L2 cache state. Network copies are 6545 * around 1.3K to 1.5K in size for historical reasons. 6546 * 6547 * Therefore, a limit of 1K bytes will be used for the 8 byte 6548 * aligned copy even for large caches and 8 MB ecache. The 6549 * infrastructure to allow different limits for different sized 6550 * caches is kept to allow further tuning in later releases. 6551 */ 6552 6553 if (min_ecache_size == 0 && use_hw_bcopy) { 6554 /* 6555 * First time through - should be before /etc/system 6556 * is read. 6557 * Could skip the checks for zero but this lets us 6558 * preserve any debugger rewrites. 6559 */ 6560 if (hw_copy_limit_1 == 0) { 6561 hw_copy_limit_1 = VIS_COPY_THRESHOLD; 6562 priv_hcl_1 = hw_copy_limit_1; 6563 } 6564 if (hw_copy_limit_2 == 0) { 6565 hw_copy_limit_2 = 2 * VIS_COPY_THRESHOLD; 6566 priv_hcl_2 = hw_copy_limit_2; 6567 } 6568 if (hw_copy_limit_4 == 0) { 6569 hw_copy_limit_4 = 4 * VIS_COPY_THRESHOLD; 6570 priv_hcl_4 = hw_copy_limit_4; 6571 } 6572 if (hw_copy_limit_8 == 0) { 6573 hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD; 6574 priv_hcl_8 = hw_copy_limit_8; 6575 } 6576 min_ecache_size = ecache_size; 6577 } else { 6578 /* 6579 * MP initialization. Called *after* /etc/system has 6580 * been parsed. One CPU has already been initialized. 6581 * Need to cater for /etc/system having scragged one 6582 * of our values. 6583 */ 6584 if (ecache_size == min_ecache_size) { 6585 /* 6586 * Same size ecache. We do nothing unless we 6587 * have a pessimistic ecache setting. In that 6588 * case we become more optimistic (if the cache is 6589 * large enough). 6590 */ 6591 if (hw_copy_limit_8 == 4 * VIS_COPY_THRESHOLD) { 6592 /* 6593 * Need to adjust hw_copy_limit* from our 6594 * pessimistic uniprocessor value to a more 6595 * optimistic UP value *iff* it hasn't been 6596 * reset. 6597 */ 6598 if ((ecache_size > 1048576) && 6599 (priv_hcl_8 == hw_copy_limit_8)) { 6600 if (ecache_size <= 2097152) 6601 hw_copy_limit_8 = 4 * 6602 VIS_COPY_THRESHOLD; 6603 else if (ecache_size <= 4194304) 6604 hw_copy_limit_8 = 4 * 6605 VIS_COPY_THRESHOLD; 6606 else 6607 hw_copy_limit_8 = 4 * 6608 VIS_COPY_THRESHOLD; 6609 priv_hcl_8 = hw_copy_limit_8; 6610 } 6611 } 6612 } else if (ecache_size < min_ecache_size) { 6613 /* 6614 * A different ecache size. Can this even happen? 6615 */ 6616 if (priv_hcl_8 == hw_copy_limit_8) { 6617 /* 6618 * The previous value that we set 6619 * is unchanged (i.e., it hasn't been 6620 * scragged by /etc/system). Rewrite it. 6621 */ 6622 if (ecache_size <= 1048576) 6623 hw_copy_limit_8 = 8 * 6624 VIS_COPY_THRESHOLD; 6625 else if (ecache_size <= 2097152) 6626 hw_copy_limit_8 = 8 * 6627 VIS_COPY_THRESHOLD; 6628 else if (ecache_size <= 4194304) 6629 hw_copy_limit_8 = 8 * 6630 VIS_COPY_THRESHOLD; 6631 else 6632 hw_copy_limit_8 = 10 * 6633 VIS_COPY_THRESHOLD; 6634 priv_hcl_8 = hw_copy_limit_8; 6635 min_ecache_size = ecache_size; 6636 } 6637 } 6638 } 6639 } 6640 6641 /* 6642 * Called from illegal instruction trap handler to see if we can attribute 6643 * the trap to a fpras check. 6644 */ 6645 int 6646 fpras_chktrap(struct regs *rp) 6647 { 6648 int op; 6649 struct fpras_chkfngrp *cgp; 6650 uintptr_t tpc = (uintptr_t)rp->r_pc; 6651 6652 if (fpras_chkfngrps == NULL) 6653 return (0); 6654 6655 cgp = &fpras_chkfngrps[CPU->cpu_id]; 6656 for (op = 0; op < FPRAS_NCOPYOPS; ++op) { 6657 if (tpc >= (uintptr_t)&cgp->fpras_fn[op].fpras_blk0 && 6658 tpc < (uintptr_t)&cgp->fpras_fn[op].fpras_chkresult) 6659 break; 6660 } 6661 if (op == FPRAS_NCOPYOPS) 6662 return (0); 6663 6664 /* 6665 * This is an fpRAS failure caught through an illegal 6666 * instruction - trampoline. 6667 */ 6668 rp->r_pc = (uintptr_t)&cgp->fpras_fn[op].fpras_trampoline; 6669 rp->r_npc = rp->r_pc + 4; 6670 return (1); 6671 } 6672 6673 /* 6674 * fpras_failure is called when a fpras check detects a bad calculation 6675 * result or an illegal instruction trap is attributed to an fpras 6676 * check. In all cases we are still bound to CPU. 6677 */ 6678 int 6679 fpras_failure(int op, int how) 6680 { 6681 int use_hw_bcopy_orig, use_hw_bzero_orig; 6682 uint_t hcl1_orig, hcl2_orig, hcl4_orig, hcl8_orig; 6683 ch_async_flt_t ch_flt; 6684 struct async_flt *aflt = (struct async_flt *)&ch_flt; 6685 struct fpras_chkfn *sfp, *cfp; 6686 uint32_t *sip, *cip; 6687 int i; 6688 6689 /* 6690 * We're running on a sick CPU. Avoid further FPU use at least for 6691 * the time in which we dispatch an ereport and (if applicable) panic. 6692 */ 6693 use_hw_bcopy_orig = use_hw_bcopy; 6694 use_hw_bzero_orig = use_hw_bzero; 6695 hcl1_orig = hw_copy_limit_1; 6696 hcl2_orig = hw_copy_limit_2; 6697 hcl4_orig = hw_copy_limit_4; 6698 hcl8_orig = hw_copy_limit_8; 6699 use_hw_bcopy = use_hw_bzero = 0; 6700 hw_copy_limit_1 = hw_copy_limit_2 = hw_copy_limit_4 = 6701 hw_copy_limit_8 = 0; 6702 6703 bzero(&ch_flt, sizeof (ch_async_flt_t)); 6704 aflt->flt_id = gethrtime_waitfree(); 6705 aflt->flt_class = CPU_FAULT; 6706 aflt->flt_inst = CPU->cpu_id; 6707 aflt->flt_status = (how << 8) | op; 6708 aflt->flt_payload = FM_EREPORT_PAYLOAD_FPU_HWCOPY; 6709 ch_flt.flt_type = CPU_FPUERR; 6710 6711 /* 6712 * We must panic if the copy operation had no lofault protection - 6713 * ie, don't panic for copyin, copyout, kcopy and bcopy called 6714 * under on_fault and do panic for unprotected bcopy and hwblkpagecopy. 6715 */ 6716 aflt->flt_panic = (curthread->t_lofault == NULL); 6717 6718 /* 6719 * XOR the source instruction block with the copied instruction 6720 * block - this will show us which bit(s) are corrupted. 6721 */ 6722 sfp = (struct fpras_chkfn *)fpras_chkfn_type1; 6723 cfp = &fpras_chkfngrps[CPU->cpu_id].fpras_fn[op]; 6724 if (op == FPRAS_BCOPY || op == FPRAS_COPYOUT) { 6725 sip = &sfp->fpras_blk0[0]; 6726 cip = &cfp->fpras_blk0[0]; 6727 } else { 6728 sip = &sfp->fpras_blk1[0]; 6729 cip = &cfp->fpras_blk1[0]; 6730 } 6731 for (i = 0; i < 16; ++i, ++sip, ++cip) 6732 ch_flt.flt_fpdata[i] = *sip ^ *cip; 6733 6734 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_FPU_HWCOPY, (void *)&ch_flt, 6735 sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 6736 6737 if (aflt->flt_panic) 6738 fm_panic("FPU failure on CPU %d", CPU->cpu_id); 6739 6740 /* 6741 * We get here for copyin/copyout and kcopy or bcopy where the 6742 * caller has used on_fault. We will flag the error so that 6743 * the process may be killed The trap_async_hwerr mechanism will 6744 * take appropriate further action (such as a reboot, contract 6745 * notification etc). Since we may be continuing we will 6746 * restore the global hardware copy acceleration switches. 6747 * 6748 * When we return from this function to the copy function we want to 6749 * avoid potentially bad data being used, ie we want the affected 6750 * copy function to return an error. The caller should therefore 6751 * invoke its lofault handler (which always exists for these functions) 6752 * which will return the appropriate error. 6753 */ 6754 ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR; 6755 aston(curthread); 6756 6757 use_hw_bcopy = use_hw_bcopy_orig; 6758 use_hw_bzero = use_hw_bzero_orig; 6759 hw_copy_limit_1 = hcl1_orig; 6760 hw_copy_limit_2 = hcl2_orig; 6761 hw_copy_limit_4 = hcl4_orig; 6762 hw_copy_limit_8 = hcl8_orig; 6763 6764 return (1); 6765 } 6766 6767 #define VIS_BLOCKSIZE 64 6768 6769 int 6770 dtrace_blksuword32_err(uintptr_t addr, uint32_t *data) 6771 { 6772 int ret, watched; 6773 6774 watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 6775 ret = dtrace_blksuword32(addr, data, 0); 6776 if (watched) 6777 watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 6778 6779 return (ret); 6780 } 6781 6782 /* 6783 * Called when a cpu enters the CPU_FAULTED state (by the cpu placing the 6784 * faulted cpu into that state). Cross-trap to the faulted cpu to clear 6785 * CEEN from the EER to disable traps for further disrupting error types 6786 * on that cpu. We could cross-call instead, but that has a larger 6787 * instruction and data footprint than cross-trapping, and the cpu is known 6788 * to be faulted. 6789 */ 6790 6791 void 6792 cpu_faulted_enter(struct cpu *cp) 6793 { 6794 xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_CLRBITS); 6795 } 6796 6797 /* 6798 * Called when a cpu leaves the CPU_FAULTED state to return to one of 6799 * offline, spare, or online (by the cpu requesting this state change). 6800 * First we cross-call to clear the AFSR (and AFSR_EXT on Panther) of 6801 * disrupting error bits that have accumulated without trapping, then 6802 * we cross-trap to re-enable CEEN controlled traps. 6803 */ 6804 void 6805 cpu_faulted_exit(struct cpu *cp) 6806 { 6807 ch_cpu_errors_t cpu_error_regs; 6808 6809 cpu_error_regs.afsr = C_AFSR_CECC_ERRS; 6810 if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) 6811 cpu_error_regs.afsr_ext &= C_AFSR_EXT_CECC_ERRS; 6812 xc_one(cp->cpu_id, (xcfunc_t *)set_cpu_error_state, 6813 (uint64_t)&cpu_error_regs, 0); 6814 6815 xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_SETBITS); 6816 } 6817 6818 /* 6819 * Return 1 if the errors in ch_flt's AFSR are secondary errors caused by 6820 * the errors in the original AFSR, 0 otherwise. 6821 * 6822 * For all procs if the initial error was a BERR or TO, then it is possible 6823 * that we may have caused a secondary BERR or TO in the process of logging the 6824 * inital error via cpu_run_bus_error_handlers(). If this is the case then 6825 * if the request was protected then a panic is still not necessary, if not 6826 * protected then aft_panic is already set - so either way there's no need 6827 * to set aft_panic for the secondary error. 6828 * 6829 * For Cheetah and Jalapeno, if the original error was a UE which occurred on 6830 * a store merge, then the error handling code will call cpu_deferred_error(). 6831 * When clear_errors() is called, it will determine that secondary errors have 6832 * occurred - in particular, the store merge also caused a EDU and WDU that 6833 * weren't discovered until this point. 6834 * 6835 * We do three checks to verify that we are in this case. If we pass all three 6836 * checks, we return 1 to indicate that we should not panic. If any unexpected 6837 * errors occur, we return 0. 6838 * 6839 * For Cheetah+ and derivative procs, the store merge causes a DUE, which is 6840 * handled in cpu_disrupting_errors(). Since this function is not even called 6841 * in the case we are interested in, we just return 0 for these processors. 6842 */ 6843 /*ARGSUSED*/ 6844 static int 6845 cpu_check_secondary_errors(ch_async_flt_t *ch_flt, uint64_t t_afsr_errs, 6846 uint64_t t_afar) 6847 { 6848 #if defined(CHEETAH_PLUS) 6849 #else /* CHEETAH_PLUS */ 6850 struct async_flt *aflt = (struct async_flt *)ch_flt; 6851 #endif /* CHEETAH_PLUS */ 6852 6853 /* 6854 * Was the original error a BERR or TO and only a BERR or TO 6855 * (multiple errors are also OK) 6856 */ 6857 if ((t_afsr_errs & ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) { 6858 /* 6859 * Is the new error a BERR or TO and only a BERR or TO 6860 * (multiple errors are also OK) 6861 */ 6862 if ((ch_flt->afsr_errs & 6863 ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) 6864 return (1); 6865 } 6866 6867 #if defined(CHEETAH_PLUS) 6868 return (0); 6869 #else /* CHEETAH_PLUS */ 6870 /* 6871 * Now look for secondary effects of a UE on cheetah/jalapeno 6872 * 6873 * Check the original error was a UE, and only a UE. Note that 6874 * the ME bit will cause us to fail this check. 6875 */ 6876 if (t_afsr_errs != C_AFSR_UE) 6877 return (0); 6878 6879 /* 6880 * Check the secondary errors were exclusively an EDU and/or WDU. 6881 */ 6882 if ((ch_flt->afsr_errs & ~(C_AFSR_EDU|C_AFSR_WDU)) != 0) 6883 return (0); 6884 6885 /* 6886 * Check the AFAR of the original error and secondary errors 6887 * match to the 64-byte boundary 6888 */ 6889 if (P2ALIGN(aflt->flt_addr, 64) != P2ALIGN(t_afar, 64)) 6890 return (0); 6891 6892 /* 6893 * We've passed all the checks, so it's a secondary error! 6894 */ 6895 return (1); 6896 #endif /* CHEETAH_PLUS */ 6897 } 6898 6899 /* 6900 * Translate the flt_bit or flt_type into an error type. First, flt_bit 6901 * is checked for any valid errors. If found, the error type is 6902 * returned. If not found, the flt_type is checked for L1$ parity errors. 6903 */ 6904 /*ARGSUSED*/ 6905 static uint8_t 6906 cpu_flt_bit_to_plat_error(struct async_flt *aflt) 6907 { 6908 #if defined(JALAPENO) 6909 /* 6910 * Currently, logging errors to the SC is not supported on Jalapeno 6911 */ 6912 return (PLAT_ECC_ERROR2_NONE); 6913 #else 6914 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 6915 6916 switch (ch_flt->flt_bit) { 6917 case C_AFSR_CE: 6918 return (PLAT_ECC_ERROR2_CE); 6919 case C_AFSR_UCC: 6920 case C_AFSR_EDC: 6921 case C_AFSR_WDC: 6922 case C_AFSR_CPC: 6923 return (PLAT_ECC_ERROR2_L2_CE); 6924 case C_AFSR_EMC: 6925 return (PLAT_ECC_ERROR2_EMC); 6926 case C_AFSR_IVC: 6927 return (PLAT_ECC_ERROR2_IVC); 6928 case C_AFSR_UE: 6929 return (PLAT_ECC_ERROR2_UE); 6930 case C_AFSR_UCU: 6931 case C_AFSR_EDU: 6932 case C_AFSR_WDU: 6933 case C_AFSR_CPU: 6934 return (PLAT_ECC_ERROR2_L2_UE); 6935 case C_AFSR_IVU: 6936 return (PLAT_ECC_ERROR2_IVU); 6937 case C_AFSR_TO: 6938 return (PLAT_ECC_ERROR2_TO); 6939 case C_AFSR_BERR: 6940 return (PLAT_ECC_ERROR2_BERR); 6941 #if defined(CHEETAH_PLUS) 6942 case C_AFSR_L3_EDC: 6943 case C_AFSR_L3_UCC: 6944 case C_AFSR_L3_CPC: 6945 case C_AFSR_L3_WDC: 6946 return (PLAT_ECC_ERROR2_L3_CE); 6947 case C_AFSR_IMC: 6948 return (PLAT_ECC_ERROR2_IMC); 6949 case C_AFSR_TSCE: 6950 return (PLAT_ECC_ERROR2_L2_TSCE); 6951 case C_AFSR_THCE: 6952 return (PLAT_ECC_ERROR2_L2_THCE); 6953 case C_AFSR_L3_MECC: 6954 return (PLAT_ECC_ERROR2_L3_MECC); 6955 case C_AFSR_L3_THCE: 6956 return (PLAT_ECC_ERROR2_L3_THCE); 6957 case C_AFSR_L3_CPU: 6958 case C_AFSR_L3_EDU: 6959 case C_AFSR_L3_UCU: 6960 case C_AFSR_L3_WDU: 6961 return (PLAT_ECC_ERROR2_L3_UE); 6962 case C_AFSR_DUE: 6963 return (PLAT_ECC_ERROR2_DUE); 6964 case C_AFSR_DTO: 6965 return (PLAT_ECC_ERROR2_DTO); 6966 case C_AFSR_DBERR: 6967 return (PLAT_ECC_ERROR2_DBERR); 6968 #endif /* CHEETAH_PLUS */ 6969 default: 6970 switch (ch_flt->flt_type) { 6971 #if defined(CPU_IMP_L1_CACHE_PARITY) 6972 case CPU_IC_PARITY: 6973 return (PLAT_ECC_ERROR2_IPE); 6974 case CPU_DC_PARITY: 6975 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 6976 if (ch_flt->parity_data.dpe.cpl_cache == 6977 CPU_PC_PARITY) { 6978 return (PLAT_ECC_ERROR2_PCACHE); 6979 } 6980 } 6981 return (PLAT_ECC_ERROR2_DPE); 6982 #endif /* CPU_IMP_L1_CACHE_PARITY */ 6983 case CPU_ITLB_PARITY: 6984 return (PLAT_ECC_ERROR2_ITLB); 6985 case CPU_DTLB_PARITY: 6986 return (PLAT_ECC_ERROR2_DTLB); 6987 default: 6988 return (PLAT_ECC_ERROR2_NONE); 6989 } 6990 } 6991 #endif /* JALAPENO */ 6992 } 6993