1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/types.h> 29 #include <sys/systm.h> 30 #include <sys/ddi.h> 31 #include <sys/sysmacros.h> 32 #include <sys/archsystm.h> 33 #include <sys/vmsystm.h> 34 #include <sys/machparam.h> 35 #include <sys/machsystm.h> 36 #include <sys/machthread.h> 37 #include <sys/cpu.h> 38 #include <sys/cmp.h> 39 #include <sys/elf_SPARC.h> 40 #include <vm/vm_dep.h> 41 #include <vm/hat_sfmmu.h> 42 #include <vm/seg_kpm.h> 43 #include <sys/cpuvar.h> 44 #include <sys/cheetahregs.h> 45 #include <sys/us3_module.h> 46 #include <sys/async.h> 47 #include <sys/cmn_err.h> 48 #include <sys/debug.h> 49 #include <sys/dditypes.h> 50 #include <sys/prom_debug.h> 51 #include <sys/prom_plat.h> 52 #include <sys/cpu_module.h> 53 #include <sys/sysmacros.h> 54 #include <sys/intreg.h> 55 #include <sys/clock.h> 56 #include <sys/platform_module.h> 57 #include <sys/machtrap.h> 58 #include <sys/ontrap.h> 59 #include <sys/panic.h> 60 #include <sys/memlist.h> 61 #include <sys/bootconf.h> 62 #include <sys/ivintr.h> 63 #include <sys/atomic.h> 64 #include <sys/taskq.h> 65 #include <sys/note.h> 66 #include <sys/ndifm.h> 67 #include <sys/ddifm.h> 68 #include <sys/fm/protocol.h> 69 #include <sys/fm/util.h> 70 #include <sys/fm/cpu/UltraSPARC-III.h> 71 #include <sys/fpras_impl.h> 72 #include <sys/dtrace.h> 73 #include <sys/watchpoint.h> 74 #include <sys/plat_ecc_unum.h> 75 #include <sys/cyclic.h> 76 #include <sys/errorq.h> 77 #include <sys/errclassify.h> 78 79 #ifdef CHEETAHPLUS_ERRATUM_25 80 #include <sys/xc_impl.h> 81 #endif /* CHEETAHPLUS_ERRATUM_25 */ 82 83 /* 84 * Note that 'Cheetah PRM' refers to: 85 * SPARC V9 JPS1 Implementation Supplement: Sun UltraSPARC-III 86 */ 87 88 /* 89 * Per CPU pointers to physical address of TL>0 logout data areas. 90 * These pointers have to be in the kernel nucleus to avoid MMU 91 * misses. 92 */ 93 uint64_t ch_err_tl1_paddrs[NCPU]; 94 95 /* 96 * One statically allocated structure to use during startup/DR 97 * to prevent unnecessary panics. 98 */ 99 ch_err_tl1_data_t ch_err_tl1_data; 100 101 /* 102 * Per CPU pending error at TL>0, used by level15 softint handler 103 */ 104 uchar_t ch_err_tl1_pending[NCPU]; 105 106 /* 107 * For deferred CE re-enable after trap. 108 */ 109 taskq_t *ch_check_ce_tq; 110 111 /* 112 * Internal functions. 113 */ 114 static int cpu_async_log_err(void *flt, errorq_elem_t *eqep); 115 static void cpu_log_diag_info(ch_async_flt_t *ch_flt); 116 static void cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 117 ecc_type_to_info_t *eccp, ch_diag_data_t *cdp); 118 static int cpu_flt_in_memory_one_event(ch_async_flt_t *ch_flt, 119 uint64_t t_afsr_bit); 120 static int clear_ecc(struct async_flt *ecc); 121 #if defined(CPU_IMP_ECACHE_ASSOC) 122 static int cpu_ecache_line_valid(ch_async_flt_t *ch_flt); 123 #endif 124 static int cpu_ecache_set_size(struct cpu *cp); 125 static int cpu_ectag_line_invalid(int cachesize, uint64_t tag); 126 static int cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr); 127 static uint64_t cpu_ectag_to_pa(int setsize, uint64_t tag); 128 static int cpu_ectag_pa_to_subblk_state(int cachesize, 129 uint64_t subaddr, uint64_t tag); 130 static void cpu_flush_ecache_line(ch_async_flt_t *ch_flt); 131 static int afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit); 132 static int afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit); 133 static int afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit); 134 static int afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit); 135 static int synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit); 136 static int cpu_get_mem_unum_synd(int synd_code, struct async_flt *, char *buf); 137 static void cpu_uninit_ecache_scrub_dr(struct cpu *cp); 138 static void cpu_scrubphys(struct async_flt *aflt); 139 static void cpu_payload_add_aflt(struct async_flt *, nvlist_t *, nvlist_t *, 140 int *, int *); 141 static void cpu_payload_add_ecache(struct async_flt *, nvlist_t *); 142 static void cpu_ereport_init(struct async_flt *aflt); 143 static int cpu_check_secondary_errors(ch_async_flt_t *, uint64_t, uint64_t); 144 static uint8_t cpu_flt_bit_to_plat_error(struct async_flt *aflt); 145 static void cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 146 uint64_t nceen, ch_cpu_logout_t *clop); 147 static int cpu_ce_delayed_ec_logout(uint64_t); 148 static int cpu_matching_ecache_line(uint64_t, void *, int, int *); 149 static int cpu_error_is_ecache_data(int, uint64_t); 150 static void cpu_fmri_cpu_set(nvlist_t *, int); 151 static int cpu_error_to_resource_type(struct async_flt *aflt); 152 153 #ifdef CHEETAHPLUS_ERRATUM_25 154 static int mondo_recover_proc(uint16_t, int); 155 static void cheetah_nudge_init(void); 156 static void cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 157 cyc_time_t *when); 158 static void cheetah_nudge_buddy(void); 159 #endif /* CHEETAHPLUS_ERRATUM_25 */ 160 161 #if defined(CPU_IMP_L1_CACHE_PARITY) 162 static void cpu_dcache_parity_info(ch_async_flt_t *ch_flt); 163 static void cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index); 164 static void cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 165 ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word); 166 static void cpu_icache_parity_info(ch_async_flt_t *ch_flt); 167 static void cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index); 168 static void cpu_pcache_parity_info(ch_async_flt_t *ch_flt); 169 static void cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index); 170 static void cpu_payload_add_dcache(struct async_flt *, nvlist_t *); 171 static void cpu_payload_add_icache(struct async_flt *, nvlist_t *); 172 #endif /* CPU_IMP_L1_CACHE_PARITY */ 173 174 int (*p2get_mem_info)(int synd_code, uint64_t paddr, 175 uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 176 int *segsp, int *banksp, int *mcidp); 177 178 /* 179 * This table is used to determine which bit(s) is(are) bad when an ECC 180 * error occurs. The array is indexed by an 9-bit syndrome. The entries 181 * of this array have the following semantics: 182 * 183 * 00-127 The number of the bad bit, when only one bit is bad. 184 * 128 ECC bit C0 is bad. 185 * 129 ECC bit C1 is bad. 186 * 130 ECC bit C2 is bad. 187 * 131 ECC bit C3 is bad. 188 * 132 ECC bit C4 is bad. 189 * 133 ECC bit C5 is bad. 190 * 134 ECC bit C6 is bad. 191 * 135 ECC bit C7 is bad. 192 * 136 ECC bit C8 is bad. 193 * 137-143 reserved for Mtag Data and ECC. 194 * 144(M2) Two bits are bad within a nibble. 195 * 145(M3) Three bits are bad within a nibble. 196 * 146(M3) Four bits are bad within a nibble. 197 * 147(M) Multiple bits (5 or more) are bad. 198 * 148 NO bits are bad. 199 * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-4,11-5. 200 */ 201 202 #define C0 128 203 #define C1 129 204 #define C2 130 205 #define C3 131 206 #define C4 132 207 #define C5 133 208 #define C6 134 209 #define C7 135 210 #define C8 136 211 #define MT0 137 /* Mtag Data bit 0 */ 212 #define MT1 138 213 #define MT2 139 214 #define MTC0 140 /* Mtag Check bit 0 */ 215 #define MTC1 141 216 #define MTC2 142 217 #define MTC3 143 218 #define M2 144 219 #define M3 145 220 #define M4 146 221 #define M 147 222 #define NA 148 223 #if defined(JALAPENO) || defined(SERRANO) 224 #define S003 149 /* Syndrome 0x003 => likely from CPU/EDU:ST/FRU/BP */ 225 #define S003MEM 150 /* Syndrome 0x003 => likely from WDU/WBP */ 226 #define SLAST S003MEM /* last special syndrome */ 227 #else /* JALAPENO || SERRANO */ 228 #define S003 149 /* Syndrome 0x003 => likely from EDU:ST */ 229 #define S071 150 /* Syndrome 0x071 => likely from WDU/CPU */ 230 #define S11C 151 /* Syndrome 0x11c => likely from BERR/DBERR */ 231 #define SLAST S11C /* last special syndrome */ 232 #endif /* JALAPENO || SERRANO */ 233 #if defined(JALAPENO) || defined(SERRANO) 234 #define BPAR0 152 /* syndrom 152 through 167 for bus parity */ 235 #define BPAR15 167 236 #endif /* JALAPENO || SERRANO */ 237 238 static uint8_t ecc_syndrome_tab[] = 239 { 240 NA, C0, C1, S003, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M, 241 C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16, 242 C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10, 243 M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M, 244 C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6, 245 M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4, 246 M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4, 247 #if defined(JALAPENO) || defined(SERRANO) 248 116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 249 #else /* JALAPENO || SERRANO */ 250 116, S071, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 251 #endif /* JALAPENO || SERRANO */ 252 C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5, 253 M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M, 254 M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2, 255 103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3, 256 M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M, 257 102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3, 258 98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M, 259 M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M, 260 C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4, 261 #if defined(JALAPENO) || defined(SERRANO) 262 M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M, 263 #else /* JALAPENO || SERRANO */ 264 M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, S11C, M, M3, M, 265 #endif /* JALAPENO || SERRANO */ 266 M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2, 267 94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M, 268 M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4, 269 89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3, 270 86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3, 271 M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2, 272 M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4, 273 77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M, 274 74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3, 275 M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M, 276 80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3, 277 M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M, 278 M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M, 279 111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M 280 }; 281 282 #define ESYND_TBL_SIZE (sizeof (ecc_syndrome_tab) / sizeof (uint8_t)) 283 284 #if !(defined(JALAPENO) || defined(SERRANO)) 285 /* 286 * This table is used to determine which bit(s) is(are) bad when a Mtag 287 * error occurs. The array is indexed by an 4-bit ECC syndrome. The entries 288 * of this array have the following semantics: 289 * 290 * -1 Invalid mtag syndrome. 291 * 137 Mtag Data 0 is bad. 292 * 138 Mtag Data 1 is bad. 293 * 139 Mtag Data 2 is bad. 294 * 140 Mtag ECC 0 is bad. 295 * 141 Mtag ECC 1 is bad. 296 * 142 Mtag ECC 2 is bad. 297 * 143 Mtag ECC 3 is bad. 298 * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-6. 299 */ 300 short mtag_syndrome_tab[] = 301 { 302 NA, MTC0, MTC1, M2, MTC2, M2, M2, MT0, MTC3, M2, M2, MT1, M2, MT2, M2, M2 303 }; 304 305 #define MSYND_TBL_SIZE (sizeof (mtag_syndrome_tab) / sizeof (short)) 306 307 #else /* !(JALAPENO || SERRANO) */ 308 309 #define BSYND_TBL_SIZE 16 310 311 #endif /* !(JALAPENO || SERRANO) */ 312 313 /* 314 * Types returned from cpu_error_to_resource_type() 315 */ 316 #define ERRTYPE_UNKNOWN 0 317 #define ERRTYPE_CPU 1 318 #define ERRTYPE_MEMORY 2 319 #define ERRTYPE_ECACHE_DATA 3 320 321 /* 322 * CE initial classification and subsequent action lookup table 323 */ 324 static ce_dispact_t ce_disp_table[CE_INITDISPTBL_SIZE]; 325 static int ce_disp_inited; 326 327 /* 328 * Set to disable leaky and partner check for memory correctables 329 */ 330 int ce_xdiag_off; 331 332 /* 333 * The following are not incremented atomically so are indicative only 334 */ 335 static int ce_xdiag_drops; 336 static int ce_xdiag_lkydrops; 337 static int ce_xdiag_ptnrdrops; 338 static int ce_xdiag_bad; 339 340 /* 341 * CE leaky check callback structure 342 */ 343 typedef struct { 344 struct async_flt *lkycb_aflt; 345 errorq_t *lkycb_eqp; 346 errorq_elem_t *lkycb_eqep; 347 } ce_lkychk_cb_t; 348 349 /* 350 * defines for various ecache_flush_flag's 351 */ 352 #define ECACHE_FLUSH_LINE 1 353 #define ECACHE_FLUSH_ALL 2 354 355 /* 356 * STICK sync 357 */ 358 #define STICK_ITERATION 10 359 #define MAX_TSKEW 1 360 #define EV_A_START 0 361 #define EV_A_END 1 362 #define EV_B_START 2 363 #define EV_B_END 3 364 #define EVENTS 4 365 366 static int64_t stick_iter = STICK_ITERATION; 367 static int64_t stick_tsk = MAX_TSKEW; 368 369 typedef enum { 370 EVENT_NULL = 0, 371 SLAVE_START, 372 SLAVE_CONT, 373 MASTER_START 374 } event_cmd_t; 375 376 static volatile event_cmd_t stick_sync_cmd = EVENT_NULL; 377 static int64_t timestamp[EVENTS]; 378 static volatile int slave_done; 379 380 #ifdef DEBUG 381 #define DSYNC_ATTEMPTS 64 382 typedef struct { 383 int64_t skew_val[DSYNC_ATTEMPTS]; 384 } ss_t; 385 386 ss_t stick_sync_stats[NCPU]; 387 #endif /* DEBUG */ 388 389 uint_t cpu_impl_dual_pgsz = 0; 390 #if defined(CPU_IMP_DUAL_PAGESIZE) 391 uint_t disable_dual_pgsz = 0; 392 #endif /* CPU_IMP_DUAL_PAGESIZE */ 393 394 /* 395 * Save the cache bootup state for use when internal 396 * caches are to be re-enabled after an error occurs. 397 */ 398 uint64_t cache_boot_state; 399 400 /* 401 * PA[22:0] represent Displacement in Safari configuration space. 402 */ 403 uint_t root_phys_addr_lo_mask = 0x7fffffu; 404 405 bus_config_eclk_t bus_config_eclk[] = { 406 #if defined(JALAPENO) || defined(SERRANO) 407 {JBUS_CONFIG_ECLK_1_DIV, JBUS_CONFIG_ECLK_1}, 408 {JBUS_CONFIG_ECLK_2_DIV, JBUS_CONFIG_ECLK_2}, 409 {JBUS_CONFIG_ECLK_32_DIV, JBUS_CONFIG_ECLK_32}, 410 #else /* JALAPENO || SERRANO */ 411 {SAFARI_CONFIG_ECLK_1_DIV, SAFARI_CONFIG_ECLK_1}, 412 {SAFARI_CONFIG_ECLK_2_DIV, SAFARI_CONFIG_ECLK_2}, 413 {SAFARI_CONFIG_ECLK_32_DIV, SAFARI_CONFIG_ECLK_32}, 414 #endif /* JALAPENO || SERRANO */ 415 {0, 0} 416 }; 417 418 /* 419 * Interval for deferred CEEN reenable 420 */ 421 int cpu_ceen_delay_secs = CPU_CEEN_DELAY_SECS; 422 423 /* 424 * set in /etc/system to control logging of user BERR/TO's 425 */ 426 int cpu_berr_to_verbose = 0; 427 428 /* 429 * set to 0 in /etc/system to defer CEEN reenable for all CEs 430 */ 431 uint64_t cpu_ce_not_deferred = CPU_CE_NOT_DEFERRED; 432 uint64_t cpu_ce_not_deferred_ext = CPU_CE_NOT_DEFERRED_EXT; 433 434 /* 435 * Set of all offline cpus 436 */ 437 cpuset_t cpu_offline_set; 438 439 static void cpu_delayed_check_ce_errors(void *); 440 static void cpu_check_ce_errors(void *); 441 void cpu_error_ecache_flush(ch_async_flt_t *); 442 static int cpu_error_ecache_flush_required(ch_async_flt_t *); 443 static void cpu_log_and_clear_ce(ch_async_flt_t *); 444 void cpu_ce_detected(ch_cpu_errors_t *, int); 445 446 /* 447 * CE Leaky check timeout in microseconds. This is chosen to be twice the 448 * memory refresh interval of current DIMMs (64ms). After initial fix that 449 * gives at least one full refresh cycle in which the cell can leak 450 * (whereafter further refreshes simply reinforce any incorrect bit value). 451 */ 452 clock_t cpu_ce_lkychk_timeout_usec = 128000; 453 454 /* 455 * CE partner check partner caching period in seconds 456 */ 457 int cpu_ce_ptnr_cachetime_sec = 60; 458 459 /* 460 * Sets trap table entry ttentry by overwriting eight instructions from ttlabel 461 */ 462 #define CH_SET_TRAP(ttentry, ttlabel) \ 463 bcopy((const void *)&ttlabel, &ttentry, 32); \ 464 flush_instr_mem((caddr_t)&ttentry, 32); 465 466 static int min_ecache_size; 467 static uint_t priv_hcl_1; 468 static uint_t priv_hcl_2; 469 static uint_t priv_hcl_4; 470 static uint_t priv_hcl_8; 471 472 void 473 cpu_setup(void) 474 { 475 extern int at_flags; 476 extern int disable_delay_tlb_flush, delay_tlb_flush; 477 extern int cpc_has_overflow_intr; 478 479 /* 480 * Setup chip-specific trap handlers. 481 */ 482 cpu_init_trap(); 483 484 cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT); 485 486 at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1 | EF_SPARC_SUN_US3; 487 488 /* 489 * save the cache bootup state. 490 */ 491 cache_boot_state = get_dcu() & DCU_CACHE; 492 493 /* 494 * Due to the number of entries in the fully-associative tlb 495 * this may have to be tuned lower than in spitfire. 496 */ 497 pp_slots = MIN(8, MAXPP_SLOTS); 498 499 /* 500 * Block stores do not invalidate all pages of the d$, pagecopy 501 * et. al. need virtual translations with virtual coloring taken 502 * into consideration. prefetch/ldd will pollute the d$ on the 503 * load side. 504 */ 505 pp_consistent_coloring = PPAGE_STORE_VCOLORING | PPAGE_LOADS_POLLUTE; 506 507 if (use_page_coloring) { 508 do_pg_coloring = 1; 509 if (use_virtual_coloring) 510 do_virtual_coloring = 1; 511 } 512 513 isa_list = 514 "sparcv9+vis2 sparcv9+vis sparcv9 " 515 "sparcv8plus+vis2 sparcv8plus+vis sparcv8plus " 516 "sparcv8 sparcv8-fsmuld sparcv7 sparc"; 517 518 /* 519 * On Panther-based machines, this should 520 * also include AV_SPARC_POPC too 521 */ 522 cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2; 523 524 /* 525 * On cheetah, there's no hole in the virtual address space 526 */ 527 hole_start = hole_end = 0; 528 529 /* 530 * The kpm mapping window. 531 * kpm_size: 532 * The size of a single kpm range. 533 * The overall size will be: kpm_size * vac_colors. 534 * kpm_vbase: 535 * The virtual start address of the kpm range within the kernel 536 * virtual address space. kpm_vbase has to be kpm_size aligned. 537 */ 538 kpm_size = (size_t)(8ull * 1024 * 1024 * 1024 * 1024); /* 8TB */ 539 kpm_size_shift = 43; 540 kpm_vbase = (caddr_t)0x8000000000000000ull; /* 8EB */ 541 kpm_smallpages = 1; 542 543 /* 544 * The traptrace code uses either %tick or %stick for 545 * timestamping. We have %stick so we can use it. 546 */ 547 traptrace_use_stick = 1; 548 549 /* 550 * Cheetah has a performance counter overflow interrupt 551 */ 552 cpc_has_overflow_intr = 1; 553 554 /* 555 * Use cheetah flush-all support 556 */ 557 if (!disable_delay_tlb_flush) 558 delay_tlb_flush = 1; 559 560 #if defined(CPU_IMP_DUAL_PAGESIZE) 561 /* 562 * Use Cheetah+ and later dual page size support. 563 */ 564 if (!disable_dual_pgsz) { 565 cpu_impl_dual_pgsz = 1; 566 } 567 #endif /* CPU_IMP_DUAL_PAGESIZE */ 568 569 /* 570 * Declare that this architecture/cpu combination does fpRAS. 571 */ 572 fpras_implemented = 1; 573 574 /* 575 * Setup CE lookup table 576 */ 577 CE_INITDISPTBL_POPULATE(ce_disp_table); 578 ce_disp_inited = 1; 579 } 580 581 /* 582 * Called by setcpudelay 583 */ 584 void 585 cpu_init_tick_freq(void) 586 { 587 /* 588 * For UltraSPARC III and beyond we want to use the 589 * system clock rate as the basis for low level timing, 590 * due to support of mixed speed CPUs and power managment. 591 */ 592 if (system_clock_freq == 0) 593 cmn_err(CE_PANIC, "setcpudelay: invalid system_clock_freq"); 594 595 sys_tick_freq = system_clock_freq; 596 } 597 598 #ifdef CHEETAHPLUS_ERRATUM_25 599 /* 600 * Tunables 601 */ 602 int cheetah_bpe_off = 0; 603 int cheetah_sendmondo_recover = 1; 604 int cheetah_sendmondo_fullscan = 0; 605 int cheetah_sendmondo_recover_delay = 5; 606 607 #define CHEETAH_LIVELOCK_MIN_DELAY 1 608 609 /* 610 * Recovery Statistics 611 */ 612 typedef struct cheetah_livelock_entry { 613 int cpuid; /* fallen cpu */ 614 int buddy; /* cpu that ran recovery */ 615 clock_t lbolt; /* when recovery started */ 616 hrtime_t recovery_time; /* time spent in recovery */ 617 } cheetah_livelock_entry_t; 618 619 #define CHEETAH_LIVELOCK_NENTRY 32 620 621 cheetah_livelock_entry_t cheetah_livelock_hist[CHEETAH_LIVELOCK_NENTRY]; 622 int cheetah_livelock_entry_nxt; 623 624 #define CHEETAH_LIVELOCK_ENTRY_NEXT(statp) { \ 625 statp = cheetah_livelock_hist + cheetah_livelock_entry_nxt; \ 626 if (++cheetah_livelock_entry_nxt >= CHEETAH_LIVELOCK_NENTRY) { \ 627 cheetah_livelock_entry_nxt = 0; \ 628 } \ 629 } 630 631 #define CHEETAH_LIVELOCK_ENTRY_SET(statp, item, val) statp->item = val 632 633 struct { 634 hrtime_t hrt; /* maximum recovery time */ 635 int recovery; /* recovered */ 636 int full_claimed; /* maximum pages claimed in full recovery */ 637 int proc_entry; /* attempted to claim TSB */ 638 int proc_tsb_scan; /* tsb scanned */ 639 int proc_tsb_partscan; /* tsb partially scanned */ 640 int proc_tsb_fullscan; /* whole tsb scanned */ 641 int proc_claimed; /* maximum pages claimed in tsb scan */ 642 int proc_user; /* user thread */ 643 int proc_kernel; /* kernel thread */ 644 int proc_onflt; /* bad stack */ 645 int proc_cpu; /* null cpu */ 646 int proc_thread; /* null thread */ 647 int proc_proc; /* null proc */ 648 int proc_as; /* null as */ 649 int proc_hat; /* null hat */ 650 int proc_hat_inval; /* hat contents don't make sense */ 651 int proc_hat_busy; /* hat is changing TSBs */ 652 int proc_tsb_reloc; /* TSB skipped because being relocated */ 653 int proc_cnum_bad; /* cnum out of range */ 654 int proc_cnum; /* last cnum processed */ 655 tte_t proc_tte; /* last tte processed */ 656 } cheetah_livelock_stat; 657 658 #define CHEETAH_LIVELOCK_STAT(item) cheetah_livelock_stat.item++ 659 660 #define CHEETAH_LIVELOCK_STATSET(item, value) \ 661 cheetah_livelock_stat.item = value 662 663 #define CHEETAH_LIVELOCK_MAXSTAT(item, value) { \ 664 if (value > cheetah_livelock_stat.item) \ 665 cheetah_livelock_stat.item = value; \ 666 } 667 668 /* 669 * Attempt to recover a cpu by claiming every cache line as saved 670 * in the TSB that the non-responsive cpu is using. Since we can't 671 * grab any adaptive lock, this is at best an attempt to do so. Because 672 * we don't grab any locks, we must operate under the protection of 673 * on_fault(). 674 * 675 * Return 1 if cpuid could be recovered, 0 if failed. 676 */ 677 int 678 mondo_recover_proc(uint16_t cpuid, int bn) 679 { 680 label_t ljb; 681 cpu_t *cp; 682 kthread_t *t; 683 proc_t *p; 684 struct as *as; 685 struct hat *hat; 686 uint_t cnum; 687 struct tsb_info *tsbinfop; 688 struct tsbe *tsbep; 689 caddr_t tsbp; 690 caddr_t end_tsbp; 691 uint64_t paddr; 692 uint64_t idsr; 693 u_longlong_t pahi, palo; 694 int pages_claimed = 0; 695 tte_t tsbe_tte; 696 int tried_kernel_tsb = 0; 697 mmu_ctx_t *mmu_ctxp; 698 699 CHEETAH_LIVELOCK_STAT(proc_entry); 700 701 if (on_fault(&ljb)) { 702 CHEETAH_LIVELOCK_STAT(proc_onflt); 703 goto badstruct; 704 } 705 706 if ((cp = cpu[cpuid]) == NULL) { 707 CHEETAH_LIVELOCK_STAT(proc_cpu); 708 goto badstruct; 709 } 710 711 if ((t = cp->cpu_thread) == NULL) { 712 CHEETAH_LIVELOCK_STAT(proc_thread); 713 goto badstruct; 714 } 715 716 if ((p = ttoproc(t)) == NULL) { 717 CHEETAH_LIVELOCK_STAT(proc_proc); 718 goto badstruct; 719 } 720 721 if ((as = p->p_as) == NULL) { 722 CHEETAH_LIVELOCK_STAT(proc_as); 723 goto badstruct; 724 } 725 726 if ((hat = as->a_hat) == NULL) { 727 CHEETAH_LIVELOCK_STAT(proc_hat); 728 goto badstruct; 729 } 730 731 if (hat != ksfmmup) { 732 CHEETAH_LIVELOCK_STAT(proc_user); 733 if (hat->sfmmu_flags & (HAT_BUSY | HAT_SWAPPED | HAT_SWAPIN)) { 734 CHEETAH_LIVELOCK_STAT(proc_hat_busy); 735 goto badstruct; 736 } 737 tsbinfop = hat->sfmmu_tsb; 738 if (tsbinfop == NULL) { 739 CHEETAH_LIVELOCK_STAT(proc_hat_inval); 740 goto badstruct; 741 } 742 tsbp = tsbinfop->tsb_va; 743 end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 744 } else { 745 CHEETAH_LIVELOCK_STAT(proc_kernel); 746 tsbinfop = NULL; 747 tsbp = ktsb_base; 748 end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 749 } 750 751 /* Verify as */ 752 if (hat->sfmmu_as != as) { 753 CHEETAH_LIVELOCK_STAT(proc_hat_inval); 754 goto badstruct; 755 } 756 757 mmu_ctxp = CPU_MMU_CTXP(cp); 758 ASSERT(mmu_ctxp); 759 cnum = hat->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; 760 CHEETAH_LIVELOCK_STATSET(proc_cnum, cnum); 761 762 if ((cnum < 0) || (cnum == INVALID_CONTEXT) || 763 (cnum >= mmu_ctxp->mmu_nctxs)) { 764 CHEETAH_LIVELOCK_STAT(proc_cnum_bad); 765 goto badstruct; 766 } 767 768 do { 769 CHEETAH_LIVELOCK_STAT(proc_tsb_scan); 770 771 /* 772 * Skip TSBs being relocated. This is important because 773 * we want to avoid the following deadlock scenario: 774 * 775 * 1) when we came in we set ourselves to "in recover" state. 776 * 2) when we try to touch TSB being relocated the mapping 777 * will be in the suspended state so we'll spin waiting 778 * for it to be unlocked. 779 * 3) when the CPU that holds the TSB mapping locked tries to 780 * unlock it it will send a xtrap which will fail to xcall 781 * us or the CPU we're trying to recover, and will in turn 782 * enter the mondo code. 783 * 4) since we are still spinning on the locked mapping 784 * no further progress will be made and the system will 785 * inevitably hard hang. 786 * 787 * A TSB not being relocated can't begin being relocated 788 * while we're accessing it because we check 789 * sendmondo_in_recover before relocating TSBs. 790 */ 791 if (hat != ksfmmup && 792 (tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 793 CHEETAH_LIVELOCK_STAT(proc_tsb_reloc); 794 goto next_tsbinfo; 795 } 796 797 for (tsbep = (struct tsbe *)tsbp; 798 tsbep < (struct tsbe *)end_tsbp; tsbep++) { 799 tsbe_tte = tsbep->tte_data; 800 801 if (tsbe_tte.tte_val == 0) { 802 /* 803 * Invalid tte 804 */ 805 continue; 806 } 807 if (tsbe_tte.tte_se) { 808 /* 809 * Don't want device registers 810 */ 811 continue; 812 } 813 if (tsbe_tte.tte_cp == 0) { 814 /* 815 * Must be cached in E$ 816 */ 817 continue; 818 } 819 CHEETAH_LIVELOCK_STATSET(proc_tte, tsbe_tte); 820 idsr = getidsr(); 821 if ((idsr & (IDSR_NACK_BIT(bn) | 822 IDSR_BUSY_BIT(bn))) == 0) { 823 CHEETAH_LIVELOCK_STAT(proc_tsb_partscan); 824 goto done; 825 } 826 pahi = tsbe_tte.tte_pahi; 827 palo = tsbe_tte.tte_palo; 828 paddr = (uint64_t)((pahi << 32) | 829 (palo << MMU_PAGESHIFT)); 830 claimlines(paddr, TTEBYTES(TTE_CSZ(&tsbe_tte)), 831 CH_ECACHE_SUBBLK_SIZE); 832 if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 833 shipit(cpuid, bn); 834 } 835 pages_claimed++; 836 } 837 next_tsbinfo: 838 if (tsbinfop != NULL) 839 tsbinfop = tsbinfop->tsb_next; 840 if (tsbinfop != NULL) { 841 tsbp = tsbinfop->tsb_va; 842 end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 843 } else if (tsbp == ktsb_base) { 844 tried_kernel_tsb = 1; 845 } else if (!tried_kernel_tsb) { 846 tsbp = ktsb_base; 847 end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 848 hat = ksfmmup; 849 tsbinfop = NULL; 850 } 851 } while (tsbinfop != NULL || 852 ((tsbp == ktsb_base) && !tried_kernel_tsb)); 853 854 CHEETAH_LIVELOCK_STAT(proc_tsb_fullscan); 855 CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 856 no_fault(); 857 idsr = getidsr(); 858 if ((idsr & (IDSR_NACK_BIT(bn) | 859 IDSR_BUSY_BIT(bn))) == 0) { 860 return (1); 861 } else { 862 return (0); 863 } 864 865 done: 866 no_fault(); 867 CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 868 return (1); 869 870 badstruct: 871 no_fault(); 872 return (0); 873 } 874 875 /* 876 * Attempt to claim ownership, temporarily, of every cache line that a 877 * non-responsive cpu might be using. This might kick that cpu out of 878 * this state. 879 * 880 * The return value indicates to the caller if we have exhausted all recovery 881 * techniques. If 1 is returned, it is useless to call this function again 882 * even for a different target CPU. 883 */ 884 int 885 mondo_recover(uint16_t cpuid, int bn) 886 { 887 struct memseg *seg; 888 uint64_t begin_pa, end_pa, cur_pa; 889 hrtime_t begin_hrt, end_hrt; 890 int retval = 0; 891 int pages_claimed = 0; 892 cheetah_livelock_entry_t *histp; 893 uint64_t idsr; 894 895 if (cas32(&sendmondo_in_recover, 0, 1) != 0) { 896 /* 897 * Wait while recovery takes place 898 */ 899 while (sendmondo_in_recover) { 900 drv_usecwait(1); 901 } 902 /* 903 * Assume we didn't claim the whole memory. If 904 * the target of this caller is not recovered, 905 * it will come back. 906 */ 907 return (retval); 908 } 909 910 CHEETAH_LIVELOCK_ENTRY_NEXT(histp) 911 CHEETAH_LIVELOCK_ENTRY_SET(histp, lbolt, lbolt); 912 CHEETAH_LIVELOCK_ENTRY_SET(histp, cpuid, cpuid); 913 CHEETAH_LIVELOCK_ENTRY_SET(histp, buddy, CPU->cpu_id); 914 915 begin_hrt = gethrtime_waitfree(); 916 /* 917 * First try to claim the lines in the TSB the target 918 * may have been using. 919 */ 920 if (mondo_recover_proc(cpuid, bn) == 1) { 921 /* 922 * Didn't claim the whole memory 923 */ 924 goto done; 925 } 926 927 /* 928 * We tried using the TSB. The target is still 929 * not recovered. Check if complete memory scan is 930 * enabled. 931 */ 932 if (cheetah_sendmondo_fullscan == 0) { 933 /* 934 * Full memory scan is disabled. 935 */ 936 retval = 1; 937 goto done; 938 } 939 940 /* 941 * Try claiming the whole memory. 942 */ 943 for (seg = memsegs; seg; seg = seg->next) { 944 begin_pa = (uint64_t)(seg->pages_base) << MMU_PAGESHIFT; 945 end_pa = (uint64_t)(seg->pages_end) << MMU_PAGESHIFT; 946 for (cur_pa = begin_pa; cur_pa < end_pa; 947 cur_pa += MMU_PAGESIZE) { 948 idsr = getidsr(); 949 if ((idsr & (IDSR_NACK_BIT(bn) | 950 IDSR_BUSY_BIT(bn))) == 0) { 951 /* 952 * Didn't claim all memory 953 */ 954 goto done; 955 } 956 claimlines(cur_pa, MMU_PAGESIZE, 957 CH_ECACHE_SUBBLK_SIZE); 958 if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 959 shipit(cpuid, bn); 960 } 961 pages_claimed++; 962 } 963 } 964 965 /* 966 * We did all we could. 967 */ 968 retval = 1; 969 970 done: 971 /* 972 * Update statistics 973 */ 974 end_hrt = gethrtime_waitfree(); 975 CHEETAH_LIVELOCK_STAT(recovery); 976 CHEETAH_LIVELOCK_MAXSTAT(hrt, (end_hrt - begin_hrt)); 977 CHEETAH_LIVELOCK_MAXSTAT(full_claimed, pages_claimed); 978 CHEETAH_LIVELOCK_ENTRY_SET(histp, recovery_time, \ 979 (end_hrt - begin_hrt)); 980 981 while (cas32(&sendmondo_in_recover, 1, 0) != 1); 982 983 return (retval); 984 } 985 986 /* 987 * This is called by the cyclic framework when this CPU becomes online 988 */ 989 /*ARGSUSED*/ 990 static void 991 cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when) 992 { 993 994 hdlr->cyh_func = (cyc_func_t)cheetah_nudge_buddy; 995 hdlr->cyh_level = CY_LOW_LEVEL; 996 hdlr->cyh_arg = NULL; 997 998 /* 999 * Stagger the start time 1000 */ 1001 when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU); 1002 if (cheetah_sendmondo_recover_delay < CHEETAH_LIVELOCK_MIN_DELAY) { 1003 cheetah_sendmondo_recover_delay = CHEETAH_LIVELOCK_MIN_DELAY; 1004 } 1005 when->cyt_interval = cheetah_sendmondo_recover_delay * NANOSEC; 1006 } 1007 1008 /* 1009 * Create a low level cyclic to send a xtrap to the next cpu online. 1010 * However, there's no need to have this running on a uniprocessor system. 1011 */ 1012 static void 1013 cheetah_nudge_init(void) 1014 { 1015 cyc_omni_handler_t hdlr; 1016 1017 if (max_ncpus == 1) { 1018 return; 1019 } 1020 1021 hdlr.cyo_online = cheetah_nudge_onln; 1022 hdlr.cyo_offline = NULL; 1023 hdlr.cyo_arg = NULL; 1024 1025 mutex_enter(&cpu_lock); 1026 (void) cyclic_add_omni(&hdlr); 1027 mutex_exit(&cpu_lock); 1028 } 1029 1030 /* 1031 * Cyclic handler to wake up buddy 1032 */ 1033 void 1034 cheetah_nudge_buddy(void) 1035 { 1036 /* 1037 * Disable kernel preemption to protect the cpu list 1038 */ 1039 kpreempt_disable(); 1040 if ((CPU->cpu_next_onln != CPU) && (sendmondo_in_recover == 0)) { 1041 xt_one(CPU->cpu_next_onln->cpu_id, (xcfunc_t *)xt_sync_tl1, 1042 0, 0); 1043 } 1044 kpreempt_enable(); 1045 } 1046 1047 #endif /* CHEETAHPLUS_ERRATUM_25 */ 1048 1049 #ifdef SEND_MONDO_STATS 1050 uint32_t x_one_stimes[64]; 1051 uint32_t x_one_ltimes[16]; 1052 uint32_t x_set_stimes[64]; 1053 uint32_t x_set_ltimes[16]; 1054 uint32_t x_set_cpus[NCPU]; 1055 uint32_t x_nack_stimes[64]; 1056 #endif 1057 1058 /* 1059 * Note: A version of this function is used by the debugger via the KDI, 1060 * and must be kept in sync with this version. Any changes made to this 1061 * function to support new chips or to accomodate errata must also be included 1062 * in the KDI-specific version. See us3_kdi.c. 1063 */ 1064 void 1065 send_one_mondo(int cpuid) 1066 { 1067 int busy, nack; 1068 uint64_t idsr, starttick, endtick, tick, lasttick; 1069 uint64_t busymask; 1070 #ifdef CHEETAHPLUS_ERRATUM_25 1071 int recovered = 0; 1072 #endif 1073 1074 CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 1075 starttick = lasttick = gettick(); 1076 shipit(cpuid, 0); 1077 endtick = starttick + xc_tick_limit; 1078 busy = nack = 0; 1079 #if defined(JALAPENO) || defined(SERRANO) 1080 /* 1081 * Lower 2 bits of the agent ID determine which BUSY/NACK pair 1082 * will be used for dispatching interrupt. For now, assume 1083 * there are no more than IDSR_BN_SETS CPUs, hence no aliasing 1084 * issues with respect to BUSY/NACK pair usage. 1085 */ 1086 busymask = IDSR_BUSY_BIT(cpuid); 1087 #else /* JALAPENO || SERRANO */ 1088 busymask = IDSR_BUSY; 1089 #endif /* JALAPENO || SERRANO */ 1090 for (;;) { 1091 idsr = getidsr(); 1092 if (idsr == 0) 1093 break; 1094 1095 tick = gettick(); 1096 /* 1097 * If there is a big jump between the current tick 1098 * count and lasttick, we have probably hit a break 1099 * point. Adjust endtick accordingly to avoid panic. 1100 */ 1101 if (tick > (lasttick + xc_tick_jump_limit)) 1102 endtick += (tick - lasttick); 1103 lasttick = tick; 1104 if (tick > endtick) { 1105 if (panic_quiesce) 1106 return; 1107 #ifdef CHEETAHPLUS_ERRATUM_25 1108 if (cheetah_sendmondo_recover && recovered == 0) { 1109 if (mondo_recover(cpuid, 0)) { 1110 /* 1111 * We claimed the whole memory or 1112 * full scan is disabled. 1113 */ 1114 recovered++; 1115 } 1116 tick = gettick(); 1117 endtick = tick + xc_tick_limit; 1118 lasttick = tick; 1119 /* 1120 * Recheck idsr 1121 */ 1122 continue; 1123 } else 1124 #endif /* CHEETAHPLUS_ERRATUM_25 */ 1125 { 1126 cmn_err(CE_PANIC, "send mondo timeout " 1127 "(target 0x%x) [%d NACK %d BUSY]", 1128 cpuid, nack, busy); 1129 } 1130 } 1131 1132 if (idsr & busymask) { 1133 busy++; 1134 continue; 1135 } 1136 drv_usecwait(1); 1137 shipit(cpuid, 0); 1138 nack++; 1139 busy = 0; 1140 } 1141 #ifdef SEND_MONDO_STATS 1142 { 1143 int n = gettick() - starttick; 1144 if (n < 8192) 1145 x_one_stimes[n >> 7]++; 1146 else 1147 x_one_ltimes[(n >> 13) & 0xf]++; 1148 } 1149 #endif 1150 } 1151 1152 void 1153 syncfpu(void) 1154 { 1155 } 1156 1157 /* 1158 * Return processor specific async error structure 1159 * size used. 1160 */ 1161 int 1162 cpu_aflt_size(void) 1163 { 1164 return (sizeof (ch_async_flt_t)); 1165 } 1166 1167 /* 1168 * Tunable to disable the checking of other cpu logout areas during panic for 1169 * potential syndrome 71 generating errors. 1170 */ 1171 int enable_check_other_cpus_logout = 1; 1172 1173 /* 1174 * Check other cpus logout area for potential synd 71 generating 1175 * errors. 1176 */ 1177 static void 1178 cpu_check_cpu_logout(int cpuid, caddr_t tpc, int tl, int ecc_type, 1179 ch_cpu_logout_t *clop) 1180 { 1181 struct async_flt *aflt; 1182 ch_async_flt_t ch_flt; 1183 uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1184 1185 if (clop == NULL || clop->clo_data.chd_afar == LOGOUT_INVALID) { 1186 return; 1187 } 1188 1189 bzero(&ch_flt, sizeof (ch_async_flt_t)); 1190 1191 t_afar = clop->clo_data.chd_afar; 1192 t_afsr = clop->clo_data.chd_afsr; 1193 t_afsr_ext = clop->clo_data.chd_afsr_ext; 1194 #if defined(SERRANO) 1195 ch_flt.afar2 = clop->clo_data.chd_afar2; 1196 #endif /* SERRANO */ 1197 1198 /* 1199 * In order to simplify code, we maintain this afsr_errs 1200 * variable which holds the aggregate of AFSR and AFSR_EXT 1201 * sticky bits. 1202 */ 1203 t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1204 (t_afsr & C_AFSR_ALL_ERRS); 1205 1206 /* Setup the async fault structure */ 1207 aflt = (struct async_flt *)&ch_flt; 1208 aflt->flt_id = gethrtime_waitfree(); 1209 ch_flt.afsr_ext = t_afsr_ext; 1210 ch_flt.afsr_errs = t_afsr_errs; 1211 aflt->flt_stat = t_afsr; 1212 aflt->flt_addr = t_afar; 1213 aflt->flt_bus_id = cpuid; 1214 aflt->flt_inst = cpuid; 1215 aflt->flt_pc = tpc; 1216 aflt->flt_prot = AFLT_PROT_NONE; 1217 aflt->flt_class = CPU_FAULT; 1218 aflt->flt_priv = ((t_afsr & C_AFSR_PRIV) != 0); 1219 aflt->flt_tl = tl; 1220 aflt->flt_status = ecc_type; 1221 aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 1222 1223 /* 1224 * Queue events on the async event queue, one event per error bit. 1225 * If no events are queued, queue an event to complain. 1226 */ 1227 if (cpu_queue_events(&ch_flt, NULL, t_afsr_errs, clop) == 0) { 1228 ch_flt.flt_type = CPU_INV_AFSR; 1229 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 1230 (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 1231 aflt->flt_panic); 1232 } 1233 1234 /* 1235 * Zero out + invalidate CPU logout. 1236 */ 1237 bzero(clop, sizeof (ch_cpu_logout_t)); 1238 clop->clo_data.chd_afar = LOGOUT_INVALID; 1239 } 1240 1241 /* 1242 * Check the logout areas of all other cpus for unlogged errors. 1243 */ 1244 static void 1245 cpu_check_other_cpus_logout(void) 1246 { 1247 int i, j; 1248 processorid_t myid; 1249 struct cpu *cp; 1250 ch_err_tl1_data_t *cl1p; 1251 1252 myid = CPU->cpu_id; 1253 for (i = 0; i < NCPU; i++) { 1254 cp = cpu[i]; 1255 1256 if ((cp == NULL) || !(cp->cpu_flags & CPU_EXISTS) || 1257 (cp->cpu_id == myid) || (CPU_PRIVATE(cp) == NULL)) { 1258 continue; 1259 } 1260 1261 /* 1262 * Check each of the tl>0 logout areas 1263 */ 1264 cl1p = CPU_PRIVATE_PTR(cp, chpr_tl1_err_data[0]); 1265 for (j = 0; j < CH_ERR_TL1_TLMAX; j++, cl1p++) { 1266 if (cl1p->ch_err_tl1_flags == 0) 1267 continue; 1268 1269 cpu_check_cpu_logout(i, (caddr_t)cl1p->ch_err_tl1_tpc, 1270 1, ECC_F_TRAP, &cl1p->ch_err_tl1_logout); 1271 } 1272 1273 /* 1274 * Check each of the remaining logout areas 1275 */ 1276 cpu_check_cpu_logout(i, NULL, 0, ECC_F_TRAP, 1277 CPU_PRIVATE_PTR(cp, chpr_fecctl0_logout)); 1278 cpu_check_cpu_logout(i, NULL, 0, ECC_C_TRAP, 1279 CPU_PRIVATE_PTR(cp, chpr_cecc_logout)); 1280 cpu_check_cpu_logout(i, NULL, 0, ECC_D_TRAP, 1281 CPU_PRIVATE_PTR(cp, chpr_async_logout)); 1282 } 1283 } 1284 1285 /* 1286 * The fast_ecc_err handler transfers control here for UCU, UCC events. 1287 * Note that we flush Ecache twice, once in the fast_ecc_err handler to 1288 * flush the error that caused the UCU/UCC, then again here at the end to 1289 * flush the TL=1 trap handler code out of the Ecache, so we can minimize 1290 * the probability of getting a TL>1 Fast ECC trap when we're fielding 1291 * another Fast ECC trap. 1292 * 1293 * Cheetah+ also handles: TSCE: No additional processing required. 1294 * Panther adds L3_UCU and L3_UCC which are reported in AFSR_EXT. 1295 * 1296 * Note that the p_clo_flags input is only valid in cases where the 1297 * cpu_private struct is not yet initialized (since that is the only 1298 * time that information cannot be obtained from the logout struct.) 1299 */ 1300 /*ARGSUSED*/ 1301 void 1302 cpu_fast_ecc_error(struct regs *rp, ulong_t p_clo_flags) 1303 { 1304 ch_cpu_logout_t *clop; 1305 uint64_t ceen, nceen; 1306 1307 /* 1308 * Get the CPU log out info. If we can't find our CPU private 1309 * pointer, then we will have to make due without any detailed 1310 * logout information. 1311 */ 1312 if (CPU_PRIVATE(CPU) == NULL) { 1313 clop = NULL; 1314 ceen = p_clo_flags & EN_REG_CEEN; 1315 nceen = p_clo_flags & EN_REG_NCEEN; 1316 } else { 1317 clop = CPU_PRIVATE_PTR(CPU, chpr_fecctl0_logout); 1318 ceen = clop->clo_flags & EN_REG_CEEN; 1319 nceen = clop->clo_flags & EN_REG_NCEEN; 1320 } 1321 1322 cpu_log_fast_ecc_error((caddr_t)rp->r_pc, 1323 (rp->r_tstate & TSTATE_PRIV) ? 1 : 0, 0, ceen, nceen, clop); 1324 } 1325 1326 /* 1327 * Log fast ecc error, called from either Fast ECC at TL=0 or Fast 1328 * ECC at TL>0. Need to supply either a error register pointer or a 1329 * cpu logout structure pointer. 1330 */ 1331 static void 1332 cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 1333 uint64_t nceen, ch_cpu_logout_t *clop) 1334 { 1335 struct async_flt *aflt; 1336 ch_async_flt_t ch_flt; 1337 uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1338 char pr_reason[MAX_REASON_STRING]; 1339 ch_cpu_errors_t cpu_error_regs; 1340 1341 bzero(&ch_flt, sizeof (ch_async_flt_t)); 1342 /* 1343 * If no cpu logout data, then we will have to make due without 1344 * any detailed logout information. 1345 */ 1346 if (clop == NULL) { 1347 ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 1348 get_cpu_error_state(&cpu_error_regs); 1349 set_cpu_error_state(&cpu_error_regs); 1350 t_afar = cpu_error_regs.afar; 1351 t_afsr = cpu_error_regs.afsr; 1352 t_afsr_ext = cpu_error_regs.afsr_ext; 1353 #if defined(SERRANO) 1354 ch_flt.afar2 = cpu_error_regs.afar2; 1355 #endif /* SERRANO */ 1356 } else { 1357 t_afar = clop->clo_data.chd_afar; 1358 t_afsr = clop->clo_data.chd_afsr; 1359 t_afsr_ext = clop->clo_data.chd_afsr_ext; 1360 #if defined(SERRANO) 1361 ch_flt.afar2 = clop->clo_data.chd_afar2; 1362 #endif /* SERRANO */ 1363 } 1364 1365 /* 1366 * In order to simplify code, we maintain this afsr_errs 1367 * variable which holds the aggregate of AFSR and AFSR_EXT 1368 * sticky bits. 1369 */ 1370 t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1371 (t_afsr & C_AFSR_ALL_ERRS); 1372 pr_reason[0] = '\0'; 1373 1374 /* Setup the async fault structure */ 1375 aflt = (struct async_flt *)&ch_flt; 1376 aflt->flt_id = gethrtime_waitfree(); 1377 ch_flt.afsr_ext = t_afsr_ext; 1378 ch_flt.afsr_errs = t_afsr_errs; 1379 aflt->flt_stat = t_afsr; 1380 aflt->flt_addr = t_afar; 1381 aflt->flt_bus_id = getprocessorid(); 1382 aflt->flt_inst = CPU->cpu_id; 1383 aflt->flt_pc = tpc; 1384 aflt->flt_prot = AFLT_PROT_NONE; 1385 aflt->flt_class = CPU_FAULT; 1386 aflt->flt_priv = priv; 1387 aflt->flt_tl = tl; 1388 aflt->flt_status = ECC_F_TRAP; 1389 aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 1390 1391 /* 1392 * XXXX - Phenomenal hack to get around Solaris not getting all the 1393 * cmn_err messages out to the console. The situation is a UCU (in 1394 * priv mode) which causes a WDU which causes a UE (on the retry). 1395 * The messages for the UCU and WDU are enqueued and then pulled off 1396 * the async queue via softint and syslogd starts to process them 1397 * but doesn't get them to the console. The UE causes a panic, but 1398 * since the UCU/WDU messages are already in transit, those aren't 1399 * on the async queue. The hack is to check if we have a matching 1400 * WDU event for the UCU, and if it matches, we're more than likely 1401 * going to panic with a UE, unless we're under protection. So, we 1402 * check to see if we got a matching WDU event and if we're under 1403 * protection. 1404 * 1405 * For Cheetah/Cheetah+/Jaguar/Jalapeno, the sequence we care about 1406 * looks like this: 1407 * UCU->WDU->UE 1408 * For Panther, it could look like either of these: 1409 * UCU---->WDU->L3_WDU->UE 1410 * L3_UCU->WDU->L3_WDU->UE 1411 */ 1412 if ((t_afsr_errs & (C_AFSR_UCU | C_AFSR_L3_UCU)) && 1413 aflt->flt_panic == 0 && aflt->flt_priv != 0 && 1414 curthread->t_ontrap == NULL && curthread->t_lofault == NULL) { 1415 get_cpu_error_state(&cpu_error_regs); 1416 aflt->flt_panic |= ((cpu_error_regs.afsr & C_AFSR_WDU) && 1417 (cpu_error_regs.afar == t_afar)); 1418 aflt->flt_panic |= ((clop == NULL) && 1419 (t_afsr_errs & C_AFSR_WDU)); 1420 } 1421 1422 /* 1423 * Queue events on the async event queue, one event per error bit. 1424 * If no events are queued or no Fast ECC events are on in the AFSR, 1425 * queue an event to complain. 1426 */ 1427 if (cpu_queue_events(&ch_flt, pr_reason, t_afsr_errs, clop) == 0 || 1428 ((t_afsr_errs & (C_AFSR_FECC_ERRS | C_AFSR_EXT_FECC_ERRS)) == 0)) { 1429 ch_flt.flt_type = CPU_INV_AFSR; 1430 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 1431 (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 1432 aflt->flt_panic); 1433 } 1434 1435 /* 1436 * Zero out + invalidate CPU logout. 1437 */ 1438 if (clop) { 1439 bzero(clop, sizeof (ch_cpu_logout_t)); 1440 clop->clo_data.chd_afar = LOGOUT_INVALID; 1441 } 1442 1443 /* 1444 * We carefully re-enable NCEEN and CEEN and then check if any deferred 1445 * or disrupting errors have happened. We do this because if a 1446 * deferred or disrupting error had occurred with NCEEN/CEEN off, the 1447 * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 1448 * CEEN works differently on Cheetah than on Spitfire. Also, we enable 1449 * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 1450 * deferred or disrupting error happening between checking the AFSR and 1451 * enabling NCEEN/CEEN. 1452 * 1453 * Note: CEEN and NCEEN are only reenabled if they were on when trap 1454 * taken. 1455 */ 1456 set_error_enable(get_error_enable() | (nceen | ceen)); 1457 if (clear_errors(&ch_flt)) { 1458 aflt->flt_panic |= ((ch_flt.afsr_errs & 1459 (C_AFSR_EXT_ASYNC_ERRS | C_AFSR_ASYNC_ERRS)) != 0); 1460 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 1461 NULL); 1462 } 1463 1464 /* 1465 * Panic here if aflt->flt_panic has been set. Enqueued errors will 1466 * be logged as part of the panic flow. 1467 */ 1468 if (aflt->flt_panic) 1469 fm_panic("%sError(s)", pr_reason); 1470 1471 /* 1472 * Flushing the Ecache here gets the part of the trap handler that 1473 * is run at TL=1 out of the Ecache. 1474 */ 1475 cpu_flush_ecache(); 1476 } 1477 1478 /* 1479 * This is called via sys_trap from pil15_interrupt code if the 1480 * corresponding entry in ch_err_tl1_pending is set. Checks the 1481 * various ch_err_tl1_data structures for valid entries based on the bit 1482 * settings in the ch_err_tl1_flags entry of the structure. 1483 */ 1484 /*ARGSUSED*/ 1485 void 1486 cpu_tl1_error(struct regs *rp, int panic) 1487 { 1488 ch_err_tl1_data_t *cl1p, cl1; 1489 int i, ncl1ps; 1490 uint64_t me_flags; 1491 uint64_t ceen, nceen; 1492 1493 if (ch_err_tl1_paddrs[CPU->cpu_id] == 0) { 1494 cl1p = &ch_err_tl1_data; 1495 ncl1ps = 1; 1496 } else if (CPU_PRIVATE(CPU) != NULL) { 1497 cl1p = CPU_PRIVATE_PTR(CPU, chpr_tl1_err_data[0]); 1498 ncl1ps = CH_ERR_TL1_TLMAX; 1499 } else { 1500 ncl1ps = 0; 1501 } 1502 1503 for (i = 0; i < ncl1ps; i++, cl1p++) { 1504 if (cl1p->ch_err_tl1_flags == 0) 1505 continue; 1506 1507 /* 1508 * Grab a copy of the logout data and invalidate 1509 * the logout area. 1510 */ 1511 cl1 = *cl1p; 1512 bzero(cl1p, sizeof (ch_err_tl1_data_t)); 1513 cl1p->ch_err_tl1_logout.clo_data.chd_afar = LOGOUT_INVALID; 1514 me_flags = CH_ERR_ME_FLAGS(cl1.ch_err_tl1_flags); 1515 1516 /* 1517 * Log "first error" in ch_err_tl1_data. 1518 */ 1519 if (cl1.ch_err_tl1_flags & CH_ERR_FECC) { 1520 ceen = get_error_enable() & EN_REG_CEEN; 1521 nceen = get_error_enable() & EN_REG_NCEEN; 1522 cpu_log_fast_ecc_error((caddr_t)cl1.ch_err_tl1_tpc, 1, 1523 1, ceen, nceen, &cl1.ch_err_tl1_logout); 1524 } 1525 #if defined(CPU_IMP_L1_CACHE_PARITY) 1526 if (cl1.ch_err_tl1_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 1527 cpu_parity_error(rp, cl1.ch_err_tl1_flags, 1528 (caddr_t)cl1.ch_err_tl1_tpc); 1529 } 1530 #endif /* CPU_IMP_L1_CACHE_PARITY */ 1531 1532 /* 1533 * Log "multiple events" in ch_err_tl1_data. Note that 1534 * we don't read and clear the AFSR/AFAR in the TL>0 code 1535 * if the structure is busy, we just do the cache flushing 1536 * we have to do and then do the retry. So the AFSR/AFAR 1537 * at this point *should* have some relevant info. If there 1538 * are no valid errors in the AFSR, we'll assume they've 1539 * already been picked up and logged. For I$/D$ parity, 1540 * we just log an event with an "Unknown" (NULL) TPC. 1541 */ 1542 if (me_flags & CH_ERR_FECC) { 1543 ch_cpu_errors_t cpu_error_regs; 1544 uint64_t t_afsr_errs; 1545 1546 /* 1547 * Get the error registers and see if there's 1548 * a pending error. If not, don't bother 1549 * generating an "Invalid AFSR" error event. 1550 */ 1551 get_cpu_error_state(&cpu_error_regs); 1552 t_afsr_errs = (cpu_error_regs.afsr_ext & 1553 C_AFSR_EXT_ALL_ERRS) | 1554 (cpu_error_regs.afsr & C_AFSR_ALL_ERRS); 1555 if (t_afsr_errs != 0) { 1556 ceen = get_error_enable() & EN_REG_CEEN; 1557 nceen = get_error_enable() & EN_REG_NCEEN; 1558 cpu_log_fast_ecc_error((caddr_t)NULL, 1, 1559 1, ceen, nceen, NULL); 1560 } 1561 } 1562 #if defined(CPU_IMP_L1_CACHE_PARITY) 1563 if (me_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 1564 cpu_parity_error(rp, me_flags, (caddr_t)NULL); 1565 } 1566 #endif /* CPU_IMP_L1_CACHE_PARITY */ 1567 } 1568 } 1569 1570 /* 1571 * Called from Fast ECC TL>0 handler in case of fatal error. 1572 * cpu_tl1_error should always find an associated ch_err_tl1_data structure, 1573 * but if we don't, we'll panic with something reasonable. 1574 */ 1575 /*ARGSUSED*/ 1576 void 1577 cpu_tl1_err_panic(struct regs *rp, ulong_t flags) 1578 { 1579 cpu_tl1_error(rp, 1); 1580 /* 1581 * Should never return, but just in case. 1582 */ 1583 fm_panic("Unsurvivable ECC Error at TL>0"); 1584 } 1585 1586 /* 1587 * The ce_err/ce_err_tl1 handlers transfer control here for CE, EMC, EDU:ST, 1588 * EDC, WDU, WDC, CPU, CPC, IVU, IVC events. 1589 * Disrupting errors controlled by NCEEN: EDU:ST, WDU, CPU, IVU 1590 * Disrupting errors controlled by CEEN: CE, EMC, EDC, WDC, CPC, IVC 1591 * 1592 * Cheetah+ also handles (No additional processing required): 1593 * DUE, DTO, DBERR (NCEEN controlled) 1594 * THCE (CEEN and ET_ECC_en controlled) 1595 * TUE (ET_ECC_en controlled) 1596 * 1597 * Panther further adds: 1598 * IMU, L3_EDU, L3_WDU, L3_CPU (NCEEN controlled) 1599 * IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE (CEEN controlled) 1600 * TUE_SH, TUE (NCEEN and L2_tag_ECC_en controlled) 1601 * L3_TUE, L3_TUE_SH (NCEEN and ET_ECC_en controlled) 1602 * THCE (CEEN and L2_tag_ECC_en controlled) 1603 * L3_THCE (CEEN and ET_ECC_en controlled) 1604 * 1605 * Note that the p_clo_flags input is only valid in cases where the 1606 * cpu_private struct is not yet initialized (since that is the only 1607 * time that information cannot be obtained from the logout struct.) 1608 */ 1609 /*ARGSUSED*/ 1610 void 1611 cpu_disrupting_error(struct regs *rp, ulong_t p_clo_flags) 1612 { 1613 struct async_flt *aflt; 1614 ch_async_flt_t ch_flt; 1615 char pr_reason[MAX_REASON_STRING]; 1616 ch_cpu_logout_t *clop; 1617 uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1618 ch_cpu_errors_t cpu_error_regs; 1619 1620 bzero(&ch_flt, sizeof (ch_async_flt_t)); 1621 /* 1622 * Get the CPU log out info. If we can't find our CPU private 1623 * pointer, then we will have to make due without any detailed 1624 * logout information. 1625 */ 1626 if (CPU_PRIVATE(CPU) == NULL) { 1627 clop = NULL; 1628 ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 1629 get_cpu_error_state(&cpu_error_regs); 1630 set_cpu_error_state(&cpu_error_regs); 1631 t_afar = cpu_error_regs.afar; 1632 t_afsr = cpu_error_regs.afsr; 1633 t_afsr_ext = cpu_error_regs.afsr_ext; 1634 #if defined(SERRANO) 1635 ch_flt.afar2 = cpu_error_regs.afar2; 1636 #endif /* SERRANO */ 1637 } else { 1638 clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 1639 t_afar = clop->clo_data.chd_afar; 1640 t_afsr = clop->clo_data.chd_afsr; 1641 t_afsr_ext = clop->clo_data.chd_afsr_ext; 1642 #if defined(SERRANO) 1643 ch_flt.afar2 = clop->clo_data.chd_afar2; 1644 #endif /* SERRANO */ 1645 } 1646 1647 /* 1648 * In order to simplify code, we maintain this afsr_errs 1649 * variable which holds the aggregate of AFSR and AFSR_EXT 1650 * sticky bits. 1651 */ 1652 t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1653 (t_afsr & C_AFSR_ALL_ERRS); 1654 1655 pr_reason[0] = '\0'; 1656 /* Setup the async fault structure */ 1657 aflt = (struct async_flt *)&ch_flt; 1658 ch_flt.afsr_ext = t_afsr_ext; 1659 ch_flt.afsr_errs = t_afsr_errs; 1660 aflt->flt_stat = t_afsr; 1661 aflt->flt_addr = t_afar; 1662 aflt->flt_pc = (caddr_t)rp->r_pc; 1663 aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 1664 aflt->flt_tl = 0; 1665 aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 1666 1667 /* 1668 * If this trap is a result of one of the errors not masked 1669 * by cpu_ce_not_deferred, we don't reenable CEEN. Instead 1670 * indicate that a timeout is to be set later. 1671 */ 1672 if (!(t_afsr_errs & (cpu_ce_not_deferred | cpu_ce_not_deferred_ext)) && 1673 !aflt->flt_panic) 1674 ch_flt.flt_trapped_ce = CE_CEEN_DEFER | CE_CEEN_TRAPPED; 1675 else 1676 ch_flt.flt_trapped_ce = CE_CEEN_NODEFER | CE_CEEN_TRAPPED; 1677 1678 /* 1679 * log the CE and clean up 1680 */ 1681 cpu_log_and_clear_ce(&ch_flt); 1682 1683 /* 1684 * We re-enable CEEN (if required) and check if any disrupting errors 1685 * have happened. We do this because if a disrupting error had occurred 1686 * with CEEN off, the trap will not be taken when CEEN is re-enabled. 1687 * Note that CEEN works differently on Cheetah than on Spitfire. Also, 1688 * we enable CEEN *before* checking the AFSR to avoid the small window 1689 * of a error happening between checking the AFSR and enabling CEEN. 1690 */ 1691 if (ch_flt.flt_trapped_ce & CE_CEEN_NODEFER) 1692 set_error_enable(get_error_enable() | EN_REG_CEEN); 1693 if (clear_errors(&ch_flt)) { 1694 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 1695 NULL); 1696 } 1697 1698 /* 1699 * Panic here if aflt->flt_panic has been set. Enqueued errors will 1700 * be logged as part of the panic flow. 1701 */ 1702 if (aflt->flt_panic) 1703 fm_panic("%sError(s)", pr_reason); 1704 } 1705 1706 /* 1707 * The async_err handler transfers control here for UE, EMU, EDU:BLD, 1708 * L3_EDU:BLD, TO, and BERR events. 1709 * Deferred errors controlled by NCEEN: UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR 1710 * 1711 * Cheetah+: No additional errors handled. 1712 * 1713 * Note that the p_clo_flags input is only valid in cases where the 1714 * cpu_private struct is not yet initialized (since that is the only 1715 * time that information cannot be obtained from the logout struct.) 1716 */ 1717 /*ARGSUSED*/ 1718 void 1719 cpu_deferred_error(struct regs *rp, ulong_t p_clo_flags) 1720 { 1721 ushort_t ttype, tl; 1722 ch_async_flt_t ch_flt; 1723 struct async_flt *aflt; 1724 int trampolined = 0; 1725 char pr_reason[MAX_REASON_STRING]; 1726 ch_cpu_logout_t *clop; 1727 uint64_t ceen, clo_flags; 1728 uint64_t log_afsr; 1729 uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1730 ch_cpu_errors_t cpu_error_regs; 1731 int expected = DDI_FM_ERR_UNEXPECTED; 1732 ddi_acc_hdl_t *hp; 1733 1734 /* 1735 * We need to look at p_flag to determine if the thread detected an 1736 * error while dumping core. We can't grab p_lock here, but it's ok 1737 * because we just need a consistent snapshot and we know that everyone 1738 * else will store a consistent set of bits while holding p_lock. We 1739 * don't have to worry about a race because SDOCORE is set once prior 1740 * to doing i/o from the process's address space and is never cleared. 1741 */ 1742 uint_t pflag = ttoproc(curthread)->p_flag; 1743 1744 bzero(&ch_flt, sizeof (ch_async_flt_t)); 1745 /* 1746 * Get the CPU log out info. If we can't find our CPU private 1747 * pointer then we will have to make due without any detailed 1748 * logout information. 1749 */ 1750 if (CPU_PRIVATE(CPU) == NULL) { 1751 clop = NULL; 1752 ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 1753 get_cpu_error_state(&cpu_error_regs); 1754 set_cpu_error_state(&cpu_error_regs); 1755 t_afar = cpu_error_regs.afar; 1756 t_afsr = cpu_error_regs.afsr; 1757 t_afsr_ext = cpu_error_regs.afsr_ext; 1758 #if defined(SERRANO) 1759 ch_flt.afar2 = cpu_error_regs.afar2; 1760 #endif /* SERRANO */ 1761 clo_flags = p_clo_flags; 1762 } else { 1763 clop = CPU_PRIVATE_PTR(CPU, chpr_async_logout); 1764 t_afar = clop->clo_data.chd_afar; 1765 t_afsr = clop->clo_data.chd_afsr; 1766 t_afsr_ext = clop->clo_data.chd_afsr_ext; 1767 #if defined(SERRANO) 1768 ch_flt.afar2 = clop->clo_data.chd_afar2; 1769 #endif /* SERRANO */ 1770 clo_flags = clop->clo_flags; 1771 } 1772 1773 /* 1774 * In order to simplify code, we maintain this afsr_errs 1775 * variable which holds the aggregate of AFSR and AFSR_EXT 1776 * sticky bits. 1777 */ 1778 t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1779 (t_afsr & C_AFSR_ALL_ERRS); 1780 pr_reason[0] = '\0'; 1781 1782 /* 1783 * Grab information encoded into our clo_flags field. 1784 */ 1785 ceen = clo_flags & EN_REG_CEEN; 1786 tl = (clo_flags & CLO_FLAGS_TL_MASK) >> CLO_FLAGS_TL_SHIFT; 1787 ttype = (clo_flags & CLO_FLAGS_TT_MASK) >> CLO_FLAGS_TT_SHIFT; 1788 1789 /* 1790 * handle the specific error 1791 */ 1792 aflt = (struct async_flt *)&ch_flt; 1793 aflt->flt_id = gethrtime_waitfree(); 1794 aflt->flt_bus_id = getprocessorid(); 1795 aflt->flt_inst = CPU->cpu_id; 1796 ch_flt.afsr_ext = t_afsr_ext; 1797 ch_flt.afsr_errs = t_afsr_errs; 1798 aflt->flt_stat = t_afsr; 1799 aflt->flt_addr = t_afar; 1800 aflt->flt_pc = (caddr_t)rp->r_pc; 1801 aflt->flt_prot = AFLT_PROT_NONE; 1802 aflt->flt_class = CPU_FAULT; 1803 aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 1804 aflt->flt_tl = (uchar_t)tl; 1805 aflt->flt_panic = ((tl != 0) || (aft_testfatal != 0) || 1806 C_AFSR_PANIC(t_afsr_errs)); 1807 aflt->flt_core = (pflag & SDOCORE) ? 1 : 0; 1808 aflt->flt_status = ((ttype == T_DATA_ERROR) ? ECC_D_TRAP : ECC_I_TRAP); 1809 1810 /* 1811 * If the trap occurred in privileged mode at TL=0, we need to check to 1812 * see if we were executing in the kernel under on_trap() or t_lofault 1813 * protection. If so, modify the saved registers so that we return 1814 * from the trap to the appropriate trampoline routine. 1815 */ 1816 if (aflt->flt_priv && tl == 0) { 1817 if (curthread->t_ontrap != NULL) { 1818 on_trap_data_t *otp = curthread->t_ontrap; 1819 1820 if (otp->ot_prot & OT_DATA_EC) { 1821 aflt->flt_prot = AFLT_PROT_EC; 1822 otp->ot_trap |= OT_DATA_EC; 1823 rp->r_pc = otp->ot_trampoline; 1824 rp->r_npc = rp->r_pc + 4; 1825 trampolined = 1; 1826 } 1827 1828 if ((t_afsr & (C_AFSR_TO | C_AFSR_BERR)) && 1829 (otp->ot_prot & OT_DATA_ACCESS)) { 1830 aflt->flt_prot = AFLT_PROT_ACCESS; 1831 otp->ot_trap |= OT_DATA_ACCESS; 1832 rp->r_pc = otp->ot_trampoline; 1833 rp->r_npc = rp->r_pc + 4; 1834 trampolined = 1; 1835 /* 1836 * for peeks and caut_gets errors are expected 1837 */ 1838 hp = (ddi_acc_hdl_t *)otp->ot_handle; 1839 if (!hp) 1840 expected = DDI_FM_ERR_PEEK; 1841 else if (hp->ah_acc.devacc_attr_access == 1842 DDI_CAUTIOUS_ACC) 1843 expected = DDI_FM_ERR_EXPECTED; 1844 } 1845 1846 } else if (curthread->t_lofault) { 1847 aflt->flt_prot = AFLT_PROT_COPY; 1848 rp->r_g1 = EFAULT; 1849 rp->r_pc = curthread->t_lofault; 1850 rp->r_npc = rp->r_pc + 4; 1851 trampolined = 1; 1852 } 1853 } 1854 1855 /* 1856 * If we're in user mode or we're doing a protected copy, we either 1857 * want the ASTON code below to send a signal to the user process 1858 * or we want to panic if aft_panic is set. 1859 * 1860 * If we're in privileged mode and we're not doing a copy, then we 1861 * need to check if we've trampolined. If we haven't trampolined, 1862 * we should panic. 1863 */ 1864 if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 1865 if (t_afsr_errs & 1866 ((C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS) & 1867 ~(C_AFSR_BERR | C_AFSR_TO))) 1868 aflt->flt_panic |= aft_panic; 1869 } else if (!trampolined) { 1870 aflt->flt_panic = 1; 1871 } 1872 1873 /* 1874 * If we've trampolined due to a privileged TO or BERR, or if an 1875 * unprivileged TO or BERR occurred, we don't want to enqueue an 1876 * event for that TO or BERR. Queue all other events (if any) besides 1877 * the TO/BERR. Since we may not be enqueing any events, we need to 1878 * ignore the number of events queued. If we haven't trampolined due 1879 * to a TO or BERR, just enqueue events normally. 1880 */ 1881 log_afsr = t_afsr_errs; 1882 if (trampolined) { 1883 log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 1884 } else if (!aflt->flt_priv) { 1885 /* 1886 * User mode, suppress messages if 1887 * cpu_berr_to_verbose is not set. 1888 */ 1889 if (!cpu_berr_to_verbose) 1890 log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 1891 } 1892 1893 /* 1894 * Log any errors that occurred 1895 */ 1896 if (((log_afsr & 1897 ((C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS) & ~C_AFSR_ME)) && 1898 cpu_queue_events(&ch_flt, pr_reason, log_afsr, clop) == 0) || 1899 (t_afsr_errs & 1900 (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) == 0) { 1901 ch_flt.flt_type = CPU_INV_AFSR; 1902 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 1903 (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 1904 aflt->flt_panic); 1905 } 1906 1907 /* 1908 * Zero out + invalidate CPU logout. 1909 */ 1910 if (clop) { 1911 bzero(clop, sizeof (ch_cpu_logout_t)); 1912 clop->clo_data.chd_afar = LOGOUT_INVALID; 1913 } 1914 1915 #if defined(JALAPENO) || defined(SERRANO) 1916 /* 1917 * UE/RUE/BERR/TO: Call our bus nexus friends to check for 1918 * IO errors that may have resulted in this trap. 1919 */ 1920 if (t_afsr & (C_AFSR_UE|C_AFSR_RUE|C_AFSR_TO|C_AFSR_BERR)) { 1921 cpu_run_bus_error_handlers(aflt, expected); 1922 } 1923 1924 /* 1925 * UE/RUE: If UE or RUE is in memory, we need to flush the bad 1926 * line from the Ecache. We also need to query the bus nexus for 1927 * fatal errors. Attempts to do diagnostic read on caches may 1928 * introduce more errors (especially when the module is bad). 1929 */ 1930 if (t_afsr & (C_AFSR_UE|C_AFSR_RUE)) { 1931 /* 1932 * Ask our bus nexus friends if they have any fatal errors. If 1933 * so, they will log appropriate error messages. 1934 */ 1935 if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 1936 aflt->flt_panic = 1; 1937 1938 /* 1939 * We got a UE or RUE and are panicking, save the fault PA in 1940 * a known location so that the platform specific panic code 1941 * can check for copyback errors. 1942 */ 1943 if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 1944 panic_aflt = *aflt; 1945 } 1946 } 1947 1948 /* 1949 * Flush Ecache line or entire Ecache 1950 */ 1951 if (t_afsr & (C_AFSR_UE | C_AFSR_RUE | C_AFSR_EDU | C_AFSR_BERR)) 1952 cpu_error_ecache_flush(&ch_flt); 1953 #else /* JALAPENO || SERRANO */ 1954 /* 1955 * UE/BERR/TO: Call our bus nexus friends to check for 1956 * IO errors that may have resulted in this trap. 1957 */ 1958 if (t_afsr & (C_AFSR_UE|C_AFSR_TO|C_AFSR_BERR)) { 1959 cpu_run_bus_error_handlers(aflt, expected); 1960 } 1961 1962 /* 1963 * UE: If the UE is in memory, we need to flush the bad 1964 * line from the Ecache. We also need to query the bus nexus for 1965 * fatal errors. Attempts to do diagnostic read on caches may 1966 * introduce more errors (especially when the module is bad). 1967 */ 1968 if (t_afsr & C_AFSR_UE) { 1969 /* 1970 * Ask our legacy bus nexus friends if they have any fatal 1971 * errors. If so, they will log appropriate error messages. 1972 */ 1973 if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 1974 aflt->flt_panic = 1; 1975 1976 /* 1977 * We got a UE and are panicking, save the fault PA in a known 1978 * location so that the platform specific panic code can check 1979 * for copyback errors. 1980 */ 1981 if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 1982 panic_aflt = *aflt; 1983 } 1984 } 1985 1986 /* 1987 * Flush Ecache line or entire Ecache 1988 */ 1989 if (t_afsr_errs & 1990 (C_AFSR_UE | C_AFSR_EDU | C_AFSR_BERR | C_AFSR_L3_EDU)) 1991 cpu_error_ecache_flush(&ch_flt); 1992 #endif /* JALAPENO || SERRANO */ 1993 1994 /* 1995 * We carefully re-enable NCEEN and CEEN and then check if any deferred 1996 * or disrupting errors have happened. We do this because if a 1997 * deferred or disrupting error had occurred with NCEEN/CEEN off, the 1998 * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 1999 * CEEN works differently on Cheetah than on Spitfire. Also, we enable 2000 * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 2001 * deferred or disrupting error happening between checking the AFSR and 2002 * enabling NCEEN/CEEN. 2003 * 2004 * Note: CEEN reenabled only if it was on when trap taken. 2005 */ 2006 set_error_enable(get_error_enable() | (EN_REG_NCEEN | ceen)); 2007 if (clear_errors(&ch_flt)) { 2008 /* 2009 * Check for secondary errors, and avoid panicking if we 2010 * have them 2011 */ 2012 if (cpu_check_secondary_errors(&ch_flt, t_afsr_errs, 2013 t_afar) == 0) { 2014 aflt->flt_panic |= ((ch_flt.afsr_errs & 2015 (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) != 0); 2016 } 2017 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 2018 NULL); 2019 } 2020 2021 /* 2022 * Panic here if aflt->flt_panic has been set. Enqueued errors will 2023 * be logged as part of the panic flow. 2024 */ 2025 if (aflt->flt_panic) 2026 fm_panic("%sError(s)", pr_reason); 2027 2028 /* 2029 * If we queued an error and we are going to return from the trap and 2030 * the error was in user mode or inside of a copy routine, set AST flag 2031 * so the queue will be drained before returning to user mode. The 2032 * AST processing will also act on our failure policy. 2033 */ 2034 if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 2035 int pcb_flag = 0; 2036 2037 if (t_afsr_errs & 2038 (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS & 2039 ~(C_AFSR_BERR | C_AFSR_TO))) 2040 pcb_flag |= ASYNC_HWERR; 2041 2042 if (t_afsr & C_AFSR_BERR) 2043 pcb_flag |= ASYNC_BERR; 2044 2045 if (t_afsr & C_AFSR_TO) 2046 pcb_flag |= ASYNC_BTO; 2047 2048 ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag; 2049 aston(curthread); 2050 } 2051 } 2052 2053 #if defined(CPU_IMP_L1_CACHE_PARITY) 2054 /* 2055 * Handling of data and instruction parity errors (traps 0x71, 0x72). 2056 * 2057 * For Panther, P$ data parity errors during floating point load hits 2058 * are also detected (reported as TT 0x71) and handled by this trap 2059 * handler. 2060 * 2061 * AFSR/AFAR are not set for parity errors, only TPC (a virtual address) 2062 * is available. 2063 */ 2064 /*ARGSUSED*/ 2065 void 2066 cpu_parity_error(struct regs *rp, uint_t flags, caddr_t tpc) 2067 { 2068 ch_async_flt_t ch_flt; 2069 struct async_flt *aflt; 2070 uchar_t tl = ((flags & CH_ERR_TL) != 0); 2071 uchar_t iparity = ((flags & CH_ERR_IPE) != 0); 2072 uchar_t panic = ((flags & CH_ERR_PANIC) != 0); 2073 char *error_class; 2074 2075 /* 2076 * Log the error. 2077 * For icache parity errors the fault address is the trap PC. 2078 * For dcache/pcache parity errors the instruction would have to 2079 * be decoded to determine the address and that isn't possible 2080 * at high PIL. 2081 */ 2082 bzero(&ch_flt, sizeof (ch_async_flt_t)); 2083 aflt = (struct async_flt *)&ch_flt; 2084 aflt->flt_id = gethrtime_waitfree(); 2085 aflt->flt_bus_id = getprocessorid(); 2086 aflt->flt_inst = CPU->cpu_id; 2087 aflt->flt_pc = tpc; 2088 aflt->flt_addr = iparity ? (uint64_t)tpc : AFLT_INV_ADDR; 2089 aflt->flt_prot = AFLT_PROT_NONE; 2090 aflt->flt_class = CPU_FAULT; 2091 aflt->flt_priv = (tl || (rp->r_tstate & TSTATE_PRIV)) ? 1 : 0; 2092 aflt->flt_tl = tl; 2093 aflt->flt_panic = panic; 2094 aflt->flt_status = iparity ? ECC_IP_TRAP : ECC_DP_TRAP; 2095 ch_flt.flt_type = iparity ? CPU_IC_PARITY : CPU_DC_PARITY; 2096 2097 if (iparity) { 2098 cpu_icache_parity_info(&ch_flt); 2099 if (ch_flt.parity_data.ipe.cpl_off != -1) 2100 error_class = FM_EREPORT_CPU_USIII_IDSPE; 2101 else if (ch_flt.parity_data.ipe.cpl_way != -1) 2102 error_class = FM_EREPORT_CPU_USIII_ITSPE; 2103 else 2104 error_class = FM_EREPORT_CPU_USIII_IPE; 2105 aflt->flt_payload = FM_EREPORT_PAYLOAD_ICACHE_PE; 2106 } else { 2107 cpu_dcache_parity_info(&ch_flt); 2108 if (ch_flt.parity_data.dpe.cpl_off != -1) 2109 error_class = FM_EREPORT_CPU_USIII_DDSPE; 2110 else if (ch_flt.parity_data.dpe.cpl_way != -1) 2111 error_class = FM_EREPORT_CPU_USIII_DTSPE; 2112 else 2113 error_class = FM_EREPORT_CPU_USIII_DPE; 2114 aflt->flt_payload = FM_EREPORT_PAYLOAD_DCACHE_PE; 2115 /* 2116 * For panther we also need to check the P$ for parity errors. 2117 */ 2118 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 2119 cpu_pcache_parity_info(&ch_flt); 2120 if (ch_flt.parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 2121 error_class = FM_EREPORT_CPU_USIII_PDSPE; 2122 aflt->flt_payload = 2123 FM_EREPORT_PAYLOAD_PCACHE_PE; 2124 } 2125 } 2126 } 2127 2128 cpu_errorq_dispatch(error_class, (void *)&ch_flt, 2129 sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 2130 2131 if (iparity) { 2132 /* 2133 * Invalidate entire I$. 2134 * This is required due to the use of diagnostic ASI 2135 * accesses that may result in a loss of I$ coherency. 2136 */ 2137 if (cache_boot_state & DCU_IC) { 2138 flush_icache(); 2139 } 2140 /* 2141 * According to section P.3.1 of the Panther PRM, we 2142 * need to do a little more for recovery on those 2143 * CPUs after encountering an I$ parity error. 2144 */ 2145 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 2146 flush_ipb(); 2147 correct_dcache_parity(dcache_size, 2148 dcache_linesize); 2149 flush_pcache(); 2150 } 2151 } else { 2152 /* 2153 * Since the valid bit is ignored when checking parity the 2154 * D$ data and tag must also be corrected. Set D$ data bits 2155 * to zero and set utag to 0, 1, 2, 3. 2156 */ 2157 correct_dcache_parity(dcache_size, dcache_linesize); 2158 2159 /* 2160 * According to section P.3.3 of the Panther PRM, we 2161 * need to do a little more for recovery on those 2162 * CPUs after encountering a D$ or P$ parity error. 2163 * 2164 * As far as clearing P$ parity errors, it is enough to 2165 * simply invalidate all entries in the P$ since P$ parity 2166 * error traps are only generated for floating point load 2167 * hits. 2168 */ 2169 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 2170 flush_icache(); 2171 flush_ipb(); 2172 flush_pcache(); 2173 } 2174 } 2175 2176 /* 2177 * Invalidate entire D$ if it was enabled. 2178 * This is done to avoid stale data in the D$ which might 2179 * occur with the D$ disabled and the trap handler doing 2180 * stores affecting lines already in the D$. 2181 */ 2182 if (cache_boot_state & DCU_DC) { 2183 flush_dcache(); 2184 } 2185 2186 /* 2187 * Restore caches to their bootup state. 2188 */ 2189 set_dcu(get_dcu() | cache_boot_state); 2190 2191 /* 2192 * Panic here if aflt->flt_panic has been set. Enqueued errors will 2193 * be logged as part of the panic flow. 2194 */ 2195 if (aflt->flt_panic) 2196 fm_panic("%sError(s)", iparity ? "IPE " : "DPE "); 2197 2198 /* 2199 * If this error occurred at TL>0 then flush the E$ here to reduce 2200 * the chance of getting an unrecoverable Fast ECC error. This 2201 * flush will evict the part of the parity trap handler that is run 2202 * at TL>1. 2203 */ 2204 if (tl) { 2205 cpu_flush_ecache(); 2206 } 2207 } 2208 2209 /* 2210 * On an I$ parity error, mark the appropriate entries in the ch_async_flt_t 2211 * to indicate which portions of the captured data should be in the ereport. 2212 */ 2213 void 2214 cpu_async_log_ic_parity_err(ch_async_flt_t *ch_flt) 2215 { 2216 int way = ch_flt->parity_data.ipe.cpl_way; 2217 int offset = ch_flt->parity_data.ipe.cpl_off; 2218 int tag_index; 2219 struct async_flt *aflt = (struct async_flt *)ch_flt; 2220 2221 2222 if ((offset != -1) || (way != -1)) { 2223 /* 2224 * Parity error in I$ tag or data 2225 */ 2226 tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 2227 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 2228 ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 2229 PN_ICIDX_TO_WAY(tag_index); 2230 else 2231 ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 2232 CH_ICIDX_TO_WAY(tag_index); 2233 ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 2234 IC_LOGFLAG_MAGIC; 2235 } else { 2236 /* 2237 * Parity error was not identified. 2238 * Log tags and data for all ways. 2239 */ 2240 for (way = 0; way < CH_ICACHE_NWAY; way++) { 2241 tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 2242 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 2243 ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 2244 PN_ICIDX_TO_WAY(tag_index); 2245 else 2246 ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 2247 CH_ICIDX_TO_WAY(tag_index); 2248 ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 2249 IC_LOGFLAG_MAGIC; 2250 } 2251 } 2252 } 2253 2254 /* 2255 * On an D$ parity error, mark the appropriate entries in the ch_async_flt_t 2256 * to indicate which portions of the captured data should be in the ereport. 2257 */ 2258 void 2259 cpu_async_log_dc_parity_err(ch_async_flt_t *ch_flt) 2260 { 2261 int way = ch_flt->parity_data.dpe.cpl_way; 2262 int offset = ch_flt->parity_data.dpe.cpl_off; 2263 int tag_index; 2264 2265 if (offset != -1) { 2266 /* 2267 * Parity error in D$ or P$ data array. 2268 * 2269 * First check to see whether the parity error is in D$ or P$ 2270 * since P$ data parity errors are reported in Panther using 2271 * the same trap. 2272 */ 2273 if (ch_flt->parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 2274 tag_index = ch_flt->parity_data.dpe.cpl_pc[way].pc_idx; 2275 ch_flt->parity_data.dpe.cpl_pc[way].pc_way = 2276 CH_PCIDX_TO_WAY(tag_index); 2277 ch_flt->parity_data.dpe.cpl_pc[way].pc_logflag = 2278 PC_LOGFLAG_MAGIC; 2279 } else { 2280 tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 2281 ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 2282 CH_DCIDX_TO_WAY(tag_index); 2283 ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 2284 DC_LOGFLAG_MAGIC; 2285 } 2286 } else if (way != -1) { 2287 /* 2288 * Parity error in D$ tag. 2289 */ 2290 tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 2291 ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 2292 CH_DCIDX_TO_WAY(tag_index); 2293 ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 2294 DC_LOGFLAG_MAGIC; 2295 } 2296 } 2297 #endif /* CPU_IMP_L1_CACHE_PARITY */ 2298 2299 /* 2300 * The cpu_async_log_err() function is called via the [uc]e_drain() function to 2301 * post-process CPU events that are dequeued. As such, it can be invoked 2302 * from softint context, from AST processing in the trap() flow, or from the 2303 * panic flow. We decode the CPU-specific data, and take appropriate actions. 2304 * Historically this entry point was used to log the actual cmn_err(9F) text; 2305 * now with FMA it is used to prepare 'flt' to be converted into an ereport. 2306 * With FMA this function now also returns a flag which indicates to the 2307 * caller whether the ereport should be posted (1) or suppressed (0). 2308 */ 2309 static int 2310 cpu_async_log_err(void *flt, errorq_elem_t *eqep) 2311 { 2312 ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt; 2313 struct async_flt *aflt = (struct async_flt *)flt; 2314 uint64_t errors; 2315 extern void memscrub_induced_error(void); 2316 2317 switch (ch_flt->flt_type) { 2318 case CPU_INV_AFSR: 2319 /* 2320 * If it is a disrupting trap and the AFSR is zero, then 2321 * the event has probably already been noted. Do not post 2322 * an ereport. 2323 */ 2324 if ((aflt->flt_status & ECC_C_TRAP) && 2325 (!(aflt->flt_stat & C_AFSR_MASK))) 2326 return (0); 2327 else 2328 return (1); 2329 case CPU_TO: 2330 case CPU_BERR: 2331 case CPU_FATAL: 2332 case CPU_FPUERR: 2333 return (1); 2334 2335 case CPU_UE_ECACHE_RETIRE: 2336 cpu_log_err(aflt); 2337 cpu_page_retire(ch_flt); 2338 return (1); 2339 2340 /* 2341 * Cases where we may want to suppress logging or perform 2342 * extended diagnostics. 2343 */ 2344 case CPU_CE: 2345 case CPU_EMC: 2346 /* 2347 * We want to skip logging and further classification 2348 * only if ALL the following conditions are true: 2349 * 2350 * 1. There is only one error 2351 * 2. That error is a correctable memory error 2352 * 3. The error is caused by the memory scrubber (in 2353 * which case the error will have occurred under 2354 * on_trap protection) 2355 * 4. The error is on a retired page 2356 * 2357 * Note: AFLT_PROT_EC is used places other than the memory 2358 * scrubber. However, none of those errors should occur 2359 * on a retired page. 2360 */ 2361 if ((ch_flt->afsr_errs & 2362 (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_CE && 2363 aflt->flt_prot == AFLT_PROT_EC) { 2364 2365 if (page_retire_check(aflt->flt_addr, NULL) == 0) { 2366 if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 2367 2368 /* 2369 * Since we're skipping logging, we'll need 2370 * to schedule the re-enabling of CEEN 2371 */ 2372 (void) timeout(cpu_delayed_check_ce_errors, 2373 (void *)(uintptr_t)aflt->flt_inst, 2374 drv_usectohz((clock_t)cpu_ceen_delay_secs 2375 * MICROSEC)); 2376 } 2377 /* 2378 * Inform memscrubber - scrubbing induced 2379 * CE on a retired page. 2380 */ 2381 memscrub_induced_error(); 2382 return (0); 2383 } 2384 } 2385 2386 /* 2387 * Perform/schedule further classification actions, but 2388 * only if the page is healthy (we don't want bad 2389 * pages inducing too much diagnostic activity). If we could 2390 * not find a page pointer then we also skip this. If 2391 * ce_scrub_xdiag_recirc returns nonzero then it has chosen 2392 * to copy and recirculate the event (for further diagnostics) 2393 * and we should not proceed to log it here. 2394 * 2395 * This must be the last step here before the cpu_log_err() 2396 * below - if an event recirculates cpu_ce_log_err() will 2397 * not call the current function but just proceed directly 2398 * to cpu_ereport_post after the cpu_log_err() avoided below. 2399 * 2400 * Note: Check cpu_impl_async_log_err if changing this 2401 */ 2402 if (page_retire_check(aflt->flt_addr, &errors) == EINVAL) { 2403 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 2404 CE_XDIAG_SKIP_NOPP); 2405 } else { 2406 if (errors != PR_OK) { 2407 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 2408 CE_XDIAG_SKIP_PAGEDET); 2409 } else if (ce_scrub_xdiag_recirc(aflt, ce_queue, eqep, 2410 offsetof(ch_async_flt_t, cmn_asyncflt))) { 2411 return (0); 2412 } 2413 } 2414 /*FALLTHRU*/ 2415 2416 /* 2417 * Cases where we just want to report the error and continue. 2418 */ 2419 case CPU_CE_ECACHE: 2420 case CPU_UE_ECACHE: 2421 case CPU_IV: 2422 case CPU_ORPH: 2423 cpu_log_err(aflt); 2424 return (1); 2425 2426 /* 2427 * Cases where we want to fall through to handle panicking. 2428 */ 2429 case CPU_UE: 2430 /* 2431 * We want to skip logging in the same conditions as the 2432 * CE case. In addition, we want to make sure we're not 2433 * panicking. 2434 */ 2435 if (!panicstr && (ch_flt->afsr_errs & 2436 (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_UE && 2437 aflt->flt_prot == AFLT_PROT_EC) { 2438 if (page_retire_check(aflt->flt_addr, NULL) == 0) { 2439 /* Zero the address to clear the error */ 2440 softcall(ecc_page_zero, (void *)aflt->flt_addr); 2441 /* 2442 * Inform memscrubber - scrubbing induced 2443 * UE on a retired page. 2444 */ 2445 memscrub_induced_error(); 2446 return (0); 2447 } 2448 } 2449 cpu_log_err(aflt); 2450 break; 2451 2452 default: 2453 /* 2454 * If the us3_common.c code doesn't know the flt_type, it may 2455 * be an implementation-specific code. Call into the impldep 2456 * backend to find out what to do: if it tells us to continue, 2457 * break and handle as if falling through from a UE; if not, 2458 * the impldep backend has handled the error and we're done. 2459 */ 2460 switch (cpu_impl_async_log_err(flt, eqep)) { 2461 case CH_ASYNC_LOG_DONE: 2462 return (1); 2463 case CH_ASYNC_LOG_RECIRC: 2464 return (0); 2465 case CH_ASYNC_LOG_CONTINUE: 2466 break; /* continue on to handle UE-like error */ 2467 default: 2468 cmn_err(CE_WARN, "discarding error 0x%p with " 2469 "invalid fault type (0x%x)", 2470 (void *)aflt, ch_flt->flt_type); 2471 return (0); 2472 } 2473 } 2474 2475 /* ... fall through from the UE case */ 2476 2477 if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) { 2478 if (!panicstr) { 2479 cpu_page_retire(ch_flt); 2480 } else { 2481 /* 2482 * Clear UEs on panic so that we don't 2483 * get haunted by them during panic or 2484 * after reboot 2485 */ 2486 cpu_clearphys(aflt); 2487 (void) clear_errors(NULL); 2488 } 2489 } 2490 2491 return (1); 2492 } 2493 2494 /* 2495 * Retire the bad page that may contain the flushed error. 2496 */ 2497 void 2498 cpu_page_retire(ch_async_flt_t *ch_flt) 2499 { 2500 struct async_flt *aflt = (struct async_flt *)ch_flt; 2501 (void) page_retire(aflt->flt_addr, PR_UE); 2502 } 2503 2504 /* 2505 * Return true if the error specified in the AFSR indicates 2506 * an E$ data error (L2$ for Cheetah/Cheetah+/Jaguar, L3$ 2507 * for Panther, none for Jalapeno/Serrano). 2508 */ 2509 /* ARGSUSED */ 2510 static int 2511 cpu_error_is_ecache_data(int cpuid, uint64_t t_afsr) 2512 { 2513 #if defined(JALAPENO) || defined(SERRANO) 2514 return (0); 2515 #elif defined(CHEETAH_PLUS) 2516 if (IS_PANTHER(cpunodes[cpuid].implementation)) 2517 return ((t_afsr & C_AFSR_EXT_L3_DATA_ERRS) != 0); 2518 return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0); 2519 #else /* CHEETAH_PLUS */ 2520 return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0); 2521 #endif 2522 } 2523 2524 /* 2525 * The cpu_log_err() function is called by cpu_async_log_err() to perform the 2526 * generic event post-processing for correctable and uncorrectable memory, 2527 * E$, and MTag errors. Historically this entry point was used to log bits of 2528 * common cmn_err(9F) text; now with FMA it is used to prepare 'flt' to be 2529 * converted into an ereport. In addition, it transmits the error to any 2530 * platform-specific service-processor FRU logging routines, if available. 2531 */ 2532 void 2533 cpu_log_err(struct async_flt *aflt) 2534 { 2535 char unum[UNUM_NAMLEN]; 2536 int synd_status, synd_code, afar_status; 2537 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 2538 2539 if (cpu_error_is_ecache_data(aflt->flt_inst, ch_flt->flt_bit)) 2540 aflt->flt_status |= ECC_ECACHE; 2541 else 2542 aflt->flt_status &= ~ECC_ECACHE; 2543 /* 2544 * Determine syndrome status. 2545 */ 2546 synd_status = afsr_to_synd_status(aflt->flt_inst, 2547 ch_flt->afsr_errs, ch_flt->flt_bit); 2548 2549 /* 2550 * Determine afar status. 2551 */ 2552 if (pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)) 2553 afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 2554 ch_flt->flt_bit); 2555 else 2556 afar_status = AFLT_STAT_INVALID; 2557 2558 synd_code = synd_to_synd_code(synd_status, 2559 aflt->flt_synd, ch_flt->flt_bit); 2560 2561 /* 2562 * If afar status is not invalid do a unum lookup. 2563 */ 2564 if (afar_status != AFLT_STAT_INVALID) { 2565 (void) cpu_get_mem_unum_synd(synd_code, aflt, unum); 2566 } else { 2567 unum[0] = '\0'; 2568 } 2569 2570 /* 2571 * Do not send the fruid message (plat_ecc_error_data_t) 2572 * to the SC if it can handle the enhanced error information 2573 * (plat_ecc_error2_data_t) or when the tunable 2574 * ecc_log_fruid_enable is set to 0. 2575 */ 2576 2577 if (&plat_ecc_capability_sc_get && 2578 plat_ecc_capability_sc_get(PLAT_ECC_ERROR_MESSAGE)) { 2579 if (&plat_log_fruid_error) 2580 plat_log_fruid_error(synd_code, aflt, unum, 2581 ch_flt->flt_bit); 2582 } 2583 2584 if (aflt->flt_func != NULL) 2585 aflt->flt_func(aflt, unum); 2586 2587 if (afar_status != AFLT_STAT_INVALID) 2588 cpu_log_diag_info(ch_flt); 2589 2590 /* 2591 * If we have a CEEN error , we do not reenable CEEN until after 2592 * we exit the trap handler. Otherwise, another error may 2593 * occur causing the handler to be entered recursively. 2594 * We set a timeout to trigger in cpu_ceen_delay_secs seconds, 2595 * to try and ensure that the CPU makes progress in the face 2596 * of a CE storm. 2597 */ 2598 if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 2599 (void) timeout(cpu_delayed_check_ce_errors, 2600 (void *)(uintptr_t)aflt->flt_inst, 2601 drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 2602 } 2603 } 2604 2605 /* 2606 * Invoked by error_init() early in startup and therefore before 2607 * startup_errorq() is called to drain any error Q - 2608 * 2609 * startup() 2610 * startup_end() 2611 * error_init() 2612 * cpu_error_init() 2613 * errorq_init() 2614 * errorq_drain() 2615 * start_other_cpus() 2616 * 2617 * The purpose of this routine is to create error-related taskqs. Taskqs 2618 * are used for this purpose because cpu_lock can't be grabbed from interrupt 2619 * context. 2620 */ 2621 void 2622 cpu_error_init(int items) 2623 { 2624 /* 2625 * Create taskq(s) to reenable CE 2626 */ 2627 ch_check_ce_tq = taskq_create("cheetah_check_ce", 1, minclsyspri, 2628 items, items, TASKQ_PREPOPULATE); 2629 } 2630 2631 void 2632 cpu_ce_log_err(struct async_flt *aflt, errorq_elem_t *eqep) 2633 { 2634 char unum[UNUM_NAMLEN]; 2635 int len; 2636 2637 switch (aflt->flt_class) { 2638 case CPU_FAULT: 2639 cpu_ereport_init(aflt); 2640 if (cpu_async_log_err(aflt, eqep)) 2641 cpu_ereport_post(aflt); 2642 break; 2643 2644 case BUS_FAULT: 2645 if (aflt->flt_func != NULL) { 2646 (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, 2647 unum, UNUM_NAMLEN, &len); 2648 aflt->flt_func(aflt, unum); 2649 } 2650 break; 2651 2652 case RECIRC_CPU_FAULT: 2653 aflt->flt_class = CPU_FAULT; 2654 cpu_log_err(aflt); 2655 cpu_ereport_post(aflt); 2656 break; 2657 2658 case RECIRC_BUS_FAULT: 2659 ASSERT(aflt->flt_class != RECIRC_BUS_FAULT); 2660 /*FALLTHRU*/ 2661 default: 2662 cmn_err(CE_WARN, "discarding CE error 0x%p with invalid " 2663 "fault class (0x%x)", (void *)aflt, aflt->flt_class); 2664 return; 2665 } 2666 } 2667 2668 /* 2669 * Scrub and classify a CE. This function must not modify the 2670 * fault structure passed to it but instead should return the classification 2671 * information. 2672 */ 2673 2674 static uchar_t 2675 cpu_ce_scrub_mem_err_common(struct async_flt *ecc, boolean_t logout_tried) 2676 { 2677 uchar_t disp = CE_XDIAG_EXTALG; 2678 on_trap_data_t otd; 2679 uint64_t orig_err; 2680 ch_cpu_logout_t *clop; 2681 2682 /* 2683 * Clear CEEN. CPU CE TL > 0 trap handling will already have done 2684 * this, but our other callers have not. Disable preemption to 2685 * avoid CPU migration so that we restore CEEN on the correct 2686 * cpu later. 2687 * 2688 * CEEN is cleared so that further CEs that our instruction and 2689 * data footprint induce do not cause use to either creep down 2690 * kernel stack to the point of overflow, or do so much CE 2691 * notification as to make little real forward progress. 2692 * 2693 * NCEEN must not be cleared. However it is possible that 2694 * our accesses to the flt_addr may provoke a bus error or timeout 2695 * if the offending address has just been unconfigured as part of 2696 * a DR action. So we must operate under on_trap protection. 2697 */ 2698 kpreempt_disable(); 2699 orig_err = get_error_enable(); 2700 if (orig_err & EN_REG_CEEN) 2701 set_error_enable(orig_err & ~EN_REG_CEEN); 2702 2703 /* 2704 * Our classification algorithm includes the line state before 2705 * the scrub; we'd like this captured after the detection and 2706 * before the algorithm below - the earlier the better. 2707 * 2708 * If we've come from a cpu CE trap then this info already exists 2709 * in the cpu logout area. 2710 * 2711 * For a CE detected by memscrub for which there was no trap 2712 * (running with CEEN off) cpu_log_and_clear_ce has called 2713 * cpu_ce_delayed_ec_logout to capture some cache data, and 2714 * marked the fault structure as incomplete as a flag to later 2715 * logging code. 2716 * 2717 * If called directly from an IO detected CE there has been 2718 * no line data capture. In this case we logout to the cpu logout 2719 * area - that's appropriate since it's the cpu cache data we need 2720 * for classification. We thus borrow the cpu logout area for a 2721 * short time, and cpu_ce_delayed_ec_logout will mark it as busy in 2722 * this time (we will invalidate it again below). 2723 * 2724 * If called from the partner check xcall handler then this cpu 2725 * (the partner) has not necessarily experienced a CE at this 2726 * address. But we want to capture line state before its scrub 2727 * attempt since we use that in our classification. 2728 */ 2729 if (logout_tried == B_FALSE) { 2730 if (!cpu_ce_delayed_ec_logout(ecc->flt_addr)) 2731 disp |= CE_XDIAG_NOLOGOUT; 2732 } 2733 2734 /* 2735 * Scrub memory, then check AFSR for errors. The AFAR we scrub may 2736 * no longer be valid (if DR'd since the initial event) so we 2737 * perform this scrub under on_trap protection. If this access is 2738 * ok then further accesses below will also be ok - DR cannot 2739 * proceed while this thread is active (preemption is disabled); 2740 * to be safe we'll nonetheless use on_trap again below. 2741 */ 2742 if (!on_trap(&otd, OT_DATA_ACCESS)) { 2743 cpu_scrubphys(ecc); 2744 } else { 2745 no_trap(); 2746 if (orig_err & EN_REG_CEEN) 2747 set_error_enable(orig_err); 2748 kpreempt_enable(); 2749 return (disp); 2750 } 2751 no_trap(); 2752 2753 /* 2754 * Did the casx read of the scrub log a CE that matches the AFAR? 2755 * Note that it's quite possible that the read sourced the data from 2756 * another cpu. 2757 */ 2758 if (clear_ecc(ecc)) 2759 disp |= CE_XDIAG_CE1; 2760 2761 /* 2762 * Read the data again. This time the read is very likely to 2763 * come from memory since the scrub induced a writeback to memory. 2764 */ 2765 if (!on_trap(&otd, OT_DATA_ACCESS)) { 2766 (void) lddphys(P2ALIGN(ecc->flt_addr, 8)); 2767 } else { 2768 no_trap(); 2769 if (orig_err & EN_REG_CEEN) 2770 set_error_enable(orig_err); 2771 kpreempt_enable(); 2772 return (disp); 2773 } 2774 no_trap(); 2775 2776 /* Did that read induce a CE that matches the AFAR? */ 2777 if (clear_ecc(ecc)) 2778 disp |= CE_XDIAG_CE2; 2779 2780 /* 2781 * Look at the logout information and record whether we found the 2782 * line in l2/l3 cache. For Panther we are interested in whether 2783 * we found it in either cache (it won't reside in both but 2784 * it is possible to read it that way given the moving target). 2785 */ 2786 clop = CPU_PRIVATE(CPU) ? CPU_PRIVATE_PTR(CPU, chpr_cecc_logout) : NULL; 2787 if (!(disp & CE_XDIAG_NOLOGOUT) && clop && 2788 clop->clo_data.chd_afar != LOGOUT_INVALID) { 2789 int hit, level; 2790 int state; 2791 int totalsize; 2792 ch_ec_data_t *ecp; 2793 2794 /* 2795 * If hit is nonzero then a match was found and hit will 2796 * be one greater than the index which hit. For Panther we 2797 * also need to pay attention to level to see which of l2$ or 2798 * l3$ it hit in. 2799 */ 2800 hit = cpu_matching_ecache_line(ecc->flt_addr, &clop->clo_data, 2801 0, &level); 2802 2803 if (hit) { 2804 --hit; 2805 disp |= CE_XDIAG_AFARMATCH; 2806 2807 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 2808 if (level == 2) 2809 ecp = &clop->clo_data.chd_l2_data[hit]; 2810 else 2811 ecp = &clop->clo_data.chd_ec_data[hit]; 2812 } else { 2813 ASSERT(level == 2); 2814 ecp = &clop->clo_data.chd_ec_data[hit]; 2815 } 2816 totalsize = cpunodes[CPU->cpu_id].ecache_size; 2817 state = cpu_ectag_pa_to_subblk_state(totalsize, 2818 ecc->flt_addr, ecp->ec_tag); 2819 2820 /* 2821 * Cheetah variants use different state encodings - 2822 * the CH_ECSTATE_* defines vary depending on the 2823 * module we're compiled for. Translate into our 2824 * one true version. Conflate Owner-Shared state 2825 * of SSM mode with Owner as victimisation of such 2826 * lines may cause a writeback. 2827 */ 2828 switch (state) { 2829 case CH_ECSTATE_MOD: 2830 disp |= EC_STATE_M; 2831 break; 2832 2833 case CH_ECSTATE_OWN: 2834 case CH_ECSTATE_OWS: 2835 disp |= EC_STATE_O; 2836 break; 2837 2838 case CH_ECSTATE_EXL: 2839 disp |= EC_STATE_E; 2840 break; 2841 2842 case CH_ECSTATE_SHR: 2843 disp |= EC_STATE_S; 2844 break; 2845 2846 default: 2847 disp |= EC_STATE_I; 2848 break; 2849 } 2850 } 2851 2852 /* 2853 * If we initiated the delayed logout then we are responsible 2854 * for invalidating the logout area. 2855 */ 2856 if (logout_tried == B_FALSE) { 2857 bzero(clop, sizeof (ch_cpu_logout_t)); 2858 clop->clo_data.chd_afar = LOGOUT_INVALID; 2859 } 2860 } 2861 2862 /* 2863 * Re-enable CEEN if we turned it off. 2864 */ 2865 if (orig_err & EN_REG_CEEN) 2866 set_error_enable(orig_err); 2867 kpreempt_enable(); 2868 2869 return (disp); 2870 } 2871 2872 /* 2873 * Scrub a correctable memory error and collect data for classification 2874 * of CE type. This function is called in the detection path, ie tl0 handling 2875 * of a correctable error trap (cpus) or interrupt (IO) at high PIL. 2876 */ 2877 void 2878 cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t logout_tried) 2879 { 2880 /* 2881 * Cheetah CE classification does not set any bits in flt_status. 2882 * Instead we will record classification datapoints in flt_disp. 2883 */ 2884 ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY); 2885 2886 /* 2887 * To check if the error detected by IO is persistent, sticky or 2888 * intermittent. This is noticed by clear_ecc(). 2889 */ 2890 if (ecc->flt_status & ECC_IOBUS) 2891 ecc->flt_stat = C_AFSR_MEMORY; 2892 2893 /* 2894 * Record information from this first part of the algorithm in 2895 * flt_disp. 2896 */ 2897 ecc->flt_disp = cpu_ce_scrub_mem_err_common(ecc, logout_tried); 2898 } 2899 2900 /* 2901 * Select a partner to perform a further CE classification check from. 2902 * Must be called with kernel preemption disabled (to stop the cpu list 2903 * from changing). The detecting cpu we are partnering has cpuid 2904 * aflt->flt_inst; we might not be running on the detecting cpu. 2905 * 2906 * Restrict choice to active cpus in the same cpu partition as ourselves in 2907 * an effort to stop bad cpus in one partition causing other partitions to 2908 * perform excessive diagnostic activity. Actually since the errorq drain 2909 * is run from a softint most of the time and that is a global mechanism 2910 * this isolation is only partial. Return NULL if we fail to find a 2911 * suitable partner. 2912 * 2913 * We prefer a partner that is in a different latency group to ourselves as 2914 * we will share fewer datapaths. If such a partner is unavailable then 2915 * choose one in the same lgroup but prefer a different chip and only allow 2916 * a sibling core if flags includes PTNR_SIBLINGOK. If all else fails and 2917 * flags includes PTNR_SELFOK then permit selection of the original detector. 2918 * 2919 * We keep a cache of the last partner selected for a cpu, and we'll try to 2920 * use that previous partner if no more than cpu_ce_ptnr_cachetime_sec seconds 2921 * have passed since that selection was made. This provides the benefit 2922 * of the point-of-view of different partners over time but without 2923 * requiring frequent cpu list traversals. 2924 */ 2925 2926 #define PTNR_SIBLINGOK 0x1 /* Allow selection of sibling core */ 2927 #define PTNR_SELFOK 0x2 /* Allow selection of cpu to "partner" itself */ 2928 2929 static cpu_t * 2930 ce_ptnr_select(struct async_flt *aflt, int flags, int *typep) 2931 { 2932 cpu_t *sp, *dtcr, *ptnr, *locptnr, *sibptnr; 2933 hrtime_t lasttime, thistime; 2934 2935 ASSERT(curthread->t_preempt > 0 || getpil() >= DISP_LEVEL); 2936 2937 dtcr = cpu[aflt->flt_inst]; 2938 2939 /* 2940 * Short-circuit for the following cases: 2941 * . the dtcr is not flagged active 2942 * . there is just one cpu present 2943 * . the detector has disappeared 2944 * . we were given a bad flt_inst cpuid; this should not happen 2945 * (eg PCI code now fills flt_inst) but if it does it is no 2946 * reason to panic. 2947 * . there is just one cpu left online in the cpu partition 2948 * 2949 * If we return NULL after this point then we do not update the 2950 * chpr_ceptnr_seltime which will cause us to perform a full lookup 2951 * again next time; this is the case where the only other cpu online 2952 * in the detector's partition is on the same chip as the detector 2953 * and since CEEN re-enable is throttled even that case should not 2954 * hurt performance. 2955 */ 2956 if (dtcr == NULL || !cpu_flagged_active(dtcr->cpu_flags)) { 2957 return (NULL); 2958 } 2959 if (ncpus == 1 || dtcr->cpu_part->cp_ncpus == 1) { 2960 if (flags & PTNR_SELFOK) { 2961 *typep = CE_XDIAG_PTNR_SELF; 2962 return (dtcr); 2963 } else { 2964 return (NULL); 2965 } 2966 } 2967 2968 thistime = gethrtime(); 2969 lasttime = CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime); 2970 2971 /* 2972 * Select a starting point. 2973 */ 2974 if (!lasttime) { 2975 /* 2976 * We've never selected a partner for this detector before. 2977 * Start the scan at the next online cpu in the same cpu 2978 * partition. 2979 */ 2980 sp = dtcr->cpu_next_part; 2981 } else if (thistime - lasttime < cpu_ce_ptnr_cachetime_sec * NANOSEC) { 2982 /* 2983 * Our last selection has not aged yet. If this partner: 2984 * . is still a valid cpu, 2985 * . is still in the same partition as the detector 2986 * . is still marked active 2987 * . satisfies the 'flags' argument criteria 2988 * then select it again without updating the timestamp. 2989 */ 2990 sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 2991 if (sp == NULL || sp->cpu_part != dtcr->cpu_part || 2992 !cpu_flagged_active(sp->cpu_flags) || 2993 (sp == dtcr && !(flags & PTNR_SELFOK)) || 2994 (sp->cpu_chip->chip_id == dtcr->cpu_chip->chip_id && 2995 !(flags & PTNR_SIBLINGOK))) { 2996 sp = dtcr->cpu_next_part; 2997 } else { 2998 if (sp->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 2999 *typep = CE_XDIAG_PTNR_REMOTE; 3000 } else if (sp == dtcr) { 3001 *typep = CE_XDIAG_PTNR_SELF; 3002 } else if (sp->cpu_chip->chip_id == 3003 dtcr->cpu_chip->chip_id) { 3004 *typep = CE_XDIAG_PTNR_SIBLING; 3005 } else { 3006 *typep = CE_XDIAG_PTNR_LOCAL; 3007 } 3008 return (sp); 3009 } 3010 } else { 3011 /* 3012 * Our last selection has aged. If it is nonetheless still a 3013 * valid cpu then start the scan at the next cpu in the 3014 * partition after our last partner. If the last selection 3015 * is no longer a valid cpu then go with our default. In 3016 * this way we slowly cycle through possible partners to 3017 * obtain multiple viewpoints over time. 3018 */ 3019 sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 3020 if (sp == NULL) { 3021 sp = dtcr->cpu_next_part; 3022 } else { 3023 sp = sp->cpu_next_part; /* may be dtcr */ 3024 if (sp->cpu_part != dtcr->cpu_part) 3025 sp = dtcr; 3026 } 3027 } 3028 3029 /* 3030 * We have a proposed starting point for our search, but if this 3031 * cpu is offline then its cpu_next_part will point to itself 3032 * so we can't use that to iterate over cpus in this partition in 3033 * the loop below. We still want to avoid iterating over cpus not 3034 * in our partition, so in the case that our starting point is offline 3035 * we will repoint it to be the detector itself; and if the detector 3036 * happens to be offline we'll return NULL from the following loop. 3037 */ 3038 if (!cpu_flagged_active(sp->cpu_flags)) { 3039 sp = dtcr; 3040 } 3041 3042 ptnr = sp; 3043 locptnr = NULL; 3044 sibptnr = NULL; 3045 do { 3046 if (ptnr == dtcr || !cpu_flagged_active(ptnr->cpu_flags)) 3047 continue; 3048 if (ptnr->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 3049 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = ptnr->cpu_id; 3050 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 3051 *typep = CE_XDIAG_PTNR_REMOTE; 3052 return (ptnr); 3053 } 3054 if (ptnr->cpu_chip->chip_id == dtcr->cpu_chip->chip_id) { 3055 if (sibptnr == NULL) 3056 sibptnr = ptnr; 3057 continue; 3058 } 3059 if (locptnr == NULL) 3060 locptnr = ptnr; 3061 } while ((ptnr = ptnr->cpu_next_part) != sp); 3062 3063 /* 3064 * A foreign partner has already been returned if one was available. 3065 * 3066 * If locptnr is not NULL it is a cpu in the same lgroup as the 3067 * detector, is active, and is not a sibling of the detector. 3068 * 3069 * If sibptnr is not NULL it is a sibling of the detector, and is 3070 * active. 3071 * 3072 * If we have to resort to using the detector itself we have already 3073 * checked that it is active. 3074 */ 3075 if (locptnr) { 3076 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = locptnr->cpu_id; 3077 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 3078 *typep = CE_XDIAG_PTNR_LOCAL; 3079 return (locptnr); 3080 } else if (sibptnr && flags & PTNR_SIBLINGOK) { 3081 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = sibptnr->cpu_id; 3082 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 3083 *typep = CE_XDIAG_PTNR_SIBLING; 3084 return (sibptnr); 3085 } else if (flags & PTNR_SELFOK) { 3086 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = dtcr->cpu_id; 3087 CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 3088 *typep = CE_XDIAG_PTNR_SELF; 3089 return (dtcr); 3090 } 3091 3092 return (NULL); 3093 } 3094 3095 /* 3096 * Cross call handler that is requested to run on the designated partner of 3097 * a cpu that experienced a possibly sticky or possibly persistnet CE. 3098 */ 3099 static void 3100 ce_ptnrchk_xc(struct async_flt *aflt, uchar_t *dispp) 3101 { 3102 *dispp = cpu_ce_scrub_mem_err_common(aflt, B_FALSE); 3103 } 3104 3105 /* 3106 * The associated errorqs are never destroyed so we do not need to deal with 3107 * them disappearing before this timeout fires. If the affected memory 3108 * has been DR'd out since the original event the scrub algrithm will catch 3109 * any errors and return null disposition info. If the original detecting 3110 * cpu has been DR'd out then ereport detector info will not be able to 3111 * lookup CPU type; with a small timeout this is unlikely. 3112 */ 3113 static void 3114 ce_lkychk_cb(ce_lkychk_cb_t *cbarg) 3115 { 3116 struct async_flt *aflt = cbarg->lkycb_aflt; 3117 uchar_t disp; 3118 cpu_t *cp; 3119 int ptnrtype; 3120 3121 kpreempt_disable(); 3122 if (cp = ce_ptnr_select(aflt, PTNR_SIBLINGOK | PTNR_SELFOK, 3123 &ptnrtype)) { 3124 xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, (uint64_t)aflt, 3125 (uint64_t)&disp); 3126 CE_XDIAG_SETLKYINFO(aflt->flt_disp, disp); 3127 CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 3128 CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 3129 } else { 3130 ce_xdiag_lkydrops++; 3131 if (ncpus > 1) 3132 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 3133 CE_XDIAG_SKIP_NOPTNR); 3134 } 3135 kpreempt_enable(); 3136 3137 errorq_commit(cbarg->lkycb_eqp, cbarg->lkycb_eqep, ERRORQ_ASYNC); 3138 kmem_free(cbarg, sizeof (ce_lkychk_cb_t)); 3139 } 3140 3141 /* 3142 * Called from errorq drain code when processing a CE error, both from 3143 * CPU and PCI drain functions. Decide what further classification actions, 3144 * if any, we will perform. Perform immediate actions now, and schedule 3145 * delayed actions as required. Note that we are no longer necessarily running 3146 * on the detecting cpu, and that the async_flt structure will not persist on 3147 * return from this function. 3148 * 3149 * Calls to this function should aim to be self-throtlling in some way. With 3150 * the delayed re-enable of CEEN the absolute rate of calls should not 3151 * be excessive. Callers should also avoid performing in-depth classification 3152 * for events in pages that are already known to be suspect. 3153 * 3154 * We return nonzero to indicate that the event has been copied and 3155 * recirculated for further testing. The caller should not log the event 3156 * in this case - it will be logged when further test results are available. 3157 * 3158 * Our possible contexts are that of errorq_drain: below lock level or from 3159 * panic context. We can assume that the cpu we are running on is online. 3160 */ 3161 3162 3163 #ifdef DEBUG 3164 static int ce_xdiag_forceaction; 3165 #endif 3166 3167 int 3168 ce_scrub_xdiag_recirc(struct async_flt *aflt, errorq_t *eqp, 3169 errorq_elem_t *eqep, size_t afltoffset) 3170 { 3171 ce_dispact_t dispact, action; 3172 cpu_t *cp; 3173 uchar_t dtcrinfo, disp; 3174 int ptnrtype; 3175 3176 if (!ce_disp_inited || panicstr || ce_xdiag_off) { 3177 ce_xdiag_drops++; 3178 return (0); 3179 } else if (!aflt->flt_in_memory) { 3180 ce_xdiag_drops++; 3181 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOTMEM); 3182 return (0); 3183 } 3184 3185 dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 3186 3187 /* 3188 * Some correctable events are not scrubbed/classified, such as those 3189 * noticed at the tail of cpu_deferred_error. So if there is no 3190 * initial detector classification go no further. 3191 */ 3192 if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) { 3193 ce_xdiag_drops++; 3194 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOSCRUB); 3195 return (0); 3196 } 3197 3198 dispact = CE_DISPACT(ce_disp_table, 3199 CE_XDIAG_AFARMATCHED(dtcrinfo), 3200 CE_XDIAG_STATE(dtcrinfo), 3201 CE_XDIAG_CE1SEEN(dtcrinfo), 3202 CE_XDIAG_CE2SEEN(dtcrinfo)); 3203 3204 3205 action = CE_ACT(dispact); /* bad lookup caught below */ 3206 #ifdef DEBUG 3207 if (ce_xdiag_forceaction != 0) 3208 action = ce_xdiag_forceaction; 3209 #endif 3210 3211 switch (action) { 3212 case CE_ACT_LKYCHK: { 3213 caddr_t ndata; 3214 errorq_elem_t *neqep; 3215 struct async_flt *ecc; 3216 ce_lkychk_cb_t *cbargp; 3217 3218 if ((ndata = errorq_elem_dup(eqp, eqep, &neqep)) == NULL) { 3219 ce_xdiag_lkydrops++; 3220 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 3221 CE_XDIAG_SKIP_DUPFAIL); 3222 break; 3223 } 3224 ecc = (struct async_flt *)(ndata + afltoffset); 3225 3226 ASSERT(ecc->flt_class == CPU_FAULT || 3227 ecc->flt_class == BUS_FAULT); 3228 ecc->flt_class = (ecc->flt_class == CPU_FAULT) ? 3229 RECIRC_CPU_FAULT : RECIRC_BUS_FAULT; 3230 3231 cbargp = kmem_alloc(sizeof (ce_lkychk_cb_t), KM_SLEEP); 3232 cbargp->lkycb_aflt = ecc; 3233 cbargp->lkycb_eqp = eqp; 3234 cbargp->lkycb_eqep = neqep; 3235 3236 (void) timeout((void (*)(void *))ce_lkychk_cb, 3237 (void *)cbargp, drv_usectohz(cpu_ce_lkychk_timeout_usec)); 3238 return (1); 3239 } 3240 3241 case CE_ACT_PTNRCHK: 3242 kpreempt_disable(); /* stop cpu list changing */ 3243 if ((cp = ce_ptnr_select(aflt, 0, &ptnrtype)) != NULL) { 3244 xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, 3245 (uint64_t)aflt, (uint64_t)&disp); 3246 CE_XDIAG_SETPTNRINFO(aflt->flt_disp, disp); 3247 CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 3248 CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 3249 } else if (ncpus > 1) { 3250 ce_xdiag_ptnrdrops++; 3251 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 3252 CE_XDIAG_SKIP_NOPTNR); 3253 } else { 3254 ce_xdiag_ptnrdrops++; 3255 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 3256 CE_XDIAG_SKIP_UNIPROC); 3257 } 3258 kpreempt_enable(); 3259 break; 3260 3261 case CE_ACT_DONE: 3262 break; 3263 3264 case CE_ACT(CE_DISP_BAD): 3265 default: 3266 #ifdef DEBUG 3267 cmn_err(CE_PANIC, "ce_scrub_post: Bad action '%d'", action); 3268 #endif 3269 ce_xdiag_bad++; 3270 CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_ACTBAD); 3271 break; 3272 } 3273 3274 return (0); 3275 } 3276 3277 /* 3278 * We route all errors through a single switch statement. 3279 */ 3280 void 3281 cpu_ue_log_err(struct async_flt *aflt) 3282 { 3283 switch (aflt->flt_class) { 3284 case CPU_FAULT: 3285 cpu_ereport_init(aflt); 3286 if (cpu_async_log_err(aflt, NULL)) 3287 cpu_ereport_post(aflt); 3288 break; 3289 3290 case BUS_FAULT: 3291 bus_async_log_err(aflt); 3292 break; 3293 3294 default: 3295 cmn_err(CE_WARN, "discarding async error %p with invalid " 3296 "fault class (0x%x)", (void *)aflt, aflt->flt_class); 3297 return; 3298 } 3299 } 3300 3301 /* 3302 * Routine for panic hook callback from panic_idle(). 3303 */ 3304 void 3305 cpu_async_panic_callb(void) 3306 { 3307 ch_async_flt_t ch_flt; 3308 struct async_flt *aflt; 3309 ch_cpu_errors_t cpu_error_regs; 3310 uint64_t afsr_errs; 3311 3312 get_cpu_error_state(&cpu_error_regs); 3313 3314 afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 3315 (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS); 3316 3317 if (afsr_errs) { 3318 3319 bzero(&ch_flt, sizeof (ch_async_flt_t)); 3320 aflt = (struct async_flt *)&ch_flt; 3321 aflt->flt_id = gethrtime_waitfree(); 3322 aflt->flt_bus_id = getprocessorid(); 3323 aflt->flt_inst = CPU->cpu_id; 3324 aflt->flt_stat = cpu_error_regs.afsr; 3325 aflt->flt_addr = cpu_error_regs.afar; 3326 aflt->flt_prot = AFLT_PROT_NONE; 3327 aflt->flt_class = CPU_FAULT; 3328 aflt->flt_priv = ((cpu_error_regs.afsr & C_AFSR_PRIV) != 0); 3329 aflt->flt_panic = 1; 3330 ch_flt.afsr_ext = cpu_error_regs.afsr_ext; 3331 ch_flt.afsr_errs = afsr_errs; 3332 #if defined(SERRANO) 3333 ch_flt.afar2 = cpu_error_regs.afar2; 3334 #endif /* SERRANO */ 3335 (void) cpu_queue_events(&ch_flt, NULL, afsr_errs, NULL); 3336 } 3337 } 3338 3339 /* 3340 * Routine to convert a syndrome into a syndrome code. 3341 */ 3342 static int 3343 synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit) 3344 { 3345 if (synd_status == AFLT_STAT_INVALID) 3346 return (-1); 3347 3348 /* 3349 * Use the syndrome to index the appropriate syndrome table, 3350 * to get the code indicating which bit(s) is(are) bad. 3351 */ 3352 if (afsr_bit & 3353 (C_AFSR_MSYND_ERRS | C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 3354 if (afsr_bit & C_AFSR_MSYND_ERRS) { 3355 #if defined(JALAPENO) || defined(SERRANO) 3356 if ((synd == 0) || (synd >= BSYND_TBL_SIZE)) 3357 return (-1); 3358 else 3359 return (BPAR0 + synd); 3360 #else /* JALAPENO || SERRANO */ 3361 if ((synd == 0) || (synd >= MSYND_TBL_SIZE)) 3362 return (-1); 3363 else 3364 return (mtag_syndrome_tab[synd]); 3365 #endif /* JALAPENO || SERRANO */ 3366 } else { 3367 if ((synd == 0) || (synd >= ESYND_TBL_SIZE)) 3368 return (-1); 3369 else 3370 return (ecc_syndrome_tab[synd]); 3371 } 3372 } else { 3373 return (-1); 3374 } 3375 } 3376 3377 int 3378 cpu_get_mem_sid(char *unum, char *buf, int buflen, int *lenp) 3379 { 3380 if (&plat_get_mem_sid) 3381 return (plat_get_mem_sid(unum, buf, buflen, lenp)); 3382 else 3383 return (ENOTSUP); 3384 } 3385 3386 int 3387 cpu_get_mem_offset(uint64_t flt_addr, uint64_t *offp) 3388 { 3389 if (&plat_get_mem_offset) 3390 return (plat_get_mem_offset(flt_addr, offp)); 3391 else 3392 return (ENOTSUP); 3393 } 3394 3395 int 3396 cpu_get_mem_addr(char *unum, char *sid, uint64_t offset, uint64_t *addrp) 3397 { 3398 if (&plat_get_mem_addr) 3399 return (plat_get_mem_addr(unum, sid, offset, addrp)); 3400 else 3401 return (ENOTSUP); 3402 } 3403 3404 /* 3405 * Routine to return a string identifying the physical name 3406 * associated with a memory/cache error. 3407 */ 3408 int 3409 cpu_get_mem_unum(int synd_status, ushort_t flt_synd, uint64_t flt_stat, 3410 uint64_t flt_addr, int flt_bus_id, int flt_in_memory, 3411 ushort_t flt_status, char *buf, int buflen, int *lenp) 3412 { 3413 int synd_code; 3414 int ret; 3415 3416 /* 3417 * An AFSR of -1 defaults to a memory syndrome. 3418 */ 3419 if (flt_stat == (uint64_t)-1) 3420 flt_stat = C_AFSR_CE; 3421 3422 synd_code = synd_to_synd_code(synd_status, flt_synd, flt_stat); 3423 3424 /* 3425 * Syndrome code must be either a single-bit error code 3426 * (0...143) or -1 for unum lookup. 3427 */ 3428 if (synd_code < 0 || synd_code >= M2) 3429 synd_code = -1; 3430 if (&plat_get_mem_unum) { 3431 if ((ret = plat_get_mem_unum(synd_code, flt_addr, flt_bus_id, 3432 flt_in_memory, flt_status, buf, buflen, lenp)) != 0) { 3433 buf[0] = '\0'; 3434 *lenp = 0; 3435 } 3436 3437 return (ret); 3438 } 3439 3440 return (ENOTSUP); 3441 } 3442 3443 /* 3444 * Wrapper for cpu_get_mem_unum() routine that takes an 3445 * async_flt struct rather than explicit arguments. 3446 */ 3447 int 3448 cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt, 3449 char *buf, int buflen, int *lenp) 3450 { 3451 /* 3452 * If we come thru here for an IO bus error aflt->flt_stat will 3453 * not be the CPU AFSR, and we pass in a -1 to cpu_get_mem_unum() 3454 * so it will interpret this as a memory error. 3455 */ 3456 return (cpu_get_mem_unum(synd_status, aflt->flt_synd, 3457 (aflt->flt_class == BUS_FAULT) ? 3458 (uint64_t)-1 : ((ch_async_flt_t *)aflt)->flt_bit, 3459 aflt->flt_addr, aflt->flt_bus_id, aflt->flt_in_memory, 3460 aflt->flt_status, buf, buflen, lenp)); 3461 } 3462 3463 /* 3464 * Return unum string given synd_code and async_flt into 3465 * the buf with size UNUM_NAMLEN 3466 */ 3467 static int 3468 cpu_get_mem_unum_synd(int synd_code, struct async_flt *aflt, char *buf) 3469 { 3470 int ret, len; 3471 3472 /* 3473 * Syndrome code must be either a single-bit error code 3474 * (0...143) or -1 for unum lookup. 3475 */ 3476 if (synd_code < 0 || synd_code >= M2) 3477 synd_code = -1; 3478 if (&plat_get_mem_unum) { 3479 if ((ret = plat_get_mem_unum(synd_code, aflt->flt_addr, 3480 aflt->flt_bus_id, aflt->flt_in_memory, 3481 aflt->flt_status, buf, UNUM_NAMLEN, &len)) != 0) { 3482 buf[0] = '\0'; 3483 } 3484 return (ret); 3485 } 3486 3487 buf[0] = '\0'; 3488 return (ENOTSUP); 3489 } 3490 3491 /* 3492 * This routine is a more generic interface to cpu_get_mem_unum() 3493 * that may be used by other modules (e.g. the 'mm' driver, through 3494 * the 'MEM_NAME' ioctl, which is used by fmd to resolve unum's 3495 * for Jalapeno/Serrano FRC/RCE or FRU/RUE paired events). 3496 */ 3497 int 3498 cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar, 3499 char *buf, int buflen, int *lenp) 3500 { 3501 int synd_status, flt_in_memory, ret; 3502 ushort_t flt_status = 0; 3503 char unum[UNUM_NAMLEN]; 3504 uint64_t t_afsr_errs; 3505 3506 /* 3507 * Check for an invalid address. 3508 */ 3509 if (afar == (uint64_t)-1) 3510 return (ENXIO); 3511 3512 if (synd == (uint64_t)-1) 3513 synd_status = AFLT_STAT_INVALID; 3514 else 3515 synd_status = AFLT_STAT_VALID; 3516 3517 flt_in_memory = (*afsr & C_AFSR_MEMORY) && 3518 pf_is_memory(afar >> MMU_PAGESHIFT); 3519 3520 /* 3521 * Get aggregate AFSR for call to cpu_error_is_ecache_data. 3522 */ 3523 if (*afsr == (uint64_t)-1) 3524 t_afsr_errs = C_AFSR_CE; 3525 else { 3526 t_afsr_errs = (*afsr & C_AFSR_ALL_ERRS); 3527 #if defined(CHEETAH_PLUS) 3528 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 3529 t_afsr_errs |= (*(afsr + 1) & C_AFSR_EXT_ALL_ERRS); 3530 #endif /* CHEETAH_PLUS */ 3531 } 3532 3533 /* 3534 * Turn on ECC_ECACHE if error type is E$ Data. 3535 */ 3536 if (cpu_error_is_ecache_data(CPU->cpu_id, t_afsr_errs)) 3537 flt_status |= ECC_ECACHE; 3538 3539 ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, t_afsr_errs, afar, 3540 CPU->cpu_id, flt_in_memory, flt_status, unum, UNUM_NAMLEN, lenp); 3541 if (ret != 0) 3542 return (ret); 3543 3544 if (*lenp >= buflen) 3545 return (ENAMETOOLONG); 3546 3547 (void) strncpy(buf, unum, buflen); 3548 3549 return (0); 3550 } 3551 3552 /* 3553 * Routine to return memory information associated 3554 * with a physical address and syndrome. 3555 */ 3556 int 3557 cpu_get_mem_info(uint64_t synd, uint64_t afar, 3558 uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 3559 int *segsp, int *banksp, int *mcidp) 3560 { 3561 int synd_status, synd_code; 3562 3563 if (afar == (uint64_t)-1) 3564 return (ENXIO); 3565 3566 if (synd == (uint64_t)-1) 3567 synd_status = AFLT_STAT_INVALID; 3568 else 3569 synd_status = AFLT_STAT_VALID; 3570 3571 synd_code = synd_to_synd_code(synd_status, synd, C_AFSR_CE); 3572 3573 if (p2get_mem_info != NULL) 3574 return ((p2get_mem_info)(synd_code, afar, 3575 mem_sizep, seg_sizep, bank_sizep, 3576 segsp, banksp, mcidp)); 3577 else 3578 return (ENOTSUP); 3579 } 3580 3581 /* 3582 * Routine to return a string identifying the physical 3583 * name associated with a cpuid. 3584 */ 3585 int 3586 cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp) 3587 { 3588 int ret; 3589 char unum[UNUM_NAMLEN]; 3590 3591 if (&plat_get_cpu_unum) { 3592 if ((ret = plat_get_cpu_unum(cpuid, unum, UNUM_NAMLEN, lenp)) 3593 != 0) 3594 return (ret); 3595 } else { 3596 return (ENOTSUP); 3597 } 3598 3599 if (*lenp >= buflen) 3600 return (ENAMETOOLONG); 3601 3602 (void) strncpy(buf, unum, buflen); 3603 3604 return (0); 3605 } 3606 3607 /* 3608 * This routine exports the name buffer size. 3609 */ 3610 size_t 3611 cpu_get_name_bufsize() 3612 { 3613 return (UNUM_NAMLEN); 3614 } 3615 3616 /* 3617 * Historical function, apparantly not used. 3618 */ 3619 /* ARGSUSED */ 3620 void 3621 cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err) 3622 {} 3623 3624 /* 3625 * Historical function only called for SBus errors in debugging. 3626 */ 3627 /*ARGSUSED*/ 3628 void 3629 read_ecc_data(struct async_flt *aflt, short verbose, short ce_err) 3630 {} 3631 3632 /* 3633 * Clear the AFSR sticky bits. The routine returns a non-zero value if 3634 * any of the AFSR's sticky errors are detected. If a non-null pointer to 3635 * an async fault structure argument is passed in, the captured error state 3636 * (AFSR, AFAR) info will be returned in the structure. 3637 */ 3638 int 3639 clear_errors(ch_async_flt_t *ch_flt) 3640 { 3641 struct async_flt *aflt = (struct async_flt *)ch_flt; 3642 ch_cpu_errors_t cpu_error_regs; 3643 3644 get_cpu_error_state(&cpu_error_regs); 3645 3646 if (ch_flt != NULL) { 3647 aflt->flt_stat = cpu_error_regs.afsr & C_AFSR_MASK; 3648 aflt->flt_addr = cpu_error_regs.afar; 3649 ch_flt->afsr_ext = cpu_error_regs.afsr_ext; 3650 ch_flt->afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 3651 (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS); 3652 #if defined(SERRANO) 3653 ch_flt->afar2 = cpu_error_regs.afar2; 3654 #endif /* SERRANO */ 3655 } 3656 3657 set_cpu_error_state(&cpu_error_regs); 3658 3659 return (((cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 3660 (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS)) != 0); 3661 } 3662 3663 /* 3664 * Clear any AFSR error bits, and check for persistence. 3665 * 3666 * It would be desirable to also insist that syndrome match. PCI handling 3667 * has already filled flt_synd. For errors trapped by CPU we only fill 3668 * flt_synd when we queue the event, so we do not have a valid flt_synd 3669 * during initial classification (it is valid if we're called as part of 3670 * subsequent low-pil additional classification attempts). We could try 3671 * to determine which syndrome to use: we know we're only called for 3672 * CE/RCE (Jalapeno & Serrano) and CE/EMC (others) so the syndrome to use 3673 * would be esynd/none and esynd/msynd, respectively. If that is 3674 * implemented then what do we do in the case that we do experience an 3675 * error on the same afar but with different syndrome? At the very least 3676 * we should count such occurences. Anyway, for now, we'll leave it as 3677 * it has been for ages. 3678 */ 3679 static int 3680 clear_ecc(struct async_flt *aflt) 3681 { 3682 ch_cpu_errors_t cpu_error_regs; 3683 3684 /* 3685 * Snapshot the AFSR and AFAR and clear any errors 3686 */ 3687 get_cpu_error_state(&cpu_error_regs); 3688 set_cpu_error_state(&cpu_error_regs); 3689 3690 /* 3691 * If any of the same memory access error bits are still on and 3692 * the AFAR matches, return that the error is persistent. 3693 */ 3694 return ((cpu_error_regs.afsr & (C_AFSR_MEMORY & aflt->flt_stat)) != 0 && 3695 cpu_error_regs.afar == aflt->flt_addr); 3696 } 3697 3698 /* 3699 * Turn off all cpu error detection, normally only used for panics. 3700 */ 3701 void 3702 cpu_disable_errors(void) 3703 { 3704 xt_all(set_error_enable_tl1, EN_REG_DISABLE, EER_SET_ABSOLUTE); 3705 3706 /* 3707 * With error detection now turned off, check the other cpus 3708 * logout areas for any unlogged errors. 3709 */ 3710 if (enable_check_other_cpus_logout) { 3711 cpu_check_other_cpus_logout(); 3712 /* 3713 * Make a second pass over the logout areas, in case 3714 * there is a failing CPU in an error-trap loop which 3715 * will write to the logout area once it is emptied. 3716 */ 3717 cpu_check_other_cpus_logout(); 3718 } 3719 } 3720 3721 /* 3722 * Enable errors. 3723 */ 3724 void 3725 cpu_enable_errors(void) 3726 { 3727 xt_all(set_error_enable_tl1, EN_REG_ENABLE, EER_SET_ABSOLUTE); 3728 } 3729 3730 /* 3731 * Flush the entire ecache using displacement flush by reading through a 3732 * physical address range twice as large as the Ecache. 3733 */ 3734 void 3735 cpu_flush_ecache(void) 3736 { 3737 flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size, 3738 cpunodes[CPU->cpu_id].ecache_linesize); 3739 } 3740 3741 /* 3742 * Return CPU E$ set size - E$ size divided by the associativity. 3743 * We use this function in places where the CPU_PRIVATE ptr may not be 3744 * initialized yet. Note that for send_mondo and in the Ecache scrubber, 3745 * we're guaranteed that CPU_PRIVATE is initialized. Also, cpunodes is set 3746 * up before the kernel switches from OBP's to the kernel's trap table, so 3747 * we don't have to worry about cpunodes being unitialized. 3748 */ 3749 int 3750 cpu_ecache_set_size(struct cpu *cp) 3751 { 3752 if (CPU_PRIVATE(cp)) 3753 return (CPU_PRIVATE_VAL(cp, chpr_ec_set_size)); 3754 3755 return (cpunodes[cp->cpu_id].ecache_size / cpu_ecache_nway()); 3756 } 3757 3758 /* 3759 * Flush Ecache line. 3760 * Uses ASI_EC_DIAG for Cheetah+ and Jalapeno. 3761 * Uses normal displacement flush for Cheetah. 3762 */ 3763 static void 3764 cpu_flush_ecache_line(ch_async_flt_t *ch_flt) 3765 { 3766 struct async_flt *aflt = (struct async_flt *)ch_flt; 3767 int ec_set_size = cpu_ecache_set_size(CPU); 3768 3769 ecache_flush_line(aflt->flt_addr, ec_set_size); 3770 } 3771 3772 /* 3773 * Scrub physical address. 3774 * Scrub code is different depending upon whether this a Cheetah+ with 2-way 3775 * Ecache or direct-mapped Ecache. 3776 */ 3777 static void 3778 cpu_scrubphys(struct async_flt *aflt) 3779 { 3780 int ec_set_size = cpu_ecache_set_size(CPU); 3781 3782 scrubphys(aflt->flt_addr, ec_set_size); 3783 } 3784 3785 /* 3786 * Clear physical address. 3787 * Scrub code is different depending upon whether this a Cheetah+ with 2-way 3788 * Ecache or direct-mapped Ecache. 3789 */ 3790 void 3791 cpu_clearphys(struct async_flt *aflt) 3792 { 3793 int lsize = cpunodes[CPU->cpu_id].ecache_linesize; 3794 int ec_set_size = cpu_ecache_set_size(CPU); 3795 3796 3797 clearphys(P2ALIGN(aflt->flt_addr, lsize), ec_set_size, lsize); 3798 } 3799 3800 #if defined(CPU_IMP_ECACHE_ASSOC) 3801 /* 3802 * Check for a matching valid line in all the sets. 3803 * If found, return set# + 1. Otherwise return 0. 3804 */ 3805 static int 3806 cpu_ecache_line_valid(ch_async_flt_t *ch_flt) 3807 { 3808 struct async_flt *aflt = (struct async_flt *)ch_flt; 3809 int totalsize = cpunodes[CPU->cpu_id].ecache_size; 3810 int ec_set_size = cpu_ecache_set_size(CPU); 3811 ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 3812 int nway = cpu_ecache_nway(); 3813 int i; 3814 3815 for (i = 0; i < nway; i++, ecp++) { 3816 if (!cpu_ectag_line_invalid(totalsize, ecp->ec_tag) && 3817 (aflt->flt_addr & P2ALIGN(C_AFAR_PA, ec_set_size)) == 3818 cpu_ectag_to_pa(ec_set_size, ecp->ec_tag)) 3819 return (i+1); 3820 } 3821 return (0); 3822 } 3823 #endif /* CPU_IMP_ECACHE_ASSOC */ 3824 3825 /* 3826 * Check whether a line in the given logout info matches the specified 3827 * fault address. If reqval is set then the line must not be Invalid. 3828 * Returns 0 on failure; on success (way + 1) is returned an *level is 3829 * set to 2 for l2$ or 3 for l3$. 3830 */ 3831 static int 3832 cpu_matching_ecache_line(uint64_t faddr, void *data, int reqval, int *level) 3833 { 3834 ch_diag_data_t *cdp = data; 3835 ch_ec_data_t *ecp; 3836 int totalsize, ec_set_size; 3837 int i, ways; 3838 int match = 0; 3839 int tagvalid; 3840 uint64_t addr, tagpa; 3841 int ispanther = IS_PANTHER(cpunodes[CPU->cpu_id].implementation); 3842 3843 /* 3844 * Check the l2$ logout data 3845 */ 3846 if (ispanther) { 3847 ecp = &cdp->chd_l2_data[0]; 3848 ec_set_size = PN_L2_SET_SIZE; 3849 ways = PN_L2_NWAYS; 3850 } else { 3851 ecp = &cdp->chd_ec_data[0]; 3852 ec_set_size = cpu_ecache_set_size(CPU); 3853 ways = cpu_ecache_nway(); 3854 totalsize = cpunodes[CPU->cpu_id].ecache_size; 3855 } 3856 /* remove low order PA bits from fault address not used in PA tag */ 3857 addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 3858 for (i = 0; i < ways; i++, ecp++) { 3859 if (ispanther) { 3860 tagpa = PN_L2TAG_TO_PA(ecp->ec_tag); 3861 tagvalid = !PN_L2_LINE_INVALID(ecp->ec_tag); 3862 } else { 3863 tagpa = cpu_ectag_to_pa(ec_set_size, ecp->ec_tag); 3864 tagvalid = !cpu_ectag_line_invalid(totalsize, 3865 ecp->ec_tag); 3866 } 3867 if (tagpa == addr && (!reqval || tagvalid)) { 3868 match = i + 1; 3869 *level = 2; 3870 break; 3871 } 3872 } 3873 3874 if (match || !ispanther) 3875 return (match); 3876 3877 /* For Panther we also check the l3$ */ 3878 ecp = &cdp->chd_ec_data[0]; 3879 ec_set_size = PN_L3_SET_SIZE; 3880 ways = PN_L3_NWAYS; 3881 addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 3882 3883 for (i = 0; i < ways; i++, ecp++) { 3884 if (PN_L3TAG_TO_PA(ecp->ec_tag) == addr && (!reqval || 3885 !PN_L3_LINE_INVALID(ecp->ec_tag))) { 3886 match = i + 1; 3887 *level = 3; 3888 break; 3889 } 3890 } 3891 3892 return (match); 3893 } 3894 3895 #if defined(CPU_IMP_L1_CACHE_PARITY) 3896 /* 3897 * Record information related to the source of an Dcache Parity Error. 3898 */ 3899 static void 3900 cpu_dcache_parity_info(ch_async_flt_t *ch_flt) 3901 { 3902 int dc_set_size = dcache_size / CH_DCACHE_NWAY; 3903 int index; 3904 3905 /* 3906 * Since instruction decode cannot be done at high PIL 3907 * just examine the entire Dcache to locate the error. 3908 */ 3909 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 3910 ch_flt->parity_data.dpe.cpl_way = -1; 3911 ch_flt->parity_data.dpe.cpl_off = -1; 3912 } 3913 for (index = 0; index < dc_set_size; index += dcache_linesize) 3914 cpu_dcache_parity_check(ch_flt, index); 3915 } 3916 3917 /* 3918 * Check all ways of the Dcache at a specified index for good parity. 3919 */ 3920 static void 3921 cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index) 3922 { 3923 int dc_set_size = dcache_size / CH_DCACHE_NWAY; 3924 uint64_t parity_bits, pbits, data_word; 3925 static int parity_bits_popc[] = { 0, 1, 1, 0 }; 3926 int way, word, data_byte; 3927 ch_dc_data_t *dcp = &ch_flt->parity_data.dpe.cpl_dc[0]; 3928 ch_dc_data_t tmp_dcp; 3929 3930 for (way = 0; way < CH_DCACHE_NWAY; way++, dcp++) { 3931 /* 3932 * Perform diagnostic read. 3933 */ 3934 get_dcache_dtag(index + way * dc_set_size, 3935 (uint64_t *)&tmp_dcp); 3936 3937 /* 3938 * Check tag for even parity. 3939 * Sum of 1 bits (including parity bit) should be even. 3940 */ 3941 if (popc64(tmp_dcp.dc_tag & CHP_DCTAG_PARMASK) & 1) { 3942 /* 3943 * If this is the first error log detailed information 3944 * about it and check the snoop tag. Otherwise just 3945 * record the fact that we found another error. 3946 */ 3947 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 3948 ch_flt->parity_data.dpe.cpl_way = way; 3949 ch_flt->parity_data.dpe.cpl_cache = 3950 CPU_DC_PARITY; 3951 ch_flt->parity_data.dpe.cpl_tag |= CHP_DC_TAG; 3952 3953 if (popc64(tmp_dcp.dc_sntag & 3954 CHP_DCSNTAG_PARMASK) & 1) { 3955 ch_flt->parity_data.dpe.cpl_tag |= 3956 CHP_DC_SNTAG; 3957 ch_flt->parity_data.dpe.cpl_lcnt++; 3958 } 3959 3960 bcopy(&tmp_dcp, dcp, sizeof (ch_dc_data_t)); 3961 } 3962 3963 ch_flt->parity_data.dpe.cpl_lcnt++; 3964 } 3965 3966 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 3967 /* 3968 * Panther has more parity bits than the other 3969 * processors for covering dcache data and so each 3970 * byte of data in each word has its own parity bit. 3971 */ 3972 parity_bits = tmp_dcp.dc_pn_data_parity; 3973 for (word = 0; word < 4; word++) { 3974 data_word = tmp_dcp.dc_data[word]; 3975 pbits = parity_bits & PN_DC_DATA_PARITY_MASK; 3976 for (data_byte = 0; data_byte < 8; 3977 data_byte++) { 3978 if (((popc64(data_word & 3979 PN_DC_DATA_PARITY_MASK)) & 1) ^ 3980 (pbits & 1)) { 3981 cpu_record_dc_data_parity( 3982 ch_flt, dcp, &tmp_dcp, way, 3983 word); 3984 } 3985 pbits >>= 1; 3986 data_word >>= 8; 3987 } 3988 parity_bits >>= 8; 3989 } 3990 } else { 3991 /* 3992 * Check data array for even parity. 3993 * The 8 parity bits are grouped into 4 pairs each 3994 * of which covers a 64-bit word. The endianness is 3995 * reversed -- the low-order parity bits cover the 3996 * high-order data words. 3997 */ 3998 parity_bits = tmp_dcp.dc_utag >> 8; 3999 for (word = 0; word < 4; word++) { 4000 pbits = (parity_bits >> (6 - word * 2)) & 3; 4001 if ((popc64(tmp_dcp.dc_data[word]) + 4002 parity_bits_popc[pbits]) & 1) { 4003 cpu_record_dc_data_parity(ch_flt, dcp, 4004 &tmp_dcp, way, word); 4005 } 4006 } 4007 } 4008 } 4009 } 4010 4011 static void 4012 cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 4013 ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word) 4014 { 4015 /* 4016 * If this is the first error log detailed information about it. 4017 * Otherwise just record the fact that we found another error. 4018 */ 4019 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 4020 ch_flt->parity_data.dpe.cpl_way = way; 4021 ch_flt->parity_data.dpe.cpl_cache = CPU_DC_PARITY; 4022 ch_flt->parity_data.dpe.cpl_off = word * 8; 4023 bcopy(src_dcp, dest_dcp, sizeof (ch_dc_data_t)); 4024 } 4025 ch_flt->parity_data.dpe.cpl_lcnt++; 4026 } 4027 4028 /* 4029 * Record information related to the source of an Icache Parity Error. 4030 * 4031 * Called with the Icache disabled so any diagnostic accesses are safe. 4032 */ 4033 static void 4034 cpu_icache_parity_info(ch_async_flt_t *ch_flt) 4035 { 4036 int ic_set_size; 4037 int ic_linesize; 4038 int index; 4039 4040 if (CPU_PRIVATE(CPU)) { 4041 ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 4042 CH_ICACHE_NWAY; 4043 ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 4044 } else { 4045 ic_set_size = icache_size / CH_ICACHE_NWAY; 4046 ic_linesize = icache_linesize; 4047 } 4048 4049 ch_flt->parity_data.ipe.cpl_way = -1; 4050 ch_flt->parity_data.ipe.cpl_off = -1; 4051 4052 for (index = 0; index < ic_set_size; index += ic_linesize) 4053 cpu_icache_parity_check(ch_flt, index); 4054 } 4055 4056 /* 4057 * Check all ways of the Icache at a specified index for good parity. 4058 */ 4059 static void 4060 cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index) 4061 { 4062 uint64_t parmask, pn_inst_parity; 4063 int ic_set_size; 4064 int ic_linesize; 4065 int flt_index, way, instr, num_instr; 4066 struct async_flt *aflt = (struct async_flt *)ch_flt; 4067 ch_ic_data_t *icp = &ch_flt->parity_data.ipe.cpl_ic[0]; 4068 ch_ic_data_t tmp_icp; 4069 4070 if (CPU_PRIVATE(CPU)) { 4071 ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 4072 CH_ICACHE_NWAY; 4073 ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 4074 } else { 4075 ic_set_size = icache_size / CH_ICACHE_NWAY; 4076 ic_linesize = icache_linesize; 4077 } 4078 4079 /* 4080 * Panther has twice as many instructions per icache line and the 4081 * instruction parity bit is in a different location. 4082 */ 4083 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 4084 num_instr = PN_IC_DATA_REG_SIZE / sizeof (uint64_t); 4085 pn_inst_parity = PN_ICDATA_PARITY_BIT_MASK; 4086 } else { 4087 num_instr = CH_IC_DATA_REG_SIZE / sizeof (uint64_t); 4088 pn_inst_parity = 0; 4089 } 4090 4091 /* 4092 * Index at which we expect to find the parity error. 4093 */ 4094 flt_index = P2ALIGN(aflt->flt_addr % ic_set_size, ic_linesize); 4095 4096 for (way = 0; way < CH_ICACHE_NWAY; way++, icp++) { 4097 /* 4098 * Diagnostic reads expect address argument in ASI format. 4099 */ 4100 get_icache_dtag(2 * (index + way * ic_set_size), 4101 (uint64_t *)&tmp_icp); 4102 4103 /* 4104 * If this is the index in which we expect to find the 4105 * error log detailed information about each of the ways. 4106 * This information will be displayed later if we can't 4107 * determine the exact way in which the error is located. 4108 */ 4109 if (flt_index == index) 4110 bcopy(&tmp_icp, icp, sizeof (ch_ic_data_t)); 4111 4112 /* 4113 * Check tag for even parity. 4114 * Sum of 1 bits (including parity bit) should be even. 4115 */ 4116 if (popc64(tmp_icp.ic_patag & CHP_ICPATAG_PARMASK) & 1) { 4117 /* 4118 * If this way is the one in which we expected 4119 * to find the error record the way and check the 4120 * snoop tag. Otherwise just record the fact we 4121 * found another error. 4122 */ 4123 if (flt_index == index) { 4124 ch_flt->parity_data.ipe.cpl_way = way; 4125 ch_flt->parity_data.ipe.cpl_tag |= CHP_IC_TAG; 4126 4127 if (popc64(tmp_icp.ic_sntag & 4128 CHP_ICSNTAG_PARMASK) & 1) { 4129 ch_flt->parity_data.ipe.cpl_tag |= 4130 CHP_IC_SNTAG; 4131 ch_flt->parity_data.ipe.cpl_lcnt++; 4132 } 4133 4134 } 4135 ch_flt->parity_data.ipe.cpl_lcnt++; 4136 continue; 4137 } 4138 4139 /* 4140 * Check instruction data for even parity. 4141 * Bits participating in parity differ for PC-relative 4142 * versus non-PC-relative instructions. 4143 */ 4144 for (instr = 0; instr < num_instr; instr++) { 4145 parmask = (tmp_icp.ic_data[instr] & 4146 CH_ICDATA_PRED_ISPCREL) ? 4147 (CHP_ICDATA_PCREL_PARMASK | pn_inst_parity) : 4148 (CHP_ICDATA_NPCREL_PARMASK | pn_inst_parity); 4149 if (popc64(tmp_icp.ic_data[instr] & parmask) & 1) { 4150 /* 4151 * If this way is the one in which we expected 4152 * to find the error record the way and offset. 4153 * Otherwise just log the fact we found another 4154 * error. 4155 */ 4156 if (flt_index == index) { 4157 ch_flt->parity_data.ipe.cpl_way = way; 4158 ch_flt->parity_data.ipe.cpl_off = 4159 instr * 4; 4160 } 4161 ch_flt->parity_data.ipe.cpl_lcnt++; 4162 continue; 4163 } 4164 } 4165 } 4166 } 4167 4168 /* 4169 * Record information related to the source of an Pcache Parity Error. 4170 */ 4171 static void 4172 cpu_pcache_parity_info(ch_async_flt_t *ch_flt) 4173 { 4174 int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 4175 int index; 4176 4177 /* 4178 * Since instruction decode cannot be done at high PIL just 4179 * examine the entire Pcache to check for any parity errors. 4180 */ 4181 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 4182 ch_flt->parity_data.dpe.cpl_way = -1; 4183 ch_flt->parity_data.dpe.cpl_off = -1; 4184 } 4185 for (index = 0; index < pc_set_size; index += CH_PCACHE_LSIZE) 4186 cpu_pcache_parity_check(ch_flt, index); 4187 } 4188 4189 /* 4190 * Check all ways of the Pcache at a specified index for good parity. 4191 */ 4192 static void 4193 cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index) 4194 { 4195 int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 4196 int pc_data_words = CH_PC_DATA_REG_SIZE / sizeof (uint64_t); 4197 int way, word, pbit, parity_bits; 4198 ch_pc_data_t *pcp = &ch_flt->parity_data.dpe.cpl_pc[0]; 4199 ch_pc_data_t tmp_pcp; 4200 4201 for (way = 0; way < CH_PCACHE_NWAY; way++, pcp++) { 4202 /* 4203 * Perform diagnostic read. 4204 */ 4205 get_pcache_dtag(index + way * pc_set_size, 4206 (uint64_t *)&tmp_pcp); 4207 /* 4208 * Check data array for odd parity. There are 8 parity 4209 * bits (bits 57:50 of ASI_PCACHE_STATUS_DATA) and each 4210 * of those bits covers exactly 8 bytes of the data 4211 * array: 4212 * 4213 * parity bit P$ data bytes covered 4214 * ---------- --------------------- 4215 * 50 63:56 4216 * 51 55:48 4217 * 52 47:40 4218 * 53 39:32 4219 * 54 31:24 4220 * 55 23:16 4221 * 56 15:8 4222 * 57 7:0 4223 */ 4224 parity_bits = PN_PC_PARITY_BITS(tmp_pcp.pc_status); 4225 for (word = 0; word < pc_data_words; word++) { 4226 pbit = (parity_bits >> (pc_data_words - word - 1)) & 1; 4227 if ((popc64(tmp_pcp.pc_data[word]) & 1) ^ pbit) { 4228 /* 4229 * If this is the first error log detailed 4230 * information about it. Otherwise just record 4231 * the fact that we found another error. 4232 */ 4233 if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 4234 ch_flt->parity_data.dpe.cpl_way = way; 4235 ch_flt->parity_data.dpe.cpl_cache = 4236 CPU_PC_PARITY; 4237 ch_flt->parity_data.dpe.cpl_off = 4238 word * sizeof (uint64_t); 4239 bcopy(&tmp_pcp, pcp, 4240 sizeof (ch_pc_data_t)); 4241 } 4242 ch_flt->parity_data.dpe.cpl_lcnt++; 4243 } 4244 } 4245 } 4246 } 4247 4248 4249 /* 4250 * Add L1 Data cache data to the ereport payload. 4251 */ 4252 static void 4253 cpu_payload_add_dcache(struct async_flt *aflt, nvlist_t *nvl) 4254 { 4255 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4256 ch_dc_data_t *dcp; 4257 ch_dc_data_t dcdata[CH_DCACHE_NWAY]; 4258 uint_t nelem; 4259 int i, ways_to_check, ways_logged = 0; 4260 4261 /* 4262 * If this is an D$ fault then there may be multiple 4263 * ways captured in the ch_parity_log_t structure. 4264 * Otherwise, there will be at most one way captured 4265 * in the ch_diag_data_t struct. 4266 * Check each way to see if it should be encoded. 4267 */ 4268 if (ch_flt->flt_type == CPU_DC_PARITY) 4269 ways_to_check = CH_DCACHE_NWAY; 4270 else 4271 ways_to_check = 1; 4272 for (i = 0; i < ways_to_check; i++) { 4273 if (ch_flt->flt_type == CPU_DC_PARITY) 4274 dcp = &ch_flt->parity_data.dpe.cpl_dc[i]; 4275 else 4276 dcp = &ch_flt->flt_diag_data.chd_dc_data; 4277 if (dcp->dc_logflag == DC_LOGFLAG_MAGIC) { 4278 bcopy(dcp, &dcdata[ways_logged], 4279 sizeof (ch_dc_data_t)); 4280 ways_logged++; 4281 } 4282 } 4283 4284 /* 4285 * Add the dcache data to the payload. 4286 */ 4287 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_WAYS, 4288 DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 4289 if (ways_logged != 0) { 4290 nelem = sizeof (ch_dc_data_t) / sizeof (uint64_t) * ways_logged; 4291 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_DATA, 4292 DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)dcdata, NULL); 4293 } 4294 } 4295 4296 /* 4297 * Add L1 Instruction cache data to the ereport payload. 4298 */ 4299 static void 4300 cpu_payload_add_icache(struct async_flt *aflt, nvlist_t *nvl) 4301 { 4302 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4303 ch_ic_data_t *icp; 4304 ch_ic_data_t icdata[CH_ICACHE_NWAY]; 4305 uint_t nelem; 4306 int i, ways_to_check, ways_logged = 0; 4307 4308 /* 4309 * If this is an I$ fault then there may be multiple 4310 * ways captured in the ch_parity_log_t structure. 4311 * Otherwise, there will be at most one way captured 4312 * in the ch_diag_data_t struct. 4313 * Check each way to see if it should be encoded. 4314 */ 4315 if (ch_flt->flt_type == CPU_IC_PARITY) 4316 ways_to_check = CH_ICACHE_NWAY; 4317 else 4318 ways_to_check = 1; 4319 for (i = 0; i < ways_to_check; i++) { 4320 if (ch_flt->flt_type == CPU_IC_PARITY) 4321 icp = &ch_flt->parity_data.ipe.cpl_ic[i]; 4322 else 4323 icp = &ch_flt->flt_diag_data.chd_ic_data; 4324 if (icp->ic_logflag == IC_LOGFLAG_MAGIC) { 4325 bcopy(icp, &icdata[ways_logged], 4326 sizeof (ch_ic_data_t)); 4327 ways_logged++; 4328 } 4329 } 4330 4331 /* 4332 * Add the icache data to the payload. 4333 */ 4334 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_WAYS, 4335 DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 4336 if (ways_logged != 0) { 4337 nelem = sizeof (ch_ic_data_t) / sizeof (uint64_t) * ways_logged; 4338 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_DATA, 4339 DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)icdata, NULL); 4340 } 4341 } 4342 4343 #endif /* CPU_IMP_L1_CACHE_PARITY */ 4344 4345 /* 4346 * Add ecache data to payload. 4347 */ 4348 static void 4349 cpu_payload_add_ecache(struct async_flt *aflt, nvlist_t *nvl) 4350 { 4351 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4352 ch_ec_data_t *ecp; 4353 ch_ec_data_t ecdata[CHD_EC_DATA_SETS]; 4354 uint_t nelem; 4355 int i, ways_logged = 0; 4356 4357 /* 4358 * Check each way to see if it should be encoded 4359 * and concatinate it into a temporary buffer. 4360 */ 4361 for (i = 0; i < CHD_EC_DATA_SETS; i++) { 4362 ecp = &ch_flt->flt_diag_data.chd_ec_data[i]; 4363 if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 4364 bcopy(ecp, &ecdata[ways_logged], 4365 sizeof (ch_ec_data_t)); 4366 ways_logged++; 4367 } 4368 } 4369 4370 /* 4371 * Panther CPUs have an additional level of cache and so 4372 * what we just collected was the L3 (ecache) and not the 4373 * L2 cache. 4374 */ 4375 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 4376 /* 4377 * Add the L3 (ecache) data to the payload. 4378 */ 4379 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_WAYS, 4380 DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 4381 if (ways_logged != 0) { 4382 nelem = sizeof (ch_ec_data_t) / 4383 sizeof (uint64_t) * ways_logged; 4384 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_DATA, 4385 DATA_TYPE_UINT64_ARRAY, nelem, 4386 (uint64_t *)ecdata, NULL); 4387 } 4388 4389 /* 4390 * Now collect the L2 cache. 4391 */ 4392 ways_logged = 0; 4393 for (i = 0; i < PN_L2_NWAYS; i++) { 4394 ecp = &ch_flt->flt_diag_data.chd_l2_data[i]; 4395 if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 4396 bcopy(ecp, &ecdata[ways_logged], 4397 sizeof (ch_ec_data_t)); 4398 ways_logged++; 4399 } 4400 } 4401 } 4402 4403 /* 4404 * Add the L2 cache data to the payload. 4405 */ 4406 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_WAYS, 4407 DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 4408 if (ways_logged != 0) { 4409 nelem = sizeof (ch_ec_data_t) / 4410 sizeof (uint64_t) * ways_logged; 4411 fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_DATA, 4412 DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)ecdata, NULL); 4413 } 4414 } 4415 4416 /* 4417 * Initialize cpu scheme for specified cpu. 4418 */ 4419 static void 4420 cpu_fmri_cpu_set(nvlist_t *cpu_fmri, int cpuid) 4421 { 4422 char sbuf[21]; /* sizeof (UINT64_MAX) + '\0' */ 4423 uint8_t mask; 4424 4425 mask = cpunodes[cpuid].version; 4426 (void) snprintf(sbuf, sizeof (sbuf), "%llX", 4427 (u_longlong_t)cpunodes[cpuid].device_id); 4428 (void) fm_fmri_cpu_set(cpu_fmri, FM_CPU_SCHEME_VERSION, NULL, 4429 cpuid, &mask, (const char *)sbuf); 4430 } 4431 4432 /* 4433 * Returns ereport resource type. 4434 */ 4435 static int 4436 cpu_error_to_resource_type(struct async_flt *aflt) 4437 { 4438 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4439 4440 switch (ch_flt->flt_type) { 4441 4442 case CPU_CE_ECACHE: 4443 case CPU_UE_ECACHE: 4444 case CPU_UE_ECACHE_RETIRE: 4445 case CPU_ORPH: 4446 /* 4447 * If AFSR error bit indicates L2$ Data for Cheetah, 4448 * Cheetah+ or Jaguar, or L3$ Data for Panther, return 4449 * E$ Data type, otherwise, return CPU type. 4450 */ 4451 if (cpu_error_is_ecache_data(aflt->flt_inst, 4452 ch_flt->flt_bit)) 4453 return (ERRTYPE_ECACHE_DATA); 4454 return (ERRTYPE_CPU); 4455 4456 case CPU_CE: 4457 case CPU_UE: 4458 case CPU_EMC: 4459 case CPU_DUE: 4460 case CPU_RCE: 4461 case CPU_RUE: 4462 case CPU_FRC: 4463 case CPU_FRU: 4464 return (ERRTYPE_MEMORY); 4465 4466 case CPU_IC_PARITY: 4467 case CPU_DC_PARITY: 4468 case CPU_FPUERR: 4469 case CPU_PC_PARITY: 4470 case CPU_ITLB_PARITY: 4471 case CPU_DTLB_PARITY: 4472 return (ERRTYPE_CPU); 4473 } 4474 return (ERRTYPE_UNKNOWN); 4475 } 4476 4477 /* 4478 * Encode the data saved in the ch_async_flt_t struct into 4479 * the FM ereport payload. 4480 */ 4481 static void 4482 cpu_payload_add_aflt(struct async_flt *aflt, nvlist_t *payload, 4483 nvlist_t *resource, int *afar_status, int *synd_status) 4484 { 4485 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4486 *synd_status = AFLT_STAT_INVALID; 4487 *afar_status = AFLT_STAT_INVALID; 4488 4489 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR) { 4490 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR, 4491 DATA_TYPE_UINT64, aflt->flt_stat, NULL); 4492 } 4493 4494 if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR_EXT) && 4495 IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 4496 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR_EXT, 4497 DATA_TYPE_UINT64, ch_flt->afsr_ext, NULL); 4498 } 4499 4500 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR_STATUS) { 4501 *afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 4502 ch_flt->flt_bit); 4503 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR_STATUS, 4504 DATA_TYPE_UINT8, (uint8_t)*afar_status, NULL); 4505 } 4506 4507 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR) { 4508 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR, 4509 DATA_TYPE_UINT64, aflt->flt_addr, NULL); 4510 } 4511 4512 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PC) { 4513 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PC, 4514 DATA_TYPE_UINT64, (uint64_t)aflt->flt_pc, NULL); 4515 } 4516 4517 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TL) { 4518 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TL, 4519 DATA_TYPE_UINT8, (uint8_t)aflt->flt_tl, NULL); 4520 } 4521 4522 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TT) { 4523 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TT, 4524 DATA_TYPE_UINT8, flt_to_trap_type(aflt), NULL); 4525 } 4526 4527 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PRIV) { 4528 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PRIV, 4529 DATA_TYPE_BOOLEAN_VALUE, 4530 (aflt->flt_priv ? B_TRUE : B_FALSE), NULL); 4531 } 4532 4533 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ME) { 4534 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ME, 4535 DATA_TYPE_BOOLEAN_VALUE, 4536 (aflt->flt_stat & C_AFSR_ME) ? B_TRUE : B_FALSE, NULL); 4537 } 4538 4539 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND_STATUS) { 4540 *synd_status = afsr_to_synd_status(aflt->flt_inst, 4541 ch_flt->afsr_errs, ch_flt->flt_bit); 4542 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND_STATUS, 4543 DATA_TYPE_UINT8, (uint8_t)*synd_status, NULL); 4544 } 4545 4546 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND) { 4547 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND, 4548 DATA_TYPE_UINT16, (uint16_t)aflt->flt_synd, NULL); 4549 } 4550 4551 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_TYPE) { 4552 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_TYPE, 4553 DATA_TYPE_STRING, flt_to_error_type(aflt), NULL); 4554 } 4555 4556 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_DISP) { 4557 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_DISP, 4558 DATA_TYPE_UINT64, aflt->flt_disp, NULL); 4559 } 4560 4561 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L2) 4562 cpu_payload_add_ecache(aflt, payload); 4563 4564 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_COPYFUNCTION) { 4565 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_COPYFUNCTION, 4566 DATA_TYPE_UINT8, (uint8_t)aflt->flt_status & 0xff, NULL); 4567 } 4568 4569 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_HOWDETECTED) { 4570 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_HOWDETECTED, 4571 DATA_TYPE_UINT8, (uint8_t)(aflt->flt_status >> 8), NULL); 4572 } 4573 4574 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_INSTRBLOCK) { 4575 fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_INSTRBLOCK, 4576 DATA_TYPE_UINT32_ARRAY, 16, 4577 (uint32_t *)&ch_flt->flt_fpdata, NULL); 4578 } 4579 4580 #if defined(CPU_IMP_L1_CACHE_PARITY) 4581 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1D) 4582 cpu_payload_add_dcache(aflt, payload); 4583 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1I) 4584 cpu_payload_add_icache(aflt, payload); 4585 #endif /* CPU_IMP_L1_CACHE_PARITY */ 4586 4587 #if defined(CHEETAH_PLUS) 4588 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1P) 4589 cpu_payload_add_pcache(aflt, payload); 4590 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_TLB) 4591 cpu_payload_add_tlb(aflt, payload); 4592 #endif /* CHEETAH_PLUS */ 4593 /* 4594 * Create the FMRI that goes into the payload 4595 * and contains the unum info if necessary. 4596 */ 4597 if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_RESOURCE) { 4598 char unum[UNUM_NAMLEN] = ""; 4599 char sid[DIMM_SERIAL_ID_LEN] = ""; 4600 int len, ret, rtype, synd_code; 4601 uint64_t offset = (uint64_t)-1; 4602 4603 rtype = cpu_error_to_resource_type(aflt); 4604 switch (rtype) { 4605 4606 case ERRTYPE_MEMORY: 4607 case ERRTYPE_ECACHE_DATA: 4608 4609 /* 4610 * Memory errors, do unum lookup 4611 */ 4612 if (*afar_status == AFLT_STAT_INVALID) 4613 break; 4614 4615 if (rtype == ERRTYPE_ECACHE_DATA) 4616 aflt->flt_status |= ECC_ECACHE; 4617 else 4618 aflt->flt_status &= ~ECC_ECACHE; 4619 4620 synd_code = synd_to_synd_code(*synd_status, 4621 aflt->flt_synd, ch_flt->flt_bit); 4622 4623 if (cpu_get_mem_unum_synd(synd_code, aflt, unum) != 0) 4624 break; 4625 4626 ret = cpu_get_mem_sid(unum, sid, DIMM_SERIAL_ID_LEN, 4627 &len); 4628 4629 if (ret == 0) { 4630 (void) cpu_get_mem_offset(aflt->flt_addr, 4631 &offset); 4632 } 4633 4634 fm_fmri_mem_set(resource, FM_MEM_SCHEME_VERSION, 4635 NULL, unum, (ret == 0) ? sid : NULL, offset); 4636 fm_payload_set(payload, 4637 FM_EREPORT_PAYLOAD_NAME_RESOURCE, 4638 DATA_TYPE_NVLIST, resource, NULL); 4639 break; 4640 4641 case ERRTYPE_CPU: 4642 /* 4643 * On-board processor array error, add cpu resource. 4644 */ 4645 cpu_fmri_cpu_set(resource, aflt->flt_inst); 4646 fm_payload_set(payload, 4647 FM_EREPORT_PAYLOAD_NAME_RESOURCE, 4648 DATA_TYPE_NVLIST, resource, NULL); 4649 break; 4650 } 4651 } 4652 } 4653 4654 /* 4655 * Initialize the way info if necessary. 4656 */ 4657 void 4658 cpu_ereport_init(struct async_flt *aflt) 4659 { 4660 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 4661 ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 4662 ch_ec_data_t *l2p = &ch_flt->flt_diag_data.chd_l2_data[0]; 4663 int i; 4664 4665 /* 4666 * Initialize the info in the CPU logout structure. 4667 * The I$/D$ way information is not initialized here 4668 * since it is captured in the logout assembly code. 4669 */ 4670 for (i = 0; i < CHD_EC_DATA_SETS; i++) 4671 (ecp + i)->ec_way = i; 4672 4673 for (i = 0; i < PN_L2_NWAYS; i++) 4674 (l2p + i)->ec_way = i; 4675 } 4676 4677 /* 4678 * Returns whether fault address is valid for this error bit and 4679 * whether the address is "in memory" (i.e. pf_is_memory returns 1). 4680 */ 4681 int 4682 cpu_flt_in_memory(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit) 4683 { 4684 struct async_flt *aflt = (struct async_flt *)ch_flt; 4685 4686 return ((t_afsr_bit & C_AFSR_MEMORY) && 4687 afsr_to_afar_status(ch_flt->afsr_errs, t_afsr_bit) == 4688 AFLT_STAT_VALID && 4689 pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)); 4690 } 4691 4692 /* 4693 * Returns whether fault address is valid based on the error bit for the 4694 * one event being queued and whether the address is "in memory". 4695 */ 4696 static int 4697 cpu_flt_in_memory_one_event(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit) 4698 { 4699 struct async_flt *aflt = (struct async_flt *)ch_flt; 4700 int afar_status; 4701 uint64_t afsr_errs, afsr_ow, *ow_bits; 4702 4703 if (!(t_afsr_bit & C_AFSR_MEMORY) || 4704 !pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)) 4705 return (0); 4706 4707 afsr_errs = ch_flt->afsr_errs; 4708 afar_status = afsr_to_afar_status(afsr_errs, t_afsr_bit); 4709 4710 switch (afar_status) { 4711 case AFLT_STAT_VALID: 4712 return (1); 4713 4714 case AFLT_STAT_AMBIGUOUS: 4715 /* 4716 * Status is ambiguous since another error bit (or bits) 4717 * of equal priority to the specified bit on in the afsr, 4718 * so check those bits. Return 1 only if the bits on in the 4719 * same class as the t_afsr_bit are also C_AFSR_MEMORY bits. 4720 * Otherwise not all the equal priority bits are for memory 4721 * errors, so return 0. 4722 */ 4723 ow_bits = afar_overwrite; 4724 while ((afsr_ow = *ow_bits++) != 0) { 4725 /* 4726 * Get other bits that are on in t_afsr_bit's priority 4727 * class to check for Memory Error bits only. 4728 */ 4729 if (afsr_ow & t_afsr_bit) { 4730 if ((afsr_errs & afsr_ow) & ~C_AFSR_MEMORY) 4731 return (0); 4732 else 4733 return (1); 4734 } 4735 } 4736 /*FALLTHRU*/ 4737 4738 default: 4739 return (0); 4740 } 4741 } 4742 4743 static void 4744 cpu_log_diag_info(ch_async_flt_t *ch_flt) 4745 { 4746 struct async_flt *aflt = (struct async_flt *)ch_flt; 4747 ch_dc_data_t *dcp = &ch_flt->flt_diag_data.chd_dc_data; 4748 ch_ic_data_t *icp = &ch_flt->flt_diag_data.chd_ic_data; 4749 ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 4750 #if defined(CPU_IMP_ECACHE_ASSOC) 4751 int i, nway; 4752 #endif /* CPU_IMP_ECACHE_ASSOC */ 4753 4754 /* 4755 * Check if the CPU log out captured was valid. 4756 */ 4757 if (ch_flt->flt_diag_data.chd_afar == LOGOUT_INVALID || 4758 ch_flt->flt_data_incomplete) 4759 return; 4760 4761 #if defined(CPU_IMP_ECACHE_ASSOC) 4762 nway = cpu_ecache_nway(); 4763 i = cpu_ecache_line_valid(ch_flt); 4764 if (i == 0 || i > nway) { 4765 for (i = 0; i < nway; i++) 4766 ecp[i].ec_logflag = EC_LOGFLAG_MAGIC; 4767 } else 4768 ecp[i - 1].ec_logflag = EC_LOGFLAG_MAGIC; 4769 #else /* CPU_IMP_ECACHE_ASSOC */ 4770 ecp->ec_logflag = EC_LOGFLAG_MAGIC; 4771 #endif /* CPU_IMP_ECACHE_ASSOC */ 4772 4773 #if defined(CHEETAH_PLUS) 4774 pn_cpu_log_diag_l2_info(ch_flt); 4775 #endif /* CHEETAH_PLUS */ 4776 4777 if (CH_DCTAG_MATCH(dcp->dc_tag, aflt->flt_addr)) { 4778 dcp->dc_way = CH_DCIDX_TO_WAY(dcp->dc_idx); 4779 dcp->dc_logflag = DC_LOGFLAG_MAGIC; 4780 } 4781 4782 if (CH_ICTAG_MATCH(icp, aflt->flt_addr)) { 4783 if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 4784 icp->ic_way = PN_ICIDX_TO_WAY(icp->ic_idx); 4785 else 4786 icp->ic_way = CH_ICIDX_TO_WAY(icp->ic_idx); 4787 icp->ic_logflag = IC_LOGFLAG_MAGIC; 4788 } 4789 } 4790 4791 /* 4792 * Cheetah ECC calculation. 4793 * 4794 * We only need to do the calculation on the data bits and can ignore check 4795 * bit and Mtag bit terms in the calculation. 4796 */ 4797 static uint64_t ch_ecc_table[9][2] = { 4798 /* 4799 * low order 64-bits high-order 64-bits 4800 */ 4801 { 0x46bffffeccd1177f, 0x488800022100014c }, 4802 { 0x42fccc81331ff77f, 0x14424f1010249184 }, 4803 { 0x8898827c222f1ffe, 0x22c1222808184aaf }, 4804 { 0xf7632203e131ccf1, 0xe1241121848292b8 }, 4805 { 0x7f5511421b113809, 0x901c88d84288aafe }, 4806 { 0x1d49412184882487, 0x8f338c87c044c6ef }, 4807 { 0xf552181014448344, 0x7ff8f4443e411911 }, 4808 { 0x2189240808f24228, 0xfeeff8cc81333f42 }, 4809 { 0x3280008440001112, 0xfee88b337ffffd62 }, 4810 }; 4811 4812 /* 4813 * 64-bit population count, use well-known popcnt trick. 4814 * We could use the UltraSPARC V9 POPC instruction, but some 4815 * CPUs including Cheetahplus and Jaguar do not support that 4816 * instruction. 4817 */ 4818 int 4819 popc64(uint64_t val) 4820 { 4821 int cnt; 4822 4823 for (cnt = 0; val != 0; val &= val - 1) 4824 cnt++; 4825 return (cnt); 4826 } 4827 4828 /* 4829 * Generate the 9 ECC bits for the 128-bit chunk based on the table above. 4830 * Note that xor'ing an odd number of 1 bits == 1 and xor'ing an even number 4831 * of 1 bits == 0, so we can just use the least significant bit of the popcnt 4832 * instead of doing all the xor's. 4833 */ 4834 uint32_t 4835 us3_gen_ecc(uint64_t data_low, uint64_t data_high) 4836 { 4837 int bitno, s; 4838 int synd = 0; 4839 4840 for (bitno = 0; bitno < 9; bitno++) { 4841 s = (popc64(data_low & ch_ecc_table[bitno][0]) + 4842 popc64(data_high & ch_ecc_table[bitno][1])) & 1; 4843 synd |= (s << bitno); 4844 } 4845 return (synd); 4846 4847 } 4848 4849 /* 4850 * Queue one event based on ecc_type_to_info entry. If the event has an AFT1 4851 * tag associated with it or is a fatal event (aflt_panic set), it is sent to 4852 * the UE event queue. Otherwise it is dispatched to the CE event queue. 4853 */ 4854 static void 4855 cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 4856 ecc_type_to_info_t *eccp, ch_diag_data_t *cdp) 4857 { 4858 struct async_flt *aflt = (struct async_flt *)ch_flt; 4859 4860 if (reason && 4861 strlen(reason) + strlen(eccp->ec_reason) < MAX_REASON_STRING) { 4862 (void) strcat(reason, eccp->ec_reason); 4863 } 4864 4865 ch_flt->flt_bit = eccp->ec_afsr_bit; 4866 ch_flt->flt_type = eccp->ec_flt_type; 4867 if (cdp != NULL && cdp->chd_afar != LOGOUT_INVALID) 4868 ch_flt->flt_diag_data = *cdp; 4869 else 4870 ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 4871 aflt->flt_in_memory = 4872 cpu_flt_in_memory_one_event(ch_flt, ch_flt->flt_bit); 4873 4874 if (ch_flt->flt_bit & C_AFSR_MSYND_ERRS) 4875 aflt->flt_synd = GET_M_SYND(aflt->flt_stat); 4876 else if (ch_flt->flt_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) 4877 aflt->flt_synd = GET_E_SYND(aflt->flt_stat); 4878 else 4879 aflt->flt_synd = 0; 4880 4881 aflt->flt_payload = eccp->ec_err_payload; 4882 4883 if (aflt->flt_panic || (eccp->ec_afsr_bit & 4884 (C_AFSR_LEVEL1 | C_AFSR_EXT_LEVEL1))) 4885 cpu_errorq_dispatch(eccp->ec_err_class, 4886 (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 4887 aflt->flt_panic); 4888 else 4889 cpu_errorq_dispatch(eccp->ec_err_class, 4890 (void *)ch_flt, sizeof (ch_async_flt_t), ce_queue, 4891 aflt->flt_panic); 4892 } 4893 4894 /* 4895 * Queue events on async event queue one event per error bit. First we 4896 * queue the events that we "expect" for the given trap, then we queue events 4897 * that we may not expect. Return number of events queued. 4898 */ 4899 int 4900 cpu_queue_events(ch_async_flt_t *ch_flt, char *reason, uint64_t t_afsr_errs, 4901 ch_cpu_logout_t *clop) 4902 { 4903 struct async_flt *aflt = (struct async_flt *)ch_flt; 4904 ecc_type_to_info_t *eccp; 4905 int nevents = 0; 4906 uint64_t primary_afar = aflt->flt_addr, primary_afsr = aflt->flt_stat; 4907 #if defined(CHEETAH_PLUS) 4908 uint64_t orig_t_afsr_errs; 4909 #endif 4910 uint64_t primary_afsr_ext = ch_flt->afsr_ext; 4911 uint64_t primary_afsr_errs = ch_flt->afsr_errs; 4912 ch_diag_data_t *cdp = NULL; 4913 4914 t_afsr_errs &= ((C_AFSR_ALL_ERRS & ~C_AFSR_ME) | C_AFSR_EXT_ALL_ERRS); 4915 4916 #if defined(CHEETAH_PLUS) 4917 orig_t_afsr_errs = t_afsr_errs; 4918 4919 /* 4920 * For Cheetah+, log the shadow AFSR/AFAR bits first. 4921 */ 4922 if (clop != NULL) { 4923 /* 4924 * Set the AFSR and AFAR fields to the shadow registers. The 4925 * flt_addr and flt_stat fields will be reset to the primaries 4926 * below, but the sdw_addr and sdw_stat will stay as the 4927 * secondaries. 4928 */ 4929 cdp = &clop->clo_sdw_data; 4930 aflt->flt_addr = ch_flt->flt_sdw_afar = cdp->chd_afar; 4931 aflt->flt_stat = ch_flt->flt_sdw_afsr = cdp->chd_afsr; 4932 ch_flt->afsr_ext = ch_flt->flt_sdw_afsr_ext = cdp->chd_afsr_ext; 4933 ch_flt->afsr_errs = (cdp->chd_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 4934 (cdp->chd_afsr & C_AFSR_ALL_ERRS); 4935 4936 /* 4937 * If the primary and shadow AFSR differ, tag the shadow as 4938 * the first fault. 4939 */ 4940 if ((primary_afar != cdp->chd_afar) || 4941 (primary_afsr_errs != ch_flt->afsr_errs)) { 4942 aflt->flt_stat |= (1ull << C_AFSR_FIRSTFLT_SHIFT); 4943 } 4944 4945 /* 4946 * Check AFSR bits as well as AFSR_EXT bits in order of 4947 * the AFAR overwrite priority. Our stored AFSR_EXT value 4948 * is expected to be zero for those CPUs which do not have 4949 * an AFSR_EXT register. 4950 */ 4951 for (eccp = ecc_type_to_info; eccp->ec_desc != NULL; eccp++) { 4952 if ((eccp->ec_afsr_bit & 4953 (ch_flt->afsr_errs & t_afsr_errs)) && 4954 ((eccp->ec_flags & aflt->flt_status) != 0)) { 4955 cpu_queue_one_event(ch_flt, reason, eccp, cdp); 4956 cdp = NULL; 4957 t_afsr_errs &= ~eccp->ec_afsr_bit; 4958 nevents++; 4959 } 4960 } 4961 4962 /* 4963 * If the ME bit is on in the primary AFSR turn all the 4964 * error bits on again that may set the ME bit to make 4965 * sure we see the ME AFSR error logs. 4966 */ 4967 if ((primary_afsr & C_AFSR_ME) != 0) 4968 t_afsr_errs = (orig_t_afsr_errs & C_AFSR_ALL_ME_ERRS); 4969 } 4970 #endif /* CHEETAH_PLUS */ 4971 4972 if (clop != NULL) 4973 cdp = &clop->clo_data; 4974 4975 /* 4976 * Queue expected errors, error bit and fault type must match 4977 * in the ecc_type_to_info table. 4978 */ 4979 for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 4980 eccp++) { 4981 if ((eccp->ec_afsr_bit & t_afsr_errs) != 0 && 4982 (eccp->ec_flags & aflt->flt_status) != 0) { 4983 #if defined(SERRANO) 4984 /* 4985 * For FRC/FRU errors on Serrano the afar2 captures 4986 * the address and the associated data is 4987 * in the shadow logout area. 4988 */ 4989 if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 4990 if (clop != NULL) 4991 cdp = &clop->clo_sdw_data; 4992 aflt->flt_addr = ch_flt->afar2; 4993 } else { 4994 if (clop != NULL) 4995 cdp = &clop->clo_data; 4996 aflt->flt_addr = primary_afar; 4997 } 4998 #else /* SERRANO */ 4999 aflt->flt_addr = primary_afar; 5000 #endif /* SERRANO */ 5001 aflt->flt_stat = primary_afsr; 5002 ch_flt->afsr_ext = primary_afsr_ext; 5003 ch_flt->afsr_errs = primary_afsr_errs; 5004 cpu_queue_one_event(ch_flt, reason, eccp, cdp); 5005 cdp = NULL; 5006 t_afsr_errs &= ~eccp->ec_afsr_bit; 5007 nevents++; 5008 } 5009 } 5010 5011 /* 5012 * Queue unexpected errors, error bit only match. 5013 */ 5014 for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 5015 eccp++) { 5016 if (eccp->ec_afsr_bit & t_afsr_errs) { 5017 #if defined(SERRANO) 5018 /* 5019 * For FRC/FRU errors on Serrano the afar2 captures 5020 * the address and the associated data is 5021 * in the shadow logout area. 5022 */ 5023 if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 5024 if (clop != NULL) 5025 cdp = &clop->clo_sdw_data; 5026 aflt->flt_addr = ch_flt->afar2; 5027 } else { 5028 if (clop != NULL) 5029 cdp = &clop->clo_data; 5030 aflt->flt_addr = primary_afar; 5031 } 5032 #else /* SERRANO */ 5033 aflt->flt_addr = primary_afar; 5034 #endif /* SERRANO */ 5035 aflt->flt_stat = primary_afsr; 5036 ch_flt->afsr_ext = primary_afsr_ext; 5037 ch_flt->afsr_errs = primary_afsr_errs; 5038 cpu_queue_one_event(ch_flt, reason, eccp, cdp); 5039 cdp = NULL; 5040 t_afsr_errs &= ~eccp->ec_afsr_bit; 5041 nevents++; 5042 } 5043 } 5044 return (nevents); 5045 } 5046 5047 /* 5048 * Return trap type number. 5049 */ 5050 uint8_t 5051 flt_to_trap_type(struct async_flt *aflt) 5052 { 5053 if (aflt->flt_status & ECC_I_TRAP) 5054 return (TRAP_TYPE_ECC_I); 5055 if (aflt->flt_status & ECC_D_TRAP) 5056 return (TRAP_TYPE_ECC_D); 5057 if (aflt->flt_status & ECC_F_TRAP) 5058 return (TRAP_TYPE_ECC_F); 5059 if (aflt->flt_status & ECC_C_TRAP) 5060 return (TRAP_TYPE_ECC_C); 5061 if (aflt->flt_status & ECC_DP_TRAP) 5062 return (TRAP_TYPE_ECC_DP); 5063 if (aflt->flt_status & ECC_IP_TRAP) 5064 return (TRAP_TYPE_ECC_IP); 5065 if (aflt->flt_status & ECC_ITLB_TRAP) 5066 return (TRAP_TYPE_ECC_ITLB); 5067 if (aflt->flt_status & ECC_DTLB_TRAP) 5068 return (TRAP_TYPE_ECC_DTLB); 5069 return (TRAP_TYPE_UNKNOWN); 5070 } 5071 5072 /* 5073 * Decide an error type based on detector and leaky/partner tests. 5074 * The following array is used for quick translation - it must 5075 * stay in sync with ce_dispact_t. 5076 */ 5077 5078 static char *cetypes[] = { 5079 CE_DISP_DESC_U, 5080 CE_DISP_DESC_I, 5081 CE_DISP_DESC_PP, 5082 CE_DISP_DESC_P, 5083 CE_DISP_DESC_L, 5084 CE_DISP_DESC_PS, 5085 CE_DISP_DESC_S 5086 }; 5087 5088 char * 5089 flt_to_error_type(struct async_flt *aflt) 5090 { 5091 ce_dispact_t dispact, disp; 5092 uchar_t dtcrinfo, ptnrinfo, lkyinfo; 5093 5094 /* 5095 * The memory payload bundle is shared by some events that do 5096 * not perform any classification. For those flt_disp will be 5097 * 0 and we will return "unknown". 5098 */ 5099 if (!ce_disp_inited || !aflt->flt_in_memory || aflt->flt_disp == 0) 5100 return (cetypes[CE_DISP_UNKNOWN]); 5101 5102 dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 5103 5104 /* 5105 * It is also possible that no scrub/classification was performed 5106 * by the detector, for instance where a disrupting error logged 5107 * in the AFSR while CEEN was off in cpu_deferred_error. 5108 */ 5109 if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) 5110 return (cetypes[CE_DISP_UNKNOWN]); 5111 5112 /* 5113 * Lookup type in initial classification/action table 5114 */ 5115 dispact = CE_DISPACT(ce_disp_table, 5116 CE_XDIAG_AFARMATCHED(dtcrinfo), 5117 CE_XDIAG_STATE(dtcrinfo), 5118 CE_XDIAG_CE1SEEN(dtcrinfo), 5119 CE_XDIAG_CE2SEEN(dtcrinfo)); 5120 5121 /* 5122 * A bad lookup is not something to panic production systems for. 5123 */ 5124 ASSERT(dispact != CE_DISP_BAD); 5125 if (dispact == CE_DISP_BAD) 5126 return (cetypes[CE_DISP_UNKNOWN]); 5127 5128 disp = CE_DISP(dispact); 5129 5130 switch (disp) { 5131 case CE_DISP_UNKNOWN: 5132 case CE_DISP_INTERMITTENT: 5133 break; 5134 5135 case CE_DISP_POSS_PERS: 5136 /* 5137 * "Possible persistent" errors to which we have applied a valid 5138 * leaky test can be separated into "persistent" or "leaky". 5139 */ 5140 lkyinfo = CE_XDIAG_LKYINFO(aflt->flt_disp); 5141 if (CE_XDIAG_TESTVALID(lkyinfo)) { 5142 if (CE_XDIAG_CE1SEEN(lkyinfo) || 5143 CE_XDIAG_CE2SEEN(lkyinfo)) 5144 disp = CE_DISP_LEAKY; 5145 else 5146 disp = CE_DISP_PERS; 5147 } 5148 break; 5149 5150 case CE_DISP_POSS_STICKY: 5151 /* 5152 * Promote "possible sticky" results that have been 5153 * confirmed by a partner test to "sticky". Unconfirmed 5154 * "possible sticky" events are left at that status - we do not 5155 * guess at any bad reader/writer etc status here. 5156 */ 5157 ptnrinfo = CE_XDIAG_PTNRINFO(aflt->flt_disp); 5158 if (CE_XDIAG_TESTVALID(ptnrinfo) && 5159 CE_XDIAG_CE1SEEN(ptnrinfo) && CE_XDIAG_CE2SEEN(ptnrinfo)) 5160 disp = CE_DISP_STICKY; 5161 5162 /* 5163 * Promote "possible sticky" results on a uniprocessor 5164 * to "sticky" 5165 */ 5166 if (disp == CE_DISP_POSS_STICKY && 5167 CE_XDIAG_SKIPCODE(disp) == CE_XDIAG_SKIP_UNIPROC) 5168 disp = CE_DISP_STICKY; 5169 break; 5170 5171 default: 5172 disp = CE_DISP_UNKNOWN; 5173 break; 5174 } 5175 5176 return (cetypes[disp]); 5177 } 5178 5179 /* 5180 * Given the entire afsr, the specific bit to check and a prioritized list of 5181 * error bits, determine the validity of the various overwrite priority 5182 * features of the AFSR/AFAR: AFAR, ESYND and MSYND, each of which have 5183 * different overwrite priorities. 5184 * 5185 * Given a specific afsr error bit and the entire afsr, there are three cases: 5186 * INVALID: The specified bit is lower overwrite priority than some other 5187 * error bit which is on in the afsr (or IVU/IVC). 5188 * VALID: The specified bit is higher priority than all other error bits 5189 * which are on in the afsr. 5190 * AMBIGUOUS: Another error bit (or bits) of equal priority to the specified 5191 * bit is on in the afsr. 5192 */ 5193 int 5194 afsr_to_overw_status(uint64_t afsr, uint64_t afsr_bit, uint64_t *ow_bits) 5195 { 5196 uint64_t afsr_ow; 5197 5198 while ((afsr_ow = *ow_bits++) != 0) { 5199 /* 5200 * If bit is in the priority class, check to see if another 5201 * bit in the same class is on => ambiguous. Otherwise, 5202 * the value is valid. If the bit is not on at this priority 5203 * class, but a higher priority bit is on, then the value is 5204 * invalid. 5205 */ 5206 if (afsr_ow & afsr_bit) { 5207 /* 5208 * If equal pri bit is on, ambiguous. 5209 */ 5210 if (afsr & (afsr_ow & ~afsr_bit)) 5211 return (AFLT_STAT_AMBIGUOUS); 5212 return (AFLT_STAT_VALID); 5213 } else if (afsr & afsr_ow) 5214 break; 5215 } 5216 5217 /* 5218 * We didn't find a match or a higher priority bit was on. Not 5219 * finding a match handles the case of invalid AFAR for IVC, IVU. 5220 */ 5221 return (AFLT_STAT_INVALID); 5222 } 5223 5224 static int 5225 afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit) 5226 { 5227 #if defined(SERRANO) 5228 if (afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) 5229 return (afsr_to_overw_status(afsr, afsr_bit, afar2_overwrite)); 5230 else 5231 #endif /* SERRANO */ 5232 return (afsr_to_overw_status(afsr, afsr_bit, afar_overwrite)); 5233 } 5234 5235 static int 5236 afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit) 5237 { 5238 return (afsr_to_overw_status(afsr, afsr_bit, esynd_overwrite)); 5239 } 5240 5241 static int 5242 afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit) 5243 { 5244 return (afsr_to_overw_status(afsr, afsr_bit, msynd_overwrite)); 5245 } 5246 5247 static int 5248 afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit) 5249 { 5250 #ifdef lint 5251 cpuid = cpuid; 5252 #endif 5253 #if defined(CHEETAH_PLUS) 5254 /* 5255 * The M_SYND overwrite policy is combined with the E_SYND overwrite 5256 * policy for Cheetah+ and separate for Panther CPUs. 5257 */ 5258 if (afsr_bit & C_AFSR_MSYND_ERRS) { 5259 if (IS_PANTHER(cpunodes[cpuid].implementation)) 5260 return (afsr_to_msynd_status(afsr, afsr_bit)); 5261 else 5262 return (afsr_to_esynd_status(afsr, afsr_bit)); 5263 } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 5264 if (IS_PANTHER(cpunodes[cpuid].implementation)) 5265 return (afsr_to_pn_esynd_status(afsr, afsr_bit)); 5266 else 5267 return (afsr_to_esynd_status(afsr, afsr_bit)); 5268 #else /* CHEETAH_PLUS */ 5269 if (afsr_bit & C_AFSR_MSYND_ERRS) { 5270 return (afsr_to_msynd_status(afsr, afsr_bit)); 5271 } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 5272 return (afsr_to_esynd_status(afsr, afsr_bit)); 5273 #endif /* CHEETAH_PLUS */ 5274 } else { 5275 return (AFLT_STAT_INVALID); 5276 } 5277 } 5278 5279 /* 5280 * Slave CPU stick synchronization. 5281 */ 5282 void 5283 sticksync_slave(void) 5284 { 5285 int i; 5286 int tries = 0; 5287 int64_t tskew; 5288 int64_t av_tskew; 5289 5290 kpreempt_disable(); 5291 /* wait for the master side */ 5292 while (stick_sync_cmd != SLAVE_START) 5293 ; 5294 /* 5295 * Synchronization should only take a few tries at most. But in the 5296 * odd case where the cpu isn't cooperating we'll keep trying. A cpu 5297 * without it's stick synchronized wouldn't be a good citizen. 5298 */ 5299 while (slave_done == 0) { 5300 /* 5301 * Time skew calculation. 5302 */ 5303 av_tskew = tskew = 0; 5304 5305 for (i = 0; i < stick_iter; i++) { 5306 /* make location hot */ 5307 timestamp[EV_A_START] = 0; 5308 stick_timestamp(×tamp[EV_A_START]); 5309 5310 /* tell the master we're ready */ 5311 stick_sync_cmd = MASTER_START; 5312 5313 /* and wait */ 5314 while (stick_sync_cmd != SLAVE_CONT) 5315 ; 5316 /* Event B end */ 5317 stick_timestamp(×tamp[EV_B_END]); 5318 5319 /* calculate time skew */ 5320 tskew = ((timestamp[EV_B_END] - timestamp[EV_B_START]) 5321 - (timestamp[EV_A_END] - 5322 timestamp[EV_A_START])) / 2; 5323 5324 /* keep running count */ 5325 av_tskew += tskew; 5326 } /* for */ 5327 5328 /* 5329 * Adjust stick for time skew if not within the max allowed; 5330 * otherwise we're all done. 5331 */ 5332 if (stick_iter != 0) 5333 av_tskew = av_tskew/stick_iter; 5334 if (ABS(av_tskew) > stick_tsk) { 5335 /* 5336 * If the skew is 1 (the slave's STICK register 5337 * is 1 STICK ahead of the master's), stick_adj 5338 * could fail to adjust the slave's STICK register 5339 * if the STICK read on the slave happens to 5340 * align with the increment of the STICK. 5341 * Therefore, we increment the skew to 2. 5342 */ 5343 if (av_tskew == 1) 5344 av_tskew++; 5345 stick_adj(-av_tskew); 5346 } else 5347 slave_done = 1; 5348 #ifdef DEBUG 5349 if (tries < DSYNC_ATTEMPTS) 5350 stick_sync_stats[CPU->cpu_id].skew_val[tries] = 5351 av_tskew; 5352 ++tries; 5353 #endif /* DEBUG */ 5354 #ifdef lint 5355 tries = tries; 5356 #endif 5357 5358 } /* while */ 5359 5360 /* allow the master to finish */ 5361 stick_sync_cmd = EVENT_NULL; 5362 kpreempt_enable(); 5363 } 5364 5365 /* 5366 * Master CPU side of stick synchronization. 5367 * - timestamp end of Event A 5368 * - timestamp beginning of Event B 5369 */ 5370 void 5371 sticksync_master(void) 5372 { 5373 int i; 5374 5375 kpreempt_disable(); 5376 /* tell the slave we've started */ 5377 slave_done = 0; 5378 stick_sync_cmd = SLAVE_START; 5379 5380 while (slave_done == 0) { 5381 for (i = 0; i < stick_iter; i++) { 5382 /* wait for the slave */ 5383 while (stick_sync_cmd != MASTER_START) 5384 ; 5385 /* Event A end */ 5386 stick_timestamp(×tamp[EV_A_END]); 5387 5388 /* make location hot */ 5389 timestamp[EV_B_START] = 0; 5390 stick_timestamp(×tamp[EV_B_START]); 5391 5392 /* tell the slave to continue */ 5393 stick_sync_cmd = SLAVE_CONT; 5394 } /* for */ 5395 5396 /* wait while slave calculates time skew */ 5397 while (stick_sync_cmd == SLAVE_CONT) 5398 ; 5399 } /* while */ 5400 kpreempt_enable(); 5401 } 5402 5403 /* 5404 * Cheetah/Cheetah+ have disrupting error for copyback's, so we don't need to 5405 * do Spitfire hack of xcall'ing all the cpus to ask to check for them. Also, 5406 * in cpu_async_panic_callb, each cpu checks for CPU events on its way to 5407 * panic idle. 5408 */ 5409 /*ARGSUSED*/ 5410 void 5411 cpu_check_allcpus(struct async_flt *aflt) 5412 {} 5413 5414 struct kmem_cache *ch_private_cache; 5415 5416 /* 5417 * Cpu private unitialization. Uninitialize the Ecache scrubber and 5418 * deallocate the scrubber data structures and cpu_private data structure. 5419 */ 5420 void 5421 cpu_uninit_private(struct cpu *cp) 5422 { 5423 cheetah_private_t *chprp = CPU_PRIVATE(cp); 5424 5425 ASSERT(chprp); 5426 cpu_uninit_ecache_scrub_dr(cp); 5427 CPU_PRIVATE(cp) = NULL; 5428 ch_err_tl1_paddrs[cp->cpu_id] = NULL; 5429 kmem_cache_free(ch_private_cache, chprp); 5430 cmp_delete_cpu(cp->cpu_id); 5431 5432 } 5433 5434 /* 5435 * Cheetah Cache Scrubbing 5436 * 5437 * The primary purpose of Cheetah cache scrubbing is to reduce the exposure 5438 * of E$ tags, D$ data, and I$ data to cosmic ray events since they are not 5439 * protected by either parity or ECC. 5440 * 5441 * We currently default the E$ and D$ scan rate to 100 (scan 10% of the 5442 * cache per second). Due to the the specifics of how the I$ control 5443 * logic works with respect to the ASI used to scrub I$ lines, the entire 5444 * I$ is scanned at once. 5445 */ 5446 5447 /* 5448 * Tuneables to enable and disable the scrubbing of the caches, and to tune 5449 * scrubbing behavior. These may be changed via /etc/system or using mdb 5450 * on a running system. 5451 */ 5452 int dcache_scrub_enable = 1; /* D$ scrubbing is on by default */ 5453 5454 /* 5455 * The following are the PIL levels that the softints/cross traps will fire at. 5456 */ 5457 uint_t ecache_scrub_pil = PIL_9; /* E$ scrub PIL for cross traps */ 5458 uint_t dcache_scrub_pil = PIL_9; /* D$ scrub PIL for cross traps */ 5459 uint_t icache_scrub_pil = PIL_9; /* I$ scrub PIL for cross traps */ 5460 5461 #if defined(JALAPENO) 5462 5463 /* 5464 * Due to several errata (82, 85, 86), we don't enable the L2$ scrubber 5465 * on Jalapeno. 5466 */ 5467 int ecache_scrub_enable = 0; 5468 5469 #else /* JALAPENO */ 5470 5471 /* 5472 * With all other cpu types, E$ scrubbing is on by default 5473 */ 5474 int ecache_scrub_enable = 1; 5475 5476 #endif /* JALAPENO */ 5477 5478 5479 #if defined(CHEETAH_PLUS) || defined(JALAPENO) || defined(SERRANO) 5480 5481 /* 5482 * The I$ scrubber tends to cause latency problems for real-time SW, so it 5483 * is disabled by default on non-Cheetah systems 5484 */ 5485 int icache_scrub_enable = 0; 5486 5487 /* 5488 * Tuneables specifying the scrub calls per second and the scan rate 5489 * for each cache 5490 * 5491 * The cyclic times are set during boot based on the following values. 5492 * Changing these values in mdb after this time will have no effect. If 5493 * a different value is desired, it must be set in /etc/system before a 5494 * reboot. 5495 */ 5496 int ecache_calls_a_sec = 1; 5497 int dcache_calls_a_sec = 2; 5498 int icache_calls_a_sec = 2; 5499 5500 int ecache_scan_rate_idle = 1; 5501 int ecache_scan_rate_busy = 1; 5502 int dcache_scan_rate_idle = 1; 5503 int dcache_scan_rate_busy = 1; 5504 int icache_scan_rate_idle = 1; 5505 int icache_scan_rate_busy = 1; 5506 5507 #else /* CHEETAH_PLUS || JALAPENO || SERRANO */ 5508 5509 int icache_scrub_enable = 1; /* I$ scrubbing is on by default */ 5510 5511 int ecache_calls_a_sec = 100; /* E$ scrub calls per seconds */ 5512 int dcache_calls_a_sec = 100; /* D$ scrub calls per seconds */ 5513 int icache_calls_a_sec = 100; /* I$ scrub calls per seconds */ 5514 5515 int ecache_scan_rate_idle = 100; /* E$ scan rate (in tenths of a %) */ 5516 int ecache_scan_rate_busy = 100; /* E$ scan rate (in tenths of a %) */ 5517 int dcache_scan_rate_idle = 100; /* D$ scan rate (in tenths of a %) */ 5518 int dcache_scan_rate_busy = 100; /* D$ scan rate (in tenths of a %) */ 5519 int icache_scan_rate_idle = 100; /* I$ scan rate (in tenths of a %) */ 5520 int icache_scan_rate_busy = 100; /* I$ scan rate (in tenths of a %) */ 5521 5522 #endif /* CHEETAH_PLUS || JALAPENO || SERRANO */ 5523 5524 /* 5525 * In order to scrub on offline cpus, a cross trap is sent. The handler will 5526 * increment the outstanding request counter and schedule a softint to run 5527 * the scrubber. 5528 */ 5529 extern xcfunc_t cache_scrubreq_tl1; 5530 5531 /* 5532 * These are the softint functions for each cache scrubber 5533 */ 5534 static uint_t scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2); 5535 static uint_t scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2); 5536 static uint_t scrub_icache_line_intr(caddr_t arg1, caddr_t arg2); 5537 5538 /* 5539 * The cache scrub info table contains cache specific information 5540 * and allows for some of the scrub code to be table driven, reducing 5541 * duplication of cache similar code. 5542 * 5543 * This table keeps a copy of the value in the calls per second variable 5544 * (?cache_calls_a_sec). This makes it much more difficult for someone 5545 * to cause us problems (for example, by setting ecache_calls_a_sec to 0 in 5546 * mdb in a misguided attempt to disable the scrubber). 5547 */ 5548 struct scrub_info { 5549 int *csi_enable; /* scrubber enable flag */ 5550 int csi_freq; /* scrubber calls per second */ 5551 int csi_index; /* index to chsm_outstanding[] */ 5552 uint64_t csi_inum; /* scrubber interrupt number */ 5553 cyclic_id_t csi_omni_cyc_id; /* omni cyclic ID */ 5554 cyclic_id_t csi_offline_cyc_id; /* offline cyclic ID */ 5555 char csi_name[3]; /* cache name for this scrub entry */ 5556 } cache_scrub_info[] = { 5557 { &ecache_scrub_enable, 0, CACHE_SCRUBBER_INFO_E, 0, 0, 0, "E$"}, 5558 { &dcache_scrub_enable, 0, CACHE_SCRUBBER_INFO_D, 0, 0, 0, "D$"}, 5559 { &icache_scrub_enable, 0, CACHE_SCRUBBER_INFO_I, 0, 0, 0, "I$"} 5560 }; 5561 5562 /* 5563 * If scrubbing is enabled, increment the outstanding request counter. If it 5564 * is 1 (meaning there were no previous requests outstanding), call 5565 * setsoftint_tl1 through xt_one_unchecked, which eventually ends up doing 5566 * a self trap. 5567 */ 5568 static void 5569 do_scrub(struct scrub_info *csi) 5570 { 5571 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5572 int index = csi->csi_index; 5573 uint32_t *outstanding = &csmp->chsm_outstanding[index]; 5574 5575 if (*(csi->csi_enable) && (csmp->chsm_enable[index])) { 5576 if (atomic_add_32_nv(outstanding, 1) == 1) { 5577 xt_one_unchecked(CPU->cpu_id, setsoftint_tl1, 5578 csi->csi_inum, 0); 5579 } 5580 } 5581 } 5582 5583 /* 5584 * Omni cyclics don't fire on offline cpus, so we use another cyclic to 5585 * cross-trap the offline cpus. 5586 */ 5587 static void 5588 do_scrub_offline(struct scrub_info *csi) 5589 { 5590 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5591 5592 if (CPUSET_ISNULL(cpu_offline_set)) { 5593 /* 5594 * No offline cpus - nothing to do 5595 */ 5596 return; 5597 } 5598 5599 if (*(csi->csi_enable) && (csmp->chsm_enable[csi->csi_index])) { 5600 xt_some(cpu_offline_set, cache_scrubreq_tl1, csi->csi_inum, 5601 csi->csi_index); 5602 } 5603 } 5604 5605 /* 5606 * This is the initial setup for the scrubber cyclics - it sets the 5607 * interrupt level, frequency, and function to call. 5608 */ 5609 /*ARGSUSED*/ 5610 static void 5611 cpu_scrub_cyclic_setup(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 5612 cyc_time_t *when) 5613 { 5614 struct scrub_info *csi = (struct scrub_info *)arg; 5615 5616 ASSERT(csi != NULL); 5617 hdlr->cyh_func = (cyc_func_t)do_scrub; 5618 hdlr->cyh_level = CY_LOW_LEVEL; 5619 hdlr->cyh_arg = arg; 5620 5621 when->cyt_when = 0; /* Start immediately */ 5622 when->cyt_interval = NANOSEC / csi->csi_freq; 5623 } 5624 5625 /* 5626 * Initialization for cache scrubbing. 5627 * This routine is called AFTER all cpus have had cpu_init_private called 5628 * to initialize their private data areas. 5629 */ 5630 void 5631 cpu_init_cache_scrub(void) 5632 { 5633 int i; 5634 struct scrub_info *csi; 5635 cyc_omni_handler_t omni_hdlr; 5636 cyc_handler_t offline_hdlr; 5637 cyc_time_t when; 5638 5639 /* 5640 * save away the maximum number of lines for the D$ 5641 */ 5642 dcache_nlines = dcache_size / dcache_linesize; 5643 5644 /* 5645 * register the softints for the cache scrubbing 5646 */ 5647 cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_inum = 5648 add_softintr(ecache_scrub_pil, scrub_ecache_line_intr, 5649 (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_E], SOFTINT_MT); 5650 cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_freq = ecache_calls_a_sec; 5651 5652 cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_inum = 5653 add_softintr(dcache_scrub_pil, scrub_dcache_line_intr, 5654 (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_D], SOFTINT_MT); 5655 cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_freq = dcache_calls_a_sec; 5656 5657 cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_inum = 5658 add_softintr(icache_scrub_pil, scrub_icache_line_intr, 5659 (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_I], SOFTINT_MT); 5660 cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_freq = icache_calls_a_sec; 5661 5662 /* 5663 * start the scrubbing for all the caches 5664 */ 5665 mutex_enter(&cpu_lock); 5666 for (i = 0; i < CACHE_SCRUBBER_COUNT; i++) { 5667 5668 csi = &cache_scrub_info[i]; 5669 5670 if (!(*csi->csi_enable)) 5671 continue; 5672 5673 /* 5674 * force the following to be true: 5675 * 1 <= calls_a_sec <= hz 5676 */ 5677 if (csi->csi_freq > hz) { 5678 cmn_err(CE_NOTE, "%s scrub calls_a_sec set too high " 5679 "(%d); resetting to hz (%d)", csi->csi_name, 5680 csi->csi_freq, hz); 5681 csi->csi_freq = hz; 5682 } else if (csi->csi_freq < 1) { 5683 cmn_err(CE_NOTE, "%s scrub calls_a_sec set too low " 5684 "(%d); resetting to 1", csi->csi_name, 5685 csi->csi_freq); 5686 csi->csi_freq = 1; 5687 } 5688 5689 omni_hdlr.cyo_online = cpu_scrub_cyclic_setup; 5690 omni_hdlr.cyo_offline = NULL; 5691 omni_hdlr.cyo_arg = (void *)csi; 5692 5693 offline_hdlr.cyh_func = (cyc_func_t)do_scrub_offline; 5694 offline_hdlr.cyh_arg = (void *)csi; 5695 offline_hdlr.cyh_level = CY_LOW_LEVEL; 5696 5697 when.cyt_when = 0; /* Start immediately */ 5698 when.cyt_interval = NANOSEC / csi->csi_freq; 5699 5700 csi->csi_omni_cyc_id = cyclic_add_omni(&omni_hdlr); 5701 csi->csi_offline_cyc_id = cyclic_add(&offline_hdlr, &when); 5702 } 5703 register_cpu_setup_func(cpu_scrub_cpu_setup, NULL); 5704 mutex_exit(&cpu_lock); 5705 } 5706 5707 /* 5708 * Indicate that the specified cpu is idle. 5709 */ 5710 void 5711 cpu_idle_ecache_scrub(struct cpu *cp) 5712 { 5713 if (CPU_PRIVATE(cp) != NULL) { 5714 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 5715 csmp->chsm_ecache_busy = ECACHE_CPU_IDLE; 5716 } 5717 } 5718 5719 /* 5720 * Indicate that the specified cpu is busy. 5721 */ 5722 void 5723 cpu_busy_ecache_scrub(struct cpu *cp) 5724 { 5725 if (CPU_PRIVATE(cp) != NULL) { 5726 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 5727 csmp->chsm_ecache_busy = ECACHE_CPU_BUSY; 5728 } 5729 } 5730 5731 /* 5732 * Initialization for cache scrubbing for the specified cpu. 5733 */ 5734 void 5735 cpu_init_ecache_scrub_dr(struct cpu *cp) 5736 { 5737 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 5738 int cpuid = cp->cpu_id; 5739 5740 /* initialize the number of lines in the caches */ 5741 csmp->chsm_ecache_nlines = cpunodes[cpuid].ecache_size / 5742 cpunodes[cpuid].ecache_linesize; 5743 csmp->chsm_icache_nlines = CPU_PRIVATE_VAL(cp, chpr_icache_size) / 5744 CPU_PRIVATE_VAL(cp, chpr_icache_linesize); 5745 5746 /* 5747 * do_scrub() and do_scrub_offline() check both the global 5748 * ?cache_scrub_enable and this per-cpu enable variable. All scrubbers 5749 * check this value before scrubbing. Currently, we use it to 5750 * disable the E$ scrubber on multi-core cpus or while running at 5751 * slowed speed. For now, just turn everything on and allow 5752 * cpu_init_private() to change it if necessary. 5753 */ 5754 csmp->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1; 5755 csmp->chsm_enable[CACHE_SCRUBBER_INFO_D] = 1; 5756 csmp->chsm_enable[CACHE_SCRUBBER_INFO_I] = 1; 5757 5758 cpu_busy_ecache_scrub(cp); 5759 } 5760 5761 /* 5762 * Un-initialization for cache scrubbing for the specified cpu. 5763 */ 5764 static void 5765 cpu_uninit_ecache_scrub_dr(struct cpu *cp) 5766 { 5767 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 5768 5769 /* 5770 * un-initialize bookkeeping for cache scrubbing 5771 */ 5772 bzero(csmp, sizeof (ch_scrub_misc_t)); 5773 5774 cpu_idle_ecache_scrub(cp); 5775 } 5776 5777 /* 5778 * Called periodically on each CPU to scrub the D$. 5779 */ 5780 static void 5781 scrub_dcache(int how_many) 5782 { 5783 int i; 5784 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5785 int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D]; 5786 5787 /* 5788 * scrub the desired number of lines 5789 */ 5790 for (i = 0; i < how_many; i++) { 5791 /* 5792 * scrub a D$ line 5793 */ 5794 dcache_inval_line(index); 5795 5796 /* 5797 * calculate the next D$ line to scrub, assumes 5798 * that dcache_nlines is a power of 2 5799 */ 5800 index = (index + 1) & (dcache_nlines - 1); 5801 } 5802 5803 /* 5804 * set the scrub index for the next visit 5805 */ 5806 csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D] = index; 5807 } 5808 5809 /* 5810 * Handler for D$ scrub inum softint. Call scrub_dcache until 5811 * we decrement the outstanding request count to zero. 5812 */ 5813 /*ARGSUSED*/ 5814 static uint_t 5815 scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2) 5816 { 5817 int i; 5818 int how_many; 5819 int outstanding; 5820 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5821 uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_D]; 5822 struct scrub_info *csi = (struct scrub_info *)arg1; 5823 int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 5824 dcache_scan_rate_idle : dcache_scan_rate_busy; 5825 5826 /* 5827 * The scan rates are expressed in units of tenths of a 5828 * percent. A scan rate of 1000 (100%) means the whole 5829 * cache is scanned every second. 5830 */ 5831 how_many = (dcache_nlines * scan_rate) / (1000 * csi->csi_freq); 5832 5833 do { 5834 outstanding = *countp; 5835 for (i = 0; i < outstanding; i++) { 5836 scrub_dcache(how_many); 5837 } 5838 } while (atomic_add_32_nv(countp, -outstanding)); 5839 5840 return (DDI_INTR_CLAIMED); 5841 } 5842 5843 /* 5844 * Called periodically on each CPU to scrub the I$. The I$ is scrubbed 5845 * by invalidating lines. Due to the characteristics of the ASI which 5846 * is used to invalidate an I$ line, the entire I$ must be invalidated 5847 * vs. an individual I$ line. 5848 */ 5849 static void 5850 scrub_icache(int how_many) 5851 { 5852 int i; 5853 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5854 int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I]; 5855 int icache_nlines = csmp->chsm_icache_nlines; 5856 5857 /* 5858 * scrub the desired number of lines 5859 */ 5860 for (i = 0; i < how_many; i++) { 5861 /* 5862 * since the entire I$ must be scrubbed at once, 5863 * wait until the index wraps to zero to invalidate 5864 * the entire I$ 5865 */ 5866 if (index == 0) { 5867 icache_inval_all(); 5868 } 5869 5870 /* 5871 * calculate the next I$ line to scrub, assumes 5872 * that chsm_icache_nlines is a power of 2 5873 */ 5874 index = (index + 1) & (icache_nlines - 1); 5875 } 5876 5877 /* 5878 * set the scrub index for the next visit 5879 */ 5880 csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I] = index; 5881 } 5882 5883 /* 5884 * Handler for I$ scrub inum softint. Call scrub_icache until 5885 * we decrement the outstanding request count to zero. 5886 */ 5887 /*ARGSUSED*/ 5888 static uint_t 5889 scrub_icache_line_intr(caddr_t arg1, caddr_t arg2) 5890 { 5891 int i; 5892 int how_many; 5893 int outstanding; 5894 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5895 uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_I]; 5896 struct scrub_info *csi = (struct scrub_info *)arg1; 5897 int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 5898 icache_scan_rate_idle : icache_scan_rate_busy; 5899 int icache_nlines = csmp->chsm_icache_nlines; 5900 5901 /* 5902 * The scan rates are expressed in units of tenths of a 5903 * percent. A scan rate of 1000 (100%) means the whole 5904 * cache is scanned every second. 5905 */ 5906 how_many = (icache_nlines * scan_rate) / (1000 * csi->csi_freq); 5907 5908 do { 5909 outstanding = *countp; 5910 for (i = 0; i < outstanding; i++) { 5911 scrub_icache(how_many); 5912 } 5913 } while (atomic_add_32_nv(countp, -outstanding)); 5914 5915 return (DDI_INTR_CLAIMED); 5916 } 5917 5918 /* 5919 * Called periodically on each CPU to scrub the E$. 5920 */ 5921 static void 5922 scrub_ecache(int how_many) 5923 { 5924 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5925 int i; 5926 int cpuid = CPU->cpu_id; 5927 int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E]; 5928 int nlines = csmp->chsm_ecache_nlines; 5929 int linesize = cpunodes[cpuid].ecache_linesize; 5930 int ec_set_size = cpu_ecache_set_size(CPU); 5931 5932 /* 5933 * scrub the desired number of lines 5934 */ 5935 for (i = 0; i < how_many; i++) { 5936 /* 5937 * scrub the E$ line 5938 */ 5939 ecache_flush_line(ecache_flushaddr + (index * linesize), 5940 ec_set_size); 5941 5942 /* 5943 * calculate the next E$ line to scrub based on twice 5944 * the number of E$ lines (to displace lines containing 5945 * flush area data), assumes that the number of lines 5946 * is a power of 2 5947 */ 5948 index = (index + 1) & ((nlines << 1) - 1); 5949 } 5950 5951 /* 5952 * set the ecache scrub index for the next visit 5953 */ 5954 csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E] = index; 5955 } 5956 5957 /* 5958 * Handler for E$ scrub inum softint. Call the E$ scrubber until 5959 * we decrement the outstanding request count to zero. 5960 * 5961 * Due to interactions with cpu_scrub_cpu_setup(), the outstanding count may 5962 * become negative after the atomic_add_32_nv(). This is not a problem, as 5963 * the next trip around the loop won't scrub anything, and the next add will 5964 * reset the count back to zero. 5965 */ 5966 /*ARGSUSED*/ 5967 static uint_t 5968 scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2) 5969 { 5970 int i; 5971 int how_many; 5972 int outstanding; 5973 ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 5974 uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_E]; 5975 struct scrub_info *csi = (struct scrub_info *)arg1; 5976 int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 5977 ecache_scan_rate_idle : ecache_scan_rate_busy; 5978 int ecache_nlines = csmp->chsm_ecache_nlines; 5979 5980 /* 5981 * The scan rates are expressed in units of tenths of a 5982 * percent. A scan rate of 1000 (100%) means the whole 5983 * cache is scanned every second. 5984 */ 5985 how_many = (ecache_nlines * scan_rate) / (1000 * csi->csi_freq); 5986 5987 do { 5988 outstanding = *countp; 5989 for (i = 0; i < outstanding; i++) { 5990 scrub_ecache(how_many); 5991 } 5992 } while (atomic_add_32_nv(countp, -outstanding)); 5993 5994 return (DDI_INTR_CLAIMED); 5995 } 5996 5997 /* 5998 * Timeout function to reenable CE 5999 */ 6000 static void 6001 cpu_delayed_check_ce_errors(void *arg) 6002 { 6003 if (!taskq_dispatch(ch_check_ce_tq, cpu_check_ce_errors, arg, 6004 TQ_NOSLEEP)) { 6005 (void) timeout(cpu_delayed_check_ce_errors, arg, 6006 drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 6007 } 6008 } 6009 6010 /* 6011 * CE Deferred Re-enable after trap. 6012 * 6013 * When the CPU gets a disrupting trap for any of the errors 6014 * controlled by the CEEN bit, CEEN is disabled in the trap handler 6015 * immediately. To eliminate the possibility of multiple CEs causing 6016 * recursive stack overflow in the trap handler, we cannot 6017 * reenable CEEN while still running in the trap handler. Instead, 6018 * after a CE is logged on a CPU, we schedule a timeout function, 6019 * cpu_check_ce_errors(), to trigger after cpu_ceen_delay_secs 6020 * seconds. This function will check whether any further CEs 6021 * have occurred on that CPU, and if none have, will reenable CEEN. 6022 * 6023 * If further CEs have occurred while CEEN is disabled, another 6024 * timeout will be scheduled. This is to ensure that the CPU can 6025 * make progress in the face of CE 'storms', and that it does not 6026 * spend all its time logging CE errors. 6027 */ 6028 static void 6029 cpu_check_ce_errors(void *arg) 6030 { 6031 int cpuid = (int)(uintptr_t)arg; 6032 cpu_t *cp; 6033 6034 /* 6035 * We acquire cpu_lock. 6036 */ 6037 ASSERT(curthread->t_pil == 0); 6038 6039 /* 6040 * verify that the cpu is still around, DR 6041 * could have got there first ... 6042 */ 6043 mutex_enter(&cpu_lock); 6044 cp = cpu_get(cpuid); 6045 if (cp == NULL) { 6046 mutex_exit(&cpu_lock); 6047 return; 6048 } 6049 /* 6050 * make sure we don't migrate across CPUs 6051 * while checking our CE status. 6052 */ 6053 kpreempt_disable(); 6054 6055 /* 6056 * If we are running on the CPU that got the 6057 * CE, we can do the checks directly. 6058 */ 6059 if (cp->cpu_id == CPU->cpu_id) { 6060 mutex_exit(&cpu_lock); 6061 cpu_check_ce(TIMEOUT_CEEN_CHECK, 0, 0, 0); 6062 kpreempt_enable(); 6063 return; 6064 } 6065 kpreempt_enable(); 6066 6067 /* 6068 * send an x-call to get the CPU that originally 6069 * got the CE to do the necessary checks. If we can't 6070 * send the x-call, reschedule the timeout, otherwise we 6071 * lose CEEN forever on that CPU. 6072 */ 6073 if (CPU_XCALL_READY(cp->cpu_id) && (!(cp->cpu_flags & CPU_QUIESCED))) { 6074 xc_one(cp->cpu_id, (xcfunc_t *)cpu_check_ce, 6075 TIMEOUT_CEEN_CHECK, 0); 6076 mutex_exit(&cpu_lock); 6077 } else { 6078 /* 6079 * When the CPU is not accepting xcalls, or 6080 * the processor is offlined, we don't want to 6081 * incur the extra overhead of trying to schedule the 6082 * CE timeout indefinitely. However, we don't want to lose 6083 * CE checking forever. 6084 * 6085 * Keep rescheduling the timeout, accepting the additional 6086 * overhead as the cost of correctness in the case where we get 6087 * a CE, disable CEEN, offline the CPU during the 6088 * the timeout interval, and then online it at some 6089 * point in the future. This is unlikely given the short 6090 * cpu_ceen_delay_secs. 6091 */ 6092 mutex_exit(&cpu_lock); 6093 (void) timeout(cpu_delayed_check_ce_errors, 6094 (void *)(uintptr_t)cp->cpu_id, 6095 drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 6096 } 6097 } 6098 6099 /* 6100 * This routine will check whether CEs have occurred while 6101 * CEEN is disabled. Any CEs detected will be logged and, if 6102 * possible, scrubbed. 6103 * 6104 * The memscrubber will also use this routine to clear any errors 6105 * caused by its scrubbing with CEEN disabled. 6106 * 6107 * flag == SCRUBBER_CEEN_CHECK 6108 * called from memscrubber, just check/scrub, no reset 6109 * paddr physical addr. for start of scrub pages 6110 * vaddr virtual addr. for scrub area 6111 * psz page size of area to be scrubbed 6112 * 6113 * flag == TIMEOUT_CEEN_CHECK 6114 * timeout function has triggered, reset timeout or CEEN 6115 * 6116 * Note: We must not migrate cpus during this function. This can be 6117 * achieved by one of: 6118 * - invoking as target of an x-call in which case we're at XCALL_PIL 6119 * The flag value must be first xcall argument. 6120 * - disabling kernel preemption. This should be done for very short 6121 * periods so is not suitable for SCRUBBER_CEEN_CHECK where we might 6122 * scrub an extended area with cpu_check_block. The call for 6123 * TIMEOUT_CEEN_CHECK uses this so cpu_check_ce must be kept 6124 * brief for this case. 6125 * - binding to a cpu, eg with thread_affinity_set(). This is used 6126 * in the SCRUBBER_CEEN_CHECK case, but is not practical for 6127 * the TIMEOUT_CEEN_CHECK because both need cpu_lock. 6128 */ 6129 void 6130 cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t psz) 6131 { 6132 ch_cpu_errors_t cpu_error_regs; 6133 uint64_t ec_err_enable; 6134 uint64_t page_offset; 6135 6136 /* Read AFSR */ 6137 get_cpu_error_state(&cpu_error_regs); 6138 6139 /* 6140 * If no CEEN errors have occurred during the timeout 6141 * interval, it is safe to re-enable CEEN and exit. 6142 */ 6143 if ((cpu_error_regs.afsr & C_AFSR_CECC_ERRS) == 0) { 6144 if (flag == TIMEOUT_CEEN_CHECK && 6145 !((ec_err_enable = get_error_enable()) & EN_REG_CEEN)) 6146 set_error_enable(ec_err_enable | EN_REG_CEEN); 6147 return; 6148 } 6149 6150 /* 6151 * Ensure that CEEN was not reenabled (maybe by DR) before 6152 * we log/clear the error. 6153 */ 6154 if ((ec_err_enable = get_error_enable()) & EN_REG_CEEN) 6155 set_error_enable(ec_err_enable & ~EN_REG_CEEN); 6156 6157 /* 6158 * log/clear the CE. If CE_CEEN_DEFER is passed, the 6159 * timeout will be rescheduled when the error is logged. 6160 */ 6161 if (!(cpu_error_regs.afsr & cpu_ce_not_deferred)) 6162 cpu_ce_detected(&cpu_error_regs, 6163 CE_CEEN_DEFER | CE_CEEN_TIMEOUT); 6164 else 6165 cpu_ce_detected(&cpu_error_regs, CE_CEEN_TIMEOUT); 6166 6167 /* 6168 * If the memory scrubber runs while CEEN is 6169 * disabled, (or if CEEN is disabled during the 6170 * scrub as a result of a CE being triggered by 6171 * it), the range being scrubbed will not be 6172 * completely cleaned. If there are multiple CEs 6173 * in the range at most two of these will be dealt 6174 * with, (one by the trap handler and one by the 6175 * timeout). It is also possible that none are dealt 6176 * with, (CEEN disabled and another CE occurs before 6177 * the timeout triggers). So to ensure that the 6178 * memory is actually scrubbed, we have to access each 6179 * memory location in the range and then check whether 6180 * that access causes a CE. 6181 */ 6182 if (flag == SCRUBBER_CEEN_CHECK && va) { 6183 if ((cpu_error_regs.afar >= pa) && 6184 (cpu_error_regs.afar < (pa + psz))) { 6185 /* 6186 * Force a load from physical memory for each 6187 * 64-byte block, then check AFSR to determine 6188 * whether this access caused an error. 6189 * 6190 * This is a slow way to do a scrub, but as it will 6191 * only be invoked when the memory scrubber actually 6192 * triggered a CE, it should not happen too 6193 * frequently. 6194 * 6195 * cut down what we need to check as the scrubber 6196 * has verified up to AFAR, so get it's offset 6197 * into the page and start there. 6198 */ 6199 page_offset = (uint64_t)(cpu_error_regs.afar & 6200 (psz - 1)); 6201 va = (caddr_t)(va + (P2ALIGN(page_offset, 64))); 6202 psz -= (uint_t)(P2ALIGN(page_offset, 64)); 6203 cpu_check_block((caddr_t)(P2ALIGN((uint64_t)va, 64)), 6204 psz); 6205 } 6206 } 6207 6208 /* 6209 * Reset error enable if this CE is not masked. 6210 */ 6211 if ((flag == TIMEOUT_CEEN_CHECK) && 6212 (cpu_error_regs.afsr & cpu_ce_not_deferred)) 6213 set_error_enable(ec_err_enable | EN_REG_CEEN); 6214 6215 } 6216 6217 /* 6218 * Attempt a cpu logout for an error that we did not trap for, such 6219 * as a CE noticed with CEEN off. It is assumed that we are still running 6220 * on the cpu that took the error and that we cannot migrate. Returns 6221 * 0 on success, otherwise nonzero. 6222 */ 6223 static int 6224 cpu_ce_delayed_ec_logout(uint64_t afar) 6225 { 6226 ch_cpu_logout_t *clop; 6227 6228 if (CPU_PRIVATE(CPU) == NULL) 6229 return (0); 6230 6231 clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 6232 if (cas64(&clop->clo_data.chd_afar, LOGOUT_INVALID, afar) != 6233 LOGOUT_INVALID) 6234 return (0); 6235 6236 cpu_delayed_logout(afar, clop); 6237 return (1); 6238 } 6239 6240 /* 6241 * We got an error while CEEN was disabled. We 6242 * need to clean up after it and log whatever 6243 * information we have on the CE. 6244 */ 6245 void 6246 cpu_ce_detected(ch_cpu_errors_t *cpu_error_regs, int flag) 6247 { 6248 ch_async_flt_t ch_flt; 6249 struct async_flt *aflt; 6250 char pr_reason[MAX_REASON_STRING]; 6251 6252 bzero(&ch_flt, sizeof (ch_async_flt_t)); 6253 ch_flt.flt_trapped_ce = flag; 6254 aflt = (struct async_flt *)&ch_flt; 6255 aflt->flt_stat = cpu_error_regs->afsr & C_AFSR_MASK; 6256 ch_flt.afsr_ext = cpu_error_regs->afsr_ext; 6257 ch_flt.afsr_errs = (cpu_error_regs->afsr_ext & C_AFSR_EXT_ALL_ERRS) | 6258 (cpu_error_regs->afsr & C_AFSR_ALL_ERRS); 6259 aflt->flt_addr = cpu_error_regs->afar; 6260 #if defined(SERRANO) 6261 ch_flt.afar2 = cpu_error_regs->afar2; 6262 #endif /* SERRANO */ 6263 aflt->flt_pc = NULL; 6264 aflt->flt_priv = ((cpu_error_regs->afsr & C_AFSR_PRIV) != 0); 6265 aflt->flt_tl = 0; 6266 aflt->flt_panic = 0; 6267 cpu_log_and_clear_ce(&ch_flt); 6268 6269 /* 6270 * check if we caused any errors during cleanup 6271 */ 6272 if (clear_errors(&ch_flt)) { 6273 pr_reason[0] = '\0'; 6274 (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 6275 NULL); 6276 } 6277 } 6278 6279 /* 6280 * Log/clear CEEN-controlled disrupting errors 6281 */ 6282 static void 6283 cpu_log_and_clear_ce(ch_async_flt_t *ch_flt) 6284 { 6285 struct async_flt *aflt; 6286 uint64_t afsr, afsr_errs; 6287 ch_cpu_logout_t *clop; 6288 char pr_reason[MAX_REASON_STRING]; 6289 on_trap_data_t *otp = curthread->t_ontrap; 6290 6291 aflt = (struct async_flt *)ch_flt; 6292 afsr = aflt->flt_stat; 6293 afsr_errs = ch_flt->afsr_errs; 6294 aflt->flt_id = gethrtime_waitfree(); 6295 aflt->flt_bus_id = getprocessorid(); 6296 aflt->flt_inst = CPU->cpu_id; 6297 aflt->flt_prot = AFLT_PROT_NONE; 6298 aflt->flt_class = CPU_FAULT; 6299 aflt->flt_status = ECC_C_TRAP; 6300 6301 pr_reason[0] = '\0'; 6302 /* 6303 * Get the CPU log out info for Disrupting Trap. 6304 */ 6305 if (CPU_PRIVATE(CPU) == NULL) { 6306 clop = NULL; 6307 ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 6308 } else { 6309 clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 6310 } 6311 6312 if (clop && ch_flt->flt_trapped_ce & CE_CEEN_TIMEOUT) { 6313 ch_cpu_errors_t cpu_error_regs; 6314 6315 get_cpu_error_state(&cpu_error_regs); 6316 (void) cpu_ce_delayed_ec_logout(cpu_error_regs.afar); 6317 clop->clo_data.chd_afsr = cpu_error_regs.afsr; 6318 clop->clo_data.chd_afar = cpu_error_regs.afar; 6319 clop->clo_data.chd_afsr_ext = cpu_error_regs.afsr_ext; 6320 clop->clo_sdw_data.chd_afsr = cpu_error_regs.shadow_afsr; 6321 clop->clo_sdw_data.chd_afar = cpu_error_regs.shadow_afar; 6322 clop->clo_sdw_data.chd_afsr_ext = 6323 cpu_error_regs.shadow_afsr_ext; 6324 #if defined(SERRANO) 6325 clop->clo_data.chd_afar2 = cpu_error_regs.afar2; 6326 #endif /* SERRANO */ 6327 ch_flt->flt_data_incomplete = 1; 6328 6329 /* 6330 * The logging/clear code expects AFSR/AFAR to be cleared. 6331 * The trap handler does it for CEEN enabled errors 6332 * so we need to do it here. 6333 */ 6334 set_cpu_error_state(&cpu_error_regs); 6335 } 6336 6337 #if defined(JALAPENO) || defined(SERRANO) 6338 /* 6339 * FRC: Can't scrub memory as we don't have AFAR for Jalapeno. 6340 * For Serrano, even thou we do have the AFAR, we still do the 6341 * scrub on the RCE side since that's where the error type can 6342 * be properly classified as intermittent, persistent, etc. 6343 * 6344 * CE/RCE: If error is in memory and AFAR is valid, scrub the memory. 6345 * Must scrub memory before cpu_queue_events, as scrubbing memory sets 6346 * the flt_status bits. 6347 */ 6348 if ((afsr & (C_AFSR_CE|C_AFSR_RCE)) && 6349 (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 6350 cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_RCE)))) { 6351 cpu_ce_scrub_mem_err(aflt, B_TRUE); 6352 } 6353 #else /* JALAPENO || SERRANO */ 6354 /* 6355 * CE/EMC: If error is in memory and AFAR is valid, scrub the memory. 6356 * Must scrub memory before cpu_queue_events, as scrubbing memory sets 6357 * the flt_status bits. 6358 */ 6359 if (afsr & (C_AFSR_CE|C_AFSR_EMC)) { 6360 if (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 6361 cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_EMC))) { 6362 cpu_ce_scrub_mem_err(aflt, B_TRUE); 6363 } 6364 } 6365 6366 #endif /* JALAPENO || SERRANO */ 6367 6368 /* 6369 * Update flt_prot if this error occurred under on_trap protection. 6370 */ 6371 if (otp != NULL && (otp->ot_prot & OT_DATA_EC)) 6372 aflt->flt_prot = AFLT_PROT_EC; 6373 6374 /* 6375 * Queue events on the async event queue, one event per error bit. 6376 */ 6377 if (cpu_queue_events(ch_flt, pr_reason, afsr_errs, clop) == 0 || 6378 (afsr_errs & (C_AFSR_CECC_ERRS | C_AFSR_EXT_CECC_ERRS)) == 0) { 6379 ch_flt->flt_type = CPU_INV_AFSR; 6380 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 6381 (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 6382 aflt->flt_panic); 6383 } 6384 6385 /* 6386 * Zero out + invalidate CPU logout. 6387 */ 6388 if (clop) { 6389 bzero(clop, sizeof (ch_cpu_logout_t)); 6390 clop->clo_data.chd_afar = LOGOUT_INVALID; 6391 } 6392 6393 /* 6394 * If either a CPC, WDC or EDC error has occurred while CEEN 6395 * was disabled, we need to flush either the entire 6396 * E$ or an E$ line. 6397 */ 6398 #if defined(JALAPENO) || defined(SERRANO) 6399 if (afsr & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_CPU | C_AFSR_WDC)) 6400 #else /* JALAPENO || SERRANO */ 6401 if (afsr_errs & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_WDC | C_AFSR_L3_EDC | 6402 C_AFSR_L3_CPC | C_AFSR_L3_WDC)) 6403 #endif /* JALAPENO || SERRANO */ 6404 cpu_error_ecache_flush(ch_flt); 6405 6406 } 6407 6408 /* 6409 * depending on the error type, we determine whether we 6410 * need to flush the entire ecache or just a line. 6411 */ 6412 static int 6413 cpu_error_ecache_flush_required(ch_async_flt_t *ch_flt) 6414 { 6415 struct async_flt *aflt; 6416 uint64_t afsr; 6417 uint64_t afsr_errs = ch_flt->afsr_errs; 6418 6419 aflt = (struct async_flt *)ch_flt; 6420 afsr = aflt->flt_stat; 6421 6422 /* 6423 * If we got multiple errors, no point in trying 6424 * the individual cases, just flush the whole cache 6425 */ 6426 if (afsr & C_AFSR_ME) { 6427 return (ECACHE_FLUSH_ALL); 6428 } 6429 6430 /* 6431 * If either a CPC, WDC or EDC error has occurred while CEEN 6432 * was disabled, we need to flush entire E$. We can't just 6433 * flush the cache line affected as the ME bit 6434 * is not set when multiple correctable errors of the same 6435 * type occur, so we might have multiple CPC or EDC errors, 6436 * with only the first recorded. 6437 */ 6438 #if defined(JALAPENO) || defined(SERRANO) 6439 if (afsr & (C_AFSR_CPC | C_AFSR_CPU | C_AFSR_EDC | C_AFSR_WDC)) { 6440 #else /* JALAPENO || SERRANO */ 6441 if (afsr_errs & (C_AFSR_CPC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_L3_CPC | 6442 C_AFSR_L3_EDC | C_AFSR_L3_WDC)) { 6443 #endif /* JALAPENO || SERRANO */ 6444 return (ECACHE_FLUSH_ALL); 6445 } 6446 6447 #if defined(JALAPENO) || defined(SERRANO) 6448 /* 6449 * If only UE or RUE is set, flush the Ecache line, otherwise 6450 * flush the entire Ecache. 6451 */ 6452 if (afsr & (C_AFSR_UE|C_AFSR_RUE)) { 6453 if ((afsr & C_AFSR_ALL_ERRS) == C_AFSR_UE || 6454 (afsr & C_AFSR_ALL_ERRS) == C_AFSR_RUE) { 6455 return (ECACHE_FLUSH_LINE); 6456 } else { 6457 return (ECACHE_FLUSH_ALL); 6458 } 6459 } 6460 #else /* JALAPENO || SERRANO */ 6461 /* 6462 * If UE only is set, flush the Ecache line, otherwise 6463 * flush the entire Ecache. 6464 */ 6465 if (afsr_errs & C_AFSR_UE) { 6466 if ((afsr_errs & (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == 6467 C_AFSR_UE) { 6468 return (ECACHE_FLUSH_LINE); 6469 } else { 6470 return (ECACHE_FLUSH_ALL); 6471 } 6472 } 6473 #endif /* JALAPENO || SERRANO */ 6474 6475 /* 6476 * EDU: If EDU only is set, flush the ecache line, otherwise 6477 * flush the entire Ecache. 6478 */ 6479 if (afsr_errs & (C_AFSR_EDU | C_AFSR_L3_EDU)) { 6480 if (((afsr_errs & ~C_AFSR_EDU) == 0) || 6481 ((afsr_errs & ~C_AFSR_L3_EDU) == 0)) { 6482 return (ECACHE_FLUSH_LINE); 6483 } else { 6484 return (ECACHE_FLUSH_ALL); 6485 } 6486 } 6487 6488 /* 6489 * BERR: If BERR only is set, flush the Ecache line, otherwise 6490 * flush the entire Ecache. 6491 */ 6492 if (afsr_errs & C_AFSR_BERR) { 6493 if ((afsr_errs & ~C_AFSR_BERR) == 0) { 6494 return (ECACHE_FLUSH_LINE); 6495 } else { 6496 return (ECACHE_FLUSH_ALL); 6497 } 6498 } 6499 6500 return (0); 6501 } 6502 6503 void 6504 cpu_error_ecache_flush(ch_async_flt_t *ch_flt) 6505 { 6506 int ecache_flush_flag = 6507 cpu_error_ecache_flush_required(ch_flt); 6508 6509 /* 6510 * Flush Ecache line or entire Ecache based on above checks. 6511 */ 6512 if (ecache_flush_flag == ECACHE_FLUSH_ALL) 6513 cpu_flush_ecache(); 6514 else if (ecache_flush_flag == ECACHE_FLUSH_LINE) { 6515 cpu_flush_ecache_line(ch_flt); 6516 } 6517 6518 } 6519 6520 /* 6521 * Extract the PA portion from the E$ tag. 6522 */ 6523 uint64_t 6524 cpu_ectag_to_pa(int setsize, uint64_t tag) 6525 { 6526 if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 6527 return (JG_ECTAG_TO_PA(setsize, tag)); 6528 else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 6529 return (PN_L3TAG_TO_PA(tag)); 6530 else 6531 return (CH_ECTAG_TO_PA(setsize, tag)); 6532 } 6533 6534 /* 6535 * Convert the E$ tag PA into an E$ subblock index. 6536 */ 6537 static int 6538 cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr) 6539 { 6540 if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 6541 return (JG_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 6542 else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 6543 /* Panther has only one subblock per line */ 6544 return (0); 6545 else 6546 return (CH_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 6547 } 6548 6549 /* 6550 * All subblocks in an E$ line must be invalid for 6551 * the line to be invalid. 6552 */ 6553 int 6554 cpu_ectag_line_invalid(int cachesize, uint64_t tag) 6555 { 6556 if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 6557 return (JG_ECTAG_LINE_INVALID(cachesize, tag)); 6558 else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 6559 return (PN_L3_LINE_INVALID(tag)); 6560 else 6561 return (CH_ECTAG_LINE_INVALID(cachesize, tag)); 6562 } 6563 6564 /* 6565 * Extract state bits for a subblock given the tag. Note that for Panther 6566 * this works on both l2 and l3 tags. 6567 */ 6568 static int 6569 cpu_ectag_pa_to_subblk_state(int cachesize, uint64_t subaddr, uint64_t tag) 6570 { 6571 if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 6572 return (JG_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 6573 else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 6574 return (tag & CH_ECSTATE_MASK); 6575 else 6576 return (CH_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 6577 } 6578 6579 /* 6580 * Cpu specific initialization. 6581 */ 6582 void 6583 cpu_mp_init(void) 6584 { 6585 #ifdef CHEETAHPLUS_ERRATUM_25 6586 if (cheetah_sendmondo_recover) { 6587 cheetah_nudge_init(); 6588 } 6589 #endif 6590 } 6591 6592 void 6593 cpu_ereport_post(struct async_flt *aflt) 6594 { 6595 char *cpu_type, buf[FM_MAX_CLASS]; 6596 nv_alloc_t *nva = NULL; 6597 nvlist_t *ereport, *detector, *resource; 6598 errorq_elem_t *eqep; 6599 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 6600 char unum[UNUM_NAMLEN]; 6601 int synd_code; 6602 uint8_t msg_type; 6603 plat_ecc_ch_async_flt_t plat_ecc_ch_flt; 6604 6605 if (aflt->flt_panic || panicstr) { 6606 eqep = errorq_reserve(ereport_errorq); 6607 if (eqep == NULL) 6608 return; 6609 ereport = errorq_elem_nvl(ereport_errorq, eqep); 6610 nva = errorq_elem_nva(ereport_errorq, eqep); 6611 } else { 6612 ereport = fm_nvlist_create(nva); 6613 } 6614 6615 /* 6616 * Create the scheme "cpu" FMRI. 6617 */ 6618 detector = fm_nvlist_create(nva); 6619 resource = fm_nvlist_create(nva); 6620 switch (cpunodes[aflt->flt_inst].implementation) { 6621 case CHEETAH_IMPL: 6622 cpu_type = FM_EREPORT_CPU_USIII; 6623 break; 6624 case CHEETAH_PLUS_IMPL: 6625 cpu_type = FM_EREPORT_CPU_USIIIplus; 6626 break; 6627 case JALAPENO_IMPL: 6628 cpu_type = FM_EREPORT_CPU_USIIIi; 6629 break; 6630 case SERRANO_IMPL: 6631 cpu_type = FM_EREPORT_CPU_USIIIiplus; 6632 break; 6633 case JAGUAR_IMPL: 6634 cpu_type = FM_EREPORT_CPU_USIV; 6635 break; 6636 case PANTHER_IMPL: 6637 cpu_type = FM_EREPORT_CPU_USIVplus; 6638 break; 6639 default: 6640 cpu_type = FM_EREPORT_CPU_UNSUPPORTED; 6641 break; 6642 } 6643 6644 cpu_fmri_cpu_set(detector, aflt->flt_inst); 6645 6646 /* 6647 * Encode all the common data into the ereport. 6648 */ 6649 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s.%s", 6650 FM_ERROR_CPU, cpu_type, aflt->flt_erpt_class); 6651 6652 fm_ereport_set(ereport, FM_EREPORT_VERSION, buf, 6653 fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, FM_ENA_FMT1), 6654 detector, NULL); 6655 6656 /* 6657 * Encode the error specific data that was saved in 6658 * the async_flt structure into the ereport. 6659 */ 6660 cpu_payload_add_aflt(aflt, ereport, resource, 6661 &plat_ecc_ch_flt.ecaf_afar_status, 6662 &plat_ecc_ch_flt.ecaf_synd_status); 6663 6664 if (aflt->flt_panic || panicstr) { 6665 errorq_commit(ereport_errorq, eqep, ERRORQ_SYNC); 6666 } else { 6667 (void) fm_ereport_post(ereport, EVCH_TRYHARD); 6668 fm_nvlist_destroy(ereport, FM_NVA_FREE); 6669 fm_nvlist_destroy(detector, FM_NVA_FREE); 6670 fm_nvlist_destroy(resource, FM_NVA_FREE); 6671 } 6672 /* 6673 * Send the enhanced error information (plat_ecc_error2_data_t) 6674 * to the SC olny if it can process it. 6675 */ 6676 6677 if (&plat_ecc_capability_sc_get && 6678 plat_ecc_capability_sc_get(PLAT_ECC_ERROR2_MESSAGE)) { 6679 msg_type = cpu_flt_bit_to_plat_error(aflt); 6680 if (msg_type != PLAT_ECC_ERROR2_NONE) { 6681 /* 6682 * If afar status is not invalid do a unum lookup. 6683 */ 6684 if (plat_ecc_ch_flt.ecaf_afar_status != 6685 AFLT_STAT_INVALID) { 6686 synd_code = synd_to_synd_code( 6687 plat_ecc_ch_flt.ecaf_synd_status, 6688 aflt->flt_synd, ch_flt->flt_bit); 6689 (void) cpu_get_mem_unum_synd(synd_code, 6690 aflt, unum); 6691 } else { 6692 unum[0] = '\0'; 6693 } 6694 plat_ecc_ch_flt.ecaf_sdw_afar = ch_flt->flt_sdw_afar; 6695 plat_ecc_ch_flt.ecaf_sdw_afsr = ch_flt->flt_sdw_afsr; 6696 plat_ecc_ch_flt.ecaf_afsr_ext = ch_flt->afsr_ext; 6697 plat_ecc_ch_flt.ecaf_sdw_afsr_ext = 6698 ch_flt->flt_sdw_afsr_ext; 6699 6700 if (&plat_log_fruid_error2) 6701 plat_log_fruid_error2(msg_type, unum, aflt, 6702 &plat_ecc_ch_flt); 6703 } 6704 } 6705 } 6706 6707 void 6708 cpu_run_bus_error_handlers(struct async_flt *aflt, int expected) 6709 { 6710 int status; 6711 ddi_fm_error_t de; 6712 6713 bzero(&de, sizeof (ddi_fm_error_t)); 6714 6715 de.fme_version = DDI_FME_VERSION; 6716 de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, 6717 FM_ENA_FMT1); 6718 de.fme_flag = expected; 6719 de.fme_bus_specific = (void *)aflt->flt_addr; 6720 status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de); 6721 if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL)) 6722 aflt->flt_panic = 1; 6723 } 6724 6725 void 6726 cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz, 6727 errorq_t *eqp, uint_t flag) 6728 { 6729 struct async_flt *aflt = (struct async_flt *)payload; 6730 6731 aflt->flt_erpt_class = error_class; 6732 errorq_dispatch(eqp, payload, payload_sz, flag); 6733 } 6734 6735 /* 6736 * This routine may be called by the IO module, but does not do 6737 * anything in this cpu module. The SERD algorithm is handled by 6738 * cpumem-diagnosis engine instead. 6739 */ 6740 /*ARGSUSED*/ 6741 void 6742 cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum) 6743 {} 6744 6745 void 6746 adjust_hw_copy_limits(int ecache_size) 6747 { 6748 /* 6749 * Set hw copy limits. 6750 * 6751 * /etc/system will be parsed later and can override one or more 6752 * of these settings. 6753 * 6754 * At this time, ecache size seems only mildly relevant. 6755 * We seem to run into issues with the d-cache and stalls 6756 * we see on misses. 6757 * 6758 * Cycle measurement indicates that 2 byte aligned copies fare 6759 * little better than doing things with VIS at around 512 bytes. 6760 * 4 byte aligned shows promise until around 1024 bytes. 8 Byte 6761 * aligned is faster whenever the source and destination data 6762 * in cache and the total size is less than 2 Kbytes. The 2K 6763 * limit seems to be driven by the 2K write cache. 6764 * When more than 2K of copies are done in non-VIS mode, stores 6765 * backup in the write cache. In VIS mode, the write cache is 6766 * bypassed, allowing faster cache-line writes aligned on cache 6767 * boundaries. 6768 * 6769 * In addition, in non-VIS mode, there is no prefetching, so 6770 * for larger copies, the advantage of prefetching to avoid even 6771 * occasional cache misses is enough to justify using the VIS code. 6772 * 6773 * During testing, it was discovered that netbench ran 3% slower 6774 * when hw_copy_limit_8 was 2K or larger. Apparently for server 6775 * applications, data is only used once (copied to the output 6776 * buffer, then copied by the network device off the system). Using 6777 * the VIS copy saves more L2 cache state. Network copies are 6778 * around 1.3K to 1.5K in size for historical reasons. 6779 * 6780 * Therefore, a limit of 1K bytes will be used for the 8 byte 6781 * aligned copy even for large caches and 8 MB ecache. The 6782 * infrastructure to allow different limits for different sized 6783 * caches is kept to allow further tuning in later releases. 6784 */ 6785 6786 if (min_ecache_size == 0 && use_hw_bcopy) { 6787 /* 6788 * First time through - should be before /etc/system 6789 * is read. 6790 * Could skip the checks for zero but this lets us 6791 * preserve any debugger rewrites. 6792 */ 6793 if (hw_copy_limit_1 == 0) { 6794 hw_copy_limit_1 = VIS_COPY_THRESHOLD; 6795 priv_hcl_1 = hw_copy_limit_1; 6796 } 6797 if (hw_copy_limit_2 == 0) { 6798 hw_copy_limit_2 = 2 * VIS_COPY_THRESHOLD; 6799 priv_hcl_2 = hw_copy_limit_2; 6800 } 6801 if (hw_copy_limit_4 == 0) { 6802 hw_copy_limit_4 = 4 * VIS_COPY_THRESHOLD; 6803 priv_hcl_4 = hw_copy_limit_4; 6804 } 6805 if (hw_copy_limit_8 == 0) { 6806 hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD; 6807 priv_hcl_8 = hw_copy_limit_8; 6808 } 6809 min_ecache_size = ecache_size; 6810 } else { 6811 /* 6812 * MP initialization. Called *after* /etc/system has 6813 * been parsed. One CPU has already been initialized. 6814 * Need to cater for /etc/system having scragged one 6815 * of our values. 6816 */ 6817 if (ecache_size == min_ecache_size) { 6818 /* 6819 * Same size ecache. We do nothing unless we 6820 * have a pessimistic ecache setting. In that 6821 * case we become more optimistic (if the cache is 6822 * large enough). 6823 */ 6824 if (hw_copy_limit_8 == 4 * VIS_COPY_THRESHOLD) { 6825 /* 6826 * Need to adjust hw_copy_limit* from our 6827 * pessimistic uniprocessor value to a more 6828 * optimistic UP value *iff* it hasn't been 6829 * reset. 6830 */ 6831 if ((ecache_size > 1048576) && 6832 (priv_hcl_8 == hw_copy_limit_8)) { 6833 if (ecache_size <= 2097152) 6834 hw_copy_limit_8 = 4 * 6835 VIS_COPY_THRESHOLD; 6836 else if (ecache_size <= 4194304) 6837 hw_copy_limit_8 = 4 * 6838 VIS_COPY_THRESHOLD; 6839 else 6840 hw_copy_limit_8 = 4 * 6841 VIS_COPY_THRESHOLD; 6842 priv_hcl_8 = hw_copy_limit_8; 6843 } 6844 } 6845 } else if (ecache_size < min_ecache_size) { 6846 /* 6847 * A different ecache size. Can this even happen? 6848 */ 6849 if (priv_hcl_8 == hw_copy_limit_8) { 6850 /* 6851 * The previous value that we set 6852 * is unchanged (i.e., it hasn't been 6853 * scragged by /etc/system). Rewrite it. 6854 */ 6855 if (ecache_size <= 1048576) 6856 hw_copy_limit_8 = 8 * 6857 VIS_COPY_THRESHOLD; 6858 else if (ecache_size <= 2097152) 6859 hw_copy_limit_8 = 8 * 6860 VIS_COPY_THRESHOLD; 6861 else if (ecache_size <= 4194304) 6862 hw_copy_limit_8 = 8 * 6863 VIS_COPY_THRESHOLD; 6864 else 6865 hw_copy_limit_8 = 10 * 6866 VIS_COPY_THRESHOLD; 6867 priv_hcl_8 = hw_copy_limit_8; 6868 min_ecache_size = ecache_size; 6869 } 6870 } 6871 } 6872 } 6873 6874 /* 6875 * Called from illegal instruction trap handler to see if we can attribute 6876 * the trap to a fpras check. 6877 */ 6878 int 6879 fpras_chktrap(struct regs *rp) 6880 { 6881 int op; 6882 struct fpras_chkfngrp *cgp; 6883 uintptr_t tpc = (uintptr_t)rp->r_pc; 6884 6885 if (fpras_chkfngrps == NULL) 6886 return (0); 6887 6888 cgp = &fpras_chkfngrps[CPU->cpu_id]; 6889 for (op = 0; op < FPRAS_NCOPYOPS; ++op) { 6890 if (tpc >= (uintptr_t)&cgp->fpras_fn[op].fpras_blk0 && 6891 tpc < (uintptr_t)&cgp->fpras_fn[op].fpras_chkresult) 6892 break; 6893 } 6894 if (op == FPRAS_NCOPYOPS) 6895 return (0); 6896 6897 /* 6898 * This is an fpRAS failure caught through an illegal 6899 * instruction - trampoline. 6900 */ 6901 rp->r_pc = (uintptr_t)&cgp->fpras_fn[op].fpras_trampoline; 6902 rp->r_npc = rp->r_pc + 4; 6903 return (1); 6904 } 6905 6906 /* 6907 * fpras_failure is called when a fpras check detects a bad calculation 6908 * result or an illegal instruction trap is attributed to an fpras 6909 * check. In all cases we are still bound to CPU. 6910 */ 6911 int 6912 fpras_failure(int op, int how) 6913 { 6914 int use_hw_bcopy_orig, use_hw_bzero_orig; 6915 uint_t hcl1_orig, hcl2_orig, hcl4_orig, hcl8_orig; 6916 ch_async_flt_t ch_flt; 6917 struct async_flt *aflt = (struct async_flt *)&ch_flt; 6918 struct fpras_chkfn *sfp, *cfp; 6919 uint32_t *sip, *cip; 6920 int i; 6921 6922 /* 6923 * We're running on a sick CPU. Avoid further FPU use at least for 6924 * the time in which we dispatch an ereport and (if applicable) panic. 6925 */ 6926 use_hw_bcopy_orig = use_hw_bcopy; 6927 use_hw_bzero_orig = use_hw_bzero; 6928 hcl1_orig = hw_copy_limit_1; 6929 hcl2_orig = hw_copy_limit_2; 6930 hcl4_orig = hw_copy_limit_4; 6931 hcl8_orig = hw_copy_limit_8; 6932 use_hw_bcopy = use_hw_bzero = 0; 6933 hw_copy_limit_1 = hw_copy_limit_2 = hw_copy_limit_4 = 6934 hw_copy_limit_8 = 0; 6935 6936 bzero(&ch_flt, sizeof (ch_async_flt_t)); 6937 aflt->flt_id = gethrtime_waitfree(); 6938 aflt->flt_class = CPU_FAULT; 6939 aflt->flt_inst = CPU->cpu_id; 6940 aflt->flt_status = (how << 8) | op; 6941 aflt->flt_payload = FM_EREPORT_PAYLOAD_FPU_HWCOPY; 6942 ch_flt.flt_type = CPU_FPUERR; 6943 6944 /* 6945 * We must panic if the copy operation had no lofault protection - 6946 * ie, don't panic for copyin, copyout, kcopy and bcopy called 6947 * under on_fault and do panic for unprotected bcopy and hwblkpagecopy. 6948 */ 6949 aflt->flt_panic = (curthread->t_lofault == NULL); 6950 6951 /* 6952 * XOR the source instruction block with the copied instruction 6953 * block - this will show us which bit(s) are corrupted. 6954 */ 6955 sfp = (struct fpras_chkfn *)fpras_chkfn_type1; 6956 cfp = &fpras_chkfngrps[CPU->cpu_id].fpras_fn[op]; 6957 if (op == FPRAS_BCOPY || op == FPRAS_COPYOUT) { 6958 sip = &sfp->fpras_blk0[0]; 6959 cip = &cfp->fpras_blk0[0]; 6960 } else { 6961 sip = &sfp->fpras_blk1[0]; 6962 cip = &cfp->fpras_blk1[0]; 6963 } 6964 for (i = 0; i < 16; ++i, ++sip, ++cip) 6965 ch_flt.flt_fpdata[i] = *sip ^ *cip; 6966 6967 cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_FPU_HWCOPY, (void *)&ch_flt, 6968 sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 6969 6970 if (aflt->flt_panic) 6971 fm_panic("FPU failure on CPU %d", CPU->cpu_id); 6972 6973 /* 6974 * We get here for copyin/copyout and kcopy or bcopy where the 6975 * caller has used on_fault. We will flag the error so that 6976 * the process may be killed The trap_async_hwerr mechanism will 6977 * take appropriate further action (such as a reboot, contract 6978 * notification etc). Since we may be continuing we will 6979 * restore the global hardware copy acceleration switches. 6980 * 6981 * When we return from this function to the copy function we want to 6982 * avoid potentially bad data being used, ie we want the affected 6983 * copy function to return an error. The caller should therefore 6984 * invoke its lofault handler (which always exists for these functions) 6985 * which will return the appropriate error. 6986 */ 6987 ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR; 6988 aston(curthread); 6989 6990 use_hw_bcopy = use_hw_bcopy_orig; 6991 use_hw_bzero = use_hw_bzero_orig; 6992 hw_copy_limit_1 = hcl1_orig; 6993 hw_copy_limit_2 = hcl2_orig; 6994 hw_copy_limit_4 = hcl4_orig; 6995 hw_copy_limit_8 = hcl8_orig; 6996 6997 return (1); 6998 } 6999 7000 #define VIS_BLOCKSIZE 64 7001 7002 int 7003 dtrace_blksuword32_err(uintptr_t addr, uint32_t *data) 7004 { 7005 int ret, watched; 7006 7007 watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 7008 ret = dtrace_blksuword32(addr, data, 0); 7009 if (watched) 7010 watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 7011 7012 return (ret); 7013 } 7014 7015 /* 7016 * Called when a cpu enters the CPU_FAULTED state (by the cpu placing the 7017 * faulted cpu into that state). Cross-trap to the faulted cpu to clear 7018 * CEEN from the EER to disable traps for further disrupting error types 7019 * on that cpu. We could cross-call instead, but that has a larger 7020 * instruction and data footprint than cross-trapping, and the cpu is known 7021 * to be faulted. 7022 */ 7023 7024 void 7025 cpu_faulted_enter(struct cpu *cp) 7026 { 7027 xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_CLRBITS); 7028 } 7029 7030 /* 7031 * Called when a cpu leaves the CPU_FAULTED state to return to one of 7032 * offline, spare, or online (by the cpu requesting this state change). 7033 * First we cross-call to clear the AFSR (and AFSR_EXT on Panther) of 7034 * disrupting error bits that have accumulated without trapping, then 7035 * we cross-trap to re-enable CEEN controlled traps. 7036 */ 7037 void 7038 cpu_faulted_exit(struct cpu *cp) 7039 { 7040 ch_cpu_errors_t cpu_error_regs; 7041 7042 cpu_error_regs.afsr = C_AFSR_CECC_ERRS; 7043 if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) 7044 cpu_error_regs.afsr_ext &= C_AFSR_EXT_CECC_ERRS; 7045 xc_one(cp->cpu_id, (xcfunc_t *)set_cpu_error_state, 7046 (uint64_t)&cpu_error_regs, 0); 7047 7048 xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_SETBITS); 7049 } 7050 7051 /* 7052 * Return 1 if the errors in ch_flt's AFSR are secondary errors caused by 7053 * the errors in the original AFSR, 0 otherwise. 7054 * 7055 * For all procs if the initial error was a BERR or TO, then it is possible 7056 * that we may have caused a secondary BERR or TO in the process of logging the 7057 * inital error via cpu_run_bus_error_handlers(). If this is the case then 7058 * if the request was protected then a panic is still not necessary, if not 7059 * protected then aft_panic is already set - so either way there's no need 7060 * to set aft_panic for the secondary error. 7061 * 7062 * For Cheetah and Jalapeno, if the original error was a UE which occurred on 7063 * a store merge, then the error handling code will call cpu_deferred_error(). 7064 * When clear_errors() is called, it will determine that secondary errors have 7065 * occurred - in particular, the store merge also caused a EDU and WDU that 7066 * weren't discovered until this point. 7067 * 7068 * We do three checks to verify that we are in this case. If we pass all three 7069 * checks, we return 1 to indicate that we should not panic. If any unexpected 7070 * errors occur, we return 0. 7071 * 7072 * For Cheetah+ and derivative procs, the store merge causes a DUE, which is 7073 * handled in cpu_disrupting_errors(). Since this function is not even called 7074 * in the case we are interested in, we just return 0 for these processors. 7075 */ 7076 /*ARGSUSED*/ 7077 static int 7078 cpu_check_secondary_errors(ch_async_flt_t *ch_flt, uint64_t t_afsr_errs, 7079 uint64_t t_afar) 7080 { 7081 #if defined(CHEETAH_PLUS) 7082 #else /* CHEETAH_PLUS */ 7083 struct async_flt *aflt = (struct async_flt *)ch_flt; 7084 #endif /* CHEETAH_PLUS */ 7085 7086 /* 7087 * Was the original error a BERR or TO and only a BERR or TO 7088 * (multiple errors are also OK) 7089 */ 7090 if ((t_afsr_errs & ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) { 7091 /* 7092 * Is the new error a BERR or TO and only a BERR or TO 7093 * (multiple errors are also OK) 7094 */ 7095 if ((ch_flt->afsr_errs & 7096 ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) 7097 return (1); 7098 } 7099 7100 #if defined(CHEETAH_PLUS) 7101 return (0); 7102 #else /* CHEETAH_PLUS */ 7103 /* 7104 * Now look for secondary effects of a UE on cheetah/jalapeno 7105 * 7106 * Check the original error was a UE, and only a UE. Note that 7107 * the ME bit will cause us to fail this check. 7108 */ 7109 if (t_afsr_errs != C_AFSR_UE) 7110 return (0); 7111 7112 /* 7113 * Check the secondary errors were exclusively an EDU and/or WDU. 7114 */ 7115 if ((ch_flt->afsr_errs & ~(C_AFSR_EDU|C_AFSR_WDU)) != 0) 7116 return (0); 7117 7118 /* 7119 * Check the AFAR of the original error and secondary errors 7120 * match to the 64-byte boundary 7121 */ 7122 if (P2ALIGN(aflt->flt_addr, 64) != P2ALIGN(t_afar, 64)) 7123 return (0); 7124 7125 /* 7126 * We've passed all the checks, so it's a secondary error! 7127 */ 7128 return (1); 7129 #endif /* CHEETAH_PLUS */ 7130 } 7131 7132 /* 7133 * Translate the flt_bit or flt_type into an error type. First, flt_bit 7134 * is checked for any valid errors. If found, the error type is 7135 * returned. If not found, the flt_type is checked for L1$ parity errors. 7136 */ 7137 /*ARGSUSED*/ 7138 static uint8_t 7139 cpu_flt_bit_to_plat_error(struct async_flt *aflt) 7140 { 7141 #if defined(JALAPENO) 7142 /* 7143 * Currently, logging errors to the SC is not supported on Jalapeno 7144 */ 7145 return (PLAT_ECC_ERROR2_NONE); 7146 #else 7147 ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 7148 7149 switch (ch_flt->flt_bit) { 7150 case C_AFSR_CE: 7151 return (PLAT_ECC_ERROR2_CE); 7152 case C_AFSR_UCC: 7153 case C_AFSR_EDC: 7154 case C_AFSR_WDC: 7155 case C_AFSR_CPC: 7156 return (PLAT_ECC_ERROR2_L2_CE); 7157 case C_AFSR_EMC: 7158 return (PLAT_ECC_ERROR2_EMC); 7159 case C_AFSR_IVC: 7160 return (PLAT_ECC_ERROR2_IVC); 7161 case C_AFSR_UE: 7162 return (PLAT_ECC_ERROR2_UE); 7163 case C_AFSR_UCU: 7164 case C_AFSR_EDU: 7165 case C_AFSR_WDU: 7166 case C_AFSR_CPU: 7167 return (PLAT_ECC_ERROR2_L2_UE); 7168 case C_AFSR_IVU: 7169 return (PLAT_ECC_ERROR2_IVU); 7170 case C_AFSR_TO: 7171 return (PLAT_ECC_ERROR2_TO); 7172 case C_AFSR_BERR: 7173 return (PLAT_ECC_ERROR2_BERR); 7174 #if defined(CHEETAH_PLUS) 7175 case C_AFSR_L3_EDC: 7176 case C_AFSR_L3_UCC: 7177 case C_AFSR_L3_CPC: 7178 case C_AFSR_L3_WDC: 7179 return (PLAT_ECC_ERROR2_L3_CE); 7180 case C_AFSR_IMC: 7181 return (PLAT_ECC_ERROR2_IMC); 7182 case C_AFSR_TSCE: 7183 return (PLAT_ECC_ERROR2_L2_TSCE); 7184 case C_AFSR_THCE: 7185 return (PLAT_ECC_ERROR2_L2_THCE); 7186 case C_AFSR_L3_MECC: 7187 return (PLAT_ECC_ERROR2_L3_MECC); 7188 case C_AFSR_L3_THCE: 7189 return (PLAT_ECC_ERROR2_L3_THCE); 7190 case C_AFSR_L3_CPU: 7191 case C_AFSR_L3_EDU: 7192 case C_AFSR_L3_UCU: 7193 case C_AFSR_L3_WDU: 7194 return (PLAT_ECC_ERROR2_L3_UE); 7195 case C_AFSR_DUE: 7196 return (PLAT_ECC_ERROR2_DUE); 7197 case C_AFSR_DTO: 7198 return (PLAT_ECC_ERROR2_DTO); 7199 case C_AFSR_DBERR: 7200 return (PLAT_ECC_ERROR2_DBERR); 7201 #endif /* CHEETAH_PLUS */ 7202 default: 7203 switch (ch_flt->flt_type) { 7204 #if defined(CPU_IMP_L1_CACHE_PARITY) 7205 case CPU_IC_PARITY: 7206 return (PLAT_ECC_ERROR2_IPE); 7207 case CPU_DC_PARITY: 7208 if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 7209 if (ch_flt->parity_data.dpe.cpl_cache == 7210 CPU_PC_PARITY) { 7211 return (PLAT_ECC_ERROR2_PCACHE); 7212 } 7213 } 7214 return (PLAT_ECC_ERROR2_DPE); 7215 #endif /* CPU_IMP_L1_CACHE_PARITY */ 7216 case CPU_ITLB_PARITY: 7217 return (PLAT_ECC_ERROR2_ITLB); 7218 case CPU_DTLB_PARITY: 7219 return (PLAT_ECC_ERROR2_DTLB); 7220 default: 7221 return (PLAT_ECC_ERROR2_NONE); 7222 } 7223 } 7224 #endif /* JALAPENO */ 7225 } 7226