/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef CHEETAHPLUS_ERRATUM_25 #include #endif /* CHEETAHPLUS_ERRATUM_25 */ /* * See comment above cpu_scrub_cpu_setup() for description */ #define SCRUBBER_NEITHER_CORE_ONLINE 0x0 #define SCRUBBER_CORE_0_ONLINE 0x1 #define SCRUBBER_CORE_1_ONLINE 0x2 #define SCRUBBER_BOTH_CORES_ONLINE (SCRUBBER_CORE_0_ONLINE | \ SCRUBBER_CORE_1_ONLINE) static int pn_matching_valid_l2_line(uint64_t faddr, ch_ec_data_t *clo_l2_data); static void cpu_async_log_tlb_parity_err(void *flt); static cpu_t *cpu_get_sibling_core(cpu_t *cpup); /* * Setup trap handlers. */ void cpu_init_trap(void) { CH_SET_TRAP(tt_pil15, ch_pil15_interrupt_instr); CH_SET_TRAP(tt0_fecc, fecc_err_instr); CH_SET_TRAP(tt1_fecc, fecc_err_tl1_instr); CH_SET_TRAP(tt1_swtrap0, fecc_err_tl1_cont_instr); CH_SET_TRAP(tt0_dperr, dcache_parity_instr); CH_SET_TRAP(tt1_dperr, dcache_parity_tl1_instr); CH_SET_TRAP(tt1_swtrap1, dcache_parity_tl1_cont_instr); CH_SET_TRAP(tt0_iperr, icache_parity_instr); CH_SET_TRAP(tt1_iperr, icache_parity_tl1_instr); CH_SET_TRAP(tt1_swtrap2, icache_parity_tl1_cont_instr); } /* * Set the magic constants of the implementation. */ /*ARGSUSED*/ void cpu_fiximp(dnode_t dnode) { int i, a; extern int vac_size, vac_shift; extern uint_t vac_mask; dcache_size = CH_DCACHE_SIZE; dcache_linesize = CH_DCACHE_LSIZE; icache_size = CHP_ICACHE_MAX_SIZE; icache_linesize = CHP_ICACHE_MIN_LSIZE; ecache_size = CH_ECACHE_MAX_SIZE; ecache_alignsize = CH_ECACHE_MAX_LSIZE; ecache_associativity = CHP_ECACHE_MIN_NWAY; /* * ecache_setsize needs to maximum of all cpu ecache setsizes */ ecache_setsize = CHP_ECACHE_MAX_SETSIZE; ASSERT(ecache_setsize >= (ecache_size / ecache_associativity)); vac_size = CH_VAC_SIZE; vac_mask = MMU_PAGEMASK & (vac_size - 1); i = 0; a = vac_size; while (a >>= 1) ++i; vac_shift = i; shm_alignment = vac_size; vac = 1; } /* * Use Panther values for Panther-only domains. * See Panther PRM, 1.5.4 Cache Hierarchy */ void cpu_fix_allpanther(void) { /* dcache same as Ch+ */ icache_size = PN_ICACHE_SIZE; icache_linesize = PN_ICACHE_LSIZE; ecache_size = PN_L3_SIZE; ecache_alignsize = PN_L3_LINESIZE; ecache_associativity = PN_L3_NWAYS; ecache_setsize = PN_L3_SET_SIZE; ASSERT(ecache_setsize >= (ecache_size / ecache_associativity)); /* vac same as Ch+ */ } void send_mondo_set(cpuset_t set) { int lo, busy, nack, shipped = 0; uint16_t i, cpuids[IDSR_BN_SETS]; uint64_t idsr, nackmask = 0, busymask, curnack, curbusy; uint64_t starttick, endtick, tick, lasttick; #if (NCPU > IDSR_BN_SETS) int index = 0; int ncpuids = 0; #endif #ifdef CHEETAHPLUS_ERRATUM_25 int recovered = 0; int cpuid; #endif ASSERT(!CPUSET_ISNULL(set)); starttick = lasttick = gettick(); #if (NCPU <= IDSR_BN_SETS) for (i = 0; i < NCPU; i++) if (CPU_IN_SET(set, i)) { shipit(i, shipped); nackmask |= IDSR_NACK_BIT(shipped); cpuids[shipped++] = i; CPUSET_DEL(set, i); if (CPUSET_ISNULL(set)) break; } CPU_STATS_ADDQ(CPU, sys, xcalls, shipped); #else for (i = 0; i < NCPU; i++) if (CPU_IN_SET(set, i)) { ncpuids++; /* * Ship only to the first (IDSR_BN_SETS) CPUs. If we * find we have shipped to more than (IDSR_BN_SETS) * CPUs, set "index" to the highest numbered CPU in * the set so we can ship to other CPUs a bit later on. */ if (shipped < IDSR_BN_SETS) { shipit(i, shipped); nackmask |= IDSR_NACK_BIT(shipped); cpuids[shipped++] = i; CPUSET_DEL(set, i); if (CPUSET_ISNULL(set)) break; } else index = (int)i; } CPU_STATS_ADDQ(CPU, sys, xcalls, ncpuids); #endif busymask = IDSR_NACK_TO_BUSY(nackmask); busy = nack = 0; endtick = starttick + xc_tick_limit; for (;;) { idsr = getidsr(); #if (NCPU <= IDSR_BN_SETS) if (idsr == 0) break; #else if (idsr == 0 && shipped == ncpuids) break; #endif tick = gettick(); /* * If there is a big jump between the current tick * count and lasttick, we have probably hit a break * point. Adjust endtick accordingly to avoid panic. */ if (tick > (lasttick + xc_tick_jump_limit)) endtick += (tick - lasttick); lasttick = tick; if (tick > endtick) { if (panic_quiesce) return; #ifdef CHEETAHPLUS_ERRATUM_25 cpuid = -1; for (i = 0; i < IDSR_BN_SETS; i++) { if (idsr & (IDSR_NACK_BIT(i) | IDSR_BUSY_BIT(i))) { cpuid = cpuids[i]; break; } } if (cheetah_sendmondo_recover && cpuid != -1 && recovered == 0) { if (mondo_recover(cpuid, i)) { /* * We claimed the whole memory or * full scan is disabled. */ recovered++; } tick = gettick(); endtick = tick + xc_tick_limit; lasttick = tick; /* * Recheck idsr */ continue; } else #endif /* CHEETAHPLUS_ERRATUM_25 */ { cmn_err(CE_CONT, "send mondo timeout " "[%d NACK %d BUSY]\nIDSR 0x%" "" PRIx64 " cpuids:", nack, busy, idsr); for (i = 0; i < IDSR_BN_SETS; i++) { if (idsr & (IDSR_NACK_BIT(i) | IDSR_BUSY_BIT(i))) { cmn_err(CE_CONT, " 0x%x", cpuids[i]); } } cmn_err(CE_CONT, "\n"); cmn_err(CE_PANIC, "send_mondo_set: timeout"); } } curnack = idsr & nackmask; curbusy = idsr & busymask; #if (NCPU > IDSR_BN_SETS) if (shipped < ncpuids) { uint64_t cpus_left; uint16_t next = (uint16_t)index; cpus_left = ~(IDSR_NACK_TO_BUSY(curnack) | curbusy) & busymask; if (cpus_left) { do { /* * Sequence through and ship to the * remainder of the CPUs in the system * (e.g. other than the first * (IDSR_BN_SETS)) in reverse order. */ lo = lowbit(cpus_left) - 1; i = IDSR_BUSY_IDX(lo); shipit(next, i); shipped++; cpuids[i] = next; /* * If we've processed all the CPUs, * exit the loop now and save * instructions. */ if (shipped == ncpuids) break; for ((index = ((int)next - 1)); index >= 0; index--) if (CPU_IN_SET(set, index)) { next = (uint16_t)index; break; } cpus_left &= ~(1ull << lo); } while (cpus_left); #ifdef CHEETAHPLUS_ERRATUM_25 /* * Clear recovered because we are sending to * a new set of targets. */ recovered = 0; #endif continue; } } #endif if (curbusy) { busy++; continue; } #ifdef SEND_MONDO_STATS { int n = gettick() - starttick; if (n < 8192) x_nack_stimes[n >> 7]++; } #endif while (gettick() < (tick + sys_clock_mhz)) ; do { lo = lowbit(curnack) - 1; i = IDSR_NACK_IDX(lo); shipit(cpuids[i], i); curnack &= ~(1ull << lo); } while (curnack); nack++; busy = 0; } #ifdef SEND_MONDO_STATS { int n = gettick() - starttick; if (n < 8192) x_set_stimes[n >> 7]++; else x_set_ltimes[(n >> 13) & 0xf]++; } x_set_cpus[shipped]++; #endif } /* * Handles error logging for implementation specific error types */ /*ARGSUSED1*/ int cpu_impl_async_log_err(void *flt, errorq_elem_t *eqep) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt; struct async_flt *aflt = (struct async_flt *)flt; switch (ch_flt->flt_type) { case CPU_IC_PARITY: cpu_async_log_ic_parity_err(flt); return (CH_ASYNC_LOG_DONE); case CPU_DC_PARITY: cpu_async_log_dc_parity_err(flt); return (CH_ASYNC_LOG_DONE); case CPU_DUE: cpu_log_err(aflt); cpu_page_retire(ch_flt); return (CH_ASYNC_LOG_DONE); case CPU_ITLB_PARITY: case CPU_DTLB_PARITY: cpu_async_log_tlb_parity_err(flt); return (CH_ASYNC_LOG_DONE); default: return (CH_ASYNC_LOG_UNKNOWN); } } /* * Figure out if Ecache is direct-mapped (Cheetah or Cheetah+ with Ecache * control ECCR_ASSOC bit off or 2-way (Cheetah+ with ECCR_ASSOC on). * We need to do this on the fly because we may have mixed Cheetah+'s with * both direct and 2-way Ecaches. Panther only supports 4-way L3$. */ int cpu_ecache_nway(void) { if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) return (PN_L3_NWAYS); return ((get_ecache_ctrl() & ECCR_ASSOC) ? 2 : 1); } /* * Note that these are entered into the table: Fatal Errors (PERR, IERR, ISAP, * EMU, IMU) first, orphaned UCU/UCC, AFAR Overwrite policy, finally IVU, IVC. * Afar overwrite policy is: * Class 4: * AFSR -- UCC, UCU, TUE, TSCE, TUE_SH * AFSR_EXT -- L3_UCC, L3_UCU, L3_TUE, L3_TUE_SH * Class 3: * AFSR -- UE, DUE, EDU, WDU, CPU * AFSR_EXT -- L3_EDU, L3_WDU, L3_CPU * Class 2: * AFSR -- CE, EDC, EMC, WDC, CPC, THCE * AFSR_EXT -- L3_EDC, L3_WDC, L3_CPC, L3_THCE * Class 1: * AFSR -- TO, DTO, BERR, DBERR */ ecc_type_to_info_t ecc_type_to_info[] = { /* Fatal Errors */ C_AFSR_PERR, "PERR ", ECC_ALL_TRAPS, CPU_FATAL, "PERR Fatal", FM_EREPORT_PAYLOAD_SYSTEM2, FM_EREPORT_CPU_USIII_PERR, C_AFSR_IERR, "IERR ", ECC_ALL_TRAPS, CPU_FATAL, "IERR Fatal", FM_EREPORT_PAYLOAD_SYSTEM2, FM_EREPORT_CPU_USIII_IERR, C_AFSR_ISAP, "ISAP ", ECC_ALL_TRAPS, CPU_FATAL, "ISAP Fatal", FM_EREPORT_PAYLOAD_SYSTEM1, FM_EREPORT_CPU_USIII_ISAP, C_AFSR_L3_TUE_SH, "L3_TUE_SH ", ECC_C_TRAP, CPU_FATAL, "L3_TUE_SH Fatal", FM_EREPORT_PAYLOAD_L3_TAG_ECC, FM_EREPORT_CPU_USIII_L3_TUE_SH, C_AFSR_L3_TUE, "L3_TUE ", ECC_C_TRAP, CPU_FATAL, "L3_TUE Fatal", FM_EREPORT_PAYLOAD_L3_TAG_ECC, FM_EREPORT_CPU_USIII_L3_TUE, C_AFSR_TUE_SH, "TUE_SH ", ECC_C_TRAP, CPU_FATAL, "TUE_SH Fatal", FM_EREPORT_PAYLOAD_L2_TAG_ECC, FM_EREPORT_CPU_USIII_TUE_SH, C_AFSR_TUE, "TUE ", ECC_ALL_TRAPS, CPU_FATAL, "TUE Fatal", FM_EREPORT_PAYLOAD_L2_TAG_ECC, FM_EREPORT_CPU_USIII_TUE, C_AFSR_EMU, "EMU ", ECC_ASYNC_TRAPS, CPU_FATAL, "EMU Fatal", FM_EREPORT_PAYLOAD_MEMORY, FM_EREPORT_CPU_USIII_EMU, C_AFSR_IMU, "IMU ", ECC_C_TRAP, CPU_FATAL, "IMU Fatal", FM_EREPORT_PAYLOAD_SYSTEM1, FM_EREPORT_CPU_USIII_IMU, /* L3$ Address parity errors are reported via the MECC bit */ C_AFSR_L3_MECC, "L3_MECC ", ECC_MECC_TRAPS, CPU_L3_ADDR_PE, "L3 Address Parity", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_MECC, /* Orphaned UCC/UCU Errors */ C_AFSR_L3_UCU, "L3_OUCU ", ECC_ORPH_TRAPS, CPU_ORPH, "Orphaned L3_UCU", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_UCU, C_AFSR_L3_UCC, "L3_OUCC ", ECC_ORPH_TRAPS, CPU_ORPH, "Orphaned L3_UCC", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_UCC, C_AFSR_UCU, "OUCU ", ECC_ORPH_TRAPS, CPU_ORPH, "Orphaned UCU", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_UCU, C_AFSR_UCC, "OUCC ", ECC_ORPH_TRAPS, CPU_ORPH, "Orphaned UCC", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_UCC, /* UCU, UCC */ C_AFSR_L3_UCU, "L3_UCU ", ECC_F_TRAP, CPU_UE_ECACHE, "L3_UCU", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_UCU, C_AFSR_L3_UCC, "L3_UCC ", ECC_F_TRAP, CPU_CE_ECACHE, "L3_UCC", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_UCC, C_AFSR_UCU, "UCU ", ECC_F_TRAP, CPU_UE_ECACHE, "UCU", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_UCU, C_AFSR_UCC, "UCC ", ECC_F_TRAP, CPU_CE_ECACHE, "UCC", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_UCC, C_AFSR_TSCE, "TSCE ", ECC_F_TRAP, CPU_CE_ECACHE, "TSCE", FM_EREPORT_PAYLOAD_L2_TAG_ECC, FM_EREPORT_CPU_USIII_TSCE, /* UE, EDU:ST, EDU:BLD, WDU, CPU */ C_AFSR_UE, "UE ", ECC_ASYNC_TRAPS, CPU_UE, "Uncorrectable system bus (UE)", FM_EREPORT_PAYLOAD_MEMORY, FM_EREPORT_CPU_USIII_UE, C_AFSR_L3_EDU, "L3_EDU ", ECC_C_TRAP, CPU_UE_ECACHE_RETIRE, "L3_EDU:ST", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_EDUST, C_AFSR_L3_EDU, "L3_EDU ", ECC_D_TRAP, CPU_UE_ECACHE_RETIRE, "L3_EDU:BLD", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_EDUBL, C_AFSR_L3_WDU, "L3_WDU ", ECC_C_TRAP, CPU_UE_ECACHE_RETIRE, "L3_WDU", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_WDU, C_AFSR_L3_CPU, "L3_CPU ", ECC_C_TRAP, CPU_UE_ECACHE, "L3_CPU", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_CPU, C_AFSR_EDU, "EDU ", ECC_C_TRAP, CPU_UE_ECACHE_RETIRE, "EDU:ST", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_EDUST, C_AFSR_EDU, "EDU ", ECC_D_TRAP, CPU_UE_ECACHE_RETIRE, "EDU:BLD", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_EDUBL, C_AFSR_WDU, "WDU ", ECC_C_TRAP, CPU_UE_ECACHE_RETIRE, "WDU", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_WDU, C_AFSR_CPU, "CPU ", ECC_C_TRAP, CPU_UE_ECACHE, "CPU", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_CPU, C_AFSR_DUE, "DUE ", ECC_C_TRAP, CPU_DUE, "DUE", FM_EREPORT_PAYLOAD_MEMORY, FM_EREPORT_CPU_USIII_DUE, /* CE, EDC, EMC, WDC, CPC */ C_AFSR_CE, "CE ", ECC_C_TRAP, CPU_CE, "Corrected system bus (CE)", FM_EREPORT_PAYLOAD_MEMORY, FM_EREPORT_CPU_USIII_CE, C_AFSR_L3_EDC, "L3_EDC ", ECC_C_TRAP, CPU_CE_ECACHE, "L3_EDC", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_EDC, C_AFSR_EDC, "EDC ", ECC_C_TRAP, CPU_CE_ECACHE, "EDC", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_EDC, C_AFSR_EMC, "EMC ", ECC_C_TRAP, CPU_EMC, "EMC", FM_EREPORT_PAYLOAD_MEMORY, FM_EREPORT_CPU_USIII_EMC, C_AFSR_L3_WDC, "L3_WDC ", ECC_C_TRAP, CPU_CE_ECACHE, "L3_WDC", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_WDC, C_AFSR_L3_CPC, "L3_CPC ", ECC_C_TRAP, CPU_CE_ECACHE, "L3_CPC", FM_EREPORT_PAYLOAD_L3_DATA, FM_EREPORT_CPU_USIII_L3_CPC, C_AFSR_L3_THCE, "L3_THCE ", ECC_C_TRAP, CPU_CE_ECACHE, "L3_THCE", FM_EREPORT_PAYLOAD_L3_TAG_ECC, FM_EREPORT_CPU_USIII_L3_THCE, C_AFSR_WDC, "WDC ", ECC_C_TRAP, CPU_CE_ECACHE, "WDC", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_WDC, C_AFSR_CPC, "CPC ", ECC_C_TRAP, CPU_CE_ECACHE, "CPC", FM_EREPORT_PAYLOAD_L2_DATA, FM_EREPORT_CPU_USIII_CPC, C_AFSR_THCE, "THCE ", ECC_C_TRAP, CPU_CE_ECACHE, "THCE", FM_EREPORT_PAYLOAD_L2_TAG_ECC, FM_EREPORT_CPU_USIII_THCE, /* TO, BERR */ C_AFSR_TO, "TO ", ECC_ASYNC_TRAPS, CPU_TO, "Timeout (TO)", FM_EREPORT_PAYLOAD_IO, FM_EREPORT_CPU_USIII_TO, C_AFSR_BERR, "BERR ", ECC_ASYNC_TRAPS, CPU_BERR, "Bus Error (BERR)", FM_EREPORT_PAYLOAD_IO, FM_EREPORT_CPU_USIII_BERR, C_AFSR_DTO, "DTO ", ECC_C_TRAP, CPU_TO, "Disrupting Timeout (DTO)", FM_EREPORT_PAYLOAD_IO, FM_EREPORT_CPU_USIII_DTO, C_AFSR_DBERR, "DBERR ", ECC_C_TRAP, CPU_BERR, "Disrupting Bus Error (DBERR)", FM_EREPORT_PAYLOAD_IO, FM_EREPORT_CPU_USIII_DBERR, /* IVU, IVC, IMC */ C_AFSR_IVU, "IVU ", ECC_C_TRAP, CPU_IV, "IVU", FM_EREPORT_PAYLOAD_SYSTEM1, FM_EREPORT_CPU_USIII_IVU, C_AFSR_IVC, "IVC ", ECC_C_TRAP, CPU_IV, "IVC", FM_EREPORT_PAYLOAD_SYSTEM1, FM_EREPORT_CPU_USIII_IVC, C_AFSR_IMC, "IMC ", ECC_C_TRAP, CPU_IV, "IMC", FM_EREPORT_PAYLOAD_SYSTEM1, FM_EREPORT_CPU_USIII_IMC, 0, NULL, 0, 0, NULL, FM_EREPORT_PAYLOAD_UNKNOWN, FM_EREPORT_CPU_USIII_UNKNOWN, }; /* * See Cheetah+ Delta PRM 10.9 and section P.6.1 of the Panther PRM * Class 4: * AFSR -- UCC, UCU, TUE, TSCE, TUE_SH * AFSR_EXT -- L3_UCC, L3_UCU, L3_TUE, L3_TUE_SH * Class 3: * AFSR -- UE, DUE, EDU, EMU, WDU, CPU * AFSR_EXT -- L3_EDU, L3_WDU, L3_CPU * Class 2: * AFSR -- CE, EDC, EMC, WDC, CPC, THCE * AFSR_EXT -- L3_EDC, L3_WDC, L3_CPC, L3_THCE * Class 1: * AFSR -- TO, DTO, BERR, DBERR * AFSR_EXT -- */ uint64_t afar_overwrite[] = { /* class 4: */ C_AFSR_UCC | C_AFSR_UCU | C_AFSR_TUE | C_AFSR_TSCE | C_AFSR_TUE_SH | C_AFSR_L3_UCC | C_AFSR_L3_UCU | C_AFSR_L3_TUE | C_AFSR_L3_TUE_SH, /* class 3: */ C_AFSR_UE | C_AFSR_DUE | C_AFSR_EDU | C_AFSR_EMU | C_AFSR_WDU | C_AFSR_CPU | C_AFSR_L3_EDU | C_AFSR_L3_WDU | C_AFSR_L3_CPU, /* class 2: */ C_AFSR_CE | C_AFSR_EDC | C_AFSR_EMC | C_AFSR_WDC | C_AFSR_CPC | C_AFSR_THCE | C_AFSR_L3_EDC | C_AFSR_L3_WDC | C_AFSR_L3_CPC | C_AFSR_L3_THCE, /* class 1: */ C_AFSR_TO | C_AFSR_DTO | C_AFSR_BERR | C_AFSR_DBERR, 0 }; /* * See Cheetah+ Delta PRM 10.9. * Class 2: UE, DUE, IVU, EDU, WDU, UCU, CPU * Class 1: CE, IVC, EDC, WDC, UCC, CPC */ uint64_t esynd_overwrite[] = { /* class 2: */ C_AFSR_UE | C_AFSR_DUE | C_AFSR_IVU | C_AFSR_EDU | C_AFSR_WDU | C_AFSR_UCU | C_AFSR_CPU, /* class 1: */ C_AFSR_CE | C_AFSR_IVC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_UCC | C_AFSR_CPC, 0 }; /* * In panther, the E_SYND overwrite policy changed a little bit * by adding one more level. * class 3: * AFSR -- UCU, UCC * AFSR_EXT -- L3_UCU, L3_UCC * Class 2: * AFSR -- UE, DUE, IVU, EDU, WDU, CPU * AFSR_EXT -- L3_EDU, L3_WDU, L3_CPU * Class 1: * AFSR -- CE, IVC, EDC, WDC, CPC * AFSR_EXT -- L3_EDC, L3_WDC, L3_CPC */ uint64_t pn_esynd_overwrite[] = { /* class 3: */ C_AFSR_UCU | C_AFSR_UCC | C_AFSR_L3_UCU | C_AFSR_L3_UCC, /* class 2: */ C_AFSR_UE | C_AFSR_DUE | C_AFSR_IVU | C_AFSR_EDU | C_AFSR_WDU | C_AFSR_CPU | C_AFSR_L3_EDU | C_AFSR_L3_WDU | C_AFSR_L3_CPU, /* class 1: */ C_AFSR_CE | C_AFSR_IVC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_CPC | C_AFSR_L3_EDC | C_AFSR_L3_WDC | C_AFSR_L3_CPC, 0 }; int afsr_to_pn_esynd_status(uint64_t afsr, uint64_t afsr_bit) { return (afsr_to_overw_status(afsr, afsr_bit, pn_esynd_overwrite)); } /* * Prioritized list of Error bits for MSYND overwrite. * See Cheetah PRM P.6.3 * Class 2: EMU * Class 1: EMC * * Panther adds IMU and IMC. */ uint64_t msynd_overwrite[] = { /* class 2: */ C_AFSR_EMU | C_AFSR_IMU, /* class 1: */ C_AFSR_EMC | C_AFSR_IMC, 0 }; /* * change cpu speed bits -- new speed will be normal-speed/divisor. * * The Jalapeno memory controllers are required to drain outstanding * memory transactions within 32 JBus clocks in order to be ready * to enter Estar mode. In some corner cases however, that time * fell short. * * A safe software solution is to force MCU to act like in Estar mode, * then delay 1us (in ppm code) prior to assert J_CHNG_L signal. * To reverse the effect, upon exiting Estar, software restores the * MCU to its original state. */ /* ARGSUSED1 */ void cpu_change_speed(uint64_t divisor, uint64_t arg2) { bus_config_eclk_t *bceclk; uint64_t reg; for (bceclk = bus_config_eclk; bceclk->divisor; bceclk++) { if (bceclk->divisor != divisor) continue; reg = get_safari_config(); reg &= ~SAFARI_CONFIG_ECLK_MASK; reg |= bceclk->mask; set_safari_config(reg); CPU->cpu_m.divisor = (uchar_t)divisor; return; } /* * We will reach here only if OBP and kernel don't agree on * the speeds supported by the CPU. */ cmn_err(CE_WARN, "cpu_change_speed: bad divisor %" PRIu64, divisor); } /* * Cpu private initialization. This includes allocating the cpu_private * data structure, initializing it, and initializing the scrubber for this * cpu. This function calls cpu_init_ecache_scrub_dr to init the scrubber. * We use kmem_cache_create for the cheetah private data structure because * it needs to be allocated on a PAGESIZE (8192) byte boundary. */ void cpu_init_private(struct cpu *cp) { cheetah_private_t *chprp; int i; ASSERT(CPU_PRIVATE(cp) == NULL); /* LINTED: E_TRUE_LOGICAL_EXPR */ ASSERT((offsetof(cheetah_private_t, chpr_tl1_err_data) + sizeof (ch_err_tl1_data_t) * CH_ERR_TL1_TLMAX) <= PAGESIZE); /* * Running with Cheetah CPUs in a Cheetah+, Jaguar, Panther or * mixed Cheetah+/Jaguar/Panther machine is not a supported * configuration. Attempting to do so may result in unpredictable * failures (e.g. running Cheetah+ CPUs with Cheetah E$ disp flush) * so don't allow it. * * This is just defensive code since this configuration mismatch * should have been caught prior to OS execution. */ if (!(IS_CHEETAH_PLUS(cpunodes[cp->cpu_id].implementation) || IS_JAGUAR(cpunodes[cp->cpu_id].implementation) || IS_PANTHER(cpunodes[cp->cpu_id].implementation))) { cmn_err(CE_PANIC, "CPU%d: UltraSPARC-III not supported" " on UltraSPARC-III+/IV/IV+ code\n", cp->cpu_id); } /* * If the ch_private_cache has not been created, create it. */ if (ch_private_cache == NULL) { ch_private_cache = kmem_cache_create("ch_private_cache", sizeof (cheetah_private_t), PAGESIZE, NULL, NULL, NULL, NULL, static_arena, 0); } chprp = CPU_PRIVATE(cp) = kmem_cache_alloc(ch_private_cache, KM_SLEEP); bzero(chprp, sizeof (cheetah_private_t)); chprp->chpr_fecctl0_logout.clo_data.chd_afar = LOGOUT_INVALID; chprp->chpr_cecc_logout.clo_data.chd_afar = LOGOUT_INVALID; chprp->chpr_async_logout.clo_data.chd_afar = LOGOUT_INVALID; chprp->chpr_tlb_logout.tlo_addr = LOGOUT_INVALID; for (i = 0; i < CH_ERR_TL1_TLMAX; i++) chprp->chpr_tl1_err_data[i].ch_err_tl1_logout.clo_data.chd_afar = LOGOUT_INVALID; /* Panther has a larger Icache compared to cheetahplus or Jaguar */ if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) { chprp->chpr_icache_size = PN_ICACHE_SIZE; chprp->chpr_icache_linesize = PN_ICACHE_LSIZE; } else { chprp->chpr_icache_size = CH_ICACHE_SIZE; chprp->chpr_icache_linesize = CH_ICACHE_LSIZE; } cpu_init_ecache_scrub_dr(cp); /* * Panther's L2$ and E$ are shared between cores, so the scrubber is * only needed on one of the cores. At this point, we assume all cores * are online, and we only enable the scrubber on core 0. */ if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) { chprp->chpr_scrub_misc.chsm_core_state = SCRUBBER_BOTH_CORES_ONLINE; if (cp->cpu_id != (processorid_t)cmp_cpu_to_chip(cp->cpu_id)) { chprp->chpr_scrub_misc.chsm_enable[ CACHE_SCRUBBER_INFO_E] = 0; } } chprp->chpr_ec_set_size = cpunodes[cp->cpu_id].ecache_size / cpu_ecache_nway(); adjust_hw_copy_limits(cpunodes[cp->cpu_id].ecache_size); ch_err_tl1_paddrs[cp->cpu_id] = va_to_pa(chprp); ASSERT(ch_err_tl1_paddrs[cp->cpu_id] != -1); } /* * Clear the error state registers for this CPU. * For Cheetah+/Jaguar, just clear the AFSR but * for Panther we also have to clear the AFSR_EXT. */ void set_cpu_error_state(ch_cpu_errors_t *cpu_error_regs) { set_asyncflt(cpu_error_regs->afsr & ~C_AFSR_FATAL_ERRS); if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { set_afsr_ext(cpu_error_regs->afsr_ext & ~C_AFSR_EXT_FATAL_ERRS); } } void pn_cpu_log_diag_l2_info(ch_async_flt_t *ch_flt) { struct async_flt *aflt = (struct async_flt *)ch_flt; ch_ec_data_t *l2_data = &ch_flt->flt_diag_data.chd_l2_data[0]; uint64_t faddr = aflt->flt_addr; uint8_t log_way_mask = 0; int i; /* * Only Panther CPUs have the additional L2$ data that needs * to be logged here */ if (!IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) return; /* * We'll use a simple bit mask to keep track of which way(s) * of the stored cache line we want to log. The idea is to * log the entry if it is a valid line and it matches our * fault AFAR. If no match is found, we will simply log all * the ways. */ for (i = 0; i < PN_L2_NWAYS; i++) if (pn_matching_valid_l2_line(faddr, &l2_data[i])) log_way_mask |= (1 << i); /* If no matching valid lines were found, we log all ways */ if (log_way_mask == 0) log_way_mask = (1 << PN_L2_NWAYS) - 1; /* Log the cache lines */ for (i = 0; i < PN_L2_NWAYS; i++) if (log_way_mask & (1 << i)) l2_data[i].ec_logflag = EC_LOGFLAG_MAGIC; } /* * For this routine to return true, the L2 tag in question must be valid * and the tag PA must match the fault address (faddr) assuming the correct * index is being used. */ static int pn_matching_valid_l2_line(uint64_t faddr, ch_ec_data_t *clo_l2_data) { if ((!PN_L2_LINE_INVALID(clo_l2_data->ec_tag)) && ((faddr & P2ALIGN(C_AFAR_PA, PN_L2_SET_SIZE)) == PN_L2TAG_TO_PA(clo_l2_data->ec_tag))) return (1); return (0); } /* * This array is used to convert the 3 digit PgSz encoding (as used in * various MMU registers such as MMU_TAG_ACCESS_EXT) into the corresponding * page size. */ static uint64_t tlb_pgsz_to_size[] = { /* 000 = 8KB: */ 0x2000, /* 001 = 64KB: */ 0x10000, /* 010 = 512KB: */ 0x80000, /* 011 = 4MB: */ 0x400000, /* 100 = 32MB: */ 0x2000000, /* 101 = 256MB: */ 0x10000000, /* undefined for encodings 110 and 111: */ 0, 0 }; /* * The itlb_parity_trap and dtlb_parity_trap handlers transfer control here * after collecting logout information related to the TLB parity error and * flushing the offending TTE entries from the ITLB or DTLB. * * DTLB traps which occur at TL>0 are not recoverable because we will most * likely be corrupting some other trap handler's alternate globals. As * such, we simply panic here when that happens. ITLB parity errors are * not expected to happen at TL>0. */ void cpu_tlb_parity_error(struct regs *rp, ulong_t trap_va, ulong_t tlb_info) { ch_async_flt_t ch_flt; struct async_flt *aflt; pn_tlb_logout_t *tlop = NULL; int immu_parity = (tlb_info & PN_TLO_INFO_IMMU) != 0; int tl1_trap = (tlb_info & PN_TLO_INFO_TL1) != 0; char *error_class; bzero(&ch_flt, sizeof (ch_async_flt_t)); /* * Get the CPU log out info. If we can't find our CPU private * pointer, or if the logout information does not correspond to * this error, then we will have to make due without detailed * logout information. */ if (CPU_PRIVATE(CPU)) { tlop = CPU_PRIVATE_PTR(CPU, chpr_tlb_logout); if ((tlop->tlo_addr != trap_va) || (tlop->tlo_info != tlb_info)) tlop = NULL; } if (tlop) { ch_flt.tlb_diag_data = *tlop; /* Zero out + invalidate TLB logout. */ bzero(tlop, sizeof (pn_tlb_logout_t)); tlop->tlo_addr = LOGOUT_INVALID; } else { /* * Copy what logout information we have and mark * it incomplete. */ ch_flt.flt_data_incomplete = 1; ch_flt.tlb_diag_data.tlo_info = tlb_info; ch_flt.tlb_diag_data.tlo_addr = trap_va; } /* * Log the error. */ aflt = (struct async_flt *)&ch_flt; aflt->flt_id = gethrtime_waitfree(); aflt->flt_bus_id = getprocessorid(); aflt->flt_inst = CPU->cpu_id; aflt->flt_pc = (caddr_t)rp->r_pc; aflt->flt_addr = trap_va; aflt->flt_prot = AFLT_PROT_NONE; aflt->flt_class = CPU_FAULT; aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; aflt->flt_tl = tl1_trap ? 1 : 0; aflt->flt_panic = tl1_trap ? 1 : 0; if (immu_parity) { aflt->flt_status = ECC_ITLB_TRAP; ch_flt.flt_type = CPU_ITLB_PARITY; error_class = FM_EREPORT_CPU_USIII_ITLBPE; aflt->flt_payload = FM_EREPORT_PAYLOAD_ITLB_PE; } else { aflt->flt_status = ECC_DTLB_TRAP; ch_flt.flt_type = CPU_DTLB_PARITY; error_class = FM_EREPORT_CPU_USIII_DTLBPE; aflt->flt_payload = FM_EREPORT_PAYLOAD_DTLB_PE; } /* * The TLB entries have already been flushed by the TL1 trap * handler so at this point the only thing left to do is log * the error message. */ if (aflt->flt_panic) { cpu_errorq_dispatch(error_class, (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); /* * Panic here if aflt->flt_panic has been set. Enqueued * errors will be logged as part of the panic flow. */ fm_panic("%sError(s)", immu_parity ? "ITLBPE " : "DTLBPE "); } else { cpu_errorq_dispatch(error_class, (void *)&ch_flt, sizeof (ch_async_flt_t), ce_queue, aflt->flt_panic); } } /* * This routine is called when a TLB parity error event is 'ue_drain'ed * or 'ce_drain'ed from the errorq. */ void cpu_async_log_tlb_parity_err(void *flt) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt; struct async_flt *aflt = (struct async_flt *)flt; #ifdef lint aflt = aflt; #endif /* * We only capture TLB information if we encountered * a TLB parity error and Panther is the only CPU which * can detect a TLB parity error. */ ASSERT(IS_PANTHER(cpunodes[aflt->flt_inst].implementation)); ASSERT((ch_flt->flt_type == CPU_ITLB_PARITY) || (ch_flt->flt_type == CPU_DTLB_PARITY)); if (ch_flt->flt_data_incomplete == 0) { if (ch_flt->flt_type == CPU_ITLB_PARITY) ch_flt->tlb_diag_data.tlo_logflag = IT_LOGFLAG_MAGIC; else /* parity error is in DTLB */ ch_flt->tlb_diag_data.tlo_logflag = DT_LOGFLAG_MAGIC; } } /* * Add L1 Prefetch cache data to the ereport payload. */ void cpu_payload_add_pcache(struct async_flt *aflt, nvlist_t *nvl) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; ch_pc_data_t *pcp; ch_pc_data_t pcdata[CH_PCACHE_NWAY]; uint_t nelem; int i, ways_logged = 0; /* * We only capture P$ information if we encountered * a P$ parity error and Panther is the only CPU which * can detect a P$ parity error. */ ASSERT(IS_PANTHER(cpunodes[aflt->flt_inst].implementation)); for (i = 0; i < CH_PCACHE_NWAY; i++) { pcp = &ch_flt->parity_data.dpe.cpl_pc[i]; if (pcp->pc_logflag == PC_LOGFLAG_MAGIC) { bcopy(pcp, &pcdata[ways_logged], sizeof (ch_pc_data_t)); ways_logged++; } } /* * Add the pcache data to the payload. */ fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1P_WAYS, DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); if (ways_logged != 0) { nelem = sizeof (ch_pc_data_t) / sizeof (uint64_t) * ways_logged; fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1P_DATA, DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)pcdata, NULL); } } /* * Add TLB diagnostic data to the ereport payload. */ void cpu_payload_add_tlb(struct async_flt *aflt, nvlist_t *nvl) { ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; uint8_t num_entries, tlb_data_words; /* * We only capture TLB information if we encountered * a TLB parity error and Panther is the only CPU which * can detect a TLB parity error. */ ASSERT(IS_PANTHER(cpunodes[aflt->flt_inst].implementation)); ASSERT((ch_flt->flt_type == CPU_ITLB_PARITY) || (ch_flt->flt_type == CPU_DTLB_PARITY)); if (ch_flt->flt_type == CPU_ITLB_PARITY) { num_entries = (uint8_t)(PN_ITLB_NWAYS * PN_NUM_512_ITLBS); tlb_data_words = sizeof (ch_tte_entry_t) / sizeof (uint64_t) * num_entries; /* * Add the TLB diagnostic data to the payload * if it was collected. */ if (ch_flt->tlb_diag_data.tlo_logflag == IT_LOGFLAG_MAGIC) { fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_ITLB_ENTRIES, DATA_TYPE_UINT8, num_entries, NULL); fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_ITLB_DATA, DATA_TYPE_UINT64_ARRAY, tlb_data_words, (uint64_t *)ch_flt->tlb_diag_data.tlo_itlb_tte, NULL); } } else { num_entries = (uint8_t)(PN_DTLB_NWAYS * PN_NUM_512_DTLBS); tlb_data_words = sizeof (ch_tte_entry_t) / sizeof (uint64_t) * num_entries; fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_VA, DATA_TYPE_UINT64, ch_flt->tlb_diag_data.tlo_addr, NULL); /* * Add the TLB diagnostic data to the payload * if it was collected. */ if (ch_flt->tlb_diag_data.tlo_logflag == DT_LOGFLAG_MAGIC) { fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_DTLB_ENTRIES, DATA_TYPE_UINT8, num_entries, NULL); fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_DTLB_DATA, DATA_TYPE_UINT64_ARRAY, tlb_data_words, (uint64_t *)ch_flt->tlb_diag_data.tlo_dtlb_tte, NULL); } } } /* * Panther Cache Scrubbing: * * In Jaguar, the E$ was split between cores, so the scrubber must run on both * cores. For Panther, however, the L2$ and L3$ are shared across cores. * Therefore, the E$ scrubber only needs to run on one of the two cores. * * There are four possible states for the E$ scrubber: * * 0. If both cores are offline, add core 0 to cpu_offline_set so that * the offline scrubber will run on it. * 1. If core 0 is online and core 1 off, we run the scrubber on core 0. * 2. If core 1 is online and core 0 off, we move the scrubber to run * on core 1. * 3. If both cores are online, only run the scrubber on core 0. * * These states are enumerated by the SCRUBBER_[BOTH|CORE|NEITHER]_* defines * above. One of those values is stored in * chpr_scrub_misc->chsm_core_state on each core. * * Also note that, for Panther, ecache_flush_line() will flush out the L2$ * before the E$, so the L2$ will be scrubbed by the E$ scrubber. No * additional code is necessary to scrub the L2$. * * For all cpu types, whenever a cpu or core is offlined, add it to * cpu_offline_set so the necessary scrubbers can still run. This is still * necessary on Panther so the D$ scrubber can still run. */ /*ARGSUSED*/ int cpu_scrub_cpu_setup(cpu_setup_t what, int cpuid, void *arg) { processorid_t core_0_id; cpu_t *core_cpus[2]; ch_scrub_misc_t *core_scrub[2]; int old_state, i; int new_state = SCRUBBER_NEITHER_CORE_ONLINE; switch (what) { case CPU_ON: case CPU_INIT: CPUSET_DEL(cpu_offline_set, cpuid); break; case CPU_OFF: CPUSET_ADD(cpu_offline_set, cpuid); break; default: return (0); } if (!IS_PANTHER(cpunodes[cpuid].implementation)) { return (0); } /* * Update the chsm_enable[CACHE_SCRUBBER_INFO_E] value * if necessary */ core_0_id = cmp_cpu_to_chip(cpuid); core_cpus[0] = cpu_get(core_0_id); core_cpus[1] = cpu_get_sibling_core(core_cpus[0]); for (i = 0; i < 2; i++) { if (core_cpus[i] == NULL) { /* * This may happen during DR - one core is offlined * and completely unconfigured before the second * core is offlined. Give up and return quietly, * since the second core should quickly be removed * anyways. */ return (0); } core_scrub[i] = CPU_PRIVATE_PTR(core_cpus[i], chpr_scrub_misc); } if (cpuid == (processorid_t)cmp_cpu_to_chip(cpuid)) { /* cpuid is core 0 */ if (cpu_is_active(core_cpus[1])) { new_state |= SCRUBBER_CORE_1_ONLINE; } if (what != CPU_OFF) { new_state |= SCRUBBER_CORE_0_ONLINE; } } else { /* cpuid is core 1 */ if (cpu_is_active(core_cpus[0])) { new_state |= SCRUBBER_CORE_0_ONLINE; } if (what != CPU_OFF) { new_state |= SCRUBBER_CORE_1_ONLINE; } } old_state = core_scrub[0]->chsm_core_state; if (old_state == new_state) { return (0); } if (old_state == SCRUBBER_CORE_1_ONLINE) { /* * We need to move the scrubber state from core 1 * back to core 0. This data is not protected by * locks, but the worst that can happen is some * lines are scrubbed multiple times. chsm_oustanding is * set to 0 to make sure an interrupt is scheduled the * first time through do_scrub(). */ core_scrub[0]->chsm_flush_index[CACHE_SCRUBBER_INFO_E] = core_scrub[1]->chsm_flush_index[CACHE_SCRUBBER_INFO_E]; core_scrub[0]->chsm_outstanding[CACHE_SCRUBBER_INFO_E] = 0; } switch (new_state) { case SCRUBBER_NEITHER_CORE_ONLINE: case SCRUBBER_BOTH_CORES_ONLINE: case SCRUBBER_CORE_0_ONLINE: core_scrub[1]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 0; core_scrub[0]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1; break; case SCRUBBER_CORE_1_ONLINE: default: /* * We need to move the scrubber state from core 0 * to core 1. */ core_scrub[1]->chsm_flush_index[CACHE_SCRUBBER_INFO_E] = core_scrub[0]->chsm_flush_index[CACHE_SCRUBBER_INFO_E]; core_scrub[1]->chsm_outstanding[CACHE_SCRUBBER_INFO_E] = 0; core_scrub[0]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 0; core_scrub[1]->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1; break; } core_scrub[0]->chsm_core_state = new_state; core_scrub[1]->chsm_core_state = new_state; return (0); } /* * Returns a pointer to the cpu structure of the argument's sibling core. * If no sibling core can be found, return NULL. */ static cpu_t * cpu_get_sibling_core(cpu_t *cpup) { cpu_t *nextp; if ((cpup == NULL) || (!cmp_cpu_is_cmp(cpup->cpu_id))) return (NULL); nextp = cpup->cpu_next_chip; if ((nextp == NULL) || (nextp == cpup)) return (NULL); return (nextp); }