1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22/* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * 26 * Assembly code support for the Cheetah module 27 */ 28 29#include "assym.h" 30 31#include <sys/asm_linkage.h> 32#include <sys/mmu.h> 33#include <vm/hat_sfmmu.h> 34#include <sys/machparam.h> 35#include <sys/machcpuvar.h> 36#include <sys/machthread.h> 37#include <sys/machtrap.h> 38#include <sys/privregs.h> 39#include <sys/asm_linkage.h> 40#include <sys/trap.h> 41#include <sys/cheetahregs.h> 42#include <sys/us3_module.h> 43#include <sys/xc_impl.h> 44#include <sys/intreg.h> 45#include <sys/async.h> 46#include <sys/clock.h> 47#include <sys/cheetahasm.h> 48 49#ifdef TRAPTRACE 50#include <sys/traptrace.h> 51#endif /* TRAPTRACE */ 52 53/* BEGIN CSTYLED */ 54 55/* 56 * Cheetah version to flush an Ecache line by index (aliased address) 57 */ 58#define ECACHE_REFLUSH_LINE(ecache_size, alias_address, scr2) \ 59 ldxa [alias_address]ASI_MEM, %g0 60 61#define ECACHE_FLUSH_LINE(physaddr, ecache_size, scr1, scr2) \ 62 xor physaddr, ecache_size, scr1; \ 63 add ecache_size, ecache_size, scr2; \ 64 sub scr2, 1, scr2; \ 65 and scr1, scr2, scr1; \ 66 ASM_LDX(scr2, ecache_flushaddr); \ 67 add scr1, scr2, scr1; \ 68 ECACHE_REFLUSH_LINE(ecache_size, scr1, scr2) 69 70/* END CSTYLED */ 71 72 73/* 74 * Fast ECC error at TL>0 handler 75 * We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter 76 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. 77 * For a complete description of the Fast ECC at TL>0 handling see the 78 * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in 79 * us3_common_asm.s 80 */ 81 82 .section ".text" 83 .align 64 84 ENTRY_NP(fast_ecc_tl1_err) 85 86 /* 87 * This macro turns off the D$/I$ if they are on and saves their 88 * original state in ch_err_tl1_tmp, saves all the %g registers in the 89 * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves 90 * the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will 91 * point to the ch_err_tl1_data structure and the original D$/I$ state 92 * will be saved in ch_err_tl1_tmp. All %g registers except for %g1 93 * will be available. 94 */ 95 CH_ERR_TL1_FECC_ENTER; 96 97 /* 98 * Get the diagnostic logout data. %g4 must be initialized to 99 * current CEEN state, %g5 must point to logout structure in 100 * ch_err_tl1_data_t. %g3 will contain the nesting count upon 101 * return. 102 */ 103 ldxa [%g0]ASI_ESTATE_ERR, %g4 104 and %g4, EN_REG_CEEN, %g4 105 add %g1, CH_ERR_TL1_LOGOUT, %g5 106 DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4) 107 108 /* 109 * If the logout nesting count is exceeded, we're probably 110 * not making any progress, try to panic instead. 111 */ 112 cmp %g3, CLO_NESTING_MAX 113 bge fecc_tl1_err 114 nop 115 116 /* 117 * Save the current CEEN and NCEEN state in %g7 and turn them off 118 * before flushing the Ecache. 119 */ 120 ldxa [%g0]ASI_ESTATE_ERR, %g7 121 andn %g7, EN_REG_CEEN | EN_REG_NCEEN, %g5 122 stxa %g5, [%g0]ASI_ESTATE_ERR 123 membar #Sync 124 125 /* 126 * Flush the Ecache, using the largest possible cache size with the 127 * smallest possible line size since we can't get the actual sizes 128 * from the cpu_node due to DTLB misses. 129 */ 130 set CH_ECACHE_8M_SIZE, %g4 131 set CH_ECACHE_MIN_LSIZE, %g5 132 133 /* 134 * Use a different flush address to avoid recursion if the error 135 * exists in ecache_flushaddr. 136 */ 137 ASM_LDX(%g6, ecache_tl1_flushaddr) 138 cmp %g6, -1 ! check if address is valid 139 be %xcc, fecc_tl1_err 140 nop 141 CH_ECACHE_FLUSHALL(%g4, %g5, %g6) 142 143 /* 144 * Restore CEEN and NCEEN to the previous state. 145 */ 146 stxa %g7, [%g0]ASI_ESTATE_ERR 147 membar #Sync 148 149 /* 150 * If we turned off the D$, then flush it and turn it back on. 151 */ 152 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 153 andcc %g3, CH_ERR_TSTATE_DC_ON, %g0 154 bz %xcc, 3f 155 nop 156 157 /* 158 * Flush the D$. 159 */ 160 ASM_LD(%g4, dcache_size) 161 ASM_LD(%g5, dcache_linesize) 162 CH_DCACHE_FLUSHALL(%g4, %g5, %g6) 163 164 /* 165 * Turn the D$ back on. 166 */ 167 ldxa [%g0]ASI_DCU, %g3 168 or %g3, DCU_DC, %g3 169 stxa %g3, [%g0]ASI_DCU 170 membar #Sync 1713: 172 /* 173 * If we turned off the I$, then flush it and turn it back on. 174 */ 175 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 176 andcc %g3, CH_ERR_TSTATE_IC_ON, %g0 177 bz %xcc, 4f 178 nop 179 180 /* 181 * Flush the I$. 182 */ 183 ASM_LD(%g4, icache_size) 184 ASM_LD(%g5, icache_linesize) 185 CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3) 186 187 /* 188 * Turn the I$ back on. Changing DCU_IC requires flush. 189 */ 190 ldxa [%g0]ASI_DCU, %g3 191 or %g3, DCU_IC, %g3 192 stxa %g3, [%g0]ASI_DCU 193 flush %g0 1944: 195 196#ifdef TRAPTRACE 197 /* 198 * Get current trap trace entry physical pointer. 199 */ 200 CPU_INDEX(%g6, %g5) 201 sll %g6, TRAPTR_SIZE_SHIFT, %g6 202 set trap_trace_ctl, %g5 203 add %g6, %g5, %g6 204 ld [%g6 + TRAPTR_LIMIT], %g5 205 tst %g5 206 be %icc, skip_traptrace 207 nop 208 ldx [%g6 + TRAPTR_PBASE], %g5 209 ld [%g6 + TRAPTR_OFFSET], %g4 210 add %g5, %g4, %g5 211 212 /* 213 * Create trap trace entry. 214 */ 215 rd %asi, %g7 216 wr %g0, TRAPTR_ASI, %asi 217 rd STICK, %g4 218 stxa %g4, [%g5 + TRAP_ENT_TICK]%asi 219 rdpr %tl, %g4 220 stha %g4, [%g5 + TRAP_ENT_TL]%asi 221 rdpr %tt, %g4 222 stha %g4, [%g5 + TRAP_ENT_TT]%asi 223 rdpr %tpc, %g4 224 stna %g4, [%g5 + TRAP_ENT_TPC]%asi 225 rdpr %tstate, %g4 226 stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi 227 stna %sp, [%g5 + TRAP_ENT_SP]%asi 228 stna %g0, [%g5 + TRAP_ENT_TR]%asi 229 wr %g0, %g7, %asi 230 ldxa [%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3 231 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4 232 wr %g0, TRAPTR_ASI, %asi 233 stna %g3, [%g5 + TRAP_ENT_F1]%asi 234 stna %g4, [%g5 + TRAP_ENT_F2]%asi 235 wr %g0, %g7, %asi 236 ldxa [%g1 + CH_ERR_TL1_AFAR]%asi, %g3 237 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4 238 wr %g0, TRAPTR_ASI, %asi 239 stna %g3, [%g5 + TRAP_ENT_F3]%asi 240 stna %g4, [%g5 + TRAP_ENT_F4]%asi 241 wr %g0, %g7, %asi 242 243 /* 244 * Advance trap trace pointer. 245 */ 246 ld [%g6 + TRAPTR_OFFSET], %g5 247 ld [%g6 + TRAPTR_LIMIT], %g4 248 st %g5, [%g6 + TRAPTR_LAST_OFFSET] 249 add %g5, TRAP_ENT_SIZE, %g5 250 sub %g4, TRAP_ENT_SIZE, %g4 251 cmp %g5, %g4 252 movge %icc, 0, %g5 253 st %g5, [%g6 + TRAPTR_OFFSET] 254skip_traptrace: 255#endif /* TRAPTRACE */ 256 257 /* 258 * If nesting count is not zero, skip all the AFSR/AFAR 259 * handling and just do the necessary cache-flushing. 260 */ 261 ldxa [%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2 262 brnz %g2, 6f 263 nop 264 265 /* 266 * If a UCU followed by a WDU has occurred go ahead and panic 267 * since a UE will occur (on the retry) before the UCU and WDU 268 * messages are enqueued. 269 */ 270 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g3 271 set 1, %g4 272 sllx %g4, C_AFSR_UCU_SHIFT, %g4 273 btst %g4, %g3 ! UCU in original AFSR? 274 bz %xcc, 6f 275 nop 276 ldxa [%g0]ASI_AFSR, %g4 ! current AFSR 277 or %g3, %g4, %g3 ! %g3 = original + current AFSR 278 set 1, %g4 279 sllx %g4, C_AFSR_WDU_SHIFT, %g4 280 btst %g4, %g3 ! WDU in original or current AFSR? 281 bnz %xcc, fecc_tl1_err 282 nop 283 2846: 285 /* 286 * We fall into this macro if we've successfully logged the error in 287 * the ch_err_tl1_data structure and want the PIL15 softint to pick 288 * it up and log it. %g1 must point to the ch_err_tl1_data structure. 289 * Restores the %g registers and issues retry. 290 */ 291 CH_ERR_TL1_EXIT; 292 293 /* 294 * Establish panic exit label. 295 */ 296 CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err); 297 298 SET_SIZE(fast_ecc_tl1_err) 299 300 301/* 302 * scrubphys - Pass in the aligned physical memory address 303 * that you want to scrub, along with the ecache set size. 304 * 305 * 1) Displacement flush the E$ line corresponding to %addr. 306 * The first ldxa guarantees that the %addr is no longer in 307 * M, O, or E (goes to I or S (if instruction fetch also happens). 308 * 2) "Write" the data using a CAS %addr,%g0,%g0. 309 * The casxa guarantees a transition from I to M or S to M. 310 * 3) Displacement flush the E$ line corresponding to %addr. 311 * The second ldxa pushes the M line out of the ecache, into the 312 * writeback buffers, on the way to memory. 313 * 4) The "membar #Sync" pushes the cache line out of the writeback 314 * buffers onto the bus, on the way to dram finally. 315 * 316 * This is a modified version of the algorithm suggested by Gary Lauterbach. 317 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line 318 * as modified, but then we found out that for spitfire, if it misses in the 319 * E$ it will probably install as an M, but if it hits in the E$, then it 320 * will stay E, if the store doesn't happen. So the first displacement flush 321 * should ensure that the CAS will miss in the E$. Arrgh. 322 */ 323 ENTRY(scrubphys) 324 rdpr %pstate, %o4 325 andn %o4, PSTATE_IE | PSTATE_AM, %o5 326 wrpr %o5, %g0, %pstate ! clear IE, AM bits 327 328 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 329 casxa [%o0]ASI_MEM, %g0, %g0 330 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 331 332 wrpr %g0, %o4, %pstate ! restore earlier pstate register value 333 334 retl 335 membar #Sync ! move the data out of the load buffer 336 SET_SIZE(scrubphys) 337 338 339/* 340 * clearphys - Pass in the physical memory address of the checkblock 341 * that you want to push out, cleared with a recognizable pattern, 342 * from the ecache. 343 * 344 * To ensure that the ecc gets recalculated after the bad data is cleared, 345 * we must write out enough data to fill the w$ line (64 bytes). So we read 346 * in an entire ecache subblock's worth of data, and write it back out. 347 * Then we overwrite the 16 bytes of bad data with the pattern. 348 */ 349 ENTRY(clearphys) 350 /* turn off IE, AM bits */ 351 rdpr %pstate, %o4 352 andn %o4, PSTATE_IE | PSTATE_AM, %o5 353 wrpr %o5, %g0, %pstate 354 355 /* turn off NCEEN */ 356 ldxa [%g0]ASI_ESTATE_ERR, %o5 357 andn %o5, EN_REG_NCEEN, %o3 358 stxa %o3, [%g0]ASI_ESTATE_ERR 359 membar #Sync 360 361 /* align address passed with 64 bytes subblock size */ 362 mov CH_ECACHE_SUBBLK_SIZE, %o2 363 andn %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1 364 365 /* move the good data into the W$ */ 3661: 367 subcc %o2, 8, %o2 368 ldxa [%g1 + %o2]ASI_MEM, %g2 369 bge 1b 370 stxa %g2, [%g1 + %o2]ASI_MEM 371 372 /* now overwrite the bad data */ 373 setx 0xbadecc00badecc01, %g1, %g2 374 stxa %g2, [%o0]ASI_MEM 375 mov 8, %g1 376 stxa %g2, [%o0 + %g1]ASI_MEM 377 378 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 379 casxa [%o0]ASI_MEM, %g0, %g0 380 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 381 382 /* clear the AFSR */ 383 ldxa [%g0]ASI_AFSR, %o1 384 stxa %o1, [%g0]ASI_AFSR 385 membar #Sync 386 387 /* turn NCEEN back on */ 388 stxa %o5, [%g0]ASI_ESTATE_ERR 389 membar #Sync 390 391 /* return and re-enable IE and AM */ 392 retl 393 wrpr %g0, %o4, %pstate 394 SET_SIZE(clearphys) 395 396 397/* 398 * Cheetah Ecache displacement flush the specified line from the E$ 399 * 400 * Register usage: 401 * %o0 - 64 bit physical address for flushing 402 * %o1 - Ecache set size 403 */ 404 ENTRY(ecache_flush_line) 405 406 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 407 408 retl 409 nop 410 SET_SIZE(ecache_flush_line) 411 412/* 413 * This routine will not be called in Cheetah systems. 414 */ 415 ENTRY(flush_ipb) 416 retl 417 nop 418 SET_SIZE(flush_ipb) 419 420