1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22/* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * 26 * Assembly code support for the jalapeno module 27 */ 28 29#pragma ident "%Z%%M% %I% %E% SMI" 30 31#if !defined(lint) 32#include "assym.h" 33#endif /* lint */ 34 35#include <sys/asm_linkage.h> 36#include <sys/mmu.h> 37#include <vm/hat_sfmmu.h> 38#include <sys/machparam.h> 39#include <sys/machcpuvar.h> 40#include <sys/machthread.h> 41#include <sys/machtrap.h> 42#include <sys/privregs.h> 43#include <sys/asm_linkage.h> 44#include <sys/trap.h> 45#include <sys/cheetahregs.h> 46#include <sys/us3_module.h> 47#include <sys/xc_impl.h> 48#include <sys/intreg.h> 49#include <sys/async.h> 50#include <sys/clock.h> 51#include <sys/cheetahasm.h> 52 53#ifdef TRAPTRACE 54#include <sys/traptrace.h> 55#endif /* TRAPTRACE */ 56 57#if !defined(lint) 58 59/* BEGIN CSTYLED */ 60 61#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 62 63#define CHK_JP_ERRATA85_ENABLED(scr, label) \ 64 ASM_LD(scr, jp_errata_85_active); \ 65 cmp scr, 1; \ 66 bne %icc, label; \ 67 nop 68 69#define SET_64BIT_PA(dest, scr, hi32, lo32) \ 70 set hi32, scr; \ 71 sllx scr, 32, scr; \ 72 sethi %hi(lo32), dest; \ 73 or dest, %lo(lo32), dest; \ 74 or scr, dest, dest 75 76/* 77 * Macro to trigger Jalapeno/Tomatillo speed change 78 * j_chng_pa - scratch register 79 * scr - scratch register 80 */ 81#define JP_ESTAR_TRIGGER(j_chng_pa, scr) \ 82 SET_64BIT_PA(j_chng_pa, scr, TOM_HIGH_PA, M_T_J_CHNG_INIT_PA); \ 83 ldxa [j_chng_pa]ASI_IO, scr; \ 845: \ 85 and scr, TOM_TRIGGER_MASK, scr; \ 86 cmp scr, TOM_TRIGGER; \ 87 be,pt %icc, 5b; /* wait while 10 */ \ 88 ldxa [j_chng_pa]ASI_IO, scr; \ 89 andn scr, TOM_TRIGGER_MASK, scr; \ 90 stxa scr, [j_chng_pa]ASI_IO; /* clear j_chng[1:0] */ \ 91 or scr, TOM_TRIGGER, scr; \ 92 stxa scr, [j_chng_pa]ASI_IO; /* trigger j_chng */ \ 93 ldxa [j_chng_pa]ASI_IO, scr; \ 946: \ 95 and scr, TOM_TRIGGER_MASK, scr; \ 96 cmp scr, TOM_TRIGGER; \ 97 be,pt %icc, 6b; /* wait while 10 */ \ 98 ldxa [j_chng_pa]ASI_IO, scr; \ 99 andn scr, TOM_TRIGGER_MASK, scr; \ 100 stxa scr, [j_chng_pa]ASI_IO; /* deassert j_chng */ 101 102/* 103 * Macro to set Jalapeno CPU speed 104 * speed - new speed constant 105 * scr1 - scratch register 106 * scr2 - scratch register 107 */ 108#define SET_JP_SPEED(speed, scr1, scr2) \ 109 ldxa [%g0]ASI_JBUS_CONFIG, scr1; \ 110 set JBUS_CONFIG_ECLK_MASK, scr2; \ 111 andn scr1, scr2, scr1; \ 112 set speed, scr2; \ 113 or scr1, scr2, scr1; \ 114 stxa scr1, [%g0]ASI_JBUS_CONFIG; 115 116/* 117 * macro to set Master Tomatillo speed 118 * speed - tomatillo speed constant 119 * tpa - tomatillo estar control register PA 120 * scr - scratch register 121 */ 122#define SET_TOM_SPEED(speed, tpa, scr) \ 123 ldxa [tpa]ASI_IO, scr; \ 124 andn scr, TOM_ESTAR_ELCK_MASK, scr; \ 125 or scr, speed, scr; \ 126 stxa scr, [tpa]ASI_IO; 127 128/* 129 * macro to check and set Slave Tomatillo speed 130 * speed - tomatillo speed constant 131 * scr1 - scratch register 132 * scr2 - scratch register 133 */ 134 135#define SET_SLAVE_T_SPEED(speed, scr1, scr2) \ 136 ldxa [%g0]ASI_JBUS_CONFIG, scr2; \ 137 srlx scr2, JBUS_SLAVE_T_PORT_BIT, scr2; \ 138 btst 1, scr2; \ 139 bz,pt %icc, 4f; \ 140 nop; \ 141 SET_64BIT_PA(scr1, scr2, TOM_HIGH_PA, S_T_ESTAR_CTRL_PA); \ 142 SET_TOM_SPEED(speed, scr1, scr2); \ 1434: 144 145 146/* 147 * macro to adjust ASI_MCU_CTL_REG1[26:25] fsm bits according to 148 * new cpu speed: fsm[1:0]=11b for full speed, fsm[1:0]=0 for estar speed 149 * value - fsm bit value constant 150 * scr1 - scratch register 151 * scr2 - scratch register 152 */ 153#define JP_ADJUST_FSM(value, scr1, scr2) \ 154 ldxa [%g0]ASI_MCU_CTRL, scr1; \ 155 set JP_MCU_FSM_MASK, scr2; \ 156 andn scr1, scr2, scr1; \ 157 set value, scr2; \ 158 or scr1, scr2, scr1; \ 159 stxa scr1, [%g0]ASI_MCU_CTRL; \ 160 membar #Sync; 161 162/* 163 * JP_FORCE_FULL_SPEED and its fellow macros are for Jalapeno 164 * workstation to work around Errata 85. The front portion of 165 * it packs JP speed(14..13) and Tomatillo speed(5..0) into one 166 * register. 167 * 168 * Current code assumes that these two fields are non-overlapping. 169 * If that assumption changes, then this code won't work. If so, we 170 * force a compile time error by not defining the JP_FORCE_FULL_SPEED 171 * and JP_RESTORE_SPEED macros below. 172 */ 173 174#if !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK) 175 176/* 177 * Macro to force Jalapeno/Tomatillo to full speed 178 * old_lvl - register used to save original cpu, tomatillo speed 179 * scr2 - scratch register 180 * scr3 - scratch register 181 * scr4 - scratch register 182 */ 183#define JP_FORCE_FULL_SPEED(old_lvl, scr2, scr3, scr4) \ 184 ldxa [%g0]ASI_JBUS_CONFIG, old_lvl; \ 185 set JBUS_CONFIG_ECLK_MASK, scr4; \ 186 and old_lvl, scr4, old_lvl; \ 187 SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA); \ 188 ldxa [scr2]ASI_IO, scr3; \ 189 set TOM_ESTAR_ELCK_MASK, scr4; \ 190 and scr3, scr4, scr3; \ 191 or old_lvl, scr3, old_lvl; \ 192 /* original jp and tomatillo speed saved in old_lvl */ \ 193 \ 194 /* either intended or currently at full speed */ \ 195 set JBUS_CONFIG_ECLK_MASK, scr4; \ 196 andcc old_lvl, scr4, %g0; \ 197 bz,pt %icc, 8f; \ 198 nop; \ 199 /* go through 1/2 speed. */ \ 200 SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4); \ 201 SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3); \ 202 SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4); \ 203 JP_ADJUST_FSM(0, scr3, scr4); \ 204 set jp_estar_tl0_data, scr3; \ 205 ldx [scr3], %g0; \ 206 membar #Sync; /* or busy wait 1us */ \ 207 JP_ESTAR_TRIGGER(scr3, scr4); \ 2088: \ 209 /* bring to 1:1 speed */ \ 210 SET_JP_SPEED(JBUS_CONFIG_ECLK_1, scr3, scr4); \ 211 SET_TOM_SPEED(TOM_FULL_SPEED, scr2, scr3); \ 212 SET_SLAVE_T_SPEED(TOM_FULL_SPEED, scr3, scr4); \ 213 JP_ADJUST_FSM(JP_MCU_FSM_MASK, scr3, scr4); \ 214 JP_ESTAR_TRIGGER(scr3, scr4) 215 216 217/* 218 * Macro to restore Jalapeno/Tomatillo to original speed 219 * old_lvl - register contains saved original cpu, tomatillo speed 220 * scr2 - scratch register 221 * scr3 - scratch register 222 * scr4 - scratch register 223 * 224 * If trap had occured in the middle of ppm cpu speed transtion, then 225 * old_lvl[31:10] contains the intended new speed written into jbus_config. 226 * if old_lvl[9:0] is inconsistent with old_lvl[31:10], then the trap surely 227 * interrupted the ppm cpu speed transition, otherwise nothing for sure. 228 * We'll restore the intended/then-current speed, that should cause no 229 * trouble to subsequent ppm cpu speed change code. 230 */ 231#define JP_RESTORE_SPEED(old_lvl, scr2, scr3, scr4) \ 232 srlx old_lvl, JBUS_CONFIG_ECLK_SHIFT, scr2; \ 233 and scr2, 3, scr2; \ 234 add scr2, 1, scr2; \ 235 cmp scr2, 3; \ 236 bne,pt %icc, 7f; \ 237 nop; \ 238 set TOM_SLOW_SPEED, scr2; \ 239 /* scr2 contains tom speed according to intended jp speed */ \ 2407: \ 241 andn old_lvl, TOM_ESTAR_ELCK_MASK, old_lvl; \ 242 or scr2, old_lvl, old_lvl; \ 243 /* updated old_lvl to contain intended jp and tom speed */ \ 244 andcc old_lvl, TOM_FULL_SPEED, %g0; \ 245 bnz,pt %icc, 9f; /* intended full, already at full */ \ 246 nop; \ 247 \ 248 /* go to half speed */ \ 249 SET_JP_SPEED(JBUS_CONFIG_ECLK_2, scr3, scr4); \ 250 SET_64BIT_PA(scr2, scr3, TOM_HIGH_PA, M_T_ESTAR_CTRL_PA); \ 251 SET_TOM_SPEED(TOM_HALF_SPEED, scr2, scr3); \ 252 SET_SLAVE_T_SPEED(TOM_HALF_SPEED, scr3, scr4); \ 253 JP_ADJUST_FSM(0, scr3, scr4); \ 254 set jp_estar_tl0_data, scr3; \ 255 ldx [scr3], %g0; \ 256 membar #Sync; \ 257 JP_ESTAR_TRIGGER(scr3, scr4); \ 258 andcc old_lvl, TOM_SLOW_SPEED, %g0; \ 259 bz,pt %icc, 9f; /* intended 1:2, already at 1:2 */ \ 260 nop; \ 261 \ 262 /* go to 1:32 speed */ \ 263 SET_JP_SPEED(JBUS_CONFIG_ECLK_32, scr3, scr4); \ 264 SET_TOM_SPEED(TOM_SLOW_SPEED, scr2, scr3); \ 265 SET_SLAVE_T_SPEED(TOM_SLOW_SPEED, scr3, scr4); \ 266 JP_ESTAR_TRIGGER(scr3, scr4); \ 2679: 268 269#endif /* !(JBUS_CONFIG_ECLK_MASK & TOM_SPEED_MASK) */ 270#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 271 272/* 273 * Jalapeno version to reflush an Ecache line by index. 274 * Will flush all 4 ways (with only one scratch register). 275 * Note that the code will be faster if we use 2 scratch registers. 276 */ 277#define ECACHE_REFLUSH_LINE(ec_set_size, index, scr1) \ 278 JP_EC_DIAG_ACCESS_MEMBAR; \ 279 ldxa [index]ASI_EC_DIAG, %g0; \ 280 JP_EC_DIAG_ACCESS_MEMBAR; \ 281 mov 1, scr1; \ 282 sllx scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1; \ 283 add scr1, index, scr1; \ 284 JP_EC_DIAG_ACCESS_MEMBAR; \ 285 ldxa [scr1]ASI_EC_DIAG, %g0; \ 286 JP_EC_DIAG_ACCESS_MEMBAR; \ 287 mov 2, scr1; \ 288 sllx scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1; \ 289 add scr1, index, scr1; \ 290 JP_EC_DIAG_ACCESS_MEMBAR; \ 291 ldxa [scr1]ASI_EC_DIAG, %g0; \ 292 JP_EC_DIAG_ACCESS_MEMBAR; \ 293 mov 3, scr1; \ 294 sllx scr1, JP_ECFLUSH_EC_WAY_SHIFT, scr1; \ 295 add scr1, index, scr1; \ 296 JP_EC_DIAG_ACCESS_MEMBAR; \ 297 ldxa [scr1]ASI_EC_DIAG, %g0; \ 298 JP_EC_DIAG_ACCESS_MEMBAR 299 300/* 301 * Jalapeno version of ecache_flush_line. Uses Jalapeno Ecache Displacement 302 * Flush feature to flush all 4 sets/ways. 303 */ 304#define ECACHE_FLUSH_LINE(physaddr, ec_set_size, scr1, scr2) \ 305 CPU_INDEX(scr1, scr2); \ 306 sllx scr1, JP_ECFLUSH_PORTID_SHIFT, scr1; \ 307 set JP_ECACHE_IDX_DISP_FLUSH, scr2; \ 308 or scr2, scr1, scr2; \ 309 sub ec_set_size, 1, scr1; \ 310 and physaddr, scr1, scr1; \ 311 or scr2, scr1, scr1; \ 312 ECACHE_REFLUSH_LINE(ec_set_size, scr1, scr2) 313 314/* 315 * Macro for getting ecache size from cpunodes structure 316 * scr1: Scratch, ecache size returned in this 317 * scr2: Scratch 318 */ 319#define GET_ECACHE_SIZE(scr1, scr2) \ 320 CPU_INDEX(scr1, scr2); \ 321 mulx scr1, CPU_NODE_SIZE, scr1; \ 322 set cpunodes + ECACHE_SIZE, scr2; \ 323 ld [scr1 + scr2], scr1 324 325/* END CSTYLED */ 326 327#endif /* !lint */ 328 329#if defined(lint) 330 331/* ARGSUSED */ 332void 333shipit(int upaid, int bn) 334{ return; } 335 336#else /* lint */ 337 338/* 339 * Ship mondo to aid using implicit busy/nack pair (bn ignored) 340 */ 341 ENTRY_NP(shipit) 342 sll %o0, IDCR_PID_SHIFT, %g1 ! IDCR<18:14> = agent id 343 or %g1, IDCR_OFFSET, %g1 ! IDCR<13:0> = 0x70 344 stxa %g0, [%g1]ASI_INTR_DISPATCH ! interrupt vector dispatch 345 membar #Sync 346 retl 347 nop 348 SET_SIZE(shipit) 349 350#endif /* lint */ 351 352 353/* 354 * flush_ecache: 355 * %o0 - 64 bit physical address 356 * %o1 - ecache size 357 * %o2 - ecache linesize 358 */ 359#if defined(lint) 360 361/*ARGSUSED*/ 362void 363flush_ecache(uint64_t physaddr, size_t ecache_size, size_t ecache_linesize) 364{} 365 366#else /* !lint */ 367 368 ENTRY(flush_ecache) 369#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 370 CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_1); 371 JP_FORCE_FULL_SPEED(%o3, %g1, %g2, %g3); /* %o3: saved speed */ 372flush_ecache_1: 373#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 374 375 /* 376 * Flush the entire Ecache using displacement flush. 377 */ 378 ECACHE_FLUSHALL(%o1, %o2, %o0, %o4) 379 380#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 381 CHK_JP_ERRATA85_ENABLED(%g1, flush_ecache_2); 382 JP_RESTORE_SPEED(%o3, %g1, %g2, %g3); /* %o3: saved speed */ 383flush_ecache_2: 384#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 385 386 retl 387 nop 388 SET_SIZE(flush_ecache) 389 390#endif /* lint */ 391 392 393#if defined(lint) 394 395void 396fast_ecc_err(void) 397{} 398 399#else /* lint */ 400 401 .section ".text" 402 .align 64 403 ENTRY_NP(fast_ecc_err) 404 405 /* 406 * Turn off CEEN and NCEEN. 407 */ 408 ldxa [%g0]ASI_ESTATE_ERR, %g3 409 andn %g3, EN_REG_NCEEN + EN_REG_CEEN, %g4 410 stxa %g4, [%g0]ASI_ESTATE_ERR 411 membar #Sync ! membar sync required 412 413 /* 414 * Do the CPU log out capture. 415 * %g3 = "failed?" return value. 416 * %g2 = Input = AFAR. Output the clo_flags info which is passed 417 * into this macro via %g4. Output only valid if cpu_private 418 * struct has not been initialized. 419 * CHPR_FECCTL0_LOGOUT = cpu logout structure offset input 420 * %g4 = Trap information stored in the cpu logout flags field 421 * %g5 = scr1 422 * %g6 = scr2 423 * %g3 = scr3 424 * %g4 = scr4 425 */ 426 and %g3, EN_REG_CEEN, %g4 ! store the CEEN value, TL=0 427 set CHPR_FECCTL0_LOGOUT, %g6 428 DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4) 429 430 /* 431 * Flush the Ecache to get the error out of the Ecache. If the UCC 432 * or UCU is on a dirty line, then the following flush will turn 433 * that into a WDC or WDU, respectively. 434 */ 435 CPU_INDEX(%g4, %g5) 436 mulx %g4, CPU_NODE_SIZE, %g4 437 set cpunodes, %g5 438 add %g4, %g5, %g4 439 ld [%g4 + ECACHE_LINESIZE], %g5 440 ld [%g4 + ECACHE_SIZE], %g4 441#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 442 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_1); 443 set jp_estar_tl0_data, %g6 444 stx %g2, [%g6 + 0] 445 stx %g3, [%g6 + 8] 446 JP_FORCE_FULL_SPEED(%g2, %g3, %g6, %g7) /* %g2: saved speed */ 447fast_ecc_err_1: 448#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 449 ECACHE_FLUSHALL(%g4, %g5, %g6, %g7) 450#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 451 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_err_2); 452 JP_RESTORE_SPEED(%g2, %g3, %g6, %g7) /* %g2: saved speed */ 453 set jp_estar_tl0_data, %g6 454 ldx [%g6 + 0], %g2 455 ldx [%g6 + 8], %g3 456fast_ecc_err_2: 457#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 458 459 /* 460 * Flush the Dcache. Since bad data could have been installed in 461 * the Dcache we must flush it before re-enabling it. 462 */ 463 ASM_LD(%g5, dcache_size) 464 ASM_LD(%g6, dcache_linesize) 465 CH_DCACHE_FLUSHALL(%g5, %g6, %g7) 466 467 /* 468 * Flush the Icache. Since we turned off the Icache to capture the 469 * Icache line it is now stale or corrupted and we must flush it 470 * before re-enabling it. 471 */ 472 GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_4); 473 ld [%g5 + CHPR_ICACHE_LINESIZE], %g6 474 ba,pt %icc, 5f 475 ld [%g5 + CHPR_ICACHE_SIZE], %g5 476fast_ecc_err_4: 477 ASM_LD(%g5, icache_size) 478 ASM_LD(%g6, icache_linesize) 4795: 480 CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4) 481 482 /* 483 * Restore the Dcache and Icache to the previous state. 484 */ 485 stxa %g1, [%g0]ASI_DCU 486 flush %g0 /* flush required after changing the IC bit */ 487 488 /* 489 * Make sure our CPU logout operation was successful. 490 */ 491 cmp %g3, %g0 492 be 8f 493 nop 494 495 /* 496 * If the logout structure had been busy, how many times have 497 * we tried to use it and failed (nesting count)? If we have 498 * already recursed a substantial number of times, then we can 499 * assume things are not going to get better by themselves and 500 * so it would be best to panic. 501 */ 502 cmp %g3, CLO_NESTING_MAX 503 blt 7f 504 nop 505 506 call ptl1_panic 507 mov PTL1_BAD_ECC, %g1 508 5097: 510 /* 511 * Otherwise, if the logout structure was busy but we have not 512 * nested more times than our maximum value, then we simply 513 * issue a retry. Our TL=0 trap handler code will check and 514 * clear the AFSR after it is done logging what is currently 515 * in the logout struct and handle this event at that time. 516 */ 517 retry 5188: 519 /* 520 * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're 521 * already at PIL 15. 522 */ 523 set cpu_fast_ecc_error, %g1 524 rdpr %pil, %g4 525 cmp %g4, PIL_14 526 ba sys_trap 527 movl %icc, PIL_14, %g4 528 529 SET_SIZE(fast_ecc_err) 530 531#endif /* lint */ 532 533 534/* 535 * Fast ECC error at TL>0 handler 536 * We get here via trap 70 at TL>0->Software trap 0 at TL>0. We enter 537 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. 538 * For a complete description of the Fast ECC at TL>0 handling see the 539 * comment block "Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy" in 540 * us3_common_asm.s 541 */ 542#if defined(lint) 543 544void 545fast_ecc_tl1_err(void) 546{} 547 548#else /* lint */ 549 550 .section ".text" 551 .align 64 552 ENTRY_NP(fast_ecc_tl1_err) 553 554 /* 555 * This macro turns off the D$/I$ if they are on and saves their 556 * original state in ch_err_tl1_tmp, saves all the %g registers in the 557 * ch_err_tl1_data structure, updates the ch_err_tl1_flags and saves 558 * the %tpc in ch_err_tl1_tpc. At the end of this macro, %g1 will 559 * point to the ch_err_tl1_data structure and the original D$/I$ state 560 * will be saved in ch_err_tl1_tmp. All %g registers except for %g1 561 * will be available. 562 */ 563 CH_ERR_TL1_FECC_ENTER; 564 565 /* 566 * Get the diagnostic logout data. %g4 must be initialized to 567 * current CEEN state, %g5 must point to logout structure in 568 * ch_err_tl1_data_t. %g3 will contain the nesting count upon 569 * return. 570 */ 571 ldxa [%g0]ASI_ESTATE_ERR, %g4 572 and %g4, EN_REG_CEEN, %g4 573 add %g1, CH_ERR_TL1_LOGOUT, %g5 574 DO_TL1_CPU_LOGOUT(%g3, %g2, %g4, %g5, %g6, %g3, %g4) 575 576 /* 577 * If the logout nesting count is exceeded, we're probably 578 * not making any progress, try to panic instead. 579 */ 580 cmp %g3, CLO_NESTING_MAX 581 bge fecc_tl1_err 582 nop 583 584 /* 585 * Save the current CEEN and NCEEN state in %g7 and turn them off 586 * before flushing the Ecache. 587 */ 588 ldxa [%g0]ASI_ESTATE_ERR, %g7 589 andn %g7, EN_REG_CEEN | EN_REG_NCEEN, %g5 590 stxa %g5, [%g0]ASI_ESTATE_ERR 591 membar #Sync 592 593 /* 594 * Flush the Ecache, using the largest possible cache size with the 595 * smallest possible line size since we can't get the actual sizes 596 * from the cpu_node due to DTLB misses. 597 */ 598 set JP_ECACHE_MAX_SIZE, %g4 599#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 600 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_1); 601 set jp_estar_tl1_data, %g6 602 stx %g2, [%g6 + 0] 603 stx %g3, [%g6 + 8] 604 JP_FORCE_FULL_SPEED(%g2, %g3, %g5, %g6) 605fast_ecc_tl1_err_1: 606#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 607 ECACHE_FLUSHALL(%g4, JP_ECACHE_MAX_LSIZE, %g5, %g6) 608#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 609 CHK_JP_ERRATA85_ENABLED(%g6, fast_ecc_tl1_err_2); 610 JP_RESTORE_SPEED(%g2, %g3, %g5, %g6) 611 set jp_estar_tl1_data, %g6 612 ldx [%g6 + 0], %g2 613 ldx [%g6 + 8], %g3 614fast_ecc_tl1_err_2: 615#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 616 617 /* 618 * Restore CEEN and NCEEN to the previous state. 619 */ 620 stxa %g7, [%g0]ASI_ESTATE_ERR 621 membar #Sync 622 623 /* 624 * If we turned off the D$, then flush it and turn it back on. 625 */ 626 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 627 andcc %g3, CH_ERR_TSTATE_DC_ON, %g0 628 bz %xcc, 3f 629 nop 630 631 /* 632 * Flush the D$. 633 */ 634 ASM_LD(%g4, dcache_size) 635 ASM_LD(%g5, dcache_linesize) 636 CH_DCACHE_FLUSHALL(%g4, %g5, %g6) 637 638 /* 639 * Turn the D$ back on. 640 */ 641 ldxa [%g0]ASI_DCU, %g3 642 or %g3, DCU_DC, %g3 643 stxa %g3, [%g0]ASI_DCU 644 membar #Sync 6453: 646 /* 647 * If we turned off the I$, then flush it and turn it back on. 648 */ 649 ldxa [%g1 + CH_ERR_TL1_TMP]%asi, %g3 650 andcc %g3, CH_ERR_TSTATE_IC_ON, %g0 651 bz %xcc, 4f 652 nop 653 654 /* 655 * Flush the I$. 656 */ 657 ASM_LD(%g4, icache_size) 658 ASM_LD(%g5, icache_linesize) 659 CH_ICACHE_FLUSHALL(%g4, %g5, %g6, %g3) 660 661 /* 662 * Turn the I$ back on. Changing DCU_IC requires flush. 663 */ 664 ldxa [%g0]ASI_DCU, %g3 665 or %g3, DCU_IC, %g3 666 stxa %g3, [%g0]ASI_DCU 667 flush %g0 6684: 669 670#ifdef TRAPTRACE 671 /* 672 * Get current trap trace entry physical pointer. 673 */ 674 CPU_INDEX(%g6, %g5) 675 sll %g6, TRAPTR_SIZE_SHIFT, %g6 676 set trap_trace_ctl, %g5 677 add %g6, %g5, %g6 678 ld [%g6 + TRAPTR_LIMIT], %g5 679 tst %g5 680 be %icc, skip_traptrace 681 nop 682 ldx [%g6 + TRAPTR_PBASE], %g5 683 ld [%g6 + TRAPTR_OFFSET], %g4 684 add %g5, %g4, %g5 685 686 /* 687 * Create trap trace entry. 688 */ 689 rd %asi, %g7 690 wr %g0, TRAPTR_ASI, %asi 691 rd STICK, %g4 692 stxa %g4, [%g5 + TRAP_ENT_TICK]%asi 693 rdpr %tl, %g4 694 stha %g4, [%g5 + TRAP_ENT_TL]%asi 695 rdpr %tt, %g4 696 stha %g4, [%g5 + TRAP_ENT_TT]%asi 697 rdpr %tpc, %g4 698 stna %g4, [%g5 + TRAP_ENT_TPC]%asi 699 rdpr %tstate, %g4 700 stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi 701 stna %sp, [%g5 + TRAP_ENT_SP]%asi 702 stna %g0, [%g5 + TRAP_ENT_TR]%asi 703 wr %g0, %g7, %asi 704 ldxa [%g1 + CH_ERR_TL1_SDW_AFAR]%asi, %g3 705 ldxa [%g1 + CH_ERR_TL1_SDW_AFSR]%asi, %g4 706 wr %g0, TRAPTR_ASI, %asi 707 stna %g3, [%g5 + TRAP_ENT_F1]%asi 708 stna %g4, [%g5 + TRAP_ENT_F2]%asi 709 wr %g0, %g7, %asi 710 ldxa [%g1 + CH_ERR_TL1_AFAR]%asi, %g3 711 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g4 712 wr %g0, TRAPTR_ASI, %asi 713 stna %g3, [%g5 + TRAP_ENT_F3]%asi 714 stna %g4, [%g5 + TRAP_ENT_F4]%asi 715 wr %g0, %g7, %asi 716 717 /* 718 * Advance trap trace pointer. 719 */ 720 ld [%g6 + TRAPTR_OFFSET], %g5 721 ld [%g6 + TRAPTR_LIMIT], %g4 722 st %g5, [%g6 + TRAPTR_LAST_OFFSET] 723 add %g5, TRAP_ENT_SIZE, %g5 724 sub %g4, TRAP_ENT_SIZE, %g4 725 cmp %g5, %g4 726 movge %icc, 0, %g5 727 st %g5, [%g6 + TRAPTR_OFFSET] 728skip_traptrace: 729#endif /* TRAPTRACE */ 730 731 /* 732 * If nesting count is not zero, skip all the AFSR/AFAR 733 * handling and just do the necessary cache-flushing. 734 */ 735 ldxa [%g1 + CH_ERR_TL1_NEST_CNT]%asi, %g2 736 brnz %g2, 6f 737 nop 738 739 /* 740 * If a UCU followed by a WDU has occurred go ahead and panic 741 * since a UE will occur (on the retry) before the UCU and WDU 742 * messages are enqueued. 743 */ 744 ldxa [%g1 + CH_ERR_TL1_AFSR]%asi, %g3 745 set 1, %g4 746 sllx %g4, C_AFSR_UCU_SHIFT, %g4 747 btst %g4, %g3 ! UCU in original AFSR? 748 bz %xcc, 6f 749 nop 750 ldxa [%g0]ASI_AFSR, %g4 ! current AFSR 751 or %g3, %g4, %g3 ! %g3 = original + current AFSR 752 set 1, %g4 753 sllx %g4, C_AFSR_WDU_SHIFT, %g4 754 btst %g4, %g3 ! WDU in original or current AFSR? 755 bnz %xcc, fecc_tl1_err 756 nop 757 7586: 759 /* 760 * We fall into this macro if we've successfully logged the error in 761 * the ch_err_tl1_data structure and want the PIL15 softint to pick 762 * it up and log it. %g1 must point to the ch_err_tl1_data structure. 763 * Restores the %g registers and issues retry. 764 */ 765 CH_ERR_TL1_EXIT; 766 /* 767 * Establish panic exit label. 768 */ 769 CH_ERR_TL1_PANIC_EXIT(fecc_tl1_err); 770 771 SET_SIZE(fast_ecc_tl1_err) 772 773#endif /* lint */ 774 775 776#if defined(lint) 777 778uint64_t 779get_jbus_config(void) 780{ return (0); } 781 782/* ARGSUSED */ 783void 784set_jbus_config(uint64_t jbus_config) 785{} 786 787/* ARGSUSED */ 788void 789set_mcu_ctl_reg1(uint64_t mcu_ctl) 790{} 791 792uint64_t 793get_mcu_ctl_reg1(void) 794{ return (0); } 795 796#else /* lint */ 797 798 ENTRY(get_jbus_config) 799 ldxa [%g0]ASI_JBUS_CONFIG, %o0 800 retl 801 nop 802 SET_SIZE(get_jbus_config) 803 804 ENTRY(set_jbus_config) 805 stxa %o0, [%g0]ASI_JBUS_CONFIG 806 membar #Sync 807 retl 808 nop 809 SET_SIZE(set_jbus_config) 810 811 812 ENTRY(get_mcu_ctl_reg1) 813 ldxa [%g0]ASI_MCU_CTRL, %o0 ! MCU control reg1 is at offset 0 814 retl 815 nop 816 SET_SIZE(get_mcu_ctl_reg1) 817 818 819 ENTRY(set_mcu_ctl_reg1) 820 stxa %o0, [%g0]ASI_MCU_CTRL ! MCU control reg1 is at offset 0 821 membar #Sync 822 retl 823 nop 824 SET_SIZE(set_mcu_ctl_reg1) 825 826#endif /* lint */ 827 828 829#if defined(lint) 830/* 831 * scrubphys - Pass in the aligned physical memory address 832 * that you want to scrub, along with the ecache set size. 833 * 834 * 1) Displacement flush the E$ line corresponding to %addr. 835 * The first ldxa guarantees that the %addr is no longer in 836 * M, O, or E (goes to I or S (if instruction fetch also happens). 837 * 2) "Write" the data using a CAS %addr,%g0,%g0. 838 * The casxa guarantees a transition from I to M or S to M. 839 * 3) Displacement flush the E$ line corresponding to %addr. 840 * The second ldxa pushes the M line out of the ecache, into the 841 * writeback buffers, on the way to memory. 842 * 4) The "membar #Sync" pushes the cache line out of the writeback 843 * buffers onto the bus, on the way to dram finally. 844 * 845 * This is a modified version of the algorithm suggested by Gary Lauterbach. 846 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line 847 * as modified, but then we found out that for spitfire, if it misses in the 848 * E$ it will probably install as an M, but if it hits in the E$, then it 849 * will stay E, if the store doesn't happen. So the first displacement flush 850 * should ensure that the CAS will miss in the E$. Arrgh. 851 */ 852/* ARGSUSED */ 853void 854scrubphys(uint64_t paddr, int ecache_set_size) 855{} 856 857#else /* lint */ 858 ENTRY(scrubphys) 859 rdpr %pstate, %o4 860 andn %o4, PSTATE_IE | PSTATE_AM, %o5 861 wrpr %o5, %g0, %pstate ! clear IE, AM bits 862 863#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 864 CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_1); 865 JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 866scrubphys_1: 867#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 868 869 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 870 casxa [%o0]ASI_MEM, %g0, %g0 871 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 872 873#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 874 CHK_JP_ERRATA85_ENABLED(%g1, scrubphys_2); 875 JP_RESTORE_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 876scrubphys_2: 877#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 878 879 wrpr %g0, %o4, %pstate ! restore earlier pstate register value 880 881 retl 882 membar #Sync ! move the data out of the load buffer 883 SET_SIZE(scrubphys) 884 885#endif /* lint */ 886 887 888#if defined(lint) 889/* 890 * clearphys - Pass in the physical memory address of the checkblock 891 * that you want to push out, cleared with a recognizable pattern, 892 * from the ecache. 893 * 894 * To ensure that the ecc gets recalculated after the bad data is cleared, 895 * we must write out enough data to fill the w$ line (64 bytes). So we read 896 * in an entire ecache subblock's worth of data, and write it back out. 897 * Then we overwrite the 16 bytes of bad data with the pattern. 898 */ 899/* ARGSUSED */ 900void 901clearphys(uint64_t paddr, int ecache_set_size, int ecache_linesize) 902{ 903} 904 905#else /* lint */ 906 ENTRY(clearphys) 907 /* turn off IE, AM bits */ 908 rdpr %pstate, %o4 909 andn %o4, PSTATE_IE | PSTATE_AM, %o5 910 wrpr %o5, %g0, %pstate 911 912 /* turn off NCEEN */ 913 ldxa [%g0]ASI_ESTATE_ERR, %o5 914 andn %o5, EN_REG_NCEEN, %o3 915 stxa %o3, [%g0]ASI_ESTATE_ERR 916 membar #Sync 917 918 /* align address passed with 64 bytes subblock size */ 919 mov CH_ECACHE_SUBBLK_SIZE, %o2 920 andn %o0, (CH_ECACHE_SUBBLK_SIZE - 1), %g1 921 922 /* move the good data into the W$ */ 9231: 924 subcc %o2, 8, %o2 925 ldxa [%g1 + %o2]ASI_MEM, %g2 926 bge 1b 927 stxa %g2, [%g1 + %o2]ASI_MEM 928 929 /* now overwrite the bad data */ 930 setx 0xbadecc00badecc01, %g1, %g2 931 stxa %g2, [%o0]ASI_MEM 932 mov 8, %g1 933 stxa %g2, [%o0 + %g1]ASI_MEM 934 935#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 936 CHK_JP_ERRATA85_ENABLED(%g1, clearphys_1); 937 JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 938clearphys_1: 939#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 940 941 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 942 casxa [%o0]ASI_MEM, %g0, %g0 943 ECACHE_REFLUSH_LINE(%o1, %o2, %o3) 944 945#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 946 CHK_JP_ERRATA85_ENABLED(%g1, clearphys_2); 947 JP_RESTORE_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 948clearphys_2: 949#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 950 951 /* clear the AFSR */ 952 ldxa [%g0]ASI_AFSR, %o1 953 stxa %o1, [%g0]ASI_AFSR 954 membar #Sync 955 956 /* turn NCEEN back on */ 957 stxa %o5, [%g0]ASI_ESTATE_ERR 958 membar #Sync 959 960 /* return and re-enable IE and AM */ 961 retl 962 wrpr %g0, %o4, %pstate 963 SET_SIZE(clearphys) 964 965#endif /* lint */ 966 967 968#if defined(lint) 969/* 970 * Jalapeno Ecache displacement flush the specified line from the E$ 971 * 972 * Register usage: 973 * %o0 - 64 bit physical address for flushing 974 * %o1 - Ecache set size 975 */ 976/*ARGSUSED*/ 977void 978ecache_flush_line(uint64_t flushaddr, int ec_set_size) 979{ 980} 981#else /* lint */ 982 ENTRY(ecache_flush_line) 983 984#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 985 CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_1); 986 JP_FORCE_FULL_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 987ecache_flush_line_1: 988#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 989 990 ECACHE_FLUSH_LINE(%o0, %o1, %o2, %o3) 991 992#if defined(JALAPENO) && defined(JALAPENO_ERRATA_85) 993 CHK_JP_ERRATA85_ENABLED(%g1, ecache_flush_line_2); 994 JP_RESTORE_SPEED(%o5, %g1, %g2, %g3) /* %o5: saved speed */ 995ecache_flush_line_2: 996#endif /* JALAPENO && JALAPENO_ERRATA_85 */ 997 998 retl 999 nop 1000 SET_SIZE(ecache_flush_line) 1001#endif /* lint */ 1002 1003 1004/* 1005 * Perform necessary cpu workaround to ensure jbus ordering. 1006 * Called only from Fire systems. 1007 * CPU's internal "invalidate FIFOs" are flushed. 1008 */ 1009 1010#if defined(lint) 1011void 1012jbus_stst_order() 1013{} 1014#else /* lint */ 1015 1016#define VIS_BLOCKSIZE 64 1017 1018 .seg ".data" 1019 .align VIS_BLOCKSIZE 1020 .type sync_buf, #object 1021sync_buf: 1022 .skip VIS_BLOCKSIZE 1023 .size sync_buf, VIS_BLOCKSIZE 1024 1025 ENTRY(jbus_stst_order) 1026 set sync_buf, %o1 1027 1028 rd %fprs, %o2 ! %o2 = saved fprs 1029 or %o2, FPRS_FEF, %o3 1030 wr %g0, %o3, %fprs ! make sure fp is enabled 1031 stda %d0, [%o1]ASI_BLK_COMMIT_P 1032 wr %o2, 0, %fprs ! restore fprs 1033 1034 retl 1035 membar #Sync 1036 SET_SIZE(jbus_stst_order) 1037 1038#endif /* lint */ 1039 1040#if defined(lint) 1041/* 1042 * This routine will not be called in Jalapeno systems. 1043 */ 1044void 1045flush_ipb(void) 1046{ return; } 1047 1048#else /* lint */ 1049 1050 ENTRY(flush_ipb) 1051 retl 1052 nop 1053 SET_SIZE(flush_ipb) 1054 1055#endif /* lint */ 1056