1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22/* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * 26 * Assembly code support for Cheetah/Cheetah+ modules 27 */ 28 29#pragma ident "%Z%%M% %I% %E% SMI" 30 31#if !defined(lint) 32#include "assym.h" 33#endif /* !lint */ 34 35#include <sys/asm_linkage.h> 36#include <sys/mmu.h> 37#include <vm/hat_sfmmu.h> 38#include <sys/machparam.h> 39#include <sys/machcpuvar.h> 40#include <sys/machthread.h> 41#include <sys/machtrap.h> 42#include <sys/privregs.h> 43#include <sys/trap.h> 44#include <sys/cheetahregs.h> 45#include <sys/us3_module.h> 46#include <sys/xc_impl.h> 47#include <sys/intreg.h> 48#include <sys/async.h> 49#include <sys/clock.h> 50#include <sys/cheetahasm.h> 51#include <sys/cmpregs.h> 52 53#ifdef TRAPTRACE 54#include <sys/traptrace.h> 55#endif /* TRAPTRACE */ 56 57#if !defined(lint) 58 59/* BEGIN CSTYLED */ 60 61#define DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3) \ 62 ldxa [%g0]ASI_DCU, tmp1 ;\ 63 btst DCU_DC, tmp1 /* is dcache enabled? */ ;\ 64 bz,pn %icc, 1f ;\ 65 ASM_LD(tmp1, dcache_linesize) ;\ 66 ASM_LD(tmp2, dflush_type) ;\ 67 cmp tmp2, FLUSHPAGE_TYPE ;\ 68 be,pt %icc, 2f ;\ 69 nop ;\ 70 sllx arg1, CHEETAH_DC_VBIT_SHIFT, arg1/* tag to compare */ ;\ 71 ASM_LD(tmp3, dcache_size) ;\ 72 cmp tmp2, FLUSHMATCH_TYPE ;\ 73 be,pt %icc, 3f ;\ 74 nop ;\ 75 /* \ 76 * flushtype = FLUSHALL_TYPE, flush the whole thing \ 77 * tmp3 = cache size \ 78 * tmp1 = cache line size \ 79 */ \ 80 sub tmp3, tmp1, tmp2 ;\ 814: \ 82 stxa %g0, [tmp2]ASI_DC_TAG ;\ 83 membar #Sync ;\ 84 cmp %g0, tmp2 ;\ 85 bne,pt %icc, 4b ;\ 86 sub tmp2, tmp1, tmp2 ;\ 87 ba,pt %icc, 1f ;\ 88 nop ;\ 89 /* \ 90 * flushtype = FLUSHPAGE_TYPE \ 91 * arg1 = pfn \ 92 * arg2 = virtual color \ 93 * tmp1 = cache line size \ 94 * tmp2 = tag from cache \ 95 * tmp3 = counter \ 96 */ \ 972: \ 98 set MMU_PAGESIZE, tmp3 ;\ 99 sllx arg1, MMU_PAGESHIFT, arg1 /* pfn to 43 bit PA */ ;\ 100 sub tmp3, tmp1, tmp3 ;\ 1014: \ 102 stxa %g0, [arg1 + tmp3]ASI_DC_INVAL ;\ 103 membar #Sync ;\ 1045: \ 105 cmp %g0, tmp3 ;\ 106 bnz,pt %icc, 4b /* branch if not done */ ;\ 107 sub tmp3, tmp1, tmp3 ;\ 108 ba,pt %icc, 1f ;\ 109 nop ;\ 110 /* \ 111 * flushtype = FLUSHMATCH_TYPE \ 112 * arg1 = tag to compare against \ 113 * tmp1 = cache line size \ 114 * tmp3 = cache size \ 115 * arg2 = counter \ 116 * tmp2 = cache tag \ 117 */ \ 1183: \ 119 sub tmp3, tmp1, arg2 ;\ 1204: \ 121 ldxa [arg2]ASI_DC_TAG, tmp2 /* read tag */ ;\ 122 btst CHEETAH_DC_VBIT_MASK, tmp2 ;\ 123 bz,pn %icc, 5f /* br if no valid sub-blocks */ ;\ 124 andn tmp2, CHEETAH_DC_VBIT_MASK, tmp2 /* clear out v bits */ ;\ 125 cmp tmp2, arg1 ;\ 126 bne,pn %icc, 5f /* branch if tag miss */ ;\ 127 nop ;\ 128 stxa %g0, [arg2]ASI_DC_TAG ;\ 129 membar #Sync ;\ 1305: \ 131 cmp %g0, arg2 ;\ 132 bne,pt %icc, 4b /* branch if not done */ ;\ 133 sub arg2, tmp1, arg2 ;\ 1341: 135 136 137/* END CSTYLED */ 138 139#endif /* !lint */ 140 141/* 142 * Cheetah MMU and Cache operations. 143 */ 144 145#if defined(lint) 146 147/* ARGSUSED */ 148void 149vtag_flushpage(caddr_t vaddr, u_int ctxnum) 150{} 151 152#else /* lint */ 153 154 ENTRY_NP(vtag_flushpage) 155 /* 156 * flush page from the tlb 157 * 158 * %o0 = vaddr 159 * %o1 = ctxnum 160 */ 161 rdpr %pstate, %o5 162#ifdef DEBUG 163 andcc %o5, PSTATE_IE, %g0 /* if interrupts already */ 164 bnz,a,pt %icc, 3f /* disabled, panic */ 165 nop 166 save %sp, -SA(MINFRAME), %sp 167 sethi %hi(sfmmu_panic1), %o0 168 call panic 169 or %o0, %lo(sfmmu_panic1), %o0 170 ret 171 restore 1723: 173#endif /* DEBUG */ 174 /* 175 * disable ints 176 */ 177 andn %o5, PSTATE_IE, %o4 178 wrpr %o4, 0, %pstate 179 180 /* 181 * Then, blow out the tlb 182 * Interrupts are disabled to prevent the primary ctx register 183 * from changing underneath us. 184 */ 185 brnz,pt %o1, 1f /* KCONTEXT */ 186 sethi %hi(FLUSH_ADDR), %o3 187 /* 188 * For KCONTEXT demaps use primary. type = page implicitly 189 */ 190 stxa %g0, [%o0]ASI_DTLB_DEMAP /* dmmu flush for KCONTEXT */ 191 stxa %g0, [%o0]ASI_ITLB_DEMAP /* immu flush for KCONTEXT */ 192 flush %o3 193 b 5f 194 nop 1951: 196 /* 197 * User demap. We need to set the primary context properly. 198 * Secondary context cannot be used for Cheetah IMMU. 199 * %o0 = vaddr 200 * %o1 = ctxnum 201 * %o3 = FLUSH_ADDR 202 */ 203 sethi %hi(ctx_pgsz_array), %o4 204 ldn [%o4 + %lo(ctx_pgsz_array)], %o4 205 brz %o4, 2f 206 nop 207 ldub [%o4 + %o1], %o4 208 sll %o4, CTXREG_EXT_SHIFT, %o4 209 or %o1, %o4, %o1 2102: 211 wrpr %g0, 1, %tl 212 set MMU_PCONTEXT, %o4 213 or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %o0, %o0 214 ldxa [%o4]ASI_DMMU, %o2 /* rd old ctxnum */ 215 stxa %o1, [%o4]ASI_DMMU /* wr new ctxum */ 2164: 217 stxa %g0, [%o0]ASI_DTLB_DEMAP 218 stxa %g0, [%o0]ASI_ITLB_DEMAP 219 stxa %o2, [%o4]ASI_DMMU /* restore old ctxnum */ 220 flush %o3 221 wrpr %g0, 0, %tl 2225: 223 retl 224 wrpr %g0, %o5, %pstate /* enable interrupts */ 225 SET_SIZE(vtag_flushpage) 226 227#endif /* lint */ 228 229 230#if defined(lint) 231 232/* ARGSUSED */ 233void 234vtag_flushctx(u_int ctxnum) 235{} 236 237#else /* lint */ 238 239 ENTRY_NP(vtag_flushctx) 240 /* 241 * flush context from the tlb 242 * 243 * %o0 = ctxnum 244 * We disable interrupts to prevent the primary ctx register changing 245 * underneath us. 246 */ 247 sethi %hi(FLUSH_ADDR), %o3 248 rdpr %pstate, %o2 249 250#ifdef DEBUG 251 andcc %o2, PSTATE_IE, %g0 /* if interrupts already */ 252 bnz,a,pt %icc, 1f /* disabled, panic */ 253 nop 254 sethi %hi(sfmmu_panic1), %o0 255 call panic 256 or %o0, %lo(sfmmu_panic1), %o0 2571: 258#endif /* DEBUG */ 259 260 sethi %hi(ctx_pgsz_array), %o4 261 ldn [%o4 + %lo(ctx_pgsz_array)], %o4 262 brz %o4, 2f 263 nop 264 ldub [%o4 + %o0], %o4 265 sll %o4, CTXREG_EXT_SHIFT, %o4 266 or %o0, %o4, %o0 2672: 268 wrpr %o2, PSTATE_IE, %pstate /* disable interrupts */ 269 set MMU_PCONTEXT, %o4 270 set DEMAP_CTX_TYPE | DEMAP_PRIMARY, %g1 271 wrpr %g0, 1, %tl 272 ldxa [%o4]ASI_DMMU, %o5 /* rd old ctxnum */ 273 stxa %o0, [%o4]ASI_DMMU /* wr new ctxum */ 2744: 275 stxa %g0, [%g1]ASI_DTLB_DEMAP 276 stxa %g0, [%g1]ASI_ITLB_DEMAP 277 stxa %o5, [%o4]ASI_DMMU /* restore old ctxnum */ 278 flush %o3 279 wrpr %g0, 0, %tl 2805: 281 retl 282 wrpr %g0, %o2, %pstate /* enable interrupts */ 283 SET_SIZE(vtag_flushctx) 284 285#endif /* lint */ 286 287 288#if defined(lint) 289 290void 291vtag_flushall(void) 292{} 293 294#else /* lint */ 295 296 ENTRY_NP2(vtag_flushall, demap_all) 297 /* 298 * flush the tlb 299 */ 300 sethi %hi(FLUSH_ADDR), %o3 301 set DEMAP_ALL_TYPE, %g1 302 stxa %g0, [%g1]ASI_DTLB_DEMAP 303 stxa %g0, [%g1]ASI_ITLB_DEMAP 304 flush %o3 305 retl 306 nop 307 SET_SIZE(demap_all) 308 SET_SIZE(vtag_flushall) 309 310#endif /* lint */ 311 312 313#if defined(lint) 314 315/* ARGSUSED */ 316void 317vtag_flushpage_tl1(uint64_t vaddr, uint64_t ctxnum) 318{} 319 320#else /* lint */ 321 322 ENTRY_NP(vtag_flushpage_tl1) 323 /* 324 * x-trap to flush page from tlb and tsb 325 * 326 * %g1 = vaddr, zero-extended on 32-bit kernel 327 * %g2 = ctxnum 328 * 329 * assumes TSBE_TAG = 0 330 */ 331 srln %g1, MMU_PAGESHIFT, %g1 332 brnz,pt %g2, 1f /* KCONTEXT */ 333 slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ 334 335 /* We need to demap in the kernel context */ 336 or DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1 337 stxa %g0, [%g1]ASI_DTLB_DEMAP 338 stxa %g0, [%g1]ASI_ITLB_DEMAP 339 retry 3401: 341 /* We need to demap in a user context */ 342 or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1 343 sethi %hi(ctx_pgsz_array), %g4 344 ldn [%g4 + %lo(ctx_pgsz_array)], %g4 345 brz %g4, 2f 346 nop 347 ldub [%g4 + %g2], %g4 348 sll %g4, CTXREG_EXT_SHIFT, %g4 349 or %g2, %g4, %g2 3502: 351 set MMU_PCONTEXT, %g4 352 ldxa [%g4]ASI_DMMU, %g5 /* rd old ctxnum */ 353 stxa %g2, [%g4]ASI_DMMU /* wr new ctxum */ 354 stxa %g0, [%g1]ASI_DTLB_DEMAP 355 stxa %g0, [%g1]ASI_ITLB_DEMAP 356 stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ 357 retry 358 SET_SIZE(vtag_flushpage_tl1) 359 360#endif /* lint */ 361 362 363#if defined(lint) 364 365/* ARGSUSED */ 366void 367vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t ctx_pgcnt) 368{} 369 370#else /* lint */ 371 372 ENTRY_NP(vtag_flush_pgcnt_tl1) 373 /* 374 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb 375 * 376 * %g1 = vaddr, zero-extended on 32-bit kernel 377 * %g2 = <zero32|ctx16|pgcnt16> 378 * 379 * NOTE: this handler relies on the fact that no 380 * interrupts or traps can occur during the loop 381 * issuing the TLB_DEMAP operations. It is assumed 382 * that interrupts are disabled and this code is 383 * fetching from the kernel locked text address. 384 * 385 * assumes TSBE_TAG = 0 386 */ 387 set 0xffff, %g4 388 and %g4, %g2, %g3 /* g3 = pgcnt */ 389 srln %g2, 16, %g2 /* g2 = ctxnum */ 390 srln %g1, MMU_PAGESHIFT, %g1 391 brnz,pt %g2, 1f /* KCONTEXT? */ 392 slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ 393 394 /* We need to demap in the kernel context */ 395 or DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1 396 set MMU_PAGESIZE, %g2 /* g2 = pgsize */ 3974: 398 stxa %g0, [%g1]ASI_DTLB_DEMAP 399 stxa %g0, [%g1]ASI_ITLB_DEMAP 400 deccc %g3 /* decr pgcnt */ 401 bnz,pt %icc,4b 402 add %g1, %g2, %g1 /* next page */ 403 retry 4041: 405 /* We need to demap in a user context */ 406 sethi %hi(ctx_pgsz_array), %g4 407 ldn [%g4 + %lo(ctx_pgsz_array)], %g4 408 brz %g4, 2f 409 or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1 410 ldub [%g4 + %g2], %g4 411 sll %g4, CTXREG_EXT_SHIFT, %g4 412 or %g2, %g4, %g2 4132: 414 set MMU_PCONTEXT, %g4 415 ldxa [%g4]ASI_DMMU, %g5 /* rd old ctxnum */ 416 stxa %g2, [%g4]ASI_DMMU /* wr new ctxum */ 417 418 set MMU_PAGESIZE, %g2 /* g2 = pgsize */ 4193: 420 stxa %g0, [%g1]ASI_DTLB_DEMAP 421 stxa %g0, [%g1]ASI_ITLB_DEMAP 422 deccc %g3 /* decr pgcnt */ 423 bnz,pt %icc,3b 424 add %g1, %g2, %g1 /* next page */ 425 426 stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ 427 retry 428 SET_SIZE(vtag_flush_pgcnt_tl1) 429 430#endif /* lint */ 431 432 433#if defined(lint) 434 435/* ARGSUSED */ 436void 437vtag_flushctx_tl1(uint64_t ctxnum, uint64_t dummy) 438{} 439 440#else /* lint */ 441 442 ENTRY_NP(vtag_flushctx_tl1) 443 /* 444 * x-trap to flush context from tlb 445 * 446 * %g1 = ctxnum 447 */ 448 sethi %hi(ctx_pgsz_array), %g4 449 ldn [%g4 + %lo(ctx_pgsz_array)], %g4 450 brz %g4, 2f 451 nop 452 ldub [%g4 + %g1], %g4 453 sll %g4, CTXREG_EXT_SHIFT, %g4 454 or %g1, %g4, %g1 4552: 456 set DEMAP_CTX_TYPE | DEMAP_PRIMARY, %g4 457 set MMU_PCONTEXT, %g3 458 ldxa [%g3]ASI_DMMU, %g5 /* rd old ctxnum */ 459 stxa %g1, [%g3]ASI_DMMU /* wr new ctxum */ 460 stxa %g0, [%g4]ASI_DTLB_DEMAP 461 stxa %g0, [%g4]ASI_ITLB_DEMAP 462 stxa %g5, [%g3]ASI_DMMU /* restore old ctxnum */ 463 retry 464 SET_SIZE(vtag_flushctx_tl1) 465 466#endif /* lint */ 467 468 469#if defined(lint) 470 471/*ARGSUSED*/ 472void 473vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2) 474{} 475 476#else /* lint */ 477 478 ENTRY_NP(vtag_flushall_tl1) 479 /* 480 * x-trap to flush tlb 481 */ 482 set DEMAP_ALL_TYPE, %g4 483 stxa %g0, [%g4]ASI_DTLB_DEMAP 484 stxa %g0, [%g4]ASI_ITLB_DEMAP 485 retry 486 SET_SIZE(vtag_flushall_tl1) 487 488#endif /* lint */ 489 490 491#if defined(lint) 492 493/* ARGSUSED */ 494void 495vac_flushpage(pfn_t pfnum, int vcolor) 496{} 497 498#else /* lint */ 499 500/* 501 * vac_flushpage(pfnum, color) 502 * Flush 1 8k page of the D-$ with physical page = pfnum 503 * Algorithm: 504 * The cheetah dcache is a 64k psuedo 4 way accaociative cache. 505 * It is virtual indexed, physically tagged cache. 506 */ 507 .seg ".data" 508 .align 8 509 .global dflush_type 510dflush_type: 511 .word FLUSHPAGE_TYPE 512 513 ENTRY(vac_flushpage) 514 /* 515 * flush page from the d$ 516 * 517 * %o0 = pfnum, %o1 = color 518 */ 519 DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4) 520 retl 521 nop 522 SET_SIZE(vac_flushpage) 523 524#endif /* lint */ 525 526 527#if defined(lint) 528 529/* ARGSUSED */ 530void 531vac_flushpage_tl1(uint64_t pfnum, uint64_t vcolor) 532{} 533 534#else /* lint */ 535 536 ENTRY_NP(vac_flushpage_tl1) 537 /* 538 * x-trap to flush page from the d$ 539 * 540 * %g1 = pfnum, %g2 = color 541 */ 542 DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5) 543 retry 544 SET_SIZE(vac_flushpage_tl1) 545 546#endif /* lint */ 547 548 549#if defined(lint) 550 551/* ARGSUSED */ 552void 553vac_flushcolor(int vcolor, pfn_t pfnum) 554{} 555 556#else /* lint */ 557 /* 558 * In UltraSPARC III flushcolor is same as as flushpage. 559 * This is because we have an ASI to flush dcache using physical 560 * address. 561 * Flushing dcache using physical address is faster because we 562 * don't have to deal with associativity of dcache. 563 * The arguments to vac_flushpage() and vac_flushcolor() are same but 564 * the order is reversed. this is because we maintain compatibility 565 * with spitfire, in which vac_flushcolor has only one argument, namely 566 * vcolor. 567 */ 568 569 ENTRY(vac_flushcolor) 570 /* 571 * %o0 = vcolor, %o1 = pfnum 572 */ 573 DCACHE_FLUSHPAGE(%o1, %o0, %o2, %o3, %o4) 574 retl 575 nop 576 SET_SIZE(vac_flushcolor) 577 578#endif /* lint */ 579 580 581#if defined(lint) 582 583/* ARGSUSED */ 584void 585vac_flushcolor_tl1(uint64_t vcolor, uint64_t pfnum) 586{} 587 588#else /* lint */ 589 590 ENTRY(vac_flushcolor_tl1) 591 /* 592 * %g1 = vcolor 593 * %g2 = pfnum 594 */ 595 DCACHE_FLUSHPAGE(%g2, %g1, %g3, %g4, %g5) 596 retry 597 SET_SIZE(vac_flushcolor_tl1) 598 599#endif /* lint */ 600 601#if defined(lint) 602 603int 604idsr_busy(void) 605{ 606 return (0); 607} 608 609#else /* lint */ 610 611/* 612 * Determine whether or not the IDSR is busy. 613 * Entry: no arguments 614 * Returns: 1 if busy, 0 otherwise 615 */ 616 ENTRY(idsr_busy) 617 ldxa [%g0]ASI_INTR_DISPATCH_STATUS, %g1 618 clr %o0 619 btst IDSR_BUSY, %g1 620 bz,a,pt %xcc, 1f 621 mov 1, %o0 6221: 623 retl 624 nop 625 SET_SIZE(idsr_busy) 626 627#endif /* lint */ 628 629#if defined(lint) 630 631/* ARGSUSED */ 632void 633init_mondo(xcfunc_t *func, uint64_t arg1, uint64_t arg2) 634{} 635 636/* ARGSUSED */ 637void 638init_mondo_nocheck(xcfunc_t *func, uint64_t arg1, uint64_t arg2) 639{} 640 641#else /* lint */ 642 643 .global _dispatch_status_busy 644_dispatch_status_busy: 645 .asciz "ASI_INTR_DISPATCH_STATUS error: busy" 646 .align 4 647 648/* 649 * Setup interrupt dispatch data registers 650 * Entry: 651 * %o0 - function or inumber to call 652 * %o1, %o2 - arguments (2 uint64_t's) 653 */ 654 .seg "text" 655 656 ENTRY(init_mondo) 657#ifdef DEBUG 658 ! 659 ! IDSR should not be busy at the moment 660 ! 661 ldxa [%g0]ASI_INTR_DISPATCH_STATUS, %g1 662 btst IDSR_BUSY, %g1 663 bz,pt %xcc, 1f 664 nop 665 sethi %hi(_dispatch_status_busy), %o0 666 call panic 667 or %o0, %lo(_dispatch_status_busy), %o0 668#endif /* DEBUG */ 669 670 ALTENTRY(init_mondo_nocheck) 671 ! 672 ! interrupt vector dispatch data reg 0 673 ! 6741: 675 mov IDDR_0, %g1 676 mov IDDR_1, %g2 677 mov IDDR_2, %g3 678 stxa %o0, [%g1]ASI_INTR_DISPATCH 679 680 ! 681 ! interrupt vector dispatch data reg 1 682 ! 683 stxa %o1, [%g2]ASI_INTR_DISPATCH 684 685 ! 686 ! interrupt vector dispatch data reg 2 687 ! 688 stxa %o2, [%g3]ASI_INTR_DISPATCH 689 690 membar #Sync 691 retl 692 nop 693 SET_SIZE(init_mondo_nocheck) 694 SET_SIZE(init_mondo) 695 696#endif /* lint */ 697 698 699#if !(defined(JALAPENO) || defined(SERRANO)) 700 701#if defined(lint) 702 703/* ARGSUSED */ 704void 705shipit(int upaid, int bn) 706{ return; } 707 708#else /* lint */ 709 710/* 711 * Ship mondo to aid using busy/nack pair bn 712 */ 713 ENTRY_NP(shipit) 714 sll %o0, IDCR_PID_SHIFT, %g1 ! IDCR<18:14> = agent id 715 sll %o1, IDCR_BN_SHIFT, %g2 ! IDCR<28:24> = b/n pair 716 or %g1, IDCR_OFFSET, %g1 ! IDCR<13:0> = 0x70 717 or %g1, %g2, %g1 718 stxa %g0, [%g1]ASI_INTR_DISPATCH ! interrupt vector dispatch 719 membar #Sync 720 retl 721 nop 722 SET_SIZE(shipit) 723 724#endif /* lint */ 725 726#endif /* !(JALAPENO || SERRANO) */ 727 728 729#if defined(lint) 730 731/* ARGSUSED */ 732void 733flush_instr_mem(caddr_t vaddr, size_t len) 734{} 735 736#else /* lint */ 737 738/* 739 * flush_instr_mem: 740 * Flush 1 page of the I-$ starting at vaddr 741 * %o0 vaddr 742 * %o1 bytes to be flushed 743 * UltraSPARC-III maintains consistency of the on-chip Instruction Cache with 744 * the stores from all processors so that a FLUSH instruction is only needed 745 * to ensure pipeline is consistent. This means a single flush is sufficient at 746 * the end of a sequence of stores that updates the instruction stream to 747 * ensure correct operation. 748 */ 749 750 ENTRY(flush_instr_mem) 751 flush %o0 ! address irrelevent 752 retl 753 nop 754 SET_SIZE(flush_instr_mem) 755 756#endif /* lint */ 757 758 759#if defined(CPU_IMP_ECACHE_ASSOC) 760 761#if defined(lint) 762 763/* ARGSUSED */ 764uint64_t 765get_ecache_ctrl(void) 766{ return (0); } 767 768#else /* lint */ 769 770 ENTRY(get_ecache_ctrl) 771 GET_CPU_IMPL(%o0) 772 cmp %o0, JAGUAR_IMPL 773 ! 774 ! Putting an ASI access in the delay slot may 775 ! cause it to be accessed, even when annulled. 776 ! 777 bne 1f 778 nop 779 ldxa [%g0]ASI_EC_CFG_TIMING, %o0 ! read Jaguar shared E$ ctrl reg 780 b 2f 781 nop 7821: 783 ldxa [%g0]ASI_EC_CTRL, %o0 ! read Ch/Ch+ E$ control reg 7842: 785 retl 786 nop 787 SET_SIZE(get_ecache_ctrl) 788 789#endif /* lint */ 790 791#endif /* CPU_IMP_ECACHE_ASSOC */ 792 793 794#if !(defined(JALAPENO) || defined(SERRANO)) 795 796/* 797 * flush_ecache: 798 * %o0 - 64 bit physical address 799 * %o1 - ecache size 800 * %o2 - ecache linesize 801 */ 802#if defined(lint) 803 804/*ARGSUSED*/ 805void 806flush_ecache(uint64_t physaddr, size_t ecache_size, size_t ecache_linesize) 807{} 808 809#else /* !lint */ 810 811 ENTRY(flush_ecache) 812 813 /* 814 * For certain CPU implementations, we have to flush the L2 cache 815 * before flushing the ecache. 816 */ 817 PN_L2_FLUSHALL(%g3, %g4, %g5) 818 819 /* 820 * Flush the entire Ecache using displacement flush. 821 */ 822 ECACHE_FLUSHALL(%o1, %o2, %o0, %o4) 823 824 retl 825 nop 826 SET_SIZE(flush_ecache) 827 828#endif /* lint */ 829 830#endif /* !(JALAPENO || SERRANO) */ 831 832 833#if defined(lint) 834 835void 836flush_dcache(void) 837{} 838 839#else /* lint */ 840 841 ENTRY(flush_dcache) 842 ASM_LD(%o0, dcache_size) 843 ASM_LD(%o1, dcache_linesize) 844 CH_DCACHE_FLUSHALL(%o0, %o1, %o2) 845 retl 846 nop 847 SET_SIZE(flush_dcache) 848 849#endif /* lint */ 850 851 852#if defined(lint) 853 854void 855flush_icache(void) 856{} 857 858#else /* lint */ 859 860 ENTRY(flush_icache) 861 GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, flush_icache_1); 862 ld [%o0 + CHPR_ICACHE_LINESIZE], %o1 863 ba,pt %icc, 2f 864 ld [%o0 + CHPR_ICACHE_SIZE], %o0 865flush_icache_1: 866 ASM_LD(%o0, icache_size) 867 ASM_LD(%o1, icache_linesize) 8682: 869 CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4) 870 retl 871 nop 872 SET_SIZE(flush_icache) 873 874#endif /* lint */ 875 876#if defined(lint) 877 878/*ARGSUSED*/ 879void 880kdi_flush_idcache(int dcache_size, int dcache_lsize, int icache_size, 881 int icache_lsize) 882{ 883} 884 885#else /* lint */ 886 887 ENTRY(kdi_flush_idcache) 888 CH_DCACHE_FLUSHALL(%o0, %o1, %g1) 889 CH_ICACHE_FLUSHALL(%o2, %o3, %g1, %g2) 890 membar #Sync 891 retl 892 nop 893 SET_SIZE(kdi_flush_idcache) 894 895#endif /* lint */ 896 897#if defined(lint) 898 899void 900flush_pcache(void) 901{} 902 903#else /* lint */ 904 905 ENTRY(flush_pcache) 906 PCACHE_FLUSHALL(%o0, %o1, %o2) 907 retl 908 nop 909 SET_SIZE(flush_pcache) 910 911#endif /* lint */ 912 913 914#if defined(CPU_IMP_L1_CACHE_PARITY) 915 916#if defined(lint) 917 918/* ARGSUSED */ 919void 920get_dcache_dtag(uint32_t dcache_idx, uint64_t *data) 921{} 922 923#else /* lint */ 924 925/* 926 * Get dcache data and tag. The Dcache data is a pointer to a ch_dc_data_t 927 * structure (see cheetahregs.h): 928 * The Dcache *should* be turned off when this code is executed. 929 */ 930 .align 128 931 ENTRY(get_dcache_dtag) 932 rdpr %pstate, %o5 933 andn %o5, PSTATE_IE | PSTATE_AM, %o3 934 wrpr %g0, %o3, %pstate 935 b 1f 936 stx %o0, [%o1 + CH_DC_IDX] 937 938 .align 128 9391: 940 ldxa [%o0]ASI_DC_TAG, %o2 941 stx %o2, [%o1 + CH_DC_TAG] 942 membar #Sync 943 ldxa [%o0]ASI_DC_UTAG, %o2 944 membar #Sync 945 stx %o2, [%o1 + CH_DC_UTAG] 946 ldxa [%o0]ASI_DC_SNP_TAG, %o2 947 stx %o2, [%o1 + CH_DC_SNTAG] 948 add %o1, CH_DC_DATA, %o1 949 clr %o3 9502: 951 membar #Sync ! required before ASI_DC_DATA 952 ldxa [%o0 + %o3]ASI_DC_DATA, %o2 953 membar #Sync ! required after ASI_DC_DATA 954 stx %o2, [%o1 + %o3] 955 cmp %o3, CH_DC_DATA_REG_SIZE - 8 956 blt 2b 957 add %o3, 8, %o3 958 959 /* 960 * Unlike other CPUs in the family, D$ data parity bits for Panther 961 * do not reside in the microtag. Instead, we have to read them 962 * using the DC_data_parity bit of ASI_DCACHE_DATA. Also, instead 963 * of just having 8 parity bits to protect all 32 bytes of data 964 * per line, we now have 32 bits of parity. 965 */ 966 GET_CPU_IMPL(%o3) 967 cmp %o3, PANTHER_IMPL 968 bne 4f 969 clr %o3 970 971 /* 972 * move our pointer to the next field where we store parity bits 973 * and add the offset of the last parity byte since we will be 974 * storing all 4 parity bytes within one 64 bit field like this: 975 * 976 * +------+------------+------------+------------+------------+ 977 * | - | DC_parity | DC_parity | DC_parity | DC_parity | 978 * | - | for word 3 | for word 2 | for word 1 | for word 0 | 979 * +------+------------+------------+------------+------------+ 980 * 63:32 31:24 23:16 15:8 7:0 981 */ 982 add %o1, CH_DC_PN_DATA_PARITY - CH_DC_DATA + 7, %o1 983 984 /* add the DC_data_parity bit into our working index */ 985 mov 1, %o2 986 sll %o2, PN_DC_DATA_PARITY_BIT_SHIFT, %o2 987 or %o0, %o2, %o0 9883: 989 membar #Sync ! required before ASI_DC_DATA 990 ldxa [%o0 + %o3]ASI_DC_DATA, %o2 991 membar #Sync ! required after ASI_DC_DATA 992 stb %o2, [%o1] 993 dec %o1 994 cmp %o3, CH_DC_DATA_REG_SIZE - 8 995 blt 3b 996 add %o3, 8, %o3 9974: 998 retl 999 wrpr %g0, %o5, %pstate 1000 SET_SIZE(get_dcache_dtag) 1001 1002#endif /* lint */ 1003 1004 1005#if defined(lint) 1006 1007/* ARGSUSED */ 1008void 1009get_icache_dtag(uint32_t ecache_idx, uint64_t *data) 1010{} 1011 1012#else /* lint */ 1013 1014/* 1015 * Get icache data and tag. The data argument is a pointer to a ch_ic_data_t 1016 * structure (see cheetahregs.h): 1017 * The Icache *Must* be turned off when this function is called. 1018 * This is because diagnostic accesses to the Icache interfere with cache 1019 * consistency. 1020 */ 1021 .align 128 1022 ENTRY(get_icache_dtag) 1023 rdpr %pstate, %o5 1024 andn %o5, PSTATE_IE | PSTATE_AM, %o3 1025 wrpr %g0, %o3, %pstate 1026 1027 stx %o0, [%o1 + CH_IC_IDX] 1028 ldxa [%o0]ASI_IC_TAG, %o2 1029 stx %o2, [%o1 + CH_IC_PATAG] 1030 add %o0, CH_ICTAG_UTAG, %o0 1031 ldxa [%o0]ASI_IC_TAG, %o2 1032 add %o0, (CH_ICTAG_UPPER - CH_ICTAG_UTAG), %o0 1033 stx %o2, [%o1 + CH_IC_UTAG] 1034 ldxa [%o0]ASI_IC_TAG, %o2 1035 add %o0, (CH_ICTAG_LOWER - CH_ICTAG_UPPER), %o0 1036 stx %o2, [%o1 + CH_IC_UPPER] 1037 ldxa [%o0]ASI_IC_TAG, %o2 1038 andn %o0, CH_ICTAG_TMASK, %o0 1039 stx %o2, [%o1 + CH_IC_LOWER] 1040 ldxa [%o0]ASI_IC_SNP_TAG, %o2 1041 stx %o2, [%o1 + CH_IC_SNTAG] 1042 add %o1, CH_IC_DATA, %o1 1043 clr %o3 10442: 1045 ldxa [%o0 + %o3]ASI_IC_DATA, %o2 1046 stx %o2, [%o1 + %o3] 1047 cmp %o3, PN_IC_DATA_REG_SIZE - 8 1048 blt 2b 1049 add %o3, 8, %o3 1050 1051 retl 1052 wrpr %g0, %o5, %pstate 1053 SET_SIZE(get_icache_dtag) 1054 1055#endif /* lint */ 1056 1057#if defined(lint) 1058 1059/* ARGSUSED */ 1060void 1061get_pcache_dtag(uint32_t pcache_idx, uint64_t *data) 1062{} 1063 1064#else /* lint */ 1065 1066/* 1067 * Get pcache data and tags. 1068 * inputs: 1069 * pcache_idx - fully constructed VA for for accessing P$ diagnostic 1070 * registers. Contains PC_way and PC_addr shifted into 1071 * the correct bit positions. See the PRM for more details. 1072 * data - pointer to a ch_pc_data_t 1073 * structure (see cheetahregs.h): 1074 */ 1075 .align 128 1076 ENTRY(get_pcache_dtag) 1077 rdpr %pstate, %o5 1078 andn %o5, PSTATE_IE | PSTATE_AM, %o3 1079 wrpr %g0, %o3, %pstate 1080 1081 stx %o0, [%o1 + CH_PC_IDX] 1082 ldxa [%o0]ASI_PC_STATUS_DATA, %o2 1083 stx %o2, [%o1 + CH_PC_STATUS] 1084 ldxa [%o0]ASI_PC_TAG, %o2 1085 stx %o2, [%o1 + CH_PC_TAG] 1086 ldxa [%o0]ASI_PC_SNP_TAG, %o2 1087 stx %o2, [%o1 + CH_PC_SNTAG] 1088 add %o1, CH_PC_DATA, %o1 1089 clr %o3 10902: 1091 ldxa [%o0 + %o3]ASI_PC_DATA, %o2 1092 stx %o2, [%o1 + %o3] 1093 cmp %o3, CH_PC_DATA_REG_SIZE - 8 1094 blt 2b 1095 add %o3, 8, %o3 1096 1097 retl 1098 wrpr %g0, %o5, %pstate 1099 SET_SIZE(get_pcache_dtag) 1100 1101#endif /* lint */ 1102 1103#endif /* CPU_IMP_L1_CACHE_PARITY */ 1104 1105#if defined(lint) 1106 1107/* ARGSUSED */ 1108void 1109set_dcu(uint64_t dcu) 1110{} 1111 1112#else /* lint */ 1113 1114/* 1115 * re-enable the i$, d$, w$, and p$ according to bootup cache state. 1116 * Turn on WE, HPE, SPE, PE, IC, and DC bits defined as DCU_CACHE. 1117 * %o0 - 64 bit constant 1118 */ 1119 ENTRY(set_dcu) 1120 stxa %o0, [%g0]ASI_DCU ! Store to DCU 1121 flush %g0 /* flush required after changing the IC bit */ 1122 retl 1123 nop 1124 SET_SIZE(set_dcu) 1125 1126#endif /* lint */ 1127 1128 1129#if defined(lint) 1130 1131uint64_t 1132get_dcu(void) 1133{ 1134 return ((uint64_t)0); 1135} 1136 1137#else /* lint */ 1138 1139/* 1140 * Return DCU register. 1141 */ 1142 ENTRY(get_dcu) 1143 ldxa [%g0]ASI_DCU, %o0 /* DCU control register */ 1144 retl 1145 nop 1146 SET_SIZE(get_dcu) 1147 1148#endif /* lint */ 1149 1150/* 1151 * Cheetah/Cheetah+ level 15 interrupt handler trap table entry. 1152 * 1153 * This handler is used to check for softints generated by error trap 1154 * handlers to report errors. On Cheetah, this mechanism is used by the 1155 * Fast ECC at TL>0 error trap handler and, on Cheetah+, by both the Fast 1156 * ECC at TL>0 error and the I$/D$ parity error at TL>0 trap handlers. 1157 * NB: Must be 8 instructions or less to fit in trap table and code must 1158 * be relocatable. 1159 */ 1160#if defined(lint) 1161 1162void 1163ch_pil15_interrupt_instr(void) 1164{} 1165 1166#else /* lint */ 1167 1168 ENTRY_NP(ch_pil15_interrupt_instr) 1169 ASM_JMP(%g1, ch_pil15_interrupt) 1170 SET_SIZE(ch_pil15_interrupt_instr) 1171 1172#endif 1173 1174 1175#if defined(lint) 1176 1177void 1178ch_pil15_interrupt(void) 1179{} 1180 1181#else /* lint */ 1182 1183 ENTRY_NP(ch_pil15_interrupt) 1184 1185 /* 1186 * Since pil_interrupt is hacked to assume that every level 15 1187 * interrupt is generated by the CPU to indicate a performance 1188 * counter overflow this gets ugly. Before calling pil_interrupt 1189 * the Error at TL>0 pending status is inspected. If it is 1190 * non-zero, then an error has occurred and it is handled. 1191 * Otherwise control is transfered to pil_interrupt. Note that if 1192 * an error is detected pil_interrupt will not be called and 1193 * overflow interrupts may be lost causing erroneous performance 1194 * measurements. However, error-recovery will have a detrimental 1195 * effect on performance anyway. 1196 */ 1197 CPU_INDEX(%g1, %g4) 1198 set ch_err_tl1_pending, %g4 1199 ldub [%g1 + %g4], %g2 1200 brz %g2, 1f 1201 nop 1202 1203 /* 1204 * We have a pending TL>0 error, clear the TL>0 pending status. 1205 */ 1206 stb %g0, [%g1 + %g4] 1207 1208 /* 1209 * Clear the softint. 1210 */ 1211 mov 1, %g5 1212 sll %g5, PIL_15, %g5 1213 wr %g5, CLEAR_SOFTINT 1214 1215 /* 1216 * For Cheetah*, call cpu_tl1_error via systrap at PIL 15 1217 * to process the Fast ECC/Cache Parity at TL>0 error. Clear 1218 * panic flag (%g2). 1219 */ 1220 set cpu_tl1_error, %g1 1221 clr %g2 1222 ba sys_trap 1223 mov PIL_15, %g4 1224 12251: 1226 /* 1227 * The logout is invalid. 1228 * 1229 * Call the default interrupt handler. 1230 */ 1231 sethi %hi(pil_interrupt), %g1 1232 jmp %g1 + %lo(pil_interrupt) 1233 mov PIL_15, %g4 1234 1235 SET_SIZE(ch_pil15_interrupt) 1236#endif 1237 1238 1239/* 1240 * Error Handling 1241 * 1242 * Cheetah provides error checking for all memory access paths between 1243 * the CPU, External Cache, Cheetah Data Switch and system bus. Error 1244 * information is logged in the AFSR, (also AFSR_EXT for Panther) and 1245 * AFAR and one of the following traps is generated (provided that it 1246 * is enabled in External Cache Error Enable Register) to handle that 1247 * error: 1248 * 1. trap 0x70: Precise trap 1249 * tt0_fecc for errors at trap level(TL)>=0 1250 * 2. trap 0x0A and 0x32: Deferred trap 1251 * async_err for errors at TL>=0 1252 * 3. trap 0x63: Disrupting trap 1253 * ce_err for errors at TL=0 1254 * (Note that trap 0x63 cannot happen at trap level > 0) 1255 * 1256 * Trap level one handlers panic the system except for the fast ecc 1257 * error handler which tries to recover from certain errors. 1258 */ 1259 1260/* 1261 * FAST ECC TRAP STRATEGY: 1262 * 1263 * Software must handle single and multi bit errors which occur due to data 1264 * or instruction cache reads from the external cache. A single or multi bit 1265 * error occuring in one of these situations results in a precise trap. 1266 * 1267 * The basic flow of this trap handler is as follows: 1268 * 1269 * 1) Record the state and then turn off the Dcache and Icache. The Dcache 1270 * is disabled because bad data could have been installed. The Icache is 1271 * turned off because we want to capture the Icache line related to the 1272 * AFAR. 1273 * 2) Disable trapping on CEEN/NCCEN errors during TL=0 processing. 1274 * 3) Park sibling core if caches are shared (to avoid race condition while 1275 * accessing shared resources such as L3 data staging register during 1276 * CPU logout. 1277 * 4) Read the AFAR and AFSR. 1278 * 5) If CPU logout structure is not being used, then: 1279 * 6) Clear all errors from the AFSR. 1280 * 7) Capture Ecache, Dcache and Icache lines in "CPU log out" structure. 1281 * 8) Flush Ecache then Flush Dcache and Icache and restore to previous 1282 * state. 1283 * 9) Unpark sibling core if we parked it earlier. 1284 * 10) call cpu_fast_ecc_error via systrap at PIL 14 unless we're already 1285 * running at PIL 15. 1286 * 6) Otherwise, if CPU logout structure is being used: 1287 * 7) Incriment the "logout busy count". 1288 * 8) Flush Ecache then Flush Dcache and Icache and restore to previous 1289 * state. 1290 * 9) Unpark sibling core if we parked it earlier. 1291 * 10) Issue a retry since the other CPU error logging code will end up 1292 * finding this error bit and logging information about it later. 1293 * 7) Alternatively (to 5 and 6 above), if the cpu_private struct is not 1294 * yet initialized such that we can't even check the logout struct, then 1295 * we place the clo_flags data into %g2 (sys_trap->have_win arg #1) and 1296 * call cpu_fast_ecc_error via systrap. The clo_flags parameter is used 1297 * to determine information such as TL, TT, CEEN settings, etc in the 1298 * high level trap handler since we don't have access to detailed logout 1299 * information in cases where the cpu_private struct is not yet 1300 * initialized. 1301 * 1302 * We flush the E$ and D$ here on TL=1 code to prevent getting nested 1303 * Fast ECC traps in the TL=0 code. If we get a Fast ECC event here in 1304 * the TL=1 code, we will go to the Fast ECC at TL>0 handler which, 1305 * since it is uses different code/data from this handler, has a better 1306 * chance of fixing things up than simply recursing through this code 1307 * again (this would probably cause an eventual kernel stack overflow). 1308 * If the Fast ECC at TL>0 handler encounters a Fast ECC error before it 1309 * can flush the E$ (or the error is a stuck-at bit), we will recurse in 1310 * the Fast ECC at TL>0 handler and eventually Red Mode. 1311 * 1312 * Note that for Cheetah (and only Cheetah), we use alias addresses for 1313 * flushing rather than ASI accesses (which don't exist on Cheetah). 1314 * Should we encounter a Fast ECC error within this handler on Cheetah, 1315 * there's a good chance it's within the ecache_flushaddr buffer (since 1316 * it's the largest piece of memory we touch in the handler and it is 1317 * usually kernel text/data). For that reason the Fast ECC at TL>0 1318 * handler for Cheetah uses an alternate buffer: ecache_tl1_flushaddr. 1319 */ 1320 1321/* 1322 * Cheetah ecc-protected E$ trap (Trap 70) at TL=0 1323 * tt0_fecc is replaced by fecc_err_instr in cpu_init_trap of the various 1324 * architecture-specific files. 1325 * NB: Must be 8 instructions or less to fit in trap table and code must 1326 * be relocatable. 1327 */ 1328 1329#if defined(lint) 1330 1331void 1332fecc_err_instr(void) 1333{} 1334 1335#else /* lint */ 1336 1337 ENTRY_NP(fecc_err_instr) 1338 membar #Sync ! Cheetah requires membar #Sync 1339 1340 /* 1341 * Save current DCU state. Turn off the Dcache and Icache. 1342 */ 1343 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 1344 andn %g1, DCU_DC + DCU_IC, %g4 1345 stxa %g4, [%g0]ASI_DCU 1346 flush %g0 /* flush required after changing the IC bit */ 1347 1348 ASM_JMP(%g4, fast_ecc_err) 1349 SET_SIZE(fecc_err_instr) 1350 1351#endif /* lint */ 1352 1353 1354#if !(defined(JALAPENO) || defined(SERRANO)) 1355 1356#if defined(lint) 1357 1358void 1359fast_ecc_err(void) 1360{} 1361 1362#else /* lint */ 1363 1364 .section ".text" 1365 .align 64 1366 ENTRY_NP(fast_ecc_err) 1367 1368 /* 1369 * Turn off CEEN and NCEEN. 1370 */ 1371 ldxa [%g0]ASI_ESTATE_ERR, %g3 1372 andn %g3, EN_REG_NCEEN + EN_REG_CEEN, %g4 1373 stxa %g4, [%g0]ASI_ESTATE_ERR 1374 membar #Sync ! membar sync required 1375 1376 /* 1377 * Check to see whether we need to park our sibling core 1378 * before recording diagnostic information from caches 1379 * which may be shared by both cores. 1380 * We use %g1 to store information about whether or not 1381 * we had to park the core (%g1 holds our DCUCR value and 1382 * we only use bits from that register which are "reserved" 1383 * to keep track of core parking) so that we know whether 1384 * or not to unpark later. %g5 and %g4 are scratch registers. 1385 */ 1386 PARK_SIBLING_CORE(%g1, %g5, %g4) 1387 1388 /* 1389 * Do the CPU log out capture. 1390 * %g3 = "failed?" return value. 1391 * %g2 = Input = AFAR. Output the clo_flags info which is passed 1392 * into this macro via %g4. Output only valid if cpu_private 1393 * struct has not been initialized. 1394 * CHPR_FECCTL0_LOGOUT = cpu logout structure offset input 1395 * %g4 = Trap information stored in the cpu logout flags field 1396 * %g5 = scr1 1397 * %g6 = scr2 1398 * %g3 = scr3 1399 * %g4 = scr4 1400 */ 1401 and %g3, EN_REG_CEEN, %g4 ! store the CEEN value, TL=0 1402 set CHPR_FECCTL0_LOGOUT, %g6 1403 DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4) 1404 1405 /* 1406 * Flush the Ecache (and L2 cache for Panther) to get the error out 1407 * of the Ecache. If the UCC or UCU is on a dirty line, then the 1408 * following flush will turn that into a WDC or WDU, respectively. 1409 */ 1410 PN_L2_FLUSHALL(%g4, %g5, %g6) 1411 1412 CPU_INDEX(%g4, %g5) 1413 mulx %g4, CPU_NODE_SIZE, %g4 1414 set cpunodes, %g5 1415 add %g4, %g5, %g4 1416 ld [%g4 + ECACHE_LINESIZE], %g5 1417 ld [%g4 + ECACHE_SIZE], %g4 1418 1419 ASM_LDX(%g6, ecache_flushaddr) 1420 ECACHE_FLUSHALL(%g4, %g5, %g6, %g7) 1421 1422 /* 1423 * Flush the Dcache. Since bad data could have been installed in 1424 * the Dcache we must flush it before re-enabling it. 1425 */ 1426 ASM_LD(%g5, dcache_size) 1427 ASM_LD(%g6, dcache_linesize) 1428 CH_DCACHE_FLUSHALL(%g5, %g6, %g7) 1429 1430 /* 1431 * Flush the Icache. Since we turned off the Icache to capture the 1432 * Icache line it is now stale or corrupted and we must flush it 1433 * before re-enabling it. 1434 */ 1435 GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_5); 1436 ld [%g5 + CHPR_ICACHE_LINESIZE], %g6 1437 ba,pt %icc, 6f 1438 ld [%g5 + CHPR_ICACHE_SIZE], %g5 1439fast_ecc_err_5: 1440 ASM_LD(%g5, icache_size) 1441 ASM_LD(%g6, icache_linesize) 14426: 1443 CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4) 1444 1445 /* 1446 * check to see whether we parked our sibling core at the start 1447 * of this handler. If so, we need to unpark it here. 1448 * We use DCUCR reserved bits (stored in %g1) to keep track of 1449 * whether or not we need to unpark. %g5 and %g4 are scratch registers. 1450 */ 1451 UNPARK_SIBLING_CORE(%g1, %g5, %g4) 1452 1453 /* 1454 * Restore the Dcache and Icache to the previous state. 1455 */ 1456 stxa %g1, [%g0]ASI_DCU 1457 flush %g0 /* flush required after changing the IC bit */ 1458 1459 /* 1460 * Make sure our CPU logout operation was successful. 1461 */ 1462 cmp %g3, %g0 1463 be 8f 1464 nop 1465 1466 /* 1467 * If the logout structure had been busy, how many times have 1468 * we tried to use it and failed (nesting count)? If we have 1469 * already recursed a substantial number of times, then we can 1470 * assume things are not going to get better by themselves and 1471 * so it would be best to panic. 1472 */ 1473 cmp %g3, CLO_NESTING_MAX 1474 blt 7f 1475 nop 1476 1477 call ptl1_panic 1478 mov PTL1_BAD_ECC, %g1 1479 14807: 1481 /* 1482 * Otherwise, if the logout structure was busy but we have not 1483 * nested more times than our maximum value, then we simply 1484 * issue a retry. Our TL=0 trap handler code will check and 1485 * clear the AFSR after it is done logging what is currently 1486 * in the logout struct and handle this event at that time. 1487 */ 1488 retry 14898: 1490 /* 1491 * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're 1492 * already at PIL 15. 1493 */ 1494 set cpu_fast_ecc_error, %g1 1495 rdpr %pil, %g4 1496 cmp %g4, PIL_14 1497 ba sys_trap 1498 movl %icc, PIL_14, %g4 1499 1500 SET_SIZE(fast_ecc_err) 1501 1502#endif /* lint */ 1503 1504#endif /* !(JALAPENO || SERRANO) */ 1505 1506 1507/* 1508 * Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy: 1509 * 1510 * The basic flow of this trap handler is as follows: 1511 * 1512 * 1) In the "trap 70" trap table code (fecc_err_tl1_instr), generate a 1513 * software trap 0 ("ta 0") to buy an extra set of %tpc, etc. which we 1514 * will use to save %g1 and %g2. 1515 * 2) At the software trap 0 at TL>0 trap table code (fecc_err_tl1_cont_instr), 1516 * we save %g1+%g2 using %tpc, %tnpc + %tstate and jump to the fast ecc 1517 * handler (using the just saved %g1). 1518 * 3) Turn off the Dcache if it was on and save the state of the Dcache 1519 * (whether on or off) in Bit2 (CH_ERR_TSTATE_DC_ON) of %tstate. 1520 * NB: we don't turn off the Icache because bad data is not installed nor 1521 * will we be doing any diagnostic accesses. 1522 * 4) compute physical address of the per-cpu/per-tl save area using %g1+%g2 1523 * 5) Save %g1-%g7 into the per-cpu/per-tl save area (%g1 + %g2 from the 1524 * %tpc, %tnpc, %tstate values previously saved). 1525 * 6) set %tl to %tl - 1. 1526 * 7) Save the appropriate flags and TPC in the ch_err_tl1_data structure. 1527 * 8) Save the value of CH_ERR_TSTATE_DC_ON in the ch_err_tl1_tmp field. 1528 * 9) For Cheetah and Jalapeno, read the AFAR and AFSR and clear. For 1529 * Cheetah+ (and later), read the shadow AFAR and AFSR but don't clear. 1530 * Save the values in ch_err_tl1_data. For Panther, read the shadow 1531 * AFSR_EXT and save the value in ch_err_tl1_data. 1532 * 10) Disable CEEN/NCEEN to prevent any disrupting/deferred errors from 1533 * being queued. We'll report them via the AFSR/AFAR capture in step 13. 1534 * 11) Flush the Ecache. 1535 * NB: the Ecache is flushed assuming the largest possible size with 1536 * the smallest possible line size since access to the cpu_nodes may 1537 * cause an unrecoverable DTLB miss. 1538 * 12) Reenable CEEN/NCEEN with the value saved from step 10. 1539 * 13) For Cheetah and Jalapeno, read the AFAR and AFSR and clear again. 1540 * For Cheetah+ (and later), read the primary AFAR and AFSR and now clear. 1541 * Save the read AFSR/AFAR values in ch_err_tl1_data. For Panther, 1542 * read and clear the primary AFSR_EXT and save it in ch_err_tl1_data. 1543 * 14) Flush and re-enable the Dcache if it was on at step 3. 1544 * 15) Do TRAPTRACE if enabled. 1545 * 16) Check if a UCU->WDU (or L3_UCU->WDU for Panther) happened, panic if so. 1546 * 17) Set the event pending flag in ch_err_tl1_pending[CPU] 1547 * 18) Cause a softint 15. The pil15_interrupt handler will inspect the 1548 * event pending flag and call cpu_tl1_error via systrap if set. 1549 * 19) Restore the registers from step 5 and issue retry. 1550 */ 1551 1552/* 1553 * Cheetah ecc-protected E$ trap (Trap 70) at TL>0 1554 * tt1_fecc is replaced by fecc_err_tl1_instr in cpu_init_trap of the various 1555 * architecture-specific files. This generates a "Software Trap 0" at TL>0, 1556 * which goes to fecc_err_tl1_cont_instr, and we continue the handling there. 1557 * NB: Must be 8 instructions or less to fit in trap table and code must 1558 * be relocatable. 1559 */ 1560 1561#if defined(lint) 1562 1563void 1564fecc_err_tl1_instr(void) 1565{} 1566 1567#else /* lint */ 1568 1569 ENTRY_NP(fecc_err_tl1_instr) 1570 CH_ERR_TL1_TRAPENTRY(SWTRAP_0); 1571 SET_SIZE(fecc_err_tl1_instr) 1572 1573#endif /* lint */ 1574 1575/* 1576 * Software trap 0 at TL>0. 1577 * tt1_swtrap0 is replaced by fecc_err_tl1_cont_instr in cpu_init_trap of 1578 * the various architecture-specific files. This is used as a continuation 1579 * of the fast ecc handling where we've bought an extra TL level, so we can 1580 * use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1 1581 * and %g2. Note that %tstate has bits 0-2 and then bits 8-19 as r/w, 1582 * there's a reserved hole from 3-7. We only use bits 0-1 and 8-9 (the low 1583 * order two bits from %g1 and %g2 respectively). 1584 * NB: Must be 8 instructions or less to fit in trap table and code must 1585 * be relocatable. 1586 */ 1587#if defined(lint) 1588 1589void 1590fecc_err_tl1_cont_instr(void) 1591{} 1592 1593#else /* lint */ 1594 1595 ENTRY_NP(fecc_err_tl1_cont_instr) 1596 CH_ERR_TL1_SWTRAPENTRY(fast_ecc_tl1_err) 1597 SET_SIZE(fecc_err_tl1_cont_instr) 1598 1599#endif /* lint */ 1600 1601 1602#if defined(lint) 1603 1604void 1605ce_err(void) 1606{} 1607 1608#else /* lint */ 1609 1610/* 1611 * The ce_err function handles disrupting trap type 0x63 at TL=0. 1612 * 1613 * AFSR errors bits which cause this trap are: 1614 * CE, EMC, EDU:ST, EDC, WDU, WDC, CPU, CPC, IVU, IVC 1615 * 1616 * NCEEN Bit of Cheetah External Cache Error Enable Register enables 1617 * the following AFSR disrupting traps: EDU:ST, WDU, CPU, IVU 1618 * 1619 * CEEN Bit of Cheetah External Cache Error Enable Register enables 1620 * the following AFSR disrupting traps: CE, EMC, EDC, WDC, CPC, IVC 1621 * 1622 * Cheetah+ also handles (No additional processing required): 1623 * DUE, DTO, DBERR (NCEEN controlled) 1624 * THCE (CEEN and ET_ECC_en controlled) 1625 * TUE (ET_ECC_en controlled) 1626 * 1627 * Panther further adds: 1628 * IMU, L3_EDU, L3_WDU, L3_CPU (NCEEN controlled) 1629 * IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE (CEEN controlled) 1630 * TUE_SH, TUE (NCEEN and L2_tag_ECC_en controlled) 1631 * L3_TUE, L3_TUE_SH (NCEEN and ET_ECC_en controlled) 1632 * THCE (CEEN and L2_tag_ECC_en controlled) 1633 * L3_THCE (CEEN and ET_ECC_en controlled) 1634 * 1635 * Steps: 1636 * 1. Disable hardware corrected disrupting errors only (CEEN) 1637 * 2. Park sibling core if caches are shared (to avoid race 1638 * condition while accessing shared resources such as L3 1639 * data staging register during CPU logout. 1640 * 3. If the CPU logout structure is not currently being used: 1641 * 4. Clear AFSR error bits 1642 * 5. Capture Ecache, Dcache and Icache lines associated 1643 * with AFAR. 1644 * 6. Unpark sibling core if we parked it earlier. 1645 * 7. call cpu_disrupting_error via sys_trap at PIL 14 1646 * unless we're already running at PIL 15. 1647 * 4. Otherwise, if the CPU logout structure is busy: 1648 * 5. Incriment "logout busy count" and place into %g3 1649 * 6. Unpark sibling core if we parked it earlier. 1650 * 7. Issue a retry since the other CPU error logging 1651 * code will end up finding this error bit and logging 1652 * information about it later. 1653 * 5. Alternatively (to 3 and 4 above), if the cpu_private struct is 1654 * not yet initialized such that we can't even check the logout 1655 * struct, then we place the clo_flags data into %g2 1656 * (sys_trap->have_win arg #1) and call cpu_disrupting_error via 1657 * systrap. The clo_flags parameter is used to determine information 1658 * such as TL, TT, CEEN settings, etc in the high level trap 1659 * handler since we don't have access to detailed logout information 1660 * in cases where the cpu_private struct is not yet initialized. 1661 * 1662 * %g3: [ logout busy count ] - arg #2 1663 * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1 1664 */ 1665 1666 .align 128 1667 ENTRY_NP(ce_err) 1668 membar #Sync ! Cheetah requires membar #Sync 1669 1670 /* 1671 * Disable trap on hardware corrected errors (CEEN) while at TL=0 1672 * to prevent recursion. 1673 */ 1674 ldxa [%g0]ASI_ESTATE_ERR, %g1 1675 bclr EN_REG_CEEN, %g1 1676 stxa %g1, [%g0]ASI_ESTATE_ERR 1677 membar #Sync ! membar sync required 1678 1679 /* 1680 * Save current DCU state. Turn off Icache to allow capture of 1681 * Icache data by DO_CPU_LOGOUT. 1682 */ 1683 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 1684 andn %g1, DCU_IC, %g4 1685 stxa %g4, [%g0]ASI_DCU 1686 flush %g0 /* flush required after changing the IC bit */ 1687 1688 /* 1689 * Check to see whether we need to park our sibling core 1690 * before recording diagnostic information from caches 1691 * which may be shared by both cores. 1692 * We use %g1 to store information about whether or not 1693 * we had to park the core (%g1 holds our DCUCR value and 1694 * we only use bits from that register which are "reserved" 1695 * to keep track of core parking) so that we know whether 1696 * or not to unpark later. %g5 and %g4 are scratch registers. 1697 */ 1698 PARK_SIBLING_CORE(%g1, %g5, %g4) 1699 1700 /* 1701 * Do the CPU log out capture. 1702 * %g3 = "failed?" return value. 1703 * %g2 = Input = AFAR. Output the clo_flags info which is passed 1704 * into this macro via %g4. Output only valid if cpu_private 1705 * struct has not been initialized. 1706 * CHPR_CECC_LOGOUT = cpu logout structure offset input 1707 * %g4 = Trap information stored in the cpu logout flags field 1708 * %g5 = scr1 1709 * %g6 = scr2 1710 * %g3 = scr3 1711 * %g4 = scr4 1712 */ 1713 clr %g4 ! TL=0 bit in afsr 1714 set CHPR_CECC_LOGOUT, %g6 1715 DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4) 1716 1717 /* 1718 * Flush the Icache. Since we turned off the Icache to capture the 1719 * Icache line it is now stale or corrupted and we must flush it 1720 * before re-enabling it. 1721 */ 1722 GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, ce_err_1); 1723 ld [%g5 + CHPR_ICACHE_LINESIZE], %g6 1724 ba,pt %icc, 2f 1725 ld [%g5 + CHPR_ICACHE_SIZE], %g5 1726ce_err_1: 1727 ASM_LD(%g5, icache_size) 1728 ASM_LD(%g6, icache_linesize) 17292: 1730 CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4) 1731 1732 /* 1733 * check to see whether we parked our sibling core at the start 1734 * of this handler. If so, we need to unpark it here. 1735 * We use DCUCR reserved bits (stored in %g1) to keep track of 1736 * whether or not we need to unpark. %g5 and %g4 are scratch registers. 1737 */ 1738 UNPARK_SIBLING_CORE(%g1, %g5, %g4) 1739 1740 /* 1741 * Restore Icache to previous state. 1742 */ 1743 stxa %g1, [%g0]ASI_DCU 1744 flush %g0 /* flush required after changing the IC bit */ 1745 1746 /* 1747 * Make sure our CPU logout operation was successful. 1748 */ 1749 cmp %g3, %g0 1750 be 4f 1751 nop 1752 1753 /* 1754 * If the logout structure had been busy, how many times have 1755 * we tried to use it and failed (nesting count)? If we have 1756 * already recursed a substantial number of times, then we can 1757 * assume things are not going to get better by themselves and 1758 * so it would be best to panic. 1759 */ 1760 cmp %g3, CLO_NESTING_MAX 1761 blt 3f 1762 nop 1763 1764 call ptl1_panic 1765 mov PTL1_BAD_ECC, %g1 1766 17673: 1768 /* 1769 * Otherwise, if the logout structure was busy but we have not 1770 * nested more times than our maximum value, then we simply 1771 * issue a retry. Our TL=0 trap handler code will check and 1772 * clear the AFSR after it is done logging what is currently 1773 * in the logout struct and handle this event at that time. 1774 */ 1775 retry 17764: 1777 /* 1778 * Call cpu_disrupting_error via systrap at PIL 14 unless we're 1779 * already at PIL 15. 1780 */ 1781 set cpu_disrupting_error, %g1 1782 rdpr %pil, %g4 1783 cmp %g4, PIL_14 1784 ba sys_trap 1785 movl %icc, PIL_14, %g4 1786 SET_SIZE(ce_err) 1787 1788#endif /* lint */ 1789 1790 1791#if defined(lint) 1792 1793/* 1794 * This trap cannot happen at TL>0 which means this routine will never 1795 * actually be called and so we treat this like a BAD TRAP panic. 1796 */ 1797void 1798ce_err_tl1(void) 1799{} 1800 1801#else /* lint */ 1802 1803 .align 64 1804 ENTRY_NP(ce_err_tl1) 1805 1806 call ptl1_panic 1807 mov PTL1_BAD_TRAP, %g1 1808 1809 SET_SIZE(ce_err_tl1) 1810 1811#endif /* lint */ 1812 1813 1814#if defined(lint) 1815 1816void 1817async_err(void) 1818{} 1819 1820#else /* lint */ 1821 1822/* 1823 * The async_err function handles deferred trap types 0xA 1824 * (instruction_access_error) and 0x32 (data_access_error) at TL>=0. 1825 * 1826 * AFSR errors bits which cause this trap are: 1827 * UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR 1828 * On some platforms, EMU may causes cheetah to pull the error pin 1829 * never giving Solaris a chance to take a trap. 1830 * 1831 * NCEEN Bit of Cheetah External Cache Error Enable Register enables 1832 * the following AFSR deferred traps: UE, EMU, EDU:BLD, TO, BERR 1833 * 1834 * Steps: 1835 * 1. Disable CEEN and NCEEN errors to prevent recursive errors. 1836 * 2. Turn D$ off per Cheetah PRM P.5 Note 6, turn I$ off to capture 1837 * I$ line in DO_CPU_LOGOUT. 1838 * 3. Park sibling core if caches are shared (to avoid race 1839 * condition while accessing shared resources such as L3 1840 * data staging register during CPU logout. 1841 * 4. If the CPU logout structure is not currently being used: 1842 * 5. Clear AFSR error bits 1843 * 6. Capture Ecache, Dcache and Icache lines associated 1844 * with AFAR. 1845 * 7. Unpark sibling core if we parked it earlier. 1846 * 8. call cpu_deferred_error via sys_trap. 1847 * 5. Otherwise, if the CPU logout structure is busy: 1848 * 6. Incriment "logout busy count" 1849 * 7. Unpark sibling core if we parked it earlier. 1850 * 8) Issue a retry since the other CPU error logging 1851 * code will end up finding this error bit and logging 1852 * information about it later. 1853 * 6. Alternatively (to 4 and 5 above), if the cpu_private struct is 1854 * not yet initialized such that we can't even check the logout 1855 * struct, then we place the clo_flags data into %g2 1856 * (sys_trap->have_win arg #1) and call cpu_deferred_error via 1857 * systrap. The clo_flags parameter is used to determine information 1858 * such as TL, TT, CEEN settings, etc in the high level trap handler 1859 * since we don't have access to detailed logout information in cases 1860 * where the cpu_private struct is not yet initialized. 1861 * 1862 * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1 1863 * %g3: [ logout busy count ] - arg #2 1864 */ 1865 1866 ENTRY_NP(async_err) 1867 membar #Sync ! Cheetah requires membar #Sync 1868 1869 /* 1870 * Disable CEEN and NCEEN. 1871 */ 1872 ldxa [%g0]ASI_ESTATE_ERR, %g3 1873 andn %g3, EN_REG_NCEEN + EN_REG_CEEN, %g4 1874 stxa %g4, [%g0]ASI_ESTATE_ERR 1875 membar #Sync ! membar sync required 1876 1877 /* 1878 * Save current DCU state. 1879 * Disable Icache to allow capture of Icache data by DO_CPU_LOGOUT. 1880 * Do this regardless of whether this is a Data Access Error or 1881 * Instruction Access Error Trap. 1882 * Disable Dcache for both Data Access Error and Instruction Access 1883 * Error per Cheetah PRM P.5 Note 6. 1884 */ 1885 ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 1886 andn %g1, DCU_IC + DCU_DC, %g4 1887 stxa %g4, [%g0]ASI_DCU 1888 flush %g0 /* flush required after changing the IC bit */ 1889 1890 /* 1891 * Check to see whether we need to park our sibling core 1892 * before recording diagnostic information from caches 1893 * which may be shared by both cores. 1894 * We use %g1 to store information about whether or not 1895 * we had to park the core (%g1 holds our DCUCR value and 1896 * we only use bits from that register which are "reserved" 1897 * to keep track of core parking) so that we know whether 1898 * or not to unpark later. %g6 and %g4 are scratch registers. 1899 */ 1900 PARK_SIBLING_CORE(%g1, %g6, %g4) 1901 1902 /* 1903 * Do the CPU logout capture. 1904 * 1905 * %g3 = "failed?" return value. 1906 * %g2 = Input = AFAR. Output the clo_flags info which is passed 1907 * into this macro via %g4. Output only valid if cpu_private 1908 * struct has not been initialized. 1909 * CHPR_ASYNC_LOGOUT = cpu logout structure offset input 1910 * %g4 = Trap information stored in the cpu logout flags field 1911 * %g5 = scr1 1912 * %g6 = scr2 1913 * %g3 = scr3 1914 * %g4 = scr4 1915 */ 1916 andcc %g5, T_TL1, %g0 1917 clr %g6 1918 movnz %xcc, 1, %g6 ! set %g6 if T_TL1 set 1919 sllx %g6, CLO_FLAGS_TL_SHIFT, %g6 1920 sllx %g5, CLO_FLAGS_TT_SHIFT, %g4 1921 set CLO_FLAGS_TT_MASK, %g2 1922 and %g4, %g2, %g4 ! ttype 1923 or %g6, %g4, %g4 ! TT and TL 1924 and %g3, EN_REG_CEEN, %g3 ! CEEN value 1925 or %g3, %g4, %g4 ! TT and TL and CEEN 1926 set CHPR_ASYNC_LOGOUT, %g6 1927 DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4) 1928 1929 /* 1930 * If the logout struct was busy, we may need to pass the 1931 * TT, TL, and CEEN information to the TL=0 handler via 1932 * systrap parameter so save it off here. 1933 */ 1934 cmp %g3, %g0 1935 be 1f 1936 nop 1937 sllx %g4, 32, %g4 1938 or %g4, %g3, %g3 19391: 1940 /* 1941 * Flush the Icache. Since we turned off the Icache to capture the 1942 * Icache line it is now stale or corrupted and we must flush it 1943 * before re-enabling it. 1944 */ 1945 GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, async_err_1); 1946 ld [%g5 + CHPR_ICACHE_LINESIZE], %g6 1947 ba,pt %icc, 2f 1948 ld [%g5 + CHPR_ICACHE_SIZE], %g5 1949async_err_1: 1950 ASM_LD(%g5, icache_size) 1951 ASM_LD(%g6, icache_linesize) 19522: 1953 CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4) 1954 1955 /* 1956 * XXX - Don't we need to flush the Dcache before turning it back 1957 * on to avoid stale or corrupt data? Was this broken? 1958 */ 1959 /* 1960 * Flush the Dcache before turning it back on since it may now 1961 * contain stale or corrupt data. 1962 */ 1963 ASM_LD(%g5, dcache_size) 1964 ASM_LD(%g6, dcache_linesize) 1965 CH_DCACHE_FLUSHALL(%g5, %g6, %g7) 1966 1967 /* 1968 * check to see whether we parked our sibling core at the start 1969 * of this handler. If so, we need to unpark it here. 1970 * We use DCUCR reserved bits (stored in %g1) to keep track of 1971 * whether or not we need to unpark. %g5 and %g7 are scratch registers. 1972 */ 1973 UNPARK_SIBLING_CORE(%g1, %g5, %g7) 1974 1975 /* 1976 * Restore Icache and Dcache to previous state. 1977 */ 1978 stxa %g1, [%g0]ASI_DCU 1979 flush %g0 /* flush required after changing the IC bit */ 1980 1981 /* 1982 * Make sure our CPU logout operation was successful. 1983 */ 1984 cmp %g3, %g0 1985 be 4f 1986 nop 1987 1988 /* 1989 * If the logout structure had been busy, how many times have 1990 * we tried to use it and failed (nesting count)? If we have 1991 * already recursed a substantial number of times, then we can 1992 * assume things are not going to get better by themselves and 1993 * so it would be best to panic. 1994 */ 1995 cmp %g3, CLO_NESTING_MAX 1996 blt 3f 1997 nop 1998 1999 call ptl1_panic 2000 mov PTL1_BAD_ECC, %g1 2001 20023: 2003 /* 2004 * Otherwise, if the logout structure was busy but we have not 2005 * nested more times than our maximum value, then we simply 2006 * issue a retry. Our TL=0 trap handler code will check and 2007 * clear the AFSR after it is done logging what is currently 2008 * in the logout struct and handle this event at that time. 2009 */ 2010 retry 20114: 2012 set cpu_deferred_error, %g1 2013 ba sys_trap 2014 mov PIL_15, %g4 ! run at pil 15 2015 SET_SIZE(async_err) 2016 2017#endif /* lint */ 2018 2019#if defined(CPU_IMP_L1_CACHE_PARITY) 2020 2021/* 2022 * D$ parity error trap (trap 71) at TL=0. 2023 * tt0_dperr is replaced by dcache_parity_instr in cpu_init_trap of 2024 * the various architecture-specific files. This merely sets up the 2025 * arguments for cpu_parity_error and calls it via sys_trap. 2026 * NB: Must be 8 instructions or less to fit in trap table and code must 2027 * be relocatable. 2028 */ 2029#if defined(lint) 2030 2031void 2032dcache_parity_instr(void) 2033{} 2034 2035#else /* lint */ 2036 ENTRY_NP(dcache_parity_instr) 2037 membar #Sync ! Cheetah+ requires membar #Sync 2038 set cpu_parity_error, %g1 2039 or %g0, CH_ERR_DPE, %g2 2040 rdpr %tpc, %g3 2041 sethi %hi(sys_trap), %g7 2042 jmp %g7 + %lo(sys_trap) 2043 mov PIL_15, %g4 ! run at pil 15 2044 SET_SIZE(dcache_parity_instr) 2045 2046#endif /* lint */ 2047 2048 2049/* 2050 * D$ parity error trap (trap 71) at TL>0. 2051 * tt1_dperr is replaced by dcache_parity_tl1_instr in cpu_init_trap of 2052 * the various architecture-specific files. This generates a "Software 2053 * Trap 1" at TL>0, which goes to dcache_parity_tl1_cont_instr, and we 2054 * continue the handling there. 2055 * NB: Must be 8 instructions or less to fit in trap table and code must 2056 * be relocatable. 2057 */ 2058#if defined(lint) 2059 2060void 2061dcache_parity_tl1_instr(void) 2062{} 2063 2064#else /* lint */ 2065 ENTRY_NP(dcache_parity_tl1_instr) 2066 CH_ERR_TL1_TRAPENTRY(SWTRAP_1); 2067 SET_SIZE(dcache_parity_tl1_instr) 2068 2069#endif /* lint */ 2070 2071 2072/* 2073 * Software trap 1 at TL>0. 2074 * tt1_swtrap1 is replaced by dcache_parity_tl1_cont_instr in cpu_init_trap 2075 * of the various architecture-specific files. This is used as a continuation 2076 * of the dcache parity handling where we've bought an extra TL level, so we 2077 * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1 2078 * and %g2. Note that %tstate has bits 0-2 and then bits 8-19 as r/w, 2079 * there's a reserved hole from 3-7. We only use bits 0-1 and 8-9 (the low 2080 * order two bits from %g1 and %g2 respectively). 2081 * NB: Must be 8 instructions or less to fit in trap table and code must 2082 * be relocatable. 2083 */ 2084#if defined(lint) 2085 2086void 2087dcache_parity_tl1_cont_instr(void) 2088{} 2089 2090#else /* lint */ 2091 ENTRY_NP(dcache_parity_tl1_cont_instr) 2092 CH_ERR_TL1_SWTRAPENTRY(dcache_parity_tl1_err); 2093 SET_SIZE(dcache_parity_tl1_cont_instr) 2094 2095#endif /* lint */ 2096 2097/* 2098 * D$ parity error at TL>0 handler 2099 * We get here via trap 71 at TL>0->Software trap 1 at TL>0. We enter 2100 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. 2101 */ 2102#if defined(lint) 2103 2104void 2105dcache_parity_tl1_err(void) 2106{} 2107 2108#else /* lint */ 2109 2110 ENTRY_NP(dcache_parity_tl1_err) 2111 2112 /* 2113 * This macro saves all the %g registers in the ch_err_tl1_data 2114 * structure, updates the ch_err_tl1_flags and saves the %tpc in 2115 * ch_err_tl1_tpc. At the end of this macro, %g1 will point to 2116 * the ch_err_tl1_data structure and %g2 will have the original 2117 * flags in the ch_err_tl1_data structure. All %g registers 2118 * except for %g1 and %g2 will be available. 2119 */ 2120 CH_ERR_TL1_ENTER(CH_ERR_DPE); 2121 2122#ifdef TRAPTRACE 2123 /* 2124 * Get current trap trace entry physical pointer. 2125 */ 2126 CPU_INDEX(%g6, %g5) 2127 sll %g6, TRAPTR_SIZE_SHIFT, %g6 2128 set trap_trace_ctl, %g5 2129 add %g6, %g5, %g6 2130 ld [%g6 + TRAPTR_LIMIT], %g5 2131 tst %g5 2132 be %icc, dpe_tl1_skip_tt 2133 nop 2134 ldx [%g6 + TRAPTR_PBASE], %g5 2135 ld [%g6 + TRAPTR_OFFSET], %g4 2136 add %g5, %g4, %g5 2137 2138 /* 2139 * Create trap trace entry. 2140 */ 2141 rd %asi, %g7 2142 wr %g0, TRAPTR_ASI, %asi 2143 rd STICK, %g4 2144 stxa %g4, [%g5 + TRAP_ENT_TICK]%asi 2145 rdpr %tl, %g4 2146 stha %g4, [%g5 + TRAP_ENT_TL]%asi 2147 rdpr %tt, %g4 2148 stha %g4, [%g5 + TRAP_ENT_TT]%asi 2149 rdpr %tpc, %g4 2150 stna %g4, [%g5 + TRAP_ENT_TPC]%asi 2151 rdpr %tstate, %g4 2152 stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi 2153 stna %sp, [%g5 + TRAP_ENT_SP]%asi 2154 stna %g0, [%g5 + TRAP_ENT_TR]%asi 2155 stna %g0, [%g5 + TRAP_ENT_F1]%asi 2156 stna %g0, [%g5 + TRAP_ENT_F2]%asi 2157 stna %g0, [%g5 + TRAP_ENT_F3]%asi 2158 stna %g0, [%g5 + TRAP_ENT_F4]%asi 2159 wr %g0, %g7, %asi 2160 2161 /* 2162 * Advance trap trace pointer. 2163 */ 2164 ld [%g6 + TRAPTR_OFFSET], %g5 2165 ld [%g6 + TRAPTR_LIMIT], %g4 2166 st %g5, [%g6 + TRAPTR_LAST_OFFSET] 2167 add %g5, TRAP_ENT_SIZE, %g5 2168 sub %g4, TRAP_ENT_SIZE, %g4 2169 cmp %g5, %g4 2170 movge %icc, 0, %g5 2171 st %g5, [%g6 + TRAPTR_OFFSET] 2172dpe_tl1_skip_tt: 2173#endif /* TRAPTRACE */ 2174 2175 /* 2176 * I$ and D$ are automatically turned off by HW when the CPU hits 2177 * a dcache or icache parity error so we will just leave those two 2178 * off for now to avoid repeating this trap. 2179 * For Panther, however, since we trap on P$ data parity errors 2180 * and HW does not automatically disable P$, we need to disable it 2181 * here so that we don't encounter any recursive traps when we 2182 * issue the retry. 2183 */ 2184 ldxa [%g0]ASI_DCU, %g3 2185 mov 1, %g4 2186 sllx %g4, DCU_PE_SHIFT, %g4 2187 andn %g3, %g4, %g3 2188 stxa %g3, [%g0]ASI_DCU 2189 membar #Sync 2190 2191 /* 2192 * We fall into this macro if we've successfully logged the error in 2193 * the ch_err_tl1_data structure and want the PIL15 softint to pick 2194 * it up and log it. %g1 must point to the ch_err_tl1_data structure. 2195 * Restores the %g registers and issues retry. 2196 */ 2197 CH_ERR_TL1_EXIT; 2198 SET_SIZE(dcache_parity_tl1_err) 2199 2200#endif /* lint */ 2201 2202/* 2203 * I$ parity error trap (trap 72) at TL=0. 2204 * tt0_iperr is replaced by icache_parity_instr in cpu_init_trap of 2205 * the various architecture-specific files. This merely sets up the 2206 * arguments for cpu_parity_error and calls it via sys_trap. 2207 * NB: Must be 8 instructions or less to fit in trap table and code must 2208 * be relocatable. 2209 */ 2210#if defined(lint) 2211 2212void 2213icache_parity_instr(void) 2214{} 2215 2216#else /* lint */ 2217 2218 ENTRY_NP(icache_parity_instr) 2219 membar #Sync ! Cheetah+ requires membar #Sync 2220 set cpu_parity_error, %g1 2221 or %g0, CH_ERR_IPE, %g2 2222 rdpr %tpc, %g3 2223 sethi %hi(sys_trap), %g7 2224 jmp %g7 + %lo(sys_trap) 2225 mov PIL_15, %g4 ! run at pil 15 2226 SET_SIZE(icache_parity_instr) 2227 2228#endif /* lint */ 2229 2230/* 2231 * I$ parity error trap (trap 72) at TL>0. 2232 * tt1_iperr is replaced by icache_parity_tl1_instr in cpu_init_trap of 2233 * the various architecture-specific files. This generates a "Software 2234 * Trap 2" at TL>0, which goes to icache_parity_tl1_cont_instr, and we 2235 * continue the handling there. 2236 * NB: Must be 8 instructions or less to fit in trap table and code must 2237 * be relocatable. 2238 */ 2239#if defined(lint) 2240 2241void 2242icache_parity_tl1_instr(void) 2243{} 2244 2245#else /* lint */ 2246 ENTRY_NP(icache_parity_tl1_instr) 2247 CH_ERR_TL1_TRAPENTRY(SWTRAP_2); 2248 SET_SIZE(icache_parity_tl1_instr) 2249 2250#endif /* lint */ 2251 2252/* 2253 * Software trap 2 at TL>0. 2254 * tt1_swtrap2 is replaced by icache_parity_tl1_cont_instr in cpu_init_trap 2255 * of the various architecture-specific files. This is used as a continuation 2256 * of the icache parity handling where we've bought an extra TL level, so we 2257 * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1 2258 * and %g2. Note that %tstate has bits 0-2 and then bits 8-19 as r/w, 2259 * there's a reserved hole from 3-7. We only use bits 0-1 and 8-9 (the low 2260 * order two bits from %g1 and %g2 respectively). 2261 * NB: Must be 8 instructions or less to fit in trap table and code must 2262 * be relocatable. 2263 */ 2264#if defined(lint) 2265 2266void 2267icache_parity_tl1_cont_instr(void) 2268{} 2269 2270#else /* lint */ 2271 ENTRY_NP(icache_parity_tl1_cont_instr) 2272 CH_ERR_TL1_SWTRAPENTRY(icache_parity_tl1_err); 2273 SET_SIZE(icache_parity_tl1_cont_instr) 2274 2275#endif /* lint */ 2276 2277 2278/* 2279 * I$ parity error at TL>0 handler 2280 * We get here via trap 72 at TL>0->Software trap 2 at TL>0. We enter 2281 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. 2282 */ 2283#if defined(lint) 2284 2285void 2286icache_parity_tl1_err(void) 2287{} 2288 2289#else /* lint */ 2290 2291 ENTRY_NP(icache_parity_tl1_err) 2292 2293 /* 2294 * This macro saves all the %g registers in the ch_err_tl1_data 2295 * structure, updates the ch_err_tl1_flags and saves the %tpc in 2296 * ch_err_tl1_tpc. At the end of this macro, %g1 will point to 2297 * the ch_err_tl1_data structure and %g2 will have the original 2298 * flags in the ch_err_tl1_data structure. All %g registers 2299 * except for %g1 and %g2 will be available. 2300 */ 2301 CH_ERR_TL1_ENTER(CH_ERR_IPE); 2302 2303#ifdef TRAPTRACE 2304 /* 2305 * Get current trap trace entry physical pointer. 2306 */ 2307 CPU_INDEX(%g6, %g5) 2308 sll %g6, TRAPTR_SIZE_SHIFT, %g6 2309 set trap_trace_ctl, %g5 2310 add %g6, %g5, %g6 2311 ld [%g6 + TRAPTR_LIMIT], %g5 2312 tst %g5 2313 be %icc, ipe_tl1_skip_tt 2314 nop 2315 ldx [%g6 + TRAPTR_PBASE], %g5 2316 ld [%g6 + TRAPTR_OFFSET], %g4 2317 add %g5, %g4, %g5 2318 2319 /* 2320 * Create trap trace entry. 2321 */ 2322 rd %asi, %g7 2323 wr %g0, TRAPTR_ASI, %asi 2324 rd STICK, %g4 2325 stxa %g4, [%g5 + TRAP_ENT_TICK]%asi 2326 rdpr %tl, %g4 2327 stha %g4, [%g5 + TRAP_ENT_TL]%asi 2328 rdpr %tt, %g4 2329 stha %g4, [%g5 + TRAP_ENT_TT]%asi 2330 rdpr %tpc, %g4 2331 stna %g4, [%g5 + TRAP_ENT_TPC]%asi 2332 rdpr %tstate, %g4 2333 stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi 2334 stna %sp, [%g5 + TRAP_ENT_SP]%asi 2335 stna %g0, [%g5 + TRAP_ENT_TR]%asi 2336 stna %g0, [%g5 + TRAP_ENT_F1]%asi 2337 stna %g0, [%g5 + TRAP_ENT_F2]%asi 2338 stna %g0, [%g5 + TRAP_ENT_F3]%asi 2339 stna %g0, [%g5 + TRAP_ENT_F4]%asi 2340 wr %g0, %g7, %asi 2341 2342 /* 2343 * Advance trap trace pointer. 2344 */ 2345 ld [%g6 + TRAPTR_OFFSET], %g5 2346 ld [%g6 + TRAPTR_LIMIT], %g4 2347 st %g5, [%g6 + TRAPTR_LAST_OFFSET] 2348 add %g5, TRAP_ENT_SIZE, %g5 2349 sub %g4, TRAP_ENT_SIZE, %g4 2350 cmp %g5, %g4 2351 movge %icc, 0, %g5 2352 st %g5, [%g6 + TRAPTR_OFFSET] 2353ipe_tl1_skip_tt: 2354#endif /* TRAPTRACE */ 2355 2356 /* 2357 * We fall into this macro if we've successfully logged the error in 2358 * the ch_err_tl1_data structure and want the PIL15 softint to pick 2359 * it up and log it. %g1 must point to the ch_err_tl1_data structure. 2360 * Restores the %g registers and issues retry. 2361 */ 2362 CH_ERR_TL1_EXIT; 2363 2364 SET_SIZE(icache_parity_tl1_err) 2365 2366#endif /* lint */ 2367 2368#endif /* CPU_IMP_L1_CACHE_PARITY */ 2369 2370 2371/* 2372 * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the 2373 * tte, the virtual address, and the ctxnum of the specified tlb entry. They 2374 * should only be used in places where you have no choice but to look at the 2375 * tlb itself. 2376 * 2377 * Note: These two routines are required by the Estar "cpr" loadable module. 2378 */ 2379 2380#if defined(lint) 2381 2382/* ARGSUSED */ 2383void 2384itlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag) 2385{} 2386 2387#else /* lint */ 2388 2389 ENTRY_NP(itlb_rd_entry) 2390 sllx %o0, 3, %o0 2391 ldxa [%o0]ASI_ITLB_ACCESS, %g1 2392 stx %g1, [%o1] 2393 ldxa [%o0]ASI_ITLB_TAGREAD, %g2 2394 set TAGREAD_CTX_MASK, %o4 2395 andn %g2, %o4, %o5 2396 retl 2397 stx %o5, [%o2] 2398 SET_SIZE(itlb_rd_entry) 2399 2400#endif /* lint */ 2401 2402 2403#if defined(lint) 2404 2405/* ARGSUSED */ 2406void 2407dtlb_rd_entry(uint_t entry, tte_t *tte, uint64_t *va_tag) 2408{} 2409 2410#else /* lint */ 2411 2412 ENTRY_NP(dtlb_rd_entry) 2413 sllx %o0, 3, %o0 2414 ldxa [%o0]ASI_DTLB_ACCESS, %g1 2415 stx %g1, [%o1] 2416 ldxa [%o0]ASI_DTLB_TAGREAD, %g2 2417 set TAGREAD_CTX_MASK, %o4 2418 andn %g2, %o4, %o5 2419 retl 2420 stx %o5, [%o2] 2421 SET_SIZE(dtlb_rd_entry) 2422#endif /* lint */ 2423 2424 2425#if !(defined(JALAPENO) || defined(SERRANO)) 2426 2427#if defined(lint) 2428 2429uint64_t 2430get_safari_config(void) 2431{ return (0); } 2432 2433#else /* lint */ 2434 2435 ENTRY(get_safari_config) 2436 ldxa [%g0]ASI_SAFARI_CONFIG, %o0 2437 retl 2438 nop 2439 SET_SIZE(get_safari_config) 2440 2441#endif /* lint */ 2442 2443 2444#if defined(lint) 2445 2446/* ARGSUSED */ 2447void 2448set_safari_config(uint64_t safari_config) 2449{} 2450 2451#else /* lint */ 2452 2453 ENTRY(set_safari_config) 2454 stxa %o0, [%g0]ASI_SAFARI_CONFIG 2455 membar #Sync 2456 retl 2457 nop 2458 SET_SIZE(set_safari_config) 2459 2460#endif /* lint */ 2461 2462#endif /* !(JALAPENO || SERRANO) */ 2463 2464 2465#if defined(lint) 2466 2467void 2468cpu_cleartickpnt(void) 2469{} 2470 2471#else /* lint */ 2472 /* 2473 * Clear the NPT (non-privileged trap) bit in the %tick/%stick 2474 * registers. In an effort to make the change in the 2475 * tick/stick counter as consistent as possible, we disable 2476 * all interrupts while we're changing the registers. We also 2477 * ensure that the read and write instructions are in the same 2478 * line in the instruction cache. 2479 */ 2480 ENTRY_NP(cpu_clearticknpt) 2481 rdpr %pstate, %g1 /* save processor state */ 2482 andn %g1, PSTATE_IE, %g3 /* turn off */ 2483 wrpr %g0, %g3, %pstate /* interrupts */ 2484 rdpr %tick, %g2 /* get tick register */ 2485 brgez,pn %g2, 1f /* if NPT bit off, we're done */ 2486 mov 1, %g3 /* create mask */ 2487 sllx %g3, 63, %g3 /* for NPT bit */ 2488 ba,a,pt %xcc, 2f 2489 .align 8 /* Ensure rd/wr in same i$ line */ 24902: 2491 rdpr %tick, %g2 /* get tick register */ 2492 wrpr %g3, %g2, %tick /* write tick register, */ 2493 /* clearing NPT bit */ 24941: 2495 rd STICK, %g2 /* get stick register */ 2496 brgez,pn %g2, 3f /* if NPT bit off, we're done */ 2497 mov 1, %g3 /* create mask */ 2498 sllx %g3, 63, %g3 /* for NPT bit */ 2499 ba,a,pt %xcc, 4f 2500 .align 8 /* Ensure rd/wr in same i$ line */ 25014: 2502 rd STICK, %g2 /* get stick register */ 2503 wr %g3, %g2, STICK /* write stick register, */ 2504 /* clearing NPT bit */ 25053: 2506 jmp %g4 + 4 2507 wrpr %g0, %g1, %pstate /* restore processor state */ 2508 2509 SET_SIZE(cpu_clearticknpt) 2510 2511#endif /* lint */ 2512 2513 2514#if defined(CPU_IMP_L1_CACHE_PARITY) 2515 2516#if defined(lint) 2517/* 2518 * correct_dcache_parity(size_t size, size_t linesize) 2519 * 2520 * Correct D$ data parity by zeroing the data and initializing microtag 2521 * for all indexes and all ways of the D$. 2522 * 2523 */ 2524/* ARGSUSED */ 2525void 2526correct_dcache_parity(size_t size, size_t linesize) 2527{} 2528 2529#else /* lint */ 2530 2531 ENTRY(correct_dcache_parity) 2532 /* 2533 * Register Usage: 2534 * 2535 * %o0 = input D$ size 2536 * %o1 = input D$ line size 2537 * %o2 = scratch 2538 * %o3 = scratch 2539 * %o4 = scratch 2540 */ 2541 2542 sub %o0, %o1, %o0 ! init cache line address 2543 2544 /* 2545 * For Panther CPUs, we also need to clear the data parity bits 2546 * using DC_data_parity bit of the ASI_DCACHE_DATA register. 2547 */ 2548 GET_CPU_IMPL(%o3) 2549 cmp %o3, PANTHER_IMPL 2550 bne 1f 2551 clr %o3 ! zero for non-Panther 2552 mov 1, %o3 2553 sll %o3, PN_DC_DATA_PARITY_BIT_SHIFT, %o3 2554 25551: 2556 /* 2557 * Set utag = way since it must be unique within an index. 2558 */ 2559 srl %o0, 14, %o2 ! get cache way (DC_way) 2560 membar #Sync ! required before ASI_DC_UTAG 2561 stxa %o2, [%o0]ASI_DC_UTAG ! set D$ utag = cache way 2562 membar #Sync ! required after ASI_DC_UTAG 2563 2564 /* 2565 * Zero line of D$ data (and data parity bits for Panther) 2566 */ 2567 sub %o1, 8, %o2 2568 or %o0, %o3, %o4 ! same address + DC_data_parity 25692: 2570 membar #Sync ! required before ASI_DC_DATA 2571 stxa %g0, [%o0 + %o2]ASI_DC_DATA ! zero 8 bytes of D$ data 2572 membar #Sync ! required after ASI_DC_DATA 2573 /* 2574 * We also clear the parity bits if this is a panther. For non-Panther 2575 * CPUs, we simply end up clearing the $data register twice. 2576 */ 2577 stxa %g0, [%o4 + %o2]ASI_DC_DATA 2578 membar #Sync 2579 2580 subcc %o2, 8, %o2 2581 bge 2b 2582 nop 2583 2584 subcc %o0, %o1, %o0 2585 bge 1b 2586 nop 2587 2588 retl 2589 nop 2590 SET_SIZE(correct_dcache_parity) 2591 2592#endif /* lint */ 2593 2594#endif /* CPU_IMP_L1_CACHE_PARITY */ 2595 2596 2597#if defined(lint) 2598/* 2599 * Get timestamp (stick). 2600 */ 2601/* ARGSUSED */ 2602void 2603stick_timestamp(int64_t *ts) 2604{ 2605} 2606 2607#else /* lint */ 2608 2609 ENTRY_NP(stick_timestamp) 2610 rd STICK, %g1 ! read stick reg 2611 sllx %g1, 1, %g1 2612 srlx %g1, 1, %g1 ! clear npt bit 2613 2614 retl 2615 stx %g1, [%o0] ! store the timestamp 2616 SET_SIZE(stick_timestamp) 2617 2618#endif /* lint */ 2619 2620 2621#if defined(lint) 2622/* 2623 * Set STICK adjusted by skew. 2624 */ 2625/* ARGSUSED */ 2626void 2627stick_adj(int64_t skew) 2628{ 2629} 2630 2631#else /* lint */ 2632 2633 ENTRY_NP(stick_adj) 2634 rdpr %pstate, %g1 ! save processor state 2635 andn %g1, PSTATE_IE, %g3 2636 ba 1f ! cache align stick adj 2637 wrpr %g0, %g3, %pstate ! turn off interrupts 2638 2639 .align 16 26401: nop 2641 2642 rd STICK, %g4 ! read stick reg 2643 add %g4, %o0, %o1 ! adjust stick with skew 2644 wr %o1, %g0, STICK ! write stick reg 2645 2646 retl 2647 wrpr %g1, %pstate ! restore processor state 2648 SET_SIZE(stick_adj) 2649 2650#endif /* lint */ 2651 2652#if defined(lint) 2653/* 2654 * Debugger-specific stick retrieval 2655 */ 2656/*ARGSUSED*/ 2657int 2658kdi_get_stick(uint64_t *stickp) 2659{ 2660 return (0); 2661} 2662 2663#else /* lint */ 2664 2665 ENTRY_NP(kdi_get_stick) 2666 rd STICK, %g1 2667 stx %g1, [%o0] 2668 retl 2669 mov %g0, %o0 2670 SET_SIZE(kdi_get_stick) 2671 2672#endif /* lint */ 2673 2674#if defined(lint) 2675/* 2676 * Invalidate the specified line from the D$. 2677 * 2678 * Register usage: 2679 * %o0 - index for the invalidation, specifies DC_way and DC_addr 2680 * 2681 * ASI_DC_TAG, 0x47, is used in the following manner. A 64-bit value is 2682 * stored to a particular DC_way and DC_addr in ASI_DC_TAG. 2683 * 2684 * The format of the stored 64-bit value is: 2685 * 2686 * +----------+--------+----------+ 2687 * | Reserved | DC_tag | DC_valid | 2688 * +----------+--------+----------+ 2689 * 63 31 30 1 0 2690 * 2691 * DC_tag is the 30-bit physical tag of the associated line. 2692 * DC_valid is the 1-bit valid field for both the physical and snoop tags. 2693 * 2694 * The format of the 64-bit DC_way and DC_addr into ASI_DC_TAG is: 2695 * 2696 * +----------+--------+----------+----------+ 2697 * | Reserved | DC_way | DC_addr | Reserved | 2698 * +----------+--------+----------+----------+ 2699 * 63 16 15 14 13 5 4 0 2700 * 2701 * DC_way is a 2-bit index that selects one of the 4 ways. 2702 * DC_addr is a 9-bit index that selects one of 512 tag/valid fields. 2703 * 2704 * Setting the DC_valid bit to zero for the specified DC_way and 2705 * DC_addr index into the D$ results in an invalidation of a D$ line. 2706 */ 2707/*ARGSUSED*/ 2708void 2709dcache_inval_line(int index) 2710{ 2711} 2712#else /* lint */ 2713 ENTRY(dcache_inval_line) 2714 sll %o0, 5, %o0 ! shift index into DC_way and DC_addr 2715 stxa %g0, [%o0]ASI_DC_TAG ! zero the DC_valid and DC_tag bits 2716 membar #Sync 2717 retl 2718 nop 2719 SET_SIZE(dcache_inval_line) 2720#endif /* lint */ 2721 2722#if defined(lint) 2723/* 2724 * Invalidate the entire I$ 2725 * 2726 * Register usage: 2727 * %o0 - specifies IC_way, IC_addr, IC_tag 2728 * %o1 - scratch 2729 * %o2 - used to save and restore DCU value 2730 * %o3 - scratch 2731 * %o5 - used to save and restore PSTATE 2732 * 2733 * Due to the behavior of the I$ control logic when accessing ASI_IC_TAG, 2734 * the I$ should be turned off. Accesses to ASI_IC_TAG may collide and 2735 * block out snoops and invalidates to the I$, causing I$ consistency 2736 * to be broken. Before turning on the I$, all I$ lines must be invalidated. 2737 * 2738 * ASI_IC_TAG, 0x67, is used in the following manner. A 64-bit value is 2739 * stored to a particular IC_way, IC_addr, IC_tag in ASI_IC_TAG. The 2740 * info below describes store (write) use of ASI_IC_TAG. Note that read 2741 * use of ASI_IC_TAG behaves differently. 2742 * 2743 * The format of the stored 64-bit value is: 2744 * 2745 * +----------+--------+---------------+-----------+ 2746 * | Reserved | Valid | IC_vpred<7:0> | Undefined | 2747 * +----------+--------+---------------+-----------+ 2748 * 63 55 54 53 46 45 0 2749 * 2750 * Valid is the 1-bit valid field for both the physical and snoop tags. 2751 * IC_vpred is the 8-bit LPB bits for 8 instructions starting at 2752 * the 32-byte boundary aligned address specified by IC_addr. 2753 * 2754 * The format of the 64-bit IC_way, IC_addr, IC_tag into ASI_IC_TAG is: 2755 * 2756 * +----------+--------+---------+--------+---------+ 2757 * | Reserved | IC_way | IC_addr | IC_tag |Reserved | 2758 * +----------+--------+---------+--------+---------+ 2759 * 63 16 15 14 13 5 4 3 2 0 2760 * 2761 * IC_way is a 2-bit index that selects one of the 4 ways. 2762 * IC_addr[13:6] is an 8-bit index that selects one of 256 valid fields. 2763 * IC_addr[5] is a "don't care" for a store. 2764 * IC_tag set to 2 specifies that the stored value is to be interpreted 2765 * as containing Valid and IC_vpred as described above. 2766 * 2767 * Setting the Valid bit to zero for the specified IC_way and 2768 * IC_addr index into the I$ results in an invalidation of an I$ line. 2769 */ 2770/*ARGSUSED*/ 2771void 2772icache_inval_all(void) 2773{ 2774} 2775#else /* lint */ 2776 ENTRY(icache_inval_all) 2777 rdpr %pstate, %o5 2778 andn %o5, PSTATE_IE, %o3 2779 wrpr %g0, %o3, %pstate ! clear IE bit 2780 2781 GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, icache_inval_all_1); 2782 ld [%o0 + CHPR_ICACHE_LINESIZE], %o1 2783 ba,pt %icc, 2f 2784 ld [%o0 + CHPR_ICACHE_SIZE], %o0 2785icache_inval_all_1: 2786 ASM_LD(%o0, icache_size) 2787 ASM_LD(%o1, icache_linesize) 27882: 2789 CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4) 2790 2791 retl 2792 wrpr %g0, %o5, %pstate ! restore earlier pstate 2793 SET_SIZE(icache_inval_all) 2794#endif /* lint */ 2795 2796 2797#if defined(lint) 2798/* ARGSUSED */ 2799void 2800cache_scrubreq_tl1(uint64_t inum, uint64_t index) 2801{ 2802} 2803 2804#else /* lint */ 2805/* 2806 * cache_scrubreq_tl1 is the crosstrap handler called on offlined cpus via a 2807 * crosstrap. It atomically increments the outstanding request counter and, 2808 * if there was not already an outstanding request, branches to setsoftint_tl1 2809 * to enqueue an intr_req for the given inum. 2810 */ 2811 2812 ! Register usage: 2813 ! 2814 ! Arguments: 2815 ! %g1 - inum 2816 ! %g2 - index into chsm_outstanding array 2817 ! 2818 ! Internal: 2819 ! %g2, %g3, %g5 - scratch 2820 ! %g4 - ptr. to scrub_misc chsm_outstanding[index]. 2821 ! %g6 - setsoftint_tl1 address 2822 2823 ENTRY_NP(cache_scrubreq_tl1) 2824 mulx %g2, CHSM_OUTSTANDING_INCR, %g2 2825 set CHPR_SCRUB_MISC + CHSM_OUTSTANDING, %g3 2826 add %g2, %g3, %g2 2827 GET_CPU_PRIVATE_PTR(%g2, %g4, %g5, 1f); 2828 ld [%g4], %g2 ! cpu's chsm_outstanding[index] 2829 ! 2830 ! no need to use atomic instructions for the following 2831 ! increment - we're at tl1 2832 ! 2833 add %g2, 0x1, %g3 2834 brnz,pn %g2, 1f ! no need to enqueue more intr_req 2835 st %g3, [%g4] ! delay - store incremented counter 2836 ASM_JMP(%g6, setsoftint_tl1) 2837 ! not reached 28381: 2839 retry 2840 SET_SIZE(cache_scrubreq_tl1) 2841 2842#endif /* lint */ 2843 2844 2845#if defined(lint) 2846 2847/* ARGSUSED */ 2848void 2849get_cpu_error_state(ch_cpu_errors_t *cpu_error_regs) 2850{} 2851 2852#else /* lint */ 2853 2854/* 2855 * Get the error state for the processor. 2856 * Note that this must not be used at TL>0 2857 */ 2858 ENTRY(get_cpu_error_state) 2859#if defined(CHEETAH_PLUS) 2860 set ASI_SHADOW_REG_VA, %o2 2861 ldxa [%o2]ASI_AFSR, %o1 ! shadow afsr reg 2862 stx %o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR] 2863 ldxa [%o2]ASI_AFAR, %o1 ! shadow afar reg 2864 stx %o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR] 2865 GET_CPU_IMPL(%o3) ! Only panther has AFSR_EXT registers 2866 cmp %o3, PANTHER_IMPL 2867 bne,a 1f 2868 stx %g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT] ! zero for non-PN 2869 set ASI_AFSR_EXT_VA, %o2 2870 ldxa [%o2]ASI_AFSR, %o1 ! afsr_ext reg 2871 stx %o1, [%o0 + CH_CPU_ERRORS_AFSR_EXT] 2872 set ASI_SHADOW_AFSR_EXT_VA, %o2 2873 ldxa [%o2]ASI_AFSR, %o1 ! shadow afsr_ext reg 2874 stx %o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] 2875 b 2f 2876 nop 28771: 2878 stx %g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] ! zero for non-PN 28792: 2880#else /* CHEETAH_PLUS */ 2881 stx %g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR] 2882 stx %g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR] 2883 stx %g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT] 2884 stx %g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] 2885#endif /* CHEETAH_PLUS */ 2886#if defined(SERRANO) 2887 /* 2888 * Serrano has an afar2 which captures the address on FRC/FRU errors. 2889 * We save this in the afar2 of the register save area. 2890 */ 2891 set ASI_MCU_AFAR2_VA, %o2 2892 ldxa [%o2]ASI_MCU_CTRL, %o1 2893 stx %o1, [%o0 + CH_CPU_ERRORS_AFAR2] 2894#endif /* SERRANO */ 2895 ldxa [%g0]ASI_AFSR, %o1 ! primary afsr reg 2896 stx %o1, [%o0 + CH_CPU_ERRORS_AFSR] 2897 ldxa [%g0]ASI_AFAR, %o1 ! primary afar reg 2898 retl 2899 stx %o1, [%o0 + CH_CPU_ERRORS_AFAR] 2900 SET_SIZE(get_cpu_error_state) 2901#endif /* lint */ 2902 2903#if defined(lint) 2904 2905/* 2906 * Check a page of memory for errors. 2907 * 2908 * Load each 64 byte block from physical memory. 2909 * Check AFSR after each load to see if an error 2910 * was caused. If so, log/scrub that error. 2911 * 2912 * Used to determine if a page contains 2913 * CEs when CEEN is disabled. 2914 */ 2915/*ARGSUSED*/ 2916void 2917cpu_check_block(caddr_t va, uint_t psz) 2918{} 2919 2920#else /* lint */ 2921 2922 ENTRY(cpu_check_block) 2923 ! 2924 ! get a new window with room for the error regs 2925 ! 2926 save %sp, -SA(MINFRAME + CH_CPU_ERROR_SIZE), %sp 2927 srl %i1, 6, %l4 ! clear top bits of psz 2928 ! and divide by 64 2929 rd %fprs, %l2 ! store FP 2930 wr %g0, FPRS_FEF, %fprs ! enable FP 29311: 2932 ldda [%i0]ASI_BLK_P, %d0 ! load a block 2933 membar #Sync 2934 ldxa [%g0]ASI_AFSR, %l3 ! read afsr reg 2935 brz,a,pt %l3, 2f ! check for error 2936 nop 2937 2938 ! 2939 ! if error, read the error regs and log it 2940 ! 2941 call get_cpu_error_state 2942 add %fp, STACK_BIAS - CH_CPU_ERROR_SIZE, %o0 2943 2944 ! 2945 ! cpu_ce_detected(ch_cpu_errors_t *, flag) 2946 ! 2947 call cpu_ce_detected ! log the error 2948 mov CE_CEEN_TIMEOUT, %o1 29492: 2950 dec %l4 ! next 64-byte block 2951 brnz,a,pt %l4, 1b 2952 add %i0, 64, %i0 ! increment block addr 2953 2954 wr %l2, %g0, %fprs ! restore FP 2955 ret 2956 restore 2957 2958 SET_SIZE(cpu_check_block) 2959 2960#endif /* lint */ 2961 2962#if defined(lint) 2963 2964/* 2965 * Perform a cpu logout called from C. This is used where we did not trap 2966 * for the error but still want to gather "what we can". Caller must make 2967 * sure cpu private area exists and that the indicated logout area is free 2968 * for use, and that we are unable to migrate cpus. 2969 */ 2970/*ARGSUSED*/ 2971void 2972cpu_delayed_logout(uint64_t afar, ch_cpu_logout_t *clop) 2973{ } 2974 2975#else 2976 ENTRY(cpu_delayed_logout) 2977 rdpr %pstate, %o2 2978 andn %o2, PSTATE_IE, %o2 2979 wrpr %g0, %o2, %pstate ! disable interrupts 2980 PARK_SIBLING_CORE(%o2, %o3, %o4) ! %o2 has DCU value 2981 add %o1, CH_CLO_DATA + CH_CHD_EC_DATA, %o1 2982 rd %asi, %g1 2983 wr %g0, ASI_P, %asi 2984 GET_ECACHE_DTAGS(%o0, %o1, %o3, %o4, %o5) 2985 wr %g1, %asi 2986 UNPARK_SIBLING_CORE(%o2, %o3, %o4) ! can use %o2 again 2987 rdpr %pstate, %o2 2988 or %o2, PSTATE_IE, %o2 2989 wrpr %g0, %o2, %pstate 2990 retl 2991 nop 2992 SET_SIZE(cpu_delayed_logout) 2993 2994#endif /* lint */ 2995 2996#if defined(lint) 2997 2998/*ARGSUSED*/ 2999int 3000dtrace_blksuword32(uintptr_t addr, uint32_t *data, int tryagain) 3001{ return (0); } 3002 3003#else 3004 3005 ENTRY(dtrace_blksuword32) 3006 save %sp, -SA(MINFRAME + 4), %sp 3007 3008 rdpr %pstate, %l1 3009 andn %l1, PSTATE_IE, %l2 ! disable interrupts to 3010 wrpr %g0, %l2, %pstate ! protect our FPU diddling 3011 3012 rd %fprs, %l0 3013 andcc %l0, FPRS_FEF, %g0 3014 bz,a,pt %xcc, 1f ! if the fpu is disabled 3015 wr %g0, FPRS_FEF, %fprs ! ... enable the fpu 3016 3017 st %f0, [%fp + STACK_BIAS - 4] ! save %f0 to the stack 30181: 3019 set 0f, %l5 3020 /* 3021 * We're about to write a block full or either total garbage 3022 * (not kernel data, don't worry) or user floating-point data 3023 * (so it only _looks_ like garbage). 3024 */ 3025 ld [%i1], %f0 ! modify the block 3026 membar #Sync 3027 stn %l5, [THREAD_REG + T_LOFAULT] ! set up the lofault handler 3028 stda %d0, [%i0]ASI_BLK_COMMIT_S ! store the modified block 3029 membar #Sync 3030 stn %g0, [THREAD_REG + T_LOFAULT] ! remove the lofault handler 3031 3032 bz,a,pt %xcc, 1f 3033 wr %g0, %l0, %fprs ! restore %fprs 3034 3035 ld [%fp + STACK_BIAS - 4], %f0 ! restore %f0 30361: 3037 3038 wrpr %g0, %l1, %pstate ! restore interrupts 3039 3040 ret 3041 restore %g0, %g0, %o0 3042 30430: 3044 membar #Sync 3045 stn %g0, [THREAD_REG + T_LOFAULT] ! remove the lofault handler 3046 3047 bz,a,pt %xcc, 1f 3048 wr %g0, %l0, %fprs ! restore %fprs 3049 3050 ld [%fp + STACK_BIAS - 4], %f0 ! restore %f0 30511: 3052 3053 wrpr %g0, %l1, %pstate ! restore interrupts 3054 3055 /* 3056 * If tryagain is set (%i2) we tail-call dtrace_blksuword32_err() 3057 * which deals with watchpoints. Otherwise, just return -1. 3058 */ 3059 brnz,pt %i2, 1f 3060 nop 3061 ret 3062 restore %g0, -1, %o0 30631: 3064 call dtrace_blksuword32_err 3065 restore 3066 3067 SET_SIZE(dtrace_blksuword32) 3068 3069#endif /* lint */ 3070 3071#ifdef CHEETAHPLUS_ERRATUM_25 3072 3073#if defined(lint) 3074/* 3075 * Claim a chunk of physical address space. 3076 */ 3077/*ARGSUSED*/ 3078void 3079claimlines(uint64_t pa, size_t sz, int stride) 3080{} 3081#else /* lint */ 3082 ENTRY(claimlines) 30831: 3084 subcc %o1, %o2, %o1 3085 add %o0, %o1, %o3 3086 bgeu,a,pt %xcc, 1b 3087 casxa [%o3]ASI_MEM, %g0, %g0 3088 membar #Sync 3089 retl 3090 nop 3091 SET_SIZE(claimlines) 3092#endif /* lint */ 3093 3094#if defined(lint) 3095/* 3096 * CPU feature initialization, 3097 * turn BPE off, 3098 * get device id. 3099 */ 3100/*ARGSUSED*/ 3101void 3102cpu_feature_init(void) 3103{} 3104#else /* lint */ 3105 ENTRY(cpu_feature_init) 3106 save %sp, -SA(MINFRAME), %sp 3107 sethi %hi(cheetah_bpe_off), %o0 3108 ld [%o0 + %lo(cheetah_bpe_off)], %o0 3109 brz %o0, 1f 3110 nop 3111 rd ASR_DISPATCH_CONTROL, %o0 3112 andn %o0, ASR_DISPATCH_CONTROL_BPE, %o0 3113 wr %o0, 0, ASR_DISPATCH_CONTROL 31141: 3115 ! 3116 ! get the device_id and store the device_id 3117 ! in the appropriate cpunodes structure 3118 ! given the cpus index 3119 ! 3120 CPU_INDEX(%o0, %o1) 3121 mulx %o0, CPU_NODE_SIZE, %o0 3122 set cpunodes + DEVICE_ID, %o1 3123 ldxa [%g0] ASI_DEVICE_SERIAL_ID, %o2 3124 stx %o2, [%o0 + %o1] 3125#ifdef CHEETAHPLUS_ERRATUM_34 3126 ! 3127 ! apply Cheetah+ erratum 34 workaround 3128 ! 3129 call itlb_erratum34_fixup 3130 nop 3131#endif /* CHEETAHPLUS_ERRATUM_34 */ 3132 ret 3133 restore 3134 SET_SIZE(cpu_feature_init) 3135#endif /* lint */ 3136 3137#if defined(lint) 3138/* 3139 * Copy a tsb entry atomically, from src to dest. 3140 * src must be 128 bit aligned. 3141 */ 3142/*ARGSUSED*/ 3143void 3144copy_tsb_entry(uintptr_t src, uintptr_t dest) 3145{} 3146#else /* lint */ 3147 ENTRY(copy_tsb_entry) 3148 ldda [%o0]ASI_NQUAD_LD, %o2 ! %o2 = tag, %o3 = data 3149 stx %o2, [%o1] 3150 stx %o3, [%o1 + 8 ] 3151 retl 3152 nop 3153 SET_SIZE(copy_tsb_entry) 3154#endif /* lint */ 3155 3156#endif /* CHEETAHPLUS_ERRATUM_25 */ 3157 3158#ifdef CHEETAHPLUS_ERRATUM_34 3159 3160#if defined(lint) 3161 3162/*ARGSUSED*/ 3163void 3164itlb_erratum34_fixup(void) 3165{} 3166 3167#else /* lint */ 3168 3169 ! 3170 ! In Cheetah+ erratum 34, under certain conditions an ITLB locked 3171 ! index 0 TTE will erroneously be displaced when a new TTE is 3172 ! loaded via ASI_ITLB_IN. In order to avoid cheetah+ erratum 34, 3173 ! locked index 0 TTEs must be relocated. 3174 ! 3175 ! NOTE: Care must be taken to avoid an ITLB miss in this routine. 3176 ! 3177 ENTRY_NP(itlb_erratum34_fixup) 3178 rdpr %pstate, %o3 3179#ifdef DEBUG 3180 andcc %o3, PSTATE_IE, %g0 ! If interrupts already 3181 bnz,pt %icc, 0f ! disabled, panic 3182 nop 3183 sethi %hi(sfmmu_panic1), %o0 3184 call panic 3185 or %o0, %lo(sfmmu_panic1), %o0 31860: 3187#endif /* DEBUG */ 3188 wrpr %o3, PSTATE_IE, %pstate ! Disable interrupts 3189 ldxa [%g0]ASI_ITLB_ACCESS, %o1 ! %o1 = entry 0 data 3190 ldxa [%g0]ASI_ITLB_TAGREAD, %o2 ! %o2 = entry 0 tag 3191 3192 cmp %o1, %g0 ! Is this entry valid? 3193 bge %xcc, 1f 3194 andcc %o1, TTE_LCK_INT, %g0 ! Is this entry locked? 3195 bnz %icc, 2f 3196 nop 31971: 3198 retl ! Nope, outta here... 3199 wrpr %g0, %o3, %pstate ! Enable interrupts 32002: 3201 sethi %hi(FLUSH_ADDR), %o4 3202 stxa %g0, [%o2]ASI_ITLB_DEMAP ! Flush this mapping 3203 flush %o4 ! Flush required for I-MMU 3204 ! 3205 ! Start search from index 1 up. This is because the kernel force 3206 ! loads its text page at index 15 in sfmmu_kernel_remap() and we 3207 ! don't want our relocated entry evicted later. 3208 ! 3209 ! NOTE: We assume that we'll be successful in finding an unlocked 3210 ! or invalid entry. If that isn't the case there are bound to 3211 ! bigger problems. 3212 ! 3213 set (1 << 3), %g3 32143: 3215 ldxa [%g3]ASI_ITLB_ACCESS, %o4 ! Load TTE from t16 3216 ! 3217 ! If this entry isn't valid, we'll choose to displace it (regardless 3218 ! of the lock bit). 3219 ! 3220 cmp %o4, %g0 ! TTE is > 0 iff not valid 3221 bge %xcc, 4f ! If invalid, go displace 3222 andcc %o4, TTE_LCK_INT, %g0 ! Check for lock bit 3223 bnz,a %icc, 3b ! If locked, look at next 3224 add %g3, (1 << 3), %g3 ! entry 32254: 3226 ! 3227 ! We found an unlocked or invalid entry; we'll explicitly load 3228 ! the former index 0 entry here. 3229 ! 3230 sethi %hi(FLUSH_ADDR), %o4 3231 set MMU_TAG_ACCESS, %g4 3232 stxa %o2, [%g4]ASI_IMMU 3233 stxa %o1, [%g3]ASI_ITLB_ACCESS 3234 flush %o4 ! Flush required for I-MMU 3235 retl 3236 wrpr %g0, %o3, %pstate ! Enable interrupts 3237 SET_SIZE(itlb_erratum34_fixup) 3238 3239#endif /* lint */ 3240 3241#endif /* CHEETAHPLUS_ERRATUM_34 */ 3242 3243