1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27#pragma ident "%Z%%M% %I% %E% SMI" 28 29#if !defined(lint) 30#include "assym.h" 31#endif /* !lint */ 32#include <sys/asm_linkage.h> 33#include <sys/privregs.h> 34#include <sys/sun4asi.h> 35#include <sys/machasi.h> 36#include <sys/hypervisor_api.h> 37#include <sys/machtrap.h> 38#include <sys/machthread.h> 39#include <sys/machbrand.h> 40#include <sys/pcb.h> 41#include <sys/pte.h> 42#include <sys/mmu.h> 43#include <sys/machpcb.h> 44#include <sys/async.h> 45#include <sys/intreg.h> 46#include <sys/scb.h> 47#include <sys/psr_compat.h> 48#include <sys/syscall.h> 49#include <sys/machparam.h> 50#include <sys/traptrace.h> 51#include <vm/hat_sfmmu.h> 52#include <sys/archsystm.h> 53#include <sys/utrap.h> 54#include <sys/clock.h> 55#include <sys/intr.h> 56#include <sys/fpu/fpu_simulator.h> 57#include <vm/seg_spt.h> 58 59/* 60 * WARNING: If you add a fast trap handler which can be invoked by a 61 * non-privileged user, you may have to use the FAST_TRAP_DONE macro 62 * instead of "done" instruction to return back to the user mode. See 63 * comments for the "fast_trap_done" entry point for more information. 64 * 65 * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the 66 * cases where you always want to process any pending interrupts before 67 * returning back to the user mode. 68 */ 69#define FAST_TRAP_DONE \ 70 ba,a fast_trap_done 71 72#define FAST_TRAP_DONE_CHK_INTR \ 73 ba,a fast_trap_done_chk_intr 74 75/* 76 * SPARC V9 Trap Table 77 * 78 * Most of the trap handlers are made from common building 79 * blocks, and some are instantiated multiple times within 80 * the trap table. So, I build a bunch of macros, then 81 * populate the table using only the macros. 82 * 83 * Many macros branch to sys_trap. Its calling convention is: 84 * %g1 kernel trap handler 85 * %g2, %g3 args for above 86 * %g4 desire %pil 87 */ 88 89#ifdef TRAPTRACE 90 91/* 92 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 93 */ 94#define TT_TRACE(label) \ 95 ba label ;\ 96 rd %pc, %g7 97#define TT_TRACE_INS 2 98 99#define TT_TRACE_L(label) \ 100 ba label ;\ 101 rd %pc, %l4 ;\ 102 clr %l4 103#define TT_TRACE_L_INS 3 104 105#else 106 107#define TT_TRACE(label) 108#define TT_TRACE_INS 0 109 110#define TT_TRACE_L(label) 111#define TT_TRACE_L_INS 0 112 113#endif 114 115/* 116 * This macro is used to update per cpu mmu stats in perf critical 117 * paths. It is only enabled in debug kernels or if SFMMU_STAT_GATHER 118 * is defined. 119 */ 120#if defined(DEBUG) || defined(SFMMU_STAT_GATHER) 121#define HAT_PERCPU_DBSTAT(stat) \ 122 mov stat, %g1 ;\ 123 ba stat_mmu ;\ 124 rd %pc, %g7 125#else 126#define HAT_PERCPU_DBSTAT(stat) 127#endif /* DEBUG || SFMMU_STAT_GATHER */ 128 129/* 130 * This first set are funneled to trap() with %tt as the type. 131 * Trap will then either panic or send the user a signal. 132 */ 133/* 134 * NOT is used for traps that just shouldn't happen. 135 * It comes in both single and quadruple flavors. 136 */ 137#if !defined(lint) 138 .global trap 139#endif /* !lint */ 140#define NOT \ 141 TT_TRACE(trace_gen) ;\ 142 set trap, %g1 ;\ 143 rdpr %tt, %g3 ;\ 144 ba,pt %xcc, sys_trap ;\ 145 sub %g0, 1, %g4 ;\ 146 .align 32 147#define NOT4 NOT; NOT; NOT; NOT 148 149#define NOTP \ 150 TT_TRACE(trace_gen) ;\ 151 ba,pt %xcc, ptl1_panic ;\ 152 mov PTL1_BAD_TRAP, %g1 ;\ 153 .align 32 154#define NOTP4 NOTP; NOTP; NOTP; NOTP 155 156 157/* 158 * BAD is used for trap vectors we don't have a kernel 159 * handler for. 160 * It also comes in single and quadruple versions. 161 */ 162#define BAD NOT 163#define BAD4 NOT4 164 165#define DONE \ 166 done; \ 167 .align 32 168 169/* 170 * TRAP vectors to the trap() function. 171 * It's main use is for user errors. 172 */ 173#if !defined(lint) 174 .global trap 175#endif /* !lint */ 176#define TRAP(arg) \ 177 TT_TRACE(trace_gen) ;\ 178 set trap, %g1 ;\ 179 mov arg, %g3 ;\ 180 ba,pt %xcc, sys_trap ;\ 181 sub %g0, 1, %g4 ;\ 182 .align 32 183 184/* 185 * SYSCALL is used for system calls on both ILP32 and LP64 kernels 186 * depending on the "which" parameter (should be syscall_trap, 187 * syscall_trap32, or nosys for unused system call traps). 188 */ 189#define SYSCALL(which) \ 190 TT_TRACE(trace_gen) ;\ 191 SYSCALL_NOTT(which) 192 193#define SYSCALL_NOTT(which) \ 194 set (which), %g1 ;\ 195 ba,pt %xcc, sys_trap ;\ 196 sub %g0, 1, %g4 ;\ 197 .align 32 198 199/* 200 * GOTO just jumps to a label. 201 * It's used for things that can be fixed without going thru sys_trap. 202 */ 203#define GOTO(label) \ 204 .global label ;\ 205 ba,a label ;\ 206 .empty ;\ 207 .align 32 208 209/* 210 * GOTO_TT just jumps to a label. 211 * correctable ECC error traps at level 0 and 1 will use this macro. 212 * It's used for things that can be fixed without going thru sys_trap. 213 */ 214#define GOTO_TT(label, ttlabel) \ 215 .global label ;\ 216 TT_TRACE(ttlabel) ;\ 217 ba,a label ;\ 218 .empty ;\ 219 .align 32 220 221/* 222 * Privileged traps 223 * Takes breakpoint if privileged, calls trap() if not. 224 */ 225#define PRIV(label) \ 226 rdpr %tstate, %g1 ;\ 227 btst TSTATE_PRIV, %g1 ;\ 228 bnz label ;\ 229 rdpr %tt, %g3 ;\ 230 set trap, %g1 ;\ 231 ba,pt %xcc, sys_trap ;\ 232 sub %g0, 1, %g4 ;\ 233 .align 32 234 235 236/* 237 * DTrace traps. 238 */ 239#define DTRACE_PID \ 240 .global dtrace_pid_probe ;\ 241 set dtrace_pid_probe, %g1 ;\ 242 ba,pt %xcc, user_trap ;\ 243 sub %g0, 1, %g4 ;\ 244 .align 32 245 246#define DTRACE_RETURN \ 247 .global dtrace_return_probe ;\ 248 set dtrace_return_probe, %g1 ;\ 249 ba,pt %xcc, user_trap ;\ 250 sub %g0, 1, %g4 ;\ 251 .align 32 252 253/* 254 * REGISTER WINDOW MANAGEMENT MACROS 255 */ 256 257/* 258 * various convenient units of padding 259 */ 260#define SKIP(n) .skip 4*(n) 261 262/* 263 * CLEAN_WINDOW is the simple handler for cleaning a register window. 264 */ 265#define CLEAN_WINDOW \ 266 TT_TRACE_L(trace_win) ;\ 267 rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin ;\ 268 clr %l0; clr %l1; clr %l2; clr %l3 ;\ 269 clr %l4; clr %l5; clr %l6; clr %l7 ;\ 270 clr %o0; clr %o1; clr %o2; clr %o3 ;\ 271 clr %o4; clr %o5; clr %o6; clr %o7 ;\ 272 retry; .align 128 273 274#if !defined(lint) 275 276/* 277 * If we get an unresolved tlb miss while in a window handler, the fault 278 * handler will resume execution at the last instruction of the window 279 * hander, instead of delivering the fault to the kernel. Spill handlers 280 * use this to spill windows into the wbuf. 281 * 282 * The mixed handler works by checking %sp, and branching to the correct 283 * handler. This is done by branching back to label 1: for 32b frames, 284 * or label 2: for 64b frames; which implies the handler order is: 32b, 285 * 64b, mixed. The 1: and 2: labels are offset into the routines to 286 * allow the branchs' delay slots to contain useful instructions. 287 */ 288 289/* 290 * SPILL_32bit spills a 32-bit-wide kernel register window. It 291 * assumes that the kernel context and the nucleus context are the 292 * same. The stack pointer is required to be eight-byte aligned even 293 * though this code only needs it to be four-byte aligned. 294 */ 295#define SPILL_32bit(tail) \ 296 srl %sp, 0, %sp ;\ 2971: st %l0, [%sp + 0] ;\ 298 st %l1, [%sp + 4] ;\ 299 st %l2, [%sp + 8] ;\ 300 st %l3, [%sp + 12] ;\ 301 st %l4, [%sp + 16] ;\ 302 st %l5, [%sp + 20] ;\ 303 st %l6, [%sp + 24] ;\ 304 st %l7, [%sp + 28] ;\ 305 st %i0, [%sp + 32] ;\ 306 st %i1, [%sp + 36] ;\ 307 st %i2, [%sp + 40] ;\ 308 st %i3, [%sp + 44] ;\ 309 st %i4, [%sp + 48] ;\ 310 st %i5, [%sp + 52] ;\ 311 st %i6, [%sp + 56] ;\ 312 st %i7, [%sp + 60] ;\ 313 TT_TRACE_L(trace_win) ;\ 314 saved ;\ 315 retry ;\ 316 SKIP(31-19-TT_TRACE_L_INS) ;\ 317 ba,a,pt %xcc, fault_32bit_/**/tail ;\ 318 .empty 319 320/* 321 * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit 322 * wide address space via the designated asi. It is used to spill 323 * non-kernel windows. The stack pointer is required to be eight-byte 324 * aligned even though this code only needs it to be four-byte 325 * aligned. 326 */ 327#define SPILL_32bit_asi(asi_num, tail) \ 328 srl %sp, 0, %sp ;\ 3291: sta %l0, [%sp + %g0]asi_num ;\ 330 mov 4, %g1 ;\ 331 sta %l1, [%sp + %g1]asi_num ;\ 332 mov 8, %g2 ;\ 333 sta %l2, [%sp + %g2]asi_num ;\ 334 mov 12, %g3 ;\ 335 sta %l3, [%sp + %g3]asi_num ;\ 336 add %sp, 16, %g4 ;\ 337 sta %l4, [%g4 + %g0]asi_num ;\ 338 sta %l5, [%g4 + %g1]asi_num ;\ 339 sta %l6, [%g4 + %g2]asi_num ;\ 340 sta %l7, [%g4 + %g3]asi_num ;\ 341 add %g4, 16, %g4 ;\ 342 sta %i0, [%g4 + %g0]asi_num ;\ 343 sta %i1, [%g4 + %g1]asi_num ;\ 344 sta %i2, [%g4 + %g2]asi_num ;\ 345 sta %i3, [%g4 + %g3]asi_num ;\ 346 add %g4, 16, %g4 ;\ 347 sta %i4, [%g4 + %g0]asi_num ;\ 348 sta %i5, [%g4 + %g1]asi_num ;\ 349 sta %i6, [%g4 + %g2]asi_num ;\ 350 sta %i7, [%g4 + %g3]asi_num ;\ 351 TT_TRACE_L(trace_win) ;\ 352 saved ;\ 353 retry ;\ 354 SKIP(31-25-TT_TRACE_L_INS) ;\ 355 ba,a,pt %xcc, fault_32bit_/**/tail ;\ 356 .empty 357 358#define SPILL_32bit_tt1(asi_num, tail) \ 359 ba,a,pt %xcc, fault_32bit_/**/tail ;\ 360 .empty ;\ 361 .align 128 362 363 364/* 365 * FILL_32bit fills a 32-bit-wide kernel register window. It assumes 366 * that the kernel context and the nucleus context are the same. The 367 * stack pointer is required to be eight-byte aligned even though this 368 * code only needs it to be four-byte aligned. 369 */ 370#define FILL_32bit(tail) \ 371 srl %sp, 0, %sp ;\ 3721: TT_TRACE_L(trace_win) ;\ 373 ld [%sp + 0], %l0 ;\ 374 ld [%sp + 4], %l1 ;\ 375 ld [%sp + 8], %l2 ;\ 376 ld [%sp + 12], %l3 ;\ 377 ld [%sp + 16], %l4 ;\ 378 ld [%sp + 20], %l5 ;\ 379 ld [%sp + 24], %l6 ;\ 380 ld [%sp + 28], %l7 ;\ 381 ld [%sp + 32], %i0 ;\ 382 ld [%sp + 36], %i1 ;\ 383 ld [%sp + 40], %i2 ;\ 384 ld [%sp + 44], %i3 ;\ 385 ld [%sp + 48], %i4 ;\ 386 ld [%sp + 52], %i5 ;\ 387 ld [%sp + 56], %i6 ;\ 388 ld [%sp + 60], %i7 ;\ 389 restored ;\ 390 retry ;\ 391 SKIP(31-19-TT_TRACE_L_INS) ;\ 392 ba,a,pt %xcc, fault_32bit_/**/tail ;\ 393 .empty 394 395/* 396 * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit 397 * wide address space via the designated asi. It is used to fill 398 * non-kernel windows. The stack pointer is required to be eight-byte 399 * aligned even though this code only needs it to be four-byte 400 * aligned. 401 */ 402#define FILL_32bit_asi(asi_num, tail) \ 403 srl %sp, 0, %sp ;\ 4041: TT_TRACE_L(trace_win) ;\ 405 mov 4, %g1 ;\ 406 lda [%sp + %g0]asi_num, %l0 ;\ 407 mov 8, %g2 ;\ 408 lda [%sp + %g1]asi_num, %l1 ;\ 409 mov 12, %g3 ;\ 410 lda [%sp + %g2]asi_num, %l2 ;\ 411 lda [%sp + %g3]asi_num, %l3 ;\ 412 add %sp, 16, %g4 ;\ 413 lda [%g4 + %g0]asi_num, %l4 ;\ 414 lda [%g4 + %g1]asi_num, %l5 ;\ 415 lda [%g4 + %g2]asi_num, %l6 ;\ 416 lda [%g4 + %g3]asi_num, %l7 ;\ 417 add %g4, 16, %g4 ;\ 418 lda [%g4 + %g0]asi_num, %i0 ;\ 419 lda [%g4 + %g1]asi_num, %i1 ;\ 420 lda [%g4 + %g2]asi_num, %i2 ;\ 421 lda [%g4 + %g3]asi_num, %i3 ;\ 422 add %g4, 16, %g4 ;\ 423 lda [%g4 + %g0]asi_num, %i4 ;\ 424 lda [%g4 + %g1]asi_num, %i5 ;\ 425 lda [%g4 + %g2]asi_num, %i6 ;\ 426 lda [%g4 + %g3]asi_num, %i7 ;\ 427 restored ;\ 428 retry ;\ 429 SKIP(31-25-TT_TRACE_L_INS) ;\ 430 ba,a,pt %xcc, fault_32bit_/**/tail ;\ 431 .empty 432 433 434/* 435 * SPILL_64bit spills a 64-bit-wide kernel register window. It 436 * assumes that the kernel context and the nucleus context are the 437 * same. The stack pointer is required to be eight-byte aligned. 438 */ 439#define SPILL_64bit(tail) \ 4402: stx %l0, [%sp + V9BIAS64 + 0] ;\ 441 stx %l1, [%sp + V9BIAS64 + 8] ;\ 442 stx %l2, [%sp + V9BIAS64 + 16] ;\ 443 stx %l3, [%sp + V9BIAS64 + 24] ;\ 444 stx %l4, [%sp + V9BIAS64 + 32] ;\ 445 stx %l5, [%sp + V9BIAS64 + 40] ;\ 446 stx %l6, [%sp + V9BIAS64 + 48] ;\ 447 stx %l7, [%sp + V9BIAS64 + 56] ;\ 448 stx %i0, [%sp + V9BIAS64 + 64] ;\ 449 stx %i1, [%sp + V9BIAS64 + 72] ;\ 450 stx %i2, [%sp + V9BIAS64 + 80] ;\ 451 stx %i3, [%sp + V9BIAS64 + 88] ;\ 452 stx %i4, [%sp + V9BIAS64 + 96] ;\ 453 stx %i5, [%sp + V9BIAS64 + 104] ;\ 454 stx %i6, [%sp + V9BIAS64 + 112] ;\ 455 stx %i7, [%sp + V9BIAS64 + 120] ;\ 456 TT_TRACE_L(trace_win) ;\ 457 saved ;\ 458 retry ;\ 459 SKIP(31-18-TT_TRACE_L_INS) ;\ 460 ba,a,pt %xcc, fault_64bit_/**/tail ;\ 461 .empty 462 463#define SPILL_64bit_ktt1(tail) \ 464 ba,a,pt %xcc, fault_64bit_/**/tail ;\ 465 .empty ;\ 466 .align 128 467 468#define SPILL_mixed_ktt1(tail) \ 469 btst 1, %sp ;\ 470 bz,a,pt %xcc, fault_32bit_/**/tail ;\ 471 srl %sp, 0, %sp ;\ 472 ba,a,pt %xcc, fault_64bit_/**/tail ;\ 473 .empty ;\ 474 .align 128 475 476/* 477 * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit 478 * wide address space via the designated asi. It is used to spill 479 * non-kernel windows. The stack pointer is required to be eight-byte 480 * aligned. 481 */ 482#define SPILL_64bit_asi(asi_num, tail) \ 483 mov 0 + V9BIAS64, %g1 ;\ 4842: stxa %l0, [%sp + %g1]asi_num ;\ 485 mov 8 + V9BIAS64, %g2 ;\ 486 stxa %l1, [%sp + %g2]asi_num ;\ 487 mov 16 + V9BIAS64, %g3 ;\ 488 stxa %l2, [%sp + %g3]asi_num ;\ 489 mov 24 + V9BIAS64, %g4 ;\ 490 stxa %l3, [%sp + %g4]asi_num ;\ 491 add %sp, 32, %g5 ;\ 492 stxa %l4, [%g5 + %g1]asi_num ;\ 493 stxa %l5, [%g5 + %g2]asi_num ;\ 494 stxa %l6, [%g5 + %g3]asi_num ;\ 495 stxa %l7, [%g5 + %g4]asi_num ;\ 496 add %g5, 32, %g5 ;\ 497 stxa %i0, [%g5 + %g1]asi_num ;\ 498 stxa %i1, [%g5 + %g2]asi_num ;\ 499 stxa %i2, [%g5 + %g3]asi_num ;\ 500 stxa %i3, [%g5 + %g4]asi_num ;\ 501 add %g5, 32, %g5 ;\ 502 stxa %i4, [%g5 + %g1]asi_num ;\ 503 stxa %i5, [%g5 + %g2]asi_num ;\ 504 stxa %i6, [%g5 + %g3]asi_num ;\ 505 stxa %i7, [%g5 + %g4]asi_num ;\ 506 TT_TRACE_L(trace_win) ;\ 507 saved ;\ 508 retry ;\ 509 SKIP(31-25-TT_TRACE_L_INS) ;\ 510 ba,a,pt %xcc, fault_64bit_/**/tail ;\ 511 .empty 512 513#define SPILL_64bit_tt1(asi_num, tail) \ 514 ba,a,pt %xcc, fault_64bit_/**/tail ;\ 515 .empty ;\ 516 .align 128 517 518/* 519 * FILL_64bit fills a 64-bit-wide kernel register window. It assumes 520 * that the kernel context and the nucleus context are the same. The 521 * stack pointer is required to be eight-byte aligned. 522 */ 523#define FILL_64bit(tail) \ 5242: TT_TRACE_L(trace_win) ;\ 525 ldx [%sp + V9BIAS64 + 0], %l0 ;\ 526 ldx [%sp + V9BIAS64 + 8], %l1 ;\ 527 ldx [%sp + V9BIAS64 + 16], %l2 ;\ 528 ldx [%sp + V9BIAS64 + 24], %l3 ;\ 529 ldx [%sp + V9BIAS64 + 32], %l4 ;\ 530 ldx [%sp + V9BIAS64 + 40], %l5 ;\ 531 ldx [%sp + V9BIAS64 + 48], %l6 ;\ 532 ldx [%sp + V9BIAS64 + 56], %l7 ;\ 533 ldx [%sp + V9BIAS64 + 64], %i0 ;\ 534 ldx [%sp + V9BIAS64 + 72], %i1 ;\ 535 ldx [%sp + V9BIAS64 + 80], %i2 ;\ 536 ldx [%sp + V9BIAS64 + 88], %i3 ;\ 537 ldx [%sp + V9BIAS64 + 96], %i4 ;\ 538 ldx [%sp + V9BIAS64 + 104], %i5 ;\ 539 ldx [%sp + V9BIAS64 + 112], %i6 ;\ 540 ldx [%sp + V9BIAS64 + 120], %i7 ;\ 541 restored ;\ 542 retry ;\ 543 SKIP(31-18-TT_TRACE_L_INS) ;\ 544 ba,a,pt %xcc, fault_64bit_/**/tail ;\ 545 .empty 546 547/* 548 * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit 549 * wide address space via the designated asi. It is used to fill 550 * non-kernel windows. The stack pointer is required to be eight-byte 551 * aligned. 552 */ 553#define FILL_64bit_asi(asi_num, tail) \ 554 mov V9BIAS64 + 0, %g1 ;\ 5552: TT_TRACE_L(trace_win) ;\ 556 ldxa [%sp + %g1]asi_num, %l0 ;\ 557 mov V9BIAS64 + 8, %g2 ;\ 558 ldxa [%sp + %g2]asi_num, %l1 ;\ 559 mov V9BIAS64 + 16, %g3 ;\ 560 ldxa [%sp + %g3]asi_num, %l2 ;\ 561 mov V9BIAS64 + 24, %g4 ;\ 562 ldxa [%sp + %g4]asi_num, %l3 ;\ 563 add %sp, 32, %g5 ;\ 564 ldxa [%g5 + %g1]asi_num, %l4 ;\ 565 ldxa [%g5 + %g2]asi_num, %l5 ;\ 566 ldxa [%g5 + %g3]asi_num, %l6 ;\ 567 ldxa [%g5 + %g4]asi_num, %l7 ;\ 568 add %g5, 32, %g5 ;\ 569 ldxa [%g5 + %g1]asi_num, %i0 ;\ 570 ldxa [%g5 + %g2]asi_num, %i1 ;\ 571 ldxa [%g5 + %g3]asi_num, %i2 ;\ 572 ldxa [%g5 + %g4]asi_num, %i3 ;\ 573 add %g5, 32, %g5 ;\ 574 ldxa [%g5 + %g1]asi_num, %i4 ;\ 575 ldxa [%g5 + %g2]asi_num, %i5 ;\ 576 ldxa [%g5 + %g3]asi_num, %i6 ;\ 577 ldxa [%g5 + %g4]asi_num, %i7 ;\ 578 restored ;\ 579 retry ;\ 580 SKIP(31-25-TT_TRACE_L_INS) ;\ 581 ba,a,pt %xcc, fault_64bit_/**/tail ;\ 582 .empty 583 584 585#endif /* !lint */ 586 587/* 588 * SPILL_mixed spills either size window, depending on 589 * whether %sp is even or odd, to a 32-bit address space. 590 * This may only be used in conjunction with SPILL_32bit/ 591 * FILL_64bit. 592 * Clear upper 32 bits of %sp if it is odd. 593 * We won't need to clear them in 64 bit kernel. 594 */ 595#define SPILL_mixed \ 596 btst 1, %sp ;\ 597 bz,a,pt %xcc, 1b ;\ 598 srl %sp, 0, %sp ;\ 599 ba,pt %xcc, 2b ;\ 600 nop ;\ 601 .align 128 602 603/* 604 * FILL_mixed(ASI) fills either size window, depending on 605 * whether %sp is even or odd, from a 32-bit address space. 606 * This may only be used in conjunction with FILL_32bit/ 607 * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be 608 * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular 609 * attention should be paid to the instructions that belong 610 * in the delay slots of the branches depending on the type 611 * of fill handler being branched to. 612 * Clear upper 32 bits of %sp if it is odd. 613 * We won't need to clear them in 64 bit kernel. 614 */ 615#define FILL_mixed \ 616 btst 1, %sp ;\ 617 bz,a,pt %xcc, 1b ;\ 618 srl %sp, 0, %sp ;\ 619 ba,pt %xcc, 2b ;\ 620 nop ;\ 621 .align 128 622 623 624/* 625 * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows, 626 * respectively, into the address space via the designated asi. The 627 * unbiased stack pointer is required to be eight-byte aligned (even for 628 * the 32-bit case even though this code does not require such strict 629 * alignment). 630 * 631 * With SPARC v9 the spill trap takes precedence over the cleanwin trap 632 * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save 633 * will cause cwp + 2 to be spilled but will not clean cwp + 1. That 634 * window may contain kernel data so in user_rtt we set wstate to call 635 * these spill handlers on the first user spill trap. These handler then 636 * spill the appropriate window but also back up a window and clean the 637 * window that didn't get a cleanwin trap. 638 */ 639#define SPILL_32clean(asi_num, tail) \ 640 srl %sp, 0, %sp ;\ 641 sta %l0, [%sp + %g0]asi_num ;\ 642 mov 4, %g1 ;\ 643 sta %l1, [%sp + %g1]asi_num ;\ 644 mov 8, %g2 ;\ 645 sta %l2, [%sp + %g2]asi_num ;\ 646 mov 12, %g3 ;\ 647 sta %l3, [%sp + %g3]asi_num ;\ 648 add %sp, 16, %g4 ;\ 649 sta %l4, [%g4 + %g0]asi_num ;\ 650 sta %l5, [%g4 + %g1]asi_num ;\ 651 sta %l6, [%g4 + %g2]asi_num ;\ 652 sta %l7, [%g4 + %g3]asi_num ;\ 653 add %g4, 16, %g4 ;\ 654 sta %i0, [%g4 + %g0]asi_num ;\ 655 sta %i1, [%g4 + %g1]asi_num ;\ 656 sta %i2, [%g4 + %g2]asi_num ;\ 657 sta %i3, [%g4 + %g3]asi_num ;\ 658 add %g4, 16, %g4 ;\ 659 sta %i4, [%g4 + %g0]asi_num ;\ 660 sta %i5, [%g4 + %g1]asi_num ;\ 661 sta %i6, [%g4 + %g2]asi_num ;\ 662 sta %i7, [%g4 + %g3]asi_num ;\ 663 TT_TRACE_L(trace_win) ;\ 664 b .spill_clean ;\ 665 mov WSTATE_USER32, %g7 ;\ 666 SKIP(31-25-TT_TRACE_L_INS) ;\ 667 ba,a,pt %xcc, fault_32bit_/**/tail ;\ 668 .empty 669 670#define SPILL_64clean(asi_num, tail) \ 671 mov 0 + V9BIAS64, %g1 ;\ 672 stxa %l0, [%sp + %g1]asi_num ;\ 673 mov 8 + V9BIAS64, %g2 ;\ 674 stxa %l1, [%sp + %g2]asi_num ;\ 675 mov 16 + V9BIAS64, %g3 ;\ 676 stxa %l2, [%sp + %g3]asi_num ;\ 677 mov 24 + V9BIAS64, %g4 ;\ 678 stxa %l3, [%sp + %g4]asi_num ;\ 679 add %sp, 32, %g5 ;\ 680 stxa %l4, [%g5 + %g1]asi_num ;\ 681 stxa %l5, [%g5 + %g2]asi_num ;\ 682 stxa %l6, [%g5 + %g3]asi_num ;\ 683 stxa %l7, [%g5 + %g4]asi_num ;\ 684 add %g5, 32, %g5 ;\ 685 stxa %i0, [%g5 + %g1]asi_num ;\ 686 stxa %i1, [%g5 + %g2]asi_num ;\ 687 stxa %i2, [%g5 + %g3]asi_num ;\ 688 stxa %i3, [%g5 + %g4]asi_num ;\ 689 add %g5, 32, %g5 ;\ 690 stxa %i4, [%g5 + %g1]asi_num ;\ 691 stxa %i5, [%g5 + %g2]asi_num ;\ 692 stxa %i6, [%g5 + %g3]asi_num ;\ 693 stxa %i7, [%g5 + %g4]asi_num ;\ 694 TT_TRACE_L(trace_win) ;\ 695 b .spill_clean ;\ 696 mov WSTATE_USER64, %g7 ;\ 697 SKIP(31-25-TT_TRACE_L_INS) ;\ 698 ba,a,pt %xcc, fault_64bit_/**/tail ;\ 699 .empty 700 701 702/* 703 * Floating point disabled. 704 */ 705#define FP_DISABLED_TRAP \ 706 TT_TRACE(trace_gen) ;\ 707 ba,pt %xcc,.fp_disabled ;\ 708 nop ;\ 709 .align 32 710 711/* 712 * Floating point exceptions. 713 */ 714#define FP_IEEE_TRAP \ 715 TT_TRACE(trace_gen) ;\ 716 ba,pt %xcc,.fp_ieee_exception ;\ 717 nop ;\ 718 .align 32 719 720#define FP_TRAP \ 721 TT_TRACE(trace_gen) ;\ 722 ba,pt %xcc,.fp_exception ;\ 723 nop ;\ 724 .align 32 725 726#if !defined(lint) 727 728/* 729 * ECACHE_ECC error traps at level 0 and level 1 730 */ 731#define ECACHE_ECC(table_name) \ 732 .global table_name ;\ 733table_name: ;\ 734 membar #Sync ;\ 735 set trap, %g1 ;\ 736 rdpr %tt, %g3 ;\ 737 ba,pt %xcc, sys_trap ;\ 738 sub %g0, 1, %g4 ;\ 739 .align 32 740 741#endif /* !lint */ 742 743/* 744 * illegal instruction trap 745 */ 746#define ILLTRAP_INSTR \ 747 membar #Sync ;\ 748 TT_TRACE(trace_gen) ;\ 749 or %g0, P_UTRAP4, %g2 ;\ 750 or %g0, T_UNIMP_INSTR, %g3 ;\ 751 sethi %hi(.check_v9utrap), %g4 ;\ 752 jmp %g4 + %lo(.check_v9utrap) ;\ 753 nop ;\ 754 .align 32 755 756/* 757 * tag overflow trap 758 */ 759#define TAG_OVERFLOW \ 760 TT_TRACE(trace_gen) ;\ 761 or %g0, P_UTRAP10, %g2 ;\ 762 or %g0, T_TAG_OVERFLOW, %g3 ;\ 763 sethi %hi(.check_v9utrap), %g4 ;\ 764 jmp %g4 + %lo(.check_v9utrap) ;\ 765 nop ;\ 766 .align 32 767 768/* 769 * divide by zero trap 770 */ 771#define DIV_BY_ZERO \ 772 TT_TRACE(trace_gen) ;\ 773 or %g0, P_UTRAP11, %g2 ;\ 774 or %g0, T_IDIV0, %g3 ;\ 775 sethi %hi(.check_v9utrap), %g4 ;\ 776 jmp %g4 + %lo(.check_v9utrap) ;\ 777 nop ;\ 778 .align 32 779 780/* 781 * trap instruction for V9 user trap handlers 782 */ 783#define TRAP_INSTR \ 784 TT_TRACE(trace_gen) ;\ 785 or %g0, T_SOFTWARE_TRAP, %g3 ;\ 786 sethi %hi(.check_v9utrap), %g4 ;\ 787 jmp %g4 + %lo(.check_v9utrap) ;\ 788 nop ;\ 789 .align 32 790#define TRP4 TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR 791 792/* 793 * LEVEL_INTERRUPT is for level N interrupts. 794 * VECTOR_INTERRUPT is for the vector trap. 795 */ 796#define LEVEL_INTERRUPT(level) \ 797 .global tt_pil/**/level ;\ 798tt_pil/**/level: ;\ 799 ba,pt %xcc, pil_interrupt ;\ 800 mov level, %g4 ;\ 801 .align 32 802 803#define LEVEL14_INTERRUPT \ 804 ba pil14_interrupt ;\ 805 mov PIL_14, %g4 ;\ 806 .align 32 807 808#define CPU_MONDO \ 809 ba,a,pt %xcc, cpu_mondo ;\ 810 .align 32 811 812#define DEV_MONDO \ 813 ba,a,pt %xcc, dev_mondo ;\ 814 .align 32 815 816/* 817 * We take over the rtba after we set our trap table and 818 * fault status area. The watchdog reset trap is now handled by the OS. 819 */ 820#define WATCHDOG_RESET \ 821 mov PTL1_BAD_WATCHDOG, %g1 ;\ 822 ba,a,pt %xcc, .watchdog_trap ;\ 823 .align 32 824 825/* 826 * RED is for traps that use the red mode handler. 827 * We should never see these either. 828 */ 829#define RED \ 830 mov PTL1_BAD_RED, %g1 ;\ 831 ba,a,pt %xcc, .watchdog_trap ;\ 832 .align 32 833 834 835/* 836 * MMU Trap Handlers. 837 */ 838 839/* 840 * synthesize for trap(): SFSR in %g3 841 */ 842#define IMMU_EXCEPTION \ 843 MMU_FAULT_STATUS_AREA(%g3) ;\ 844 rdpr %tpc, %g2 ;\ 845 ldx [%g3 + MMFSA_I_TYPE], %g1 ;\ 846 ldx [%g3 + MMFSA_I_CTX], %g3 ;\ 847 sllx %g3, SFSR_CTX_SHIFT, %g3 ;\ 848 or %g3, %g1, %g3 ;\ 849 ba,pt %xcc, .mmu_exception_end ;\ 850 mov T_INSTR_EXCEPTION, %g1 ;\ 851 .align 32 852 853/* 854 * synthesize for trap(): TAG_ACCESS in %g2, SFSR in %g3 855 */ 856#define DMMU_EXCEPTION \ 857 ba,a,pt %xcc, .dmmu_exception ;\ 858 .align 32 859 860/* 861 * synthesize for trap(): SFAR in %g2, SFSR in %g3 862 */ 863#define DMMU_EXC_AG_PRIV \ 864 MMU_FAULT_STATUS_AREA(%g3) ;\ 865 ldx [%g3 + MMFSA_D_ADDR], %g2 ;\ 866 /* Fault type not available in MMU fault status area */ ;\ 867 mov MMFSA_F_PRVACT, %g1 ;\ 868 ldx [%g3 + MMFSA_D_CTX], %g3 ;\ 869 sllx %g3, SFSR_CTX_SHIFT, %g3 ;\ 870 ba,pt %xcc, .mmu_priv_exception ;\ 871 or %g3, %g1, %g3 ;\ 872 .align 32 873 874/* 875 * synthesize for trap(): SFAR in %g2, SFSR in %g3 876 */ 877#define DMMU_EXC_AG_NOT_ALIGNED \ 878 MMU_FAULT_STATUS_AREA(%g3) ;\ 879 ldx [%g3 + MMFSA_D_ADDR], %g2 ;\ 880 /* Fault type not available in MMU fault status area */ ;\ 881 mov MMFSA_F_UNALIGN, %g1 ;\ 882 ldx [%g3 + MMFSA_D_CTX], %g3 ;\ 883 sllx %g3, SFSR_CTX_SHIFT, %g3 ;\ 884 ba,pt %xcc, .mmu_exception_not_aligned ;\ 885 or %g3, %g1, %g3 /* SFSR */ ;\ 886 .align 32 887/* 888 * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2) 889 */ 890 891/* 892 * synthesize for trap(): SFAR in %g2, SFSR in %g3 893 */ 894#define DMMU_EXC_LDDF_NOT_ALIGNED \ 895 ba,a,pt %xcc, .dmmu_exc_lddf_not_aligned ;\ 896 .align 32 897/* 898 * synthesize for trap(): SFAR in %g2, SFSR in %g3 899 */ 900#define DMMU_EXC_STDF_NOT_ALIGNED \ 901 ba,a,pt %xcc, .dmmu_exc_stdf_not_aligned ;\ 902 .align 32 903 904#if defined(cscope) 905/* 906 * Define labels to direct cscope quickly to labels that 907 * are generated by macro expansion of DTLB_MISS(). 908 */ 909 .global tt0_dtlbmiss 910tt0_dtlbmiss: 911 .global tt1_dtlbmiss 912tt1_dtlbmiss: 913 nop 914#endif 915 916/* 917 * Data miss handler (must be exactly 32 instructions) 918 * 919 * This handler is invoked only if the hypervisor has been instructed 920 * not to do any TSB walk. 921 * 922 * Kernel and invalid context cases are handled by the sfmmu_kdtlb_miss 923 * handler. 924 * 925 * User TLB miss handling depends upon whether a user process has one or 926 * two TSBs. User TSB information (physical base and size code) is kept 927 * in two dedicated scratchpad registers. Absence of a user TSB (primarily 928 * second TSB) is indicated by a negative value (-1) in that register. 929 */ 930 931/* 932 * synthesize for miss handler: pseudo-tag access in %g2 (with context "type" 933 * (0=kernel, 1=invalid, or 2=user) rather than context ID) 934 */ 935#define DTLB_MISS(table_name) ;\ 936 .global table_name/**/_dtlbmiss ;\ 937table_name/**/_dtlbmiss: ;\ 938 HAT_PERCPU_DBSTAT(TSBMISS_DTLBMISS) /* 3 instr ifdef DEBUG */ ;\ 939 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) /* 8 instr */ ;\ 940 cmp %g3, INVALID_CONTEXT ;\ 941 ble,pn %xcc, sfmmu_kdtlb_miss ;\ 942 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ 943 mov SCRATCHPAD_UTSBREG2, %g1 ;\ 944 ldxa [%g1]ASI_SCRATCHPAD, %g1 /* get 2nd tsbreg */ ;\ 945 brgez,pn %g1, sfmmu_udtlb_slowpath /* branch if 2 TSBs */ ;\ 946 nop ;\ 947 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) /* 11 instr */ ;\ 948 ba,pt %xcc, sfmmu_udtlb_fastpath /* no 4M TSB, miss */ ;\ 949 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ 950 .align 128 951 952 953#if defined(cscope) 954/* 955 * Define labels to direct cscope quickly to labels that 956 * are generated by macro expansion of ITLB_MISS(). 957 */ 958 .global tt0_itlbmiss 959tt0_itlbmiss: 960 .global tt1_itlbmiss 961tt1_itlbmiss: 962 nop 963#endif 964 965/* 966 * Instruction miss handler. 967 * 968 * This handler is invoked only if the hypervisor has been instructed 969 * not to do any TSB walk. 970 * 971 * ldda instructions will have their ASI patched 972 * by sfmmu_patch_ktsb at runtime. 973 * MUST be EXACTLY 32 instructions or we'll break. 974 */ 975 976/* 977 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type" 978 * (0=kernel, 1=invalid, or 2=user) rather than context ID) 979 */ 980#define ITLB_MISS(table_name) \ 981 .global table_name/**/_itlbmiss ;\ 982table_name/**/_itlbmiss: ;\ 983 HAT_PERCPU_DBSTAT(TSBMISS_ITLBMISS) /* 3 instr ifdef DEBUG */ ;\ 984 GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3) /* 8 instr */ ;\ 985 cmp %g3, INVALID_CONTEXT ;\ 986 ble,pn %xcc, sfmmu_kitlb_miss ;\ 987 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ 988 mov SCRATCHPAD_UTSBREG2, %g1 ;\ 989 ldxa [%g1]ASI_SCRATCHPAD, %g1 /* get 2nd tsbreg */ ;\ 990 brgez,pn %g1, sfmmu_uitlb_slowpath /* branch if 2 TSBs */ ;\ 991 nop ;\ 992 GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5) /* 11 instr */ ;\ 993 ba,pt %xcc, sfmmu_uitlb_fastpath /* no 4M TSB, miss */ ;\ 994 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ 995 .align 128 996 997#define DTSB_MISS \ 998 GOTO_TT(sfmmu_slow_dmmu_miss,trace_dmmu) 999 1000#define ITSB_MISS \ 1001 GOTO_TT(sfmmu_slow_immu_miss,trace_immu) 1002 1003/* 1004 * This macro is the first level handler for fast protection faults. 1005 * It first demaps the tlb entry which generated the fault and then 1006 * attempts to set the modify bit on the hash. It needs to be 1007 * exactly 32 instructions. 1008 */ 1009/* 1010 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type" 1011 * (0=kernel, 1=invalid, or 2=user) rather than context ID) 1012 */ 1013#define DTLB_PROT \ 1014 GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3) /* 8 instr */ ;\ 1015 /* ;\ 1016 * g2 = pseudo-tag access register (ctx type rather than ctx ID) ;\ 1017 * g3 = ctx type (0, 1, or 2) ;\ 1018 */ ;\ 1019 TT_TRACE(trace_dataprot) /* 2 instr ifdef TRAPTRACE */ ;\ 1020 /* clobbers g1 and g6 XXXQ? */ ;\ 1021 brnz,pt %g3, sfmmu_uprot_trap /* user trap */ ;\ 1022 nop ;\ 1023 ba,a,pt %xcc, sfmmu_kprot_trap /* kernel trap */ ;\ 1024 .align 128 1025 1026#define DMMU_EXCEPTION_TL1 ;\ 1027 ba,a,pt %xcc, mmu_trap_tl1 ;\ 1028 .align 32 1029 1030#define MISALIGN_ADDR_TL1 ;\ 1031 ba,a,pt %xcc, mmu_trap_tl1 ;\ 1032 .align 32 1033 1034/* 1035 * Trace a tsb hit 1036 * g1 = tsbe pointer (in/clobbered) 1037 * g2 = tag access register (in) 1038 * g3 - g4 = scratch (clobbered) 1039 * g5 = tsbe data (in) 1040 * g6 = scratch (clobbered) 1041 * g7 = pc we jumped here from (in) 1042 * ttextra = value to OR in to trap type (%tt) (in) 1043 */ 1044#ifdef TRAPTRACE 1045#define TRACE_TSBHIT(ttextra) \ 1046 membar #Sync ;\ 1047 sethi %hi(FLUSH_ADDR), %g6 ;\ 1048 flush %g6 ;\ 1049 TRACE_PTR(%g3, %g6) ;\ 1050 GET_TRACE_TICK(%g6) ;\ 1051 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi ;\ 1052 stna %g2, [%g3 + TRAP_ENT_SP]%asi /* tag access */ ;\ 1053 stna %g5, [%g3 + TRAP_ENT_F1]%asi /* tsb data */ ;\ 1054 rdpr %tnpc, %g6 ;\ 1055 stna %g6, [%g3 + TRAP_ENT_F2]%asi ;\ 1056 stna %g1, [%g3 + TRAP_ENT_F3]%asi /* tsb pointer */ ;\ 1057 stna %g0, [%g3 + TRAP_ENT_F4]%asi ;\ 1058 rdpr %tpc, %g6 ;\ 1059 stna %g6, [%g3 + TRAP_ENT_TPC]%asi ;\ 1060 TRACE_SAVE_TL_GL_REGS(%g3, %g6) ;\ 1061 rdpr %tt, %g6 ;\ 1062 or %g6, (ttextra), %g1 ;\ 1063 stha %g1, [%g3 + TRAP_ENT_TT]%asi ;\ 1064 MMU_FAULT_STATUS_AREA(%g4) ;\ 1065 mov MMFSA_D_ADDR, %g1 ;\ 1066 cmp %g6, FAST_IMMU_MISS_TT ;\ 1067 move %xcc, MMFSA_I_ADDR, %g1 ;\ 1068 cmp %g6, T_INSTR_MMU_MISS ;\ 1069 move %xcc, MMFSA_I_ADDR, %g1 ;\ 1070 ldx [%g4 + %g1], %g1 ;\ 1071 stxa %g1, [%g3 + TRAP_ENT_TSTATE]%asi /* fault addr */ ;\ 1072 mov MMFSA_D_CTX, %g1 ;\ 1073 cmp %g6, FAST_IMMU_MISS_TT ;\ 1074 move %xcc, MMFSA_I_CTX, %g1 ;\ 1075 cmp %g6, T_INSTR_MMU_MISS ;\ 1076 move %xcc, MMFSA_I_CTX, %g1 ;\ 1077 ldx [%g4 + %g1], %g1 ;\ 1078 stna %g1, [%g3 + TRAP_ENT_TR]%asi ;\ 1079 TRACE_NEXT(%g3, %g4, %g6) 1080#else 1081#define TRACE_TSBHIT(ttextra) 1082#endif 1083 1084 1085#if defined(lint) 1086 1087struct scb trap_table; 1088struct scb scb; /* trap_table/scb are the same object */ 1089 1090#else /* lint */ 1091 1092/* 1093 * ======================================================================= 1094 * SPARC V9 TRAP TABLE 1095 * 1096 * The trap table is divided into two halves: the first half is used when 1097 * taking traps when TL=0; the second half is used when taking traps from 1098 * TL>0. Note that handlers in the second half of the table might not be able 1099 * to make the same assumptions as handlers in the first half of the table. 1100 * 1101 * Worst case trap nesting so far: 1102 * 1103 * at TL=0 client issues software trap requesting service 1104 * at TL=1 nucleus wants a register window 1105 * at TL=2 register window clean/spill/fill takes a TLB miss 1106 * at TL=3 processing TLB miss 1107 * at TL=4 handle asynchronous error 1108 * 1109 * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode". 1110 * 1111 * ======================================================================= 1112 */ 1113 .section ".text" 1114 .align 4 1115 .global trap_table, scb, trap_table0, trap_table1, etrap_table 1116 .type trap_table, #object 1117 .type trap_table0, #object 1118 .type trap_table1, #object 1119 .type scb, #object 1120trap_table: 1121scb: 1122trap_table0: 1123 /* hardware traps */ 1124 NOT; /* 000 reserved */ 1125 RED; /* 001 power on reset */ 1126 WATCHDOG_RESET; /* 002 watchdog reset */ 1127 RED; /* 003 externally initiated reset */ 1128 RED; /* 004 software initiated reset */ 1129 RED; /* 005 red mode exception */ 1130 NOT; NOT; /* 006 - 007 reserved */ 1131 IMMU_EXCEPTION; /* 008 instruction access exception */ 1132 ITSB_MISS; /* 009 instruction access MMU miss */ 1133 NOT; /* 00A reserved */ 1134 NOT; NOT4; /* 00B - 00F reserved */ 1135 ILLTRAP_INSTR; /* 010 illegal instruction */ 1136 TRAP(T_PRIV_INSTR); /* 011 privileged opcode */ 1137 TRAP(T_UNIMP_LDD); /* 012 unimplemented LDD */ 1138 TRAP(T_UNIMP_STD); /* 013 unimplemented STD */ 1139 NOT4; NOT4; NOT4; /* 014 - 01F reserved */ 1140 FP_DISABLED_TRAP; /* 020 fp disabled */ 1141 FP_IEEE_TRAP; /* 021 fp exception ieee 754 */ 1142 FP_TRAP; /* 022 fp exception other */ 1143 TAG_OVERFLOW; /* 023 tag overflow */ 1144 CLEAN_WINDOW; /* 024 - 027 clean window */ 1145 DIV_BY_ZERO; /* 028 division by zero */ 1146 NOT; /* 029 internal processor error */ 1147 NOT; NOT; NOT4; /* 02A - 02F reserved */ 1148 DMMU_EXCEPTION; /* 030 data access exception */ 1149 DTSB_MISS; /* 031 data access MMU miss */ 1150 NOT; /* 032 reserved */ 1151 NOT; /* 033 data access protection */ 1152 DMMU_EXC_AG_NOT_ALIGNED; /* 034 mem address not aligned */ 1153 DMMU_EXC_LDDF_NOT_ALIGNED; /* 035 LDDF mem address not aligned */ 1154 DMMU_EXC_STDF_NOT_ALIGNED; /* 036 STDF mem address not aligned */ 1155 DMMU_EXC_AG_PRIV; /* 037 privileged action */ 1156 NOT; /* 038 LDQF mem address not aligned */ 1157 NOT; /* 039 STQF mem address not aligned */ 1158 NOT; NOT; NOT4; /* 03A - 03F reserved */ 1159 NOT; /* 040 async data error */ 1160 LEVEL_INTERRUPT(1); /* 041 interrupt level 1 */ 1161 LEVEL_INTERRUPT(2); /* 042 interrupt level 2 */ 1162 LEVEL_INTERRUPT(3); /* 043 interrupt level 3 */ 1163 LEVEL_INTERRUPT(4); /* 044 interrupt level 4 */ 1164 LEVEL_INTERRUPT(5); /* 045 interrupt level 5 */ 1165 LEVEL_INTERRUPT(6); /* 046 interrupt level 6 */ 1166 LEVEL_INTERRUPT(7); /* 047 interrupt level 7 */ 1167 LEVEL_INTERRUPT(8); /* 048 interrupt level 8 */ 1168 LEVEL_INTERRUPT(9); /* 049 interrupt level 9 */ 1169 LEVEL_INTERRUPT(10); /* 04A interrupt level 10 */ 1170 LEVEL_INTERRUPT(11); /* 04B interrupt level 11 */ 1171 LEVEL_INTERRUPT(12); /* 04C interrupt level 12 */ 1172 LEVEL_INTERRUPT(13); /* 04D interrupt level 13 */ 1173 LEVEL14_INTERRUPT; /* 04E interrupt level 14 */ 1174 LEVEL_INTERRUPT(15); /* 04F interrupt level 15 */ 1175 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F reserved */ 1176 NOT; /* 060 interrupt vector */ 1177 GOTO(kmdb_trap); /* 061 PA watchpoint */ 1178 GOTO(kmdb_trap); /* 062 VA watchpoint */ 1179 NOT; /* 063 reserved */ 1180 ITLB_MISS(tt0); /* 064 instruction access MMU miss */ 1181 DTLB_MISS(tt0); /* 068 data access MMU miss */ 1182 DTLB_PROT; /* 06C data access protection */ 1183 NOT; /* 070 reserved */ 1184 NOT; /* 071 reserved */ 1185 NOT; /* 072 reserved */ 1186 NOT; /* 073 reserved */ 1187 NOT4; NOT4 /* 074 - 07B reserved */ 1188 CPU_MONDO; /* 07C cpu_mondo */ 1189 DEV_MONDO; /* 07D dev_mondo */ 1190 GOTO_TT(resumable_error, trace_gen); /* 07E resumable error */ 1191 GOTO_TT(nonresumable_error, trace_gen); /* 07F non-reasumable error */ 1192 NOT4; /* 080 spill 0 normal */ 1193 SPILL_32bit_asi(ASI_AIUP,sn0); /* 084 spill 1 normal */ 1194 SPILL_64bit_asi(ASI_AIUP,sn0); /* 088 spill 2 normal */ 1195 SPILL_32clean(ASI_AIUP,sn0); /* 08C spill 3 normal */ 1196 SPILL_64clean(ASI_AIUP,sn0); /* 090 spill 4 normal */ 1197 SPILL_32bit(not); /* 094 spill 5 normal */ 1198 SPILL_64bit(not); /* 098 spill 6 normal */ 1199 SPILL_mixed; /* 09C spill 7 normal */ 1200 NOT4; /* 0A0 spill 0 other */ 1201 SPILL_32bit_asi(ASI_AIUS,so0); /* 0A4 spill 1 other */ 1202 SPILL_64bit_asi(ASI_AIUS,so0); /* 0A8 spill 2 other */ 1203 SPILL_32bit_asi(ASI_AIUS,so0); /* 0AC spill 3 other */ 1204 SPILL_64bit_asi(ASI_AIUS,so0); /* 0B0 spill 4 other */ 1205 NOT4; /* 0B4 spill 5 other */ 1206 NOT4; /* 0B8 spill 6 other */ 1207 NOT4; /* 0BC spill 7 other */ 1208 NOT4; /* 0C0 fill 0 normal */ 1209 FILL_32bit_asi(ASI_AIUP,fn0); /* 0C4 fill 1 normal */ 1210 FILL_64bit_asi(ASI_AIUP,fn0); /* 0C8 fill 2 normal */ 1211 FILL_32bit_asi(ASI_AIUP,fn0); /* 0CC fill 3 normal */ 1212 FILL_64bit_asi(ASI_AIUP,fn0); /* 0D0 fill 4 normal */ 1213 FILL_32bit(not); /* 0D4 fill 5 normal */ 1214 FILL_64bit(not); /* 0D8 fill 6 normal */ 1215 FILL_mixed; /* 0DC fill 7 normal */ 1216 NOT4; /* 0E0 fill 0 other */ 1217 NOT4; /* 0E4 fill 1 other */ 1218 NOT4; /* 0E8 fill 2 other */ 1219 NOT4; /* 0EC fill 3 other */ 1220 NOT4; /* 0F0 fill 4 other */ 1221 NOT4; /* 0F4 fill 5 other */ 1222 NOT4; /* 0F8 fill 6 other */ 1223 NOT4; /* 0FC fill 7 other */ 1224 /* user traps */ 1225 GOTO(syscall_trap_4x); /* 100 old system call */ 1226 TRAP(T_BREAKPOINT); /* 101 user breakpoint */ 1227 TRAP(T_DIV0); /* 102 user divide by zero */ 1228 GOTO(.flushw); /* 103 flush windows */ 1229 GOTO(.clean_windows); /* 104 clean windows */ 1230 BAD; /* 105 range check ?? */ 1231 GOTO(.fix_alignment); /* 106 do unaligned references */ 1232 BAD; /* 107 unused */ 1233#ifndef DEBUG 1234 SYSCALL(syscall_trap32); /* 108 ILP32 system call on LP64 */ 1235#else /* DEBUG */ 1236 GOTO(syscall_wrapper32); /* 108 ILP32 system call on LP64 */ 1237#endif /* DEBUG */ 1238 GOTO(set_trap0_addr); /* 109 set trap0 address */ 1239 BAD; BAD; BAD4; /* 10A - 10F unused */ 1240 TRP4; TRP4; TRP4; TRP4; /* 110 - 11F V9 user trap handlers */ 1241 GOTO(.getcc); /* 120 get condition codes */ 1242 GOTO(.setcc); /* 121 set condition codes */ 1243 GOTO(.getpsr); /* 122 get psr */ 1244 GOTO(.setpsr); /* 123 set psr (some fields) */ 1245 GOTO(get_timestamp); /* 124 get timestamp */ 1246 GOTO(get_virtime); /* 125 get lwp virtual time */ 1247 PRIV(self_xcall); /* 126 self xcall */ 1248 GOTO(get_hrestime); /* 127 get hrestime */ 1249 BAD; /* 128 ST_SETV9STACK */ 1250 GOTO(.getlgrp); /* 129 get lgrpid */ 1251 BAD; BAD; BAD4; /* 12A - 12F unused */ 1252 BAD4; BAD4; /* 130 - 137 unused */ 1253 DTRACE_PID; /* 138 dtrace pid tracing provider */ 1254 BAD; /* 139 unused */ 1255 DTRACE_RETURN; /* 13A dtrace pid return probe */ 1256 BAD; BAD4; /* 13B - 13F unused */ 1257#ifndef DEBUG 1258 SYSCALL(syscall_trap); /* 140 LP64 system call */ 1259#else /* DEBUG */ 1260 GOTO(syscall_wrapper); /* 140 LP64 system call */ 1261#endif /* DEBUG */ 1262 SYSCALL(nosys); /* 141 unused system call trap */ 1263#ifdef DEBUG_USER_TRAPTRACECTL 1264 GOTO(.traptrace_freeze); /* 142 freeze traptrace */ 1265 GOTO(.traptrace_unfreeze); /* 143 unfreeze traptrace */ 1266#else 1267 SYSCALL(nosys); /* 142 unused system call trap */ 1268 SYSCALL(nosys); /* 143 unused system call trap */ 1269#endif 1270 BAD4; BAD4; BAD4; /* 144 - 14F unused */ 1271 BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */ 1272 BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */ 1273 BAD; /* 170 - unused */ 1274 BAD; /* 171 - unused */ 1275 BAD; BAD; /* 172 - 173 unused */ 1276 BAD4; BAD4; /* 174 - 17B unused */ 1277#ifdef PTL1_PANIC_DEBUG 1278 mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic); 1279 /* 17C test ptl1_panic */ 1280#else 1281 BAD; /* 17C unused */ 1282#endif /* PTL1_PANIC_DEBUG */ 1283 PRIV(kmdb_trap); /* 17D kmdb enter (L1-A) */ 1284 PRIV(kmdb_trap); /* 17E kmdb breakpoint */ 1285 PRIV(obp_bpt); /* 17F obp breakpoint */ 1286 /* reserved */ 1287 NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */ 1288 NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */ 1289 NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */ 1290 NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */ 1291 NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */ 1292 NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */ 1293 NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */ 1294 NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */ 1295 .size trap_table0, (.-trap_table0) 1296trap_table1: 1297 NOT4; NOT4; /* 000 - 007 unused */ 1298 NOT; /* 008 instruction access exception */ 1299 ITSB_MISS; /* 009 instruction access MMU miss */ 1300 NOT; /* 00A reserved */ 1301 NOT; NOT4; /* 00B - 00F unused */ 1302 NOT4; NOT4; NOT4; NOT4; /* 010 - 01F unused */ 1303 NOT4; /* 020 - 023 unused */ 1304 CLEAN_WINDOW; /* 024 - 027 clean window */ 1305 NOT4; NOT4; /* 028 - 02F unused */ 1306 DMMU_EXCEPTION_TL1; /* 030 data access exception */ 1307 DTSB_MISS; /* 031 data access MMU miss */ 1308 NOT; /* 032 reserved */ 1309 NOT; /* 033 unused */ 1310 MISALIGN_ADDR_TL1; /* 034 mem address not aligned */ 1311 NOT; NOT; NOT; NOT4; NOT4 /* 035 - 03F unused */ 1312 NOT4; NOT4; NOT4; NOT4; /* 040 - 04F unused */ 1313 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F unused */ 1314 NOT; /* 060 unused */ 1315 GOTO(kmdb_trap_tl1); /* 061 PA watchpoint */ 1316 GOTO(kmdb_trap_tl1); /* 062 VA watchpoint */ 1317 NOT; /* 063 reserved */ 1318 ITLB_MISS(tt1); /* 064 instruction access MMU miss */ 1319 DTLB_MISS(tt1); /* 068 data access MMU miss */ 1320 DTLB_PROT; /* 06C data access protection */ 1321 NOT; /* 070 reserved */ 1322 NOT; /* 071 reserved */ 1323 NOT; /* 072 reserved */ 1324 NOT; /* 073 reserved */ 1325 NOT4; NOT4; /* 074 - 07B reserved */ 1326 NOT; /* 07C reserved */ 1327 NOT; /* 07D reserved */ 1328 NOT; /* 07E resumable error */ 1329 GOTO_TT(nonresumable_error, trace_gen); /* 07F nonresumable error */ 1330 NOTP4; /* 080 spill 0 normal */ 1331 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 084 spill 1 normal */ 1332 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 088 spill 2 normal */ 1333 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 08C spill 3 normal */ 1334 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 090 spill 4 normal */ 1335 NOTP4; /* 094 spill 5 normal */ 1336 SPILL_64bit_ktt1(sk); /* 098 spill 6 normal */ 1337 SPILL_mixed_ktt1(sk); /* 09C spill 7 normal */ 1338 NOTP4; /* 0A0 spill 0 other */ 1339 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0A4 spill 1 other */ 1340 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0A8 spill 2 other */ 1341 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0AC spill 3 other */ 1342 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0B0 spill 4 other */ 1343 NOTP4; /* 0B4 spill 5 other */ 1344 NOTP4; /* 0B8 spill 6 other */ 1345 NOTP4; /* 0BC spill 7 other */ 1346 NOT4; /* 0C0 fill 0 normal */ 1347 NOT4; /* 0C4 fill 1 normal */ 1348 NOT4; /* 0C8 fill 2 normal */ 1349 NOT4; /* 0CC fill 3 normal */ 1350 NOT4; /* 0D0 fill 4 normal */ 1351 NOT4; /* 0D4 fill 5 normal */ 1352 NOT4; /* 0D8 fill 6 normal */ 1353 NOT4; /* 0DC fill 7 normal */ 1354 NOT4; NOT4; NOT4; NOT4; /* 0E0 - 0EF unused */ 1355 NOT4; NOT4; NOT4; NOT4; /* 0F0 - 0FF unused */ 1356/* 1357 * Code running at TL>0 does not use soft traps, so 1358 * we can truncate the table here. 1359 * However: 1360 * sun4v uses (hypervisor) ta instructions at TL > 0, so 1361 * provide a safety net for now. 1362 */ 1363 /* soft traps */ 1364 BAD4; BAD4; BAD4; BAD4; /* 100 - 10F unused */ 1365 BAD4; BAD4; BAD4; BAD4; /* 110 - 11F unused */ 1366 BAD4; BAD4; BAD4; BAD4; /* 120 - 12F unused */ 1367 BAD4; BAD4; BAD4; BAD4; /* 130 - 13F unused */ 1368 BAD4; BAD4; BAD4; BAD4; /* 140 - 14F unused */ 1369 BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */ 1370 BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */ 1371 BAD4; BAD4; BAD4; BAD4; /* 170 - 17F unused */ 1372 /* reserved */ 1373 NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */ 1374 NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */ 1375 NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */ 1376 NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */ 1377 NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */ 1378 NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */ 1379 NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */ 1380 NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */ 1381etrap_table: 1382 .size trap_table1, (.-trap_table1) 1383 .size trap_table, (.-trap_table) 1384 .size scb, (.-scb) 1385 1386/* 1387 * We get to exec_fault in the case of an instruction miss and tte 1388 * has no execute bit set. We go to tl0 to handle it. 1389 * 1390 * g1 = tsbe pointer (in/clobbered) 1391 * g2 = tag access register (in) 1392 * g3 - g4 = scratch (clobbered) 1393 * g5 = tsbe data (in) 1394 * g6 = scratch (clobbered) 1395 * g7 = pc we jumped here from (in) 1396 */ 1397/* 1398 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type" 1399 * (0=kernel, 1=invalid, or 2=user) rather than context ID) 1400 */ 1401 ALTENTRY(exec_fault) 1402 TRACE_TSBHIT(TT_MMU_EXEC) 1403 MMU_FAULT_STATUS_AREA(%g4) 1404 ldx [%g4 + MMFSA_I_ADDR], %g2 /* g2 = address */ 1405 ldx [%g4 + MMFSA_I_CTX], %g3 /* g3 = ctx */ 1406 srlx %g2, MMU_PAGESHIFT, %g2 ! align address to page boundry 1407 cmp %g3, USER_CONTEXT_TYPE 1408 sllx %g2, MMU_PAGESHIFT, %g2 1409 movgu %icc, USER_CONTEXT_TYPE, %g3 1410 or %g2, %g3, %g2 /* TAG_ACCESS */ 1411 mov T_INSTR_MMU_MISS, %g3 ! arg2 = traptype 1412 set trap, %g1 1413 ba,pt %xcc, sys_trap 1414 mov -1, %g4 1415 1416.mmu_exception_not_aligned: 1417 /* %g2 = sfar, %g3 = sfsr */ 1418 rdpr %tstate, %g1 1419 btst TSTATE_PRIV, %g1 1420 bnz,pn %icc, 2f 1421 nop 1422 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1423 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1424 ldn [%g1 + T_PROCP], %g1 ! load proc pointer 1425 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? 1426 brz,pt %g5, 2f 1427 nop 1428 ldn [%g5 + P_UTRAP15], %g5 ! unaligned utrap? 1429 brz,pn %g5, 2f 1430 nop 1431 btst 1, %sp 1432 bz,pt %xcc, 1f ! 32 bit user program 1433 nop 1434 ba,pt %xcc, .setup_v9utrap ! 64 bit user program 1435 nop 14361: 1437 ba,pt %xcc, .setup_utrap 1438 or %g2, %g0, %g7 14392: 1440 ba,pt %xcc, .mmu_exception_end 1441 mov T_ALIGNMENT, %g1 1442 1443.mmu_priv_exception: 1444 rdpr %tstate, %g1 1445 btst TSTATE_PRIV, %g1 1446 bnz,pn %icc, 1f 1447 nop 1448 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1449 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1450 ldn [%g1 + T_PROCP], %g1 ! load proc pointer 1451 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? 1452 brz,pt %g5, 1f 1453 nop 1454 ldn [%g5 + P_UTRAP16], %g5 1455 brnz,pt %g5, .setup_v9utrap 1456 nop 14571: 1458 mov T_PRIV_INSTR, %g1 1459 1460.mmu_exception_end: 1461 CPU_INDEX(%g4, %g5) 1462 set cpu_core, %g5 1463 sllx %g4, CPU_CORE_SHIFT, %g4 1464 add %g4, %g5, %g4 1465 lduh [%g4 + CPUC_DTRACE_FLAGS], %g5 1466 andcc %g5, CPU_DTRACE_NOFAULT, %g0 1467 bz 1f 1468 or %g5, CPU_DTRACE_BADADDR, %g5 1469 stuh %g5, [%g4 + CPUC_DTRACE_FLAGS] 1470 done 1471 14721: 1473 sllx %g3, 32, %g3 1474 or %g3, %g1, %g3 1475 set trap, %g1 1476 ba,pt %xcc, sys_trap 1477 sub %g0, 1, %g4 1478 1479.fp_disabled: 1480 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1481 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1482 rdpr %tstate, %g4 1483 btst TSTATE_PRIV, %g4 1484 bnz,a,pn %icc, ptl1_panic 1485 mov PTL1_BAD_FPTRAP, %g1 1486 1487 ldn [%g1 + T_PROCP], %g1 ! load proc pointer 1488 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? 1489 brz,a,pt %g5, 2f 1490 nop 1491 ldn [%g5 + P_UTRAP7], %g5 ! fp_disabled utrap? 1492 brz,a,pn %g5, 2f 1493 nop 1494 btst 1, %sp 1495 bz,a,pt %xcc, 1f ! 32 bit user program 1496 nop 1497 ba,a,pt %xcc, .setup_v9utrap ! 64 bit user program 1498 nop 14991: 1500 ba,pt %xcc, .setup_utrap 1501 or %g0, %g0, %g7 15022: 1503 set fp_disabled, %g1 1504 ba,pt %xcc, sys_trap 1505 sub %g0, 1, %g4 1506 1507.fp_ieee_exception: 1508 rdpr %tstate, %g1 1509 btst TSTATE_PRIV, %g1 1510 bnz,a,pn %icc, ptl1_panic 1511 mov PTL1_BAD_FPTRAP, %g1 1512 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1513 stx %fsr, [%g1 + CPU_TMP1] 1514 ldx [%g1 + CPU_TMP1], %g2 1515 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1516 ldn [%g1 + T_PROCP], %g1 ! load proc pointer 1517 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? 1518 brz,a,pt %g5, 1f 1519 nop 1520 ldn [%g5 + P_UTRAP8], %g5 1521 brnz,a,pt %g5, .setup_v9utrap 1522 nop 15231: 1524 set _fp_ieee_exception, %g1 1525 ba,pt %xcc, sys_trap 1526 sub %g0, 1, %g4 1527 1528/* 1529 * Register Inputs: 1530 * %g5 user trap handler 1531 * %g7 misaligned addr - for alignment traps only 1532 */ 1533.setup_utrap: 1534 set trap, %g1 ! setup in case we go 1535 mov T_FLUSH_PCB, %g3 ! through sys_trap on 1536 sub %g0, 1, %g4 ! the save instruction below 1537 1538 /* 1539 * If the DTrace pid provider is single stepping a copied-out 1540 * instruction, t->t_dtrace_step will be set. In that case we need 1541 * to abort the single-stepping (since execution of the instruction 1542 * was interrupted) and use the value of t->t_dtrace_npc as the %npc. 1543 */ 1544 save %sp, -SA(MINFRAME32), %sp ! window for trap handler 1545 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1546 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1547 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step 1548 rdpr %tnpc, %l2 ! arg1 == tnpc 1549 brz,pt %g2, 1f 1550 rdpr %tpc, %l1 ! arg0 == tpc 1551 1552 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast 1553 ldn [%g1 + T_DTRACE_NPC], %l2 ! arg1 = t->t_dtrace_npc (step) 1554 brz,pt %g2, 1f 1555 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags 1556 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast 15571: 1558 mov %g7, %l3 ! arg2 == misaligned address 1559 1560 rdpr %tstate, %g1 ! cwp for trap handler 1561 rdpr %cwp, %g4 1562 bclr TSTATE_CWP_MASK, %g1 1563 wrpr %g1, %g4, %tstate 1564 wrpr %g0, %g5, %tnpc ! trap handler address 1565 FAST_TRAP_DONE 1566 /* NOTREACHED */ 1567 1568.check_v9utrap: 1569 rdpr %tstate, %g1 1570 btst TSTATE_PRIV, %g1 1571 bnz,a,pn %icc, 3f 1572 nop 1573 CPU_ADDR(%g4, %g1) ! load CPU struct addr 1574 ldn [%g4 + CPU_THREAD], %g5 ! load thread pointer 1575 ldn [%g5 + T_PROCP], %g5 ! load proc pointer 1576 ldn [%g5 + P_UTRAPS], %g5 ! are there utraps? 1577 1578 cmp %g3, T_SOFTWARE_TRAP 1579 bne,a,pt %icc, 1f 1580 nop 1581 1582 brz,pt %g5, 3f ! if p_utraps == NULL goto trap() 1583 rdpr %tt, %g3 ! delay - get actual hw trap type 1584 1585 sub %g3, 254, %g1 ! UT_TRAP_INSTRUCTION_16 = p_utraps[18] 1586 ba,pt %icc, 2f 1587 smul %g1, CPTRSIZE, %g2 15881: 1589 brz,a,pt %g5, 3f ! if p_utraps == NULL goto trap() 1590 nop 1591 1592 cmp %g3, T_UNIMP_INSTR 1593 bne,a,pt %icc, 2f 1594 nop 1595 1596 mov 1, %g1 1597 st %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR 1598 rdpr %tpc, %g1 ! ld trapping instruction using 1599 lduwa [%g1]ASI_AIUP, %g1 ! "AS IF USER" ASI which could fault 1600 st %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR 1601 1602 sethi %hi(0xc1c00000), %g4 ! setup mask for illtrap instruction 1603 andcc %g1, %g4, %g4 ! and instruction with mask 1604 bnz,a,pt %icc, 3f ! if %g4 == zero, %g1 is an ILLTRAP 1605 nop ! fall thru to setup 16062: 1607 ldn [%g5 + %g2], %g5 1608 brnz,a,pt %g5, .setup_v9utrap 1609 nop 16103: 1611 set trap, %g1 1612 ba,pt %xcc, sys_trap 1613 sub %g0, 1, %g4 1614 /* NOTREACHED */ 1615 1616/* 1617 * Register Inputs: 1618 * %g5 user trap handler 1619 */ 1620.setup_v9utrap: 1621 set trap, %g1 ! setup in case we go 1622 mov T_FLUSH_PCB, %g3 ! through sys_trap on 1623 sub %g0, 1, %g4 ! the save instruction below 1624 1625 /* 1626 * If the DTrace pid provider is single stepping a copied-out 1627 * instruction, t->t_dtrace_step will be set. In that case we need 1628 * to abort the single-stepping (since execution of the instruction 1629 * was interrupted) and use the value of t->t_dtrace_npc as the %npc. 1630 */ 1631 save %sp, -SA(MINFRAME64), %sp ! window for trap handler 1632 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1633 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1634 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step 1635 rdpr %tnpc, %l7 ! arg1 == tnpc 1636 brz,pt %g2, 1f 1637 rdpr %tpc, %l6 ! arg0 == tpc 1638 1639 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast 1640 ldn [%g1 + T_DTRACE_NPC], %l7 ! arg1 == t->t_dtrace_npc (step) 1641 brz,pt %g2, 1f 1642 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags 1643 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast 16441: 1645 rdpr %tstate, %g2 ! cwp for trap handler 1646 rdpr %cwp, %g4 1647 bclr TSTATE_CWP_MASK, %g2 1648 wrpr %g2, %g4, %tstate 1649 1650 ldn [%g1 + T_PROCP], %g4 ! load proc pointer 1651 ldn [%g4 + P_AS], %g4 ! load as pointer 1652 ldn [%g4 + A_USERLIMIT], %g4 ! load as userlimit 1653 cmp %l7, %g4 ! check for single-step set 1654 bne,pt %xcc, 4f 1655 nop 1656 ldn [%g1 + T_LWP], %g1 ! load klwp pointer 1657 ld [%g1 + PCB_STEP], %g4 ! load single-step flag 1658 cmp %g4, STEP_ACTIVE ! step flags set in pcb? 1659 bne,pt %icc, 4f 1660 nop 1661 stn %g5, [%g1 + PCB_TRACEPC] ! save trap handler addr in pcb 1662 mov %l7, %g4 ! on entry to precise user trap 1663 add %l6, 4, %l7 ! handler, %l6 == pc, %l7 == npc 1664 ! at time of trap 1665 wrpr %g0, %g4, %tnpc ! generate FLTBOUNDS, 1666 ! %g4 == userlimit 1667 FAST_TRAP_DONE 1668 /* NOTREACHED */ 16694: 1670 wrpr %g0, %g5, %tnpc ! trap handler address 1671 FAST_TRAP_DONE_CHK_INTR 1672 /* NOTREACHED */ 1673 1674.fp_exception: 1675 CPU_ADDR(%g1, %g4) 1676 stx %fsr, [%g1 + CPU_TMP1] 1677 ldx [%g1 + CPU_TMP1], %g2 1678 1679 /* 1680 * Cheetah takes unfinished_FPop trap for certain range of operands 1681 * to the "fitos" instruction. Instead of going through the slow 1682 * software emulation path, we try to simulate the "fitos" instruction 1683 * via "fitod" and "fdtos" provided the following conditions are met: 1684 * 1685 * fpu_exists is set (if DEBUG) 1686 * not in privileged mode 1687 * ftt is unfinished_FPop 1688 * NXM IEEE trap is not enabled 1689 * instruction at %tpc is "fitos" 1690 * 1691 * Usage: 1692 * %g1 per cpu address 1693 * %g2 %fsr 1694 * %g6 user instruction 1695 * 1696 * Note that we can take a memory access related trap while trying 1697 * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR 1698 * flag to catch those traps and let the SFMMU code deal with page 1699 * fault and data access exception. 1700 */ 1701#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 1702 sethi %hi(fpu_exists), %g7 1703 ld [%g7 + %lo(fpu_exists)], %g7 1704 brz,pn %g7, .fp_exception_cont 1705 nop 1706#endif 1707 rdpr %tstate, %g7 ! branch if in privileged mode 1708 btst TSTATE_PRIV, %g7 1709 bnz,pn %xcc, .fp_exception_cont 1710 srl %g2, FSR_FTT_SHIFT, %g7 ! extract ftt from %fsr 1711 and %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7 1712 cmp %g7, FTT_UNFIN 1713 set FSR_TEM_NX, %g5 1714 bne,pn %xcc, .fp_exception_cont ! branch if NOT unfinished_FPop 1715 andcc %g2, %g5, %g0 1716 bne,pn %xcc, .fp_exception_cont ! branch if FSR_TEM_NX enabled 1717 rdpr %tpc, %g5 ! get faulting PC 1718 1719 or %g0, 1, %g7 1720 st %g7, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag 1721 lda [%g5]ASI_USER, %g6 ! get user's instruction 1722 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 1723 1724 set FITOS_INSTR_MASK, %g7 1725 and %g6, %g7, %g7 1726 set FITOS_INSTR, %g5 1727 cmp %g7, %g5 1728 bne,pn %xcc, .fp_exception_cont ! branch if not FITOS_INSTR 1729 nop 1730 1731 /* 1732 * This is unfinished FPops trap for "fitos" instruction. We 1733 * need to simulate "fitos" via "fitod" and "fdtos" instruction 1734 * sequence. 1735 * 1736 * We need a temporary FP register to do the conversion. Since 1737 * both source and destination operands for the "fitos" instruction 1738 * have to be within %f0-%f31, we use an FP register from the upper 1739 * half to guarantee that it won't collide with the source or the 1740 * dest operand. However, we do have to save and restore its value. 1741 * 1742 * We use %d62 as a temporary FP register for the conversion and 1743 * branch to appropriate instruction within the conversion tables 1744 * based upon the rs2 and rd values. 1745 */ 1746 1747 std %d62, [%g1 + CPU_TMP1] ! save original value 1748 1749 srl %g6, FITOS_RS2_SHIFT, %g7 1750 and %g7, FITOS_REG_MASK, %g7 1751 set _fitos_fitod_table, %g4 1752 sllx %g7, 2, %g7 1753 jmp %g4 + %g7 1754 ba,pt %xcc, _fitos_fitod_done 1755 .empty 1756 1757_fitos_fitod_table: 1758 fitod %f0, %d62 1759 fitod %f1, %d62 1760 fitod %f2, %d62 1761 fitod %f3, %d62 1762 fitod %f4, %d62 1763 fitod %f5, %d62 1764 fitod %f6, %d62 1765 fitod %f7, %d62 1766 fitod %f8, %d62 1767 fitod %f9, %d62 1768 fitod %f10, %d62 1769 fitod %f11, %d62 1770 fitod %f12, %d62 1771 fitod %f13, %d62 1772 fitod %f14, %d62 1773 fitod %f15, %d62 1774 fitod %f16, %d62 1775 fitod %f17, %d62 1776 fitod %f18, %d62 1777 fitod %f19, %d62 1778 fitod %f20, %d62 1779 fitod %f21, %d62 1780 fitod %f22, %d62 1781 fitod %f23, %d62 1782 fitod %f24, %d62 1783 fitod %f25, %d62 1784 fitod %f26, %d62 1785 fitod %f27, %d62 1786 fitod %f28, %d62 1787 fitod %f29, %d62 1788 fitod %f30, %d62 1789 fitod %f31, %d62 1790_fitos_fitod_done: 1791 1792 /* 1793 * Now convert data back into single precision 1794 */ 1795 srl %g6, FITOS_RD_SHIFT, %g7 1796 and %g7, FITOS_REG_MASK, %g7 1797 set _fitos_fdtos_table, %g4 1798 sllx %g7, 2, %g7 1799 jmp %g4 + %g7 1800 ba,pt %xcc, _fitos_fdtos_done 1801 .empty 1802 1803_fitos_fdtos_table: 1804 fdtos %d62, %f0 1805 fdtos %d62, %f1 1806 fdtos %d62, %f2 1807 fdtos %d62, %f3 1808 fdtos %d62, %f4 1809 fdtos %d62, %f5 1810 fdtos %d62, %f6 1811 fdtos %d62, %f7 1812 fdtos %d62, %f8 1813 fdtos %d62, %f9 1814 fdtos %d62, %f10 1815 fdtos %d62, %f11 1816 fdtos %d62, %f12 1817 fdtos %d62, %f13 1818 fdtos %d62, %f14 1819 fdtos %d62, %f15 1820 fdtos %d62, %f16 1821 fdtos %d62, %f17 1822 fdtos %d62, %f18 1823 fdtos %d62, %f19 1824 fdtos %d62, %f20 1825 fdtos %d62, %f21 1826 fdtos %d62, %f22 1827 fdtos %d62, %f23 1828 fdtos %d62, %f24 1829 fdtos %d62, %f25 1830 fdtos %d62, %f26 1831 fdtos %d62, %f27 1832 fdtos %d62, %f28 1833 fdtos %d62, %f29 1834 fdtos %d62, %f30 1835 fdtos %d62, %f31 1836_fitos_fdtos_done: 1837 1838 ldd [%g1 + CPU_TMP1], %d62 ! restore %d62 1839 1840#if DEBUG 1841 /* 1842 * Update FPop_unfinished trap kstat 1843 */ 1844 set fpustat+FPUSTAT_UNFIN_KSTAT, %g7 1845 ldx [%g7], %g5 18461: 1847 add %g5, 1, %g6 1848 1849 casxa [%g7] ASI_N, %g5, %g6 1850 cmp %g5, %g6 1851 bne,a,pn %xcc, 1b 1852 or %g0, %g6, %g5 1853 1854 /* 1855 * Update fpu_sim_fitos kstat 1856 */ 1857 set fpuinfo+FPUINFO_FITOS_KSTAT, %g7 1858 ldx [%g7], %g5 18591: 1860 add %g5, 1, %g6 1861 1862 casxa [%g7] ASI_N, %g5, %g6 1863 cmp %g5, %g6 1864 bne,a,pn %xcc, 1b 1865 or %g0, %g6, %g5 1866#endif /* DEBUG */ 1867 1868 FAST_TRAP_DONE 1869 1870.fp_exception_cont: 1871 /* 1872 * Let _fp_exception deal with simulating FPop instruction. 1873 * Note that we need to pass %fsr in %g2 (already read above). 1874 */ 1875 1876 set _fp_exception, %g1 1877 ba,pt %xcc, sys_trap 1878 sub %g0, 1, %g4 1879 1880 1881/* 1882 * Register windows 1883 */ 1884.flushw: 1885.clean_windows: 1886 rdpr %tnpc, %g1 1887 wrpr %g1, %tpc 1888 add %g1, 4, %g1 1889 wrpr %g1, %tnpc 1890 set trap, %g1 1891 mov T_FLUSH_PCB, %g3 1892 ba,pt %xcc, sys_trap 1893 sub %g0, 1, %g4 1894 1895/* 1896 * .spill_clean: clean the previous window, restore the wstate, and 1897 * "done". 1898 * 1899 * Entry: %g7 contains new wstate 1900 */ 1901.spill_clean: 1902 sethi %hi(nwin_minus_one), %g5 1903 ld [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1 1904 rdpr %cwp, %g6 ! %g6 = %cwp 1905 deccc %g6 ! %g6-- 1906 movneg %xcc, %g5, %g6 ! if (%g6<0) %g6 = nwin-1 1907 wrpr %g6, %cwp 1908 TT_TRACE_L(trace_win) 1909 clr %l0 1910 clr %l1 1911 clr %l2 1912 clr %l3 1913 clr %l4 1914 clr %l5 1915 clr %l6 1916 clr %l7 1917 wrpr %g0, %g7, %wstate 1918 saved 1919 retry ! restores correct %cwp 1920 1921.fix_alignment: 1922 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 1923 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1924 ldn [%g1 + T_PROCP], %g1 1925 mov 1, %g2 1926 stb %g2, [%g1 + P_FIXALIGNMENT] 1927 FAST_TRAP_DONE 1928 1929#define STDF_REG(REG, ADDR, TMP) \ 1930 sll REG, 3, REG ;\ 1931mark1: set start1, TMP ;\ 1932 jmp REG + TMP ;\ 1933 nop ;\ 1934start1: ba,pt %xcc, done1 ;\ 1935 std %f0, [ADDR + CPU_TMP1] ;\ 1936 ba,pt %xcc, done1 ;\ 1937 std %f32, [ADDR + CPU_TMP1] ;\ 1938 ba,pt %xcc, done1 ;\ 1939 std %f2, [ADDR + CPU_TMP1] ;\ 1940 ba,pt %xcc, done1 ;\ 1941 std %f34, [ADDR + CPU_TMP1] ;\ 1942 ba,pt %xcc, done1 ;\ 1943 std %f4, [ADDR + CPU_TMP1] ;\ 1944 ba,pt %xcc, done1 ;\ 1945 std %f36, [ADDR + CPU_TMP1] ;\ 1946 ba,pt %xcc, done1 ;\ 1947 std %f6, [ADDR + CPU_TMP1] ;\ 1948 ba,pt %xcc, done1 ;\ 1949 std %f38, [ADDR + CPU_TMP1] ;\ 1950 ba,pt %xcc, done1 ;\ 1951 std %f8, [ADDR + CPU_TMP1] ;\ 1952 ba,pt %xcc, done1 ;\ 1953 std %f40, [ADDR + CPU_TMP1] ;\ 1954 ba,pt %xcc, done1 ;\ 1955 std %f10, [ADDR + CPU_TMP1] ;\ 1956 ba,pt %xcc, done1 ;\ 1957 std %f42, [ADDR + CPU_TMP1] ;\ 1958 ba,pt %xcc, done1 ;\ 1959 std %f12, [ADDR + CPU_TMP1] ;\ 1960 ba,pt %xcc, done1 ;\ 1961 std %f44, [ADDR + CPU_TMP1] ;\ 1962 ba,pt %xcc, done1 ;\ 1963 std %f14, [ADDR + CPU_TMP1] ;\ 1964 ba,pt %xcc, done1 ;\ 1965 std %f46, [ADDR + CPU_TMP1] ;\ 1966 ba,pt %xcc, done1 ;\ 1967 std %f16, [ADDR + CPU_TMP1] ;\ 1968 ba,pt %xcc, done1 ;\ 1969 std %f48, [ADDR + CPU_TMP1] ;\ 1970 ba,pt %xcc, done1 ;\ 1971 std %f18, [ADDR + CPU_TMP1] ;\ 1972 ba,pt %xcc, done1 ;\ 1973 std %f50, [ADDR + CPU_TMP1] ;\ 1974 ba,pt %xcc, done1 ;\ 1975 std %f20, [ADDR + CPU_TMP1] ;\ 1976 ba,pt %xcc, done1 ;\ 1977 std %f52, [ADDR + CPU_TMP1] ;\ 1978 ba,pt %xcc, done1 ;\ 1979 std %f22, [ADDR + CPU_TMP1] ;\ 1980 ba,pt %xcc, done1 ;\ 1981 std %f54, [ADDR + CPU_TMP1] ;\ 1982 ba,pt %xcc, done1 ;\ 1983 std %f24, [ADDR + CPU_TMP1] ;\ 1984 ba,pt %xcc, done1 ;\ 1985 std %f56, [ADDR + CPU_TMP1] ;\ 1986 ba,pt %xcc, done1 ;\ 1987 std %f26, [ADDR + CPU_TMP1] ;\ 1988 ba,pt %xcc, done1 ;\ 1989 std %f58, [ADDR + CPU_TMP1] ;\ 1990 ba,pt %xcc, done1 ;\ 1991 std %f28, [ADDR + CPU_TMP1] ;\ 1992 ba,pt %xcc, done1 ;\ 1993 std %f60, [ADDR + CPU_TMP1] ;\ 1994 ba,pt %xcc, done1 ;\ 1995 std %f30, [ADDR + CPU_TMP1] ;\ 1996 ba,pt %xcc, done1 ;\ 1997 std %f62, [ADDR + CPU_TMP1] ;\ 1998done1: 1999 2000#define LDDF_REG(REG, ADDR, TMP) \ 2001 sll REG, 3, REG ;\ 2002mark2: set start2, TMP ;\ 2003 jmp REG + TMP ;\ 2004 nop ;\ 2005start2: ba,pt %xcc, done2 ;\ 2006 ldd [ADDR + CPU_TMP1], %f0 ;\ 2007 ba,pt %xcc, done2 ;\ 2008 ldd [ADDR + CPU_TMP1], %f32 ;\ 2009 ba,pt %xcc, done2 ;\ 2010 ldd [ADDR + CPU_TMP1], %f2 ;\ 2011 ba,pt %xcc, done2 ;\ 2012 ldd [ADDR + CPU_TMP1], %f34 ;\ 2013 ba,pt %xcc, done2 ;\ 2014 ldd [ADDR + CPU_TMP1], %f4 ;\ 2015 ba,pt %xcc, done2 ;\ 2016 ldd [ADDR + CPU_TMP1], %f36 ;\ 2017 ba,pt %xcc, done2 ;\ 2018 ldd [ADDR + CPU_TMP1], %f6 ;\ 2019 ba,pt %xcc, done2 ;\ 2020 ldd [ADDR + CPU_TMP1], %f38 ;\ 2021 ba,pt %xcc, done2 ;\ 2022 ldd [ADDR + CPU_TMP1], %f8 ;\ 2023 ba,pt %xcc, done2 ;\ 2024 ldd [ADDR + CPU_TMP1], %f40 ;\ 2025 ba,pt %xcc, done2 ;\ 2026 ldd [ADDR + CPU_TMP1], %f10 ;\ 2027 ba,pt %xcc, done2 ;\ 2028 ldd [ADDR + CPU_TMP1], %f42 ;\ 2029 ba,pt %xcc, done2 ;\ 2030 ldd [ADDR + CPU_TMP1], %f12 ;\ 2031 ba,pt %xcc, done2 ;\ 2032 ldd [ADDR + CPU_TMP1], %f44 ;\ 2033 ba,pt %xcc, done2 ;\ 2034 ldd [ADDR + CPU_TMP1], %f14 ;\ 2035 ba,pt %xcc, done2 ;\ 2036 ldd [ADDR + CPU_TMP1], %f46 ;\ 2037 ba,pt %xcc, done2 ;\ 2038 ldd [ADDR + CPU_TMP1], %f16 ;\ 2039 ba,pt %xcc, done2 ;\ 2040 ldd [ADDR + CPU_TMP1], %f48 ;\ 2041 ba,pt %xcc, done2 ;\ 2042 ldd [ADDR + CPU_TMP1], %f18 ;\ 2043 ba,pt %xcc, done2 ;\ 2044 ldd [ADDR + CPU_TMP1], %f50 ;\ 2045 ba,pt %xcc, done2 ;\ 2046 ldd [ADDR + CPU_TMP1], %f20 ;\ 2047 ba,pt %xcc, done2 ;\ 2048 ldd [ADDR + CPU_TMP1], %f52 ;\ 2049 ba,pt %xcc, done2 ;\ 2050 ldd [ADDR + CPU_TMP1], %f22 ;\ 2051 ba,pt %xcc, done2 ;\ 2052 ldd [ADDR + CPU_TMP1], %f54 ;\ 2053 ba,pt %xcc, done2 ;\ 2054 ldd [ADDR + CPU_TMP1], %f24 ;\ 2055 ba,pt %xcc, done2 ;\ 2056 ldd [ADDR + CPU_TMP1], %f56 ;\ 2057 ba,pt %xcc, done2 ;\ 2058 ldd [ADDR + CPU_TMP1], %f26 ;\ 2059 ba,pt %xcc, done2 ;\ 2060 ldd [ADDR + CPU_TMP1], %f58 ;\ 2061 ba,pt %xcc, done2 ;\ 2062 ldd [ADDR + CPU_TMP1], %f28 ;\ 2063 ba,pt %xcc, done2 ;\ 2064 ldd [ADDR + CPU_TMP1], %f60 ;\ 2065 ba,pt %xcc, done2 ;\ 2066 ldd [ADDR + CPU_TMP1], %f30 ;\ 2067 ba,pt %xcc, done2 ;\ 2068 ldd [ADDR + CPU_TMP1], %f62 ;\ 2069done2: 2070 2071.lddf_exception_not_aligned: 2072 /* %g2 = sfar, %g3 = sfsr */ 2073 mov %g2, %g5 ! stash sfar 2074#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 2075 sethi %hi(fpu_exists), %g2 ! check fpu_exists 2076 ld [%g2 + %lo(fpu_exists)], %g2 2077 brz,a,pn %g2, 4f 2078 nop 2079#endif 2080 CPU_ADDR(%g1, %g4) 2081 or %g0, 1, %g4 2082 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag 2083 2084 rdpr %tpc, %g2 2085 lda [%g2]ASI_AIUP, %g6 ! get the user's lddf instruction 2086 srl %g6, 23, %g1 ! using ldda or not? 2087 and %g1, 1, %g1 2088 brz,a,pt %g1, 2f ! check for ldda instruction 2089 nop 2090 srl %g6, 13, %g1 ! check immflag 2091 and %g1, 1, %g1 2092 rdpr %tstate, %g2 ! %tstate in %g2 2093 brnz,a,pn %g1, 1f 2094 srl %g2, 31, %g1 ! get asi from %tstate 2095 srl %g6, 5, %g1 ! get asi from instruction 2096 and %g1, 0xFF, %g1 ! imm_asi field 20971: 2098 cmp %g1, ASI_P ! primary address space 2099 be,a,pt %icc, 2f 2100 nop 2101 cmp %g1, ASI_PNF ! primary no fault address space 2102 be,a,pt %icc, 2f 2103 nop 2104 cmp %g1, ASI_S ! secondary address space 2105 be,a,pt %icc, 2f 2106 nop 2107 cmp %g1, ASI_SNF ! secondary no fault address space 2108 bne,a,pn %icc, 3f 2109 nop 21102: 2111 lduwa [%g5]ASI_USER, %g7 ! get first half of misaligned data 2112 add %g5, 4, %g5 ! increment misaligned data address 2113 lduwa [%g5]ASI_USER, %g5 ! get second half of misaligned data 2114 2115 sllx %g7, 32, %g7 2116 or %g5, %g7, %g5 ! combine data 2117 CPU_ADDR(%g7, %g1) ! save data on a per-cpu basis 2118 stx %g5, [%g7 + CPU_TMP1] ! save in cpu_tmp1 2119 2120 srl %g6, 25, %g3 ! %g6 has the instruction 2121 and %g3, 0x1F, %g3 ! %g3 has rd 2122 LDDF_REG(%g3, %g7, %g4) 2123 2124 CPU_ADDR(%g1, %g4) 2125 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 2126 FAST_TRAP_DONE 21273: 2128 CPU_ADDR(%g1, %g4) 2129 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 21304: 2131 set T_USER, %g3 ! trap type in %g3 2132 or %g3, T_LDDF_ALIGN, %g3 2133 mov %g5, %g2 ! misaligned vaddr in %g2 2134 set fpu_trap, %g1 ! goto C for the little and 2135 ba,pt %xcc, sys_trap ! no fault little asi's 2136 sub %g0, 1, %g4 2137 2138.stdf_exception_not_aligned: 2139 /* %g2 = sfar, %g3 = sfsr */ 2140 mov %g2, %g5 2141 2142#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 2143 sethi %hi(fpu_exists), %g7 ! check fpu_exists 2144 ld [%g7 + %lo(fpu_exists)], %g3 2145 brz,a,pn %g3, 4f 2146 nop 2147#endif 2148 CPU_ADDR(%g1, %g4) 2149 or %g0, 1, %g4 2150 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag 2151 2152 rdpr %tpc, %g2 2153 lda [%g2]ASI_AIUP, %g6 ! get the user's stdf instruction 2154 2155 srl %g6, 23, %g1 ! using stda or not? 2156 and %g1, 1, %g1 2157 brz,a,pt %g1, 2f ! check for stda instruction 2158 nop 2159 srl %g6, 13, %g1 ! check immflag 2160 and %g1, 1, %g1 2161 rdpr %tstate, %g2 ! %tstate in %g2 2162 brnz,a,pn %g1, 1f 2163 srl %g2, 31, %g1 ! get asi from %tstate 2164 srl %g6, 5, %g1 ! get asi from instruction 2165 and %g1, 0xff, %g1 ! imm_asi field 21661: 2167 cmp %g1, ASI_P ! primary address space 2168 be,a,pt %icc, 2f 2169 nop 2170 cmp %g1, ASI_S ! secondary address space 2171 bne,a,pn %icc, 3f 2172 nop 21732: 2174 srl %g6, 25, %g6 2175 and %g6, 0x1F, %g6 ! %g6 has rd 2176 CPU_ADDR(%g7, %g1) 2177 STDF_REG(%g6, %g7, %g4) ! STDF_REG(REG, ADDR, TMP) 2178 2179 ldx [%g7 + CPU_TMP1], %g6 2180 srlx %g6, 32, %g7 2181 stuwa %g7, [%g5]ASI_USER ! first half 2182 add %g5, 4, %g5 ! increment misaligned data address 2183 stuwa %g6, [%g5]ASI_USER ! second half 2184 2185 CPU_ADDR(%g1, %g4) 2186 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 2187 FAST_TRAP_DONE 21883: 2189 CPU_ADDR(%g1, %g4) 2190 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 21914: 2192 set T_USER, %g3 ! trap type in %g3 2193 or %g3, T_STDF_ALIGN, %g3 2194 mov %g5, %g2 ! misaligned vaddr in %g2 2195 set fpu_trap, %g1 ! goto C for the little and 2196 ba,pt %xcc, sys_trap ! nofault little asi's 2197 sub %g0, 1, %g4 2198 2199#ifdef DEBUG_USER_TRAPTRACECTL 2200 2201.traptrace_freeze: 2202 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4 2203 TT_TRACE_L(trace_win) 2204 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0 2205 set trap_freeze, %g1 2206 mov 1, %g2 2207 st %g2, [%g1] 2208 FAST_TRAP_DONE 2209 2210.traptrace_unfreeze: 2211 set trap_freeze, %g1 2212 st %g0, [%g1] 2213 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4 2214 TT_TRACE_L(trace_win) 2215 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0 2216 FAST_TRAP_DONE 2217 2218#endif /* DEBUG_USER_TRAPTRACECTL */ 2219 2220.getcc: 2221 CPU_ADDR(%g1, %g2) 2222 stx %o0, [%g1 + CPU_TMP1] ! save %o0 2223 rdpr %tstate, %g3 ! get tstate 2224 srlx %g3, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr 2225 set PSR_ICC, %g2 2226 and %o0, %g2, %o0 ! mask out the rest 2227 srl %o0, PSR_ICC_SHIFT, %o0 ! right justify 2228 wrpr %g0, 0, %gl 2229 mov %o0, %g1 ! move ccr to normal %g1 2230 wrpr %g0, 1, %gl 2231 ! cannot assume globals retained their values after increasing %gl 2232 CPU_ADDR(%g1, %g2) 2233 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0 2234 FAST_TRAP_DONE 2235 2236.setcc: 2237 CPU_ADDR(%g1, %g2) 2238 stx %o0, [%g1 + CPU_TMP1] ! save %o0 2239 wrpr %g0, 0, %gl 2240 mov %g1, %o0 2241 wrpr %g0, 1, %gl 2242 ! cannot assume globals retained their values after increasing %gl 2243 CPU_ADDR(%g1, %g2) 2244 sll %o0, PSR_ICC_SHIFT, %g2 2245 set PSR_ICC, %g3 2246 and %g2, %g3, %g2 ! mask out rest 2247 sllx %g2, PSR_TSTATE_CC_SHIFT, %g2 2248 rdpr %tstate, %g3 ! get tstate 2249 srl %g3, 0, %g3 ! clear upper word 2250 or %g3, %g2, %g3 ! or in new bits 2251 wrpr %g3, %tstate 2252 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0 2253 FAST_TRAP_DONE 2254 2255/* 2256 * getpsr(void) 2257 * Note that the xcc part of the ccr is not provided. 2258 * The V8 code shows why the V9 trap is not faster: 2259 * #define GETPSR_TRAP() \ 2260 * mov %psr, %i0; jmp %l2; rett %l2+4; nop; 2261 */ 2262 2263 .type .getpsr, #function 2264.getpsr: 2265 rdpr %tstate, %g1 ! get tstate 2266 srlx %g1, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr 2267 set PSR_ICC, %g2 2268 and %o0, %g2, %o0 ! mask out the rest 2269 2270 rd %fprs, %g1 ! get fprs 2271 and %g1, FPRS_FEF, %g2 ! mask out dirty upper/lower 2272 sllx %g2, PSR_FPRS_FEF_SHIFT, %g2 ! shift fef to V8 psr.ef 2273 or %o0, %g2, %o0 ! or result into psr.ef 2274 2275 set V9_PSR_IMPLVER, %g2 ! SI assigned impl/ver: 0xef 2276 or %o0, %g2, %o0 ! or psr.impl/ver 2277 FAST_TRAP_DONE 2278 SET_SIZE(.getpsr) 2279 2280/* 2281 * setpsr(newpsr) 2282 * Note that there is no support for ccr.xcc in the V9 code. 2283 */ 2284 2285 .type .setpsr, #function 2286.setpsr: 2287 rdpr %tstate, %g1 ! get tstate 2288! setx TSTATE_V8_UBITS, %g2 2289 or %g0, CCR_ICC, %g3 2290 sllx %g3, TSTATE_CCR_SHIFT, %g2 2291 2292 andn %g1, %g2, %g1 ! zero current user bits 2293 set PSR_ICC, %g2 2294 and %g2, %o0, %g2 ! clear all but psr.icc bits 2295 sllx %g2, PSR_TSTATE_CC_SHIFT, %g3 ! shift to tstate.ccr.icc 2296 wrpr %g1, %g3, %tstate ! write tstate 2297 2298 set PSR_EF, %g2 2299 and %g2, %o0, %g2 ! clear all but fp enable bit 2300 srlx %g2, PSR_FPRS_FEF_SHIFT, %g4 ! shift ef to V9 fprs.fef 2301 wr %g0, %g4, %fprs ! write fprs 2302 2303 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 2304 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer 2305 ldn [%g2 + T_LWP], %g3 ! load klwp pointer 2306 ldn [%g3 + LWP_FPU], %g2 ! get lwp_fpu pointer 2307 stuw %g4, [%g2 + FPU_FPRS] ! write fef value to fpu_fprs 2308 srlx %g4, 2, %g4 ! shift fef value to bit 0 2309 stub %g4, [%g2 + FPU_EN] ! write fef value to fpu_en 2310 FAST_TRAP_DONE 2311 SET_SIZE(.setpsr) 2312 2313/* 2314 * getlgrp 2315 * get home lgrpid on which the calling thread is currently executing. 2316 */ 2317 .type .getlgrp, #function 2318.getlgrp: 2319 ! Thanks for the incredibly helpful comments 2320 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 2321 ld [%g1 + CPU_ID], %o0 ! load cpu_id 2322 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer 2323 ldn [%g2 + T_LPL], %g2 ! load lpl pointer 2324 ld [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid 2325 sra %g1, 0, %o1 2326 FAST_TRAP_DONE 2327 SET_SIZE(.getlgrp) 2328 2329/* 2330 * Entry for old 4.x trap (trap 0). 2331 */ 2332 ENTRY_NP(syscall_trap_4x) 2333 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 2334 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer 2335 ldn [%g2 + T_LWP], %g2 ! load klwp pointer 2336 ld [%g2 + PCB_TRAP0], %g2 ! lwp->lwp_pcb.pcb_trap0addr 2337 brz,pn %g2, 1f ! has it been set? 2338 st %l0, [%g1 + CPU_TMP1] ! delay - save some locals 2339 st %l1, [%g1 + CPU_TMP2] 2340 rdpr %tnpc, %l1 ! save old tnpc 2341 wrpr %g0, %g2, %tnpc ! setup tnpc 2342 2343 mov %g1, %l0 ! save CPU struct addr 2344 wrpr %g0, 0, %gl 2345 mov %l1, %g6 ! pass tnpc to user code in %g6 2346 wrpr %g0, 1, %gl 2347 ld [%l0 + CPU_TMP2], %l1 ! restore locals 2348 ld [%l0 + CPU_TMP1], %l0 2349 FAST_TRAP_DONE_CHK_INTR 23501: 2351 ! 2352 ! check for old syscall mmap which is the only different one which 2353 ! must be the same. Others are handled in the compatibility library. 2354 ! 2355 mov %g1, %l0 ! save CPU struct addr 2356 wrpr %g0, 0, %gl 2357 cmp %g1, OSYS_mmap ! compare to old 4.x mmap 2358 movz %icc, SYS_mmap, %g1 2359 wrpr %g0, 1, %gl 2360 ld [%l0 + CPU_TMP1], %l0 2361 SYSCALL(syscall_trap32) 2362 SET_SIZE(syscall_trap_4x) 2363 2364/* 2365 * Handler for software trap 9. 2366 * Set trap0 emulation address for old 4.x system call trap. 2367 * XXX - this should be a system call. 2368 */ 2369 ENTRY_NP(set_trap0_addr) 2370 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 2371 st %l0, [%g1 + CPU_TMP1] ! save some locals 2372 st %l1, [%g1 + CPU_TMP2] 2373 mov %g1, %l0 ! preserve CPU addr 2374 wrpr %g0, 0, %gl 2375 mov %g1, %l1 2376 wrpr %g0, 1, %gl 2377 ! cannot assume globals retained their values after increasing %gl 2378 ldn [%l0 + CPU_THREAD], %g2 ! load thread pointer 2379 ldn [%g2 + T_LWP], %g2 ! load klwp pointer 2380 andn %l1, 3, %l1 ! force alignment 2381 st %l1, [%g2 + PCB_TRAP0] ! lwp->lwp_pcb.pcb_trap0addr 2382 ld [%l0 + CPU_TMP2], %l1 ! restore locals 2383 ld [%l0 + CPU_TMP1], %l0 2384 FAST_TRAP_DONE 2385 SET_SIZE(set_trap0_addr) 2386 2387/* 2388 * mmu_trap_tl1 2389 * trap handler for unexpected mmu traps. 2390 * simply checks if the trap was a user lddf/stdf alignment trap, in which 2391 * case we go to fpu_trap or a user trap from the window handler, in which 2392 * case we go save the state on the pcb. Otherwise, we go to ptl1_panic. 2393 */ 2394 .type mmu_trap_tl1, #function 2395mmu_trap_tl1: 2396#ifdef TRAPTRACE 2397 TRACE_PTR(%g5, %g6) 2398 GET_TRACE_TICK(%g6) 2399 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi 2400 TRACE_SAVE_TL_GL_REGS(%g5, %g6) 2401 rdpr %tt, %g6 2402 stha %g6, [%g5 + TRAP_ENT_TT]%asi 2403 rdpr %tstate, %g6 2404 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi 2405 stna %sp, [%g5 + TRAP_ENT_SP]%asi 2406 stna %g0, [%g5 + TRAP_ENT_TR]%asi 2407 rdpr %tpc, %g6 2408 stna %g6, [%g5 + TRAP_ENT_TPC]%asi 2409 MMU_FAULT_STATUS_AREA(%g6) 2410 ldx [%g6 + MMFSA_D_ADDR], %g6 2411 stna %g6, [%g5 + TRAP_ENT_F1]%asi ! MMU fault address 2412 CPU_PADDR(%g7, %g6); 2413 add %g7, CPU_TL1_HDLR, %g7 2414 lda [%g7]ASI_MEM, %g6 2415 stna %g6, [%g5 + TRAP_ENT_F2]%asi 2416 MMU_FAULT_STATUS_AREA(%g6) 2417 ldx [%g6 + MMFSA_D_TYPE], %g7 ! XXXQ should be a MMFSA_F_ constant? 2418 ldx [%g6 + MMFSA_D_CTX], %g6 2419 sllx %g6, SFSR_CTX_SHIFT, %g6 2420 or %g6, %g7, %g6 2421 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! MMU context/type 2422 set 0xdeadbeef, %g6 2423 stna %g6, [%g5 + TRAP_ENT_F4]%asi 2424 TRACE_NEXT(%g5, %g6, %g7) 2425#endif /* TRAPTRACE */ 2426 CPU_PADDR(%g7, %g6); 2427 add %g7, CPU_TL1_HDLR, %g7 ! %g7 = &cpu_m.tl1_hdlr (PA) 2428 lda [%g7]ASI_MEM, %g6 2429 brz,a,pt %g6, 1f 2430 nop 2431 sta %g0, [%g7]ASI_MEM 2432 ! XXXQ need to setup registers for sfmmu_mmu_trap? 2433 ba,a,pt %xcc, sfmmu_mmu_trap ! handle page faults 24341: 2435 rdpr %tpc, %g7 2436 /* in user_rtt? */ 2437 set rtt_fill_start, %g6 2438 cmp %g7, %g6 2439 blu,pn %xcc, 6f 2440 .empty 2441 set rtt_fill_end, %g6 2442 cmp %g7, %g6 2443 bgeu,pn %xcc, 6f 2444 nop 2445 set fault_rtt_fn1, %g7 2446 ba,a 7f 24476: 2448 ! check to see if the trap pc is in a window spill/fill handling 2449 rdpr %tpc, %g7 2450 /* tpc should be in the trap table */ 2451 set trap_table, %g6 2452 cmp %g7, %g6 2453 blu,a,pn %xcc, ptl1_panic 2454 mov PTL1_BAD_MMUTRAP, %g1 2455 set etrap_table, %g6 2456 cmp %g7, %g6 2457 bgeu,a,pn %xcc, ptl1_panic 2458 mov PTL1_BAD_MMUTRAP, %g1 2459 ! pc is inside the trap table, convert to trap type 2460 srl %g7, 5, %g6 ! XXXQ need #define 2461 and %g6, 0x1ff, %g6 ! XXXQ need #define 2462 ! and check for a window trap type 2463 and %g6, WTRAP_TTMASK, %g6 2464 cmp %g6, WTRAP_TYPE 2465 bne,a,pn %xcc, ptl1_panic 2466 mov PTL1_BAD_MMUTRAP, %g1 2467 andn %g7, WTRAP_ALIGN, %g7 /* 128 byte aligned */ 2468 add %g7, WTRAP_FAULTOFF, %g7 2469 24707: 2471 ! Arguments are passed in the global set active after the 2472 ! 'done' instruction. Before switching sets, must save 2473 ! the calculated next pc 2474 wrpr %g0, %g7, %tnpc 2475 wrpr %g0, 1, %gl 2476 rdpr %tt, %g5 2477 MMU_FAULT_STATUS_AREA(%g7) 2478 cmp %g5, T_ALIGNMENT 2479 be,pn %xcc, 1f 2480 ldx [%g7 + MMFSA_D_ADDR], %g6 2481 ldx [%g7 + MMFSA_D_CTX], %g7 2482 srlx %g6, MMU_PAGESHIFT, %g6 /* align address */ 2483 cmp %g7, USER_CONTEXT_TYPE 2484 sllx %g6, MMU_PAGESHIFT, %g6 2485 movgu %icc, USER_CONTEXT_TYPE, %g7 2486 or %g6, %g7, %g6 /* TAG_ACCESS */ 24871: 2488 done 2489 SET_SIZE(mmu_trap_tl1) 2490 2491/* 2492 * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers. These 2493 * traps are valid only when kmdb is loaded. When the debugger is active, 2494 * the code below is rewritten to transfer control to the appropriate 2495 * debugger entry points. 2496 */ 2497 .global kmdb_trap 2498 .align 8 2499kmdb_trap: 2500 ba,a trap_table0 2501 jmp %g1 + 0 2502 nop 2503 2504 .global kmdb_trap_tl1 2505 .align 8 2506kmdb_trap_tl1: 2507 ba,a trap_table0 2508 jmp %g1 + 0 2509 nop 2510 2511/* 2512 * This entry is copied from OBP's trap table during boot. 2513 */ 2514 .global obp_bpt 2515 .align 8 2516obp_bpt: 2517 NOT 2518 2519 2520 2521#ifdef TRAPTRACE 2522/* 2523 * TRAPTRACE support. 2524 * labels here are branched to with "rd %pc, %g7" in the delay slot. 2525 * Return is done by "jmp %g7 + 4". 2526 */ 2527 2528trace_dmmu: 2529 TRACE_PTR(%g3, %g6) 2530 GET_TRACE_TICK(%g6) 2531 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi 2532 TRACE_SAVE_TL_GL_REGS(%g3, %g6) 2533 rdpr %tt, %g6 2534 stha %g6, [%g3 + TRAP_ENT_TT]%asi 2535 rdpr %tstate, %g6 2536 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi 2537 stna %sp, [%g3 + TRAP_ENT_SP]%asi 2538 rdpr %tpc, %g6 2539 stna %g6, [%g3 + TRAP_ENT_TPC]%asi 2540 MMU_FAULT_STATUS_AREA(%g6) 2541 ldx [%g6 + MMFSA_D_ADDR], %g4 2542 stxa %g4, [%g3 + TRAP_ENT_TR]%asi 2543 ldx [%g6 + MMFSA_D_CTX], %g4 2544 stxa %g4, [%g3 + TRAP_ENT_F1]%asi 2545 ldx [%g6 + MMFSA_D_TYPE], %g4 2546 stxa %g4, [%g3 + TRAP_ENT_F2]%asi 2547 stxa %g6, [%g3 + TRAP_ENT_F3]%asi 2548 stna %g0, [%g3 + TRAP_ENT_F4]%asi 2549 TRACE_NEXT(%g3, %g4, %g5) 2550 jmp %g7 + 4 2551 nop 2552 2553trace_immu: 2554 TRACE_PTR(%g3, %g6) 2555 GET_TRACE_TICK(%g6) 2556 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi 2557 TRACE_SAVE_TL_GL_REGS(%g3, %g6) 2558 rdpr %tt, %g6 2559 stha %g6, [%g3 + TRAP_ENT_TT]%asi 2560 rdpr %tstate, %g6 2561 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi 2562 stna %sp, [%g3 + TRAP_ENT_SP]%asi 2563 rdpr %tpc, %g6 2564 stna %g6, [%g3 + TRAP_ENT_TPC]%asi 2565 MMU_FAULT_STATUS_AREA(%g6) 2566 ldx [%g6 + MMFSA_I_ADDR], %g4 2567 stxa %g4, [%g3 + TRAP_ENT_TR]%asi 2568 ldx [%g6 + MMFSA_I_CTX], %g4 2569 stxa %g4, [%g3 + TRAP_ENT_F1]%asi 2570 ldx [%g6 + MMFSA_I_TYPE], %g4 2571 stxa %g4, [%g3 + TRAP_ENT_F2]%asi 2572 stxa %g6, [%g3 + TRAP_ENT_F3]%asi 2573 stna %g0, [%g3 + TRAP_ENT_F4]%asi 2574 TRACE_NEXT(%g3, %g4, %g5) 2575 jmp %g7 + 4 2576 nop 2577 2578trace_gen: 2579 TRACE_PTR(%g3, %g6) 2580 GET_TRACE_TICK(%g6) 2581 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi 2582 TRACE_SAVE_TL_GL_REGS(%g3, %g6) 2583 rdpr %tt, %g6 2584 stha %g6, [%g3 + TRAP_ENT_TT]%asi 2585 rdpr %tstate, %g6 2586 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi 2587 stna %sp, [%g3 + TRAP_ENT_SP]%asi 2588 rdpr %tpc, %g6 2589 stna %g6, [%g3 + TRAP_ENT_TPC]%asi 2590 stna %g0, [%g3 + TRAP_ENT_TR]%asi 2591 stna %g0, [%g3 + TRAP_ENT_F1]%asi 2592 stna %g0, [%g3 + TRAP_ENT_F2]%asi 2593 stna %g0, [%g3 + TRAP_ENT_F3]%asi 2594 stna %g0, [%g3 + TRAP_ENT_F4]%asi 2595 TRACE_NEXT(%g3, %g4, %g5) 2596 jmp %g7 + 4 2597 nop 2598 2599trace_win: 2600 TRACE_WIN_INFO(0, %l0, %l1, %l2) 2601 ! Keep the locals as clean as possible, caller cleans %l4 2602 clr %l2 2603 clr %l1 2604 jmp %l4 + 4 2605 clr %l0 2606 2607/* 2608 * Trace a tsb hit 2609 * g1 = tsbe pointer (in/clobbered) 2610 * g2 = tag access register (in) 2611 * g3 - g4 = scratch (clobbered) 2612 * g5 = tsbe data (in) 2613 * g6 = scratch (clobbered) 2614 * g7 = pc we jumped here from (in) 2615 */ 2616 2617 ! Do not disturb %g5, it will be used after the trace 2618 ALTENTRY(trace_tsbhit) 2619 TRACE_TSBHIT(0) 2620 jmp %g7 + 4 2621 nop 2622 2623/* 2624 * Trace a TSB miss 2625 * 2626 * g1 = tsb8k pointer (in) 2627 * g2 = tag access register (in) 2628 * g3 = tsb4m pointer (in) 2629 * g4 = tsbe tag (in/clobbered) 2630 * g5 - g6 = scratch (clobbered) 2631 * g7 = pc we jumped here from (in) 2632 */ 2633 .global trace_tsbmiss 2634trace_tsbmiss: 2635 membar #Sync 2636 sethi %hi(FLUSH_ADDR), %g6 2637 flush %g6 2638 TRACE_PTR(%g5, %g6) 2639 GET_TRACE_TICK(%g6) 2640 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi 2641 stna %g2, [%g5 + TRAP_ENT_SP]%asi ! tag access 2642 stna %g4, [%g5 + TRAP_ENT_F1]%asi ! XXX? tsb tag 2643 rdpr %tnpc, %g6 2644 stna %g6, [%g5 + TRAP_ENT_F2]%asi 2645 stna %g1, [%g5 + TRAP_ENT_F3]%asi ! tsb8k pointer 2646 rdpr %tpc, %g6 2647 stna %g6, [%g5 + TRAP_ENT_TPC]%asi 2648 TRACE_SAVE_TL_GL_REGS(%g5, %g6) 2649 rdpr %tt, %g6 2650 or %g6, TT_MMU_MISS, %g4 2651 stha %g4, [%g5 + TRAP_ENT_TT]%asi 2652 mov MMFSA_D_ADDR, %g4 2653 cmp %g6, FAST_IMMU_MISS_TT 2654 move %xcc, MMFSA_I_ADDR, %g4 2655 cmp %g6, T_INSTR_MMU_MISS 2656 move %xcc, MMFSA_I_ADDR, %g4 2657 MMU_FAULT_STATUS_AREA(%g6) 2658 ldx [%g6 + %g4], %g6 2659 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! tag target 2660 cmp %g4, MMFSA_D_ADDR 2661 move %xcc, MMFSA_D_CTX, %g4 2662 movne %xcc, MMFSA_I_CTX, %g4 2663 MMU_FAULT_STATUS_AREA(%g6) 2664 ldx [%g6 + %g4], %g6 2665 stxa %g6, [%g5 + TRAP_ENT_F4]%asi ! context ID 2666 stna %g3, [%g5 + TRAP_ENT_TR]%asi ! tsb4m pointer 2667 TRACE_NEXT(%g5, %g4, %g6) 2668 jmp %g7 + 4 2669 nop 2670 2671/* 2672 * g2 = tag access register (in) 2673 * g3 = ctx type (0, 1 or 2) (in) (not used) 2674 */ 2675trace_dataprot: 2676 membar #Sync 2677 sethi %hi(FLUSH_ADDR), %g6 2678 flush %g6 2679 TRACE_PTR(%g1, %g6) 2680 GET_TRACE_TICK(%g6) 2681 stxa %g6, [%g1 + TRAP_ENT_TICK]%asi 2682 rdpr %tpc, %g6 2683 stna %g6, [%g1 + TRAP_ENT_TPC]%asi 2684 rdpr %tstate, %g6 2685 stxa %g6, [%g1 + TRAP_ENT_TSTATE]%asi 2686 stna %g2, [%g1 + TRAP_ENT_SP]%asi ! tag access reg 2687 stna %g0, [%g1 + TRAP_ENT_F1]%asi 2688 stna %g0, [%g1 + TRAP_ENT_F2]%asi 2689 stna %g0, [%g1 + TRAP_ENT_F3]%asi 2690 stna %g0, [%g1 + TRAP_ENT_F4]%asi 2691 TRACE_SAVE_TL_GL_REGS(%g1, %g6) 2692 rdpr %tt, %g6 2693 stha %g6, [%g1 + TRAP_ENT_TT]%asi 2694 mov MMFSA_D_CTX, %g4 2695 cmp %g6, FAST_IMMU_MISS_TT 2696 move %xcc, MMFSA_I_CTX, %g4 2697 cmp %g6, T_INSTR_MMU_MISS 2698 move %xcc, MMFSA_I_CTX, %g4 2699 MMU_FAULT_STATUS_AREA(%g6) 2700 ldx [%g6 + %g4], %g6 2701 stxa %g6, [%g1 + TRAP_ENT_TR]%asi ! context ID 2702 TRACE_NEXT(%g1, %g4, %g5) 2703 jmp %g7 + 4 2704 nop 2705 2706#endif /* TRAPTRACE */ 2707 2708/* 2709 * Handle watchdog reset trap. Enable the MMU using the MMU_ENABLE 2710 * HV service, which requires the return target to be specified as a VA 2711 * since we are enabling the MMU. We set the target to ptl1_panic. 2712 */ 2713 2714 .type .watchdog_trap, #function 2715.watchdog_trap: 2716 mov 1, %o0 2717 setx ptl1_panic, %g2, %o1 2718 mov MMU_ENABLE, %o5 2719 ta FAST_TRAP 2720 done 2721 SET_SIZE(.watchdog_trap) 2722/* 2723 * synthesize for trap(): SFAR in %g2, SFSR in %g3 2724 */ 2725 .type .dmmu_exc_lddf_not_aligned, #function 2726.dmmu_exc_lddf_not_aligned: 2727 MMU_FAULT_STATUS_AREA(%g3) 2728 ldx [%g3 + MMFSA_D_ADDR], %g2 2729 /* Fault type not available in MMU fault status area */ 2730 mov MMFSA_F_UNALIGN, %g1 2731 ldx [%g3 + MMFSA_D_CTX], %g3 2732 sllx %g3, SFSR_CTX_SHIFT, %g3 2733 btst 1, %sp 2734 bnz,pt %xcc, .lddf_exception_not_aligned 2735 or %g3, %g1, %g3 /* SFSR */ 2736 ba,a,pt %xcc, .mmu_exception_not_aligned 2737 SET_SIZE(.dmmu_exc_lddf_not_aligned) 2738 2739/* 2740 * synthesize for trap(): SFAR in %g2, SFSR in %g3 2741 */ 2742 .type .dmmu_exc_stdf_not_aligned, #function 2743.dmmu_exc_stdf_not_aligned: 2744 MMU_FAULT_STATUS_AREA(%g3) 2745 ldx [%g3 + MMFSA_D_ADDR], %g2 2746 /* Fault type not available in MMU fault status area */ 2747 mov MMFSA_F_UNALIGN, %g1 2748 ldx [%g3 + MMFSA_D_CTX], %g3 2749 sllx %g3, SFSR_CTX_SHIFT, %g3 2750 btst 1, %sp 2751 bnz,pt %xcc, .stdf_exception_not_aligned 2752 or %g3, %g1, %g3 /* SFSR */ 2753 ba,a,pt %xcc, .mmu_exception_not_aligned 2754 SET_SIZE(.dmmu_exc_stdf_not_aligned) 2755 2756 .type .dmmu_exception, #function 2757.dmmu_exception: 2758 MMU_FAULT_STATUS_AREA(%g3) 2759 ldx [%g3 + MMFSA_D_ADDR], %g2 2760 ldx [%g3 + MMFSA_D_TYPE], %g1 2761 ldx [%g3 + MMFSA_D_CTX], %g4 2762 srlx %g2, MMU_PAGESHIFT, %g2 /* align address */ 2763 sllx %g2, MMU_PAGESHIFT, %g2 2764 sllx %g4, SFSR_CTX_SHIFT, %g3 2765 or %g3, %g1, %g3 /* SFSR */ 2766 cmp %g4, USER_CONTEXT_TYPE 2767 movgeu %icc, USER_CONTEXT_TYPE, %g4 2768 or %g2, %g4, %g2 /* TAG_ACCESS */ 2769 ba,pt %xcc, .mmu_exception_end 2770 mov T_DATA_EXCEPTION, %g1 2771 SET_SIZE(.dmmu_exception) 2772/* 2773 * expects offset into tsbmiss area in %g1 and return pc in %g7 2774 */ 2775stat_mmu: 2776 CPU_INDEX(%g5, %g6) 2777 sethi %hi(tsbmiss_area), %g6 2778 sllx %g5, TSBMISS_SHIFT, %g5 2779 or %g6, %lo(tsbmiss_area), %g6 2780 add %g6, %g5, %g6 /* g6 = tsbmiss area */ 2781 ld [%g6 + %g1], %g5 2782 add %g5, 1, %g5 2783 jmp %g7 + 4 2784 st %g5, [%g6 + %g1] 2785 2786 2787/* 2788 * fast_trap_done, fast_trap_done_chk_intr: 2789 * 2790 * Due to the design of UltraSPARC pipeline, pending interrupts are not 2791 * taken immediately after a RETRY or DONE instruction which causes IE to 2792 * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed 2793 * to execute first before taking any interrupts. If that instruction 2794 * results in other traps, and if the corresponding trap handler runs 2795 * entirely at TL=1 with interrupts disabled, then pending interrupts 2796 * won't be taken until after yet another instruction following the %tpc 2797 * or %tnpc. 2798 * 2799 * A malicious user program can use this feature to block out interrupts 2800 * for extended durations, which can result in send_mondo_timeout kernel 2801 * panic. 2802 * 2803 * This problem is addressed by servicing any pending interrupts via 2804 * sys_trap before returning back to the user mode from a fast trap 2805 * handler. The "done" instruction within a fast trap handler, which 2806 * runs entirely at TL=1 with interrupts disabled, is replaced with the 2807 * FAST_TRAP_DONE macro, which branches control to this fast_trap_done 2808 * entry point. 2809 * 2810 * We check for any pending interrupts here and force a sys_trap to 2811 * service those interrupts, if any. To minimize overhead, pending 2812 * interrupts are checked if the %tpc happens to be at 16K boundary, 2813 * which allows a malicious program to execute at most 4K consecutive 2814 * instructions before we service any pending interrupts. If a worst 2815 * case fast trap handler takes about 2 usec, then interrupts will be 2816 * blocked for at most 8 msec, less than a clock tick. 2817 * 2818 * For the cases where we don't know if the %tpc will cross a 16K 2819 * boundary, we can't use the above optimization and always process 2820 * any pending interrupts via fast_frap_done_chk_intr entry point. 2821 * 2822 * Entry Conditions: 2823 * %pstate am:0 priv:1 ie:0 2824 * globals are AG (not normal globals) 2825 */ 2826 2827 .global fast_trap_done, fast_trap_done_chk_intr 2828fast_trap_done: 2829 rdpr %tpc, %g5 2830 sethi %hi(0xffffc000), %g6 ! 1's complement of 0x3fff 2831 andncc %g5, %g6, %g0 ! check lower 14 bits of %tpc 2832 bz,pn %icc, 1f ! branch if zero (lower 32 bits only) 2833 nop 2834 done 2835 2836fast_trap_done_chk_intr: 28371: rd SOFTINT, %g6 2838 brnz,pn %g6, 2f ! branch if any pending intr 2839 nop 2840 done 2841 28422: 2843 /* 2844 * We get here if there are any pending interrupts. 2845 * Adjust %tpc/%tnpc as we'll be resuming via "retry" 2846 * instruction. 2847 */ 2848 rdpr %tnpc, %g5 2849 wrpr %g0, %g5, %tpc 2850 add %g5, 4, %g5 2851 wrpr %g0, %g5, %tnpc 2852 2853 /* 2854 * Force a dummy sys_trap call so that interrupts can be serviced. 2855 */ 2856 set fast_trap_dummy_call, %g1 2857 ba,pt %xcc, sys_trap 2858 mov -1, %g4 2859 2860fast_trap_dummy_call: 2861 retl 2862 nop 2863 2864#ifdef DEBUG 2865/* 2866 * Currently we only support syscall interposition for branded zones on 2867 * DEBUG kernels. The only brand that makes use of this functionality is 2868 * the fake Solaris 10 brand. Since this brand is only used for exercising 2869 * the framework, we don't want this overhead incurred on production 2870 * systems. 2871 */ 2872#define BRAND_CALLBACK(callback_id) \ 2873 CPU_ADDR(%g2, %g1) /* load CPU struct addr to %g2 */ ;\ 2874 ldn [%g2 + CPU_THREAD], %g3 /* load thread pointer */ ;\ 2875 ldn [%g3 + T_PROCP], %g3 /* get proc pointer */ ;\ 2876 ldn [%g3 + P_BRAND], %g3 /* get brand pointer */ ;\ 2877 brz %g3, 1f /* No brand? No callback. */ ;\ 2878 nop ;\ 2879 ldn [%g3 + B_MACHOPS], %g3 /* get machops list */ ;\ 2880 ldn [%g3 + (callback_id << 3)], %g3 ;\ 2881 brz %g3, 1f ;\ 2882 /* \ 2883 * This isn't pretty. We want a low-latency way for the callback \ 2884 * routine to decline to do anything. We just pass in an address \ 2885 * the routine can directly jmp back to, pretending that nothing \ 2886 * has happened. \ 2887 * \ 2888 * %g1: return address (where the brand handler jumps back to) \ 2889 * %g2: address of CPU structure \ 2890 * %g3: address of brand handler (where we will jump to) \ 2891 */ \ 2892 mov %pc, %g1 ;\ 2893 add %g1, 16, %g1 ;\ 2894 jmp %g3 ;\ 2895 nop ;\ 28961: 2897 2898 ENTRY_NP(syscall_wrapper32) 2899 TT_TRACE(trace_gen) 2900 BRAND_CALLBACK(BRAND_CB_SYSCALL32) 2901 SYSCALL_NOTT(syscall_trap32) 2902 SET_SIZE(syscall_wrapper32) 2903 2904 ENTRY_NP(syscall_wrapper) 2905 TT_TRACE(trace_gen) 2906 BRAND_CALLBACK(BRAND_CB_SYSCALL) 2907 SYSCALL_NOTT(syscall_trap) 2908 SET_SIZE(syscall_wrapper) 2909 2910#endif /* DEBUG */ 2911 2912#endif /* lint */ 2913