1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#include "assym.h" 27#include <sys/asm_linkage.h> 28#include <sys/privregs.h> 29#include <sys/sun4asi.h> 30#include <sys/spitregs.h> 31#include <sys/cheetahregs.h> 32#include <sys/machtrap.h> 33#include <sys/machthread.h> 34#include <sys/machbrand.h> 35#include <sys/pcb.h> 36#include <sys/pte.h> 37#include <sys/mmu.h> 38#include <sys/machpcb.h> 39#include <sys/async.h> 40#include <sys/intreg.h> 41#include <sys/scb.h> 42#include <sys/psr_compat.h> 43#include <sys/syscall.h> 44#include <sys/machparam.h> 45#include <sys/traptrace.h> 46#include <vm/hat_sfmmu.h> 47#include <sys/archsystm.h> 48#include <sys/utrap.h> 49#include <sys/clock.h> 50#include <sys/intr.h> 51#include <sys/fpu/fpu_simulator.h> 52#include <vm/seg_spt.h> 53 54/* 55 * WARNING: If you add a fast trap handler which can be invoked by a 56 * non-privileged user, you may have to use the FAST_TRAP_DONE macro 57 * instead of "done" instruction to return back to the user mode. See 58 * comments for the "fast_trap_done" entry point for more information. 59 * 60 * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the 61 * cases where you always want to process any pending interrupts before 62 * returning back to the user mode. 63 */ 64#define FAST_TRAP_DONE \ 65 ba,a fast_trap_done 66 67#define FAST_TRAP_DONE_CHK_INTR \ 68 ba,a fast_trap_done_chk_intr 69 70/* 71 * SPARC V9 Trap Table 72 * 73 * Most of the trap handlers are made from common building 74 * blocks, and some are instantiated multiple times within 75 * the trap table. So, I build a bunch of macros, then 76 * populate the table using only the macros. 77 * 78 * Many macros branch to sys_trap. Its calling convention is: 79 * %g1 kernel trap handler 80 * %g2, %g3 args for above 81 * %g4 desire %pil 82 */ 83 84#ifdef TRAPTRACE 85 86/* 87 * Tracing macro. Adds two instructions if TRAPTRACE is defined. 88 */ 89#define TT_TRACE(label) \ 90 ba label ;\ 91 rd %pc, %g7 92#define TT_TRACE_INS 2 93 94#define TT_TRACE_L(label) \ 95 ba label ;\ 96 rd %pc, %l4 ;\ 97 clr %l4 98#define TT_TRACE_L_INS 3 99 100#else 101 102#define TT_TRACE(label) 103#define TT_TRACE_INS 0 104 105#define TT_TRACE_L(label) 106#define TT_TRACE_L_INS 0 107 108#endif 109 110/* 111 * This first set are funneled to trap() with %tt as the type. 112 * Trap will then either panic or send the user a signal. 113 */ 114/* 115 * NOT is used for traps that just shouldn't happen. 116 * It comes in both single and quadruple flavors. 117 */ 118 .global trap 119#define NOT \ 120 TT_TRACE(trace_gen) ;\ 121 set trap, %g1 ;\ 122 rdpr %tt, %g3 ;\ 123 ba,pt %xcc, sys_trap ;\ 124 sub %g0, 1, %g4 ;\ 125 .align 32 126#define NOT4 NOT; NOT; NOT; NOT 127/* 128 * RED is for traps that use the red mode handler. 129 * We should never see these either. 130 */ 131#define RED NOT 132/* 133 * BAD is used for trap vectors we don't have a kernel 134 * handler for. 135 * It also comes in single and quadruple versions. 136 */ 137#define BAD NOT 138#define BAD4 NOT4 139 140#define DONE \ 141 done; \ 142 .align 32 143 144/* 145 * TRAP vectors to the trap() function. 146 * It's main use is for user errors. 147 */ 148 .global trap 149#define TRAP(arg) \ 150 TT_TRACE(trace_gen) ;\ 151 set trap, %g1 ;\ 152 mov arg, %g3 ;\ 153 ba,pt %xcc, sys_trap ;\ 154 sub %g0, 1, %g4 ;\ 155 .align 32 156 157/* 158 * SYSCALL is used for unsupported syscall interfaces (with 'which' 159 * set to 'nosys') and legacy support of old SunOS 4.x syscalls (with 160 * 'which' set to 'syscall_trap32'). 161 * 162 * The SYSCALL_TRAP* macros are used for syscall entry points. 163 * SYSCALL_TRAP is used to support LP64 syscalls and SYSCALL_TRAP32 164 * is used to support ILP32. Each macro can only be used once 165 * since they each define a symbol. The symbols are used as hot patch 166 * points by the brand infrastructure to dynamically enable and disable 167 * brand syscall interposition. See the comments around BRAND_CALLBACK 168 * and brand_plat_interposition_enable() for more information. 169 */ 170#define SYSCALL_NOTT(which) \ 171 set (which), %g1 ;\ 172 ba,pt %xcc, sys_trap ;\ 173 sub %g0, 1, %g4 ;\ 174 .align 32 175 176#define SYSCALL(which) \ 177 TT_TRACE(trace_gen) ;\ 178 SYSCALL_NOTT(which) 179 180#define SYSCALL_TRAP32 \ 181 TT_TRACE(trace_gen) ;\ 182 ALTENTRY(syscall_trap32_patch_point) \ 183 SYSCALL_NOTT(syscall_trap32) 184 185#define SYSCALL_TRAP \ 186 TT_TRACE(trace_gen) ;\ 187 ALTENTRY(syscall_trap_patch_point) \ 188 SYSCALL_NOTT(syscall_trap) 189 190#define FLUSHW(h_name) \ 191 .global h_name ;\ 192h_name: ;\ 193 set trap, %g1 ;\ 194 mov T_FLUSHW, %g3 ;\ 195 sub %g0, 1, %g4 ;\ 196 save ;\ 197 flushw ;\ 198 restore ;\ 199 FAST_TRAP_DONE ;\ 200 .align 32 201 202/* 203 * GOTO just jumps to a label. 204 * It's used for things that can be fixed without going thru sys_trap. 205 */ 206#define GOTO(label) \ 207 .global label ;\ 208 ba,a label ;\ 209 .empty ;\ 210 .align 32 211 212/* 213 * GOTO_TT just jumps to a label. 214 * correctable ECC error traps at level 0 and 1 will use this macro. 215 * It's used for things that can be fixed without going thru sys_trap. 216 */ 217#define GOTO_TT(label, ttlabel) \ 218 .global label ;\ 219 TT_TRACE(ttlabel) ;\ 220 ba,a label ;\ 221 .empty ;\ 222 .align 32 223 224/* 225 * Privileged traps 226 * Takes breakpoint if privileged, calls trap() if not. 227 */ 228#define PRIV(label) \ 229 rdpr %tstate, %g1 ;\ 230 btst TSTATE_PRIV, %g1 ;\ 231 bnz label ;\ 232 rdpr %tt, %g3 ;\ 233 set trap, %g1 ;\ 234 ba,pt %xcc, sys_trap ;\ 235 sub %g0, 1, %g4 ;\ 236 .align 32 237 238 239/* 240 * DTrace traps. 241 */ 242#define DTRACE_PID \ 243 .global dtrace_pid_probe ;\ 244 set dtrace_pid_probe, %g1 ;\ 245 ba,pt %xcc, user_trap ;\ 246 sub %g0, 1, %g4 ;\ 247 .align 32 248 249#define DTRACE_RETURN \ 250 .global dtrace_return_probe ;\ 251 set dtrace_return_probe, %g1 ;\ 252 ba,pt %xcc, user_trap ;\ 253 sub %g0, 1, %g4 ;\ 254 .align 32 255 256/* 257 * REGISTER WINDOW MANAGEMENT MACROS 258 */ 259 260/* 261 * various convenient units of padding 262 */ 263#define SKIP(n) .skip 4*(n) 264 265/* 266 * CLEAN_WINDOW is the simple handler for cleaning a register window. 267 */ 268#define CLEAN_WINDOW \ 269 TT_TRACE_L(trace_win) ;\ 270 rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin ;\ 271 clr %l0; clr %l1; clr %l2; clr %l3 ;\ 272 clr %l4; clr %l5; clr %l6; clr %l7 ;\ 273 clr %o0; clr %o1; clr %o2; clr %o3 ;\ 274 clr %o4; clr %o5; clr %o6; clr %o7 ;\ 275 retry; .align 128 276 277/* 278 * If we get an unresolved tlb miss while in a window handler, the fault 279 * handler will resume execution at the last instruction of the window 280 * hander, instead of delivering the fault to the kernel. Spill handlers 281 * use this to spill windows into the wbuf. 282 * 283 * The mixed handler works by checking %sp, and branching to the correct 284 * handler. This is done by branching back to label 1: for 32b frames, 285 * or label 2: for 64b frames; which implies the handler order is: 32b, 286 * 64b, mixed. The 1: and 2: labels are offset into the routines to 287 * allow the branchs' delay slots to contain useful instructions. 288 */ 289 290/* 291 * SPILL_32bit spills a 32-bit-wide kernel register window. It 292 * assumes that the kernel context and the nucleus context are the 293 * same. The stack pointer is required to be eight-byte aligned even 294 * though this code only needs it to be four-byte aligned. 295 */ 296#define SPILL_32bit(tail) \ 297 srl %sp, 0, %sp ;\ 2981: st %l0, [%sp + 0] ;\ 299 st %l1, [%sp + 4] ;\ 300 st %l2, [%sp + 8] ;\ 301 st %l3, [%sp + 12] ;\ 302 st %l4, [%sp + 16] ;\ 303 st %l5, [%sp + 20] ;\ 304 st %l6, [%sp + 24] ;\ 305 st %l7, [%sp + 28] ;\ 306 st %i0, [%sp + 32] ;\ 307 st %i1, [%sp + 36] ;\ 308 st %i2, [%sp + 40] ;\ 309 st %i3, [%sp + 44] ;\ 310 st %i4, [%sp + 48] ;\ 311 st %i5, [%sp + 52] ;\ 312 st %i6, [%sp + 56] ;\ 313 st %i7, [%sp + 60] ;\ 314 TT_TRACE_L(trace_win) ;\ 315 saved ;\ 316 retry ;\ 317 SKIP(31-19-TT_TRACE_L_INS) ;\ 318 ba,a,pt %xcc, fault_32bit_##tail ;\ 319 .empty 320 321/* 322 * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit 323 * wide address space via the designated asi. It is used to spill 324 * non-kernel windows. The stack pointer is required to be eight-byte 325 * aligned even though this code only needs it to be four-byte 326 * aligned. 327 */ 328#define SPILL_32bit_asi(asi_num, tail) \ 329 srl %sp, 0, %sp ;\ 3301: sta %l0, [%sp + %g0]asi_num ;\ 331 mov 4, %g1 ;\ 332 sta %l1, [%sp + %g1]asi_num ;\ 333 mov 8, %g2 ;\ 334 sta %l2, [%sp + %g2]asi_num ;\ 335 mov 12, %g3 ;\ 336 sta %l3, [%sp + %g3]asi_num ;\ 337 add %sp, 16, %g4 ;\ 338 sta %l4, [%g4 + %g0]asi_num ;\ 339 sta %l5, [%g4 + %g1]asi_num ;\ 340 sta %l6, [%g4 + %g2]asi_num ;\ 341 sta %l7, [%g4 + %g3]asi_num ;\ 342 add %g4, 16, %g4 ;\ 343 sta %i0, [%g4 + %g0]asi_num ;\ 344 sta %i1, [%g4 + %g1]asi_num ;\ 345 sta %i2, [%g4 + %g2]asi_num ;\ 346 sta %i3, [%g4 + %g3]asi_num ;\ 347 add %g4, 16, %g4 ;\ 348 sta %i4, [%g4 + %g0]asi_num ;\ 349 sta %i5, [%g4 + %g1]asi_num ;\ 350 sta %i6, [%g4 + %g2]asi_num ;\ 351 sta %i7, [%g4 + %g3]asi_num ;\ 352 TT_TRACE_L(trace_win) ;\ 353 saved ;\ 354 retry ;\ 355 SKIP(31-25-TT_TRACE_L_INS) ;\ 356 ba,a,pt %xcc, fault_32bit_##tail ;\ 357 .empty 358 359/* 360 * SPILL_32bit_tt1 spills a 32-bit-wide register window into a 32-bit 361 * wide address space via the designated asi. It is used to spill 362 * windows at tl>1 where performance isn't the primary concern and 363 * where we don't want to use unnecessary registers. The stack 364 * pointer is required to be eight-byte aligned even though this code 365 * only needs it to be four-byte aligned. 366 */ 367#define SPILL_32bit_tt1(asi_num, tail) \ 368 mov asi_num, %asi ;\ 3691: srl %sp, 0, %sp ;\ 370 sta %l0, [%sp + 0]%asi ;\ 371 sta %l1, [%sp + 4]%asi ;\ 372 sta %l2, [%sp + 8]%asi ;\ 373 sta %l3, [%sp + 12]%asi ;\ 374 sta %l4, [%sp + 16]%asi ;\ 375 sta %l5, [%sp + 20]%asi ;\ 376 sta %l6, [%sp + 24]%asi ;\ 377 sta %l7, [%sp + 28]%asi ;\ 378 sta %i0, [%sp + 32]%asi ;\ 379 sta %i1, [%sp + 36]%asi ;\ 380 sta %i2, [%sp + 40]%asi ;\ 381 sta %i3, [%sp + 44]%asi ;\ 382 sta %i4, [%sp + 48]%asi ;\ 383 sta %i5, [%sp + 52]%asi ;\ 384 sta %i6, [%sp + 56]%asi ;\ 385 sta %i7, [%sp + 60]%asi ;\ 386 TT_TRACE_L(trace_win) ;\ 387 saved ;\ 388 retry ;\ 389 SKIP(31-20-TT_TRACE_L_INS) ;\ 390 ba,a,pt %xcc, fault_32bit_##tail ;\ 391 .empty 392 393 394/* 395 * FILL_32bit fills a 32-bit-wide kernel register window. It assumes 396 * that the kernel context and the nucleus context are the same. The 397 * stack pointer is required to be eight-byte aligned even though this 398 * code only needs it to be four-byte aligned. 399 */ 400#define FILL_32bit(tail) \ 401 srl %sp, 0, %sp ;\ 4021: TT_TRACE_L(trace_win) ;\ 403 ld [%sp + 0], %l0 ;\ 404 ld [%sp + 4], %l1 ;\ 405 ld [%sp + 8], %l2 ;\ 406 ld [%sp + 12], %l3 ;\ 407 ld [%sp + 16], %l4 ;\ 408 ld [%sp + 20], %l5 ;\ 409 ld [%sp + 24], %l6 ;\ 410 ld [%sp + 28], %l7 ;\ 411 ld [%sp + 32], %i0 ;\ 412 ld [%sp + 36], %i1 ;\ 413 ld [%sp + 40], %i2 ;\ 414 ld [%sp + 44], %i3 ;\ 415 ld [%sp + 48], %i4 ;\ 416 ld [%sp + 52], %i5 ;\ 417 ld [%sp + 56], %i6 ;\ 418 ld [%sp + 60], %i7 ;\ 419 restored ;\ 420 retry ;\ 421 SKIP(31-19-TT_TRACE_L_INS) ;\ 422 ba,a,pt %xcc, fault_32bit_##tail ;\ 423 .empty 424 425/* 426 * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit 427 * wide address space via the designated asi. It is used to fill 428 * non-kernel windows. The stack pointer is required to be eight-byte 429 * aligned even though this code only needs it to be four-byte 430 * aligned. 431 */ 432#define FILL_32bit_asi(asi_num, tail) \ 433 srl %sp, 0, %sp ;\ 4341: TT_TRACE_L(trace_win) ;\ 435 mov 4, %g1 ;\ 436 lda [%sp + %g0]asi_num, %l0 ;\ 437 mov 8, %g2 ;\ 438 lda [%sp + %g1]asi_num, %l1 ;\ 439 mov 12, %g3 ;\ 440 lda [%sp + %g2]asi_num, %l2 ;\ 441 lda [%sp + %g3]asi_num, %l3 ;\ 442 add %sp, 16, %g4 ;\ 443 lda [%g4 + %g0]asi_num, %l4 ;\ 444 lda [%g4 + %g1]asi_num, %l5 ;\ 445 lda [%g4 + %g2]asi_num, %l6 ;\ 446 lda [%g4 + %g3]asi_num, %l7 ;\ 447 add %g4, 16, %g4 ;\ 448 lda [%g4 + %g0]asi_num, %i0 ;\ 449 lda [%g4 + %g1]asi_num, %i1 ;\ 450 lda [%g4 + %g2]asi_num, %i2 ;\ 451 lda [%g4 + %g3]asi_num, %i3 ;\ 452 add %g4, 16, %g4 ;\ 453 lda [%g4 + %g0]asi_num, %i4 ;\ 454 lda [%g4 + %g1]asi_num, %i5 ;\ 455 lda [%g4 + %g2]asi_num, %i6 ;\ 456 lda [%g4 + %g3]asi_num, %i7 ;\ 457 restored ;\ 458 retry ;\ 459 SKIP(31-25-TT_TRACE_L_INS) ;\ 460 ba,a,pt %xcc, fault_32bit_##tail ;\ 461 .empty 462 463/* 464 * FILL_32bit_tt1 fills a 32-bit-wide register window from a 32-bit 465 * wide address space via the designated asi. It is used to fill 466 * windows at tl>1 where performance isn't the primary concern and 467 * where we don't want to use unnecessary registers. The stack 468 * pointer is required to be eight-byte aligned even though this code 469 * only needs it to be four-byte aligned. 470 */ 471#define FILL_32bit_tt1(asi_num, tail) \ 472 mov asi_num, %asi ;\ 4731: srl %sp, 0, %sp ;\ 474 TT_TRACE_L(trace_win) ;\ 475 lda [%sp + 0]%asi, %l0 ;\ 476 lda [%sp + 4]%asi, %l1 ;\ 477 lda [%sp + 8]%asi, %l2 ;\ 478 lda [%sp + 12]%asi, %l3 ;\ 479 lda [%sp + 16]%asi, %l4 ;\ 480 lda [%sp + 20]%asi, %l5 ;\ 481 lda [%sp + 24]%asi, %l6 ;\ 482 lda [%sp + 28]%asi, %l7 ;\ 483 lda [%sp + 32]%asi, %i0 ;\ 484 lda [%sp + 36]%asi, %i1 ;\ 485 lda [%sp + 40]%asi, %i2 ;\ 486 lda [%sp + 44]%asi, %i3 ;\ 487 lda [%sp + 48]%asi, %i4 ;\ 488 lda [%sp + 52]%asi, %i5 ;\ 489 lda [%sp + 56]%asi, %i6 ;\ 490 lda [%sp + 60]%asi, %i7 ;\ 491 restored ;\ 492 retry ;\ 493 SKIP(31-20-TT_TRACE_L_INS) ;\ 494 ba,a,pt %xcc, fault_32bit_##tail ;\ 495 .empty 496 497 498/* 499 * SPILL_64bit spills a 64-bit-wide kernel register window. It 500 * assumes that the kernel context and the nucleus context are the 501 * same. The stack pointer is required to be eight-byte aligned. 502 */ 503#define SPILL_64bit(tail) \ 5042: stx %l0, [%sp + V9BIAS64 + 0] ;\ 505 stx %l1, [%sp + V9BIAS64 + 8] ;\ 506 stx %l2, [%sp + V9BIAS64 + 16] ;\ 507 stx %l3, [%sp + V9BIAS64 + 24] ;\ 508 stx %l4, [%sp + V9BIAS64 + 32] ;\ 509 stx %l5, [%sp + V9BIAS64 + 40] ;\ 510 stx %l6, [%sp + V9BIAS64 + 48] ;\ 511 stx %l7, [%sp + V9BIAS64 + 56] ;\ 512 stx %i0, [%sp + V9BIAS64 + 64] ;\ 513 stx %i1, [%sp + V9BIAS64 + 72] ;\ 514 stx %i2, [%sp + V9BIAS64 + 80] ;\ 515 stx %i3, [%sp + V9BIAS64 + 88] ;\ 516 stx %i4, [%sp + V9BIAS64 + 96] ;\ 517 stx %i5, [%sp + V9BIAS64 + 104] ;\ 518 stx %i6, [%sp + V9BIAS64 + 112] ;\ 519 stx %i7, [%sp + V9BIAS64 + 120] ;\ 520 TT_TRACE_L(trace_win) ;\ 521 saved ;\ 522 retry ;\ 523 SKIP(31-18-TT_TRACE_L_INS) ;\ 524 ba,a,pt %xcc, fault_64bit_##tail ;\ 525 .empty 526 527/* 528 * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit 529 * wide address space via the designated asi. It is used to spill 530 * non-kernel windows. The stack pointer is required to be eight-byte 531 * aligned. 532 */ 533#define SPILL_64bit_asi(asi_num, tail) \ 534 mov 0 + V9BIAS64, %g1 ;\ 5352: stxa %l0, [%sp + %g1]asi_num ;\ 536 mov 8 + V9BIAS64, %g2 ;\ 537 stxa %l1, [%sp + %g2]asi_num ;\ 538 mov 16 + V9BIAS64, %g3 ;\ 539 stxa %l2, [%sp + %g3]asi_num ;\ 540 mov 24 + V9BIAS64, %g4 ;\ 541 stxa %l3, [%sp + %g4]asi_num ;\ 542 add %sp, 32, %g5 ;\ 543 stxa %l4, [%g5 + %g1]asi_num ;\ 544 stxa %l5, [%g5 + %g2]asi_num ;\ 545 stxa %l6, [%g5 + %g3]asi_num ;\ 546 stxa %l7, [%g5 + %g4]asi_num ;\ 547 add %g5, 32, %g5 ;\ 548 stxa %i0, [%g5 + %g1]asi_num ;\ 549 stxa %i1, [%g5 + %g2]asi_num ;\ 550 stxa %i2, [%g5 + %g3]asi_num ;\ 551 stxa %i3, [%g5 + %g4]asi_num ;\ 552 add %g5, 32, %g5 ;\ 553 stxa %i4, [%g5 + %g1]asi_num ;\ 554 stxa %i5, [%g5 + %g2]asi_num ;\ 555 stxa %i6, [%g5 + %g3]asi_num ;\ 556 stxa %i7, [%g5 + %g4]asi_num ;\ 557 TT_TRACE_L(trace_win) ;\ 558 saved ;\ 559 retry ;\ 560 SKIP(31-25-TT_TRACE_L_INS) ;\ 561 ba,a,pt %xcc, fault_64bit_##tail ;\ 562 .empty 563 564/* 565 * SPILL_64bit_tt1 spills a 64-bit-wide register window into a 64-bit 566 * wide address space via the designated asi. It is used to spill 567 * windows at tl>1 where performance isn't the primary concern and 568 * where we don't want to use unnecessary registers. The stack 569 * pointer is required to be eight-byte aligned. 570 */ 571#define SPILL_64bit_tt1(asi_num, tail) \ 572 mov asi_num, %asi ;\ 5732: stxa %l0, [%sp + V9BIAS64 + 0]%asi ;\ 574 stxa %l1, [%sp + V9BIAS64 + 8]%asi ;\ 575 stxa %l2, [%sp + V9BIAS64 + 16]%asi ;\ 576 stxa %l3, [%sp + V9BIAS64 + 24]%asi ;\ 577 stxa %l4, [%sp + V9BIAS64 + 32]%asi ;\ 578 stxa %l5, [%sp + V9BIAS64 + 40]%asi ;\ 579 stxa %l6, [%sp + V9BIAS64 + 48]%asi ;\ 580 stxa %l7, [%sp + V9BIAS64 + 56]%asi ;\ 581 stxa %i0, [%sp + V9BIAS64 + 64]%asi ;\ 582 stxa %i1, [%sp + V9BIAS64 + 72]%asi ;\ 583 stxa %i2, [%sp + V9BIAS64 + 80]%asi ;\ 584 stxa %i3, [%sp + V9BIAS64 + 88]%asi ;\ 585 stxa %i4, [%sp + V9BIAS64 + 96]%asi ;\ 586 stxa %i5, [%sp + V9BIAS64 + 104]%asi ;\ 587 stxa %i6, [%sp + V9BIAS64 + 112]%asi ;\ 588 stxa %i7, [%sp + V9BIAS64 + 120]%asi ;\ 589 TT_TRACE_L(trace_win) ;\ 590 saved ;\ 591 retry ;\ 592 SKIP(31-19-TT_TRACE_L_INS) ;\ 593 ba,a,pt %xcc, fault_64bit_##tail ;\ 594 .empty 595 596 597/* 598 * FILL_64bit fills a 64-bit-wide kernel register window. It assumes 599 * that the kernel context and the nucleus context are the same. The 600 * stack pointer is required to be eight-byte aligned. 601 */ 602#define FILL_64bit(tail) \ 6032: TT_TRACE_L(trace_win) ;\ 604 ldx [%sp + V9BIAS64 + 0], %l0 ;\ 605 ldx [%sp + V9BIAS64 + 8], %l1 ;\ 606 ldx [%sp + V9BIAS64 + 16], %l2 ;\ 607 ldx [%sp + V9BIAS64 + 24], %l3 ;\ 608 ldx [%sp + V9BIAS64 + 32], %l4 ;\ 609 ldx [%sp + V9BIAS64 + 40], %l5 ;\ 610 ldx [%sp + V9BIAS64 + 48], %l6 ;\ 611 ldx [%sp + V9BIAS64 + 56], %l7 ;\ 612 ldx [%sp + V9BIAS64 + 64], %i0 ;\ 613 ldx [%sp + V9BIAS64 + 72], %i1 ;\ 614 ldx [%sp + V9BIAS64 + 80], %i2 ;\ 615 ldx [%sp + V9BIAS64 + 88], %i3 ;\ 616 ldx [%sp + V9BIAS64 + 96], %i4 ;\ 617 ldx [%sp + V9BIAS64 + 104], %i5 ;\ 618 ldx [%sp + V9BIAS64 + 112], %i6 ;\ 619 ldx [%sp + V9BIAS64 + 120], %i7 ;\ 620 restored ;\ 621 retry ;\ 622 SKIP(31-18-TT_TRACE_L_INS) ;\ 623 ba,a,pt %xcc, fault_64bit_##tail ;\ 624 .empty 625 626/* 627 * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit 628 * wide address space via the designated asi. It is used to fill 629 * non-kernel windows. The stack pointer is required to be eight-byte 630 * aligned. 631 */ 632#define FILL_64bit_asi(asi_num, tail) \ 633 mov V9BIAS64 + 0, %g1 ;\ 6342: TT_TRACE_L(trace_win) ;\ 635 ldxa [%sp + %g1]asi_num, %l0 ;\ 636 mov V9BIAS64 + 8, %g2 ;\ 637 ldxa [%sp + %g2]asi_num, %l1 ;\ 638 mov V9BIAS64 + 16, %g3 ;\ 639 ldxa [%sp + %g3]asi_num, %l2 ;\ 640 mov V9BIAS64 + 24, %g4 ;\ 641 ldxa [%sp + %g4]asi_num, %l3 ;\ 642 add %sp, 32, %g5 ;\ 643 ldxa [%g5 + %g1]asi_num, %l4 ;\ 644 ldxa [%g5 + %g2]asi_num, %l5 ;\ 645 ldxa [%g5 + %g3]asi_num, %l6 ;\ 646 ldxa [%g5 + %g4]asi_num, %l7 ;\ 647 add %g5, 32, %g5 ;\ 648 ldxa [%g5 + %g1]asi_num, %i0 ;\ 649 ldxa [%g5 + %g2]asi_num, %i1 ;\ 650 ldxa [%g5 + %g3]asi_num, %i2 ;\ 651 ldxa [%g5 + %g4]asi_num, %i3 ;\ 652 add %g5, 32, %g5 ;\ 653 ldxa [%g5 + %g1]asi_num, %i4 ;\ 654 ldxa [%g5 + %g2]asi_num, %i5 ;\ 655 ldxa [%g5 + %g3]asi_num, %i6 ;\ 656 ldxa [%g5 + %g4]asi_num, %i7 ;\ 657 restored ;\ 658 retry ;\ 659 SKIP(31-25-TT_TRACE_L_INS) ;\ 660 ba,a,pt %xcc, fault_64bit_##tail ;\ 661 .empty 662 663/* 664 * FILL_64bit_tt1 fills a 64-bit-wide register window from a 64-bit 665 * wide address space via the designated asi. It is used to fill 666 * windows at tl>1 where performance isn't the primary concern and 667 * where we don't want to use unnecessary registers. The stack 668 * pointer is required to be eight-byte aligned. 669 */ 670#define FILL_64bit_tt1(asi_num, tail) \ 671 mov asi_num, %asi ;\ 672 TT_TRACE_L(trace_win) ;\ 673 ldxa [%sp + V9BIAS64 + 0]%asi, %l0 ;\ 674 ldxa [%sp + V9BIAS64 + 8]%asi, %l1 ;\ 675 ldxa [%sp + V9BIAS64 + 16]%asi, %l2 ;\ 676 ldxa [%sp + V9BIAS64 + 24]%asi, %l3 ;\ 677 ldxa [%sp + V9BIAS64 + 32]%asi, %l4 ;\ 678 ldxa [%sp + V9BIAS64 + 40]%asi, %l5 ;\ 679 ldxa [%sp + V9BIAS64 + 48]%asi, %l6 ;\ 680 ldxa [%sp + V9BIAS64 + 56]%asi, %l7 ;\ 681 ldxa [%sp + V9BIAS64 + 64]%asi, %i0 ;\ 682 ldxa [%sp + V9BIAS64 + 72]%asi, %i1 ;\ 683 ldxa [%sp + V9BIAS64 + 80]%asi, %i2 ;\ 684 ldxa [%sp + V9BIAS64 + 88]%asi, %i3 ;\ 685 ldxa [%sp + V9BIAS64 + 96]%asi, %i4 ;\ 686 ldxa [%sp + V9BIAS64 + 104]%asi, %i5 ;\ 687 ldxa [%sp + V9BIAS64 + 112]%asi, %i6 ;\ 688 ldxa [%sp + V9BIAS64 + 120]%asi, %i7 ;\ 689 restored ;\ 690 retry ;\ 691 SKIP(31-19-TT_TRACE_L_INS) ;\ 692 ba,a,pt %xcc, fault_64bit_##tail ;\ 693 .empty 694 695/* 696 * SPILL_mixed spills either size window, depending on 697 * whether %sp is even or odd, to a 32-bit address space. 698 * This may only be used in conjunction with SPILL_32bit/ 699 * SPILL_64bit. New versions of SPILL_mixed_{tt1,asi} would be 700 * needed for use with SPILL_{32,64}bit_{tt1,asi}. Particular 701 * attention should be paid to the instructions that belong 702 * in the delay slots of the branches depending on the type 703 * of spill handler being branched to. 704 * Clear upper 32 bits of %sp if it is odd. 705 * We won't need to clear them in 64 bit kernel. 706 */ 707#define SPILL_mixed \ 708 btst 1, %sp ;\ 709 bz,a,pt %xcc, 1b ;\ 710 srl %sp, 0, %sp ;\ 711 ba,pt %xcc, 2b ;\ 712 nop ;\ 713 .align 128 714 715/* 716 * FILL_mixed(ASI) fills either size window, depending on 717 * whether %sp is even or odd, from a 32-bit address space. 718 * This may only be used in conjunction with FILL_32bit/ 719 * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be 720 * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular 721 * attention should be paid to the instructions that belong 722 * in the delay slots of the branches depending on the type 723 * of fill handler being branched to. 724 * Clear upper 32 bits of %sp if it is odd. 725 * We won't need to clear them in 64 bit kernel. 726 */ 727#define FILL_mixed \ 728 btst 1, %sp ;\ 729 bz,a,pt %xcc, 1b ;\ 730 srl %sp, 0, %sp ;\ 731 ba,pt %xcc, 2b ;\ 732 nop ;\ 733 .align 128 734 735 736/* 737 * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows, 738 * respectively, into the address space via the designated asi. The 739 * unbiased stack pointer is required to be eight-byte aligned (even for 740 * the 32-bit case even though this code does not require such strict 741 * alignment). 742 * 743 * With SPARC v9 the spill trap takes precedence over the cleanwin trap 744 * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save 745 * will cause cwp + 2 to be spilled but will not clean cwp + 1. That 746 * window may contain kernel data so in user_rtt we set wstate to call 747 * these spill handlers on the first user spill trap. These handler then 748 * spill the appropriate window but also back up a window and clean the 749 * window that didn't get a cleanwin trap. 750 */ 751#define SPILL_32clean(asi_num, tail) \ 752 srl %sp, 0, %sp ;\ 753 sta %l0, [%sp + %g0]asi_num ;\ 754 mov 4, %g1 ;\ 755 sta %l1, [%sp + %g1]asi_num ;\ 756 mov 8, %g2 ;\ 757 sta %l2, [%sp + %g2]asi_num ;\ 758 mov 12, %g3 ;\ 759 sta %l3, [%sp + %g3]asi_num ;\ 760 add %sp, 16, %g4 ;\ 761 sta %l4, [%g4 + %g0]asi_num ;\ 762 sta %l5, [%g4 + %g1]asi_num ;\ 763 sta %l6, [%g4 + %g2]asi_num ;\ 764 sta %l7, [%g4 + %g3]asi_num ;\ 765 add %g4, 16, %g4 ;\ 766 sta %i0, [%g4 + %g0]asi_num ;\ 767 sta %i1, [%g4 + %g1]asi_num ;\ 768 sta %i2, [%g4 + %g2]asi_num ;\ 769 sta %i3, [%g4 + %g3]asi_num ;\ 770 add %g4, 16, %g4 ;\ 771 sta %i4, [%g4 + %g0]asi_num ;\ 772 sta %i5, [%g4 + %g1]asi_num ;\ 773 sta %i6, [%g4 + %g2]asi_num ;\ 774 sta %i7, [%g4 + %g3]asi_num ;\ 775 TT_TRACE_L(trace_win) ;\ 776 b .spill_clean ;\ 777 mov WSTATE_USER32, %g7 ;\ 778 SKIP(31-25-TT_TRACE_L_INS) ;\ 779 ba,a,pt %xcc, fault_32bit_##tail ;\ 780 .empty 781 782#define SPILL_64clean(asi_num, tail) \ 783 mov 0 + V9BIAS64, %g1 ;\ 784 stxa %l0, [%sp + %g1]asi_num ;\ 785 mov 8 + V9BIAS64, %g2 ;\ 786 stxa %l1, [%sp + %g2]asi_num ;\ 787 mov 16 + V9BIAS64, %g3 ;\ 788 stxa %l2, [%sp + %g3]asi_num ;\ 789 mov 24 + V9BIAS64, %g4 ;\ 790 stxa %l3, [%sp + %g4]asi_num ;\ 791 add %sp, 32, %g5 ;\ 792 stxa %l4, [%g5 + %g1]asi_num ;\ 793 stxa %l5, [%g5 + %g2]asi_num ;\ 794 stxa %l6, [%g5 + %g3]asi_num ;\ 795 stxa %l7, [%g5 + %g4]asi_num ;\ 796 add %g5, 32, %g5 ;\ 797 stxa %i0, [%g5 + %g1]asi_num ;\ 798 stxa %i1, [%g5 + %g2]asi_num ;\ 799 stxa %i2, [%g5 + %g3]asi_num ;\ 800 stxa %i3, [%g5 + %g4]asi_num ;\ 801 add %g5, 32, %g5 ;\ 802 stxa %i4, [%g5 + %g1]asi_num ;\ 803 stxa %i5, [%g5 + %g2]asi_num ;\ 804 stxa %i6, [%g5 + %g3]asi_num ;\ 805 stxa %i7, [%g5 + %g4]asi_num ;\ 806 TT_TRACE_L(trace_win) ;\ 807 b .spill_clean ;\ 808 mov WSTATE_USER64, %g7 ;\ 809 SKIP(31-25-TT_TRACE_L_INS) ;\ 810 ba,a,pt %xcc, fault_64bit_##tail ;\ 811 .empty 812 813 814/* 815 * Floating point disabled. 816 */ 817#define FP_DISABLED_TRAP \ 818 TT_TRACE(trace_gen) ;\ 819 ba,pt %xcc,.fp_disabled ;\ 820 nop ;\ 821 .align 32 822 823/* 824 * Floating point exceptions. 825 */ 826#define FP_IEEE_TRAP \ 827 TT_TRACE(trace_gen) ;\ 828 ba,pt %xcc,.fp_ieee_exception ;\ 829 nop ;\ 830 .align 32 831 832#define FP_TRAP \ 833 TT_TRACE(trace_gen) ;\ 834 ba,pt %xcc,.fp_exception ;\ 835 nop ;\ 836 .align 32 837 838/* 839 * asynchronous traps at level 0 and level 1 840 * 841 * The first instruction must be a membar for UltraSPARC-III 842 * to stop RED state entry if the store queue has many 843 * pending bad stores (PRM, Chapter 11). 844 */ 845#define ASYNC_TRAP(ttype, ttlabel, table_name)\ 846 .global table_name ;\ 847table_name: ;\ 848 membar #Sync ;\ 849 TT_TRACE(ttlabel) ;\ 850 ba async_err ;\ 851 mov ttype, %g5 ;\ 852 .align 32 853 854/* 855 * Defaults to BAD entry, but establishes label to be used for 856 * architecture-specific overwrite of trap table entry. 857 */ 858#define LABELED_BAD(table_name) \ 859 .global table_name ;\ 860table_name: ;\ 861 BAD 862 863/* 864 * illegal instruction trap 865 */ 866#define ILLTRAP_INSTR \ 867 membar #Sync ;\ 868 TT_TRACE(trace_gen) ;\ 869 or %g0, P_UTRAP4, %g2 ;\ 870 or %g0, T_UNIMP_INSTR, %g3 ;\ 871 sethi %hi(.check_v9utrap), %g4 ;\ 872 jmp %g4 + %lo(.check_v9utrap) ;\ 873 nop ;\ 874 .align 32 875 876/* 877 * tag overflow trap 878 */ 879#define TAG_OVERFLOW \ 880 TT_TRACE(trace_gen) ;\ 881 or %g0, P_UTRAP10, %g2 ;\ 882 or %g0, T_TAG_OVERFLOW, %g3 ;\ 883 sethi %hi(.check_v9utrap), %g4 ;\ 884 jmp %g4 + %lo(.check_v9utrap) ;\ 885 nop ;\ 886 .align 32 887 888/* 889 * divide by zero trap 890 */ 891#define DIV_BY_ZERO \ 892 TT_TRACE(trace_gen) ;\ 893 or %g0, P_UTRAP11, %g2 ;\ 894 or %g0, T_IDIV0, %g3 ;\ 895 sethi %hi(.check_v9utrap), %g4 ;\ 896 jmp %g4 + %lo(.check_v9utrap) ;\ 897 nop ;\ 898 .align 32 899 900/* 901 * trap instruction for V9 user trap handlers 902 */ 903#define TRAP_INSTR \ 904 TT_TRACE(trace_gen) ;\ 905 or %g0, T_SOFTWARE_TRAP, %g3 ;\ 906 sethi %hi(.check_v9utrap), %g4 ;\ 907 jmp %g4 + %lo(.check_v9utrap) ;\ 908 nop ;\ 909 .align 32 910#define TRP4 TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR 911 912/* 913 * LEVEL_INTERRUPT is for level N interrupts. 914 * VECTOR_INTERRUPT is for the vector trap. 915 */ 916#define LEVEL_INTERRUPT(level) \ 917 .global tt_pil##level ;\ 918tt_pil##level: ;\ 919 ba,pt %xcc, pil_interrupt ;\ 920 mov level, %g4 ;\ 921 .align 32 922 923#define LEVEL14_INTERRUPT \ 924 ba pil14_interrupt ;\ 925 mov PIL_14, %g4 ;\ 926 .align 32 927 928#define LEVEL15_INTERRUPT \ 929 ba pil15_interrupt ;\ 930 mov PIL_15, %g4 ;\ 931 .align 32 932 933#define VECTOR_INTERRUPT \ 934 ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g1 ;\ 935 btst IRSR_BUSY, %g1 ;\ 936 bnz,pt %xcc, vec_interrupt ;\ 937 nop ;\ 938 ba,a,pt %xcc, vec_intr_spurious ;\ 939 .empty ;\ 940 .align 32 941 942/* 943 * MMU Trap Handlers. 944 */ 945#define SWITCH_GLOBALS /* mmu->alt, alt->mmu */ \ 946 rdpr %pstate, %g5 ;\ 947 wrpr %g5, PSTATE_MG | PSTATE_AG, %pstate 948 949#define IMMU_EXCEPTION \ 950 membar #Sync ;\ 951 SWITCH_GLOBALS ;\ 952 wr %g0, ASI_IMMU, %asi ;\ 953 rdpr %tpc, %g2 ;\ 954 ldxa [MMU_SFSR]%asi, %g3 ;\ 955 ba,pt %xcc, .mmu_exception_end ;\ 956 mov T_INSTR_EXCEPTION, %g1 ;\ 957 .align 32 958 959#define DMMU_EXCEPTION \ 960 SWITCH_GLOBALS ;\ 961 wr %g0, ASI_DMMU, %asi ;\ 962 ldxa [MMU_TAG_ACCESS]%asi, %g2 ;\ 963 ldxa [MMU_SFSR]%asi, %g3 ;\ 964 ba,pt %xcc, .mmu_exception_end ;\ 965 mov T_DATA_EXCEPTION, %g1 ;\ 966 .align 32 967 968#define DMMU_EXC_AG_PRIV \ 969 wr %g0, ASI_DMMU, %asi ;\ 970 ldxa [MMU_SFAR]%asi, %g2 ;\ 971 ba,pt %xcc, .mmu_priv_exception ;\ 972 ldxa [MMU_SFSR]%asi, %g3 ;\ 973 .align 32 974 975#define DMMU_EXC_AG_NOT_ALIGNED \ 976 wr %g0, ASI_DMMU, %asi ;\ 977 ldxa [MMU_SFAR]%asi, %g2 ;\ 978 ba,pt %xcc, .mmu_exception_not_aligned ;\ 979 ldxa [MMU_SFSR]%asi, %g3 ;\ 980 .align 32 981 982/* 983 * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2) 984 */ 985#define DMMU_EXC_LDDF_NOT_ALIGNED \ 986 btst 1, %sp ;\ 987 bnz,pt %xcc, .lddf_exception_not_aligned ;\ 988 wr %g0, ASI_DMMU, %asi ;\ 989 ldxa [MMU_SFAR]%asi, %g2 ;\ 990 ba,pt %xcc, .mmu_exception_not_aligned ;\ 991 ldxa [MMU_SFSR]%asi, %g3 ;\ 992 .align 32 993 994#define DMMU_EXC_STDF_NOT_ALIGNED \ 995 btst 1, %sp ;\ 996 bnz,pt %xcc, .stdf_exception_not_aligned ;\ 997 wr %g0, ASI_DMMU, %asi ;\ 998 ldxa [MMU_SFAR]%asi, %g2 ;\ 999 ba,pt %xcc, .mmu_exception_not_aligned ;\ 1000 ldxa [MMU_SFSR]%asi, %g3 ;\ 1001 .align 32 1002 1003/* 1004 * Flush the TLB using either the primary, secondary, or nucleus flush 1005 * operation based on whether the ctx from the tag access register matches 1006 * the primary or secondary context (flush the nucleus if neither matches). 1007 * 1008 * Requires a membar #Sync before next ld/st. 1009 * exits with: 1010 * g2 = tag access register 1011 * g3 = ctx number 1012 */ 1013#if TAGACC_CTX_MASK != CTXREG_CTX_MASK 1014#error "TAGACC_CTX_MASK != CTXREG_CTX_MASK" 1015#endif 1016#define DTLB_DEMAP_ENTRY \ 1017 mov MMU_TAG_ACCESS, %g1 ;\ 1018 mov MMU_PCONTEXT, %g5 ;\ 1019 ldxa [%g1]ASI_DMMU, %g2 ;\ 1020 sethi %hi(TAGACC_CTX_MASK), %g4 ;\ 1021 or %g4, %lo(TAGACC_CTX_MASK), %g4 ;\ 1022 and %g2, %g4, %g3 /* g3 = ctx */ ;\ 1023 ldxa [%g5]ASI_DMMU, %g6 /* g6 = primary ctx */ ;\ 1024 and %g6, %g4, %g6 /* &= CTXREG_CTX_MASK */ ;\ 1025 cmp %g3, %g6 ;\ 1026 be,pt %xcc, 1f ;\ 1027 andn %g2, %g4, %g1 /* ctx = primary */ ;\ 1028 mov MMU_SCONTEXT, %g5 ;\ 1029 ldxa [%g5]ASI_DMMU, %g6 /* g6 = secondary ctx */ ;\ 1030 and %g6, %g4, %g6 /* &= CTXREG_CTX_MASK */ ;\ 1031 cmp %g3, %g6 ;\ 1032 be,a,pt %xcc, 1f ;\ 1033 or %g1, DEMAP_SECOND, %g1 ;\ 1034 or %g1, DEMAP_NUCLEUS, %g1 ;\ 10351: stxa %g0, [%g1]ASI_DTLB_DEMAP /* MMU_DEMAP_PAGE */ ;\ 1036 membar #Sync 1037 1038#if defined(cscope) 1039/* 1040 * Define labels to direct cscope quickly to labels that 1041 * are generated by macro expansion of DTLB_MISS(). 1042 */ 1043 .global tt0_dtlbmiss 1044tt0_dtlbmiss: 1045 .global tt1_dtlbmiss 1046tt1_dtlbmiss: 1047 nop 1048#endif 1049 1050/* 1051 * Needs to be exactly 32 instructions 1052 * 1053 * UTLB NOTE: If we don't hit on the 8k pointer then we branch 1054 * to a special 4M tsb handler. It would be nice if that handler 1055 * could live in this file but currently it seems better to allow 1056 * it to fall thru to sfmmu_tsb_miss. 1057 */ 1058#ifdef UTSB_PHYS 1059#define DTLB_MISS(table_name) ;\ 1060 .global table_name##_dtlbmiss ;\ 1061table_name##_dtlbmiss: ;\ 1062 mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\ 1063 ldxa [%g0]ASI_DMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\ 1064 ldxa [%g6]ASI_DMMU, %g2 /* g2 = tag access */ ;\ 1065 sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\ 1066 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\ 1067 cmp %g3, INVALID_CONTEXT ;\ 1068 ble,pn %xcc, sfmmu_kdtlb_miss ;\ 1069 srax %g2, PREDISM_BASESHIFT, %g6 /* g6 > 0 ISM predicted */ ;\ 1070 brgz,pn %g6, sfmmu_udtlb_slowpath_ismpred ;\ 1071 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ 1072 ldda [%g1]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, %g5 data */;\ 1073 cmp %g4, %g7 ;\ 1074 bne,pn %xcc, sfmmu_udtlb_slowpath_noismpred ;\ 1075 nop ;\ 1076 TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\ 1077 stxa %g5, [%g0]ASI_DTLB_IN /* trapstat expects TTE */ ;\ 1078 retry /* in %g5 */ ;\ 1079 unimp 0 ;\ 1080 unimp 0 ;\ 1081 unimp 0 ;\ 1082 unimp 0 ;\ 1083 unimp 0 ;\ 1084 unimp 0 ;\ 1085 unimp 0 ;\ 1086 unimp 0 ;\ 1087 unimp 0 ;\ 1088 unimp 0 ;\ 1089 unimp 0 ;\ 1090 unimp 0 ;\ 1091 unimp 0 ;\ 1092 unimp 0 ;\ 1093 .align 128 1094 1095#else /* UTSB_PHYS */ 1096#define DTLB_MISS(table_name) ;\ 1097 .global table_name##_dtlbmiss ;\ 1098table_name##_dtlbmiss: ;\ 1099 mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\ 1100 ldxa [%g0]ASI_DMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\ 1101 ldxa [%g6]ASI_DMMU, %g2 /* g2 = tag access */ ;\ 1102 sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\ 1103 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\ 1104 cmp %g3, INVALID_CONTEXT ;\ 1105 ble,pn %xcc, sfmmu_kdtlb_miss ;\ 1106 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ 1107 brlz,pn %g1, sfmmu_udtlb_slowpath ;\ 1108 nop ;\ 1109 ldda [%g1]ASI_NQUAD_LD, %g4 /* g4 = tag, %g5 data */ ;\ 1110 cmp %g4, %g7 ;\ 1111 bne,pn %xcc, sfmmu_tsb_miss_tt /* no 4M TSB, miss */ ;\ 1112 mov -1, %g3 /* set 4M tsbe ptr to -1 */ ;\ 1113 TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\ 1114 stxa %g5, [%g0]ASI_DTLB_IN /* trapstat expects TTE */ ;\ 1115 retry /* in %g5 */ ;\ 1116 unimp 0 ;\ 1117 unimp 0 ;\ 1118 unimp 0 ;\ 1119 unimp 0 ;\ 1120 unimp 0 ;\ 1121 unimp 0 ;\ 1122 unimp 0 ;\ 1123 unimp 0 ;\ 1124 unimp 0 ;\ 1125 unimp 0 ;\ 1126 unimp 0 ;\ 1127 unimp 0 ;\ 1128 unimp 0 ;\ 1129 unimp 0 ;\ 1130 .align 128 1131#endif /* UTSB_PHYS */ 1132 1133#if defined(cscope) 1134/* 1135 * Define labels to direct cscope quickly to labels that 1136 * are generated by macro expansion of ITLB_MISS(). 1137 */ 1138 .global tt0_itlbmiss 1139tt0_itlbmiss: 1140 .global tt1_itlbmiss 1141tt1_itlbmiss: 1142 nop 1143#endif 1144 1145/* 1146 * Instruction miss handler. 1147 * ldda instructions will have their ASI patched 1148 * by sfmmu_patch_ktsb at runtime. 1149 * MUST be EXACTLY 32 instructions or we'll break. 1150 */ 1151#ifdef UTSB_PHYS 1152#define ITLB_MISS(table_name) \ 1153 .global table_name##_itlbmiss ;\ 1154table_name##_itlbmiss: ;\ 1155 mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\ 1156 ldxa [%g0]ASI_IMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\ 1157 ldxa [%g6]ASI_IMMU, %g2 /* g2 = tag access */ ;\ 1158 sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\ 1159 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\ 1160 cmp %g3, INVALID_CONTEXT ;\ 1161 ble,pn %xcc, sfmmu_kitlb_miss ;\ 1162 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ 1163 ldda [%g1]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */ ;\ 1164 cmp %g4, %g7 ;\ 1165 bne,pn %xcc, sfmmu_uitlb_slowpath ;\ 1166 andcc %g5, TTE_EXECPRM_INT, %g0 /* check execute bit */ ;\ 1167 bz,pn %icc, exec_fault ;\ 1168 nop ;\ 1169 TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\ 1170 stxa %g5, [%g0]ASI_ITLB_IN /* trapstat expects %g5 */ ;\ 1171 retry ;\ 1172 unimp 0 ;\ 1173 unimp 0 ;\ 1174 unimp 0 ;\ 1175 unimp 0 ;\ 1176 unimp 0 ;\ 1177 unimp 0 ;\ 1178 unimp 0 ;\ 1179 unimp 0 ;\ 1180 unimp 0 ;\ 1181 unimp 0 ;\ 1182 unimp 0 ;\ 1183 unimp 0 ;\ 1184 unimp 0 ;\ 1185 unimp 0 ;\ 1186 .align 128 1187 1188#else /* UTSB_PHYS */ 1189#define ITLB_MISS(table_name) \ 1190 .global table_name##_itlbmiss ;\ 1191table_name##_itlbmiss: ;\ 1192 mov MMU_TAG_ACCESS, %g6 /* select tag acc */ ;\ 1193 ldxa [%g0]ASI_IMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\ 1194 ldxa [%g6]ASI_IMMU, %g2 /* g2 = tag access */ ;\ 1195 sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\ 1196 srlx %g3, TAGACC_CTX_LSHIFT, %g3 /* g3 = ctx */ ;\ 1197 cmp %g3, INVALID_CONTEXT ;\ 1198 ble,pn %xcc, sfmmu_kitlb_miss ;\ 1199 srlx %g2, TAG_VALO_SHIFT, %g7 /* g7 = tsb tag */ ;\ 1200 brlz,pn %g1, sfmmu_uitlb_slowpath /* if >1 TSB branch */ ;\ 1201 nop ;\ 1202 ldda [%g1]ASI_NQUAD_LD, %g4 /* g4 = tag, g5 = data */ ;\ 1203 cmp %g4, %g7 ;\ 1204 bne,pn %xcc, sfmmu_tsb_miss_tt /* br if 8k ptr miss */ ;\ 1205 mov -1, %g3 /* set 4M TSB ptr to -1 */ ;\ 1206 andcc %g5, TTE_EXECPRM_INT, %g0 /* check execute bit */ ;\ 1207 bz,pn %icc, exec_fault ;\ 1208 nop ;\ 1209 TT_TRACE(trace_tsbhit) /* 2 instr ifdef TRAPTRACE */ ;\ 1210 stxa %g5, [%g0]ASI_ITLB_IN /* trapstat expects %g5 */ ;\ 1211 retry ;\ 1212 unimp 0 ;\ 1213 unimp 0 ;\ 1214 unimp 0 ;\ 1215 unimp 0 ;\ 1216 unimp 0 ;\ 1217 unimp 0 ;\ 1218 unimp 0 ;\ 1219 unimp 0 ;\ 1220 unimp 0 ;\ 1221 unimp 0 ;\ 1222 unimp 0 ;\ 1223 .align 128 1224#endif /* UTSB_PHYS */ 1225 1226 1227/* 1228 * This macro is the first level handler for fast protection faults. 1229 * It first demaps the tlb entry which generated the fault and then 1230 * attempts to set the modify bit on the hash. It needs to be 1231 * exactly 32 instructions. 1232 */ 1233#define DTLB_PROT \ 1234 DTLB_DEMAP_ENTRY /* 20 instructions */ ;\ 1235 /* ;\ 1236 * At this point: ;\ 1237 * g1 = ???? ;\ 1238 * g2 = tag access register ;\ 1239 * g3 = ctx number ;\ 1240 * g4 = ???? ;\ 1241 */ ;\ 1242 TT_TRACE(trace_dataprot) /* 2 instr ifdef TRAPTRACE */ ;\ 1243 /* clobbers g1 and g6 */ ;\ 1244 ldxa [%g0]ASI_DMMU_TSB_8K, %g1 /* g1 = tsbe ptr */ ;\ 1245 brnz,pt %g3, sfmmu_uprot_trap /* user trap */ ;\ 1246 nop ;\ 1247 ba,a,pt %xcc, sfmmu_kprot_trap /* kernel trap */ ;\ 1248 unimp 0 ;\ 1249 unimp 0 ;\ 1250 unimp 0 ;\ 1251 unimp 0 ;\ 1252 unimp 0 ;\ 1253 unimp 0 ;\ 1254 .align 128 1255 1256#define DMMU_EXCEPTION_TL1 ;\ 1257 SWITCH_GLOBALS ;\ 1258 ba,a,pt %xcc, mmu_trap_tl1 ;\ 1259 nop ;\ 1260 .align 32 1261 1262#define MISALIGN_ADDR_TL1 ;\ 1263 ba,a,pt %xcc, mmu_trap_tl1 ;\ 1264 nop ;\ 1265 .align 32 1266 1267/* 1268 * Trace a tsb hit 1269 * g1 = tsbe pointer (in/clobbered) 1270 * g2 = tag access register (in) 1271 * g3 - g4 = scratch (clobbered) 1272 * g5 = tsbe data (in) 1273 * g6 = scratch (clobbered) 1274 * g7 = pc we jumped here from (in) 1275 * ttextra = value to OR in to trap type (%tt) (in) 1276 */ 1277#ifdef TRAPTRACE 1278#define TRACE_TSBHIT(ttextra) \ 1279 membar #Sync ;\ 1280 sethi %hi(FLUSH_ADDR), %g6 ;\ 1281 flush %g6 ;\ 1282 TRACE_PTR(%g3, %g6) ;\ 1283 GET_TRACE_TICK(%g6, %g4) ;\ 1284 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi ;\ 1285 stxa %g2, [%g3 + TRAP_ENT_SP]%asi /* tag access */ ;\ 1286 stxa %g5, [%g3 + TRAP_ENT_F1]%asi /* tsb data */ ;\ 1287 rdpr %tnpc, %g6 ;\ 1288 stxa %g6, [%g3 + TRAP_ENT_F2]%asi ;\ 1289 stxa %g1, [%g3 + TRAP_ENT_F3]%asi /* tsb pointer */ ;\ 1290 stxa %g0, [%g3 + TRAP_ENT_F4]%asi ;\ 1291 rdpr %tpc, %g6 ;\ 1292 stxa %g6, [%g3 + TRAP_ENT_TPC]%asi ;\ 1293 rdpr %tl, %g6 ;\ 1294 stha %g6, [%g3 + TRAP_ENT_TL]%asi ;\ 1295 rdpr %tt, %g6 ;\ 1296 or %g6, (ttextra), %g6 ;\ 1297 stha %g6, [%g3 + TRAP_ENT_TT]%asi ;\ 1298 ldxa [%g0]ASI_IMMU, %g1 /* tag target */ ;\ 1299 ldxa [%g0]ASI_DMMU, %g4 ;\ 1300 cmp %g6, FAST_IMMU_MISS_TT ;\ 1301 movne %icc, %g4, %g1 ;\ 1302 stxa %g1, [%g3 + TRAP_ENT_TSTATE]%asi /* tsb tag */ ;\ 1303 stxa %g0, [%g3 + TRAP_ENT_TR]%asi ;\ 1304 TRACE_NEXT(%g3, %g4, %g6) 1305#else 1306#define TRACE_TSBHIT(ttextra) 1307#endif 1308 1309/* 1310 * ======================================================================= 1311 * SPARC V9 TRAP TABLE 1312 * 1313 * The trap table is divided into two halves: the first half is used when 1314 * taking traps when TL=0; the second half is used when taking traps from 1315 * TL>0. Note that handlers in the second half of the table might not be able 1316 * to make the same assumptions as handlers in the first half of the table. 1317 * 1318 * Worst case trap nesting so far: 1319 * 1320 * at TL=0 client issues software trap requesting service 1321 * at TL=1 nucleus wants a register window 1322 * at TL=2 register window clean/spill/fill takes a TLB miss 1323 * at TL=3 processing TLB miss 1324 * at TL=4 handle asynchronous error 1325 * 1326 * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode". 1327 * 1328 * ======================================================================= 1329 */ 1330 .section ".text" 1331 .align 4 1332 .global trap_table, scb, trap_table0, trap_table1, etrap_table 1333 .type trap_table, #object 1334 .type scb, #object 1335trap_table: 1336scb: 1337trap_table0: 1338 /* hardware traps */ 1339 NOT; /* 000 reserved */ 1340 RED; /* 001 power on reset */ 1341 RED; /* 002 watchdog reset */ 1342 RED; /* 003 externally initiated reset */ 1343 RED; /* 004 software initiated reset */ 1344 RED; /* 005 red mode exception */ 1345 NOT; NOT; /* 006 - 007 reserved */ 1346 IMMU_EXCEPTION; /* 008 instruction access exception */ 1347 NOT; /* 009 instruction access MMU miss */ 1348 ASYNC_TRAP(T_INSTR_ERROR, trace_gen, tt0_iae); 1349 /* 00A instruction access error */ 1350 NOT; NOT4; /* 00B - 00F reserved */ 1351 ILLTRAP_INSTR; /* 010 illegal instruction */ 1352 TRAP(T_PRIV_INSTR); /* 011 privileged opcode */ 1353 NOT; /* 012 unimplemented LDD */ 1354 NOT; /* 013 unimplemented STD */ 1355 NOT4; NOT4; NOT4; /* 014 - 01F reserved */ 1356 FP_DISABLED_TRAP; /* 020 fp disabled */ 1357 FP_IEEE_TRAP; /* 021 fp exception ieee 754 */ 1358 FP_TRAP; /* 022 fp exception other */ 1359 TAG_OVERFLOW; /* 023 tag overflow */ 1360 CLEAN_WINDOW; /* 024 - 027 clean window */ 1361 DIV_BY_ZERO; /* 028 division by zero */ 1362 NOT; /* 029 internal processor error */ 1363 NOT; NOT; NOT4; /* 02A - 02F reserved */ 1364 DMMU_EXCEPTION; /* 030 data access exception */ 1365 NOT; /* 031 data access MMU miss */ 1366 ASYNC_TRAP(T_DATA_ERROR, trace_gen, tt0_dae); 1367 /* 032 data access error */ 1368 NOT; /* 033 data access protection */ 1369 DMMU_EXC_AG_NOT_ALIGNED; /* 034 mem address not aligned */ 1370 DMMU_EXC_LDDF_NOT_ALIGNED; /* 035 LDDF mem address not aligned */ 1371 DMMU_EXC_STDF_NOT_ALIGNED; /* 036 STDF mem address not aligned */ 1372 DMMU_EXC_AG_PRIV; /* 037 privileged action */ 1373 NOT; /* 038 LDQF mem address not aligned */ 1374 NOT; /* 039 STQF mem address not aligned */ 1375 NOT; NOT; NOT4; /* 03A - 03F reserved */ 1376 LABELED_BAD(tt0_asdat); /* 040 async data error */ 1377 LEVEL_INTERRUPT(1); /* 041 interrupt level 1 */ 1378 LEVEL_INTERRUPT(2); /* 042 interrupt level 2 */ 1379 LEVEL_INTERRUPT(3); /* 043 interrupt level 3 */ 1380 LEVEL_INTERRUPT(4); /* 044 interrupt level 4 */ 1381 LEVEL_INTERRUPT(5); /* 045 interrupt level 5 */ 1382 LEVEL_INTERRUPT(6); /* 046 interrupt level 6 */ 1383 LEVEL_INTERRUPT(7); /* 047 interrupt level 7 */ 1384 LEVEL_INTERRUPT(8); /* 048 interrupt level 8 */ 1385 LEVEL_INTERRUPT(9); /* 049 interrupt level 9 */ 1386 LEVEL_INTERRUPT(10); /* 04A interrupt level 10 */ 1387 LEVEL_INTERRUPT(11); /* 04B interrupt level 11 */ 1388 LEVEL_INTERRUPT(12); /* 04C interrupt level 12 */ 1389 LEVEL_INTERRUPT(13); /* 04D interrupt level 13 */ 1390 LEVEL14_INTERRUPT; /* 04E interrupt level 14 */ 1391 LEVEL15_INTERRUPT; /* 04F interrupt level 15 */ 1392 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F reserved */ 1393 VECTOR_INTERRUPT; /* 060 interrupt vector */ 1394 GOTO(kmdb_trap); /* 061 PA watchpoint */ 1395 GOTO(kmdb_trap); /* 062 VA watchpoint */ 1396 GOTO_TT(ce_err, trace_gen); /* 063 corrected ECC error */ 1397 ITLB_MISS(tt0); /* 064 instruction access MMU miss */ 1398 DTLB_MISS(tt0); /* 068 data access MMU miss */ 1399 DTLB_PROT; /* 06C data access protection */ 1400 LABELED_BAD(tt0_fecc); /* 070 fast ecache ECC error */ 1401 LABELED_BAD(tt0_dperr); /* 071 Cheetah+ dcache parity error */ 1402 LABELED_BAD(tt0_iperr); /* 072 Cheetah+ icache parity error */ 1403 NOT; /* 073 reserved */ 1404 NOT4; NOT4; NOT4; /* 074 - 07F reserved */ 1405 NOT4; /* 080 spill 0 normal */ 1406 SPILL_32bit_asi(ASI_AIUP,sn0); /* 084 spill 1 normal */ 1407 SPILL_64bit_asi(ASI_AIUP,sn0); /* 088 spill 2 normal */ 1408 SPILL_32clean(ASI_AIUP,sn0); /* 08C spill 3 normal */ 1409 SPILL_64clean(ASI_AIUP,sn0); /* 090 spill 4 normal */ 1410 SPILL_32bit(not); /* 094 spill 5 normal */ 1411 SPILL_64bit(not); /* 098 spill 6 normal */ 1412 SPILL_mixed; /* 09C spill 7 normal */ 1413 NOT4; /* 0A0 spill 0 other */ 1414 SPILL_32bit_asi(ASI_AIUS,so0); /* 0A4 spill 1 other */ 1415 SPILL_64bit_asi(ASI_AIUS,so0); /* 0A8 spill 2 other */ 1416 SPILL_32bit_asi(ASI_AIUS,so0); /* 0AC spill 3 other */ 1417 SPILL_64bit_asi(ASI_AIUS,so0); /* 0B0 spill 4 other */ 1418 NOT4; /* 0B4 spill 5 other */ 1419 NOT4; /* 0B8 spill 6 other */ 1420 NOT4; /* 0BC spill 7 other */ 1421 NOT4; /* 0C0 fill 0 normal */ 1422 FILL_32bit_asi(ASI_AIUP,fn0); /* 0C4 fill 1 normal */ 1423 FILL_64bit_asi(ASI_AIUP,fn0); /* 0C8 fill 2 normal */ 1424 FILL_32bit_asi(ASI_AIUP,fn0); /* 0CC fill 3 normal */ 1425 FILL_64bit_asi(ASI_AIUP,fn0); /* 0D0 fill 4 normal */ 1426 FILL_32bit(not); /* 0D4 fill 5 normal */ 1427 FILL_64bit(not); /* 0D8 fill 6 normal */ 1428 FILL_mixed; /* 0DC fill 7 normal */ 1429 NOT4; /* 0E0 fill 0 other */ 1430 NOT4; /* 0E4 fill 1 other */ 1431 NOT4; /* 0E8 fill 2 other */ 1432 NOT4; /* 0EC fill 3 other */ 1433 NOT4; /* 0F0 fill 4 other */ 1434 NOT4; /* 0F4 fill 5 other */ 1435 NOT4; /* 0F8 fill 6 other */ 1436 NOT4; /* 0FC fill 7 other */ 1437 /* user traps */ 1438 GOTO(syscall_trap_4x); /* 100 old system call */ 1439 TRAP(T_BREAKPOINT); /* 101 user breakpoint */ 1440 TRAP(T_DIV0); /* 102 user divide by zero */ 1441 FLUSHW(tt0_flushw); /* 103 flush windows */ 1442 GOTO(.clean_windows); /* 104 clean windows */ 1443 BAD; /* 105 range check ?? */ 1444 GOTO(.fix_alignment); /* 106 do unaligned references */ 1445 BAD; /* 107 unused */ 1446 SYSCALL_TRAP32; /* 108 ILP32 system call on LP64 */ 1447 GOTO(set_trap0_addr); /* 109 set trap0 address */ 1448 BAD; BAD; BAD4; /* 10A - 10F unused */ 1449 TRP4; TRP4; TRP4; TRP4; /* 110 - 11F V9 user trap handlers */ 1450 GOTO(.getcc); /* 120 get condition codes */ 1451 GOTO(.setcc); /* 121 set condition codes */ 1452 GOTO(.getpsr); /* 122 get psr */ 1453 GOTO(.setpsr); /* 123 set psr (some fields) */ 1454 GOTO(get_timestamp); /* 124 get timestamp */ 1455 GOTO(get_virtime); /* 125 get lwp virtual time */ 1456 PRIV(self_xcall); /* 126 self xcall */ 1457 GOTO(get_hrestime); /* 127 get hrestime */ 1458 BAD; /* 128 ST_SETV9STACK */ 1459 GOTO(.getlgrp); /* 129 get lgrpid */ 1460 BAD; BAD; BAD4; /* 12A - 12F unused */ 1461 BAD4; BAD4; /* 130 - 137 unused */ 1462 DTRACE_PID; /* 138 dtrace pid tracing provider */ 1463 BAD; /* 139 unused */ 1464 DTRACE_RETURN; /* 13A dtrace pid return probe */ 1465 BAD; BAD4; /* 13B - 13F unused */ 1466 SYSCALL_TRAP; /* 140 LP64 system call */ 1467 SYSCALL(nosys); /* 141 unused system call trap */ 1468#ifdef DEBUG_USER_TRAPTRACECTL 1469 GOTO(.traptrace_freeze); /* 142 freeze traptrace */ 1470 GOTO(.traptrace_unfreeze); /* 143 unfreeze traptrace */ 1471#else 1472 SYSCALL(nosys); /* 142 unused system call trap */ 1473 SYSCALL(nosys); /* 143 unused system call trap */ 1474#endif 1475 BAD4; BAD4; BAD4; /* 144 - 14F unused */ 1476 BAD4; BAD4; BAD4; BAD4; /* 150 - 15F unused */ 1477 BAD4; BAD4; BAD4; BAD4; /* 160 - 16F unused */ 1478 BAD; /* 170 - unused */ 1479 BAD; /* 171 - unused */ 1480 BAD; BAD; /* 172 - 173 unused */ 1481 BAD4; BAD4; /* 174 - 17B unused */ 1482#ifdef PTL1_PANIC_DEBUG 1483 mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic); 1484 /* 17C test ptl1_panic */ 1485#else 1486 BAD; /* 17C unused */ 1487#endif /* PTL1_PANIC_DEBUG */ 1488 PRIV(kmdb_trap); /* 17D kmdb enter (L1-A) */ 1489 PRIV(kmdb_trap); /* 17E kmdb breakpoint */ 1490 PRIV(kctx_obp_bpt); /* 17F obp breakpoint */ 1491 /* reserved */ 1492 NOT4; NOT4; NOT4; NOT4; /* 180 - 18F reserved */ 1493 NOT4; NOT4; NOT4; NOT4; /* 190 - 19F reserved */ 1494 NOT4; NOT4; NOT4; NOT4; /* 1A0 - 1AF reserved */ 1495 NOT4; NOT4; NOT4; NOT4; /* 1B0 - 1BF reserved */ 1496 NOT4; NOT4; NOT4; NOT4; /* 1C0 - 1CF reserved */ 1497 NOT4; NOT4; NOT4; NOT4; /* 1D0 - 1DF reserved */ 1498 NOT4; NOT4; NOT4; NOT4; /* 1E0 - 1EF reserved */ 1499 NOT4; NOT4; NOT4; NOT4; /* 1F0 - 1FF reserved */ 1500trap_table1: 1501 NOT4; NOT4; NOT; NOT; /* 000 - 009 unused */ 1502 ASYNC_TRAP(T_INSTR_ERROR + T_TL1, trace_gen, tt1_iae); 1503 /* 00A instruction access error */ 1504 NOT; NOT4; /* 00B - 00F unused */ 1505 NOT4; NOT4; NOT4; NOT4; /* 010 - 01F unused */ 1506 NOT4; /* 020 - 023 unused */ 1507 CLEAN_WINDOW; /* 024 - 027 clean window */ 1508 NOT4; NOT4; /* 028 - 02F unused */ 1509 DMMU_EXCEPTION_TL1; /* 030 data access exception */ 1510 NOT; /* 031 unused */ 1511 ASYNC_TRAP(T_DATA_ERROR + T_TL1, trace_gen, tt1_dae); 1512 /* 032 data access error */ 1513 NOT; /* 033 unused */ 1514 MISALIGN_ADDR_TL1; /* 034 mem address not aligned */ 1515 NOT; NOT; NOT; NOT4; NOT4 /* 035 - 03F unused */ 1516 LABELED_BAD(tt1_asdat); /* 040 async data error */ 1517 NOT; NOT; NOT; /* 041 - 043 unused */ 1518 NOT4; NOT4; NOT4; /* 044 - 04F unused */ 1519 NOT4; NOT4; NOT4; NOT4; /* 050 - 05F unused */ 1520 NOT; /* 060 unused */ 1521 GOTO(kmdb_trap_tl1); /* 061 PA watchpoint */ 1522 GOTO(kmdb_trap_tl1); /* 062 VA watchpoint */ 1523 GOTO_TT(ce_err_tl1, trace_gen); /* 063 corrected ECC error */ 1524 ITLB_MISS(tt1); /* 064 instruction access MMU miss */ 1525 DTLB_MISS(tt1); /* 068 data access MMU miss */ 1526 DTLB_PROT; /* 06C data access protection */ 1527 LABELED_BAD(tt1_fecc); /* 070 fast ecache ECC error */ 1528 LABELED_BAD(tt1_dperr); /* 071 Cheetah+ dcache parity error */ 1529 LABELED_BAD(tt1_iperr); /* 072 Cheetah+ icache parity error */ 1530 NOT; /* 073 reserved */ 1531 NOT4; NOT4; NOT4; /* 074 - 07F reserved */ 1532 NOT4; /* 080 spill 0 normal */ 1533 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 084 spill 1 normal */ 1534 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 088 spill 2 normal */ 1535 SPILL_32bit_tt1(ASI_AIUP,sn1); /* 08C spill 3 normal */ 1536 SPILL_64bit_tt1(ASI_AIUP,sn1); /* 090 spill 4 normal */ 1537 SPILL_32bit(not); /* 094 spill 5 normal */ 1538 SPILL_64bit(not); /* 098 spill 6 normal */ 1539 SPILL_mixed; /* 09C spill 7 normal */ 1540 NOT4; /* 0A0 spill 0 other */ 1541 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0A4 spill 1 other */ 1542 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0A8 spill 2 other */ 1543 SPILL_32bit_tt1(ASI_AIUS,so1); /* 0AC spill 3 other */ 1544 SPILL_64bit_tt1(ASI_AIUS,so1); /* 0B0 spill 4 other */ 1545 NOT4; /* 0B4 spill 5 other */ 1546 NOT4; /* 0B8 spill 6 other */ 1547 NOT4; /* 0BC spill 7 other */ 1548 NOT4; /* 0C0 fill 0 normal */ 1549 FILL_32bit_tt1(ASI_AIUP,fn1); /* 0C4 fill 1 normal */ 1550 FILL_64bit_tt1(ASI_AIUP,fn1); /* 0C8 fill 2 normal */ 1551 FILL_32bit_tt1(ASI_AIUP,fn1); /* 0CC fill 3 normal */ 1552 FILL_64bit_tt1(ASI_AIUP,fn1); /* 0D0 fill 4 normal */ 1553 FILL_32bit(not); /* 0D4 fill 5 normal */ 1554 FILL_64bit(not); /* 0D8 fill 6 normal */ 1555 FILL_mixed; /* 0DC fill 7 normal */ 1556 NOT4; NOT4; NOT4; NOT4; /* 0E0 - 0EF unused */ 1557 NOT4; NOT4; NOT4; NOT4; /* 0F0 - 0FF unused */ 1558 LABELED_BAD(tt1_swtrap0); /* 100 fast ecache ECC error (cont) */ 1559 LABELED_BAD(tt1_swtrap1); /* 101 Ch+ D$ parity error (cont) */ 1560 LABELED_BAD(tt1_swtrap2); /* 102 Ch+ I$ parity error (cont) */ 1561 NOT; /* 103 reserved */ 1562/* 1563 * We only reserve the above four special case soft traps for code running 1564 * at TL>0, so we can truncate the trap table here. 1565 */ 1566etrap_table: 1567 .size trap_table, (.-trap_table) 1568 .size scb, (.-scb) 1569 1570/* 1571 * We get to exec_fault in the case of an instruction miss and tte 1572 * has no execute bit set. We go to tl0 to handle it. 1573 * 1574 * g1 = tsbe pointer (in/clobbered) 1575 * g2 = tag access register (in) 1576 * g3 - g4 = scratch (clobbered) 1577 * g5 = tsbe data (in) 1578 * g6 = scratch (clobbered) 1579 */ 1580 ALTENTRY(exec_fault) 1581 TRACE_TSBHIT(0x200) 1582 SWITCH_GLOBALS 1583 mov MMU_TAG_ACCESS, %g4 1584 ldxa [%g4]ASI_IMMU, %g2 ! arg1 = addr 1585 mov T_INSTR_MMU_MISS, %g3 ! arg2 = traptype 1586 set trap, %g1 1587 ba,pt %xcc, sys_trap 1588 mov -1, %g4 1589 1590.mmu_exception_not_aligned: 1591 rdpr %tstate, %g1 1592 btst TSTATE_PRIV, %g1 1593 bnz,pn %icc, 2f 1594 nop 1595 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1596 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1597 ldn [%g1 + T_PROCP], %g1 ! load proc pointer 1598 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? 1599 brz,pt %g5, 2f 1600 nop 1601 ldn [%g5 + P_UTRAP15], %g5 ! unaligned utrap? 1602 brz,pn %g5, 2f 1603 nop 1604 btst 1, %sp 1605 bz,pt %xcc, 1f ! 32 bit user program 1606 nop 1607 ba,pt %xcc, .setup_v9utrap ! 64 bit user program 1608 nop 16091: 1610 ba,pt %xcc, .setup_utrap 1611 or %g2, %g0, %g7 16122: 1613 ba,pt %xcc, .mmu_exception_end 1614 mov T_ALIGNMENT, %g1 1615 1616.mmu_priv_exception: 1617 rdpr %tstate, %g1 1618 btst TSTATE_PRIV, %g1 1619 bnz,pn %icc, 1f 1620 nop 1621 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1622 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1623 ldn [%g1 + T_PROCP], %g1 ! load proc pointer 1624 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? 1625 brz,pt %g5, 1f 1626 nop 1627 ldn [%g5 + P_UTRAP16], %g5 1628 brnz,pt %g5, .setup_v9utrap 1629 nop 16301: 1631 mov T_PRIV_INSTR, %g1 1632 1633.mmu_exception_end: 1634 CPU_INDEX(%g4, %g5) 1635 set cpu_core, %g5 1636 sllx %g4, CPU_CORE_SHIFT, %g4 1637 add %g4, %g5, %g4 1638 lduh [%g4 + CPUC_DTRACE_FLAGS], %g5 1639 andcc %g5, CPU_DTRACE_NOFAULT, %g0 1640 bz %xcc, .mmu_exception_tlb_chk 1641 or %g5, CPU_DTRACE_BADADDR, %g5 1642 stuh %g5, [%g4 + CPUC_DTRACE_FLAGS] 1643 done 1644 1645.mmu_exception_tlb_chk: 1646 GET_CPU_IMPL(%g5) ! check SFSR.FT to see if this 1647 cmp %g5, PANTHER_IMPL ! is a TLB parity error. But 1648 bne 2f ! we only do this check while 1649 mov 1, %g4 ! running on Panther CPUs 1650 sllx %g4, PN_SFSR_PARITY_SHIFT, %g4 ! since US-I/II use the same 1651 andcc %g3, %g4, %g0 ! bit for something else which 1652 bz 2f ! will be handled later. 1653 nop 1654.mmu_exception_is_tlb_parity: 1655 .weak itlb_parity_trap 1656 .weak dtlb_parity_trap 1657 set itlb_parity_trap, %g4 1658 cmp %g1, T_INSTR_EXCEPTION ! branch to the itlb or 1659 be 3f ! dtlb parity handler 1660 nop ! if this trap is due 1661 set dtlb_parity_trap, %g4 1662 cmp %g1, T_DATA_EXCEPTION ! to a IMMU exception 1663 be 3f ! or DMMU exception. 1664 nop 16652: 1666 sllx %g3, 32, %g3 1667 or %g3, %g1, %g3 1668 set trap, %g1 1669 ba,pt %xcc, sys_trap 1670 sub %g0, 1, %g4 16713: 1672 jmp %g4 ! off to the appropriate 1673 nop ! TLB parity handler 1674 1675.fp_disabled: 1676 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1677 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1678#ifdef SF_ERRATA_30 /* call causes fp-disabled */ 1679 brz,a,pn %g1, 2f 1680 nop 1681#endif 1682 rdpr %tstate, %g4 1683 btst TSTATE_PRIV, %g4 1684#ifdef SF_ERRATA_30 /* call causes fp-disabled */ 1685 bnz,pn %icc, 2f 1686 nop 1687#else 1688 bnz,a,pn %icc, ptl1_panic 1689 mov PTL1_BAD_FPTRAP, %g1 1690#endif 1691 ldn [%g1 + T_PROCP], %g1 ! load proc pointer 1692 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? 1693 brz,a,pt %g5, 2f 1694 nop 1695 ldn [%g5 + P_UTRAP7], %g5 ! fp_disabled utrap? 1696 brz,a,pn %g5, 2f 1697 nop 1698 btst 1, %sp 1699 bz,a,pt %xcc, 1f ! 32 bit user program 1700 nop 1701 ba,a,pt %xcc, .setup_v9utrap ! 64 bit user program 1702 nop 17031: 1704 ba,pt %xcc, .setup_utrap 1705 or %g0, %g0, %g7 17062: 1707 set fp_disabled, %g1 1708 ba,pt %xcc, sys_trap 1709 sub %g0, 1, %g4 1710 1711.fp_ieee_exception: 1712 rdpr %tstate, %g1 1713 btst TSTATE_PRIV, %g1 1714 bnz,a,pn %icc, ptl1_panic 1715 mov PTL1_BAD_FPTRAP, %g1 1716 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1717 stx %fsr, [%g1 + CPU_TMP1] 1718 ldx [%g1 + CPU_TMP1], %g2 1719 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1720 ldn [%g1 + T_PROCP], %g1 ! load proc pointer 1721 ldn [%g1 + P_UTRAPS], %g5 ! are there utraps? 1722 brz,a,pt %g5, 1f 1723 nop 1724 ldn [%g5 + P_UTRAP8], %g5 1725 brnz,a,pt %g5, .setup_v9utrap 1726 nop 17271: 1728 set _fp_ieee_exception, %g1 1729 ba,pt %xcc, sys_trap 1730 sub %g0, 1, %g4 1731 1732/* 1733 * Register Inputs: 1734 * %g5 user trap handler 1735 * %g7 misaligned addr - for alignment traps only 1736 */ 1737.setup_utrap: 1738 set trap, %g1 ! setup in case we go 1739 mov T_FLUSH_PCB, %g3 ! through sys_trap on 1740 sub %g0, 1, %g4 ! the save instruction below 1741 1742 /* 1743 * If the DTrace pid provider is single stepping a copied-out 1744 * instruction, t->t_dtrace_step will be set. In that case we need 1745 * to abort the single-stepping (since execution of the instruction 1746 * was interrupted) and use the value of t->t_dtrace_npc as the %npc. 1747 */ 1748 save %sp, -SA(MINFRAME32), %sp ! window for trap handler 1749 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1750 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1751 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step 1752 rdpr %tnpc, %l2 ! arg1 == tnpc 1753 brz,pt %g2, 1f 1754 rdpr %tpc, %l1 ! arg0 == tpc 1755 1756 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast 1757 ldn [%g1 + T_DTRACE_NPC], %l2 ! arg1 = t->t_dtrace_npc (step) 1758 brz,pt %g2, 1f 1759 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags 1760 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast 17611: 1762 mov %g7, %l3 ! arg2 == misaligned address 1763 1764 rdpr %tstate, %g1 ! cwp for trap handler 1765 rdpr %cwp, %g4 1766 bclr TSTATE_CWP_MASK, %g1 1767 wrpr %g1, %g4, %tstate 1768 wrpr %g0, %g5, %tnpc ! trap handler address 1769 FAST_TRAP_DONE 1770 /* NOTREACHED */ 1771 1772.check_v9utrap: 1773 rdpr %tstate, %g1 1774 btst TSTATE_PRIV, %g1 1775 bnz,a,pn %icc, 3f 1776 nop 1777 CPU_ADDR(%g4, %g1) ! load CPU struct addr 1778 ldn [%g4 + CPU_THREAD], %g5 ! load thread pointer 1779 ldn [%g5 + T_PROCP], %g5 ! load proc pointer 1780 ldn [%g5 + P_UTRAPS], %g5 ! are there utraps? 1781 1782 cmp %g3, T_SOFTWARE_TRAP 1783 bne,a,pt %icc, 1f 1784 nop 1785 1786 brz,pt %g5, 3f ! if p_utraps == NULL goto trap() 1787 rdpr %tt, %g3 ! delay - get actual hw trap type 1788 1789 sub %g3, 254, %g1 ! UT_TRAP_INSTRUCTION_16 = p_utraps[18] 1790 ba,pt %icc, 2f 1791 smul %g1, CPTRSIZE, %g2 17921: 1793 brz,a,pt %g5, 3f ! if p_utraps == NULL goto trap() 1794 nop 1795 1796 cmp %g3, T_UNIMP_INSTR 1797 bne,a,pt %icc, 2f 1798 nop 1799 1800 mov 1, %g1 1801 st %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR 1802 rdpr %tpc, %g1 ! ld trapping instruction using 1803 lduwa [%g1]ASI_AIUP, %g1 ! "AS IF USER" ASI which could fault 1804 st %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR 1805 1806 sethi %hi(0xc1c00000), %g4 ! setup mask for illtrap instruction 1807 andcc %g1, %g4, %g4 ! and instruction with mask 1808 bnz,a,pt %icc, 3f ! if %g4 == zero, %g1 is an ILLTRAP 1809 nop ! fall thru to setup 18102: 1811 ldn [%g5 + %g2], %g5 1812 brnz,a,pt %g5, .setup_v9utrap 1813 nop 18143: 1815 set trap, %g1 1816 ba,pt %xcc, sys_trap 1817 sub %g0, 1, %g4 1818 /* NOTREACHED */ 1819 1820/* 1821 * Register Inputs: 1822 * %g5 user trap handler 1823 */ 1824.setup_v9utrap: 1825 set trap, %g1 ! setup in case we go 1826 mov T_FLUSH_PCB, %g3 ! through sys_trap on 1827 sub %g0, 1, %g4 ! the save instruction below 1828 1829 /* 1830 * If the DTrace pid provider is single stepping a copied-out 1831 * instruction, t->t_dtrace_step will be set. In that case we need 1832 * to abort the single-stepping (since execution of the instruction 1833 * was interrupted) and use the value of t->t_dtrace_npc as the %npc. 1834 */ 1835 save %sp, -SA(MINFRAME64), %sp ! window for trap handler 1836 CPU_ADDR(%g1, %g4) ! load CPU struct addr 1837 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 1838 ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step 1839 rdpr %tnpc, %l7 ! arg1 == tnpc 1840 brz,pt %g2, 1f 1841 rdpr %tpc, %l6 ! arg0 == tpc 1842 1843 ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast 1844 ldn [%g1 + T_DTRACE_NPC], %l7 ! arg1 == t->t_dtrace_npc (step) 1845 brz,pt %g2, 1f 1846 st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags 1847 stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast 18481: 1849 rdpr %tstate, %g2 ! cwp for trap handler 1850 rdpr %cwp, %g4 1851 bclr TSTATE_CWP_MASK, %g2 1852 wrpr %g2, %g4, %tstate 1853 1854 ldn [%g1 + T_PROCP], %g4 ! load proc pointer 1855 ldn [%g4 + P_AS], %g4 ! load as pointer 1856 ldn [%g4 + A_USERLIMIT], %g4 ! load as userlimit 1857 cmp %l7, %g4 ! check for single-step set 1858 bne,pt %xcc, 4f 1859 nop 1860 ldn [%g1 + T_LWP], %g1 ! load klwp pointer 1861 ld [%g1 + PCB_STEP], %g4 ! load single-step flag 1862 cmp %g4, STEP_ACTIVE ! step flags set in pcb? 1863 bne,pt %icc, 4f 1864 nop 1865 stn %g5, [%g1 + PCB_TRACEPC] ! save trap handler addr in pcb 1866 mov %l7, %g4 ! on entry to precise user trap 1867 add %l6, 4, %l7 ! handler, %l6 == pc, %l7 == npc 1868 ! at time of trap 1869 wrpr %g0, %g4, %tnpc ! generate FLTBOUNDS, 1870 ! %g4 == userlimit 1871 FAST_TRAP_DONE 1872 /* NOTREACHED */ 18734: 1874 wrpr %g0, %g5, %tnpc ! trap handler address 1875 FAST_TRAP_DONE_CHK_INTR 1876 /* NOTREACHED */ 1877 1878.fp_exception: 1879 CPU_ADDR(%g1, %g4) 1880 stx %fsr, [%g1 + CPU_TMP1] 1881 ldx [%g1 + CPU_TMP1], %g2 1882 1883 /* 1884 * Cheetah takes unfinished_FPop trap for certain range of operands 1885 * to the "fitos" instruction. Instead of going through the slow 1886 * software emulation path, we try to simulate the "fitos" instruction 1887 * via "fitod" and "fdtos" provided the following conditions are met: 1888 * 1889 * fpu_exists is set (if DEBUG) 1890 * not in privileged mode 1891 * ftt is unfinished_FPop 1892 * NXM IEEE trap is not enabled 1893 * instruction at %tpc is "fitos" 1894 * 1895 * Usage: 1896 * %g1 per cpu address 1897 * %g2 %fsr 1898 * %g6 user instruction 1899 * 1900 * Note that we can take a memory access related trap while trying 1901 * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR 1902 * flag to catch those traps and let the SFMMU code deal with page 1903 * fault and data access exception. 1904 */ 1905#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 1906 sethi %hi(fpu_exists), %g7 1907 ld [%g7 + %lo(fpu_exists)], %g7 1908 brz,pn %g7, .fp_exception_cont 1909 nop 1910#endif 1911 rdpr %tstate, %g7 ! branch if in privileged mode 1912 btst TSTATE_PRIV, %g7 1913 bnz,pn %xcc, .fp_exception_cont 1914 srl %g2, FSR_FTT_SHIFT, %g7 ! extract ftt from %fsr 1915 and %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7 1916 cmp %g7, FTT_UNFIN 1917 set FSR_TEM_NX, %g5 1918 bne,pn %xcc, .fp_exception_cont ! branch if NOT unfinished_FPop 1919 andcc %g2, %g5, %g0 1920 bne,pn %xcc, .fp_exception_cont ! branch if FSR_TEM_NX enabled 1921 rdpr %tpc, %g5 ! get faulting PC 1922 1923 or %g0, 1, %g7 1924 st %g7, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag 1925 lda [%g5]ASI_USER, %g6 ! get user's instruction 1926 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 1927 1928 set FITOS_INSTR_MASK, %g7 1929 and %g6, %g7, %g7 1930 set FITOS_INSTR, %g5 1931 cmp %g7, %g5 1932 bne,pn %xcc, .fp_exception_cont ! branch if not FITOS_INSTR 1933 nop 1934 1935 /* 1936 * This is unfinished FPops trap for "fitos" instruction. We 1937 * need to simulate "fitos" via "fitod" and "fdtos" instruction 1938 * sequence. 1939 * 1940 * We need a temporary FP register to do the conversion. Since 1941 * both source and destination operands for the "fitos" instruction 1942 * have to be within %f0-%f31, we use an FP register from the upper 1943 * half to guarantee that it won't collide with the source or the 1944 * dest operand. However, we do have to save and restore its value. 1945 * 1946 * We use %d62 as a temporary FP register for the conversion and 1947 * branch to appropriate instruction within the conversion tables 1948 * based upon the rs2 and rd values. 1949 */ 1950 1951 std %d62, [%g1 + CPU_TMP1] ! save original value 1952 1953 srl %g6, FITOS_RS2_SHIFT, %g7 1954 and %g7, FITOS_REG_MASK, %g7 1955 set _fitos_fitod_table, %g4 1956 sllx %g7, 2, %g7 1957 jmp %g4 + %g7 1958 ba,pt %xcc, _fitos_fitod_done 1959 .empty 1960 1961_fitos_fitod_table: 1962 fitod %f0, %d62 1963 fitod %f1, %d62 1964 fitod %f2, %d62 1965 fitod %f3, %d62 1966 fitod %f4, %d62 1967 fitod %f5, %d62 1968 fitod %f6, %d62 1969 fitod %f7, %d62 1970 fitod %f8, %d62 1971 fitod %f9, %d62 1972 fitod %f10, %d62 1973 fitod %f11, %d62 1974 fitod %f12, %d62 1975 fitod %f13, %d62 1976 fitod %f14, %d62 1977 fitod %f15, %d62 1978 fitod %f16, %d62 1979 fitod %f17, %d62 1980 fitod %f18, %d62 1981 fitod %f19, %d62 1982 fitod %f20, %d62 1983 fitod %f21, %d62 1984 fitod %f22, %d62 1985 fitod %f23, %d62 1986 fitod %f24, %d62 1987 fitod %f25, %d62 1988 fitod %f26, %d62 1989 fitod %f27, %d62 1990 fitod %f28, %d62 1991 fitod %f29, %d62 1992 fitod %f30, %d62 1993 fitod %f31, %d62 1994_fitos_fitod_done: 1995 1996 /* 1997 * Now convert data back into single precision 1998 */ 1999 srl %g6, FITOS_RD_SHIFT, %g7 2000 and %g7, FITOS_REG_MASK, %g7 2001 set _fitos_fdtos_table, %g4 2002 sllx %g7, 2, %g7 2003 jmp %g4 + %g7 2004 ba,pt %xcc, _fitos_fdtos_done 2005 .empty 2006 2007_fitos_fdtos_table: 2008 fdtos %d62, %f0 2009 fdtos %d62, %f1 2010 fdtos %d62, %f2 2011 fdtos %d62, %f3 2012 fdtos %d62, %f4 2013 fdtos %d62, %f5 2014 fdtos %d62, %f6 2015 fdtos %d62, %f7 2016 fdtos %d62, %f8 2017 fdtos %d62, %f9 2018 fdtos %d62, %f10 2019 fdtos %d62, %f11 2020 fdtos %d62, %f12 2021 fdtos %d62, %f13 2022 fdtos %d62, %f14 2023 fdtos %d62, %f15 2024 fdtos %d62, %f16 2025 fdtos %d62, %f17 2026 fdtos %d62, %f18 2027 fdtos %d62, %f19 2028 fdtos %d62, %f20 2029 fdtos %d62, %f21 2030 fdtos %d62, %f22 2031 fdtos %d62, %f23 2032 fdtos %d62, %f24 2033 fdtos %d62, %f25 2034 fdtos %d62, %f26 2035 fdtos %d62, %f27 2036 fdtos %d62, %f28 2037 fdtos %d62, %f29 2038 fdtos %d62, %f30 2039 fdtos %d62, %f31 2040_fitos_fdtos_done: 2041 2042 ldd [%g1 + CPU_TMP1], %d62 ! restore %d62 2043 2044#if DEBUG 2045 /* 2046 * Update FPop_unfinished trap kstat 2047 */ 2048 set fpustat+FPUSTAT_UNFIN_KSTAT, %g7 2049 ldx [%g7], %g5 20501: 2051 add %g5, 1, %g6 2052 2053 casxa [%g7] ASI_N, %g5, %g6 2054 cmp %g5, %g6 2055 bne,a,pn %xcc, 1b 2056 or %g0, %g6, %g5 2057 2058 /* 2059 * Update fpu_sim_fitos kstat 2060 */ 2061 set fpuinfo+FPUINFO_FITOS_KSTAT, %g7 2062 ldx [%g7], %g5 20631: 2064 add %g5, 1, %g6 2065 2066 casxa [%g7] ASI_N, %g5, %g6 2067 cmp %g5, %g6 2068 bne,a,pn %xcc, 1b 2069 or %g0, %g6, %g5 2070#endif /* DEBUG */ 2071 2072 FAST_TRAP_DONE 2073 2074.fp_exception_cont: 2075 /* 2076 * Let _fp_exception deal with simulating FPop instruction. 2077 * Note that we need to pass %fsr in %g2 (already read above). 2078 */ 2079 2080 set _fp_exception, %g1 2081 ba,pt %xcc, sys_trap 2082 sub %g0, 1, %g4 2083 2084 .global opl_cleanw_patch 2085opl_cleanw_patch: 2086.clean_windows: 2087 set trap, %g1 2088 mov T_FLUSH_PCB, %g3 2089 sub %g0, 1, %g4 2090 save 2091 flushw 2092 restore 2093 wrpr %g0, %g0, %cleanwin ! no clean windows 2094 2095 CPU_ADDR(%g4, %g5) 2096 ldn [%g4 + CPU_MPCB], %g4 2097 brz,a,pn %g4, 1f 2098 nop 2099 ld [%g4 + MPCB_WSTATE], %g5 2100 add %g5, WSTATE_CLEAN_OFFSET, %g5 2101 wrpr %g0, %g5, %wstate 21021: FAST_TRAP_DONE 2103 2104/* 2105 * .spill_clean: clean the previous window, restore the wstate, and 2106 * "done". 2107 * 2108 * Entry: %g7 contains new wstate 2109 */ 2110.spill_clean: 2111 sethi %hi(nwin_minus_one), %g5 2112 ld [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1 2113 rdpr %cwp, %g6 ! %g6 = %cwp 2114 deccc %g6 ! %g6-- 2115 movneg %xcc, %g5, %g6 ! if (%g6<0) %g6 = nwin-1 2116 wrpr %g6, %cwp 2117 TT_TRACE_L(trace_win) 2118 clr %l0 2119 clr %l1 2120 clr %l2 2121 clr %l3 2122 clr %l4 2123 clr %l5 2124 clr %l6 2125 clr %l7 2126 wrpr %g0, %g7, %wstate 2127 saved 2128 retry ! restores correct %cwp 2129 2130.fix_alignment: 2131 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 2132 ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer 2133 ldn [%g1 + T_PROCP], %g1 2134 mov 1, %g2 2135 stb %g2, [%g1 + P_FIXALIGNMENT] 2136 FAST_TRAP_DONE 2137 2138#define STDF_REG(REG, ADDR, TMP) \ 2139 sll REG, 3, REG ;\ 2140mark1: set start1, TMP ;\ 2141 jmp REG + TMP ;\ 2142 nop ;\ 2143start1: ba,pt %xcc, done1 ;\ 2144 std %f0, [ADDR + CPU_TMP1] ;\ 2145 ba,pt %xcc, done1 ;\ 2146 std %f32, [ADDR + CPU_TMP1] ;\ 2147 ba,pt %xcc, done1 ;\ 2148 std %f2, [ADDR + CPU_TMP1] ;\ 2149 ba,pt %xcc, done1 ;\ 2150 std %f34, [ADDR + CPU_TMP1] ;\ 2151 ba,pt %xcc, done1 ;\ 2152 std %f4, [ADDR + CPU_TMP1] ;\ 2153 ba,pt %xcc, done1 ;\ 2154 std %f36, [ADDR + CPU_TMP1] ;\ 2155 ba,pt %xcc, done1 ;\ 2156 std %f6, [ADDR + CPU_TMP1] ;\ 2157 ba,pt %xcc, done1 ;\ 2158 std %f38, [ADDR + CPU_TMP1] ;\ 2159 ba,pt %xcc, done1 ;\ 2160 std %f8, [ADDR + CPU_TMP1] ;\ 2161 ba,pt %xcc, done1 ;\ 2162 std %f40, [ADDR + CPU_TMP1] ;\ 2163 ba,pt %xcc, done1 ;\ 2164 std %f10, [ADDR + CPU_TMP1] ;\ 2165 ba,pt %xcc, done1 ;\ 2166 std %f42, [ADDR + CPU_TMP1] ;\ 2167 ba,pt %xcc, done1 ;\ 2168 std %f12, [ADDR + CPU_TMP1] ;\ 2169 ba,pt %xcc, done1 ;\ 2170 std %f44, [ADDR + CPU_TMP1] ;\ 2171 ba,pt %xcc, done1 ;\ 2172 std %f14, [ADDR + CPU_TMP1] ;\ 2173 ba,pt %xcc, done1 ;\ 2174 std %f46, [ADDR + CPU_TMP1] ;\ 2175 ba,pt %xcc, done1 ;\ 2176 std %f16, [ADDR + CPU_TMP1] ;\ 2177 ba,pt %xcc, done1 ;\ 2178 std %f48, [ADDR + CPU_TMP1] ;\ 2179 ba,pt %xcc, done1 ;\ 2180 std %f18, [ADDR + CPU_TMP1] ;\ 2181 ba,pt %xcc, done1 ;\ 2182 std %f50, [ADDR + CPU_TMP1] ;\ 2183 ba,pt %xcc, done1 ;\ 2184 std %f20, [ADDR + CPU_TMP1] ;\ 2185 ba,pt %xcc, done1 ;\ 2186 std %f52, [ADDR + CPU_TMP1] ;\ 2187 ba,pt %xcc, done1 ;\ 2188 std %f22, [ADDR + CPU_TMP1] ;\ 2189 ba,pt %xcc, done1 ;\ 2190 std %f54, [ADDR + CPU_TMP1] ;\ 2191 ba,pt %xcc, done1 ;\ 2192 std %f24, [ADDR + CPU_TMP1] ;\ 2193 ba,pt %xcc, done1 ;\ 2194 std %f56, [ADDR + CPU_TMP1] ;\ 2195 ba,pt %xcc, done1 ;\ 2196 std %f26, [ADDR + CPU_TMP1] ;\ 2197 ba,pt %xcc, done1 ;\ 2198 std %f58, [ADDR + CPU_TMP1] ;\ 2199 ba,pt %xcc, done1 ;\ 2200 std %f28, [ADDR + CPU_TMP1] ;\ 2201 ba,pt %xcc, done1 ;\ 2202 std %f60, [ADDR + CPU_TMP1] ;\ 2203 ba,pt %xcc, done1 ;\ 2204 std %f30, [ADDR + CPU_TMP1] ;\ 2205 ba,pt %xcc, done1 ;\ 2206 std %f62, [ADDR + CPU_TMP1] ;\ 2207done1: 2208 2209#define LDDF_REG(REG, ADDR, TMP) \ 2210 sll REG, 3, REG ;\ 2211mark2: set start2, TMP ;\ 2212 jmp REG + TMP ;\ 2213 nop ;\ 2214start2: ba,pt %xcc, done2 ;\ 2215 ldd [ADDR + CPU_TMP1], %f0 ;\ 2216 ba,pt %xcc, done2 ;\ 2217 ldd [ADDR + CPU_TMP1], %f32 ;\ 2218 ba,pt %xcc, done2 ;\ 2219 ldd [ADDR + CPU_TMP1], %f2 ;\ 2220 ba,pt %xcc, done2 ;\ 2221 ldd [ADDR + CPU_TMP1], %f34 ;\ 2222 ba,pt %xcc, done2 ;\ 2223 ldd [ADDR + CPU_TMP1], %f4 ;\ 2224 ba,pt %xcc, done2 ;\ 2225 ldd [ADDR + CPU_TMP1], %f36 ;\ 2226 ba,pt %xcc, done2 ;\ 2227 ldd [ADDR + CPU_TMP1], %f6 ;\ 2228 ba,pt %xcc, done2 ;\ 2229 ldd [ADDR + CPU_TMP1], %f38 ;\ 2230 ba,pt %xcc, done2 ;\ 2231 ldd [ADDR + CPU_TMP1], %f8 ;\ 2232 ba,pt %xcc, done2 ;\ 2233 ldd [ADDR + CPU_TMP1], %f40 ;\ 2234 ba,pt %xcc, done2 ;\ 2235 ldd [ADDR + CPU_TMP1], %f10 ;\ 2236 ba,pt %xcc, done2 ;\ 2237 ldd [ADDR + CPU_TMP1], %f42 ;\ 2238 ba,pt %xcc, done2 ;\ 2239 ldd [ADDR + CPU_TMP1], %f12 ;\ 2240 ba,pt %xcc, done2 ;\ 2241 ldd [ADDR + CPU_TMP1], %f44 ;\ 2242 ba,pt %xcc, done2 ;\ 2243 ldd [ADDR + CPU_TMP1], %f14 ;\ 2244 ba,pt %xcc, done2 ;\ 2245 ldd [ADDR + CPU_TMP1], %f46 ;\ 2246 ba,pt %xcc, done2 ;\ 2247 ldd [ADDR + CPU_TMP1], %f16 ;\ 2248 ba,pt %xcc, done2 ;\ 2249 ldd [ADDR + CPU_TMP1], %f48 ;\ 2250 ba,pt %xcc, done2 ;\ 2251 ldd [ADDR + CPU_TMP1], %f18 ;\ 2252 ba,pt %xcc, done2 ;\ 2253 ldd [ADDR + CPU_TMP1], %f50 ;\ 2254 ba,pt %xcc, done2 ;\ 2255 ldd [ADDR + CPU_TMP1], %f20 ;\ 2256 ba,pt %xcc, done2 ;\ 2257 ldd [ADDR + CPU_TMP1], %f52 ;\ 2258 ba,pt %xcc, done2 ;\ 2259 ldd [ADDR + CPU_TMP1], %f22 ;\ 2260 ba,pt %xcc, done2 ;\ 2261 ldd [ADDR + CPU_TMP1], %f54 ;\ 2262 ba,pt %xcc, done2 ;\ 2263 ldd [ADDR + CPU_TMP1], %f24 ;\ 2264 ba,pt %xcc, done2 ;\ 2265 ldd [ADDR + CPU_TMP1], %f56 ;\ 2266 ba,pt %xcc, done2 ;\ 2267 ldd [ADDR + CPU_TMP1], %f26 ;\ 2268 ba,pt %xcc, done2 ;\ 2269 ldd [ADDR + CPU_TMP1], %f58 ;\ 2270 ba,pt %xcc, done2 ;\ 2271 ldd [ADDR + CPU_TMP1], %f28 ;\ 2272 ba,pt %xcc, done2 ;\ 2273 ldd [ADDR + CPU_TMP1], %f60 ;\ 2274 ba,pt %xcc, done2 ;\ 2275 ldd [ADDR + CPU_TMP1], %f30 ;\ 2276 ba,pt %xcc, done2 ;\ 2277 ldd [ADDR + CPU_TMP1], %f62 ;\ 2278done2: 2279 2280.lddf_exception_not_aligned: 2281 /* 2282 * Cheetah overwrites SFAR on a DTLB miss, hence read it now. 2283 */ 2284 ldxa [MMU_SFAR]%asi, %g5 ! misaligned vaddr in %g5 2285 2286#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 2287 sethi %hi(fpu_exists), %g2 ! check fpu_exists 2288 ld [%g2 + %lo(fpu_exists)], %g2 2289 brz,a,pn %g2, 4f 2290 nop 2291#endif 2292 CPU_ADDR(%g1, %g4) 2293 or %g0, 1, %g4 2294 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag 2295 2296 rdpr %tpc, %g2 2297 lda [%g2]ASI_AIUP, %g6 ! get the user's lddf instruction 2298 srl %g6, 23, %g1 ! using ldda or not? 2299 and %g1, 1, %g1 2300 brz,a,pt %g1, 2f ! check for ldda instruction 2301 nop 2302 srl %g6, 13, %g1 ! check immflag 2303 and %g1, 1, %g1 2304 rdpr %tstate, %g2 ! %tstate in %g2 2305 brnz,a,pn %g1, 1f 2306 srl %g2, 31, %g1 ! get asi from %tstate 2307 srl %g6, 5, %g1 ! get asi from instruction 2308 and %g1, 0xFF, %g1 ! imm_asi field 23091: 2310 cmp %g1, ASI_P ! primary address space 2311 be,a,pt %icc, 2f 2312 nop 2313 cmp %g1, ASI_PNF ! primary no fault address space 2314 be,a,pt %icc, 2f 2315 nop 2316 cmp %g1, ASI_S ! secondary address space 2317 be,a,pt %icc, 2f 2318 nop 2319 cmp %g1, ASI_SNF ! secondary no fault address space 2320 bne,a,pn %icc, 3f 2321 nop 23222: 2323 lduwa [%g5]ASI_USER, %g7 ! get first half of misaligned data 2324 add %g5, 4, %g5 ! increment misaligned data address 2325 lduwa [%g5]ASI_USER, %g5 ! get second half of misaligned data 2326 2327 sllx %g7, 32, %g7 2328 or %g5, %g7, %g5 ! combine data 2329 CPU_ADDR(%g7, %g1) ! save data on a per-cpu basis 2330 stx %g5, [%g7 + CPU_TMP1] ! save in cpu_tmp1 2331 2332 srl %g6, 25, %g3 ! %g6 has the instruction 2333 and %g3, 0x1F, %g3 ! %g3 has rd 2334 LDDF_REG(%g3, %g7, %g4) 2335 2336 CPU_ADDR(%g1, %g4) 2337 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 2338 FAST_TRAP_DONE 23393: 2340 CPU_ADDR(%g1, %g4) 2341 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 23424: 2343 set T_USER, %g3 ! trap type in %g3 2344 or %g3, T_LDDF_ALIGN, %g3 2345 mov %g5, %g2 ! misaligned vaddr in %g2 2346 set fpu_trap, %g1 ! goto C for the little and 2347 ba,pt %xcc, sys_trap ! no fault little asi's 2348 sub %g0, 1, %g4 2349 2350.stdf_exception_not_aligned: 2351 /* 2352 * Cheetah overwrites SFAR on a DTLB miss, hence read it now. 2353 */ 2354 ldxa [MMU_SFAR]%asi, %g5 ! misaligned vaddr in %g5 2355 2356#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 2357 sethi %hi(fpu_exists), %g7 ! check fpu_exists 2358 ld [%g7 + %lo(fpu_exists)], %g3 2359 brz,a,pn %g3, 4f 2360 nop 2361#endif 2362 CPU_ADDR(%g1, %g4) 2363 or %g0, 1, %g4 2364 st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag 2365 2366 rdpr %tpc, %g2 2367 lda [%g2]ASI_AIUP, %g6 ! get the user's stdf instruction 2368 2369 srl %g6, 23, %g1 ! using stda or not? 2370 and %g1, 1, %g1 2371 brz,a,pt %g1, 2f ! check for stda instruction 2372 nop 2373 srl %g6, 13, %g1 ! check immflag 2374 and %g1, 1, %g1 2375 rdpr %tstate, %g2 ! %tstate in %g2 2376 brnz,a,pn %g1, 1f 2377 srl %g2, 31, %g1 ! get asi from %tstate 2378 srl %g6, 5, %g1 ! get asi from instruction 2379 and %g1, 0xFF, %g1 ! imm_asi field 23801: 2381 cmp %g1, ASI_P ! primary address space 2382 be,a,pt %icc, 2f 2383 nop 2384 cmp %g1, ASI_S ! secondary address space 2385 bne,a,pn %icc, 3f 2386 nop 23872: 2388 srl %g6, 25, %g6 2389 and %g6, 0x1F, %g6 ! %g6 has rd 2390 CPU_ADDR(%g7, %g1) 2391 STDF_REG(%g6, %g7, %g4) ! STDF_REG(REG, ADDR, TMP) 2392 2393 ldx [%g7 + CPU_TMP1], %g6 2394 srlx %g6, 32, %g7 2395 stuwa %g7, [%g5]ASI_USER ! first half 2396 add %g5, 4, %g5 ! increment misaligned data address 2397 stuwa %g6, [%g5]ASI_USER ! second half 2398 2399 CPU_ADDR(%g1, %g4) 2400 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 2401 FAST_TRAP_DONE 24023: 2403 CPU_ADDR(%g1, %g4) 2404 st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag 24054: 2406 set T_USER, %g3 ! trap type in %g3 2407 or %g3, T_STDF_ALIGN, %g3 2408 mov %g5, %g2 ! misaligned vaddr in %g2 2409 set fpu_trap, %g1 ! goto C for the little and 2410 ba,pt %xcc, sys_trap ! nofault little asi's 2411 sub %g0, 1, %g4 2412 2413#ifdef DEBUG_USER_TRAPTRACECTL 2414 2415.traptrace_freeze: 2416 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4 2417 TT_TRACE_L(trace_win) 2418 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0 2419 set trap_freeze, %g1 2420 mov 1, %g2 2421 st %g2, [%g1] 2422 FAST_TRAP_DONE 2423 2424.traptrace_unfreeze: 2425 set trap_freeze, %g1 2426 st %g0, [%g1] 2427 mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4 2428 TT_TRACE_L(trace_win) 2429 mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0 2430 FAST_TRAP_DONE 2431 2432#endif /* DEBUG_USER_TRAPTRACECTL */ 2433 2434.getcc: 2435 CPU_ADDR(%g1, %g2) 2436 stx %o0, [%g1 + CPU_TMP1] ! save %o0 2437 stx %o1, [%g1 + CPU_TMP2] ! save %o1 2438 rdpr %tstate, %g3 ! get tstate 2439 srlx %g3, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr 2440 set PSR_ICC, %g2 2441 and %o0, %g2, %o0 ! mask out the rest 2442 srl %o0, PSR_ICC_SHIFT, %o0 ! right justify 2443 rdpr %pstate, %o1 2444 wrpr %o1, PSTATE_AG, %pstate ! get into normal globals 2445 mov %o0, %g1 ! move ccr to normal %g1 2446 wrpr %g0, %o1, %pstate ! back into alternate globals 2447 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0 2448 ldx [%g1 + CPU_TMP2], %o1 ! restore %o1 2449 FAST_TRAP_DONE 2450 2451.setcc: 2452 CPU_ADDR(%g1, %g2) 2453 stx %o0, [%g1 + CPU_TMP1] ! save %o0 2454 stx %o1, [%g1 + CPU_TMP2] ! save %o1 2455 rdpr %pstate, %o0 2456 wrpr %o0, PSTATE_AG, %pstate ! get into normal globals 2457 mov %g1, %o1 2458 wrpr %g0, %o0, %pstate ! back to alternates 2459 sll %o1, PSR_ICC_SHIFT, %g2 2460 set PSR_ICC, %g3 2461 and %g2, %g3, %g2 ! mask out rest 2462 sllx %g2, PSR_TSTATE_CC_SHIFT, %g2 2463 rdpr %tstate, %g3 ! get tstate 2464 srl %g3, 0, %g3 ! clear upper word 2465 or %g3, %g2, %g3 ! or in new bits 2466 wrpr %g3, %tstate 2467 ldx [%g1 + CPU_TMP1], %o0 ! restore %o0 2468 ldx [%g1 + CPU_TMP2], %o1 ! restore %o1 2469 FAST_TRAP_DONE 2470 2471/* 2472 * getpsr(void) 2473 * Note that the xcc part of the ccr is not provided. 2474 * The V8 code shows why the V9 trap is not faster: 2475 * #define GETPSR_TRAP() \ 2476 * mov %psr, %i0; jmp %l2; rett %l2+4; nop; 2477 */ 2478 2479 .type .getpsr, #function 2480.getpsr: 2481 rdpr %tstate, %g1 ! get tstate 2482 srlx %g1, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr 2483 set PSR_ICC, %g2 2484 and %o0, %g2, %o0 ! mask out the rest 2485 2486 rd %fprs, %g1 ! get fprs 2487 and %g1, FPRS_FEF, %g2 ! mask out dirty upper/lower 2488 sllx %g2, PSR_FPRS_FEF_SHIFT, %g2 ! shift fef to V8 psr.ef 2489 or %o0, %g2, %o0 ! or result into psr.ef 2490 2491 set V9_PSR_IMPLVER, %g2 ! SI assigned impl/ver: 0xef 2492 or %o0, %g2, %o0 ! or psr.impl/ver 2493 FAST_TRAP_DONE 2494 SET_SIZE(.getpsr) 2495 2496/* 2497 * setpsr(newpsr) 2498 * Note that there is no support for ccr.xcc in the V9 code. 2499 */ 2500 2501 .type .setpsr, #function 2502.setpsr: 2503 rdpr %tstate, %g1 ! get tstate 2504! setx TSTATE_V8_UBITS, %g2 2505 or %g0, CCR_ICC, %g3 2506 sllx %g3, TSTATE_CCR_SHIFT, %g2 2507 2508 andn %g1, %g2, %g1 ! zero current user bits 2509 set PSR_ICC, %g2 2510 and %g2, %o0, %g2 ! clear all but psr.icc bits 2511 sllx %g2, PSR_TSTATE_CC_SHIFT, %g3 ! shift to tstate.ccr.icc 2512 wrpr %g1, %g3, %tstate ! write tstate 2513 2514 set PSR_EF, %g2 2515 and %g2, %o0, %g2 ! clear all but fp enable bit 2516 srlx %g2, PSR_FPRS_FEF_SHIFT, %g4 ! shift ef to V9 fprs.fef 2517 wr %g0, %g4, %fprs ! write fprs 2518 2519 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 2520 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer 2521 ldn [%g2 + T_LWP], %g3 ! load klwp pointer 2522 ldn [%g3 + LWP_FPU], %g2 ! get lwp_fpu pointer 2523 stuw %g4, [%g2 + FPU_FPRS] ! write fef value to fpu_fprs 2524 srlx %g4, 2, %g4 ! shift fef value to bit 0 2525 stub %g4, [%g2 + FPU_EN] ! write fef value to fpu_en 2526 FAST_TRAP_DONE 2527 SET_SIZE(.setpsr) 2528 2529/* 2530 * getlgrp 2531 * get home lgrpid on which the calling thread is currently executing. 2532 */ 2533 .type .getlgrp, #function 2534.getlgrp: 2535 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 2536 ld [%g1 + CPU_ID], %o0 ! load cpu_id 2537 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer 2538 ldn [%g2 + T_LPL], %g2 ! load lpl pointer 2539 ld [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid 2540 sra %g1, 0, %o1 2541 FAST_TRAP_DONE 2542 SET_SIZE(.getlgrp) 2543 2544/* 2545 * Entry for old 4.x trap (trap 0). 2546 */ 2547 ENTRY_NP(syscall_trap_4x) 2548 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 2549 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer 2550 ldn [%g2 + T_LWP], %g2 ! load klwp pointer 2551 ld [%g2 + PCB_TRAP0], %g2 ! lwp->lwp_pcb.pcb_trap0addr 2552 brz,pn %g2, 1f ! has it been set? 2553 st %l0, [%g1 + CPU_TMP1] ! delay - save some locals 2554 st %l1, [%g1 + CPU_TMP2] 2555 rdpr %tnpc, %l1 ! save old tnpc 2556 wrpr %g0, %g2, %tnpc ! setup tnpc 2557 2558 rdpr %pstate, %l0 2559 wrpr %l0, PSTATE_AG, %pstate ! switch to normal globals 2560 mov %l1, %g6 ! pass tnpc to user code in %g6 2561 wrpr %l0, %g0, %pstate ! switch back to alternate globals 2562 2563 ! Note that %g1 still contains CPU struct addr 2564 ld [%g1 + CPU_TMP2], %l1 ! restore locals 2565 ld [%g1 + CPU_TMP1], %l0 2566 FAST_TRAP_DONE_CHK_INTR 25671: 2568 mov %g1, %l0 2569 st %l1, [%g1 + CPU_TMP2] 2570 rdpr %pstate, %l1 2571 wrpr %l1, PSTATE_AG, %pstate 2572 ! 2573 ! check for old syscall mmap which is the only different one which 2574 ! must be the same. Others are handled in the compatibility library. 2575 ! 2576 cmp %g1, OSYS_mmap ! compare to old 4.x mmap 2577 movz %icc, SYS_mmap, %g1 2578 wrpr %g0, %l1, %pstate 2579 ld [%l0 + CPU_TMP2], %l1 ! restore locals 2580 ld [%l0 + CPU_TMP1], %l0 2581 SYSCALL(syscall_trap32) 2582 SET_SIZE(syscall_trap_4x) 2583 2584/* 2585 * Handler for software trap 9. 2586 * Set trap0 emulation address for old 4.x system call trap. 2587 * XXX - this should be a system call. 2588 */ 2589 ENTRY_NP(set_trap0_addr) 2590 CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2 2591 ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer 2592 ldn [%g2 + T_LWP], %g2 ! load klwp pointer 2593 st %l0, [%g1 + CPU_TMP1] ! save some locals 2594 st %l1, [%g1 + CPU_TMP2] 2595 rdpr %pstate, %l0 2596 wrpr %l0, PSTATE_AG, %pstate 2597 mov %g1, %l1 2598 wrpr %g0, %l0, %pstate 2599 andn %l1, 3, %l1 ! force alignment 2600 st %l1, [%g2 + PCB_TRAP0] ! lwp->lwp_pcb.pcb_trap0addr 2601 ld [%g1 + CPU_TMP1], %l0 ! restore locals 2602 ld [%g1 + CPU_TMP2], %l1 2603 FAST_TRAP_DONE 2604 SET_SIZE(set_trap0_addr) 2605 2606/* 2607 * mmu_trap_tl1 2608 * trap handler for unexpected mmu traps. 2609 * simply checks if the trap was a user lddf/stdf alignment trap, in which 2610 * case we go to fpu_trap or a user trap from the window handler, in which 2611 * case we go save the state on the pcb. Otherwise, we go to ptl1_panic. 2612 */ 2613 .type mmu_trap_tl1, #function 2614mmu_trap_tl1: 2615#ifdef TRAPTRACE 2616 TRACE_PTR(%g5, %g6) 2617 GET_TRACE_TICK(%g6, %g7) 2618 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi 2619 rdpr %tl, %g6 2620 stha %g6, [%g5 + TRAP_ENT_TL]%asi 2621 rdpr %tt, %g6 2622 stha %g6, [%g5 + TRAP_ENT_TT]%asi 2623 rdpr %tstate, %g6 2624 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi 2625 stna %sp, [%g5 + TRAP_ENT_SP]%asi 2626 stna %g0, [%g5 + TRAP_ENT_TR]%asi 2627 rdpr %tpc, %g6 2628 stna %g6, [%g5 + TRAP_ENT_TPC]%asi 2629 set MMU_SFAR, %g6 2630 ldxa [%g6]ASI_DMMU, %g6 2631 stxa %g6, [%g5 + TRAP_ENT_F1]%asi 2632 CPU_PADDR(%g7, %g6); 2633 add %g7, CPU_TL1_HDLR, %g7 2634 lda [%g7]ASI_MEM, %g6 2635 stxa %g6, [%g5 + TRAP_ENT_F2]%asi 2636 set 0xdeadbeef, %g6 2637 stna %g6, [%g5 + TRAP_ENT_F3]%asi 2638 stna %g6, [%g5 + TRAP_ENT_F4]%asi 2639 TRACE_NEXT(%g5, %g6, %g7) 2640#endif /* TRAPTRACE */ 2641 2642 GET_CPU_IMPL(%g5) 2643 cmp %g5, PANTHER_IMPL 2644 bne mmu_trap_tl1_4 2645 nop 2646 rdpr %tt, %g5 2647 cmp %g5, T_DATA_EXCEPTION 2648 bne mmu_trap_tl1_4 2649 nop 2650 wr %g0, ASI_DMMU, %asi 2651 ldxa [MMU_SFSR]%asi, %g5 2652 mov 1, %g6 2653 sllx %g6, PN_SFSR_PARITY_SHIFT, %g6 2654 andcc %g5, %g6, %g0 2655 bz mmu_trap_tl1_4 2656 2657 /* 2658 * We are running on a Panther and have hit a DTLB parity error. 2659 */ 2660 ldxa [MMU_TAG_ACCESS]%asi, %g2 2661 mov %g5, %g3 2662 ba,pt %xcc, .mmu_exception_is_tlb_parity 2663 mov T_DATA_EXCEPTION, %g1 2664 2665mmu_trap_tl1_4: 2666 CPU_PADDR(%g7, %g6); 2667 add %g7, CPU_TL1_HDLR, %g7 ! %g7 = &cpu_m.tl1_hdlr (PA) 2668 /* 2669 * AM is cleared on trap, so addresses are 64 bit 2670 */ 2671 lda [%g7]ASI_MEM, %g6 2672 brz,a,pt %g6, 1f 2673 nop 2674 /* 2675 * We are going to update cpu_m.tl1_hdlr using physical address. 2676 * Flush the D$ line, so that stale data won't be accessed later. 2677 */ 2678 CPU_ADDR(%g6, %g5) 2679 add %g6, CPU_TL1_HDLR, %g6 ! %g6 = &cpu_m.tl1_hdlr (VA) 2680 GET_CPU_IMPL(%g5) 2681 cmp %g5, CHEETAH_IMPL 2682 bl,pt %icc, 3f 2683 cmp %g5, SPITFIRE_IMPL 2684 stxa %g0, [%g7]ASI_DC_INVAL 2685 membar #Sync 2686 ba,pt %xcc, 2f 2687 nop 26883: 2689 bl,pt %icc, 2f 2690 sethi %hi(dcache_line_mask), %g5 2691 ld [%g5 + %lo(dcache_line_mask)], %g5 2692 and %g6, %g5, %g5 2693 stxa %g0, [%g5]ASI_DC_TAG 2694 membar #Sync 26952: 2696 sta %g0, [%g7]ASI_MEM 2697 SWITCH_GLOBALS ! back to mmu globals 2698 ba,a,pt %xcc, sfmmu_mmu_trap ! handle page faults 26991: 2700 rdpr %tt, %g5 2701 rdpr %tl, %g7 2702 sub %g7, 1, %g6 2703 wrpr %g6, %tl 2704 rdpr %tt, %g6 2705 wrpr %g7, %tl 2706 and %g6, WTRAP_TTMASK, %g6 2707 cmp %g6, WTRAP_TYPE 2708 bne,a,pn %xcc, ptl1_panic 2709 mov PTL1_BAD_MMUTRAP, %g1 2710 rdpr %tpc, %g7 2711 /* tpc should be in the trap table */ 2712 set trap_table, %g6 2713 cmp %g7, %g6 2714 blt,a,pn %xcc, ptl1_panic 2715 mov PTL1_BAD_MMUTRAP, %g1 2716 set etrap_table, %g6 2717 cmp %g7, %g6 2718 bge,a,pn %xcc, ptl1_panic 2719 mov PTL1_BAD_MMUTRAP, %g1 2720 cmp %g5, T_ALIGNMENT 2721 move %icc, MMU_SFAR, %g6 2722 movne %icc, MMU_TAG_ACCESS, %g6 2723 ldxa [%g6]ASI_DMMU, %g6 2724 andn %g7, WTRAP_ALIGN, %g7 /* 128 byte aligned */ 2725 add %g7, WTRAP_FAULTOFF, %g7 2726 wrpr %g0, %g7, %tnpc 2727 done 2728 SET_SIZE(mmu_trap_tl1) 2729 2730/* 2731 * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers. These 2732 * traps are valid only when kmdb is loaded. When the debugger is active, 2733 * the code below is rewritten to transfer control to the appropriate 2734 * debugger entry points. 2735 */ 2736 .global kmdb_trap 2737 .align 8 2738kmdb_trap: 2739 ba,a trap_table0 2740 jmp %g1 + 0 2741 nop 2742 2743 .global kmdb_trap_tl1 2744 .align 8 2745kmdb_trap_tl1: 2746 ba,a trap_table0 2747 jmp %g1 + 0 2748 nop 2749 2750/* 2751 * This entry is copied from OBP's trap table during boot. 2752 */ 2753 .global obp_bpt 2754 .align 8 2755obp_bpt: 2756 NOT 2757 2758/* 2759 * if kernel, set PCONTEXT to 0 for debuggers 2760 * if user, clear nucleus page sizes 2761 */ 2762 .global kctx_obp_bpt 2763kctx_obp_bpt: 2764 set obp_bpt, %g2 27651: 2766#ifndef _OPL 2767 mov MMU_PCONTEXT, %g1 2768 ldxa [%g1]ASI_DMMU, %g1 2769 srlx %g1, CTXREG_NEXT_SHIFT, %g3 2770 brz,pt %g3, 3f ! nucleus pgsz is 0, no problem 2771 sllx %g3, CTXREG_NEXT_SHIFT, %g3 2772 set CTXREG_CTX_MASK, %g4 ! check Pcontext 2773 btst %g4, %g1 2774 bz,a,pt %xcc, 2f 2775 clr %g3 ! kernel: PCONTEXT=0 2776 xor %g3, %g1, %g3 ! user: clr N_pgsz0/1 bits 27772: 2778 set DEMAP_ALL_TYPE, %g1 2779 stxa %g0, [%g1]ASI_DTLB_DEMAP 2780 stxa %g0, [%g1]ASI_ITLB_DEMAP 2781 mov MMU_PCONTEXT, %g1 2782 stxa %g3, [%g1]ASI_DMMU 2783 membar #Sync 2784 sethi %hi(FLUSH_ADDR), %g1 2785 flush %g1 ! flush required by immu 2786#endif /* _OPL */ 27873: 2788 jmp %g2 2789 nop 2790 2791 2792#ifdef TRAPTRACE 2793/* 2794 * TRAPTRACE support. 2795 * labels here are branched to with "rd %pc, %g7" in the delay slot. 2796 * Return is done by "jmp %g7 + 4". 2797 */ 2798 2799trace_gen: 2800 TRACE_PTR(%g3, %g6) 2801 GET_TRACE_TICK(%g6, %g4) 2802 stxa %g6, [%g3 + TRAP_ENT_TICK]%asi 2803 rdpr %tl, %g6 2804 stha %g6, [%g3 + TRAP_ENT_TL]%asi 2805 rdpr %tt, %g6 2806 stha %g6, [%g3 + TRAP_ENT_TT]%asi 2807 rdpr %tstate, %g6 2808 stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi 2809 stna %sp, [%g3 + TRAP_ENT_SP]%asi 2810 rdpr %tpc, %g6 2811 stna %g6, [%g3 + TRAP_ENT_TPC]%asi 2812 TRACE_NEXT(%g3, %g4, %g5) 2813 jmp %g7 + 4 2814 nop 2815 2816trace_win: 2817 TRACE_WIN_INFO(0, %l0, %l1, %l2) 2818 ! Keep the locals as clean as possible, caller cleans %l4 2819 clr %l2 2820 clr %l1 2821 jmp %l4 + 4 2822 clr %l0 2823 2824/* 2825 * Trace a tsb hit 2826 * g1 = tsbe pointer (in/clobbered) 2827 * g2 = tag access register (in) 2828 * g3 - g4 = scratch (clobbered) 2829 * g5 = tsbe data (in) 2830 * g6 = scratch (clobbered) 2831 * g7 = pc we jumped here from (in) 2832 */ 2833 2834 ! Do not disturb %g5, it will be used after the trace 2835 ALTENTRY(trace_tsbhit) 2836 TRACE_TSBHIT(0) 2837 jmp %g7 + 4 2838 nop 2839 2840/* 2841 * Trace a TSB miss 2842 * 2843 * g1 = tsb8k pointer (in) 2844 * g2 = tag access register (in) 2845 * g3 = tsb4m pointer (in) 2846 * g4 = tsbe tag (in/clobbered) 2847 * g5 - g6 = scratch (clobbered) 2848 * g7 = pc we jumped here from (in) 2849 */ 2850 .global trace_tsbmiss 2851trace_tsbmiss: 2852 membar #Sync 2853 sethi %hi(FLUSH_ADDR), %g6 2854 flush %g6 2855 TRACE_PTR(%g5, %g6) 2856 stxa %g2, [%g5 + TRAP_ENT_SP]%asi ! tag access 2857 stxa %g4, [%g5 + TRAP_ENT_F1]%asi ! tsb tag 2858 GET_TRACE_TICK(%g6, %g4) 2859 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi 2860 rdpr %tnpc, %g6 2861 stxa %g6, [%g5 + TRAP_ENT_F2]%asi 2862 stna %g1, [%g5 + TRAP_ENT_F3]%asi ! tsb8k pointer 2863 srlx %g1, 32, %g6 2864 stna %g6, [%g5 + TRAP_ENT_F4]%asi ! huh? 2865 rdpr %tpc, %g6 2866 stna %g6, [%g5 + TRAP_ENT_TPC]%asi 2867 rdpr %tl, %g6 2868 stha %g6, [%g5 + TRAP_ENT_TL]%asi 2869 rdpr %tt, %g6 2870 or %g6, TT_MMU_MISS, %g4 2871 stha %g4, [%g5 + TRAP_ENT_TT]%asi 2872 cmp %g6, FAST_IMMU_MISS_TT 2873 be,a %icc, 1f 2874 ldxa [%g0]ASI_IMMU, %g6 2875 ldxa [%g0]ASI_DMMU, %g6 28761: stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! tag target 2877 stxa %g3, [%g5 + TRAP_ENT_TR]%asi ! tsb4m pointer 2878 TRACE_NEXT(%g5, %g4, %g6) 2879 jmp %g7 + 4 2880 nop 2881 2882/* 2883 * g2 = tag access register (in) 2884 * g3 = ctx number (in) 2885 */ 2886trace_dataprot: 2887 membar #Sync 2888 sethi %hi(FLUSH_ADDR), %g6 2889 flush %g6 2890 TRACE_PTR(%g1, %g6) 2891 GET_TRACE_TICK(%g6, %g5) 2892 stxa %g6, [%g1 + TRAP_ENT_TICK]%asi 2893 rdpr %tpc, %g6 2894 stna %g6, [%g1 + TRAP_ENT_TPC]%asi 2895 rdpr %tstate, %g6 2896 stxa %g6, [%g1 + TRAP_ENT_TSTATE]%asi 2897 stxa %g2, [%g1 + TRAP_ENT_SP]%asi ! tag access reg 2898 stxa %g0, [%g1 + TRAP_ENT_TR]%asi 2899 stxa %g0, [%g1 + TRAP_ENT_F1]%asi 2900 stxa %g0, [%g1 + TRAP_ENT_F2]%asi 2901 stxa %g0, [%g1 + TRAP_ENT_F3]%asi 2902 stxa %g0, [%g1 + TRAP_ENT_F4]%asi 2903 rdpr %tl, %g6 2904 stha %g6, [%g1 + TRAP_ENT_TL]%asi 2905 rdpr %tt, %g6 2906 stha %g6, [%g1 + TRAP_ENT_TT]%asi 2907 TRACE_NEXT(%g1, %g4, %g5) 2908 jmp %g7 + 4 2909 nop 2910 2911#endif /* TRAPTRACE */ 2912 2913 .align 32 2914 .global pil15_epilogue 2915pil15_epilogue: 2916 ba pil_interrupt_common 2917 nop 2918 .align 32 2919 2920/* 2921 * fast_trap_done, fast_trap_done_chk_intr: 2922 * 2923 * Due to the design of UltraSPARC pipeline, pending interrupts are not 2924 * taken immediately after a RETRY or DONE instruction which causes IE to 2925 * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed 2926 * to execute first before taking any interrupts. If that instruction 2927 * results in other traps, and if the corresponding trap handler runs 2928 * entirely at TL=1 with interrupts disabled, then pending interrupts 2929 * won't be taken until after yet another instruction following the %tpc 2930 * or %tnpc. 2931 * 2932 * A malicious user program can use this feature to block out interrupts 2933 * for extended durations, which can result in send_mondo_timeout kernel 2934 * panic. 2935 * 2936 * This problem is addressed by servicing any pending interrupts via 2937 * sys_trap before returning back to the user mode from a fast trap 2938 * handler. The "done" instruction within a fast trap handler, which 2939 * runs entirely at TL=1 with interrupts disabled, is replaced with the 2940 * FAST_TRAP_DONE macro, which branches control to this fast_trap_done 2941 * entry point. 2942 * 2943 * We check for any pending interrupts here and force a sys_trap to 2944 * service those interrupts, if any. To minimize overhead, pending 2945 * interrupts are checked if the %tpc happens to be at 16K boundary, 2946 * which allows a malicious program to execute at most 4K consecutive 2947 * instructions before we service any pending interrupts. If a worst 2948 * case fast trap handler takes about 2 usec, then interrupts will be 2949 * blocked for at most 8 msec, less than a clock tick. 2950 * 2951 * For the cases where we don't know if the %tpc will cross a 16K 2952 * boundary, we can't use the above optimization and always process 2953 * any pending interrupts via fast_frap_done_chk_intr entry point. 2954 * 2955 * Entry Conditions: 2956 * %pstate am:0 priv:1 ie:0 2957 * globals are AG (not normal globals) 2958 */ 2959 2960 .global fast_trap_done, fast_trap_done_chk_intr 2961fast_trap_done: 2962 rdpr %tpc, %g5 2963 sethi %hi(0xffffc000), %g6 ! 1's complement of 0x3fff 2964 andncc %g5, %g6, %g0 ! check lower 14 bits of %tpc 2965 bz,a,pn %icc, 1f ! branch if zero (lower 32 bits only) 2966 ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g5 2967 done 2968 2969 ALTENTRY(fast_trap_done_check_interrupts) 2970fast_trap_done_chk_intr: 2971 ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g5 2972 29731: rd SOFTINT, %g6 2974 and %g5, IRSR_BUSY, %g5 2975 orcc %g5, %g6, %g0 2976 bnz,pn %xcc, 2f ! branch if any pending intr 2977 nop 2978 done 2979 29802: 2981 /* 2982 * We get here if there are any pending interrupts. 2983 * Adjust %tpc/%tnpc as we'll be resuming via "retry" 2984 * instruction. 2985 */ 2986 rdpr %tnpc, %g5 2987 wrpr %g0, %g5, %tpc 2988 add %g5, 4, %g5 2989 wrpr %g0, %g5, %tnpc 2990 2991 /* 2992 * Force a dummy sys_trap call so that interrupts can be serviced. 2993 */ 2994 set fast_trap_dummy_call, %g1 2995 ba,pt %xcc, sys_trap 2996 mov -1, %g4 2997 2998fast_trap_dummy_call: 2999 retl 3000 nop 3001 3002/* 3003 * Currently the brand syscall interposition code is not enabled by 3004 * default. Instead, when a branded zone is first booted the brand 3005 * infrastructure will patch the trap table so that the syscall 3006 * entry points are redirected to syscall_wrapper32 and syscall_wrapper 3007 * for ILP32 and LP64 syscalls respectively. this is done in 3008 * brand_plat_interposition_enable(). Note that the syscall wrappers 3009 * below do not collect any trap trace data since the syscall hot patch 3010 * points are reached after trap trace data has already been collected. 3011 */ 3012#define BRAND_CALLBACK(callback_id) \ 3013 CPU_ADDR(%g2, %g1) /* load CPU struct addr to %g2 */ ;\ 3014 ldn [%g2 + CPU_THREAD], %g3 /* load thread pointer */ ;\ 3015 ldn [%g3 + T_PROCP], %g3 /* get proc pointer */ ;\ 3016 ldn [%g3 + P_BRAND], %g3 /* get brand pointer */ ;\ 3017 brz %g3, 1f /* No brand? No callback. */ ;\ 3018 nop ;\ 3019 ldn [%g3 + B_MACHOPS], %g3 /* get machops list */ ;\ 3020 ldn [%g3 + (callback_id << 3)], %g3 ;\ 3021 brz %g3, 1f ;\ 3022 /* \ 3023 * This isn't pretty. We want a low-latency way for the callback \ 3024 * routine to decline to do anything. We just pass in an address \ 3025 * the routine can directly jmp back to, pretending that nothing \ 3026 * has happened. \ 3027 * \ 3028 * %g1: return address (where the brand handler jumps back to) \ 3029 * %g2: address of CPU structure \ 3030 * %g3: address of brand handler (where we will jump to) \ 3031 */ \ 3032 mov %pc, %g1 ;\ 3033 add %g1, 16, %g1 ;\ 3034 jmp %g3 ;\ 3035 nop ;\ 30361: 3037 3038 ENTRY_NP(syscall_wrapper32) 3039 BRAND_CALLBACK(BRAND_CB_SYSCALL32) 3040 SYSCALL_NOTT(syscall_trap32) 3041 SET_SIZE(syscall_wrapper32) 3042 3043 ENTRY_NP(syscall_wrapper) 3044 BRAND_CALLBACK(BRAND_CB_SYSCALL) 3045 SYSCALL_NOTT(syscall_trap) 3046 SET_SIZE(syscall_wrapper) 3047