1/* 2 * Low-level exception handling 3 * 4 * This file is subject to the terms and conditions of the GNU General Public 5 * License. See the file "COPYING" in the main directory of this archive 6 * for more details. 7 * 8 * Copyright (C) 2004 - 2008 by Tensilica Inc. 9 * Copyright (C) 2015 Cadence Design Systems Inc. 10 * 11 * Chris Zankel <chris@zankel.net> 12 * 13 */ 14 15#include <linux/linkage.h> 16#include <linux/pgtable.h> 17#include <asm/asm-offsets.h> 18#include <asm/asmmacro.h> 19#include <asm/processor.h> 20#include <asm/coprocessor.h> 21#include <asm/thread_info.h> 22#include <asm/asm-uaccess.h> 23#include <asm/unistd.h> 24#include <asm/ptrace.h> 25#include <asm/current.h> 26#include <asm/page.h> 27#include <asm/signal.h> 28#include <asm/tlbflush.h> 29#include <variant/tie-asm.h> 30 31/* Unimplemented features. */ 32 33#undef KERNEL_STACK_OVERFLOW_CHECK 34 35/* Not well tested. 36 * 37 * - fast_coprocessor 38 */ 39 40/* 41 * Macro to find first bit set in WINDOWBASE from the left + 1 42 * 43 * 100....0 -> 1 44 * 010....0 -> 2 45 * 000....1 -> WSBITS 46 */ 47 48 .macro ffs_ws bit mask 49 50#if XCHAL_HAVE_NSA 51 nsau \bit, \mask # 32-WSBITS ... 31 (32 iff 0) 52 addi \bit, \bit, WSBITS - 32 + 1 # uppest bit set -> return 1 53#else 54 movi \bit, WSBITS 55#if WSBITS > 16 56 _bltui \mask, 0x10000, 99f 57 addi \bit, \bit, -16 58 extui \mask, \mask, 16, 16 59#endif 60#if WSBITS > 8 6199: _bltui \mask, 0x100, 99f 62 addi \bit, \bit, -8 63 srli \mask, \mask, 8 64#endif 6599: _bltui \mask, 0x10, 99f 66 addi \bit, \bit, -4 67 srli \mask, \mask, 4 6899: _bltui \mask, 0x4, 99f 69 addi \bit, \bit, -2 70 srli \mask, \mask, 2 7199: _bltui \mask, 0x2, 99f 72 addi \bit, \bit, -1 7399: 74 75#endif 76 .endm 77 78 79 .macro irq_save flags tmp 80#if XTENSA_FAKE_NMI 81#if defined(CONFIG_DEBUG_KERNEL) && (LOCKLEVEL | TOPLEVEL) >= XCHAL_DEBUGLEVEL 82 rsr \flags, ps 83 extui \tmp, \flags, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH 84 bgei \tmp, LOCKLEVEL, 99f 85 rsil \tmp, LOCKLEVEL 8699: 87#else 88 movi \tmp, LOCKLEVEL 89 rsr \flags, ps 90 or \flags, \flags, \tmp 91 xsr \flags, ps 92 rsync 93#endif 94#else 95 rsil \flags, LOCKLEVEL 96#endif 97 .endm 98 99/* ----------------- DEFAULT FIRST LEVEL EXCEPTION HANDLERS ----------------- */ 100 101/* 102 * First-level exception handler for user exceptions. 103 * Save some special registers, extra states and all registers in the AR 104 * register file that were in use in the user task, and jump to the common 105 * exception code. 106 * We save SAR (used to calculate WMASK), and WB and WS (we don't have to 107 * save them for kernel exceptions). 108 * 109 * Entry condition for user_exception: 110 * 111 * a0: trashed, original value saved on stack (PT_AREG0) 112 * a1: a1 113 * a2: new stack pointer, original value in depc 114 * a3: a3 115 * depc: a2, original value saved on stack (PT_DEPC) 116 * excsave1: dispatch table 117 * 118 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 119 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 120 * 121 * Entry condition for _user_exception: 122 * 123 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC 124 * excsave has been restored, and 125 * stack pointer (a1) has been set. 126 * 127 * Note: _user_exception might be at an odd address. Don't use call0..call12 128 */ 129 .literal_position 130 131ENTRY(user_exception) 132 133 /* Save a1, a2, a3, and set SP. */ 134 135 rsr a0, depc 136 s32i a1, a2, PT_AREG1 137 s32i a0, a2, PT_AREG2 138 s32i a3, a2, PT_AREG3 139 mov a1, a2 140 141 .globl _user_exception 142_user_exception: 143 144 /* Save SAR and turn off single stepping */ 145 146 movi a2, 0 147 wsr a2, depc # terminate user stack trace with 0 148 rsr a3, sar 149 xsr a2, icountlevel 150 s32i a3, a1, PT_SAR 151 s32i a2, a1, PT_ICOUNTLEVEL 152 153#if XCHAL_HAVE_THREADPTR 154 rur a2, threadptr 155 s32i a2, a1, PT_THREADPTR 156#endif 157 158 /* Rotate ws so that the current windowbase is at bit0. */ 159 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */ 160 161#if defined(USER_SUPPORT_WINDOWED) 162 rsr a2, windowbase 163 rsr a3, windowstart 164 ssr a2 165 s32i a2, a1, PT_WINDOWBASE 166 s32i a3, a1, PT_WINDOWSTART 167 slli a2, a3, 32-WSBITS 168 src a2, a3, a2 169 srli a2, a2, 32-WSBITS 170 s32i a2, a1, PT_WMASK # needed for restoring registers 171#else 172 movi a2, 0 173 movi a3, 1 174 s32i a2, a1, PT_WINDOWBASE 175 s32i a3, a1, PT_WINDOWSTART 176 s32i a3, a1, PT_WMASK 177#endif 178 179 /* Save only live registers. */ 180 181UABI_W _bbsi.l a2, 1, 1f 182 s32i a4, a1, PT_AREG4 183 s32i a5, a1, PT_AREG5 184 s32i a6, a1, PT_AREG6 185 s32i a7, a1, PT_AREG7 186UABI_W _bbsi.l a2, 2, 1f 187 s32i a8, a1, PT_AREG8 188 s32i a9, a1, PT_AREG9 189 s32i a10, a1, PT_AREG10 190 s32i a11, a1, PT_AREG11 191UABI_W _bbsi.l a2, 3, 1f 192 s32i a12, a1, PT_AREG12 193 s32i a13, a1, PT_AREG13 194 s32i a14, a1, PT_AREG14 195 s32i a15, a1, PT_AREG15 196 197#if defined(USER_SUPPORT_WINDOWED) 198 _bnei a2, 1, 1f # only one valid frame? 199 200 /* Only one valid frame, skip saving regs. */ 201 202 j 2f 203 204 /* Save the remaining registers. 205 * We have to save all registers up to the first '1' from 206 * the right, except the current frame (bit 0). 207 * Assume a2 is: 001001000110001 208 * All register frames starting from the top field to the marked '1' 209 * must be saved. 210 */ 211 2121: addi a3, a2, -1 # eliminate '1' in bit 0: yyyyxxww0 213 neg a3, a3 # yyyyxxww0 -> YYYYXXWW1+1 214 and a3, a3, a2 # max. only one bit is set 215 216 /* Find number of frames to save */ 217 218 ffs_ws a0, a3 # number of frames to the '1' from left 219 220 /* Store information into WMASK: 221 * bits 0..3: xxx1 masked lower 4 bits of the rotated windowstart, 222 * bits 4...: number of valid 4-register frames 223 */ 224 225 slli a3, a0, 4 # number of frames to save in bits 8..4 226 extui a2, a2, 0, 4 # mask for the first 16 registers 227 or a2, a3, a2 228 s32i a2, a1, PT_WMASK # needed when we restore the reg-file 229 230 /* Save 4 registers at a time */ 231 2321: rotw -1 233 s32i a0, a5, PT_AREG_END - 16 234 s32i a1, a5, PT_AREG_END - 12 235 s32i a2, a5, PT_AREG_END - 8 236 s32i a3, a5, PT_AREG_END - 4 237 addi a0, a4, -1 238 addi a1, a5, -16 239 _bnez a0, 1b 240 241 /* WINDOWBASE still in SAR! */ 242 243 rsr a2, sar # original WINDOWBASE 244 movi a3, 1 245 ssl a2 246 sll a3, a3 247 wsr a3, windowstart # set corresponding WINDOWSTART bit 248 wsr a2, windowbase # and WINDOWSTART 249 rsync 250 251 /* We are back to the original stack pointer (a1) */ 252#endif 2532: /* Now, jump to the common exception handler. */ 254 255 j common_exception 256 257ENDPROC(user_exception) 258 259/* 260 * First-level exit handler for kernel exceptions 261 * Save special registers and the live window frame. 262 * Note: Even though we changes the stack pointer, we don't have to do a 263 * MOVSP here, as we do that when we return from the exception. 264 * (See comment in the kernel exception exit code) 265 * 266 * Entry condition for kernel_exception: 267 * 268 * a0: trashed, original value saved on stack (PT_AREG0) 269 * a1: a1 270 * a2: new stack pointer, original in DEPC 271 * a3: a3 272 * depc: a2, original value saved on stack (PT_DEPC) 273 * excsave_1: dispatch table 274 * 275 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 276 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 277 * 278 * Entry condition for _kernel_exception: 279 * 280 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC 281 * excsave has been restored, and 282 * stack pointer (a1) has been set. 283 * 284 * Note: _kernel_exception might be at an odd address. Don't use call0..call12 285 */ 286 287ENTRY(kernel_exception) 288 289 /* Save a1, a2, a3, and set SP. */ 290 291 rsr a0, depc # get a2 292 s32i a1, a2, PT_AREG1 293 s32i a0, a2, PT_AREG2 294 s32i a3, a2, PT_AREG3 295 mov a1, a2 296 297 .globl _kernel_exception 298_kernel_exception: 299 300 /* Save SAR and turn off single stepping */ 301 302 movi a2, 0 303 rsr a3, sar 304 xsr a2, icountlevel 305 s32i a3, a1, PT_SAR 306 s32i a2, a1, PT_ICOUNTLEVEL 307 308#if defined(__XTENSA_WINDOWED_ABI__) 309 /* Rotate ws so that the current windowbase is at bit0. */ 310 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */ 311 312 rsr a2, windowbase # don't need to save these, we only 313 rsr a3, windowstart # need shifted windowstart: windowmask 314 ssr a2 315 slli a2, a3, 32-WSBITS 316 src a2, a3, a2 317 srli a2, a2, 32-WSBITS 318 s32i a2, a1, PT_WMASK # needed for kernel_exception_exit 319#endif 320 321 /* Save only the live window-frame */ 322 323KABI_W _bbsi.l a2, 1, 1f 324 s32i a4, a1, PT_AREG4 325 s32i a5, a1, PT_AREG5 326 s32i a6, a1, PT_AREG6 327 s32i a7, a1, PT_AREG7 328KABI_W _bbsi.l a2, 2, 1f 329 s32i a8, a1, PT_AREG8 330 s32i a9, a1, PT_AREG9 331 s32i a10, a1, PT_AREG10 332 s32i a11, a1, PT_AREG11 333KABI_W _bbsi.l a2, 3, 1f 334 s32i a12, a1, PT_AREG12 335 s32i a13, a1, PT_AREG13 336 s32i a14, a1, PT_AREG14 337 s32i a15, a1, PT_AREG15 338 339#ifdef __XTENSA_WINDOWED_ABI__ 340 _bnei a2, 1, 1f 341 /* Copy spill slots of a0 and a1 to imitate movsp 342 * in order to keep exception stack continuous 343 */ 344 l32i a3, a1, PT_SIZE 345 l32i a0, a1, PT_SIZE + 4 346 s32e a3, a1, -16 347 s32e a0, a1, -12 348#endif 3491: 350 l32i a0, a1, PT_AREG0 # restore saved a0 351 wsr a0, depc 352 353#ifdef KERNEL_STACK_OVERFLOW_CHECK 354 355 /* Stack overflow check, for debugging */ 356 extui a2, a1, TASK_SIZE_BITS,XX 357 movi a3, SIZE?? 358 _bge a2, a3, out_of_stack_panic 359 360#endif 361 362/* 363 * This is the common exception handler. 364 * We get here from the user exception handler or simply by falling through 365 * from the kernel exception handler. 366 * Save the remaining special registers, switch to kernel mode, and jump 367 * to the second-level exception handler. 368 * 369 */ 370 371common_exception: 372 373 /* Save some registers, disable loops and clear the syscall flag. */ 374 375 rsr a2, debugcause 376 rsr a3, epc1 377 s32i a2, a1, PT_DEBUGCAUSE 378 s32i a3, a1, PT_PC 379 380 movi a2, NO_SYSCALL 381 rsr a3, excvaddr 382 s32i a2, a1, PT_SYSCALL 383 movi a2, 0 384 s32i a3, a1, PT_EXCVADDR 385#if XCHAL_HAVE_LOOPS 386 xsr a2, lcount 387 s32i a2, a1, PT_LCOUNT 388#endif 389 390#if XCHAL_HAVE_EXCLUSIVE 391 /* Clear exclusive access monitor set by interrupted code */ 392 clrex 393#endif 394 395 /* It is now save to restore the EXC_TABLE_FIXUP variable. */ 396 397 rsr a2, exccause 398 movi a3, 0 399 rsr a0, excsave1 400 s32i a2, a1, PT_EXCCAUSE 401 s32i a3, a0, EXC_TABLE_FIXUP 402 403 /* All unrecoverable states are saved on stack, now, and a1 is valid. 404 * Now we can allow exceptions again. In case we've got an interrupt 405 * PS.INTLEVEL is set to LOCKLEVEL disabling furhter interrupts, 406 * otherwise it's left unchanged. 407 * 408 * Set PS(EXCM = 0, UM = 0, RING = 0, OWB = 0, WOE = 1, INTLEVEL = X) 409 */ 410 411 rsr a3, ps 412 s32i a3, a1, PT_PS # save ps 413 414#if XTENSA_FAKE_NMI 415 /* Correct PS needs to be saved in the PT_PS: 416 * - in case of exception or level-1 interrupt it's in the PS, 417 * and is already saved. 418 * - in case of medium level interrupt it's in the excsave2. 419 */ 420 movi a0, EXCCAUSE_MAPPED_NMI 421 extui a3, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH 422 beq a2, a0, .Lmedium_level_irq 423 bnei a2, EXCCAUSE_LEVEL1_INTERRUPT, .Lexception 424 beqz a3, .Llevel1_irq # level-1 IRQ sets ps.intlevel to 0 425 426.Lmedium_level_irq: 427 rsr a0, excsave2 428 s32i a0, a1, PT_PS # save medium-level interrupt ps 429 bgei a3, LOCKLEVEL, .Lexception 430 431.Llevel1_irq: 432 movi a3, LOCKLEVEL 433 434.Lexception: 435KABI_W movi a0, PS_WOE_MASK 436KABI_W or a3, a3, a0 437#else 438 addi a2, a2, -EXCCAUSE_LEVEL1_INTERRUPT 439 movi a0, LOCKLEVEL 440 extui a3, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH 441 # a3 = PS.INTLEVEL 442 moveqz a3, a0, a2 # a3 = LOCKLEVEL iff interrupt 443KABI_W movi a2, PS_WOE_MASK 444KABI_W or a3, a3, a2 445 rsr a2, exccause 446#endif 447 448 /* restore return address (or 0 if return to userspace) */ 449 rsr a0, depc 450 wsr a3, ps 451 rsync # PS.WOE => rsync => overflow 452 453 /* Save lbeg, lend */ 454#if XCHAL_HAVE_LOOPS 455 rsr a4, lbeg 456 rsr a3, lend 457 s32i a4, a1, PT_LBEG 458 s32i a3, a1, PT_LEND 459#endif 460 461 /* Save SCOMPARE1 */ 462 463#if XCHAL_HAVE_S32C1I 464 rsr a3, scompare1 465 s32i a3, a1, PT_SCOMPARE1 466#endif 467 468 /* Save optional registers. */ 469 470 save_xtregs_opt a1 a3 a4 a5 a6 a7 PT_XTREGS_OPT 471 472 /* Go to second-level dispatcher. Set up parameters to pass to the 473 * exception handler and call the exception handler. 474 */ 475 476 rsr a4, excsave1 477 addx4 a4, a2, a4 478 l32i a4, a4, EXC_TABLE_DEFAULT # load handler 479 mov abi_arg1, a2 # pass EXCCAUSE 480 mov abi_arg0, a1 # pass stack frame 481 482 /* Call the second-level handler */ 483 484 abi_callx a4 485 486 /* Jump here for exception exit */ 487 .global common_exception_return 488common_exception_return: 489 490#if XTENSA_FAKE_NMI 491 l32i a2, a1, PT_EXCCAUSE 492 movi a3, EXCCAUSE_MAPPED_NMI 493 beq a2, a3, .LNMIexit 494#endif 4951: 496 irq_save a2, a3 497#ifdef CONFIG_TRACE_IRQFLAGS 498 abi_call trace_hardirqs_off 499#endif 500 501 /* Jump if we are returning from kernel exceptions. */ 502 503 l32i abi_saved1, a1, PT_PS 504 GET_THREAD_INFO(a2, a1) 505 l32i a4, a2, TI_FLAGS 506 _bbci.l abi_saved1, PS_UM_BIT, 6f 507 508 /* Specific to a user exception exit: 509 * We need to check some flags for signal handling and rescheduling, 510 * and have to restore WB and WS, extra states, and all registers 511 * in the register file that were in use in the user task. 512 * Note that we don't disable interrupts here. 513 */ 514 515 _bbsi.l a4, TIF_NEED_RESCHED, 3f 516 movi a2, _TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NOTIFY_SIGNAL 517 bnone a4, a2, 5f 518 5192: l32i a4, a1, PT_DEPC 520 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f 521 522 /* Call do_signal() */ 523 524#ifdef CONFIG_TRACE_IRQFLAGS 525 abi_call trace_hardirqs_on 526#endif 527 rsil a2, 0 528 mov abi_arg0, a1 529 abi_call do_notify_resume # int do_notify_resume(struct pt_regs*) 530 j 1b 531 5323: /* Reschedule */ 533 534#ifdef CONFIG_TRACE_IRQFLAGS 535 abi_call trace_hardirqs_on 536#endif 537 rsil a2, 0 538 abi_call schedule # void schedule (void) 539 j 1b 540 541#ifdef CONFIG_PREEMPTION 5426: 543 _bbci.l a4, TIF_NEED_RESCHED, 4f 544 545 /* Check current_thread_info->preempt_count */ 546 547 l32i a4, a2, TI_PRE_COUNT 548 bnez a4, 4f 549 abi_call preempt_schedule_irq 550 j 4f 551#endif 552 553#if XTENSA_FAKE_NMI 554.LNMIexit: 555 l32i abi_saved1, a1, PT_PS 556 _bbci.l abi_saved1, PS_UM_BIT, 4f 557#endif 558 5595: 560#ifdef CONFIG_HAVE_HW_BREAKPOINT 561 _bbci.l a4, TIF_DB_DISABLED, 7f 562 abi_call restore_dbreak 5637: 564#endif 565#ifdef CONFIG_DEBUG_TLB_SANITY 566 l32i a4, a1, PT_DEPC 567 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f 568 abi_call check_tlb_sanity 569#endif 5706: 5714: 572#ifdef CONFIG_TRACE_IRQFLAGS 573 extui a4, abi_saved1, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH 574 bgei a4, LOCKLEVEL, 1f 575 abi_call trace_hardirqs_on 5761: 577#endif 578 /* Restore optional registers. */ 579 580 load_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT 581 582 /* Restore SCOMPARE1 */ 583 584#if XCHAL_HAVE_S32C1I 585 l32i a2, a1, PT_SCOMPARE1 586 wsr a2, scompare1 587#endif 588 wsr abi_saved1, ps /* disable interrupts */ 589 590 _bbci.l abi_saved1, PS_UM_BIT, kernel_exception_exit 591 592user_exception_exit: 593 594 /* Restore the state of the task and return from the exception. */ 595 596#if defined(USER_SUPPORT_WINDOWED) 597 /* Switch to the user thread WINDOWBASE. Save SP temporarily in DEPC */ 598 599 l32i a2, a1, PT_WINDOWBASE 600 l32i a3, a1, PT_WINDOWSTART 601 wsr a1, depc # use DEPC as temp storage 602 wsr a3, windowstart # restore WINDOWSTART 603 ssr a2 # preserve user's WB in the SAR 604 wsr a2, windowbase # switch to user's saved WB 605 rsync 606 rsr a1, depc # restore stack pointer 607 l32i a2, a1, PT_WMASK # register frames saved (in bits 4...9) 608 rotw -1 # we restore a4..a7 609 _bltui a6, 16, 1f # only have to restore current window? 610 611 /* The working registers are a0 and a3. We are restoring to 612 * a4..a7. Be careful not to destroy what we have just restored. 613 * Note: wmask has the format YYYYM: 614 * Y: number of registers saved in groups of 4 615 * M: 4 bit mask of first 16 registers 616 */ 617 618 mov a2, a6 619 mov a3, a5 620 6212: rotw -1 # a0..a3 become a4..a7 622 addi a3, a7, -4*4 # next iteration 623 addi a2, a6, -16 # decrementing Y in WMASK 624 l32i a4, a3, PT_AREG_END + 0 625 l32i a5, a3, PT_AREG_END + 4 626 l32i a6, a3, PT_AREG_END + 8 627 l32i a7, a3, PT_AREG_END + 12 628 _bgeui a2, 16, 2b 629 630 /* Clear unrestored registers (don't leak anything to user-land */ 631 6321: rsr a0, windowbase 633 rsr a3, sar 634 sub a3, a0, a3 635 beqz a3, 2f 636 extui a3, a3, 0, WBBITS 637 6381: rotw -1 639 addi a3, a7, -1 640 movi a4, 0 641 movi a5, 0 642 movi a6, 0 643 movi a7, 0 644 bgei a3, 1, 1b 645 646 /* We are back were we were when we started. 647 * Note: a2 still contains WMASK (if we've returned to the original 648 * frame where we had loaded a2), or at least the lower 4 bits 649 * (if we have restored WSBITS-1 frames). 650 */ 6512: 652#else 653 movi a2, 1 654#endif 655#if XCHAL_HAVE_THREADPTR 656 l32i a3, a1, PT_THREADPTR 657 wur a3, threadptr 658#endif 659 660 j common_exception_exit 661 662 /* This is the kernel exception exit. 663 * We avoided to do a MOVSP when we entered the exception, but we 664 * have to do it here. 665 */ 666 667kernel_exception_exit: 668 669#if defined(__XTENSA_WINDOWED_ABI__) 670 /* Check if we have to do a movsp. 671 * 672 * We only have to do a movsp if the previous window-frame has 673 * been spilled to the *temporary* exception stack instead of the 674 * task's stack. This is the case if the corresponding bit in 675 * WINDOWSTART for the previous window-frame was set before 676 * (not spilled) but is zero now (spilled). 677 * If this bit is zero, all other bits except the one for the 678 * current window frame are also zero. So, we can use a simple test: 679 * 'and' WINDOWSTART and WINDOWSTART-1: 680 * 681 * (XXXXXX1[0]* - 1) AND XXXXXX1[0]* = XXXXXX0[0]* 682 * 683 * The result is zero only if one bit was set. 684 * 685 * (Note: We might have gone through several task switches before 686 * we come back to the current task, so WINDOWBASE might be 687 * different from the time the exception occurred.) 688 */ 689 690 /* Test WINDOWSTART before and after the exception. 691 * We actually have WMASK, so we only have to test if it is 1 or not. 692 */ 693 694 l32i a2, a1, PT_WMASK 695 _beqi a2, 1, common_exception_exit # Spilled before exception,jump 696 697 /* Test WINDOWSTART now. If spilled, do the movsp */ 698 699 rsr a3, windowstart 700 addi a0, a3, -1 701 and a3, a3, a0 702 _bnez a3, common_exception_exit 703 704 /* Do a movsp (we returned from a call4, so we have at least a0..a7) */ 705 706 addi a0, a1, -16 707 l32i a3, a0, 0 708 l32i a4, a0, 4 709 s32i a3, a1, PT_SIZE+0 710 s32i a4, a1, PT_SIZE+4 711 l32i a3, a0, 8 712 l32i a4, a0, 12 713 s32i a3, a1, PT_SIZE+8 714 s32i a4, a1, PT_SIZE+12 715 716 /* Common exception exit. 717 * We restore the special register and the current window frame, and 718 * return from the exception. 719 * 720 * Note: We expect a2 to hold PT_WMASK 721 */ 722#else 723 movi a2, 1 724#endif 725 726common_exception_exit: 727 728 /* Restore address registers. */ 729 730 _bbsi.l a2, 1, 1f 731 l32i a4, a1, PT_AREG4 732 l32i a5, a1, PT_AREG5 733 l32i a6, a1, PT_AREG6 734 l32i a7, a1, PT_AREG7 735 _bbsi.l a2, 2, 1f 736 l32i a8, a1, PT_AREG8 737 l32i a9, a1, PT_AREG9 738 l32i a10, a1, PT_AREG10 739 l32i a11, a1, PT_AREG11 740 _bbsi.l a2, 3, 1f 741 l32i a12, a1, PT_AREG12 742 l32i a13, a1, PT_AREG13 743 l32i a14, a1, PT_AREG14 744 l32i a15, a1, PT_AREG15 745 746 /* Restore PC, SAR */ 747 7481: l32i a2, a1, PT_PC 749 l32i a3, a1, PT_SAR 750 wsr a2, epc1 751 wsr a3, sar 752 753 /* Restore LBEG, LEND, LCOUNT */ 754#if XCHAL_HAVE_LOOPS 755 l32i a2, a1, PT_LBEG 756 l32i a3, a1, PT_LEND 757 wsr a2, lbeg 758 l32i a2, a1, PT_LCOUNT 759 wsr a3, lend 760 wsr a2, lcount 761#endif 762 763 /* We control single stepping through the ICOUNTLEVEL register. */ 764 765 l32i a2, a1, PT_ICOUNTLEVEL 766 movi a3, -2 767 wsr a2, icountlevel 768 wsr a3, icount 769 770 /* Check if it was double exception. */ 771 772 l32i a0, a1, PT_DEPC 773 l32i a3, a1, PT_AREG3 774 l32i a2, a1, PT_AREG2 775 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f 776 777 /* Restore a0...a3 and return */ 778 779 l32i a0, a1, PT_AREG0 780 l32i a1, a1, PT_AREG1 781 rfe 782 7831: wsr a0, depc 784 l32i a0, a1, PT_AREG0 785 l32i a1, a1, PT_AREG1 786 rfde 787 788ENDPROC(kernel_exception) 789 790/* 791 * Debug exception handler. 792 * 793 * Currently, we don't support KGDB, so only user application can be debugged. 794 * 795 * When we get here, a0 is trashed and saved to excsave[debuglevel] 796 */ 797 798 .literal_position 799 800ENTRY(debug_exception) 801 802 rsr a0, SREG_EPS + XCHAL_DEBUGLEVEL 803 bbsi.l a0, PS_EXCM_BIT, 1f # exception mode 804 805 /* Set EPC1 and EXCCAUSE */ 806 807 wsr a2, depc # save a2 temporarily 808 rsr a2, SREG_EPC + XCHAL_DEBUGLEVEL 809 wsr a2, epc1 810 811 movi a2, EXCCAUSE_MAPPED_DEBUG 812 wsr a2, exccause 813 814 /* Restore PS to the value before the debug exc but with PS.EXCM set.*/ 815 816 movi a2, 1 << PS_EXCM_BIT 817 or a2, a0, a2 818 wsr a2, ps 819 820 /* Switch to kernel/user stack, restore jump vector, and save a0 */ 821 822 bbsi.l a2, PS_UM_BIT, 2f # jump if user mode 823 824 addi a2, a1, -16-PT_SIZE # assume kernel stack 8253: 826 l32i a0, a3, DT_DEBUG_SAVE 827 s32i a1, a2, PT_AREG1 828 s32i a0, a2, PT_AREG0 829 movi a0, 0 830 s32i a0, a2, PT_DEPC # mark it as a regular exception 831 xsr a3, SREG_EXCSAVE + XCHAL_DEBUGLEVEL 832 xsr a0, depc 833 s32i a3, a2, PT_AREG3 834 s32i a0, a2, PT_AREG2 835 mov a1, a2 836 837 /* Debug exception is handled as an exception, so interrupts will 838 * likely be enabled in the common exception handler. Disable 839 * preemption if we have HW breakpoints to preserve DEBUGCAUSE.DBNUM 840 * meaning. 841 */ 842#if defined(CONFIG_PREEMPT_COUNT) && defined(CONFIG_HAVE_HW_BREAKPOINT) 843 GET_THREAD_INFO(a2, a1) 844 l32i a3, a2, TI_PRE_COUNT 845 addi a3, a3, 1 846 s32i a3, a2, TI_PRE_COUNT 847#endif 848 849 rsr a2, ps 850 bbsi.l a2, PS_UM_BIT, _user_exception 851 j _kernel_exception 852 8532: rsr a2, excsave1 854 l32i a2, a2, EXC_TABLE_KSTK # load kernel stack pointer 855 j 3b 856 857#ifdef CONFIG_HAVE_HW_BREAKPOINT 858 /* Debug exception while in exception mode. This may happen when 859 * window overflow/underflow handler or fast exception handler hits 860 * data breakpoint, in which case save and disable all data 861 * breakpoints, single-step faulting instruction and restore data 862 * breakpoints. 863 */ 8641: 865 bbci.l a0, PS_UM_BIT, 1b # jump if kernel mode 866 867 rsr a0, debugcause 868 bbsi.l a0, DEBUGCAUSE_DBREAK_BIT, .Ldebug_save_dbreak 869 870 .set _index, 0 871 .rept XCHAL_NUM_DBREAK 872 l32i a0, a3, DT_DBREAKC_SAVE + _index * 4 873 wsr a0, SREG_DBREAKC + _index 874 .set _index, _index + 1 875 .endr 876 877 l32i a0, a3, DT_ICOUNT_LEVEL_SAVE 878 wsr a0, icountlevel 879 880 l32i a0, a3, DT_ICOUNT_SAVE 881 xsr a0, icount 882 883 l32i a0, a3, DT_DEBUG_SAVE 884 xsr a3, SREG_EXCSAVE + XCHAL_DEBUGLEVEL 885 rfi XCHAL_DEBUGLEVEL 886 887.Ldebug_save_dbreak: 888 .set _index, 0 889 .rept XCHAL_NUM_DBREAK 890 movi a0, 0 891 xsr a0, SREG_DBREAKC + _index 892 s32i a0, a3, DT_DBREAKC_SAVE + _index * 4 893 .set _index, _index + 1 894 .endr 895 896 movi a0, XCHAL_EXCM_LEVEL + 1 897 xsr a0, icountlevel 898 s32i a0, a3, DT_ICOUNT_LEVEL_SAVE 899 900 movi a0, 0xfffffffe 901 xsr a0, icount 902 s32i a0, a3, DT_ICOUNT_SAVE 903 904 l32i a0, a3, DT_DEBUG_SAVE 905 xsr a3, SREG_EXCSAVE + XCHAL_DEBUGLEVEL 906 rfi XCHAL_DEBUGLEVEL 907#else 908 /* Debug exception while in exception mode. Should not happen. */ 9091: j 1b // FIXME!! 910#endif 911 912ENDPROC(debug_exception) 913 914/* 915 * We get here in case of an unrecoverable exception. 916 * The only thing we can do is to be nice and print a panic message. 917 * We only produce a single stack frame for panic, so ??? 918 * 919 * 920 * Entry conditions: 921 * 922 * - a0 contains the caller address; original value saved in excsave1. 923 * - the original a0 contains a valid return address (backtrace) or 0. 924 * - a2 contains a valid stackpointer 925 * 926 * Notes: 927 * 928 * - If the stack pointer could be invalid, the caller has to setup a 929 * dummy stack pointer (e.g. the stack of the init_task) 930 * 931 * - If the return address could be invalid, the caller has to set it 932 * to 0, so the backtrace would stop. 933 * 934 */ 935 .align 4 936unrecoverable_text: 937 .ascii "Unrecoverable error in exception handler\0" 938 939 .literal_position 940 941ENTRY(unrecoverable_exception) 942 943#if XCHAL_HAVE_WINDOWED 944 movi a0, 1 945 movi a1, 0 946 947 wsr a0, windowstart 948 wsr a1, windowbase 949 rsync 950#endif 951 952 movi a1, KERNEL_PS_WOE_MASK | LOCKLEVEL 953 wsr a1, ps 954 rsync 955 956 movi a1, init_task 957 movi a0, 0 958 addi a1, a1, PT_REGS_OFFSET 959 960 movi abi_arg0, unrecoverable_text 961 abi_call panic 962 9631: j 1b 964 965ENDPROC(unrecoverable_exception) 966 967/* -------------------------- FAST EXCEPTION HANDLERS ----------------------- */ 968 969 __XTENSA_HANDLER 970 .literal_position 971 972/* 973 * Fast-handler for alloca exceptions 974 * 975 * The ALLOCA handler is entered when user code executes the MOVSP 976 * instruction and the caller's frame is not in the register file. 977 * 978 * This algorithm was taken from the Ross Morley's RTOS Porting Layer: 979 * 980 * /home/ross/rtos/porting/XtensaRTOS-PortingLayer-20090507/xtensa_vectors.S 981 * 982 * It leverages the existing window spill/fill routines and their support for 983 * double exceptions. The 'movsp' instruction will only cause an exception if 984 * the next window needs to be loaded. In fact this ALLOCA exception may be 985 * replaced at some point by changing the hardware to do a underflow exception 986 * of the proper size instead. 987 * 988 * This algorithm simply backs out the register changes started by the user 989 * exception handler, makes it appear that we have started a window underflow 990 * by rotating the window back and then setting the old window base (OWB) in 991 * the 'ps' register with the rolled back window base. The 'movsp' instruction 992 * will be re-executed and this time since the next window frames is in the 993 * active AR registers it won't cause an exception. 994 * 995 * If the WindowUnderflow code gets a TLB miss the page will get mapped 996 * the partial WindowUnderflow will be handled in the double exception 997 * handler. 998 * 999 * Entry condition: 1000 * 1001 * a0: trashed, original value saved on stack (PT_AREG0) 1002 * a1: a1 1003 * a2: new stack pointer, original in DEPC 1004 * a3: a3 1005 * depc: a2, original value saved on stack (PT_DEPC) 1006 * excsave_1: dispatch table 1007 * 1008 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 1009 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 1010 */ 1011 1012ENTRY(fast_alloca) 1013 rsr a0, windowbase 1014 rotw -1 1015 rsr a2, ps 1016 extui a3, a2, PS_OWB_SHIFT, PS_OWB_WIDTH 1017 xor a3, a3, a4 1018 l32i a4, a6, PT_AREG0 1019 l32i a1, a6, PT_DEPC 1020 rsr a6, depc 1021 wsr a1, depc 1022 slli a3, a3, PS_OWB_SHIFT 1023 xor a2, a2, a3 1024 wsr a2, ps 1025 rsync 1026 1027 _bbci.l a4, 31, 4f 1028 rotw -1 1029 _bbci.l a8, 30, 8f 1030 rotw -1 1031 j _WindowUnderflow12 10328: j _WindowUnderflow8 10334: j _WindowUnderflow4 1034ENDPROC(fast_alloca) 1035 1036#ifdef CONFIG_USER_ABI_CALL0_PROBE 1037/* 1038 * fast illegal instruction handler. 1039 * 1040 * This is used to fix up user PS.WOE on the exception caused 1041 * by the first opcode related to register window. If PS.WOE is 1042 * already set it goes directly to the common user exception handler. 1043 * 1044 * Entry condition: 1045 * 1046 * a0: trashed, original value saved on stack (PT_AREG0) 1047 * a1: a1 1048 * a2: new stack pointer, original in DEPC 1049 * a3: a3 1050 * depc: a2, original value saved on stack (PT_DEPC) 1051 * excsave_1: dispatch table 1052 */ 1053 1054ENTRY(fast_illegal_instruction_user) 1055 1056 rsr a0, ps 1057 bbsi.l a0, PS_WOE_BIT, 1f 1058 s32i a3, a2, PT_AREG3 1059 movi a3, PS_WOE_MASK 1060 or a0, a0, a3 1061 wsr a0, ps 1062 l32i a3, a2, PT_AREG3 1063 l32i a0, a2, PT_AREG0 1064 rsr a2, depc 1065 rfe 10661: 1067 call0 user_exception 1068 1069ENDPROC(fast_illegal_instruction_user) 1070#endif 1071 1072 /* 1073 * fast system calls. 1074 * 1075 * WARNING: The kernel doesn't save the entire user context before 1076 * handling a fast system call. These functions are small and short, 1077 * usually offering some functionality not available to user tasks. 1078 * 1079 * BE CAREFUL TO PRESERVE THE USER'S CONTEXT. 1080 * 1081 * Entry condition: 1082 * 1083 * a0: trashed, original value saved on stack (PT_AREG0) 1084 * a1: a1 1085 * a2: new stack pointer, original in DEPC 1086 * a3: a3 1087 * depc: a2, original value saved on stack (PT_DEPC) 1088 * excsave_1: dispatch table 1089 */ 1090 1091ENTRY(fast_syscall_user) 1092 1093 /* Skip syscall. */ 1094 1095 rsr a0, epc1 1096 addi a0, a0, 3 1097 wsr a0, epc1 1098 1099 l32i a0, a2, PT_DEPC 1100 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable 1101 1102 rsr a0, depc # get syscall-nr 1103 _beqz a0, fast_syscall_spill_registers 1104 _beqi a0, __NR_xtensa, fast_syscall_xtensa 1105 1106 call0 user_exception 1107 1108ENDPROC(fast_syscall_user) 1109 1110ENTRY(fast_syscall_unrecoverable) 1111 1112 /* Restore all states. */ 1113 1114 l32i a0, a2, PT_AREG0 # restore a0 1115 xsr a2, depc # restore a2, depc 1116 1117 wsr a0, excsave1 1118 call0 unrecoverable_exception 1119 1120ENDPROC(fast_syscall_unrecoverable) 1121 1122/* 1123 * sysxtensa syscall handler 1124 * 1125 * int sysxtensa (SYS_XTENSA_ATOMIC_SET, ptr, val, unused); 1126 * int sysxtensa (SYS_XTENSA_ATOMIC_ADD, ptr, val, unused); 1127 * int sysxtensa (SYS_XTENSA_ATOMIC_EXG_ADD, ptr, val, unused); 1128 * int sysxtensa (SYS_XTENSA_ATOMIC_CMP_SWP, ptr, oldval, newval); 1129 * a2 a6 a3 a4 a5 1130 * 1131 * Entry condition: 1132 * 1133 * a0: a2 (syscall-nr), original value saved on stack (PT_AREG0) 1134 * a1: a1 1135 * a2: new stack pointer, original in a0 and DEPC 1136 * a3: a3 1137 * a4..a15: unchanged 1138 * depc: a2, original value saved on stack (PT_DEPC) 1139 * excsave_1: dispatch table 1140 * 1141 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 1142 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 1143 * 1144 * Note: we don't have to save a2; a2 holds the return value 1145 */ 1146 1147 .literal_position 1148 1149#ifdef CONFIG_FAST_SYSCALL_XTENSA 1150 1151ENTRY(fast_syscall_xtensa) 1152 1153 s32i a7, a2, PT_AREG7 # we need an additional register 1154 movi a7, 4 # sizeof(unsigned int) 1155 access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp 1156 1157 _bgeui a6, SYS_XTENSA_COUNT, .Lill 1158 _bnei a6, SYS_XTENSA_ATOMIC_CMP_SWP, .Lnswp 1159 1160 /* Fall through for ATOMIC_CMP_SWP. */ 1161 1162.Lswp: /* Atomic compare and swap */ 1163 1164EX(.Leac) l32i a0, a3, 0 # read old value 1165 bne a0, a4, 1f # same as old value? jump 1166EX(.Leac) s32i a5, a3, 0 # different, modify value 1167 l32i a7, a2, PT_AREG7 # restore a7 1168 l32i a0, a2, PT_AREG0 # restore a0 1169 movi a2, 1 # and return 1 1170 rfe 1171 11721: l32i a7, a2, PT_AREG7 # restore a7 1173 l32i a0, a2, PT_AREG0 # restore a0 1174 movi a2, 0 # return 0 (note that we cannot set 1175 rfe 1176 1177.Lnswp: /* Atomic set, add, and exg_add. */ 1178 1179EX(.Leac) l32i a7, a3, 0 # orig 1180 addi a6, a6, -SYS_XTENSA_ATOMIC_SET 1181 add a0, a4, a7 # + arg 1182 moveqz a0, a4, a6 # set 1183 addi a6, a6, SYS_XTENSA_ATOMIC_SET 1184EX(.Leac) s32i a0, a3, 0 # write new value 1185 1186 mov a0, a2 1187 mov a2, a7 1188 l32i a7, a0, PT_AREG7 # restore a7 1189 l32i a0, a0, PT_AREG0 # restore a0 1190 rfe 1191 1192.Leac: l32i a7, a2, PT_AREG7 # restore a7 1193 l32i a0, a2, PT_AREG0 # restore a0 1194 movi a2, -EFAULT 1195 rfe 1196 1197.Lill: l32i a7, a2, PT_AREG7 # restore a7 1198 l32i a0, a2, PT_AREG0 # restore a0 1199 movi a2, -EINVAL 1200 rfe 1201 1202ENDPROC(fast_syscall_xtensa) 1203 1204#else /* CONFIG_FAST_SYSCALL_XTENSA */ 1205 1206ENTRY(fast_syscall_xtensa) 1207 1208 l32i a0, a2, PT_AREG0 # restore a0 1209 movi a2, -ENOSYS 1210 rfe 1211 1212ENDPROC(fast_syscall_xtensa) 1213 1214#endif /* CONFIG_FAST_SYSCALL_XTENSA */ 1215 1216 1217/* fast_syscall_spill_registers. 1218 * 1219 * Entry condition: 1220 * 1221 * a0: trashed, original value saved on stack (PT_AREG0) 1222 * a1: a1 1223 * a2: new stack pointer, original in DEPC 1224 * a3: a3 1225 * depc: a2, original value saved on stack (PT_DEPC) 1226 * excsave_1: dispatch table 1227 * 1228 * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler. 1229 */ 1230 1231#ifdef CONFIG_FAST_SYSCALL_SPILL_REGISTERS 1232 1233ENTRY(fast_syscall_spill_registers) 1234 1235 /* Register a FIXUP handler (pass current wb as a parameter) */ 1236 1237 xsr a3, excsave1 1238 movi a0, fast_syscall_spill_registers_fixup 1239 s32i a0, a3, EXC_TABLE_FIXUP 1240 rsr a0, windowbase 1241 s32i a0, a3, EXC_TABLE_PARAM 1242 xsr a3, excsave1 # restore a3 and excsave_1 1243 1244 /* Save a3, a4 and SAR on stack. */ 1245 1246 rsr a0, sar 1247 s32i a3, a2, PT_AREG3 1248 s32i a0, a2, PT_SAR 1249 1250 /* The spill routine might clobber a4, a7, a8, a11, a12, and a15. */ 1251 1252 s32i a4, a2, PT_AREG4 1253 s32i a7, a2, PT_AREG7 1254 s32i a8, a2, PT_AREG8 1255 s32i a11, a2, PT_AREG11 1256 s32i a12, a2, PT_AREG12 1257 s32i a15, a2, PT_AREG15 1258 1259 /* 1260 * Rotate ws so that the current windowbase is at bit 0. 1261 * Assume ws = xxxwww1yy (www1 current window frame). 1262 * Rotate ws right so that a4 = yyxxxwww1. 1263 */ 1264 1265 rsr a0, windowbase 1266 rsr a3, windowstart # a3 = xxxwww1yy 1267 ssr a0 # holds WB 1268 slli a0, a3, WSBITS 1269 or a3, a3, a0 # a3 = xxxwww1yyxxxwww1yy 1270 srl a3, a3 # a3 = 00xxxwww1yyxxxwww1 1271 1272 /* We are done if there are no more than the current register frame. */ 1273 1274 extui a3, a3, 1, WSBITS-1 # a3 = 0yyxxxwww 1275 movi a0, (1 << (WSBITS-1)) 1276 _beqz a3, .Lnospill # only one active frame? jump 1277 1278 /* We want 1 at the top, so that we return to the current windowbase */ 1279 1280 or a3, a3, a0 # 1yyxxxwww 1281 1282 /* Skip empty frames - get 'oldest' WINDOWSTART-bit. */ 1283 1284 wsr a3, windowstart # save shifted windowstart 1285 neg a0, a3 1286 and a3, a0, a3 # first bit set from right: 000010000 1287 1288 ffs_ws a0, a3 # a0: shifts to skip empty frames 1289 movi a3, WSBITS 1290 sub a0, a3, a0 # WSBITS-a0:number of 0-bits from right 1291 ssr a0 # save in SAR for later. 1292 1293 rsr a3, windowbase 1294 add a3, a3, a0 1295 wsr a3, windowbase 1296 rsync 1297 1298 rsr a3, windowstart 1299 srl a3, a3 # shift windowstart 1300 1301 /* WB is now just one frame below the oldest frame in the register 1302 window. WS is shifted so the oldest frame is in bit 0, thus, WB 1303 and WS differ by one 4-register frame. */ 1304 1305 /* Save frames. Depending what call was used (call4, call8, call12), 1306 * we have to save 4,8. or 12 registers. 1307 */ 1308 1309 1310.Lloop: _bbsi.l a3, 1, .Lc4 1311 _bbci.l a3, 2, .Lc12 1312 1313.Lc8: s32e a4, a13, -16 1314 l32e a4, a5, -12 1315 s32e a8, a4, -32 1316 s32e a5, a13, -12 1317 s32e a6, a13, -8 1318 s32e a7, a13, -4 1319 s32e a9, a4, -28 1320 s32e a10, a4, -24 1321 s32e a11, a4, -20 1322 srli a11, a3, 2 # shift windowbase by 2 1323 rotw 2 1324 _bnei a3, 1, .Lloop 1325 j .Lexit 1326 1327.Lc4: s32e a4, a9, -16 1328 s32e a5, a9, -12 1329 s32e a6, a9, -8 1330 s32e a7, a9, -4 1331 1332 srli a7, a3, 1 1333 rotw 1 1334 _bnei a3, 1, .Lloop 1335 j .Lexit 1336 1337.Lc12: _bbci.l a3, 3, .Linvalid_mask # bit 2 shouldn't be zero! 1338 1339 /* 12-register frame (call12) */ 1340 1341 l32e a0, a5, -12 1342 s32e a8, a0, -48 1343 mov a8, a0 1344 1345 s32e a9, a8, -44 1346 s32e a10, a8, -40 1347 s32e a11, a8, -36 1348 s32e a12, a8, -32 1349 s32e a13, a8, -28 1350 s32e a14, a8, -24 1351 s32e a15, a8, -20 1352 srli a15, a3, 3 1353 1354 /* The stack pointer for a4..a7 is out of reach, so we rotate the 1355 * window, grab the stackpointer, and rotate back. 1356 * Alternatively, we could also use the following approach, but that 1357 * makes the fixup routine much more complicated: 1358 * rotw 1 1359 * s32e a0, a13, -16 1360 * ... 1361 * rotw 2 1362 */ 1363 1364 rotw 1 1365 mov a4, a13 1366 rotw -1 1367 1368 s32e a4, a8, -16 1369 s32e a5, a8, -12 1370 s32e a6, a8, -8 1371 s32e a7, a8, -4 1372 1373 rotw 3 1374 1375 _beqi a3, 1, .Lexit 1376 j .Lloop 1377 1378.Lexit: 1379 1380 /* Done. Do the final rotation and set WS */ 1381 1382 rotw 1 1383 rsr a3, windowbase 1384 ssl a3 1385 movi a3, 1 1386 sll a3, a3 1387 wsr a3, windowstart 1388.Lnospill: 1389 1390 /* Advance PC, restore registers and SAR, and return from exception. */ 1391 1392 l32i a3, a2, PT_SAR 1393 l32i a0, a2, PT_AREG0 1394 wsr a3, sar 1395 l32i a3, a2, PT_AREG3 1396 1397 /* Restore clobbered registers. */ 1398 1399 l32i a4, a2, PT_AREG4 1400 l32i a7, a2, PT_AREG7 1401 l32i a8, a2, PT_AREG8 1402 l32i a11, a2, PT_AREG11 1403 l32i a12, a2, PT_AREG12 1404 l32i a15, a2, PT_AREG15 1405 1406 movi a2, 0 1407 rfe 1408 1409.Linvalid_mask: 1410 1411 /* We get here because of an unrecoverable error in the window 1412 * registers, so set up a dummy frame and kill the user application. 1413 * Note: We assume EXC_TABLE_KSTK contains a valid stack pointer. 1414 */ 1415 1416 movi a0, 1 1417 movi a1, 0 1418 1419 wsr a0, windowstart 1420 wsr a1, windowbase 1421 rsync 1422 1423 movi a0, 0 1424 1425 rsr a3, excsave1 1426 l32i a1, a3, EXC_TABLE_KSTK 1427 1428 movi a4, KERNEL_PS_WOE_MASK | LOCKLEVEL 1429 wsr a4, ps 1430 rsync 1431 1432 movi abi_arg0, SIGSEGV 1433 abi_call do_exit 1434 1435 /* shouldn't return, so panic */ 1436 1437 wsr a0, excsave1 1438 call0 unrecoverable_exception # should not return 14391: j 1b 1440 1441 1442ENDPROC(fast_syscall_spill_registers) 1443 1444/* Fixup handler. 1445 * 1446 * We get here if the spill routine causes an exception, e.g. tlb miss. 1447 * We basically restore WINDOWBASE and WINDOWSTART to the condition when 1448 * we entered the spill routine and jump to the user exception handler. 1449 * 1450 * Note that we only need to restore the bits in windowstart that have not 1451 * been spilled yet by the _spill_register routine. Luckily, a3 contains a 1452 * rotated windowstart with only those bits set for frames that haven't been 1453 * spilled yet. Because a3 is rotated such that bit 0 represents the register 1454 * frame for the current windowbase - 1, we need to rotate a3 left by the 1455 * value of the current windowbase + 1 and move it to windowstart. 1456 * 1457 * a0: value of depc, original value in depc 1458 * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE 1459 * a3: exctable, original value in excsave1 1460 */ 1461 1462ENTRY(fast_syscall_spill_registers_fixup) 1463 1464 rsr a2, windowbase # get current windowbase (a2 is saved) 1465 xsr a0, depc # restore depc and a0 1466 ssl a2 # set shift (32 - WB) 1467 1468 /* We need to make sure the current registers (a0-a3) are preserved. 1469 * To do this, we simply set the bit for the current window frame 1470 * in WS, so that the exception handlers save them to the task stack. 1471 * 1472 * Note: we use a3 to set the windowbase, so we take a special care 1473 * of it, saving it in the original _spill_registers frame across 1474 * the exception handler call. 1475 */ 1476 1477 xsr a3, excsave1 # get spill-mask 1478 slli a3, a3, 1 # shift left by one 1479 addi a3, a3, 1 # set the bit for the current window frame 1480 1481 slli a2, a3, 32-WSBITS 1482 src a2, a3, a2 # a2 = xxwww1yyxxxwww1yy...... 1483 wsr a2, windowstart # set corrected windowstart 1484 1485 srli a3, a3, 1 1486 rsr a2, excsave1 1487 l32i a2, a2, EXC_TABLE_DOUBLE_SAVE # restore a2 1488 xsr a2, excsave1 1489 s32i a3, a2, EXC_TABLE_DOUBLE_SAVE # save a3 1490 l32i a3, a2, EXC_TABLE_PARAM # original WB (in user task) 1491 xsr a2, excsave1 1492 1493 /* Return to the original (user task) WINDOWBASE. 1494 * We leave the following frame behind: 1495 * a0, a1, a2 same 1496 * a3: trashed (saved in EXC_TABLE_DOUBLE_SAVE) 1497 * depc: depc (we have to return to that address) 1498 * excsave_1: exctable 1499 */ 1500 1501 wsr a3, windowbase 1502 rsync 1503 1504 /* We are now in the original frame when we entered _spill_registers: 1505 * a0: return address 1506 * a1: used, stack pointer 1507 * a2: kernel stack pointer 1508 * a3: available 1509 * depc: exception address 1510 * excsave: exctable 1511 * Note: This frame might be the same as above. 1512 */ 1513 1514 /* Setup stack pointer. */ 1515 1516 addi a2, a2, -PT_USER_SIZE 1517 s32i a0, a2, PT_AREG0 1518 1519 /* Make sure we return to this fixup handler. */ 1520 1521 movi a3, fast_syscall_spill_registers_fixup_return 1522 s32i a3, a2, PT_DEPC # setup depc 1523 1524 /* Jump to the exception handler. */ 1525 1526 rsr a3, excsave1 1527 rsr a0, exccause 1528 addx4 a0, a0, a3 # find entry in table 1529 l32i a0, a0, EXC_TABLE_FAST_USER # load handler 1530 l32i a3, a3, EXC_TABLE_DOUBLE_SAVE 1531 jx a0 1532 1533ENDPROC(fast_syscall_spill_registers_fixup) 1534 1535ENTRY(fast_syscall_spill_registers_fixup_return) 1536 1537 /* When we return here, all registers have been restored (a2: DEPC) */ 1538 1539 wsr a2, depc # exception address 1540 1541 /* Restore fixup handler. */ 1542 1543 rsr a2, excsave1 1544 s32i a3, a2, EXC_TABLE_DOUBLE_SAVE 1545 movi a3, fast_syscall_spill_registers_fixup 1546 s32i a3, a2, EXC_TABLE_FIXUP 1547 rsr a3, windowbase 1548 s32i a3, a2, EXC_TABLE_PARAM 1549 l32i a2, a2, EXC_TABLE_KSTK 1550 1551 /* Load WB at the time the exception occurred. */ 1552 1553 rsr a3, sar # WB is still in SAR 1554 neg a3, a3 1555 wsr a3, windowbase 1556 rsync 1557 1558 rsr a3, excsave1 1559 l32i a3, a3, EXC_TABLE_DOUBLE_SAVE 1560 1561 rfde 1562 1563ENDPROC(fast_syscall_spill_registers_fixup_return) 1564 1565#else /* CONFIG_FAST_SYSCALL_SPILL_REGISTERS */ 1566 1567ENTRY(fast_syscall_spill_registers) 1568 1569 l32i a0, a2, PT_AREG0 # restore a0 1570 movi a2, -ENOSYS 1571 rfe 1572 1573ENDPROC(fast_syscall_spill_registers) 1574 1575#endif /* CONFIG_FAST_SYSCALL_SPILL_REGISTERS */ 1576 1577#ifdef CONFIG_MMU 1578/* 1579 * We should never get here. Bail out! 1580 */ 1581 1582ENTRY(fast_second_level_miss_double_kernel) 1583 15841: 1585 call0 unrecoverable_exception # should not return 15861: j 1b 1587 1588ENDPROC(fast_second_level_miss_double_kernel) 1589 1590/* First-level entry handler for user, kernel, and double 2nd-level 1591 * TLB miss exceptions. Note that for now, user and kernel miss 1592 * exceptions share the same entry point and are handled identically. 1593 * 1594 * An old, less-efficient C version of this function used to exist. 1595 * We include it below, interleaved as comments, for reference. 1596 * 1597 * Entry condition: 1598 * 1599 * a0: trashed, original value saved on stack (PT_AREG0) 1600 * a1: a1 1601 * a2: new stack pointer, original in DEPC 1602 * a3: a3 1603 * depc: a2, original value saved on stack (PT_DEPC) 1604 * excsave_1: dispatch table 1605 * 1606 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 1607 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 1608 */ 1609 1610ENTRY(fast_second_level_miss) 1611 1612 /* Save a1 and a3. Note: we don't expect a double exception. */ 1613 1614 s32i a1, a2, PT_AREG1 1615 s32i a3, a2, PT_AREG3 1616 1617 /* We need to map the page of PTEs for the user task. Find 1618 * the pointer to that page. Also, it's possible for tsk->mm 1619 * to be NULL while tsk->active_mm is nonzero if we faulted on 1620 * a vmalloc address. In that rare case, we must use 1621 * active_mm instead to avoid a fault in this handler. See 1622 * 1623 * http://mail.nl.linux.org/linux-mm/2002-08/msg00258.html 1624 * (or search Internet on "mm vs. active_mm") 1625 * 1626 * if (!mm) 1627 * mm = tsk->active_mm; 1628 * pgd = pgd_offset (mm, regs->excvaddr); 1629 * pmd = pmd_offset (pgd, regs->excvaddr); 1630 * pmdval = *pmd; 1631 */ 1632 1633 GET_CURRENT(a1,a2) 1634 l32i a0, a1, TASK_MM # tsk->mm 1635 beqz a0, 9f 1636 16378: rsr a3, excvaddr # fault address 1638 _PGD_OFFSET(a0, a3, a1) 1639 l32i a0, a0, 0 # read pmdval 1640 beqz a0, 2f 1641 1642 /* Read ptevaddr and convert to top of page-table page. 1643 * 1644 * vpnval = read_ptevaddr_register() & PAGE_MASK; 1645 * vpnval += DTLB_WAY_PGTABLE; 1646 * pteval = mk_pte (virt_to_page(pmd_val(pmdval)), PAGE_KERNEL); 1647 * write_dtlb_entry (pteval, vpnval); 1648 * 1649 * The messy computation for 'pteval' above really simplifies 1650 * into the following: 1651 * 1652 * pteval = ((pmdval - PAGE_OFFSET + PHYS_OFFSET) & PAGE_MASK) 1653 * | PAGE_DIRECTORY 1654 */ 1655 1656 movi a1, (PHYS_OFFSET - PAGE_OFFSET) & 0xffffffff 1657 add a0, a0, a1 # pmdval - PAGE_OFFSET 1658 extui a1, a0, 0, PAGE_SHIFT # ... & PAGE_MASK 1659 xor a0, a0, a1 1660 1661 movi a1, _PAGE_DIRECTORY 1662 or a0, a0, a1 # ... | PAGE_DIRECTORY 1663 1664 /* 1665 * We utilize all three wired-ways (7-9) to hold pmd translations. 1666 * Memory regions are mapped to the DTLBs according to bits 28 and 29. 1667 * This allows to map the three most common regions to three different 1668 * DTLBs: 1669 * 0,1 -> way 7 program (0040.0000) and virtual (c000.0000) 1670 * 2 -> way 8 shared libaries (2000.0000) 1671 * 3 -> way 0 stack (3000.0000) 1672 */ 1673 1674 extui a3, a3, 28, 2 # addr. bit 28 and 29 0,1,2,3 1675 rsr a1, ptevaddr 1676 addx2 a3, a3, a3 # -> 0,3,6,9 1677 srli a1, a1, PAGE_SHIFT 1678 extui a3, a3, 2, 2 # -> 0,0,1,2 1679 slli a1, a1, PAGE_SHIFT # ptevaddr & PAGE_MASK 1680 addi a3, a3, DTLB_WAY_PGD 1681 add a1, a1, a3 # ... + way_number 1682 16833: wdtlb a0, a1 1684 dsync 1685 1686 /* Exit critical section. */ 1687 16884: rsr a3, excsave1 1689 movi a0, 0 1690 s32i a0, a3, EXC_TABLE_FIXUP 1691 1692 /* Restore the working registers, and return. */ 1693 1694 l32i a0, a2, PT_AREG0 1695 l32i a1, a2, PT_AREG1 1696 l32i a3, a2, PT_AREG3 1697 l32i a2, a2, PT_DEPC 1698 1699 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f 1700 1701 /* Restore excsave1 and return. */ 1702 1703 rsr a2, depc 1704 rfe 1705 1706 /* Return from double exception. */ 1707 17081: xsr a2, depc 1709 esync 1710 rfde 1711 17129: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0 1713 bnez a0, 8b 1714 1715 /* Even more unlikely case active_mm == 0. 1716 * We can get here with NMI in the middle of context_switch that 1717 * touches vmalloc area. 1718 */ 1719 movi a0, init_mm 1720 j 8b 1721 1722#if (DCACHE_WAY_SIZE > PAGE_SIZE) 1723 17242: /* Special case for cache aliasing. 1725 * We (should) only get here if a clear_user_page, copy_user_page 1726 * or the aliased cache flush functions got preemptively interrupted 1727 * by another task. Re-establish temporary mapping to the 1728 * TLBTEMP_BASE areas. 1729 */ 1730 1731 /* We shouldn't be in a double exception */ 1732 1733 l32i a0, a2, PT_DEPC 1734 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f 1735 1736 /* Make sure the exception originated in the special functions */ 1737 1738 movi a0, __tlbtemp_mapping_start 1739 rsr a3, epc1 1740 bltu a3, a0, 2f 1741 movi a0, __tlbtemp_mapping_end 1742 bgeu a3, a0, 2f 1743 1744 /* Check if excvaddr was in one of the TLBTEMP_BASE areas. */ 1745 1746 movi a3, TLBTEMP_BASE_1 1747 rsr a0, excvaddr 1748 bltu a0, a3, 2f 1749 1750 addi a1, a0, -TLBTEMP_SIZE 1751 bgeu a1, a3, 2f 1752 1753 /* Check if we have to restore an ITLB mapping. */ 1754 1755 movi a1, __tlbtemp_mapping_itlb 1756 rsr a3, epc1 1757 sub a3, a3, a1 1758 1759 /* Calculate VPN */ 1760 1761 movi a1, PAGE_MASK 1762 and a1, a1, a0 1763 1764 /* Jump for ITLB entry */ 1765 1766 bgez a3, 1f 1767 1768 /* We can use up to two TLBTEMP areas, one for src and one for dst. */ 1769 1770 extui a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1 1771 add a1, a3, a1 1772 1773 /* PPN is in a6 for the first TLBTEMP area and in a7 for the second. */ 1774 1775 mov a0, a6 1776 movnez a0, a7, a3 1777 j 3b 1778 1779 /* ITLB entry. We only use dst in a6. */ 1780 17811: witlb a6, a1 1782 isync 1783 j 4b 1784 1785 1786#endif // DCACHE_WAY_SIZE > PAGE_SIZE 1787 1788 17892: /* Invalid PGD, default exception handling */ 1790 1791 rsr a1, depc 1792 s32i a1, a2, PT_AREG2 1793 mov a1, a2 1794 1795 rsr a2, ps 1796 bbsi.l a2, PS_UM_BIT, 1f 1797 call0 _kernel_exception 17981: call0 _user_exception 1799 1800ENDPROC(fast_second_level_miss) 1801 1802/* 1803 * StoreProhibitedException 1804 * 1805 * Update the pte and invalidate the itlb mapping for this pte. 1806 * 1807 * Entry condition: 1808 * 1809 * a0: trashed, original value saved on stack (PT_AREG0) 1810 * a1: a1 1811 * a2: new stack pointer, original in DEPC 1812 * a3: a3 1813 * depc: a2, original value saved on stack (PT_DEPC) 1814 * excsave_1: dispatch table 1815 * 1816 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC 1817 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception 1818 */ 1819 1820ENTRY(fast_store_prohibited) 1821 1822 /* Save a1 and a3. */ 1823 1824 s32i a1, a2, PT_AREG1 1825 s32i a3, a2, PT_AREG3 1826 1827 GET_CURRENT(a1,a2) 1828 l32i a0, a1, TASK_MM # tsk->mm 1829 beqz a0, 9f 1830 18318: rsr a1, excvaddr # fault address 1832 _PGD_OFFSET(a0, a1, a3) 1833 l32i a0, a0, 0 1834 beqz a0, 2f 1835 1836 /* 1837 * Note that we test _PAGE_WRITABLE_BIT only if PTE is present 1838 * and is not PAGE_NONE. See pgtable.h for possible PTE layouts. 1839 */ 1840 1841 _PTE_OFFSET(a0, a1, a3) 1842 l32i a3, a0, 0 # read pteval 1843 movi a1, _PAGE_CA_INVALID 1844 ball a3, a1, 2f 1845 bbci.l a3, _PAGE_WRITABLE_BIT, 2f 1846 1847 movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE 1848 or a3, a3, a1 1849 rsr a1, excvaddr 1850 s32i a3, a0, 0 1851 1852 /* We need to flush the cache if we have page coloring. */ 1853#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK 1854 dhwb a0, 0 1855#endif 1856 pdtlb a0, a1 1857 wdtlb a3, a0 1858 1859 /* Exit critical section. */ 1860 1861 movi a0, 0 1862 rsr a3, excsave1 1863 s32i a0, a3, EXC_TABLE_FIXUP 1864 1865 /* Restore the working registers, and return. */ 1866 1867 l32i a3, a2, PT_AREG3 1868 l32i a1, a2, PT_AREG1 1869 l32i a0, a2, PT_AREG0 1870 l32i a2, a2, PT_DEPC 1871 1872 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f 1873 1874 rsr a2, depc 1875 rfe 1876 1877 /* Double exception. Restore FIXUP handler and return. */ 1878 18791: xsr a2, depc 1880 esync 1881 rfde 1882 18839: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0 1884 j 8b 1885 18862: /* If there was a problem, handle fault in C */ 1887 1888 rsr a3, depc # still holds a2 1889 s32i a3, a2, PT_AREG2 1890 mov a1, a2 1891 1892 rsr a2, ps 1893 bbsi.l a2, PS_UM_BIT, 1f 1894 call0 _kernel_exception 18951: call0 _user_exception 1896 1897ENDPROC(fast_store_prohibited) 1898 1899#endif /* CONFIG_MMU */ 1900 1901 .text 1902/* 1903 * System Calls. 1904 * 1905 * void system_call (struct pt_regs* regs, int exccause) 1906 * a2 a3 1907 */ 1908 .literal_position 1909 1910ENTRY(system_call) 1911 1912#if defined(__XTENSA_WINDOWED_ABI__) 1913 abi_entry_default 1914#elif defined(__XTENSA_CALL0_ABI__) 1915 abi_entry(12) 1916 1917 s32i a0, sp, 0 1918 s32i abi_saved0, sp, 4 1919 s32i abi_saved1, sp, 8 1920 mov abi_saved0, a2 1921#else 1922#error Unsupported Xtensa ABI 1923#endif 1924 1925 /* regs->syscall = regs->areg[2] */ 1926 1927 l32i a7, abi_saved0, PT_AREG2 1928 s32i a7, abi_saved0, PT_SYSCALL 1929 1930 GET_THREAD_INFO(a4, a1) 1931 l32i abi_saved1, a4, TI_FLAGS 1932 movi a4, _TIF_WORK_MASK 1933 and abi_saved1, abi_saved1, a4 1934 beqz abi_saved1, 1f 1935 1936 mov abi_arg0, abi_saved0 1937 abi_call do_syscall_trace_enter 1938 beqz abi_rv, .Lsyscall_exit 1939 l32i a7, abi_saved0, PT_SYSCALL 1940 19411: 1942 /* syscall = sys_call_table[syscall_nr] */ 1943 1944 movi a4, sys_call_table 1945 movi a5, __NR_syscalls 1946 movi abi_rv, -ENOSYS 1947 bgeu a7, a5, 1f 1948 1949 addx4 a4, a7, a4 1950 l32i abi_tmp0, a4, 0 1951 1952 /* Load args: arg0 - arg5 are passed via regs. */ 1953 1954 l32i abi_arg0, abi_saved0, PT_AREG6 1955 l32i abi_arg1, abi_saved0, PT_AREG3 1956 l32i abi_arg2, abi_saved0, PT_AREG4 1957 l32i abi_arg3, abi_saved0, PT_AREG5 1958 l32i abi_arg4, abi_saved0, PT_AREG8 1959 l32i abi_arg5, abi_saved0, PT_AREG9 1960 1961 abi_callx abi_tmp0 1962 19631: /* regs->areg[2] = return_value */ 1964 1965 s32i abi_rv, abi_saved0, PT_AREG2 1966 bnez abi_saved1, 1f 1967.Lsyscall_exit: 1968#if defined(__XTENSA_WINDOWED_ABI__) 1969 abi_ret_default 1970#elif defined(__XTENSA_CALL0_ABI__) 1971 l32i a0, sp, 0 1972 l32i abi_saved0, sp, 4 1973 l32i abi_saved1, sp, 8 1974 abi_ret(12) 1975#else 1976#error Unsupported Xtensa ABI 1977#endif 1978 19791: 1980 mov abi_arg0, abi_saved0 1981 abi_call do_syscall_trace_leave 1982 j .Lsyscall_exit 1983 1984ENDPROC(system_call) 1985 1986/* 1987 * Spill live registers on the kernel stack macro. 1988 * 1989 * Entry condition: ps.woe is set, ps.excm is cleared 1990 * Exit condition: windowstart has single bit set 1991 * May clobber: a12, a13 1992 */ 1993 .macro spill_registers_kernel 1994 1995#if XCHAL_NUM_AREGS > 16 1996 call12 1f 1997 _j 2f 1998 retw 1999 .align 4 20001: 2001 _entry a1, 48 2002 addi a12, a0, 3 2003#if XCHAL_NUM_AREGS > 32 2004 .rept (XCHAL_NUM_AREGS - 32) / 12 2005 _entry a1, 48 2006 mov a12, a0 2007 .endr 2008#endif 2009 _entry a1, 16 2010#if XCHAL_NUM_AREGS % 12 == 0 2011 mov a8, a8 2012#elif XCHAL_NUM_AREGS % 12 == 4 2013 mov a12, a12 2014#elif XCHAL_NUM_AREGS % 12 == 8 2015 mov a4, a4 2016#endif 2017 retw 20182: 2019#else 2020 mov a12, a12 2021#endif 2022 .endm 2023 2024/* 2025 * Task switch. 2026 * 2027 * struct task* _switch_to (struct task* prev, struct task* next) 2028 * a2 a2 a3 2029 */ 2030 2031ENTRY(_switch_to) 2032 2033#if defined(__XTENSA_WINDOWED_ABI__) 2034 abi_entry(XTENSA_SPILL_STACK_RESERVE) 2035#elif defined(__XTENSA_CALL0_ABI__) 2036 abi_entry(16) 2037 2038 s32i a12, sp, 0 2039 s32i a13, sp, 4 2040 s32i a14, sp, 8 2041 s32i a15, sp, 12 2042#else 2043#error Unsupported Xtensa ABI 2044#endif 2045 mov a11, a3 # and 'next' (a3) 2046 2047 l32i a4, a2, TASK_THREAD_INFO 2048 l32i a5, a3, TASK_THREAD_INFO 2049 2050 save_xtregs_user a4 a6 a8 a9 a12 a13 THREAD_XTREGS_USER 2051 2052#if THREAD_RA > 1020 || THREAD_SP > 1020 2053 addi a10, a2, TASK_THREAD 2054 s32i a0, a10, THREAD_RA - TASK_THREAD # save return address 2055 s32i a1, a10, THREAD_SP - TASK_THREAD # save stack pointer 2056#else 2057 s32i a0, a2, THREAD_RA # save return address 2058 s32i a1, a2, THREAD_SP # save stack pointer 2059#endif 2060 2061#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) 2062 movi a6, __stack_chk_guard 2063 l32i a8, a3, TASK_STACK_CANARY 2064 s32i a8, a6, 0 2065#endif 2066 2067 /* Disable ints while we manipulate the stack pointer. */ 2068 2069 irq_save a14, a3 2070 rsync 2071 2072 /* Switch CPENABLE */ 2073 2074#if (XTENSA_HAVE_COPROCESSORS || XTENSA_HAVE_IO_PORTS) 2075 l32i a3, a5, THREAD_CPENABLE 2076 xsr a3, cpenable 2077 s32i a3, a4, THREAD_CPENABLE 2078#endif 2079 2080#if XCHAL_HAVE_EXCLUSIVE 2081 l32i a3, a5, THREAD_ATOMCTL8 2082 getex a3 2083 s32i a3, a4, THREAD_ATOMCTL8 2084#endif 2085 2086 /* Flush register file. */ 2087 2088#if defined(__XTENSA_WINDOWED_ABI__) 2089 spill_registers_kernel 2090#endif 2091 2092 /* Set kernel stack (and leave critical section) 2093 * Note: It's save to set it here. The stack will not be overwritten 2094 * because the kernel stack will only be loaded again after 2095 * we return from kernel space. 2096 */ 2097 2098 rsr a3, excsave1 # exc_table 2099 addi a7, a5, PT_REGS_OFFSET 2100 s32i a7, a3, EXC_TABLE_KSTK 2101 2102 /* restore context of the task 'next' */ 2103 2104 l32i a0, a11, THREAD_RA # restore return address 2105 l32i a1, a11, THREAD_SP # restore stack pointer 2106 2107 load_xtregs_user a5 a6 a8 a9 a12 a13 THREAD_XTREGS_USER 2108 2109 wsr a14, ps 2110 rsync 2111 2112#if defined(__XTENSA_WINDOWED_ABI__) 2113 abi_ret(XTENSA_SPILL_STACK_RESERVE) 2114#elif defined(__XTENSA_CALL0_ABI__) 2115 l32i a12, sp, 0 2116 l32i a13, sp, 4 2117 l32i a14, sp, 8 2118 l32i a15, sp, 12 2119 abi_ret(16) 2120#else 2121#error Unsupported Xtensa ABI 2122#endif 2123 2124ENDPROC(_switch_to) 2125 2126ENTRY(ret_from_fork) 2127 2128 /* void schedule_tail (struct task_struct *prev) 2129 * Note: prev is still in abi_arg0 (return value from fake call frame) 2130 */ 2131 abi_call schedule_tail 2132 2133 mov abi_arg0, a1 2134 abi_call do_syscall_trace_leave 2135 j common_exception_return 2136 2137ENDPROC(ret_from_fork) 2138 2139/* 2140 * Kernel thread creation helper 2141 * On entry, set up by copy_thread: abi_saved0 = thread_fn, 2142 * abi_saved1 = thread_fn arg. Left from _switch_to: abi_arg0 = prev 2143 */ 2144ENTRY(ret_from_kernel_thread) 2145 2146 abi_call schedule_tail 2147 mov abi_arg0, abi_saved1 2148 abi_callx abi_saved0 2149 j common_exception_return 2150 2151ENDPROC(ret_from_kernel_thread) 2152