1/* 2 * PowerPC version 3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 4 * Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP 5 * Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com> 6 * Adapted for Power Macintosh by Paul Mackerras. 7 * Low-level exception handlers and MMU support 8 * rewritten by Paul Mackerras. 9 * Copyright (C) 1996 Paul Mackerras. 10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net). 11 * 12 * This file contains the system call entry code, context switch 13 * code, and exception/interrupt return code for PowerPC. 14 * 15 * This program is free software; you can redistribute it and/or 16 * modify it under the terms of the GNU General Public License 17 * as published by the Free Software Foundation; either version 18 * 2 of the License, or (at your option) any later version. 19 * 20 */ 21 22#include <linux/config.h> 23#include <linux/errno.h> 24#include <linux/sys.h> 25#include <linux/threads.h> 26#include <asm/reg.h> 27#include <asm/page.h> 28#include <asm/mmu.h> 29#include <asm/cputable.h> 30#include <asm/thread_info.h> 31#include <asm/ppc_asm.h> 32#include <asm/asm-offsets.h> 33#include <asm/unistd.h> 34 35#undef SHOW_SYSCALLS 36#undef SHOW_SYSCALLS_TASK 37 38/* 39 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE. 40 */ 41#if MSR_KERNEL >= 0x10000 42#define LOAD_MSR_KERNEL(r, x) lis r,(x)@h; ori r,r,(x)@l 43#else 44#define LOAD_MSR_KERNEL(r, x) li r,(x) 45#endif 46 47#ifdef CONFIG_BOOKE 48#include "head_booke.h" 49#define TRANSFER_TO_HANDLER_EXC_LEVEL(exc_level) \ 50 mtspr exc_level##_SPRG,r8; \ 51 BOOKE_LOAD_EXC_LEVEL_STACK(exc_level); \ 52 lwz r0,GPR10-INT_FRAME_SIZE(r8); \ 53 stw r0,GPR10(r11); \ 54 lwz r0,GPR11-INT_FRAME_SIZE(r8); \ 55 stw r0,GPR11(r11); \ 56 mfspr r8,exc_level##_SPRG 57 58 .globl mcheck_transfer_to_handler 59mcheck_transfer_to_handler: 60 TRANSFER_TO_HANDLER_EXC_LEVEL(MCHECK) 61 b transfer_to_handler_full 62 63 .globl debug_transfer_to_handler 64debug_transfer_to_handler: 65 TRANSFER_TO_HANDLER_EXC_LEVEL(DEBUG) 66 b transfer_to_handler_full 67 68 .globl crit_transfer_to_handler 69crit_transfer_to_handler: 70 TRANSFER_TO_HANDLER_EXC_LEVEL(CRIT) 71 /* fall through */ 72#endif 73 74#ifdef CONFIG_40x 75 .globl crit_transfer_to_handler 76crit_transfer_to_handler: 77 lwz r0,crit_r10@l(0) 78 stw r0,GPR10(r11) 79 lwz r0,crit_r11@l(0) 80 stw r0,GPR11(r11) 81 /* fall through */ 82#endif 83 84/* 85 * This code finishes saving the registers to the exception frame 86 * and jumps to the appropriate handler for the exception, turning 87 * on address translation. 88 * Note that we rely on the caller having set cr0.eq iff the exception 89 * occurred in kernel mode (i.e. MSR:PR = 0). 90 */ 91 .globl transfer_to_handler_full 92transfer_to_handler_full: 93 SAVE_NVGPRS(r11) 94 /* fall through */ 95 96 .globl transfer_to_handler 97transfer_to_handler: 98 stw r2,GPR2(r11) 99 stw r12,_NIP(r11) 100 stw r9,_MSR(r11) 101 andi. r2,r9,MSR_PR 102 mfctr r12 103 mfspr r2,SPRN_XER 104 stw r12,_CTR(r11) 105 stw r2,_XER(r11) 106 mfspr r12,SPRN_SPRG3 107 addi r2,r12,-THREAD 108 tovirt(r2,r2) /* set r2 to current */ 109 beq 2f /* if from user, fix up THREAD.regs */ 110 addi r11,r1,STACK_FRAME_OVERHEAD 111 stw r11,PT_REGS(r12) 112#if defined(CONFIG_40x) || defined(CONFIG_BOOKE) 113 /* Check to see if the dbcr0 register is set up to debug. Use the 114 single-step bit to do this. */ 115 lwz r12,THREAD_DBCR0(r12) 116 andis. r12,r12,DBCR0_IC@h 117 beq+ 3f 118 /* From user and task is ptraced - load up global dbcr0 */ 119 li r12,-1 /* clear all pending debug events */ 120 mtspr SPRN_DBSR,r12 121 lis r11,global_dbcr0@ha 122 tophys(r11,r11) 123 addi r11,r11,global_dbcr0@l 124 lwz r12,0(r11) 125 mtspr SPRN_DBCR0,r12 126 lwz r12,4(r11) 127 addi r12,r12,-1 128 stw r12,4(r11) 129#endif 130 b 3f 1312: /* if from kernel, check interrupted DOZE/NAP mode and 132 * check for stack overflow 133 */ 134#ifdef CONFIG_6xx 135 mfspr r11,SPRN_HID0 136 mtcr r11 137BEGIN_FTR_SECTION 138 bt- 8,4f /* Check DOZE */ 139END_FTR_SECTION_IFSET(CPU_FTR_CAN_DOZE) 140BEGIN_FTR_SECTION 141 bt- 9,4f /* Check NAP */ 142END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) 143#endif /* CONFIG_6xx */ 144 .globl transfer_to_handler_cont 145transfer_to_handler_cont: 146 lwz r11,THREAD_INFO-THREAD(r12) 147 cmplw r1,r11 /* if r1 <= current->thread_info */ 148 ble- stack_ovf /* then the kernel stack overflowed */ 1493: 150 mflr r9 151 lwz r11,0(r9) /* virtual address of handler */ 152 lwz r9,4(r9) /* where to go when done */ 153 FIX_SRR1(r10,r12) 154 mtspr SPRN_SRR0,r11 155 mtspr SPRN_SRR1,r10 156 mtlr r9 157 SYNC 158 RFI /* jump to handler, enable MMU */ 159 160#ifdef CONFIG_6xx 1614: b power_save_6xx_restore 162#endif 163 164/* 165 * On kernel stack overflow, load up an initial stack pointer 166 * and call StackOverflow(regs), which should not return. 167 */ 168stack_ovf: 169 /* sometimes we use a statically-allocated stack, which is OK. */ 170 lis r11,_end@h 171 ori r11,r11,_end@l 172 cmplw r1,r11 173 ble 3b /* r1 <= &_end is OK */ 174 SAVE_NVGPRS(r11) 175 addi r3,r1,STACK_FRAME_OVERHEAD 176 lis r1,init_thread_union@ha 177 addi r1,r1,init_thread_union@l 178 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD 179 lis r9,StackOverflow@ha 180 addi r9,r9,StackOverflow@l 181 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 182 FIX_SRR1(r10,r12) 183 mtspr SPRN_SRR0,r9 184 mtspr SPRN_SRR1,r10 185 SYNC 186 RFI 187 188/* 189 * Handle a system call. 190 */ 191 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f 192 .stabs "entry_32.S",N_SO,0,0,0f 1930: 194 195_GLOBAL(DoSyscall) 196 stw r0,THREAD+LAST_SYSCALL(r2) 197 stw r3,ORIG_GPR3(r1) 198 li r12,0 199 stw r12,RESULT(r1) 200 lwz r11,_CCR(r1) /* Clear SO bit in CR */ 201 rlwinm r11,r11,0,4,2 202 stw r11,_CCR(r1) 203#ifdef SHOW_SYSCALLS 204 bl do_show_syscall 205#endif /* SHOW_SYSCALLS */ 206 rlwinm r10,r1,0,0,(31-THREAD_SHIFT) /* current_thread_info() */ 207 lwz r11,TI_FLAGS(r10) 208 andi. r11,r11,_TIF_SYSCALL_T_OR_A 209 bne- syscall_dotrace 210syscall_dotrace_cont: 211 cmplwi 0,r0,NR_syscalls 212 lis r10,sys_call_table@h 213 ori r10,r10,sys_call_table@l 214 slwi r0,r0,2 215 bge- 66f 216 lwzx r10,r10,r0 /* Fetch system call handler [ptr] */ 217 mtlr r10 218 addi r9,r1,STACK_FRAME_OVERHEAD 219 PPC440EP_ERR42 220 blrl /* Call handler */ 221 .globl ret_from_syscall 222ret_from_syscall: 223#ifdef SHOW_SYSCALLS 224 bl do_show_syscall_exit 225#endif 226 mr r6,r3 227 rlwinm r12,r1,0,0,(31-THREAD_SHIFT) /* current_thread_info() */ 228 /* disable interrupts so current_thread_info()->flags can't change */ 229 LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */ 230 SYNC 231 MTMSRD(r10) 232 lwz r9,TI_FLAGS(r12) 233 li r8,-_LAST_ERRNO 234 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK) 235 bne- syscall_exit_work 236 cmplw 0,r3,r8 237 blt+ syscall_exit_cont 238 lwz r11,_CCR(r1) /* Load CR */ 239 neg r3,r3 240 oris r11,r11,0x1000 /* Set SO bit in CR */ 241 stw r11,_CCR(r1) 242syscall_exit_cont: 243#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 244 /* If the process has its own DBCR0 value, load it up. The single 245 step bit tells us that dbcr0 should be loaded. */ 246 lwz r0,THREAD+THREAD_DBCR0(r2) 247 andis. r10,r0,DBCR0_IC@h 248 bnel- load_dbcr0 249#endif 250 stwcx. r0,0,r1 /* to clear the reservation */ 251 lwz r4,_LINK(r1) 252 lwz r5,_CCR(r1) 253 mtlr r4 254 mtcr r5 255 lwz r7,_NIP(r1) 256 lwz r8,_MSR(r1) 257 FIX_SRR1(r8, r0) 258 lwz r2,GPR2(r1) 259 lwz r1,GPR1(r1) 260 mtspr SPRN_SRR0,r7 261 mtspr SPRN_SRR1,r8 262 SYNC 263 RFI 264 26566: li r3,-ENOSYS 266 b ret_from_syscall 267 268 .globl ret_from_fork 269ret_from_fork: 270 REST_NVGPRS(r1) 271 bl schedule_tail 272 li r3,0 273 b ret_from_syscall 274 275/* Traced system call support */ 276syscall_dotrace: 277 SAVE_NVGPRS(r1) 278 li r0,0xc00 279 stw r0,_TRAP(r1) 280 addi r3,r1,STACK_FRAME_OVERHEAD 281 bl do_syscall_trace_enter 282 lwz r0,GPR0(r1) /* Restore original registers */ 283 lwz r3,GPR3(r1) 284 lwz r4,GPR4(r1) 285 lwz r5,GPR5(r1) 286 lwz r6,GPR6(r1) 287 lwz r7,GPR7(r1) 288 lwz r8,GPR8(r1) 289 REST_NVGPRS(r1) 290 b syscall_dotrace_cont 291 292syscall_exit_work: 293 andi. r0,r9,_TIF_RESTOREALL 294 beq+ 0f 295 REST_NVGPRS(r1) 296 b 2f 2970: cmplw 0,r3,r8 298 blt+ 1f 299 andi. r0,r9,_TIF_NOERROR 300 bne- 1f 301 lwz r11,_CCR(r1) /* Load CR */ 302 neg r3,r3 303 oris r11,r11,0x1000 /* Set SO bit in CR */ 304 stw r11,_CCR(r1) 305 3061: stw r6,RESULT(r1) /* Save result */ 307 stw r3,GPR3(r1) /* Update return value */ 3082: andi. r0,r9,(_TIF_PERSYSCALL_MASK) 309 beq 4f 310 311 /* Clear per-syscall TIF flags if any are set. */ 312 313 li r11,_TIF_PERSYSCALL_MASK 314 addi r12,r12,TI_FLAGS 3153: lwarx r8,0,r12 316 andc r8,r8,r11 317#ifdef CONFIG_IBM405_ERR77 318 dcbt 0,r12 319#endif 320 stwcx. r8,0,r12 321 bne- 3b 322 subi r12,r12,TI_FLAGS 323 3244: /* Anything which requires enabling interrupts? */ 325 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP) 326 beq ret_from_except 327 328 /* Re-enable interrupts */ 329 ori r10,r10,MSR_EE 330 SYNC 331 MTMSRD(r10) 332 333 /* Save NVGPRS if they're not saved already */ 334 lwz r4,_TRAP(r1) 335 andi. r4,r4,1 336 beq 5f 337 SAVE_NVGPRS(r1) 338 li r4,0xc00 339 stw r4,_TRAP(r1) 3405: 341 addi r3,r1,STACK_FRAME_OVERHEAD 342 bl do_syscall_trace_leave 343 b ret_from_except_full 344 345#ifdef SHOW_SYSCALLS 346do_show_syscall: 347#ifdef SHOW_SYSCALLS_TASK 348 lis r11,show_syscalls_task@ha 349 lwz r11,show_syscalls_task@l(r11) 350 cmp 0,r2,r11 351 bnelr 352#endif 353 stw r31,GPR31(r1) 354 mflr r31 355 lis r3,7f@ha 356 addi r3,r3,7f@l 357 lwz r4,GPR0(r1) 358 lwz r5,GPR3(r1) 359 lwz r6,GPR4(r1) 360 lwz r7,GPR5(r1) 361 lwz r8,GPR6(r1) 362 lwz r9,GPR7(r1) 363 bl printk 364 lis r3,77f@ha 365 addi r3,r3,77f@l 366 lwz r4,GPR8(r1) 367 mr r5,r2 368 bl printk 369 lwz r0,GPR0(r1) 370 lwz r3,GPR3(r1) 371 lwz r4,GPR4(r1) 372 lwz r5,GPR5(r1) 373 lwz r6,GPR6(r1) 374 lwz r7,GPR7(r1) 375 lwz r8,GPR8(r1) 376 mtlr r31 377 lwz r31,GPR31(r1) 378 blr 379 380do_show_syscall_exit: 381#ifdef SHOW_SYSCALLS_TASK 382 lis r11,show_syscalls_task@ha 383 lwz r11,show_syscalls_task@l(r11) 384 cmp 0,r2,r11 385 bnelr 386#endif 387 stw r31,GPR31(r1) 388 mflr r31 389 stw r3,RESULT(r1) /* Save result */ 390 mr r4,r3 391 lis r3,79f@ha 392 addi r3,r3,79f@l 393 bl printk 394 lwz r3,RESULT(r1) 395 mtlr r31 396 lwz r31,GPR31(r1) 397 blr 398 3997: .string "syscall %d(%x, %x, %x, %x, %x, " 40077: .string "%x), current=%p\n" 40179: .string " -> %x\n" 402 .align 2,0 403 404#ifdef SHOW_SYSCALLS_TASK 405 .data 406 .globl show_syscalls_task 407show_syscalls_task: 408 .long -1 409 .text 410#endif 411#endif /* SHOW_SYSCALLS */ 412 413/* 414 * The fork/clone functions need to copy the full register set into 415 * the child process. Therefore we need to save all the nonvolatile 416 * registers (r13 - r31) before calling the C code. 417 */ 418 .globl ppc_fork 419ppc_fork: 420 SAVE_NVGPRS(r1) 421 lwz r0,_TRAP(r1) 422 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 423 stw r0,_TRAP(r1) /* register set saved */ 424 b sys_fork 425 426 .globl ppc_vfork 427ppc_vfork: 428 SAVE_NVGPRS(r1) 429 lwz r0,_TRAP(r1) 430 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 431 stw r0,_TRAP(r1) /* register set saved */ 432 b sys_vfork 433 434 .globl ppc_clone 435ppc_clone: 436 SAVE_NVGPRS(r1) 437 lwz r0,_TRAP(r1) 438 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 439 stw r0,_TRAP(r1) /* register set saved */ 440 b sys_clone 441 442 .globl ppc_swapcontext 443ppc_swapcontext: 444 SAVE_NVGPRS(r1) 445 lwz r0,_TRAP(r1) 446 rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ 447 stw r0,_TRAP(r1) /* register set saved */ 448 b sys_swapcontext 449 450/* 451 * Top-level page fault handling. 452 * This is in assembler because if do_page_fault tells us that 453 * it is a bad kernel page fault, we want to save the non-volatile 454 * registers before calling bad_page_fault. 455 */ 456 .globl handle_page_fault 457handle_page_fault: 458 stw r4,_DAR(r1) 459 addi r3,r1,STACK_FRAME_OVERHEAD 460 bl do_page_fault 461 cmpwi r3,0 462 beq+ ret_from_except 463 SAVE_NVGPRS(r1) 464 lwz r0,_TRAP(r1) 465 clrrwi r0,r0,1 466 stw r0,_TRAP(r1) 467 mr r5,r3 468 addi r3,r1,STACK_FRAME_OVERHEAD 469 lwz r4,_DAR(r1) 470 bl bad_page_fault 471 b ret_from_except_full 472 473/* 474 * This routine switches between two different tasks. The process 475 * state of one is saved on its kernel stack. Then the state 476 * of the other is restored from its kernel stack. The memory 477 * management hardware is updated to the second process's state. 478 * Finally, we can return to the second process. 479 * On entry, r3 points to the THREAD for the current task, r4 480 * points to the THREAD for the new task. 481 * 482 * This routine is always called with interrupts disabled. 483 * 484 * Note: there are two ways to get to the "going out" portion 485 * of this code; either by coming in via the entry (_switch) 486 * or via "fork" which must set up an environment equivalent 487 * to the "_switch" path. If you change this , you'll have to 488 * change the fork code also. 489 * 490 * The code which creates the new task context is in 'copy_thread' 491 * in arch/ppc/kernel/process.c 492 */ 493_GLOBAL(_switch) 494 stwu r1,-INT_FRAME_SIZE(r1) 495 mflr r0 496 stw r0,INT_FRAME_SIZE+4(r1) 497 /* r3-r12 are caller saved -- Cort */ 498 SAVE_NVGPRS(r1) 499 stw r0,_NIP(r1) /* Return to switch caller */ 500 mfmsr r11 501 li r0,MSR_FP /* Disable floating-point */ 502#ifdef CONFIG_ALTIVEC 503BEGIN_FTR_SECTION 504 oris r0,r0,MSR_VEC@h /* Disable altivec */ 505 mfspr r12,SPRN_VRSAVE /* save vrsave register value */ 506 stw r12,THREAD+THREAD_VRSAVE(r2) 507END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 508#endif /* CONFIG_ALTIVEC */ 509#ifdef CONFIG_SPE 510 oris r0,r0,MSR_SPE@h /* Disable SPE */ 511 mfspr r12,SPRN_SPEFSCR /* save spefscr register value */ 512 stw r12,THREAD+THREAD_SPEFSCR(r2) 513#endif /* CONFIG_SPE */ 514 and. r0,r0,r11 /* FP or altivec or SPE enabled? */ 515 beq+ 1f 516 andc r11,r11,r0 517 MTMSRD(r11) 518 isync 5191: stw r11,_MSR(r1) 520 mfcr r10 521 stw r10,_CCR(r1) 522 stw r1,KSP(r3) /* Set old stack pointer */ 523 524#ifdef CONFIG_SMP 525 /* We need a sync somewhere here to make sure that if the 526 * previous task gets rescheduled on another CPU, it sees all 527 * stores it has performed on this one. 528 */ 529 sync 530#endif /* CONFIG_SMP */ 531 532 tophys(r0,r4) 533 CLR_TOP32(r0) 534 mtspr SPRN_SPRG3,r0 /* Update current THREAD phys addr */ 535 lwz r1,KSP(r4) /* Load new stack pointer */ 536 537 /* save the old current 'last' for return value */ 538 mr r3,r2 539 addi r2,r4,-THREAD /* Update current */ 540 541#ifdef CONFIG_ALTIVEC 542BEGIN_FTR_SECTION 543 lwz r0,THREAD+THREAD_VRSAVE(r2) 544 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */ 545END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 546#endif /* CONFIG_ALTIVEC */ 547#ifdef CONFIG_SPE 548 lwz r0,THREAD+THREAD_SPEFSCR(r2) 549 mtspr SPRN_SPEFSCR,r0 /* restore SPEFSCR reg */ 550#endif /* CONFIG_SPE */ 551 552 lwz r0,_CCR(r1) 553 mtcrf 0xFF,r0 554 /* r3-r12 are destroyed -- Cort */ 555 REST_NVGPRS(r1) 556 557 lwz r4,_NIP(r1) /* Return to _switch caller in new task */ 558 mtlr r4 559 addi r1,r1,INT_FRAME_SIZE 560 blr 561 562 .globl fast_exception_return 563fast_exception_return: 564#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 565 andi. r10,r9,MSR_RI /* check for recoverable interrupt */ 566 beq 1f /* if not, we've got problems */ 567#endif 568 5692: REST_4GPRS(3, r11) 570 lwz r10,_CCR(r11) 571 REST_GPR(1, r11) 572 mtcr r10 573 lwz r10,_LINK(r11) 574 mtlr r10 575 REST_GPR(10, r11) 576 mtspr SPRN_SRR1,r9 577 mtspr SPRN_SRR0,r12 578 REST_GPR(9, r11) 579 REST_GPR(12, r11) 580 lwz r11,GPR11(r11) 581 SYNC 582 RFI 583 584#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 585/* check if the exception happened in a restartable section */ 5861: lis r3,exc_exit_restart_end@ha 587 addi r3,r3,exc_exit_restart_end@l 588 cmplw r12,r3 589 bge 3f 590 lis r4,exc_exit_restart@ha 591 addi r4,r4,exc_exit_restart@l 592 cmplw r12,r4 593 blt 3f 594 lis r3,fee_restarts@ha 595 tophys(r3,r3) 596 lwz r5,fee_restarts@l(r3) 597 addi r5,r5,1 598 stw r5,fee_restarts@l(r3) 599 mr r12,r4 /* restart at exc_exit_restart */ 600 b 2b 601 602 .comm fee_restarts,4 603 604/* aargh, a nonrecoverable interrupt, panic */ 605/* aargh, we don't know which trap this is */ 606/* but the 601 doesn't implement the RI bit, so assume it's OK */ 6073: 608BEGIN_FTR_SECTION 609 b 2b 610END_FTR_SECTION_IFSET(CPU_FTR_601) 611 li r10,-1 612 stw r10,_TRAP(r11) 613 addi r3,r1,STACK_FRAME_OVERHEAD 614 lis r10,MSR_KERNEL@h 615 ori r10,r10,MSR_KERNEL@l 616 bl transfer_to_handler_full 617 .long nonrecoverable_exception 618 .long ret_from_except 619#endif 620 621 .globl ret_from_except_full 622ret_from_except_full: 623 REST_NVGPRS(r1) 624 /* fall through */ 625 626 .globl ret_from_except 627ret_from_except: 628 /* Hard-disable interrupts so that current_thread_info()->flags 629 * can't change between when we test it and when we return 630 * from the interrupt. */ 631 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 632 SYNC /* Some chip revs have problems here... */ 633 MTMSRD(r10) /* disable interrupts */ 634 635 lwz r3,_MSR(r1) /* Returning to user mode? */ 636 andi. r0,r3,MSR_PR 637 beq resume_kernel 638 639user_exc_return: /* r10 contains MSR_KERNEL here */ 640 /* Check current_thread_info()->flags */ 641 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 642 lwz r9,TI_FLAGS(r9) 643 andi. r0,r9,(_TIF_SIGPENDING|_TIF_RESTORE_SIGMASK|_TIF_NEED_RESCHED) 644 bne do_work 645 646restore_user: 647#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 648 /* Check whether this process has its own DBCR0 value. The single 649 step bit tells us that dbcr0 should be loaded. */ 650 lwz r0,THREAD+THREAD_DBCR0(r2) 651 andis. r10,r0,DBCR0_IC@h 652 bnel- load_dbcr0 653#endif 654 655#ifdef CONFIG_PREEMPT 656 b restore 657 658/* N.B. the only way to get here is from the beq following ret_from_except. */ 659resume_kernel: 660 /* check current_thread_info->preempt_count */ 661 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 662 lwz r0,TI_PREEMPT(r9) 663 cmpwi 0,r0,0 /* if non-zero, just restore regs and return */ 664 bne restore 665 lwz r0,TI_FLAGS(r9) 666 andi. r0,r0,_TIF_NEED_RESCHED 667 beq+ restore 668 andi. r0,r3,MSR_EE /* interrupts off? */ 669 beq restore /* don't schedule if so */ 6701: bl preempt_schedule_irq 671 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 672 lwz r3,TI_FLAGS(r9) 673 andi. r0,r3,_TIF_NEED_RESCHED 674 bne- 1b 675#else 676resume_kernel: 677#endif /* CONFIG_PREEMPT */ 678 679 /* interrupts are hard-disabled at this point */ 680restore: 681 lwz r0,GPR0(r1) 682 lwz r2,GPR2(r1) 683 REST_4GPRS(3, r1) 684 REST_2GPRS(7, r1) 685 686 lwz r10,_XER(r1) 687 lwz r11,_CTR(r1) 688 mtspr SPRN_XER,r10 689 mtctr r11 690 691 PPC405_ERR77(0,r1) 692 stwcx. r0,0,r1 /* to clear the reservation */ 693 694#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 695 lwz r9,_MSR(r1) 696 andi. r10,r9,MSR_RI /* check if this exception occurred */ 697 beql nonrecoverable /* at a bad place (MSR:RI = 0) */ 698 699 lwz r10,_CCR(r1) 700 lwz r11,_LINK(r1) 701 mtcrf 0xFF,r10 702 mtlr r11 703 704 /* 705 * Once we put values in SRR0 and SRR1, we are in a state 706 * where exceptions are not recoverable, since taking an 707 * exception will trash SRR0 and SRR1. Therefore we clear the 708 * MSR:RI bit to indicate this. If we do take an exception, 709 * we can't return to the point of the exception but we 710 * can restart the exception exit path at the label 711 * exc_exit_restart below. -- paulus 712 */ 713 LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI) 714 SYNC 715 MTMSRD(r10) /* clear the RI bit */ 716 .globl exc_exit_restart 717exc_exit_restart: 718 lwz r9,_MSR(r1) 719 lwz r12,_NIP(r1) 720 FIX_SRR1(r9,r10) 721 mtspr SPRN_SRR0,r12 722 mtspr SPRN_SRR1,r9 723 REST_4GPRS(9, r1) 724 lwz r1,GPR1(r1) 725 .globl exc_exit_restart_end 726exc_exit_restart_end: 727 SYNC 728 RFI 729 730#else /* !(CONFIG_4xx || CONFIG_BOOKE) */ 731 /* 732 * This is a bit different on 4xx/Book-E because it doesn't have 733 * the RI bit in the MSR. 734 * The TLB miss handler checks if we have interrupted 735 * the exception exit path and restarts it if so 736 * (well maybe one day it will... :). 737 */ 738 lwz r11,_LINK(r1) 739 mtlr r11 740 lwz r10,_CCR(r1) 741 mtcrf 0xff,r10 742 REST_2GPRS(9, r1) 743 .globl exc_exit_restart 744exc_exit_restart: 745 lwz r11,_NIP(r1) 746 lwz r12,_MSR(r1) 747exc_exit_start: 748 mtspr SPRN_SRR0,r11 749 mtspr SPRN_SRR1,r12 750 REST_2GPRS(11, r1) 751 lwz r1,GPR1(r1) 752 .globl exc_exit_restart_end 753exc_exit_restart_end: 754 PPC405_ERR77_SYNC 755 rfi 756 b . /* prevent prefetch past rfi */ 757 758/* 759 * Returning from a critical interrupt in user mode doesn't need 760 * to be any different from a normal exception. For a critical 761 * interrupt in the kernel, we just return (without checking for 762 * preemption) since the interrupt may have happened at some crucial 763 * place (e.g. inside the TLB miss handler), and because we will be 764 * running with r1 pointing into critical_stack, not the current 765 * process's kernel stack (and therefore current_thread_info() will 766 * give the wrong answer). 767 * We have to restore various SPRs that may have been in use at the 768 * time of the critical interrupt. 769 * 770 */ 771#ifdef CONFIG_40x 772#define PPC_40x_TURN_OFF_MSR_DR \ 773 /* avoid any possible TLB misses here by turning off MSR.DR, we \ 774 * assume the instructions here are mapped by a pinned TLB entry */ \ 775 li r10,MSR_IR; \ 776 mtmsr r10; \ 777 isync; \ 778 tophys(r1, r1); 779#else 780#define PPC_40x_TURN_OFF_MSR_DR 781#endif 782 783#define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \ 784 REST_NVGPRS(r1); \ 785 lwz r3,_MSR(r1); \ 786 andi. r3,r3,MSR_PR; \ 787 LOAD_MSR_KERNEL(r10,MSR_KERNEL); \ 788 bne user_exc_return; \ 789 lwz r0,GPR0(r1); \ 790 lwz r2,GPR2(r1); \ 791 REST_4GPRS(3, r1); \ 792 REST_2GPRS(7, r1); \ 793 lwz r10,_XER(r1); \ 794 lwz r11,_CTR(r1); \ 795 mtspr SPRN_XER,r10; \ 796 mtctr r11; \ 797 PPC405_ERR77(0,r1); \ 798 stwcx. r0,0,r1; /* to clear the reservation */ \ 799 lwz r11,_LINK(r1); \ 800 mtlr r11; \ 801 lwz r10,_CCR(r1); \ 802 mtcrf 0xff,r10; \ 803 PPC_40x_TURN_OFF_MSR_DR; \ 804 lwz r9,_DEAR(r1); \ 805 lwz r10,_ESR(r1); \ 806 mtspr SPRN_DEAR,r9; \ 807 mtspr SPRN_ESR,r10; \ 808 lwz r11,_NIP(r1); \ 809 lwz r12,_MSR(r1); \ 810 mtspr exc_lvl_srr0,r11; \ 811 mtspr exc_lvl_srr1,r12; \ 812 lwz r9,GPR9(r1); \ 813 lwz r12,GPR12(r1); \ 814 lwz r10,GPR10(r1); \ 815 lwz r11,GPR11(r1); \ 816 lwz r1,GPR1(r1); \ 817 PPC405_ERR77_SYNC; \ 818 exc_lvl_rfi; \ 819 b .; /* prevent prefetch past exc_lvl_rfi */ 820 821 .globl ret_from_crit_exc 822ret_from_crit_exc: 823 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, RFCI) 824 825#ifdef CONFIG_BOOKE 826 .globl ret_from_debug_exc 827ret_from_debug_exc: 828 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, RFDI) 829 830 .globl ret_from_mcheck_exc 831ret_from_mcheck_exc: 832 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, RFMCI) 833#endif /* CONFIG_BOOKE */ 834 835/* 836 * Load the DBCR0 value for a task that is being ptraced, 837 * having first saved away the global DBCR0. Note that r0 838 * has the dbcr0 value to set upon entry to this. 839 */ 840load_dbcr0: 841 mfmsr r10 /* first disable debug exceptions */ 842 rlwinm r10,r10,0,~MSR_DE 843 mtmsr r10 844 isync 845 mfspr r10,SPRN_DBCR0 846 lis r11,global_dbcr0@ha 847 addi r11,r11,global_dbcr0@l 848 stw r10,0(r11) 849 mtspr SPRN_DBCR0,r0 850 lwz r10,4(r11) 851 addi r10,r10,1 852 stw r10,4(r11) 853 li r11,-1 854 mtspr SPRN_DBSR,r11 /* clear all pending debug events */ 855 blr 856 857 .comm global_dbcr0,8 858#endif /* !(CONFIG_4xx || CONFIG_BOOKE) */ 859 860do_work: /* r10 contains MSR_KERNEL here */ 861 andi. r0,r9,_TIF_NEED_RESCHED 862 beq do_user_signal 863 864do_resched: /* r10 contains MSR_KERNEL here */ 865 ori r10,r10,MSR_EE 866 SYNC 867 MTMSRD(r10) /* hard-enable interrupts */ 868 bl schedule 869recheck: 870 LOAD_MSR_KERNEL(r10,MSR_KERNEL) 871 SYNC 872 MTMSRD(r10) /* disable interrupts */ 873 rlwinm r9,r1,0,0,(31-THREAD_SHIFT) 874 lwz r9,TI_FLAGS(r9) 875 andi. r0,r9,_TIF_NEED_RESCHED 876 bne- do_resched 877 andi. r0,r9,_TIF_SIGPENDING|_TIF_RESTORE_SIGMASK 878 beq restore_user 879do_user_signal: /* r10 contains MSR_KERNEL here */ 880 ori r10,r10,MSR_EE 881 SYNC 882 MTMSRD(r10) /* hard-enable interrupts */ 883 /* save r13-r31 in the exception frame, if not already done */ 884 lwz r3,_TRAP(r1) 885 andi. r0,r3,1 886 beq 2f 887 SAVE_NVGPRS(r1) 888 rlwinm r3,r3,0,0,30 889 stw r3,_TRAP(r1) 8902: li r3,0 891 addi r4,r1,STACK_FRAME_OVERHEAD 892 bl do_signal 893 REST_NVGPRS(r1) 894 b recheck 895 896/* 897 * We come here when we are at the end of handling an exception 898 * that occurred at a place where taking an exception will lose 899 * state information, such as the contents of SRR0 and SRR1. 900 */ 901nonrecoverable: 902 lis r10,exc_exit_restart_end@ha 903 addi r10,r10,exc_exit_restart_end@l 904 cmplw r12,r10 905 bge 3f 906 lis r11,exc_exit_restart@ha 907 addi r11,r11,exc_exit_restart@l 908 cmplw r12,r11 909 blt 3f 910 lis r10,ee_restarts@ha 911 lwz r12,ee_restarts@l(r10) 912 addi r12,r12,1 913 stw r12,ee_restarts@l(r10) 914 mr r12,r11 /* restart at exc_exit_restart */ 915 blr 9163: /* OK, we can't recover, kill this process */ 917 /* but the 601 doesn't implement the RI bit, so assume it's OK */ 918BEGIN_FTR_SECTION 919 blr 920END_FTR_SECTION_IFSET(CPU_FTR_601) 921 lwz r3,_TRAP(r1) 922 andi. r0,r3,1 923 beq 4f 924 SAVE_NVGPRS(r1) 925 rlwinm r3,r3,0,0,30 926 stw r3,_TRAP(r1) 9274: addi r3,r1,STACK_FRAME_OVERHEAD 928 bl nonrecoverable_exception 929 /* shouldn't return */ 930 b 4b 931 932 .comm ee_restarts,4 933 934/* 935 * PROM code for specific machines follows. Put it 936 * here so it's easy to add arch-specific sections later. 937 * -- Cort 938 */ 939#ifdef CONFIG_PPC_RTAS 940/* 941 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be 942 * called with the MMU off. 943 */ 944_GLOBAL(enter_rtas) 945 stwu r1,-INT_FRAME_SIZE(r1) 946 mflr r0 947 stw r0,INT_FRAME_SIZE+4(r1) 948 LOAD_REG_ADDR(r4, rtas) 949 lis r6,1f@ha /* physical return address for rtas */ 950 addi r6,r6,1f@l 951 tophys(r6,r6) 952 tophys(r7,r1) 953 lwz r8,RTASENTRY(r4) 954 lwz r4,RTASBASE(r4) 955 mfmsr r9 956 stw r9,8(r1) 957 LOAD_MSR_KERNEL(r0,MSR_KERNEL) 958 SYNC /* disable interrupts so SRR0/1 */ 959 MTMSRD(r0) /* don't get trashed */ 960 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR) 961 mtlr r6 962 mtspr SPRN_SPRG2,r7 963 mtspr SPRN_SRR0,r8 964 mtspr SPRN_SRR1,r9 965 RFI 9661: tophys(r9,r1) 967 lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */ 968 lwz r9,8(r9) /* original msr value */ 969 FIX_SRR1(r9,r0) 970 addi r1,r1,INT_FRAME_SIZE 971 li r0,0 972 mtspr SPRN_SPRG2,r0 973 mtspr SPRN_SRR0,r8 974 mtspr SPRN_SRR1,r9 975 RFI /* return to caller */ 976 977 .globl machine_check_in_rtas 978machine_check_in_rtas: 979 twi 31,0,0 980 /* XXX load up BATs and panic */ 981 982#endif /* CONFIG_PPC_RTAS */ 983