1/* 2 * This file contains the 64-bit "server" PowerPC variant 3 * of the low level exception handling including exception 4 * vectors, exception return, part of the slb and stab 5 * handling and other fixed offset specific things. 6 * 7 * This file is meant to be #included from head_64.S due to 8 * position dependant assembly. 9 * 10 * Most of this originates from head_64.S and thus has the same 11 * copyright history. 12 * 13 */ 14 15#include <asm/exception-64s.h> 16 17/* 18 * We layout physical memory as follows: 19 * 0x0000 - 0x00ff : Secondary processor spin code 20 * 0x0100 - 0x2fff : pSeries Interrupt prologs 21 * 0x3000 - 0x5fff : interrupt support, iSeries and common interrupt prologs 22 * 0x6000 - 0x6fff : Initial (CPU0) segment table 23 * 0x7000 - 0x7fff : FWNMI data area 24 * 0x8000 - : Early init and support code 25 */ 26 27/* 28 * This is the start of the interrupt handlers for pSeries 29 * This code runs with relocation off. 30 * Code from here to __end_interrupts gets copied down to real 31 * address 0x100 when we are running a relocatable kernel. 32 * Therefore any relative branches in this section must only 33 * branch to labels in this section. 34 */ 35 . = 0x100 36 .globl __start_interrupts 37__start_interrupts: 38 39 STD_EXCEPTION_PSERIES(0x100, system_reset) 40 41 . = 0x200 42_machine_check_pSeries: 43 HMT_MEDIUM 44 DO_KVM 0x200 45 mtspr SPRN_SPRG_SCRATCH0,r13 /* save r13 */ 46 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common) 47 48 . = 0x300 49 .globl data_access_pSeries 50data_access_pSeries: 51 HMT_MEDIUM 52 DO_KVM 0x300 53 mtspr SPRN_SPRG_SCRATCH0,r13 54BEGIN_FTR_SECTION 55 mfspr r13,SPRN_SPRG_PACA 56 std r9,PACA_EXSLB+EX_R9(r13) 57 std r10,PACA_EXSLB+EX_R10(r13) 58 mfspr r10,SPRN_DAR 59 mfspr r9,SPRN_DSISR 60 srdi r10,r10,60 61 rlwimi r10,r9,16,0x20 62 mfcr r9 63 cmpwi r10,0x2c 64 beq do_stab_bolted_pSeries 65 ld r10,PACA_EXSLB+EX_R10(r13) 66 std r11,PACA_EXGEN+EX_R11(r13) 67 ld r11,PACA_EXSLB+EX_R9(r13) 68 std r12,PACA_EXGEN+EX_R12(r13) 69 mfspr r12,SPRN_SPRG_SCRATCH0 70 std r10,PACA_EXGEN+EX_R10(r13) 71 std r11,PACA_EXGEN+EX_R9(r13) 72 std r12,PACA_EXGEN+EX_R13(r13) 73 EXCEPTION_PROLOG_PSERIES_1(data_access_common) 74FTR_SECTION_ELSE 75 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common) 76ALT_FTR_SECTION_END_IFCLR(CPU_FTR_SLB) 77 78 . = 0x380 79 .globl data_access_slb_pSeries 80data_access_slb_pSeries: 81 HMT_MEDIUM 82 DO_KVM 0x380 83 mtspr SPRN_SPRG_SCRATCH0,r13 84 mfspr r13,SPRN_SPRG_PACA /* get paca address into r13 */ 85 std r3,PACA_EXSLB+EX_R3(r13) 86 mfspr r3,SPRN_DAR 87 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */ 88 mfcr r9 89#ifdef __DISABLED__ 90 /* Keep that around for when we re-implement dynamic VSIDs */ 91 cmpdi r3,0 92 bge slb_miss_user_pseries 93#endif /* __DISABLED__ */ 94 std r10,PACA_EXSLB+EX_R10(r13) 95 std r11,PACA_EXSLB+EX_R11(r13) 96 std r12,PACA_EXSLB+EX_R12(r13) 97 mfspr r10,SPRN_SPRG_SCRATCH0 98 std r10,PACA_EXSLB+EX_R13(r13) 99 mfspr r12,SPRN_SRR1 /* and SRR1 */ 100#ifndef CONFIG_RELOCATABLE 101 b .slb_miss_realmode 102#else 103 /* 104 * We can't just use a direct branch to .slb_miss_realmode 105 * because the distance from here to there depends on where 106 * the kernel ends up being put. 107 */ 108 mfctr r11 109 ld r10,PACAKBASE(r13) 110 LOAD_HANDLER(r10, .slb_miss_realmode) 111 mtctr r10 112 bctr 113#endif 114 115 STD_EXCEPTION_PSERIES(0x400, instruction_access) 116 117 . = 0x480 118 .globl instruction_access_slb_pSeries 119instruction_access_slb_pSeries: 120 HMT_MEDIUM 121 DO_KVM 0x480 122 mtspr SPRN_SPRG_SCRATCH0,r13 123 mfspr r13,SPRN_SPRG_PACA /* get paca address into r13 */ 124 std r3,PACA_EXSLB+EX_R3(r13) 125 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */ 126 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */ 127 mfcr r9 128#ifdef __DISABLED__ 129 /* Keep that around for when we re-implement dynamic VSIDs */ 130 cmpdi r3,0 131 bge slb_miss_user_pseries 132#endif /* __DISABLED__ */ 133 std r10,PACA_EXSLB+EX_R10(r13) 134 std r11,PACA_EXSLB+EX_R11(r13) 135 std r12,PACA_EXSLB+EX_R12(r13) 136 mfspr r10,SPRN_SPRG_SCRATCH0 137 std r10,PACA_EXSLB+EX_R13(r13) 138 mfspr r12,SPRN_SRR1 /* and SRR1 */ 139#ifndef CONFIG_RELOCATABLE 140 b .slb_miss_realmode 141#else 142 mfctr r11 143 ld r10,PACAKBASE(r13) 144 LOAD_HANDLER(r10, .slb_miss_realmode) 145 mtctr r10 146 bctr 147#endif 148 149 MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt) 150 STD_EXCEPTION_PSERIES(0x600, alignment) 151 STD_EXCEPTION_PSERIES(0x700, program_check) 152 STD_EXCEPTION_PSERIES(0x800, fp_unavailable) 153 MASKABLE_EXCEPTION_PSERIES(0x900, decrementer) 154 STD_EXCEPTION_PSERIES(0xa00, trap_0a) 155 STD_EXCEPTION_PSERIES(0xb00, trap_0b) 156 157 . = 0xc00 158 .globl system_call_pSeries 159system_call_pSeries: 160 HMT_MEDIUM 161 DO_KVM 0xc00 162BEGIN_FTR_SECTION 163 cmpdi r0,0x1ebe 164 beq- 1f 165END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 166 mr r9,r13 167 mfspr r13,SPRN_SPRG_PACA 168 mfspr r11,SPRN_SRR0 169 ld r12,PACAKBASE(r13) 170 ld r10,PACAKMSR(r13) 171 LOAD_HANDLER(r12, system_call_entry) 172 mtspr SPRN_SRR0,r12 173 mfspr r12,SPRN_SRR1 174 mtspr SPRN_SRR1,r10 175 rfid 176 b . /* prevent speculative execution */ 177 178/* Fast LE/BE switch system call */ 1791: mfspr r12,SPRN_SRR1 180 xori r12,r12,MSR_LE 181 mtspr SPRN_SRR1,r12 182 rfid /* return to userspace */ 183 b . 184 185 STD_EXCEPTION_PSERIES(0xd00, single_step) 186 STD_EXCEPTION_PSERIES(0xe00, trap_0e) 187 188 /* We need to deal with the Altivec unavailable exception 189 * here which is at 0xf20, thus in the middle of the 190 * prolog code of the PerformanceMonitor one. A little 191 * trickery is thus necessary 192 */ 193performance_monitor_pSeries_1: 194 . = 0xf00 195 DO_KVM 0xf00 196 b performance_monitor_pSeries 197 198altivec_unavailable_pSeries_1: 199 . = 0xf20 200 DO_KVM 0xf20 201 b altivec_unavailable_pSeries 202 203vsx_unavailable_pSeries_1: 204 . = 0xf40 205 DO_KVM 0xf40 206 b vsx_unavailable_pSeries 207 208#ifdef CONFIG_CBE_RAS 209 HSTD_EXCEPTION_PSERIES(0x1200, cbe_system_error) 210#endif /* CONFIG_CBE_RAS */ 211 STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint) 212#ifdef CONFIG_CBE_RAS 213 HSTD_EXCEPTION_PSERIES(0x1600, cbe_maintenance) 214#endif /* CONFIG_CBE_RAS */ 215 STD_EXCEPTION_PSERIES(0x1700, altivec_assist) 216#ifdef CONFIG_CBE_RAS 217 HSTD_EXCEPTION_PSERIES(0x1800, cbe_thermal) 218#endif /* CONFIG_CBE_RAS */ 219 220 . = 0x3000 221 222/*** pSeries interrupt support ***/ 223 224 /* moved from 0xf00 */ 225 STD_EXCEPTION_PSERIES(., performance_monitor) 226 STD_EXCEPTION_PSERIES(., altivec_unavailable) 227 STD_EXCEPTION_PSERIES(., vsx_unavailable) 228 229/* 230 * An interrupt came in while soft-disabled; clear EE in SRR1, 231 * clear paca->hard_enabled and return. 232 */ 233masked_interrupt: 234 stb r10,PACAHARDIRQEN(r13) 235 mtcrf 0x80,r9 236 ld r9,PACA_EXGEN+EX_R9(r13) 237 mfspr r10,SPRN_SRR1 238 rldicl r10,r10,48,1 /* clear MSR_EE */ 239 rotldi r10,r10,16 240 mtspr SPRN_SRR1,r10 241 ld r10,PACA_EXGEN+EX_R10(r13) 242 mfspr r13,SPRN_SPRG_SCRATCH0 243 rfid 244 b . 245 246 .align 7 247do_stab_bolted_pSeries: 248 std r11,PACA_EXSLB+EX_R11(r13) 249 std r12,PACA_EXSLB+EX_R12(r13) 250 mfspr r10,SPRN_SPRG_SCRATCH0 251 std r10,PACA_EXSLB+EX_R13(r13) 252 EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted) 253 254#ifdef CONFIG_PPC_PSERIES 255/* 256 * Vectors for the FWNMI option. Share common code. 257 */ 258 .globl system_reset_fwnmi 259 .align 7 260system_reset_fwnmi: 261 HMT_MEDIUM 262 mtspr SPRN_SPRG_SCRATCH0,r13 /* save r13 */ 263 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common) 264 265 .globl machine_check_fwnmi 266 .align 7 267machine_check_fwnmi: 268 HMT_MEDIUM 269 mtspr SPRN_SPRG_SCRATCH0,r13 /* save r13 */ 270 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common) 271 272#endif /* CONFIG_PPC_PSERIES */ 273 274#ifdef __DISABLED__ 275/* 276 * This is used for when the SLB miss handler has to go virtual, 277 * which doesn't happen for now anymore but will once we re-implement 278 * dynamic VSIDs for shared page tables 279 */ 280slb_miss_user_pseries: 281 std r10,PACA_EXGEN+EX_R10(r13) 282 std r11,PACA_EXGEN+EX_R11(r13) 283 std r12,PACA_EXGEN+EX_R12(r13) 284 mfspr r10,SPRG_SCRATCH0 285 ld r11,PACA_EXSLB+EX_R9(r13) 286 ld r12,PACA_EXSLB+EX_R3(r13) 287 std r10,PACA_EXGEN+EX_R13(r13) 288 std r11,PACA_EXGEN+EX_R9(r13) 289 std r12,PACA_EXGEN+EX_R3(r13) 290 clrrdi r12,r13,32 291 mfmsr r10 292 mfspr r11,SRR0 /* save SRR0 */ 293 ori r12,r12,slb_miss_user_common@l /* virt addr of handler */ 294 ori r10,r10,MSR_IR|MSR_DR|MSR_RI 295 mtspr SRR0,r12 296 mfspr r12,SRR1 /* and SRR1 */ 297 mtspr SRR1,r10 298 rfid 299 b . /* prevent spec. execution */ 300#endif /* __DISABLED__ */ 301 302 .align 7 303 .globl __end_interrupts 304__end_interrupts: 305 306/* 307 * Code from here down to __end_handlers is invoked from the 308 * exception prologs above. Because the prologs assemble the 309 * addresses of these handlers using the LOAD_HANDLER macro, 310 * which uses an addi instruction, these handlers must be in 311 * the first 32k of the kernel image. 312 */ 313 314/*** Common interrupt handlers ***/ 315 316 STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception) 317 318 /* 319 * Machine check is different because we use a different 320 * save area: PACA_EXMC instead of PACA_EXGEN. 321 */ 322 .align 7 323 .globl machine_check_common 324machine_check_common: 325 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC) 326 FINISH_NAP 327 DISABLE_INTS 328 bl .save_nvgprs 329 addi r3,r1,STACK_FRAME_OVERHEAD 330 bl .machine_check_exception 331 b .ret_from_except 332 333 STD_EXCEPTION_COMMON_LITE(0x900, decrementer, .timer_interrupt) 334 STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception) 335 STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception) 336 STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception) 337 STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception) 338 STD_EXCEPTION_COMMON_IDLE(0xf00, performance_monitor, .performance_monitor_exception) 339 STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception) 340#ifdef CONFIG_ALTIVEC 341 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception) 342#else 343 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception) 344#endif 345#ifdef CONFIG_CBE_RAS 346 STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception) 347 STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception) 348 STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception) 349#endif /* CONFIG_CBE_RAS */ 350 351 .align 7 352system_call_entry: 353 b system_call_common 354 355/* 356 * Here we have detected that the kernel stack pointer is bad. 357 * R9 contains the saved CR, r13 points to the paca, 358 * r10 contains the (bad) kernel stack pointer, 359 * r11 and r12 contain the saved SRR0 and SRR1. 360 * We switch to using an emergency stack, save the registers there, 361 * and call kernel_bad_stack(), which panics. 362 */ 363bad_stack: 364 ld r1,PACAEMERGSP(r13) 365 subi r1,r1,64+INT_FRAME_SIZE 366 std r9,_CCR(r1) 367 std r10,GPR1(r1) 368 std r11,_NIP(r1) 369 std r12,_MSR(r1) 370 mfspr r11,SPRN_DAR 371 mfspr r12,SPRN_DSISR 372 std r11,_DAR(r1) 373 std r12,_DSISR(r1) 374 mflr r10 375 mfctr r11 376 mfxer r12 377 std r10,_LINK(r1) 378 std r11,_CTR(r1) 379 std r12,_XER(r1) 380 SAVE_GPR(0,r1) 381 SAVE_GPR(2,r1) 382 SAVE_4GPRS(3,r1) 383 SAVE_2GPRS(7,r1) 384 SAVE_10GPRS(12,r1) 385 SAVE_10GPRS(22,r1) 386 lhz r12,PACA_TRAP_SAVE(r13) 387 std r12,_TRAP(r1) 388 addi r11,r1,INT_FRAME_SIZE 389 std r11,0(r1) 390 li r12,0 391 std r12,0(r11) 392 ld r2,PACATOC(r13) 3931: addi r3,r1,STACK_FRAME_OVERHEAD 394 bl .kernel_bad_stack 395 b 1b 396 397/* 398 * Here r13 points to the paca, r9 contains the saved CR, 399 * SRR0 and SRR1 are saved in r11 and r12, 400 * r9 - r13 are saved in paca->exgen. 401 */ 402 .align 7 403 .globl data_access_common 404data_access_common: 405 mfspr r10,SPRN_DAR 406 std r10,PACA_EXGEN+EX_DAR(r13) 407 mfspr r10,SPRN_DSISR 408 stw r10,PACA_EXGEN+EX_DSISR(r13) 409 EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN) 410 ld r3,PACA_EXGEN+EX_DAR(r13) 411 lwz r4,PACA_EXGEN+EX_DSISR(r13) 412 li r5,0x300 413 b .do_hash_page /* Try to handle as hpte fault */ 414 415 .align 7 416 .globl instruction_access_common 417instruction_access_common: 418 EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN) 419 ld r3,_NIP(r1) 420 andis. r4,r12,0x5820 421 li r5,0x400 422 b .do_hash_page /* Try to handle as hpte fault */ 423 424/* 425 * Here is the common SLB miss user that is used when going to virtual 426 * mode for SLB misses, that is currently not used 427 */ 428#ifdef __DISABLED__ 429 .align 7 430 .globl slb_miss_user_common 431slb_miss_user_common: 432 mflr r10 433 std r3,PACA_EXGEN+EX_DAR(r13) 434 stw r9,PACA_EXGEN+EX_CCR(r13) 435 std r10,PACA_EXGEN+EX_LR(r13) 436 std r11,PACA_EXGEN+EX_SRR0(r13) 437 bl .slb_allocate_user 438 439 ld r10,PACA_EXGEN+EX_LR(r13) 440 ld r3,PACA_EXGEN+EX_R3(r13) 441 lwz r9,PACA_EXGEN+EX_CCR(r13) 442 ld r11,PACA_EXGEN+EX_SRR0(r13) 443 mtlr r10 444 beq- slb_miss_fault 445 446 andi. r10,r12,MSR_RI /* check for unrecoverable exception */ 447 beq- unrecov_user_slb 448 mfmsr r10 449 450.machine push 451.machine "power4" 452 mtcrf 0x80,r9 453.machine pop 454 455 clrrdi r10,r10,2 /* clear RI before setting SRR0/1 */ 456 mtmsrd r10,1 457 458 mtspr SRR0,r11 459 mtspr SRR1,r12 460 461 ld r9,PACA_EXGEN+EX_R9(r13) 462 ld r10,PACA_EXGEN+EX_R10(r13) 463 ld r11,PACA_EXGEN+EX_R11(r13) 464 ld r12,PACA_EXGEN+EX_R12(r13) 465 ld r13,PACA_EXGEN+EX_R13(r13) 466 rfid 467 b . 468 469slb_miss_fault: 470 EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN) 471 ld r4,PACA_EXGEN+EX_DAR(r13) 472 li r5,0 473 std r4,_DAR(r1) 474 std r5,_DSISR(r1) 475 b handle_page_fault 476 477unrecov_user_slb: 478 EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN) 479 DISABLE_INTS 480 bl .save_nvgprs 4811: addi r3,r1,STACK_FRAME_OVERHEAD 482 bl .unrecoverable_exception 483 b 1b 484 485#endif /* __DISABLED__ */ 486 487 488/* 489 * r13 points to the PACA, r9 contains the saved CR, 490 * r12 contain the saved SRR1, SRR0 is still ready for return 491 * r3 has the faulting address 492 * r9 - r13 are saved in paca->exslb. 493 * r3 is saved in paca->slb_r3 494 * We assume we aren't going to take any exceptions during this procedure. 495 */ 496_GLOBAL(slb_miss_realmode) 497 mflr r10 498#ifdef CONFIG_RELOCATABLE 499 mtctr r11 500#endif 501 502 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */ 503 std r10,PACA_EXSLB+EX_LR(r13) /* save LR */ 504 505 bl .slb_allocate_realmode 506 507 /* All done -- return from exception. */ 508 509 ld r10,PACA_EXSLB+EX_LR(r13) 510 ld r3,PACA_EXSLB+EX_R3(r13) 511 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */ 512#ifdef CONFIG_PPC_ISERIES 513BEGIN_FW_FTR_SECTION 514 ld r11,PACALPPACAPTR(r13) 515 ld r11,LPPACASRR0(r11) /* get SRR0 value */ 516END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 517#endif /* CONFIG_PPC_ISERIES */ 518 519 mtlr r10 520 521 andi. r10,r12,MSR_RI /* check for unrecoverable exception */ 522 beq- 2f 523 524.machine push 525.machine "power4" 526 mtcrf 0x80,r9 527 mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */ 528.machine pop 529 530#ifdef CONFIG_PPC_ISERIES 531BEGIN_FW_FTR_SECTION 532 mtspr SPRN_SRR0,r11 533 mtspr SPRN_SRR1,r12 534END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 535#endif /* CONFIG_PPC_ISERIES */ 536 ld r9,PACA_EXSLB+EX_R9(r13) 537 ld r10,PACA_EXSLB+EX_R10(r13) 538 ld r11,PACA_EXSLB+EX_R11(r13) 539 ld r12,PACA_EXSLB+EX_R12(r13) 540 ld r13,PACA_EXSLB+EX_R13(r13) 541 rfid 542 b . /* prevent speculative execution */ 543 5442: 545#ifdef CONFIG_PPC_ISERIES 546BEGIN_FW_FTR_SECTION 547 b unrecov_slb 548END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 549#endif /* CONFIG_PPC_ISERIES */ 550 mfspr r11,SPRN_SRR0 551 ld r10,PACAKBASE(r13) 552 LOAD_HANDLER(r10,unrecov_slb) 553 mtspr SPRN_SRR0,r10 554 ld r10,PACAKMSR(r13) 555 mtspr SPRN_SRR1,r10 556 rfid 557 b . 558 559unrecov_slb: 560 EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB) 561 DISABLE_INTS 562 bl .save_nvgprs 5631: addi r3,r1,STACK_FRAME_OVERHEAD 564 bl .unrecoverable_exception 565 b 1b 566 567 .align 7 568 .globl hardware_interrupt_common 569 .globl hardware_interrupt_entry 570hardware_interrupt_common: 571 EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN) 572 FINISH_NAP 573hardware_interrupt_entry: 574 DISABLE_INTS 575BEGIN_FTR_SECTION 576 bl .ppc64_runlatch_on 577END_FTR_SECTION_IFSET(CPU_FTR_CTRL) 578 addi r3,r1,STACK_FRAME_OVERHEAD 579 bl .do_IRQ 580 b .ret_from_except_lite 581 582#ifdef CONFIG_PPC_970_NAP 583power4_fixup_nap: 584 andc r9,r9,r10 585 std r9,TI_LOCAL_FLAGS(r11) 586 ld r10,_LINK(r1) /* make idle task do the */ 587 std r10,_NIP(r1) /* equivalent of a blr */ 588 blr 589#endif 590 591 .align 7 592 .globl alignment_common 593alignment_common: 594 mfspr r10,SPRN_DAR 595 std r10,PACA_EXGEN+EX_DAR(r13) 596 mfspr r10,SPRN_DSISR 597 stw r10,PACA_EXGEN+EX_DSISR(r13) 598 EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN) 599 ld r3,PACA_EXGEN+EX_DAR(r13) 600 lwz r4,PACA_EXGEN+EX_DSISR(r13) 601 std r3,_DAR(r1) 602 std r4,_DSISR(r1) 603 bl .save_nvgprs 604 addi r3,r1,STACK_FRAME_OVERHEAD 605 ENABLE_INTS 606 bl .alignment_exception 607 b .ret_from_except 608 609 .align 7 610 .globl program_check_common 611program_check_common: 612 EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN) 613 bl .save_nvgprs 614 addi r3,r1,STACK_FRAME_OVERHEAD 615 ENABLE_INTS 616 bl .program_check_exception 617 b .ret_from_except 618 619 .align 7 620 .globl fp_unavailable_common 621fp_unavailable_common: 622 EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN) 623 bne 1f /* if from user, just load it up */ 624 bl .save_nvgprs 625 addi r3,r1,STACK_FRAME_OVERHEAD 626 ENABLE_INTS 627 bl .kernel_fp_unavailable_exception 628 BUG_OPCODE 6291: bl .load_up_fpu 630 b fast_exception_return 631 632 .align 7 633 .globl altivec_unavailable_common 634altivec_unavailable_common: 635 EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN) 636#ifdef CONFIG_ALTIVEC 637BEGIN_FTR_SECTION 638 beq 1f 639 bl .load_up_altivec 640 b fast_exception_return 6411: 642END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 643#endif 644 bl .save_nvgprs 645 addi r3,r1,STACK_FRAME_OVERHEAD 646 ENABLE_INTS 647 bl .altivec_unavailable_exception 648 b .ret_from_except 649 650 .align 7 651 .globl vsx_unavailable_common 652vsx_unavailable_common: 653 EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN) 654#ifdef CONFIG_VSX 655BEGIN_FTR_SECTION 656 bne .load_up_vsx 6571: 658END_FTR_SECTION_IFSET(CPU_FTR_VSX) 659#endif 660 bl .save_nvgprs 661 addi r3,r1,STACK_FRAME_OVERHEAD 662 ENABLE_INTS 663 bl .vsx_unavailable_exception 664 b .ret_from_except 665 666 .align 7 667 .globl __end_handlers 668__end_handlers: 669 670/* 671 * Return from an exception with minimal checks. 672 * The caller is assumed to have done EXCEPTION_PROLOG_COMMON. 673 * If interrupts have been enabled, or anything has been 674 * done that might have changed the scheduling status of 675 * any task or sent any task a signal, you should use 676 * ret_from_except or ret_from_except_lite instead of this. 677 */ 678fast_exc_return_irq: /* restores irq state too */ 679 ld r3,SOFTE(r1) 680 TRACE_AND_RESTORE_IRQ(r3); 681 ld r12,_MSR(r1) 682 rldicl r4,r12,49,63 /* get MSR_EE to LSB */ 683 stb r4,PACAHARDIRQEN(r13) /* restore paca->hard_enabled */ 684 b 1f 685 686 .globl fast_exception_return 687fast_exception_return: 688 ld r12,_MSR(r1) 6891: ld r11,_NIP(r1) 690 andi. r3,r12,MSR_RI /* check if RI is set */ 691 beq- unrecov_fer 692 693#ifdef CONFIG_VIRT_CPU_ACCOUNTING 694 andi. r3,r12,MSR_PR 695 beq 2f 696 ACCOUNT_CPU_USER_EXIT(r3, r4) 6972: 698#endif 699 700 ld r3,_CCR(r1) 701 ld r4,_LINK(r1) 702 ld r5,_CTR(r1) 703 ld r6,_XER(r1) 704 mtcr r3 705 mtlr r4 706 mtctr r5 707 mtxer r6 708 REST_GPR(0, r1) 709 REST_8GPRS(2, r1) 710 711 mfmsr r10 712 rldicl r10,r10,48,1 /* clear EE */ 713 rldicr r10,r10,16,61 /* clear RI (LE is 0 already) */ 714 mtmsrd r10,1 715 716 mtspr SPRN_SRR1,r12 717 mtspr SPRN_SRR0,r11 718 REST_4GPRS(10, r1) 719 ld r1,GPR1(r1) 720 rfid 721 b . /* prevent speculative execution */ 722 723unrecov_fer: 724 bl .save_nvgprs 7251: addi r3,r1,STACK_FRAME_OVERHEAD 726 bl .unrecoverable_exception 727 b 1b 728 729 730/* 731 * Hash table stuff 732 */ 733 .align 7 734_STATIC(do_hash_page) 735 std r3,_DAR(r1) 736 std r4,_DSISR(r1) 737 738 andis. r0,r4,0xa410 /* weird error? */ 739 bne- handle_page_fault /* if not, try to insert a HPTE */ 740 andis. r0,r4,DSISR_DABRMATCH@h 741 bne- handle_dabr_fault 742 743BEGIN_FTR_SECTION 744 andis. r0,r4,0x0020 /* Is it a segment table fault? */ 745 bne- do_ste_alloc /* If so handle it */ 746END_FTR_SECTION_IFCLR(CPU_FTR_SLB) 747 748 clrrdi r11,r1,THREAD_SHIFT 749 lwz r0,TI_PREEMPT(r11) /* If we're in an "NMI" */ 750 andis. r0,r0,NMI_MASK@h /* (i.e. an irq when soft-disabled) */ 751 bne 77f /* then don't call hash_page now */ 752 753 /* 754 * On iSeries, we soft-disable interrupts here, then 755 * hard-enable interrupts so that the hash_page code can spin on 756 * the hash_table_lock without problems on a shared processor. 757 */ 758 DISABLE_INTS 759 760 /* 761 * Currently, trace_hardirqs_off() will be called by DISABLE_INTS 762 * and will clobber volatile registers when irq tracing is enabled 763 * so we need to reload them. It may be possible to be smarter here 764 * and move the irq tracing elsewhere but let's keep it simple for 765 * now 766 */ 767#ifdef CONFIG_TRACE_IRQFLAGS 768 ld r3,_DAR(r1) 769 ld r4,_DSISR(r1) 770 ld r5,_TRAP(r1) 771 ld r12,_MSR(r1) 772 clrrdi r5,r5,4 773#endif /* CONFIG_TRACE_IRQFLAGS */ 774 /* 775 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are 776 * accessing a userspace segment (even from the kernel). We assume 777 * kernel addresses always have the high bit set. 778 */ 779 rlwinm r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */ 780 rotldi r0,r3,15 /* Move high bit into MSR_PR posn */ 781 orc r0,r12,r0 /* MSR_PR | ~high_bit */ 782 rlwimi r4,r0,32-13,30,30 /* becomes _PAGE_USER access bit */ 783 ori r4,r4,1 /* add _PAGE_PRESENT */ 784 rlwimi r4,r5,22+2,31-2,31-2 /* Set _PAGE_EXEC if trap is 0x400 */ 785 786 /* 787 * r3 contains the faulting address 788 * r4 contains the required access permissions 789 * r5 contains the trap number 790 * 791 * at return r3 = 0 for success 792 */ 793 bl .hash_page /* build HPTE if possible */ 794 cmpdi r3,0 /* see if hash_page succeeded */ 795 796BEGIN_FW_FTR_SECTION 797 /* 798 * If we had interrupts soft-enabled at the point where the 799 * DSI/ISI occurred, and an interrupt came in during hash_page, 800 * handle it now. 801 * We jump to ret_from_except_lite rather than fast_exception_return 802 * because ret_from_except_lite will check for and handle pending 803 * interrupts if necessary. 804 */ 805 beq 13f 806END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 807 808BEGIN_FW_FTR_SECTION 809 /* 810 * Here we have interrupts hard-disabled, so it is sufficient 811 * to restore paca->{soft,hard}_enable and get out. 812 */ 813 beq fast_exc_return_irq /* Return from exception on success */ 814END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES) 815 816 /* For a hash failure, we don't bother re-enabling interrupts */ 817 ble- 12f 818 819 /* 820 * hash_page couldn't handle it, set soft interrupt enable back 821 * to what it was before the trap. Note that .raw_local_irq_restore 822 * handles any interrupts pending at this point. 823 */ 824 ld r3,SOFTE(r1) 825 TRACE_AND_RESTORE_IRQ_PARTIAL(r3, 11f) 826 bl .raw_local_irq_restore 827 b 11f 828 829/* We have a data breakpoint exception - handle it */ 830handle_dabr_fault: 831 ld r4,_DAR(r1) 832 ld r5,_DSISR(r1) 833 addi r3,r1,STACK_FRAME_OVERHEAD 834 bl .do_dabr 835 b .ret_from_except_lite 836 837/* Here we have a page fault that hash_page can't handle. */ 838handle_page_fault: 839 ENABLE_INTS 84011: ld r4,_DAR(r1) 841 ld r5,_DSISR(r1) 842 addi r3,r1,STACK_FRAME_OVERHEAD 843 bl .do_page_fault 844 cmpdi r3,0 845 beq+ 13f 846 bl .save_nvgprs 847 mr r5,r3 848 addi r3,r1,STACK_FRAME_OVERHEAD 849 lwz r4,_DAR(r1) 850 bl .bad_page_fault 851 b .ret_from_except 852 85313: b .ret_from_except_lite 854 855/* We have a page fault that hash_page could handle but HV refused 856 * the PTE insertion 857 */ 85812: bl .save_nvgprs 859 mr r5,r3 860 addi r3,r1,STACK_FRAME_OVERHEAD 861 ld r4,_DAR(r1) 862 bl .low_hash_fault 863 b .ret_from_except 864 865/* 866 * We come here as a result of a DSI at a point where we don't want 867 * to call hash_page, such as when we are accessing memory (possibly 868 * user memory) inside a PMU interrupt that occurred while interrupts 869 * were soft-disabled. We want to invoke the exception handler for 870 * the access, or panic if there isn't a handler. 871 */ 87277: bl .save_nvgprs 873 mr r4,r3 874 addi r3,r1,STACK_FRAME_OVERHEAD 875 li r5,SIGSEGV 876 bl .bad_page_fault 877 b .ret_from_except 878 879 /* here we have a segment miss */ 880do_ste_alloc: 881 bl .ste_allocate /* try to insert stab entry */ 882 cmpdi r3,0 883 bne- handle_page_fault 884 b fast_exception_return 885 886/* 887 * r13 points to the PACA, r9 contains the saved CR, 888 * r11 and r12 contain the saved SRR0 and SRR1. 889 * r9 - r13 are saved in paca->exslb. 890 * We assume we aren't going to take any exceptions during this procedure. 891 * We assume (DAR >> 60) == 0xc. 892 */ 893 .align 7 894_GLOBAL(do_stab_bolted) 895 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */ 896 std r11,PACA_EXSLB+EX_SRR0(r13) /* save SRR0 in exc. frame */ 897 898 /* Hash to the primary group */ 899 ld r10,PACASTABVIRT(r13) 900 mfspr r11,SPRN_DAR 901 srdi r11,r11,28 902 rldimi r10,r11,7,52 /* r10 = first ste of the group */ 903 904 /* Calculate VSID */ 905 /* This is a kernel address, so protovsid = ESID */ 906 ASM_VSID_SCRAMBLE(r11, r9, 256M) 907 rldic r9,r11,12,16 /* r9 = vsid << 12 */ 908 909 /* Search the primary group for a free entry */ 9101: ld r11,0(r10) /* Test valid bit of the current ste */ 911 andi. r11,r11,0x80 912 beq 2f 913 addi r10,r10,16 914 andi. r11,r10,0x70 915 bne 1b 916 917 /* Stick for only searching the primary group for now. */ 918 /* At least for now, we use a very simple random castout scheme */ 919 /* Use the TB as a random number ; OR in 1 to avoid entry 0 */ 920 mftb r11 921 rldic r11,r11,4,57 /* r11 = (r11 << 4) & 0x70 */ 922 ori r11,r11,0x10 923 924 /* r10 currently points to an ste one past the group of interest */ 925 /* make it point to the randomly selected entry */ 926 subi r10,r10,128 927 or r10,r10,r11 /* r10 is the entry to invalidate */ 928 929 isync /* mark the entry invalid */ 930 ld r11,0(r10) 931 rldicl r11,r11,56,1 /* clear the valid bit */ 932 rotldi r11,r11,8 933 std r11,0(r10) 934 sync 935 936 clrrdi r11,r11,28 /* Get the esid part of the ste */ 937 slbie r11 938 9392: std r9,8(r10) /* Store the vsid part of the ste */ 940 eieio 941 942 mfspr r11,SPRN_DAR /* Get the new esid */ 943 clrrdi r11,r11,28 /* Permits a full 32b of ESID */ 944 ori r11,r11,0x90 /* Turn on valid and kp */ 945 std r11,0(r10) /* Put new entry back into the stab */ 946 947 sync 948 949 /* All done -- return from exception. */ 950 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */ 951 ld r11,PACA_EXSLB+EX_SRR0(r13) /* get saved SRR0 */ 952 953 andi. r10,r12,MSR_RI 954 beq- unrecov_slb 955 956 mtcrf 0x80,r9 /* restore CR */ 957 958 mfmsr r10 959 clrrdi r10,r10,2 960 mtmsrd r10,1 961 962 mtspr SPRN_SRR0,r11 963 mtspr SPRN_SRR1,r12 964 ld r9,PACA_EXSLB+EX_R9(r13) 965 ld r10,PACA_EXSLB+EX_R10(r13) 966 ld r11,PACA_EXSLB+EX_R11(r13) 967 ld r12,PACA_EXSLB+EX_R12(r13) 968 ld r13,PACA_EXSLB+EX_R13(r13) 969 rfid 970 b . /* prevent speculative execution */ 971 972/* 973 * Space for CPU0's segment table. 974 * 975 * On iSeries, the hypervisor must fill in at least one entry before 976 * we get control (with relocate on). The address is given to the hv 977 * as a page number (see xLparMap below), so this must be at a 978 * fixed address (the linker can't compute (u64)&initial_stab >> 979 * PAGE_SHIFT). 980 */ 981 . = STAB0_OFFSET /* 0x6000 */ 982 .globl initial_stab 983initial_stab: 984 .space 4096 985 986#ifdef CONFIG_PPC_PSERIES 987/* 988 * Data area reserved for FWNMI option. 989 * This address (0x7000) is fixed by the RPA. 990 */ 991 .= 0x7000 992 .globl fwnmi_data_area 993fwnmi_data_area: 994#endif /* CONFIG_PPC_PSERIES */ 995 996 /* iSeries does not use the FWNMI stuff, so it is safe to put 997 * this here, even if we later allow kernels that will boot on 998 * both pSeries and iSeries */ 999#ifdef CONFIG_PPC_ISERIES 1000 . = LPARMAP_PHYS 1001 .globl xLparMap 1002xLparMap: 1003 .quad HvEsidsToMap /* xNumberEsids */ 1004 .quad HvRangesToMap /* xNumberRanges */ 1005 .quad STAB0_PAGE /* xSegmentTableOffs */ 1006 .zero 40 /* xRsvd */ 1007 /* xEsids (HvEsidsToMap entries of 2 quads) */ 1008 .quad PAGE_OFFSET_ESID /* xKernelEsid */ 1009 .quad PAGE_OFFSET_VSID /* xKernelVsid */ 1010 .quad VMALLOC_START_ESID /* xKernelEsid */ 1011 .quad VMALLOC_START_VSID /* xKernelVsid */ 1012 /* xRanges (HvRangesToMap entries of 3 quads) */ 1013 .quad HvPagesToMap /* xPages */ 1014 .quad 0 /* xOffset */ 1015 .quad PAGE_OFFSET_VSID << (SID_SHIFT - HW_PAGE_SHIFT) /* xVPN */ 1016 1017#endif /* CONFIG_PPC_ISERIES */ 1018 1019#ifdef CONFIG_PPC_PSERIES 1020 . = 0x8000 1021#endif /* CONFIG_PPC_PSERIES */ 1022