1/* 2 * PowerPC version 3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 4 * 5 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP 6 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu> 7 * Adapted for Power Macintosh by Paul Mackerras. 8 * Low-level exception handlers and MMU support 9 * rewritten by Paul Mackerras. 10 * Copyright (C) 1996 Paul Mackerras. 11 * 12 * Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and 13 * Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com 14 * 15 * This file contains the low-level support and setup for the 16 * PowerPC-64 platform, including trap and interrupt dispatch. 17 * 18 * This program is free software; you can redistribute it and/or 19 * modify it under the terms of the GNU General Public License 20 * as published by the Free Software Foundation; either version 21 * 2 of the License, or (at your option) any later version. 22 */ 23 24#include <linux/threads.h> 25#include <asm/reg.h> 26#include <asm/page.h> 27#include <asm/mmu.h> 28#include <asm/ppc_asm.h> 29#include <asm/asm-offsets.h> 30#include <asm/bug.h> 31#include <asm/cputable.h> 32#include <asm/setup.h> 33#include <asm/hvcall.h> 34#include <asm/iseries/lpar_map.h> 35#include <asm/thread_info.h> 36#include <asm/firmware.h> 37#include <asm/page_64.h> 38#include <asm/exception.h> 39#include <asm/irqflags.h> 40 41/* 42 * We layout physical memory as follows: 43 * 0x0000 - 0x00ff : Secondary processor spin code 44 * 0x0100 - 0x2fff : pSeries Interrupt prologs 45 * 0x3000 - 0x5fff : interrupt support, iSeries and common interrupt prologs 46 * 0x6000 - 0x6fff : Initial (CPU0) segment table 47 * 0x7000 - 0x7fff : FWNMI data area 48 * 0x8000 - : Early init and support code 49 */ 50 51/* 52 * SPRG Usage 53 * 54 * Register Definition 55 * 56 * SPRG0 reserved for hypervisor 57 * SPRG1 temp - used to save gpr 58 * SPRG2 temp - used to save gpr 59 * SPRG3 virt addr of paca 60 */ 61 62/* 63 * Entering into this code we make the following assumptions: 64 * For pSeries: 65 * 1. The MMU is off & open firmware is running in real mode. 66 * 2. The kernel is entered at __start 67 * 68 * For iSeries: 69 * 1. The MMU is on (as it always is for iSeries) 70 * 2. The kernel is entered at system_reset_iSeries 71 */ 72 73 .text 74 .globl _stext 75_stext: 76_GLOBAL(__start) 77 /* NOP this out unconditionally */ 78BEGIN_FTR_SECTION 79 b .__start_initialization_multiplatform 80END_FTR_SECTION(0, 1) 81 82 /* Catch branch to 0 in real mode */ 83 trap 84 85 /* Secondary processors spin on this value until it becomes nonzero. 86 * When it does it contains the real address of the descriptor 87 * of the function that the cpu should jump to to continue 88 * initialization. 89 */ 90 .globl __secondary_hold_spinloop 91__secondary_hold_spinloop: 92 .llong 0x0 93 94 /* Secondary processors write this value with their cpu # */ 95 /* after they enter the spin loop immediately below. */ 96 .globl __secondary_hold_acknowledge 97__secondary_hold_acknowledge: 98 .llong 0x0 99 100#ifdef CONFIG_PPC_ISERIES 101 /* 102 * At offset 0x20, there is a pointer to iSeries LPAR data. 103 * This is required by the hypervisor 104 */ 105 . = 0x20 106 .llong hvReleaseData-KERNELBASE 107#endif /* CONFIG_PPC_ISERIES */ 108 109#ifdef CONFIG_CRASH_DUMP 110 /* This flag is set to 1 by a loader if the kernel should run 111 * at the loaded address instead of the linked address. This 112 * is used by kexec-tools to keep the the kdump kernel in the 113 * crash_kernel region. The loader is responsible for 114 * observing the alignment requirement. 115 */ 116 /* Do not move this variable as kexec-tools knows about it. */ 117 . = 0x5c 118 .globl __run_at_load 119__run_at_load: 120 .long 0x72756e30 /* "run0" -- relocate to 0 by default */ 121#endif 122 123 . = 0x60 124/* 125 * The following code is used to hold secondary processors 126 * in a spin loop after they have entered the kernel, but 127 * before the bulk of the kernel has been relocated. This code 128 * is relocated to physical address 0x60 before prom_init is run. 129 * All of it must fit below the first exception vector at 0x100. 130 * Use .globl here not _GLOBAL because we want __secondary_hold 131 * to be the actual text address, not a descriptor. 132 */ 133 .globl __secondary_hold 134__secondary_hold: 135 mfmsr r24 136 ori r24,r24,MSR_RI 137 mtmsrd r24 /* RI on */ 138 139 /* Grab our physical cpu number */ 140 mr r24,r3 141 142 /* Tell the master cpu we're here */ 143 /* Relocation is off & we are located at an address less */ 144 /* than 0x100, so only need to grab low order offset. */ 145 std r24,__secondary_hold_acknowledge-_stext(0) 146 sync 147 148 /* All secondary cpus wait here until told to start. */ 149100: ld r4,__secondary_hold_spinloop-_stext(0) 150 cmpdi 0,r4,0 151 beq 100b 152 153#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC) 154 ld r4,0(r4) /* deref function descriptor */ 155 mtctr r4 156 mr r3,r24 157 bctr 158#else 159 BUG_OPCODE 160#endif 161 162/* This value is used to mark exception frames on the stack. */ 163 .section ".toc","aw" 164exception_marker: 165 .tc ID_72656773_68657265[TC],0x7265677368657265 166 .text 167 168/* 169 * This is the start of the interrupt handlers for pSeries 170 * This code runs with relocation off. 171 * Code from here to __end_interrupts gets copied down to real 172 * address 0x100 when we are running a relocatable kernel. 173 * Therefore any relative branches in this section must only 174 * branch to labels in this section. 175 */ 176 . = 0x100 177 .globl __start_interrupts 178__start_interrupts: 179 180 STD_EXCEPTION_PSERIES(0x100, system_reset) 181 182 . = 0x200 183_machine_check_pSeries: 184 HMT_MEDIUM 185 mtspr SPRN_SPRG1,r13 /* save r13 */ 186 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common) 187 188 . = 0x300 189 .globl data_access_pSeries 190data_access_pSeries: 191 HMT_MEDIUM 192 mtspr SPRN_SPRG1,r13 193BEGIN_FTR_SECTION 194 mtspr SPRN_SPRG2,r12 195 mfspr r13,SPRN_DAR 196 mfspr r12,SPRN_DSISR 197 srdi r13,r13,60 198 rlwimi r13,r12,16,0x20 199 mfcr r12 200 cmpwi r13,0x2c 201 beq do_stab_bolted_pSeries 202 mtcrf 0x80,r12 203 mfspr r12,SPRN_SPRG2 204END_FTR_SECTION_IFCLR(CPU_FTR_SLB) 205 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common) 206 207 . = 0x380 208 .globl data_access_slb_pSeries 209data_access_slb_pSeries: 210 HMT_MEDIUM 211 mtspr SPRN_SPRG1,r13 212 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */ 213 std r3,PACA_EXSLB+EX_R3(r13) 214 mfspr r3,SPRN_DAR 215 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */ 216 mfcr r9 217#ifdef __DISABLED__ 218 /* Keep that around for when we re-implement dynamic VSIDs */ 219 cmpdi r3,0 220 bge slb_miss_user_pseries 221#endif /* __DISABLED__ */ 222 std r10,PACA_EXSLB+EX_R10(r13) 223 std r11,PACA_EXSLB+EX_R11(r13) 224 std r12,PACA_EXSLB+EX_R12(r13) 225 mfspr r10,SPRN_SPRG1 226 std r10,PACA_EXSLB+EX_R13(r13) 227 mfspr r12,SPRN_SRR1 /* and SRR1 */ 228#ifndef CONFIG_RELOCATABLE 229 b .slb_miss_realmode 230#else 231 /* 232 * We can't just use a direct branch to .slb_miss_realmode 233 * because the distance from here to there depends on where 234 * the kernel ends up being put. 235 */ 236 mfctr r11 237 ld r10,PACAKBASE(r13) 238 LOAD_HANDLER(r10, .slb_miss_realmode) 239 mtctr r10 240 bctr 241#endif 242 243 STD_EXCEPTION_PSERIES(0x400, instruction_access) 244 245 . = 0x480 246 .globl instruction_access_slb_pSeries 247instruction_access_slb_pSeries: 248 HMT_MEDIUM 249 mtspr SPRN_SPRG1,r13 250 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */ 251 std r3,PACA_EXSLB+EX_R3(r13) 252 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */ 253 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */ 254 mfcr r9 255#ifdef __DISABLED__ 256 /* Keep that around for when we re-implement dynamic VSIDs */ 257 cmpdi r3,0 258 bge slb_miss_user_pseries 259#endif /* __DISABLED__ */ 260 std r10,PACA_EXSLB+EX_R10(r13) 261 std r11,PACA_EXSLB+EX_R11(r13) 262 std r12,PACA_EXSLB+EX_R12(r13) 263 mfspr r10,SPRN_SPRG1 264 std r10,PACA_EXSLB+EX_R13(r13) 265 mfspr r12,SPRN_SRR1 /* and SRR1 */ 266#ifndef CONFIG_RELOCATABLE 267 b .slb_miss_realmode 268#else 269 mfctr r11 270 ld r10,PACAKBASE(r13) 271 LOAD_HANDLER(r10, .slb_miss_realmode) 272 mtctr r10 273 bctr 274#endif 275 276 MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt) 277 STD_EXCEPTION_PSERIES(0x600, alignment) 278 STD_EXCEPTION_PSERIES(0x700, program_check) 279 STD_EXCEPTION_PSERIES(0x800, fp_unavailable) 280 MASKABLE_EXCEPTION_PSERIES(0x900, decrementer) 281 STD_EXCEPTION_PSERIES(0xa00, trap_0a) 282 STD_EXCEPTION_PSERIES(0xb00, trap_0b) 283 284 . = 0xc00 285 .globl system_call_pSeries 286system_call_pSeries: 287 HMT_MEDIUM 288BEGIN_FTR_SECTION 289 cmpdi r0,0x1ebe 290 beq- 1f 291END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) 292 mr r9,r13 293 mfspr r13,SPRN_SPRG3 294 mfspr r11,SPRN_SRR0 295 ld r12,PACAKBASE(r13) 296 ld r10,PACAKMSR(r13) 297 LOAD_HANDLER(r12, system_call_entry) 298 mtspr SPRN_SRR0,r12 299 mfspr r12,SPRN_SRR1 300 mtspr SPRN_SRR1,r10 301 rfid 302 b . /* prevent speculative execution */ 303 304/* Fast LE/BE switch system call */ 3051: mfspr r12,SPRN_SRR1 306 xori r12,r12,MSR_LE 307 mtspr SPRN_SRR1,r12 308 rfid /* return to userspace */ 309 b . 310 311 STD_EXCEPTION_PSERIES(0xd00, single_step) 312 STD_EXCEPTION_PSERIES(0xe00, trap_0e) 313 314 /* We need to deal with the Altivec unavailable exception 315 * here which is at 0xf20, thus in the middle of the 316 * prolog code of the PerformanceMonitor one. A little 317 * trickery is thus necessary 318 */ 319 . = 0xf00 320 b performance_monitor_pSeries 321 322 . = 0xf20 323 b altivec_unavailable_pSeries 324 325 . = 0xf40 326 b vsx_unavailable_pSeries 327 328#ifdef CONFIG_CBE_RAS 329 HSTD_EXCEPTION_PSERIES(0x1200, cbe_system_error) 330#endif /* CONFIG_CBE_RAS */ 331 STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint) 332#ifdef CONFIG_CBE_RAS 333 HSTD_EXCEPTION_PSERIES(0x1600, cbe_maintenance) 334#endif /* CONFIG_CBE_RAS */ 335 STD_EXCEPTION_PSERIES(0x1700, altivec_assist) 336#ifdef CONFIG_CBE_RAS 337 HSTD_EXCEPTION_PSERIES(0x1800, cbe_thermal) 338#endif /* CONFIG_CBE_RAS */ 339 340 . = 0x3000 341 342/*** pSeries interrupt support ***/ 343 344 /* moved from 0xf00 */ 345 STD_EXCEPTION_PSERIES(., performance_monitor) 346 STD_EXCEPTION_PSERIES(., altivec_unavailable) 347 STD_EXCEPTION_PSERIES(., vsx_unavailable) 348 349/* 350 * An interrupt came in while soft-disabled; clear EE in SRR1, 351 * clear paca->hard_enabled and return. 352 */ 353masked_interrupt: 354 stb r10,PACAHARDIRQEN(r13) 355 mtcrf 0x80,r9 356 ld r9,PACA_EXGEN+EX_R9(r13) 357 mfspr r10,SPRN_SRR1 358 rldicl r10,r10,48,1 /* clear MSR_EE */ 359 rotldi r10,r10,16 360 mtspr SPRN_SRR1,r10 361 ld r10,PACA_EXGEN+EX_R10(r13) 362 mfspr r13,SPRN_SPRG1 363 rfid 364 b . 365 366 .align 7 367do_stab_bolted_pSeries: 368 mtcrf 0x80,r12 369 mfspr r12,SPRN_SPRG2 370 EXCEPTION_PROLOG_PSERIES(PACA_EXSLB, .do_stab_bolted) 371 372#ifdef CONFIG_PPC_PSERIES 373/* 374 * Vectors for the FWNMI option. Share common code. 375 */ 376 .globl system_reset_fwnmi 377 .align 7 378system_reset_fwnmi: 379 HMT_MEDIUM 380 mtspr SPRN_SPRG1,r13 /* save r13 */ 381 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common) 382 383 .globl machine_check_fwnmi 384 .align 7 385machine_check_fwnmi: 386 HMT_MEDIUM 387 mtspr SPRN_SPRG1,r13 /* save r13 */ 388 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common) 389 390#endif /* CONFIG_PPC_PSERIES */ 391 392#ifdef __DISABLED__ 393/* 394 * This is used for when the SLB miss handler has to go virtual, 395 * which doesn't happen for now anymore but will once we re-implement 396 * dynamic VSIDs for shared page tables 397 */ 398slb_miss_user_pseries: 399 std r10,PACA_EXGEN+EX_R10(r13) 400 std r11,PACA_EXGEN+EX_R11(r13) 401 std r12,PACA_EXGEN+EX_R12(r13) 402 mfspr r10,SPRG1 403 ld r11,PACA_EXSLB+EX_R9(r13) 404 ld r12,PACA_EXSLB+EX_R3(r13) 405 std r10,PACA_EXGEN+EX_R13(r13) 406 std r11,PACA_EXGEN+EX_R9(r13) 407 std r12,PACA_EXGEN+EX_R3(r13) 408 clrrdi r12,r13,32 409 mfmsr r10 410 mfspr r11,SRR0 /* save SRR0 */ 411 ori r12,r12,slb_miss_user_common@l /* virt addr of handler */ 412 ori r10,r10,MSR_IR|MSR_DR|MSR_RI 413 mtspr SRR0,r12 414 mfspr r12,SRR1 /* and SRR1 */ 415 mtspr SRR1,r10 416 rfid 417 b . /* prevent spec. execution */ 418#endif /* __DISABLED__ */ 419 420 .align 7 421 .globl __end_interrupts 422__end_interrupts: 423 424/* 425 * Code from here down to __end_handlers is invoked from the 426 * exception prologs above. Because the prologs assemble the 427 * addresses of these handlers using the LOAD_HANDLER macro, 428 * which uses an addi instruction, these handlers must be in 429 * the first 32k of the kernel image. 430 */ 431 432/*** Common interrupt handlers ***/ 433 434 STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception) 435 436 /* 437 * Machine check is different because we use a different 438 * save area: PACA_EXMC instead of PACA_EXGEN. 439 */ 440 .align 7 441 .globl machine_check_common 442machine_check_common: 443 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC) 444 FINISH_NAP 445 DISABLE_INTS 446 bl .save_nvgprs 447 addi r3,r1,STACK_FRAME_OVERHEAD 448 bl .machine_check_exception 449 b .ret_from_except 450 451 STD_EXCEPTION_COMMON_LITE(0x900, decrementer, .timer_interrupt) 452 STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception) 453 STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception) 454 STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception) 455 STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception) 456 STD_EXCEPTION_COMMON_IDLE(0xf00, performance_monitor, .performance_monitor_exception) 457 STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception) 458#ifdef CONFIG_ALTIVEC 459 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception) 460#else 461 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception) 462#endif 463#ifdef CONFIG_CBE_RAS 464 STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception) 465 STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception) 466 STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception) 467#endif /* CONFIG_CBE_RAS */ 468 469 .align 7 470system_call_entry: 471 b system_call_common 472 473/* 474 * Here we have detected that the kernel stack pointer is bad. 475 * R9 contains the saved CR, r13 points to the paca, 476 * r10 contains the (bad) kernel stack pointer, 477 * r11 and r12 contain the saved SRR0 and SRR1. 478 * We switch to using an emergency stack, save the registers there, 479 * and call kernel_bad_stack(), which panics. 480 */ 481bad_stack: 482 ld r1,PACAEMERGSP(r13) 483 subi r1,r1,64+INT_FRAME_SIZE 484 std r9,_CCR(r1) 485 std r10,GPR1(r1) 486 std r11,_NIP(r1) 487 std r12,_MSR(r1) 488 mfspr r11,SPRN_DAR 489 mfspr r12,SPRN_DSISR 490 std r11,_DAR(r1) 491 std r12,_DSISR(r1) 492 mflr r10 493 mfctr r11 494 mfxer r12 495 std r10,_LINK(r1) 496 std r11,_CTR(r1) 497 std r12,_XER(r1) 498 SAVE_GPR(0,r1) 499 SAVE_GPR(2,r1) 500 SAVE_4GPRS(3,r1) 501 SAVE_2GPRS(7,r1) 502 SAVE_10GPRS(12,r1) 503 SAVE_10GPRS(22,r1) 504 lhz r12,PACA_TRAP_SAVE(r13) 505 std r12,_TRAP(r1) 506 addi r11,r1,INT_FRAME_SIZE 507 std r11,0(r1) 508 li r12,0 509 std r12,0(r11) 510 ld r2,PACATOC(r13) 5111: addi r3,r1,STACK_FRAME_OVERHEAD 512 bl .kernel_bad_stack 513 b 1b 514 515/* 516 * Here r13 points to the paca, r9 contains the saved CR, 517 * SRR0 and SRR1 are saved in r11 and r12, 518 * r9 - r13 are saved in paca->exgen. 519 */ 520 .align 7 521 .globl data_access_common 522data_access_common: 523 mfspr r10,SPRN_DAR 524 std r10,PACA_EXGEN+EX_DAR(r13) 525 mfspr r10,SPRN_DSISR 526 stw r10,PACA_EXGEN+EX_DSISR(r13) 527 EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN) 528 ld r3,PACA_EXGEN+EX_DAR(r13) 529 lwz r4,PACA_EXGEN+EX_DSISR(r13) 530 li r5,0x300 531 b .do_hash_page /* Try to handle as hpte fault */ 532 533 .align 7 534 .globl instruction_access_common 535instruction_access_common: 536 EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN) 537 ld r3,_NIP(r1) 538 andis. r4,r12,0x5820 539 li r5,0x400 540 b .do_hash_page /* Try to handle as hpte fault */ 541 542/* 543 * Here is the common SLB miss user that is used when going to virtual 544 * mode for SLB misses, that is currently not used 545 */ 546#ifdef __DISABLED__ 547 .align 7 548 .globl slb_miss_user_common 549slb_miss_user_common: 550 mflr r10 551 std r3,PACA_EXGEN+EX_DAR(r13) 552 stw r9,PACA_EXGEN+EX_CCR(r13) 553 std r10,PACA_EXGEN+EX_LR(r13) 554 std r11,PACA_EXGEN+EX_SRR0(r13) 555 bl .slb_allocate_user 556 557 ld r10,PACA_EXGEN+EX_LR(r13) 558 ld r3,PACA_EXGEN+EX_R3(r13) 559 lwz r9,PACA_EXGEN+EX_CCR(r13) 560 ld r11,PACA_EXGEN+EX_SRR0(r13) 561 mtlr r10 562 beq- slb_miss_fault 563 564 andi. r10,r12,MSR_RI /* check for unrecoverable exception */ 565 beq- unrecov_user_slb 566 mfmsr r10 567 568.machine push 569.machine "power4" 570 mtcrf 0x80,r9 571.machine pop 572 573 clrrdi r10,r10,2 /* clear RI before setting SRR0/1 */ 574 mtmsrd r10,1 575 576 mtspr SRR0,r11 577 mtspr SRR1,r12 578 579 ld r9,PACA_EXGEN+EX_R9(r13) 580 ld r10,PACA_EXGEN+EX_R10(r13) 581 ld r11,PACA_EXGEN+EX_R11(r13) 582 ld r12,PACA_EXGEN+EX_R12(r13) 583 ld r13,PACA_EXGEN+EX_R13(r13) 584 rfid 585 b . 586 587slb_miss_fault: 588 EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN) 589 ld r4,PACA_EXGEN+EX_DAR(r13) 590 li r5,0 591 std r4,_DAR(r1) 592 std r5,_DSISR(r1) 593 b handle_page_fault 594 595unrecov_user_slb: 596 EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN) 597 DISABLE_INTS 598 bl .save_nvgprs 5991: addi r3,r1,STACK_FRAME_OVERHEAD 600 bl .unrecoverable_exception 601 b 1b 602 603#endif /* __DISABLED__ */ 604 605 606/* 607 * r13 points to the PACA, r9 contains the saved CR, 608 * r12 contain the saved SRR1, SRR0 is still ready for return 609 * r3 has the faulting address 610 * r9 - r13 are saved in paca->exslb. 611 * r3 is saved in paca->slb_r3 612 * We assume we aren't going to take any exceptions during this procedure. 613 */ 614_GLOBAL(slb_miss_realmode) 615 mflr r10 616#ifdef CONFIG_RELOCATABLE 617 mtctr r11 618#endif 619 620 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */ 621 std r10,PACA_EXSLB+EX_LR(r13) /* save LR */ 622 623 bl .slb_allocate_realmode 624 625 /* All done -- return from exception. */ 626 627 ld r10,PACA_EXSLB+EX_LR(r13) 628 ld r3,PACA_EXSLB+EX_R3(r13) 629 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */ 630#ifdef CONFIG_PPC_ISERIES 631BEGIN_FW_FTR_SECTION 632 ld r11,PACALPPACAPTR(r13) 633 ld r11,LPPACASRR0(r11) /* get SRR0 value */ 634END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 635#endif /* CONFIG_PPC_ISERIES */ 636 637 mtlr r10 638 639 andi. r10,r12,MSR_RI /* check for unrecoverable exception */ 640 beq- 2f 641 642.machine push 643.machine "power4" 644 mtcrf 0x80,r9 645 mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */ 646.machine pop 647 648#ifdef CONFIG_PPC_ISERIES 649BEGIN_FW_FTR_SECTION 650 mtspr SPRN_SRR0,r11 651 mtspr SPRN_SRR1,r12 652END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 653#endif /* CONFIG_PPC_ISERIES */ 654 ld r9,PACA_EXSLB+EX_R9(r13) 655 ld r10,PACA_EXSLB+EX_R10(r13) 656 ld r11,PACA_EXSLB+EX_R11(r13) 657 ld r12,PACA_EXSLB+EX_R12(r13) 658 ld r13,PACA_EXSLB+EX_R13(r13) 659 rfid 660 b . /* prevent speculative execution */ 661 6622: 663#ifdef CONFIG_PPC_ISERIES 664BEGIN_FW_FTR_SECTION 665 b unrecov_slb 666END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 667#endif /* CONFIG_PPC_ISERIES */ 668 mfspr r11,SPRN_SRR0 669 ld r10,PACAKBASE(r13) 670 LOAD_HANDLER(r10,unrecov_slb) 671 mtspr SPRN_SRR0,r10 672 ld r10,PACAKMSR(r13) 673 mtspr SPRN_SRR1,r10 674 rfid 675 b . 676 677unrecov_slb: 678 EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB) 679 DISABLE_INTS 680 bl .save_nvgprs 6811: addi r3,r1,STACK_FRAME_OVERHEAD 682 bl .unrecoverable_exception 683 b 1b 684 685 .align 7 686 .globl hardware_interrupt_common 687 .globl hardware_interrupt_entry 688hardware_interrupt_common: 689 EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN) 690 FINISH_NAP 691hardware_interrupt_entry: 692 DISABLE_INTS 693BEGIN_FTR_SECTION 694 bl .ppc64_runlatch_on 695END_FTR_SECTION_IFSET(CPU_FTR_CTRL) 696 addi r3,r1,STACK_FRAME_OVERHEAD 697 bl .do_IRQ 698 b .ret_from_except_lite 699 700#ifdef CONFIG_PPC_970_NAP 701power4_fixup_nap: 702 andc r9,r9,r10 703 std r9,TI_LOCAL_FLAGS(r11) 704 ld r10,_LINK(r1) /* make idle task do the */ 705 std r10,_NIP(r1) /* equivalent of a blr */ 706 blr 707#endif 708 709 .align 7 710 .globl alignment_common 711alignment_common: 712 mfspr r10,SPRN_DAR 713 std r10,PACA_EXGEN+EX_DAR(r13) 714 mfspr r10,SPRN_DSISR 715 stw r10,PACA_EXGEN+EX_DSISR(r13) 716 EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN) 717 ld r3,PACA_EXGEN+EX_DAR(r13) 718 lwz r4,PACA_EXGEN+EX_DSISR(r13) 719 std r3,_DAR(r1) 720 std r4,_DSISR(r1) 721 bl .save_nvgprs 722 addi r3,r1,STACK_FRAME_OVERHEAD 723 ENABLE_INTS 724 bl .alignment_exception 725 b .ret_from_except 726 727 .align 7 728 .globl program_check_common 729program_check_common: 730 EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN) 731 bl .save_nvgprs 732 addi r3,r1,STACK_FRAME_OVERHEAD 733 ENABLE_INTS 734 bl .program_check_exception 735 b .ret_from_except 736 737 .align 7 738 .globl fp_unavailable_common 739fp_unavailable_common: 740 EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN) 741 bne 1f /* if from user, just load it up */ 742 bl .save_nvgprs 743 addi r3,r1,STACK_FRAME_OVERHEAD 744 ENABLE_INTS 745 bl .kernel_fp_unavailable_exception 746 BUG_OPCODE 7471: bl .load_up_fpu 748 b fast_exception_return 749 750 .align 7 751 .globl altivec_unavailable_common 752altivec_unavailable_common: 753 EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN) 754#ifdef CONFIG_ALTIVEC 755BEGIN_FTR_SECTION 756 beq 1f 757 bl .load_up_altivec 758 b fast_exception_return 7591: 760END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 761#endif 762 bl .save_nvgprs 763 addi r3,r1,STACK_FRAME_OVERHEAD 764 ENABLE_INTS 765 bl .altivec_unavailable_exception 766 b .ret_from_except 767 768 .align 7 769 .globl vsx_unavailable_common 770vsx_unavailable_common: 771 EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN) 772#ifdef CONFIG_VSX 773BEGIN_FTR_SECTION 774 bne .load_up_vsx 7751: 776END_FTR_SECTION_IFSET(CPU_FTR_VSX) 777#endif 778 bl .save_nvgprs 779 addi r3,r1,STACK_FRAME_OVERHEAD 780 ENABLE_INTS 781 bl .vsx_unavailable_exception 782 b .ret_from_except 783 784 .align 7 785 .globl __end_handlers 786__end_handlers: 787 788/* 789 * Return from an exception with minimal checks. 790 * The caller is assumed to have done EXCEPTION_PROLOG_COMMON. 791 * If interrupts have been enabled, or anything has been 792 * done that might have changed the scheduling status of 793 * any task or sent any task a signal, you should use 794 * ret_from_except or ret_from_except_lite instead of this. 795 */ 796fast_exc_return_irq: /* restores irq state too */ 797 ld r3,SOFTE(r1) 798 TRACE_AND_RESTORE_IRQ(r3); 799 ld r12,_MSR(r1) 800 rldicl r4,r12,49,63 /* get MSR_EE to LSB */ 801 stb r4,PACAHARDIRQEN(r13) /* restore paca->hard_enabled */ 802 b 1f 803 804 .globl fast_exception_return 805fast_exception_return: 806 ld r12,_MSR(r1) 8071: ld r11,_NIP(r1) 808 andi. r3,r12,MSR_RI /* check if RI is set */ 809 beq- unrecov_fer 810 811#ifdef CONFIG_VIRT_CPU_ACCOUNTING 812 andi. r3,r12,MSR_PR 813 beq 2f 814 ACCOUNT_CPU_USER_EXIT(r3, r4) 8152: 816#endif 817 818 ld r3,_CCR(r1) 819 ld r4,_LINK(r1) 820 ld r5,_CTR(r1) 821 ld r6,_XER(r1) 822 mtcr r3 823 mtlr r4 824 mtctr r5 825 mtxer r6 826 REST_GPR(0, r1) 827 REST_8GPRS(2, r1) 828 829 mfmsr r10 830 rldicl r10,r10,48,1 /* clear EE */ 831 rldicr r10,r10,16,61 /* clear RI (LE is 0 already) */ 832 mtmsrd r10,1 833 834 mtspr SPRN_SRR1,r12 835 mtspr SPRN_SRR0,r11 836 REST_4GPRS(10, r1) 837 ld r1,GPR1(r1) 838 rfid 839 b . /* prevent speculative execution */ 840 841unrecov_fer: 842 bl .save_nvgprs 8431: addi r3,r1,STACK_FRAME_OVERHEAD 844 bl .unrecoverable_exception 845 b 1b 846 847#ifdef CONFIG_ALTIVEC 848/* 849 * load_up_altivec(unused, unused, tsk) 850 * Disable VMX for the task which had it previously, 851 * and save its vector registers in its thread_struct. 852 * Enables the VMX for use in the kernel on return. 853 * On SMP we know the VMX is free, since we give it up every 854 * switch (ie, no lazy save of the vector registers). 855 * On entry: r13 == 'current' && last_task_used_altivec != 'current' 856 */ 857_STATIC(load_up_altivec) 858 mfmsr r5 /* grab the current MSR */ 859 oris r5,r5,MSR_VEC@h 860 mtmsrd r5 /* enable use of VMX now */ 861 isync 862 863/* 864 * For SMP, we don't do lazy VMX switching because it just gets too 865 * horrendously complex, especially when a task switches from one CPU 866 * to another. Instead we call giveup_altvec in switch_to. 867 * VRSAVE isn't dealt with here, that is done in the normal context 868 * switch code. Note that we could rely on vrsave value to eventually 869 * avoid saving all of the VREGs here... 870 */ 871#ifndef CONFIG_SMP 872 ld r3,last_task_used_altivec@got(r2) 873 ld r4,0(r3) 874 cmpdi 0,r4,0 875 beq 1f 876 /* Save VMX state to last_task_used_altivec's THREAD struct */ 877 addi r4,r4,THREAD 878 SAVE_32VRS(0,r5,r4) 879 mfvscr vr0 880 li r10,THREAD_VSCR 881 stvx vr0,r10,r4 882 /* Disable VMX for last_task_used_altivec */ 883 ld r5,PT_REGS(r4) 884 ld r4,_MSR-STACK_FRAME_OVERHEAD(r5) 885 lis r6,MSR_VEC@h 886 andc r4,r4,r6 887 std r4,_MSR-STACK_FRAME_OVERHEAD(r5) 8881: 889#endif /* CONFIG_SMP */ 890 /* Hack: if we get an altivec unavailable trap with VRSAVE 891 * set to all zeros, we assume this is a broken application 892 * that fails to set it properly, and thus we switch it to 893 * all 1's 894 */ 895 mfspr r4,SPRN_VRSAVE 896 cmpdi 0,r4,0 897 bne+ 1f 898 li r4,-1 899 mtspr SPRN_VRSAVE,r4 9001: 901 /* enable use of VMX after return */ 902 ld r4,PACACURRENT(r13) 903 addi r5,r4,THREAD /* Get THREAD */ 904 oris r12,r12,MSR_VEC@h 905 std r12,_MSR(r1) 906 li r4,1 907 li r10,THREAD_VSCR 908 stw r4,THREAD_USED_VR(r5) 909 lvx vr0,r10,r5 910 mtvscr vr0 911 REST_32VRS(0,r4,r5) 912#ifndef CONFIG_SMP 913 /* Update last_task_used_math to 'current' */ 914 subi r4,r5,THREAD /* Back to 'current' */ 915 std r4,0(r3) 916#endif /* CONFIG_SMP */ 917 /* restore registers and return */ 918 blr 919#endif /* CONFIG_ALTIVEC */ 920 921#ifdef CONFIG_VSX 922/* 923 * load_up_vsx(unused, unused, tsk) 924 * Disable VSX for the task which had it previously, 925 * and save its vector registers in its thread_struct. 926 * Reuse the fp and vsx saves, but first check to see if they have 927 * been saved already. 928 * On entry: r13 == 'current' && last_task_used_vsx != 'current' 929 */ 930_STATIC(load_up_vsx) 931/* Load FP and VSX registers if they haven't been done yet */ 932 andi. r5,r12,MSR_FP 933 beql+ load_up_fpu /* skip if already loaded */ 934 andis. r5,r12,MSR_VEC@h 935 beql+ load_up_altivec /* skip if already loaded */ 936 937#ifndef CONFIG_SMP 938 ld r3,last_task_used_vsx@got(r2) 939 ld r4,0(r3) 940 cmpdi 0,r4,0 941 beq 1f 942 /* Disable VSX for last_task_used_vsx */ 943 addi r4,r4,THREAD 944 ld r5,PT_REGS(r4) 945 ld r4,_MSR-STACK_FRAME_OVERHEAD(r5) 946 lis r6,MSR_VSX@h 947 andc r6,r4,r6 948 std r6,_MSR-STACK_FRAME_OVERHEAD(r5) 9491: 950#endif /* CONFIG_SMP */ 951 ld r4,PACACURRENT(r13) 952 addi r4,r4,THREAD /* Get THREAD */ 953 li r6,1 954 stw r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */ 955 /* enable use of VSX after return */ 956 oris r12,r12,MSR_VSX@h 957 std r12,_MSR(r1) 958#ifndef CONFIG_SMP 959 /* Update last_task_used_math to 'current' */ 960 ld r4,PACACURRENT(r13) 961 std r4,0(r3) 962#endif /* CONFIG_SMP */ 963 b fast_exception_return 964#endif /* CONFIG_VSX */ 965 966/* 967 * Hash table stuff 968 */ 969 .align 7 970_STATIC(do_hash_page) 971 std r3,_DAR(r1) 972 std r4,_DSISR(r1) 973 974 andis. r0,r4,0xa450 /* weird error? */ 975 bne- handle_page_fault /* if not, try to insert a HPTE */ 976BEGIN_FTR_SECTION 977 andis. r0,r4,0x0020 /* Is it a segment table fault? */ 978 bne- do_ste_alloc /* If so handle it */ 979END_FTR_SECTION_IFCLR(CPU_FTR_SLB) 980 981 /* 982 * On iSeries, we soft-disable interrupts here, then 983 * hard-enable interrupts so that the hash_page code can spin on 984 * the hash_table_lock without problems on a shared processor. 985 */ 986 DISABLE_INTS 987 988 /* 989 * Currently, trace_hardirqs_off() will be called by DISABLE_INTS 990 * and will clobber volatile registers when irq tracing is enabled 991 * so we need to reload them. It may be possible to be smarter here 992 * and move the irq tracing elsewhere but let's keep it simple for 993 * now 994 */ 995#ifdef CONFIG_TRACE_IRQFLAGS 996 ld r3,_DAR(r1) 997 ld r4,_DSISR(r1) 998 ld r5,_TRAP(r1) 999 ld r12,_MSR(r1) 1000 clrrdi r5,r5,4 1001#endif /* CONFIG_TRACE_IRQFLAGS */ 1002 /* 1003 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are 1004 * accessing a userspace segment (even from the kernel). We assume 1005 * kernel addresses always have the high bit set. 1006 */ 1007 rlwinm r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */ 1008 rotldi r0,r3,15 /* Move high bit into MSR_PR posn */ 1009 orc r0,r12,r0 /* MSR_PR | ~high_bit */ 1010 rlwimi r4,r0,32-13,30,30 /* becomes _PAGE_USER access bit */ 1011 ori r4,r4,1 /* add _PAGE_PRESENT */ 1012 rlwimi r4,r5,22+2,31-2,31-2 /* Set _PAGE_EXEC if trap is 0x400 */ 1013 1014 /* 1015 * r3 contains the faulting address 1016 * r4 contains the required access permissions 1017 * r5 contains the trap number 1018 * 1019 * at return r3 = 0 for success 1020 */ 1021 bl .hash_page /* build HPTE if possible */ 1022 cmpdi r3,0 /* see if hash_page succeeded */ 1023 1024BEGIN_FW_FTR_SECTION 1025 /* 1026 * If we had interrupts soft-enabled at the point where the 1027 * DSI/ISI occurred, and an interrupt came in during hash_page, 1028 * handle it now. 1029 * We jump to ret_from_except_lite rather than fast_exception_return 1030 * because ret_from_except_lite will check for and handle pending 1031 * interrupts if necessary. 1032 */ 1033 beq 13f 1034END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 1035 1036BEGIN_FW_FTR_SECTION 1037 /* 1038 * Here we have interrupts hard-disabled, so it is sufficient 1039 * to restore paca->{soft,hard}_enable and get out. 1040 */ 1041 beq fast_exc_return_irq /* Return from exception on success */ 1042END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES) 1043 1044 /* For a hash failure, we don't bother re-enabling interrupts */ 1045 ble- 12f 1046 1047 /* 1048 * hash_page couldn't handle it, set soft interrupt enable back 1049 * to what it was before the trap. Note that .raw_local_irq_restore 1050 * handles any interrupts pending at this point. 1051 */ 1052 ld r3,SOFTE(r1) 1053 TRACE_AND_RESTORE_IRQ_PARTIAL(r3, 11f) 1054 bl .raw_local_irq_restore 1055 b 11f 1056 1057/* Here we have a page fault that hash_page can't handle. */ 1058handle_page_fault: 1059 ENABLE_INTS 106011: ld r4,_DAR(r1) 1061 ld r5,_DSISR(r1) 1062 addi r3,r1,STACK_FRAME_OVERHEAD 1063 bl .do_page_fault 1064 cmpdi r3,0 1065 beq+ 13f 1066 bl .save_nvgprs 1067 mr r5,r3 1068 addi r3,r1,STACK_FRAME_OVERHEAD 1069 lwz r4,_DAR(r1) 1070 bl .bad_page_fault 1071 b .ret_from_except 1072 107313: b .ret_from_except_lite 1074 1075/* We have a page fault that hash_page could handle but HV refused 1076 * the PTE insertion 1077 */ 107812: bl .save_nvgprs 1079 mr r5,r3 1080 addi r3,r1,STACK_FRAME_OVERHEAD 1081 ld r4,_DAR(r1) 1082 bl .low_hash_fault 1083 b .ret_from_except 1084 1085 /* here we have a segment miss */ 1086do_ste_alloc: 1087 bl .ste_allocate /* try to insert stab entry */ 1088 cmpdi r3,0 1089 bne- handle_page_fault 1090 b fast_exception_return 1091 1092/* 1093 * r13 points to the PACA, r9 contains the saved CR, 1094 * r11 and r12 contain the saved SRR0 and SRR1. 1095 * r9 - r13 are saved in paca->exslb. 1096 * We assume we aren't going to take any exceptions during this procedure. 1097 * We assume (DAR >> 60) == 0xc. 1098 */ 1099 .align 7 1100_GLOBAL(do_stab_bolted) 1101 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */ 1102 std r11,PACA_EXSLB+EX_SRR0(r13) /* save SRR0 in exc. frame */ 1103 1104 /* Hash to the primary group */ 1105 ld r10,PACASTABVIRT(r13) 1106 mfspr r11,SPRN_DAR 1107 srdi r11,r11,28 1108 rldimi r10,r11,7,52 /* r10 = first ste of the group */ 1109 1110 /* Calculate VSID */ 1111 /* This is a kernel address, so protovsid = ESID */ 1112 ASM_VSID_SCRAMBLE(r11, r9, 256M) 1113 rldic r9,r11,12,16 /* r9 = vsid << 12 */ 1114 1115 /* Search the primary group for a free entry */ 11161: ld r11,0(r10) /* Test valid bit of the current ste */ 1117 andi. r11,r11,0x80 1118 beq 2f 1119 addi r10,r10,16 1120 andi. r11,r10,0x70 1121 bne 1b 1122 1123 /* Stick for only searching the primary group for now. */ 1124 /* At least for now, we use a very simple random castout scheme */ 1125 /* Use the TB as a random number ; OR in 1 to avoid entry 0 */ 1126 mftb r11 1127 rldic r11,r11,4,57 /* r11 = (r11 << 4) & 0x70 */ 1128 ori r11,r11,0x10 1129 1130 /* r10 currently points to an ste one past the group of interest */ 1131 /* make it point to the randomly selected entry */ 1132 subi r10,r10,128 1133 or r10,r10,r11 /* r10 is the entry to invalidate */ 1134 1135 isync /* mark the entry invalid */ 1136 ld r11,0(r10) 1137 rldicl r11,r11,56,1 /* clear the valid bit */ 1138 rotldi r11,r11,8 1139 std r11,0(r10) 1140 sync 1141 1142 clrrdi r11,r11,28 /* Get the esid part of the ste */ 1143 slbie r11 1144 11452: std r9,8(r10) /* Store the vsid part of the ste */ 1146 eieio 1147 1148 mfspr r11,SPRN_DAR /* Get the new esid */ 1149 clrrdi r11,r11,28 /* Permits a full 32b of ESID */ 1150 ori r11,r11,0x90 /* Turn on valid and kp */ 1151 std r11,0(r10) /* Put new entry back into the stab */ 1152 1153 sync 1154 1155 /* All done -- return from exception. */ 1156 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */ 1157 ld r11,PACA_EXSLB+EX_SRR0(r13) /* get saved SRR0 */ 1158 1159 andi. r10,r12,MSR_RI 1160 beq- unrecov_slb 1161 1162 mtcrf 0x80,r9 /* restore CR */ 1163 1164 mfmsr r10 1165 clrrdi r10,r10,2 1166 mtmsrd r10,1 1167 1168 mtspr SPRN_SRR0,r11 1169 mtspr SPRN_SRR1,r12 1170 ld r9,PACA_EXSLB+EX_R9(r13) 1171 ld r10,PACA_EXSLB+EX_R10(r13) 1172 ld r11,PACA_EXSLB+EX_R11(r13) 1173 ld r12,PACA_EXSLB+EX_R12(r13) 1174 ld r13,PACA_EXSLB+EX_R13(r13) 1175 rfid 1176 b . /* prevent speculative execution */ 1177 1178/* 1179 * Space for CPU0's segment table. 1180 * 1181 * On iSeries, the hypervisor must fill in at least one entry before 1182 * we get control (with relocate on). The address is given to the hv 1183 * as a page number (see xLparMap below), so this must be at a 1184 * fixed address (the linker can't compute (u64)&initial_stab >> 1185 * PAGE_SHIFT). 1186 */ 1187 . = STAB0_OFFSET /* 0x6000 */ 1188 .globl initial_stab 1189initial_stab: 1190 .space 4096 1191 1192#ifdef CONFIG_PPC_PSERIES 1193/* 1194 * Data area reserved for FWNMI option. 1195 * This address (0x7000) is fixed by the RPA. 1196 */ 1197 .= 0x7000 1198 .globl fwnmi_data_area 1199fwnmi_data_area: 1200#endif /* CONFIG_PPC_PSERIES */ 1201 1202 /* iSeries does not use the FWNMI stuff, so it is safe to put 1203 * this here, even if we later allow kernels that will boot on 1204 * both pSeries and iSeries */ 1205#ifdef CONFIG_PPC_ISERIES 1206 . = LPARMAP_PHYS 1207 .globl xLparMap 1208xLparMap: 1209 .quad HvEsidsToMap /* xNumberEsids */ 1210 .quad HvRangesToMap /* xNumberRanges */ 1211 .quad STAB0_PAGE /* xSegmentTableOffs */ 1212 .zero 40 /* xRsvd */ 1213 /* xEsids (HvEsidsToMap entries of 2 quads) */ 1214 .quad PAGE_OFFSET_ESID /* xKernelEsid */ 1215 .quad PAGE_OFFSET_VSID /* xKernelVsid */ 1216 .quad VMALLOC_START_ESID /* xKernelEsid */ 1217 .quad VMALLOC_START_VSID /* xKernelVsid */ 1218 /* xRanges (HvRangesToMap entries of 3 quads) */ 1219 .quad HvPagesToMap /* xPages */ 1220 .quad 0 /* xOffset */ 1221 .quad PAGE_OFFSET_VSID << (SID_SHIFT - HW_PAGE_SHIFT) /* xVPN */ 1222 1223#endif /* CONFIG_PPC_ISERIES */ 1224 1225#ifdef CONFIG_PPC_PSERIES 1226 . = 0x8000 1227#endif /* CONFIG_PPC_PSERIES */ 1228 1229/* 1230 * On pSeries and most other platforms, secondary processors spin 1231 * in the following code. 1232 * At entry, r3 = this processor's number (physical cpu id) 1233 */ 1234_GLOBAL(generic_secondary_smp_init) 1235 mr r24,r3 1236 1237 /* turn on 64-bit mode */ 1238 bl .enable_64b_mode 1239 1240 /* get the TOC pointer (real address) */ 1241 bl .relative_toc 1242 1243 /* Set up a paca value for this processor. Since we have the 1244 * physical cpu id in r24, we need to search the pacas to find 1245 * which logical id maps to our physical one. 1246 */ 1247 LOAD_REG_ADDR(r13, paca) /* Get base vaddr of paca array */ 1248 li r5,0 /* logical cpu id */ 12491: lhz r6,PACAHWCPUID(r13) /* Load HW procid from paca */ 1250 cmpw r6,r24 /* Compare to our id */ 1251 beq 2f 1252 addi r13,r13,PACA_SIZE /* Loop to next PACA on miss */ 1253 addi r5,r5,1 1254 cmpwi r5,NR_CPUS 1255 blt 1b 1256 1257 mr r3,r24 /* not found, copy phys to r3 */ 1258 b .kexec_wait /* next kernel might do better */ 1259 12602: mtspr SPRN_SPRG3,r13 /* Save vaddr of paca in SPRG3 */ 1261 /* From now on, r24 is expected to be logical cpuid */ 1262 mr r24,r5 12633: HMT_LOW 1264 lbz r23,PACAPROCSTART(r13) /* Test if this processor should */ 1265 /* start. */ 1266 1267#ifndef CONFIG_SMP 1268 b 3b /* Never go on non-SMP */ 1269#else 1270 cmpwi 0,r23,0 1271 beq 3b /* Loop until told to go */ 1272 1273 sync /* order paca.run and cur_cpu_spec */ 1274 1275 /* See if we need to call a cpu state restore handler */ 1276 LOAD_REG_ADDR(r23, cur_cpu_spec) 1277 ld r23,0(r23) 1278 ld r23,CPU_SPEC_RESTORE(r23) 1279 cmpdi 0,r23,0 1280 beq 4f 1281 ld r23,0(r23) 1282 mtctr r23 1283 bctrl 1284 12854: /* Create a temp kernel stack for use before relocation is on. */ 1286 ld r1,PACAEMERGSP(r13) 1287 subi r1,r1,STACK_FRAME_OVERHEAD 1288 1289 b __secondary_start 1290#endif 1291 1292/* 1293 * Turn the MMU off. 1294 * Assumes we're mapped EA == RA if the MMU is on. 1295 */ 1296_STATIC(__mmu_off) 1297 mfmsr r3 1298 andi. r0,r3,MSR_IR|MSR_DR 1299 beqlr 1300 mflr r4 1301 andc r3,r3,r0 1302 mtspr SPRN_SRR0,r4 1303 mtspr SPRN_SRR1,r3 1304 sync 1305 rfid 1306 b . /* prevent speculative execution */ 1307 1308 1309/* 1310 * Here is our main kernel entry point. We support currently 2 kind of entries 1311 * depending on the value of r5. 1312 * 1313 * r5 != NULL -> OF entry, we go to prom_init, "legacy" parameter content 1314 * in r3...r7 1315 * 1316 * r5 == NULL -> kexec style entry. r3 is a physical pointer to the 1317 * DT block, r4 is a physical pointer to the kernel itself 1318 * 1319 */ 1320_GLOBAL(__start_initialization_multiplatform) 1321 /* Make sure we are running in 64 bits mode */ 1322 bl .enable_64b_mode 1323 1324 /* Get TOC pointer (current runtime address) */ 1325 bl .relative_toc 1326 1327 /* find out where we are now */ 1328 bcl 20,31,$+4 13290: mflr r26 /* r26 = runtime addr here */ 1330 addis r26,r26,(_stext - 0b)@ha 1331 addi r26,r26,(_stext - 0b)@l /* current runtime base addr */ 1332 1333 /* 1334 * Are we booted from a PROM Of-type client-interface ? 1335 */ 1336 cmpldi cr0,r5,0 1337 beq 1f 1338 b .__boot_from_prom /* yes -> prom */ 13391: 1340 /* Save parameters */ 1341 mr r31,r3 1342 mr r30,r4 1343 1344 /* Setup some critical 970 SPRs before switching MMU off */ 1345 mfspr r0,SPRN_PVR 1346 srwi r0,r0,16 1347 cmpwi r0,0x39 /* 970 */ 1348 beq 1f 1349 cmpwi r0,0x3c /* 970FX */ 1350 beq 1f 1351 cmpwi r0,0x44 /* 970MP */ 1352 beq 1f 1353 cmpwi r0,0x45 /* 970GX */ 1354 bne 2f 13551: bl .__cpu_preinit_ppc970 13562: 1357 1358 /* Switch off MMU if not already off */ 1359 bl .__mmu_off 1360 b .__after_prom_start 1361 1362_INIT_STATIC(__boot_from_prom) 1363#ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE 1364 /* Save parameters */ 1365 mr r31,r3 1366 mr r30,r4 1367 mr r29,r5 1368 mr r28,r6 1369 mr r27,r7 1370 1371 /* 1372 * Align the stack to 16-byte boundary 1373 * Depending on the size and layout of the ELF sections in the initial 1374 * boot binary, the stack pointer may be unaligned on PowerMac 1375 */ 1376 rldicr r1,r1,0,59 1377 1378#ifdef CONFIG_RELOCATABLE 1379 /* Relocate code for where we are now */ 1380 mr r3,r26 1381 bl .relocate 1382#endif 1383 1384 /* Restore parameters */ 1385 mr r3,r31 1386 mr r4,r30 1387 mr r5,r29 1388 mr r6,r28 1389 mr r7,r27 1390 1391 /* Do all of the interaction with OF client interface */ 1392 mr r8,r26 1393 bl .prom_init 1394#endif /* #CONFIG_PPC_OF_BOOT_TRAMPOLINE */ 1395 1396 /* We never return. We also hit that trap if trying to boot 1397 * from OF while CONFIG_PPC_OF_BOOT_TRAMPOLINE isn't selected */ 1398 trap 1399 1400_STATIC(__after_prom_start) 1401#ifdef CONFIG_RELOCATABLE 1402 /* process relocations for the final address of the kernel */ 1403 lis r25,PAGE_OFFSET@highest /* compute virtual base of kernel */ 1404 sldi r25,r25,32 1405#ifdef CONFIG_CRASH_DUMP 1406 lwz r7,__run_at_load-_stext(r26) 1407 cmplwi cr0,r7,1 /* kdump kernel ? - stay where we are */ 1408 bne 1f 1409 add r25,r25,r26 1410#endif 14111: mr r3,r25 1412 bl .relocate 1413#endif 1414 1415/* 1416 * We need to run with _stext at physical address PHYSICAL_START. 1417 * This will leave some code in the first 256B of 1418 * real memory, which are reserved for software use. 1419 * 1420 * Note: This process overwrites the OF exception vectors. 1421 */ 1422 li r3,0 /* target addr */ 1423 mr. r4,r26 /* In some cases the loader may */ 1424 beq 9f /* have already put us at zero */ 1425 li r6,0x100 /* Start offset, the first 0x100 */ 1426 /* bytes were copied earlier. */ 1427 1428#ifdef CONFIG_CRASH_DUMP 1429/* 1430 * Check if the kernel has to be running as relocatable kernel based on the 1431 * variable __run_at_load, if it is set the kernel is treated as relocatable 1432 * kernel, otherwise it will be moved to PHYSICAL_START 1433 */ 1434 lwz r7,__run_at_load-_stext(r26) 1435 cmplwi cr0,r7,1 1436 bne 3f 1437 1438 li r5,__end_interrupts - _stext /* just copy interrupts */ 1439 b 5f 14403: 1441#endif 1442 lis r5,(copy_to_here - _stext)@ha 1443 addi r5,r5,(copy_to_here - _stext)@l /* # bytes of memory to copy */ 1444 1445 bl .copy_and_flush /* copy the first n bytes */ 1446 /* this includes the code being */ 1447 /* executed here. */ 1448 addis r8,r3,(4f - _stext)@ha /* Jump to the copy of this code */ 1449 addi r8,r8,(4f - _stext)@l /* that we just made */ 1450 mtctr r8 1451 bctr 1452 1453p_end: .llong _end - _stext 1454 14554: /* Now copy the rest of the kernel up to _end */ 1456 addis r5,r26,(p_end - _stext)@ha 1457 ld r5,(p_end - _stext)@l(r5) /* get _end */ 14585: bl .copy_and_flush /* copy the rest */ 1459 14609: b .start_here_multiplatform 1461 1462/* 1463 * Copy routine used to copy the kernel to start at physical address 0 1464 * and flush and invalidate the caches as needed. 1465 * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset 1466 * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5. 1467 * 1468 * Note: this routine *only* clobbers r0, r6 and lr 1469 */ 1470_GLOBAL(copy_and_flush) 1471 addi r5,r5,-8 1472 addi r6,r6,-8 14734: li r0,8 /* Use the smallest common */ 1474 /* denominator cache line */ 1475 /* size. This results in */ 1476 /* extra cache line flushes */ 1477 /* but operation is correct. */ 1478 /* Can't get cache line size */ 1479 /* from NACA as it is being */ 1480 /* moved too. */ 1481 1482 mtctr r0 /* put # words/line in ctr */ 14833: addi r6,r6,8 /* copy a cache line */ 1484 ldx r0,r6,r4 1485 stdx r0,r6,r3 1486 bdnz 3b 1487 dcbst r6,r3 /* write it to memory */ 1488 sync 1489 icbi r6,r3 /* flush the icache line */ 1490 cmpld 0,r6,r5 1491 blt 4b 1492 sync 1493 addi r5,r5,8 1494 addi r6,r6,8 1495 blr 1496 1497.align 8 1498copy_to_here: 1499 1500#ifdef CONFIG_SMP 1501#ifdef CONFIG_PPC_PMAC 1502/* 1503 * On PowerMac, secondary processors starts from the reset vector, which 1504 * is temporarily turned into a call to one of the functions below. 1505 */ 1506 .section ".text"; 1507 .align 2 ; 1508 1509 .globl __secondary_start_pmac_0 1510__secondary_start_pmac_0: 1511 /* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */ 1512 li r24,0 1513 b 1f 1514 li r24,1 1515 b 1f 1516 li r24,2 1517 b 1f 1518 li r24,3 15191: 1520 1521_GLOBAL(pmac_secondary_start) 1522 /* turn on 64-bit mode */ 1523 bl .enable_64b_mode 1524 1525 li r0,0 1526 mfspr r3,SPRN_HID4 1527 rldimi r3,r0,40,23 /* clear bit 23 (rm_ci) */ 1528 sync 1529 mtspr SPRN_HID4,r3 1530 isync 1531 sync 1532 slbia 1533 1534 /* get TOC pointer (real address) */ 1535 bl .relative_toc 1536 1537 /* Copy some CPU settings from CPU 0 */ 1538 bl .__restore_cpu_ppc970 1539 1540 /* pSeries do that early though I don't think we really need it */ 1541 mfmsr r3 1542 ori r3,r3,MSR_RI 1543 mtmsrd r3 /* RI on */ 1544 1545 /* Set up a paca value for this processor. */ 1546 LOAD_REG_ADDR(r4,paca) /* Get base vaddr of paca array */ 1547 mulli r13,r24,PACA_SIZE /* Calculate vaddr of right paca */ 1548 add r13,r13,r4 /* for this processor. */ 1549 mtspr SPRN_SPRG3,r13 /* Save vaddr of paca in SPRG3 */ 1550 1551 /* Create a temp kernel stack for use before relocation is on. */ 1552 ld r1,PACAEMERGSP(r13) 1553 subi r1,r1,STACK_FRAME_OVERHEAD 1554 1555 b __secondary_start 1556 1557#endif /* CONFIG_PPC_PMAC */ 1558 1559/* 1560 * This function is called after the master CPU has released the 1561 * secondary processors. The execution environment is relocation off. 1562 * The paca for this processor has the following fields initialized at 1563 * this point: 1564 * 1. Processor number 1565 * 2. Segment table pointer (virtual address) 1566 * On entry the following are set: 1567 * r1 = stack pointer. vaddr for iSeries, raddr (temp stack) for pSeries 1568 * r24 = cpu# (in Linux terms) 1569 * r13 = paca virtual address 1570 * SPRG3 = paca virtual address 1571 */ 1572 .globl __secondary_start 1573__secondary_start: 1574 /* Set thread priority to MEDIUM */ 1575 HMT_MEDIUM 1576 1577 /* Do early setup for that CPU (stab, slb, hash table pointer) */ 1578 bl .early_setup_secondary 1579 1580 /* Initialize the kernel stack. Just a repeat for iSeries. */ 1581 LOAD_REG_ADDR(r3, current_set) 1582 sldi r28,r24,3 /* get current_set[cpu#] */ 1583 ldx r1,r3,r28 1584 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD 1585 std r1,PACAKSAVE(r13) 1586 1587 /* Clear backchain so we get nice backtraces */ 1588 li r7,0 1589 mtlr r7 1590 1591 /* enable MMU and jump to start_secondary */ 1592 LOAD_REG_ADDR(r3, .start_secondary_prolog) 1593 LOAD_REG_IMMEDIATE(r4, MSR_KERNEL) 1594#ifdef CONFIG_PPC_ISERIES 1595BEGIN_FW_FTR_SECTION 1596 ori r4,r4,MSR_EE 1597 li r8,1 1598 stb r8,PACAHARDIRQEN(r13) 1599END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 1600#endif 1601BEGIN_FW_FTR_SECTION 1602 stb r7,PACAHARDIRQEN(r13) 1603END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES) 1604 stb r7,PACASOFTIRQEN(r13) 1605 1606 mtspr SPRN_SRR0,r3 1607 mtspr SPRN_SRR1,r4 1608 rfid 1609 b . /* prevent speculative execution */ 1610 1611/* 1612 * Running with relocation on at this point. All we want to do is 1613 * zero the stack back-chain pointer and get the TOC virtual address 1614 * before going into C code. 1615 */ 1616_GLOBAL(start_secondary_prolog) 1617 ld r2,PACATOC(r13) 1618 li r3,0 1619 std r3,0(r1) /* Zero the stack frame pointer */ 1620 bl .start_secondary 1621 b . 1622#endif 1623 1624/* 1625 * This subroutine clobbers r11 and r12 1626 */ 1627_GLOBAL(enable_64b_mode) 1628 mfmsr r11 /* grab the current MSR */ 1629 li r12,(MSR_SF | MSR_ISF)@highest 1630 sldi r12,r12,48 1631 or r11,r11,r12 1632 mtmsrd r11 1633 isync 1634 blr 1635 1636/* 1637 * This puts the TOC pointer into r2, offset by 0x8000 (as expected 1638 * by the toolchain). It computes the correct value for wherever we 1639 * are running at the moment, using position-independent code. 1640 */ 1641_GLOBAL(relative_toc) 1642 mflr r0 1643 bcl 20,31,$+4 16440: mflr r9 1645 ld r2,(p_toc - 0b)(r9) 1646 add r2,r2,r9 1647 mtlr r0 1648 blr 1649 1650p_toc: .llong __toc_start + 0x8000 - 0b 1651 1652/* 1653 * This is where the main kernel code starts. 1654 */ 1655_INIT_STATIC(start_here_multiplatform) 1656 /* set up the TOC (real address) */ 1657 bl .relative_toc 1658 1659 /* Clear out the BSS. It may have been done in prom_init, 1660 * already but that's irrelevant since prom_init will soon 1661 * be detached from the kernel completely. Besides, we need 1662 * to clear it now for kexec-style entry. 1663 */ 1664 LOAD_REG_ADDR(r11,__bss_stop) 1665 LOAD_REG_ADDR(r8,__bss_start) 1666 sub r11,r11,r8 /* bss size */ 1667 addi r11,r11,7 /* round up to an even double word */ 1668 srdi. r11,r11,3 /* shift right by 3 */ 1669 beq 4f 1670 addi r8,r8,-8 1671 li r0,0 1672 mtctr r11 /* zero this many doublewords */ 16733: stdu r0,8(r8) 1674 bdnz 3b 16754: 1676 1677 mfmsr r6 1678 ori r6,r6,MSR_RI 1679 mtmsrd r6 /* RI on */ 1680 1681#ifdef CONFIG_RELOCATABLE 1682 /* Save the physical address we're running at in kernstart_addr */ 1683 LOAD_REG_ADDR(r4, kernstart_addr) 1684 clrldi r0,r25,2 1685 std r0,0(r4) 1686#endif 1687 1688 /* The following gets the stack set up with the regs */ 1689 /* pointing to the real addr of the kernel stack. This is */ 1690 /* all done to support the C function call below which sets */ 1691 /* up the htab. This is done because we have relocated the */ 1692 /* kernel but are still running in real mode. */ 1693 1694 LOAD_REG_ADDR(r3,init_thread_union) 1695 1696 /* set up a stack pointer */ 1697 addi r1,r3,THREAD_SIZE 1698 li r0,0 1699 stdu r0,-STACK_FRAME_OVERHEAD(r1) 1700 1701 /* Do very early kernel initializations, including initial hash table, 1702 * stab and slb setup before we turn on relocation. */ 1703 1704 /* Restore parameters passed from prom_init/kexec */ 1705 mr r3,r31 1706 bl .early_setup /* also sets r13 and SPRG3 */ 1707 1708 LOAD_REG_ADDR(r3, .start_here_common) 1709 ld r4,PACAKMSR(r13) 1710 mtspr SPRN_SRR0,r3 1711 mtspr SPRN_SRR1,r4 1712 rfid 1713 b . /* prevent speculative execution */ 1714 1715 /* This is where all platforms converge execution */ 1716_INIT_GLOBAL(start_here_common) 1717 /* relocation is on at this point */ 1718 std r1,PACAKSAVE(r13) 1719 1720 /* Load the TOC (virtual address) */ 1721 ld r2,PACATOC(r13) 1722 1723 bl .setup_system 1724 1725 /* Load up the kernel context */ 17265: 1727 li r5,0 1728 stb r5,PACASOFTIRQEN(r13) /* Soft Disabled */ 1729#ifdef CONFIG_PPC_ISERIES 1730BEGIN_FW_FTR_SECTION 1731 mfmsr r5 1732 ori r5,r5,MSR_EE /* Hard Enabled on iSeries*/ 1733 mtmsrd r5 1734 li r5,1 1735END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) 1736#endif 1737 stb r5,PACAHARDIRQEN(r13) /* Hard Disabled on others */ 1738 1739 bl .start_kernel 1740 1741 /* Not reached */ 1742 BUG_OPCODE 1743 1744/* 1745 * We put a few things here that have to be page-aligned. 1746 * This stuff goes at the beginning of the bss, which is page-aligned. 1747 */ 1748 .section ".bss" 1749 1750 .align PAGE_SHIFT 1751 1752 .globl empty_zero_page 1753empty_zero_page: 1754 .space PAGE_SIZE 1755 1756 .globl swapper_pg_dir 1757swapper_pg_dir: 1758 .space PGD_TABLE_SIZE 1759