1/*- 2 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com> 3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com> 4 * Copyright (C) 2006 Juniper Networks, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31/*- 32 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 33 * Copyright (C) 1995, 1996 TooLs GmbH. 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed by TooLs GmbH. 47 * 4. The name of TooLs GmbH may not be used to endorse or promote products 48 * derived from this software without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 60 * 61 * from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $ 62 */ 63 64/* 65 * NOTICE: This is not a standalone file. to use it, #include it in 66 * your port's locore.S, like so: 67 * 68 * #include <powerpc/booke/trap_subr.S> 69 */ 70 71/* 72 * SPRG usage notes 73 * 74 * SPRG0 - pcpu pointer 75 * SPRG1 - all interrupts except TLB miss, critical, machine check 76 * SPRG2 - critical 77 * SPRG3 - machine check 78 * SPRG4-6 - scratch 79 * 80 */ 81 82/* Get the per-CPU data structure */ 83#define GET_CPUINFO(r) mfsprg0 r 84 85#define RES_GRANULE 32 86#define RES_LOCK 0 /* offset to the 'lock' word */ 87#define RES_RECURSE 4 /* offset to the 'recurse' word */ 88 89/* 90 * Standard interrupt prolog 91 * 92 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP 93 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave}) 94 * isrr0-1 - save restore registers with CPU state at interrupt time (may be 95 * SRR0-1, CSRR0-1, MCSRR0-1 96 * 97 * 1. saves in the given savearea: 98 * - R30-31 99 * - DEAR, ESR 100 * - xSRR0-1 101 * 102 * 2. saves CR -> R30 103 * 104 * 3. switches to kstack if needed 105 * 106 * 4. notes: 107 * - R31 can be used as scratch register until a new frame is layed on 108 * the stack with FRAME_SETUP 109 * 110 * - potential TLB miss: NO. Saveareas are always acessible via TLB1 111 * permanent entries, and within this prolog we do not dereference any 112 * locations potentially not in the TLB 113 */ 114#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ 115 mtspr sprg_sp, %r1; /* Save SP */ \ 116 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 117 stw %r30, (savearea+CPUSAVE_R30)(%r1); \ 118 stw %r31, (savearea+CPUSAVE_R31)(%r1); \ 119 mfdear %r30; \ 120 mfesr %r31; \ 121 stw %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 122 stw %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 123 mfspr %r30, isrr0; \ 124 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 125 stw %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 126 stw %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 127 isync; \ 128 mfspr %r1, sprg_sp; /* Restore SP */ \ 129 mfcr %r30; /* Save CR */ \ 130 /* switch to per-thread kstack if intr taken in user mode */ \ 131 mtcr %r31; /* MSR at interrupt time */ \ 132 bf 17, 1f; \ 133 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 134 lwz %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1351: 136 137#define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ 138 mtspr sprg_sp, %r1; /* Save SP */ \ 139 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 140 stw %r30, (savearea+CPUSAVE_R30)(%r1); \ 141 stw %r31, (savearea+CPUSAVE_R31)(%r1); \ 142 mfdear %r30; \ 143 mfesr %r31; \ 144 stw %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 145 stw %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 146 mfspr %r30, isrr0; \ 147 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 148 stw %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 149 stw %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 150 mfspr %r30, SPR_SRR0; \ 151 mfspr %r31, SPR_SRR1; /* MSR at interrupt time */ \ 152 stw %r30, (savearea+CPUSAVE_SRR0+8)(%r1); \ 153 stw %r31, (savearea+CPUSAVE_SRR1+8)(%r1); \ 154 isync; \ 155 mfspr %r1, sprg_sp; /* Restore SP */ \ 156 mfcr %r30; /* Save CR */ \ 157 /* switch to per-thread kstack if intr taken in user mode */ \ 158 mtcr %r31; /* MSR at interrupt time */ \ 159 bf 17, 1f; \ 160 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 161 lwz %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1621: 163 164/* 165 * FRAME_SETUP assumes: 166 * SPRG{1-3} SP at the time interrupt occured 167 * savearea r30-r31, DEAR, ESR, xSRR0-1 168 * r30 CR 169 * r31 scratch 170 * r1 kernel stack 171 * 172 * sprg_sp - SPRG reg containing SP at the time interrupt occured 173 * savearea - temp save 174 * exc - exception number (EXC_xxx) 175 * 176 * 1. sets a new frame 177 * 2. saves in the frame: 178 * - R0, R1 (SP at the time of interrupt), R2, LR, CR 179 * - R3-31 (R30-31 first restored from savearea) 180 * - XER, CTR, DEAR, ESR (from savearea), xSRR0-1 181 * 182 * Notes: 183 * - potential TLB miss: YES, since we make dereferences to kstack, which 184 * can happen not covered (we can have up to two DTLB misses if fortunate 185 * enough i.e. when kstack crosses page boundary and both pages are 186 * untranslated) 187 */ 188#define FRAME_SETUP(sprg_sp, savearea, exc) \ 189 mfspr %r31, sprg_sp; /* get saved SP */ \ 190 /* establish a new stack frame and put everything on it */ \ 191 stwu %r31, -FRAMELEN(%r1); \ 192 stw %r0, FRAME_0+8(%r1); /* save r0 in the trapframe */ \ 193 stw %r31, FRAME_1+8(%r1); /* save SP " " */ \ 194 stw %r2, FRAME_2+8(%r1); /* save r2 " " */ \ 195 mflr %r31; \ 196 stw %r31, FRAME_LR+8(%r1); /* save LR " " */ \ 197 stw %r30, FRAME_CR+8(%r1); /* save CR " " */ \ 198 GET_CPUINFO(%r2); \ 199 lwz %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \ 200 lwz %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \ 201 /* save R3-31 */ \ 202 stmw %r3, FRAME_3+8(%r1) ; \ 203 /* save DEAR, ESR */ \ 204 lwz %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2); \ 205 lwz %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2); \ 206 stw %r28, FRAME_BOOKE_DEAR+8(%r1); \ 207 stw %r29, FRAME_BOOKE_ESR+8(%r1); \ 208 /* save XER, CTR, exc number */ \ 209 mfxer %r3; \ 210 mfctr %r4; \ 211 stw %r3, FRAME_XER+8(%r1); \ 212 stw %r4, FRAME_CTR+8(%r1); \ 213 li %r5, exc; \ 214 stw %r5, FRAME_EXC+8(%r1); \ 215 /* save DBCR0 */ \ 216 mfspr %r3, SPR_DBCR0; \ 217 stw %r3, FRAME_BOOKE_DBCR0+8(%r1); \ 218 /* save xSSR0-1 */ \ 219 lwz %r30, (savearea+CPUSAVE_SRR0)(%r2); \ 220 lwz %r31, (savearea+CPUSAVE_SRR1)(%r2); \ 221 stw %r30, FRAME_SRR0+8(%r1); \ 222 stw %r31, FRAME_SRR1+8(%r1); \ 223 lwz %r2,PC_CURTHREAD(%r2) /* set curthread pointer */ 224 225/* 226 * 227 * isrr0-1 - save restore registers to restore CPU state to (may be 228 * SRR0-1, CSRR0-1, MCSRR0-1 229 * 230 * Notes: 231 * - potential TLB miss: YES. The deref'd kstack may be not covered 232 */ 233#define FRAME_LEAVE(isrr0, isrr1) \ 234 /* restore CTR, XER, LR, CR */ \ 235 lwz %r4, FRAME_CTR+8(%r1); \ 236 lwz %r5, FRAME_XER+8(%r1); \ 237 lwz %r6, FRAME_LR+8(%r1); \ 238 lwz %r7, FRAME_CR+8(%r1); \ 239 mtctr %r4; \ 240 mtxer %r5; \ 241 mtlr %r6; \ 242 mtcr %r7; \ 243 /* restore DBCR0 */ \ 244 lwz %r4, FRAME_BOOKE_DBCR0+8(%r1); \ 245 mtspr SPR_DBCR0, %r4; \ 246 /* restore xSRR0-1 */ \ 247 lwz %r30, FRAME_SRR0+8(%r1); \ 248 lwz %r31, FRAME_SRR1+8(%r1); \ 249 mtspr isrr0, %r30; \ 250 mtspr isrr1, %r31; \ 251 /* restore R2-31, SP */ \ 252 lmw %r2, FRAME_2+8(%r1) ; \ 253 lwz %r0, FRAME_0+8(%r1); \ 254 lwz %r1, FRAME_1+8(%r1); \ 255 isync 256 257/* 258 * TLB miss prolog 259 * 260 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area 261 * 262 * Notes: 263 * - potential TLB miss: NO. It is crucial that we do not generate a TLB 264 * miss within the TLB prolog itself! 265 * - TLBSAVE is always translated 266 */ 267#define TLB_PROLOG \ 268 mtsprg4 %r1; /* Save SP */ \ 269 mtsprg5 %r28; \ 270 mtsprg6 %r29; \ 271 /* calculate TLB nesting level and TLBSAVE instance address */ \ 272 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 273 lwz %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 274 rlwinm %r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */ \ 275 addi %r28, %r28, 1; \ 276 stw %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 277 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 278 add %r1, %r1, %r29; /* current TLBSAVE ptr */ \ 279 \ 280 /* save R20-31 */ \ 281 mfsprg5 %r28; \ 282 mfsprg6 %r29; \ 283 stmw %r20, (TLBSAVE_BOOKE_R20)(%r1); \ 284 /* save LR, CR */ \ 285 mflr %r30; \ 286 mfcr %r31; \ 287 stw %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 288 stw %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 289 /* save SRR0-1 */ \ 290 mfsrr0 %r30; /* execution addr at interrupt time */ \ 291 mfsrr1 %r31; /* MSR at interrupt time*/ \ 292 stw %r30, (TLBSAVE_BOOKE_SRR0)(%r1); /* save SRR0 */ \ 293 stw %r31, (TLBSAVE_BOOKE_SRR1)(%r1); /* save SRR1 */ \ 294 isync; \ 295 mfsprg4 %r1 296 297/* 298 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area 299 * 300 * same notes as for the TLB_PROLOG 301 */ 302#define TLB_RESTORE \ 303 mtsprg4 %r1; /* Save SP */ \ 304 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 305 /* calculate TLB nesting level and TLBSAVE instance addr */ \ 306 lwz %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 307 subi %r28, %r28, 1; \ 308 stw %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 309 rlwinm %r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */ \ 310 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 311 add %r1, %r1, %r29; \ 312 \ 313 /* restore LR, CR */ \ 314 lwz %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 315 lwz %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 316 mtlr %r30; \ 317 mtcr %r31; \ 318 /* restore SRR0-1 */ \ 319 lwz %r30, (TLBSAVE_BOOKE_SRR0)(%r1); \ 320 lwz %r31, (TLBSAVE_BOOKE_SRR1)(%r1); \ 321 mtsrr0 %r30; \ 322 mtsrr1 %r31; \ 323 /* restore R20-31 */ \ 324 lmw %r20, (TLBSAVE_BOOKE_R20)(%r1); \ 325 mfsprg4 %r1 326 327#ifdef SMP 328#define TLB_LOCK \ 329 GET_CPUINFO(%r20); \ 330 lwz %r21, PC_CURTHREAD(%r20); \ 331 lwz %r22, PC_BOOKE_TLB_LOCK(%r20); \ 332 \ 3331: lwarx %r23, 0, %r22; \ 334 cmpwi %r23, TLB_UNLOCKED; \ 335 beq 2f; \ 336 \ 337 /* check if this is recursion */ \ 338 cmplw cr0, %r21, %r23; \ 339 bne- 1b; \ 340 \ 3412: /* try to acquire lock */ \ 342 stwcx. %r21, 0, %r22; \ 343 bne- 1b; \ 344 \ 345 /* got it, update recursion counter */ \ 346 lwz %r21, RES_RECURSE(%r22); \ 347 addi %r21, %r21, 1; \ 348 stw %r21, RES_RECURSE(%r22); \ 349 isync; \ 350 msync 351 352#define TLB_UNLOCK \ 353 GET_CPUINFO(%r20); \ 354 lwz %r21, PC_CURTHREAD(%r20); \ 355 lwz %r22, PC_BOOKE_TLB_LOCK(%r20); \ 356 \ 357 /* update recursion counter */ \ 358 lwz %r23, RES_RECURSE(%r22); \ 359 subi %r23, %r23, 1; \ 360 stw %r23, RES_RECURSE(%r22); \ 361 \ 362 cmpwi %r23, 0; \ 363 bne 1f; \ 364 isync; \ 365 msync; \ 366 \ 367 /* release the lock */ \ 368 li %r23, TLB_UNLOCKED; \ 369 stw %r23, 0(%r22); \ 3701: isync; \ 371 msync 372#else 373#define TLB_LOCK 374#define TLB_UNLOCK 375#endif /* SMP */ 376 377#define INTERRUPT(label) \ 378 .globl label; \ 379 .align 5; \ 380 CNAME(label): 381 382/* 383 * Interrupt handling routines in BookE can be flexibly placed and do not have 384 * to live in pre-defined vectors location. Note they need to be TLB-mapped at 385 * all times in order to be able to handle exceptions. We thus arrange for 386 * them to be part of kernel text which is always TLB-accessible. 387 * 388 * The interrupt handling routines have to be 16 bytes aligned: we align them 389 * to 32 bytes (cache line length) which supposedly performs better. 390 * 391 */ 392 .text 393 .globl CNAME(interrupt_vector_base) 394 .align 5 395interrupt_vector_base: 396/***************************************************************************** 397 * Catch-all handler to handle uninstalled IVORs 398 ****************************************************************************/ 399INTERRUPT(int_unknown) 400 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 401 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD) 402 b trap_common 403 404/***************************************************************************** 405 * Critical input interrupt 406 ****************************************************************************/ 407INTERRUPT(int_critical_input) 408 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 409 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT) 410 addi %r3, %r1, 8 411 bl CNAME(powerpc_interrupt) 412 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) 413 rfci 414 415 416/***************************************************************************** 417 * Machine check interrupt 418 ****************************************************************************/ 419INTERRUPT(int_machine_check) 420 STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1) 421 FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK) 422 addi %r3, %r1, 8 423 bl CNAME(powerpc_interrupt) 424 FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1) 425 rfmci 426 427 428/***************************************************************************** 429 * Data storage interrupt 430 ****************************************************************************/ 431INTERRUPT(int_data_storage) 432 STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1) 433 FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI) 434 b trap_common 435 436 437/***************************************************************************** 438 * Instruction storage interrupt 439 ****************************************************************************/ 440INTERRUPT(int_instr_storage) 441 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 442 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI) 443 b trap_common 444 445 446/***************************************************************************** 447 * External input interrupt 448 ****************************************************************************/ 449INTERRUPT(int_external_input) 450 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 451 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI) 452 addi %r3, %r1, 8 453 bl CNAME(powerpc_interrupt) 454 b clear_we 455 456 457INTERRUPT(int_alignment) 458 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 459 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI) 460 b trap_common 461 462 463INTERRUPT(int_program) 464 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 465 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM) 466 b trap_common 467 468 469INTERRUPT(int_fpu) 470 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 471 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU) 472 b trap_common 473 474 475/***************************************************************************** 476 * System call 477 ****************************************************************************/ 478INTERRUPT(int_syscall) 479 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 480 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC) 481 b trap_common 482 483 484/***************************************************************************** 485 * Decrementer interrupt 486 ****************************************************************************/ 487INTERRUPT(int_decrementer) 488 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 489 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR) 490 addi %r3, %r1, 8 491 bl CNAME(powerpc_interrupt) 492 b clear_we 493 494 495/***************************************************************************** 496 * Fixed interval timer 497 ****************************************************************************/ 498INTERRUPT(int_fixed_interval_timer) 499 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 500 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT) 501 b trap_common 502 503 504/***************************************************************************** 505 * Watchdog interrupt 506 ****************************************************************************/ 507INTERRUPT(int_watchdog) 508 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 509 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG) 510 b trap_common 511 512 513/***************************************************************************** 514 * Altivec Unavailable interrupt 515 ****************************************************************************/ 516INTERRUPT(int_vec) 517 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 518 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC) 519 b trap_common 520 521 522/***************************************************************************** 523 * Watchdog interrupt 524 ****************************************************************************/ 525INTERRUPT(int_vecast) 526 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 527 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E) 528 b trap_common 529 530 531#ifdef HWPMC_HOOKS 532/***************************************************************************** 533 * PMC Interrupt 534 ****************************************************************************/ 535INTERRUPT(int_performance_counter) 536 STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 537 FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF) 538 addi %r3, %r1, 8 539 bl CNAME(powerpc_interrupt) 540 b trapexit 541#endif 542 543 544/***************************************************************************** 545 * Data TLB miss interrupt 546 * 547 * There can be nested TLB misses - while handling a TLB miss we reference 548 * data structures that may be not covered by translations. We support up to 549 * TLB_NESTED_MAX-1 nested misses. 550 * 551 * Registers use: 552 * r31 - dear 553 * r30 - unused 554 * r29 - saved mas0 555 * r28 - saved mas1 556 * r27 - saved mas2 557 * r26 - pmap address 558 * r25 - pte address 559 * 560 * r20:r23 - scratch registers 561 ****************************************************************************/ 562INTERRUPT(int_data_tlb_error) 563 TLB_PROLOG 564 TLB_LOCK 565 566 mfdear %r31 567 568 /* 569 * Save MAS0-MAS2 registers. There might be another tlb miss during 570 * pte lookup overwriting current contents (which was hw filled). 571 */ 572 mfspr %r29, SPR_MAS0 573 mfspr %r28, SPR_MAS1 574 mfspr %r27, SPR_MAS2 575 576 /* Check faulting address. */ 577 lis %r21, VM_MAXUSER_ADDRESS@h 578 ori %r21, %r21, VM_MAXUSER_ADDRESS@l 579 cmplw cr0, %r31, %r21 580 blt search_user_pmap 581 582 /* If it's kernel address, allow only supervisor mode misses. */ 583 mfsrr1 %r21 584 mtcr %r21 585 bt 17, search_failed /* check MSR[PR] */ 586 587search_kernel_pmap: 588 /* Load r26 with kernel_pmap address */ 589 bl 1f 590 .long kernel_pmap_store-. 5911: mflr %r21 592 lwz %r26, 0(%r21) 593 add %r26, %r21, %r26 /* kernel_pmap_store in r26 */ 594 595 /* Force kernel tid, set TID to 0 in MAS1. */ 596 li %r21, 0 597 rlwimi %r28, %r21, 0, 8, 15 /* clear TID bits */ 598 599tlb_miss_handle: 600 /* This may result in nested tlb miss. */ 601 bl pte_lookup /* returns PTE address in R25 */ 602 603 cmpwi %r25, 0 /* pte found? */ 604 beq search_failed 605 606 /* Finish up, write TLB entry. */ 607 bl tlb_fill_entry 608 609tlb_miss_return: 610 TLB_UNLOCK 611 TLB_RESTORE 612 rfi 613 614search_user_pmap: 615 /* Load r26 with current user space process pmap */ 616 GET_CPUINFO(%r26) 617 lwz %r26, PC_CURPMAP(%r26) 618 619 b tlb_miss_handle 620 621search_failed: 622 /* 623 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with 624 * the faulting virtual address anyway, but put a fake RPN and no 625 * access rights. This should cause a following {D,I}SI exception. 626 */ 627 lis %r23, 0xffff0000@h /* revoke all permissions */ 628 629 /* Load MAS registers. */ 630 mtspr SPR_MAS0, %r29 631 isync 632 mtspr SPR_MAS1, %r28 633 isync 634 mtspr SPR_MAS2, %r27 635 isync 636 mtspr SPR_MAS3, %r23 637 isync 638 639 bl zero_mas7 640 bl zero_mas8 641 642 tlbwe 643 msync 644 isync 645 b tlb_miss_return 646 647/***************************************************************************** 648 * 649 * Return pte address that corresponds to given pmap/va. If there is no valid 650 * entry return 0. 651 * 652 * input: r26 - pmap 653 * input: r31 - dear 654 * output: r25 - pte address 655 * 656 * scratch regs used: r21 657 * 658 ****************************************************************************/ 659pte_lookup: 660 cmpwi %r26, 0 661 beq 1f /* fail quickly if pmap is invalid */ 662 663 srwi %r21, %r31, PDIR_SHIFT /* pdir offset */ 664 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ 665 666 addi %r25, %r26, PM_PDIR /* pmap pm_dir[] address */ 667 add %r25, %r25, %r21 /* offset within pm_pdir[] table */ 668 /* 669 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx] 670 * This load may cause a Data TLB miss for non-kernel pmap! 671 */ 672 lwz %r25, 0(%r25) 673 cmpwi %r25, 0 674 beq 2f 675 676 lis %r21, PTBL_MASK@h 677 ori %r21, %r21, PTBL_MASK@l 678 and %r21, %r21, %r31 679 680 /* ptbl offset, multiply by ptbl entry size */ 681 srwi %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT) 682 683 add %r25, %r25, %r21 /* address of pte entry */ 684 /* 685 * Get pte->flags 686 * This load may cause a Data TLB miss for non-kernel pmap! 687 */ 688 lwz %r21, PTE_FLAGS(%r25) 689 andi. %r21, %r21, PTE_VALID@l 690 bne 2f 6911: 692 li %r25, 0 6932: 694 blr 695 696/***************************************************************************** 697 * 698 * Load MAS1-MAS3 registers with data, write TLB entry 699 * 700 * input: 701 * r29 - mas0 702 * r28 - mas1 703 * r27 - mas2 704 * r25 - pte 705 * 706 * output: none 707 * 708 * scratch regs: r21-r23 709 * 710 ****************************************************************************/ 711tlb_fill_entry: 712 /* 713 * Update PTE flags: we have to do it atomically, as pmap_protect() 714 * running on other CPUs could attempt to update the flags at the same 715 * time. 716 */ 717 li %r23, PTE_FLAGS 7181: 719 lwarx %r21, %r23, %r25 /* get pte->flags */ 720 oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */ 721 722 andi. %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */ 723 beq 2f 724 ori %r21, %r21, PTE_MODIFIED@l /* set modified bit */ 7252: 726 stwcx. %r21, %r23, %r25 /* write it back */ 727 bne- 1b 728 729 /* Update MAS2. */ 730 rlwimi %r27, %r21, 13, 27, 30 /* insert WIMG bits from pte */ 731 732 /* Setup MAS3 value in r23. */ 733 lwz %r23, PTE_RPN(%r25) /* get pte->rpn */ 734 rlwinm %r22, %r23, 20, 0, 11 /* extract MAS3 portion of RPN */ 735 736 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ 737 rlwimi %r22, %r21, 20, 12, 19 /* insert lower 8 RPN bits to MAS3 */ 738 rlwinm %r23, %r23, 20, 24, 31 /* MAS7 portion of RPN */ 739 740 /* Load MAS registers. */ 741 mtspr SPR_MAS0, %r29 742 isync 743 mtspr SPR_MAS1, %r28 744 isync 745 mtspr SPR_MAS2, %r27 746 isync 747 mtspr SPR_MAS3, %r22 748 isync 749 mtspr SPR_MAS7, %r23 750 isync 751 752 mflr %r21 753 bl zero_mas8 754 mtlr %r21 755 756 tlbwe 757 isync 758 msync 759 blr 760 761/***************************************************************************** 762 * Instruction TLB miss interrupt 763 * 764 * Same notes as for the Data TLB miss 765 ****************************************************************************/ 766INTERRUPT(int_inst_tlb_error) 767 TLB_PROLOG 768 TLB_LOCK 769 770 mfsrr0 %r31 /* faulting address */ 771 772 /* 773 * Save MAS0-MAS2 registers. There might be another tlb miss during pte 774 * lookup overwriting current contents (which was hw filled). 775 */ 776 mfspr %r29, SPR_MAS0 777 mfspr %r28, SPR_MAS1 778 mfspr %r27, SPR_MAS2 779 780 mfsrr1 %r21 781 mtcr %r21 782 783 /* check MSR[PR] */ 784 bt 17, search_user_pmap 785 b search_kernel_pmap 786 787 788 .globl interrupt_vector_top 789interrupt_vector_top: 790 791/***************************************************************************** 792 * Debug interrupt 793 ****************************************************************************/ 794INTERRUPT(int_debug) 795 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 796 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) 797 GET_CPUINFO(%r3) 798 lwz %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3) 799 bl 0f 800 .long interrupt_vector_base-. 801 .long interrupt_vector_top-. 8020: mflr %r5 803 lwz %r4,0(%r5) /* interrupt_vector_base in r4 */ 804 add %r4,%r4,%r5 805 cmplw cr0, %r3, %r4 806 blt 1f 807 lwz %r4,4(%r5) /* interrupt_vector_top in r4 */ 808 add %r4,%r4,%r5 809 addi %r4,%r4,4 810 cmplw cr0, %r3, %r4 811 bge 1f 812 /* Disable single-stepping for the interrupt handlers. */ 813 lwz %r3, FRAME_SRR1+8(%r1); 814 rlwinm %r3, %r3, 0, 23, 21 815 stw %r3, FRAME_SRR1+8(%r1); 816 /* Restore srr0 and srr1 as they could have been clobbered. */ 817 GET_CPUINFO(%r4) 818 lwz %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0+8)(%r4); 819 mtspr SPR_SRR0, %r3 820 lwz %r4, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR1+8)(%r4); 821 mtspr SPR_SRR1, %r4 822 b 9f 8231: 824 addi %r3, %r1, 8 825 bl CNAME(trap) 826 /* 827 * Handle ASTs, needed for proper support of single-stepping. 828 * We actually need to return to the process with an rfi. 829 */ 830 b trapexit 8319: 832 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) 833 rfci 834 835 836/***************************************************************************** 837 * Common trap code 838 ****************************************************************************/ 839trap_common: 840 /* Call C trap dispatcher */ 841 addi %r3, %r1, 8 842 bl CNAME(trap) 843 844 .globl CNAME(trapexit) /* exported for db_backtrace use */ 845CNAME(trapexit): 846 /* disable interrupts */ 847 wrteei 0 848 849 /* Test AST pending - makes sense for user process only */ 850 lwz %r5, FRAME_SRR1+8(%r1) 851 mtcr %r5 852 bf 17, 1f 853 854 GET_CPUINFO(%r3) 855 lwz %r4, PC_CURTHREAD(%r3) 856 lwz %r4, TD_FLAGS(%r4) 857 lis %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h 858 ori %r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l 859 and. %r4, %r4, %r5 860 beq 1f 861 862 /* re-enable interrupts before calling ast() */ 863 wrteei 1 864 865 addi %r3, %r1, 8 866 bl CNAME(ast) 867 .globl CNAME(asttrapexit) /* db_backtrace code sentinel #2 */ 868CNAME(asttrapexit): 869 b trapexit /* test ast ret value ? */ 8701: 871 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 872 rfi 873 874 875#if defined(KDB) 876/* 877 * Deliberate entry to dbtrap 878 */ 879 .globl CNAME(breakpoint) 880CNAME(breakpoint): 881 mtsprg1 %r1 882 mfmsr %r3 883 mtsrr1 %r3 884 andi. %r3, %r3, ~(PSL_EE | PSL_ME)@l 885 mtmsr %r3 /* disable interrupts */ 886 isync 887 GET_CPUINFO(%r3) 888 stw %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3) 889 stw %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3) 890 891 mflr %r31 892 mtsrr0 %r31 893 894 mfdear %r30 895 mfesr %r31 896 stw %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3) 897 stw %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3) 898 899 mfsrr0 %r30 900 mfsrr1 %r31 901 stw %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3) 902 stw %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3) 903 isync 904 905 mfcr %r30 906 907/* 908 * Now the kdb trap catching code. 909 */ 910dbtrap: 911 FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG) 912/* Call C trap code: */ 913 addi %r3, %r1, 8 914 bl CNAME(db_trap_glue) 915 or. %r3, %r3, %r3 916 bne dbleave 917/* This wasn't for KDB, so switch to real trap: */ 918 b trap_common 919 920dbleave: 921 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 922 rfi 923#endif /* KDB */ 924 925clear_we: 926 lwz %r3, (FRAME_SRR1+8)(%r1) 927 rlwinm %r3, %r3, 0, 14, 12 928 stw %r3, (FRAME_SRR1+8)(%r1) 929 b trapexit 930 931#ifdef SMP 932ENTRY(tlb_lock) 933 GET_CPUINFO(%r5) 934 lwz %r5, PC_CURTHREAD(%r5) 9351: lwarx %r4, 0, %r3 936 cmpwi %r4, TLB_UNLOCKED 937 bne 1b 938 stwcx. %r5, 0, %r3 939 bne- 1b 940 isync 941 msync 942 blr 943 944ENTRY(tlb_unlock) 945 isync 946 msync 947 li %r4, TLB_UNLOCKED 948 stw %r4, 0(%r3) 949 isync 950 msync 951 blr 952 953/* 954 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes); 955 * only a single word from this granule will actually be used as a spin lock 956 * for mutual exclusion between TLB miss handler and pmap layer that 957 * manipulates page table contents. 958 */ 959 .data 960 .align 5 961GLOBAL(tlb0_miss_locks) 962 .space RES_GRANULE * MAXCPU 963#endif 964