1/*- 2 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com> 3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com> 4 * Copyright (C) 2006 Juniper Networks, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29/*- 30 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 31 * Copyright (C) 1995, 1996 TooLs GmbH. 32 * All rights reserved. 33 * 34 * Redistribution and use in source and binary forms, with or without 35 * modification, are permitted provided that the following conditions 36 * are met: 37 * 1. Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * 2. Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in the 41 * documentation and/or other materials provided with the distribution. 42 * 3. All advertising materials mentioning features or use of this software 43 * must display the following acknowledgement: 44 * This product includes software developed by TooLs GmbH. 45 * 4. The name of TooLs GmbH may not be used to endorse or promote products 46 * derived from this software without specific prior written permission. 47 * 48 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 49 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 50 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 51 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 53 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 54 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 55 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 56 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 57 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 58 * 59 * from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $ 60 */ 61 62/* 63 * NOTICE: This is not a standalone file. to use it, #include it in 64 * your port's locore.S, like so: 65 * 66 * #include <powerpc/booke/trap_subr.S> 67 */ 68 69/* 70 * SPRG usage notes 71 * 72 * SPRG0 - pcpu pointer 73 * SPRG1 - all interrupts except TLB miss, critical, machine check 74 * SPRG2 - critical 75 * SPRG3 - machine check 76 * SPRG4-6 - scratch 77 * 78 */ 79 80/* Get the per-CPU data structure */ 81#define GET_CPUINFO(r) mfsprg0 r 82 83#define RES_GRANULE 64 84#define RES_LOCK 0 /* offset to the 'lock' word */ 85#ifdef __powerpc64__ 86#define RES_RECURSE 8 /* offset to the 'recurse' word */ 87#else 88#define RES_RECURSE 4 /* offset to the 'recurse' word */ 89#endif 90 91/* 92 * Standard interrupt prolog 93 * 94 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP 95 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave}) 96 * isrr0-1 - save restore registers with CPU state at interrupt time (may be 97 * SRR0-1, CSRR0-1, MCSRR0-1 98 * 99 * 1. saves in the given savearea: 100 * - R30-31 101 * - DEAR, ESR 102 * - xSRR0-1 103 * 104 * 2. saves CR -> R30 105 * 106 * 3. switches to kstack if needed 107 * 108 * 4. notes: 109 * - R31 can be used as scratch register until a new frame is laid on 110 * the stack with FRAME_SETUP 111 * 112 * - potential TLB miss: NO. Saveareas are always acessible via TLB1 113 * permanent entries, and within this prolog we do not dereference any 114 * locations potentially not in the TLB 115 */ 116#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ 117 mtspr sprg_sp, %r1; /* Save SP */ \ 118 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 119 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \ 120 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \ 121 mfspr %r30, SPR_DEAR; \ 122 mfspr %r31, SPR_ESR; \ 123 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 124 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 125 mfspr %r30, isrr0; \ 126 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 127 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 128 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 129 isync; \ 130 mfspr %r1, sprg_sp; /* Restore SP */ \ 131 mfcr %r30; /* Save CR */ \ 132 /* switch to per-thread kstack if intr taken in user mode */ \ 133 mtcr %r31; /* MSR at interrupt time */ \ 134 bf 17, 1f; \ 135 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 136 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1371: 138 139#define STANDARD_CRIT_PROLOG(sprg_sp, savearea, stack, isrr0, isrr1) \ 140 mtspr sprg_sp, %r1; /* Save SP */ \ 141 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 142 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \ 143 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \ 144 mfspr %r30, SPR_DEAR; \ 145 mfspr %r31, SPR_ESR; \ 146 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 147 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 148 mfspr %r30, SPR_SRR0; \ 149 mfspr %r31, SPR_SRR1; \ 150 STORE %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1); \ 151 STORE %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1); \ 152 mfspr %r30, isrr0; \ 153 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 154 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 155 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 156 isync; \ 157 mfcr %r30; /* Save CR */ \ 158 mtcr %r31; /* MSR at interrupt time */ \ 159 /* \ 160 * Never inherit the interrupted r1. A critical interrupt can \ 161 * land partway through a non-critical prolog, where r1 holds \ 162 * the pcpu pointer rather than a stack, and laying a frame \ 163 * there walks straight into the adjacent CPU's pcpu. \ 164 */ \ 165 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 166 bf 17, 1f; \ 167 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 168 b 2f; \ 1691: LOAD %r1, stack(%r1); \ 1702: 171 172/* 173 * Put back the SRR0-1 saved by STANDARD_CRIT_PROLOG. 174 * 175 * A critical interrupt can land in the middle of a non-critical prolog, 176 * before it has stashed SRR0-1, or between the mtsrr0/mtsrr1 and the rfi of 177 * a TLB miss return. Nested exceptions taken by the C dispatcher clobber 178 * them either way, so restore before returning. 179 */ 180#define CRIT_SRR_RESTORE(savearea) \ 181 GET_CPUINFO(%r3); \ 182 LOAD %r4, (savearea+BOOKE_CRITSAVE_SRR0)(%r3); \ 183 LOAD %r5, (savearea+BOOKE_CRITSAVE_SRR1)(%r3); \ 184 mtspr SPR_SRR0, %r4; \ 185 mtspr SPR_SRR1, %r5 186 187/* 188 * FRAME_SETUP assumes: 189 * SPRG{1-3} SP at the time interrupt occurred 190 * savearea r30-r31, DEAR, ESR, xSRR0-1 191 * r30 CR 192 * r31 scratch 193 * r1 kernel stack 194 * 195 * sprg_sp - SPRG reg containing SP at the time interrupt occurred 196 * savearea - temp save 197 * exc - exception number (EXC_xxx) 198 * 199 * 1. sets a new frame 200 * 2. saves in the frame: 201 * - R0, R1 (SP at the time of interrupt), R2, LR, CR 202 * - R3-31 (R30-31 first restored from savearea) 203 * - XER, CTR, DEAR, ESR (from savearea), xSRR0-1 204 * 205 * Notes: 206 * - potential TLB miss: YES, since we make dereferences to kstack, which 207 * can happen not covered (we can have up to two DTLB misses if fortunate 208 * enough i.e. when kstack crosses page boundary and both pages are 209 * untranslated) 210 */ 211#ifdef __powerpc64__ 212#define SAVE_REGS(r) \ 213 std %r3, FRAME_3+CALLSIZE(r); \ 214 std %r4, FRAME_4+CALLSIZE(r); \ 215 std %r5, FRAME_5+CALLSIZE(r); \ 216 std %r6, FRAME_6+CALLSIZE(r); \ 217 std %r7, FRAME_7+CALLSIZE(r); \ 218 std %r8, FRAME_8+CALLSIZE(r); \ 219 std %r9, FRAME_9+CALLSIZE(r); \ 220 std %r10, FRAME_10+CALLSIZE(r); \ 221 std %r11, FRAME_11+CALLSIZE(r); \ 222 std %r12, FRAME_12+CALLSIZE(r); \ 223 std %r13, FRAME_13+CALLSIZE(r); \ 224 std %r14, FRAME_14+CALLSIZE(r); \ 225 std %r15, FRAME_15+CALLSIZE(r); \ 226 std %r16, FRAME_16+CALLSIZE(r); \ 227 std %r17, FRAME_17+CALLSIZE(r); \ 228 std %r18, FRAME_18+CALLSIZE(r); \ 229 std %r19, FRAME_19+CALLSIZE(r); \ 230 std %r20, FRAME_20+CALLSIZE(r); \ 231 std %r21, FRAME_21+CALLSIZE(r); \ 232 std %r22, FRAME_22+CALLSIZE(r); \ 233 std %r23, FRAME_23+CALLSIZE(r); \ 234 std %r24, FRAME_24+CALLSIZE(r); \ 235 std %r25, FRAME_25+CALLSIZE(r); \ 236 std %r26, FRAME_26+CALLSIZE(r); \ 237 std %r27, FRAME_27+CALLSIZE(r); \ 238 std %r28, FRAME_28+CALLSIZE(r); \ 239 std %r29, FRAME_29+CALLSIZE(r); \ 240 std %r30, FRAME_30+CALLSIZE(r); \ 241 std %r31, FRAME_31+CALLSIZE(r) 242#define LD_REGS(r) \ 243 ld %r3, FRAME_3+CALLSIZE(r); \ 244 ld %r4, FRAME_4+CALLSIZE(r); \ 245 ld %r5, FRAME_5+CALLSIZE(r); \ 246 ld %r6, FRAME_6+CALLSIZE(r); \ 247 ld %r7, FRAME_7+CALLSIZE(r); \ 248 ld %r8, FRAME_8+CALLSIZE(r); \ 249 ld %r9, FRAME_9+CALLSIZE(r); \ 250 ld %r10, FRAME_10+CALLSIZE(r); \ 251 ld %r11, FRAME_11+CALLSIZE(r); \ 252 ld %r12, FRAME_12+CALLSIZE(r); \ 253 ld %r13, FRAME_13+CALLSIZE(r); \ 254 ld %r14, FRAME_14+CALLSIZE(r); \ 255 ld %r15, FRAME_15+CALLSIZE(r); \ 256 ld %r16, FRAME_16+CALLSIZE(r); \ 257 ld %r17, FRAME_17+CALLSIZE(r); \ 258 ld %r18, FRAME_18+CALLSIZE(r); \ 259 ld %r19, FRAME_19+CALLSIZE(r); \ 260 ld %r20, FRAME_20+CALLSIZE(r); \ 261 ld %r21, FRAME_21+CALLSIZE(r); \ 262 ld %r22, FRAME_22+CALLSIZE(r); \ 263 ld %r23, FRAME_23+CALLSIZE(r); \ 264 ld %r24, FRAME_24+CALLSIZE(r); \ 265 ld %r25, FRAME_25+CALLSIZE(r); \ 266 ld %r26, FRAME_26+CALLSIZE(r); \ 267 ld %r27, FRAME_27+CALLSIZE(r); \ 268 ld %r28, FRAME_28+CALLSIZE(r); \ 269 ld %r29, FRAME_29+CALLSIZE(r); \ 270 ld %r30, FRAME_30+CALLSIZE(r); \ 271 ld %r31, FRAME_31+CALLSIZE(r) 272#else 273#define SAVE_REGS(r) \ 274 stmw %r3, FRAME_3+CALLSIZE(r) 275#define LD_REGS(r) \ 276 lmw %r3, FRAME_3+CALLSIZE(r) 277#endif 278#define FRAME_SETUP(sprg_sp, savearea, exc) \ 279 mfspr %r31, sprg_sp; /* get saved SP */ \ 280 /* establish a new stack frame and put everything on it */ \ 281 STU %r31, -(FRAMELEN+REDZONE)(%r1); \ 282 STORE %r0, FRAME_0+CALLSIZE(%r1); /* save r0 in the trapframe */ \ 283 STORE %r31, FRAME_1+CALLSIZE(%r1); /* save SP " " */ \ 284 STORE %r2, FRAME_2+CALLSIZE(%r1); /* save r2 " " */ \ 285 mflr %r31; \ 286 STORE %r31, FRAME_LR+CALLSIZE(%r1); /* save LR " " */ \ 287 STORE %r30, FRAME_CR+CALLSIZE(%r1); /* save CR " " */ \ 288 GET_CPUINFO(%r2); \ 289 LOAD %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \ 290 LOAD %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \ 291 /* save R3-31 */ \ 292 SAVE_REGS(%r1); \ 293 /* save DEAR, ESR */ \ 294 LOAD %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2); \ 295 LOAD %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2); \ 296 STORE %r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1); \ 297 STORE %r29, FRAME_BOOKE_ESR+CALLSIZE(%r1); \ 298 /* save XER, CTR, exc number */ \ 299 mfxer %r3; \ 300 mfctr %r4; \ 301 STORE %r3, FRAME_XER+CALLSIZE(%r1); \ 302 STORE %r4, FRAME_CTR+CALLSIZE(%r1); \ 303 li %r5, exc; \ 304 STORE %r5, FRAME_EXC+CALLSIZE(%r1); \ 305 /* save DBCR0 */ \ 306 mfspr %r3, SPR_DBCR0; \ 307 STORE %r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \ 308 /* save xSSR0-1 */ \ 309 LOAD %r30, (savearea+CPUSAVE_SRR0)(%r2); \ 310 LOAD %r31, (savearea+CPUSAVE_SRR1)(%r2); \ 311 STORE %r30, FRAME_SRR0+CALLSIZE(%r1); \ 312 STORE %r31, FRAME_SRR1+CALLSIZE(%r1); \ 313 LOAD THREAD_REG, PC_CURTHREAD(%r2); \ 314 315/* 316 * 317 * sprg_sp - SPRG{1-3} reg free to use as a scratch once r1 is restored 318 * savearea - temp save area (pc_{tempsave, critsave, mchksave, dbsave}) 319 * isrr0-1 - save restore registers to restore CPU state to (may be 320 * SRR0-1, CSRR0-1, MCSRR0-1 321 * 322 * Notes: 323 * - potential TLB miss: YES. The deref'd kstack may be not covered 324 * 325 * - xSRR0-1 are staged in the savearea rather than written to the SPRs up 326 * front. A DTLB miss on the kstack below is not maskable by wrteei, and 327 * the hardware overwrites SRR0-1 on entry to the miss handler, which then 328 * restores its own values -- so anything parked in SRR0-1 across the 329 * register reload is lost, and the rfi returns into the middle of this 330 * macro instead of to the interrupted context. Everything after the last 331 * kstack access touches only pcpu and SPRs, which cannot fault. 332 */ 333#define FRAME_LEAVE(sprg_sp, savearea, isrr0, isrr1) \ 334 wrteei 0; \ 335 /* restore CTR, XER, LR, CR */ \ 336 LOAD %r4, FRAME_CTR+CALLSIZE(%r1); \ 337 LOAD %r5, FRAME_XER+CALLSIZE(%r1); \ 338 LOAD %r6, FRAME_LR+CALLSIZE(%r1); \ 339 LOAD %r7, FRAME_CR+CALLSIZE(%r1); \ 340 mtctr %r4; \ 341 mtxer %r5; \ 342 mtlr %r6; \ 343 mtcr %r7; \ 344 /* restore DBCR0 */ \ 345 LOAD %r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \ 346 mtspr SPR_DBCR0, %r4; \ 347 /* stage xSRR0-1 where a TLB miss cannot reach them */ \ 348 LOAD %r30, FRAME_SRR0+CALLSIZE(%r1); \ 349 LOAD %r31, FRAME_SRR1+CALLSIZE(%r1); \ 350 GET_CPUINFO(%r4); \ 351 STORE %r30, (savearea+CPUSAVE_SRR0)(%r4); \ 352 STORE %r31, (savearea+CPUSAVE_SRR1)(%r4); \ 353 /* restore R2-31, SP */ \ 354 LD_REGS(%r1); \ 355 LOAD %r2, FRAME_2+CALLSIZE(%r1); \ 356 LOAD %r0, FRAME_0+CALLSIZE(%r1); \ 357 LOAD %r1, FRAME_1+CALLSIZE(%r1); \ 358 /* no kstack references past here; park r3 to free a scratch */ \ 359 mtspr sprg_sp, %r3; \ 360 GET_CPUINFO(%r3); \ 361 LOAD %r3, (savearea+CPUSAVE_SRR0)(%r3); \ 362 mtspr isrr0, %r3; \ 363 GET_CPUINFO(%r3); \ 364 LOAD %r3, (savearea+CPUSAVE_SRR1)(%r3); \ 365 mtspr isrr1, %r3; \ 366 mfspr %r3, sprg_sp; \ 367 isync 368 369/* 370 * TLB miss prolog 371 * 372 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area 373 * 374 * Notes: 375 * - potential TLB miss: NO. It is crucial that we do not generate a TLB 376 * miss within the TLB prolog itself! 377 * - TLBSAVE is always translated 378 */ 379#ifdef __powerpc64__ 380#define TLB_SAVE_REGS(br) \ 381 std %r20, (TLBSAVE_BOOKE_R20)(br); \ 382 std %r21, (TLBSAVE_BOOKE_R21)(br); \ 383 std %r22, (TLBSAVE_BOOKE_R22)(br); \ 384 std %r23, (TLBSAVE_BOOKE_R23)(br); \ 385 std %r24, (TLBSAVE_BOOKE_R24)(br); \ 386 std %r25, (TLBSAVE_BOOKE_R25)(br); \ 387 std %r26, (TLBSAVE_BOOKE_R26)(br); \ 388 std %r27, (TLBSAVE_BOOKE_R27)(br); \ 389 std %r28, (TLBSAVE_BOOKE_R28)(br); \ 390 std %r29, (TLBSAVE_BOOKE_R29)(br); \ 391 std %r30, (TLBSAVE_BOOKE_R30)(br); \ 392 std %r31, (TLBSAVE_BOOKE_R31)(br); 393#define TLB_RESTORE_REGS(br) \ 394 ld %r20, (TLBSAVE_BOOKE_R20)(br); \ 395 ld %r21, (TLBSAVE_BOOKE_R21)(br); \ 396 ld %r22, (TLBSAVE_BOOKE_R22)(br); \ 397 ld %r23, (TLBSAVE_BOOKE_R23)(br); \ 398 ld %r24, (TLBSAVE_BOOKE_R24)(br); \ 399 ld %r25, (TLBSAVE_BOOKE_R25)(br); \ 400 ld %r26, (TLBSAVE_BOOKE_R26)(br); \ 401 ld %r27, (TLBSAVE_BOOKE_R27)(br); \ 402 ld %r28, (TLBSAVE_BOOKE_R28)(br); \ 403 ld %r29, (TLBSAVE_BOOKE_R29)(br); \ 404 ld %r30, (TLBSAVE_BOOKE_R30)(br); \ 405 ld %r31, (TLBSAVE_BOOKE_R31)(br); 406#define TLB_NEST(outr,inr) \ 407 rlwinm outr, inr, 7, 23, 24; /* 8 x TLBSAVE_LEN */ 408#else 409#define TLB_SAVE_REGS(br) \ 410 stmw %r20, TLBSAVE_BOOKE_R20(br) 411#define TLB_RESTORE_REGS(br) \ 412 lmw %r20, TLBSAVE_BOOKE_R20(br) 413#define TLB_NEST(outr,inr) \ 414 rlwinm outr, inr, 6, 24, 25; /* 4 x TLBSAVE_LEN */ 415#endif 416#define TLB_PROLOG \ 417 mtspr SPR_SPRG4, %r1; /* Save SP */ \ 418 mtspr SPR_SPRG5, %r28; \ 419 mtspr SPR_SPRG6, %r29; \ 420 /* calculate TLB nesting level and TLBSAVE instance address */ \ 421 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 422 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 423 TLB_NEST(%r29,%r28); \ 424 addi %r28, %r28, 1; \ 425 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 426 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 427 add %r1, %r1, %r29; /* current TLBSAVE ptr */ \ 428 \ 429 /* save R20-31 */ \ 430 mfspr %r28, SPR_SPRG5; \ 431 mfspr %r29, SPR_SPRG6; \ 432 TLB_SAVE_REGS(%r1); \ 433 /* save LR, CR */ \ 434 mflr %r30; \ 435 mfcr %r31; \ 436 STORE %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 437 STORE %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 438 /* save SRR0-1 */ \ 439 mfsrr0 %r30; /* execution addr at interrupt time */ \ 440 mfsrr1 %r31; /* MSR at interrupt time*/ \ 441 STORE %r30, (TLBSAVE_BOOKE_SRR0)(%r1); /* save SRR0 */ \ 442 STORE %r31, (TLBSAVE_BOOKE_SRR1)(%r1); /* save SRR1 */ \ 443 isync; \ 444 mfspr %r1, SPR_SPRG4 445 446/* 447 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area 448 * 449 * same notes as for the TLB_PROLOG 450 */ 451#define TLB_RESTORE \ 452 mtspr SPR_SPRG4, %r1; /* Save SP */ \ 453 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 454 /* calculate TLB nesting level and TLBSAVE instance addr */ \ 455 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 456 subi %r28, %r28, 1; \ 457 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 458 TLB_NEST(%r29,%r28); \ 459 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 460 add %r1, %r1, %r29; \ 461 \ 462 /* restore LR, CR */ \ 463 LOAD %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 464 LOAD %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 465 mtlr %r30; \ 466 mtcr %r31; \ 467 /* restore SRR0-1 */ \ 468 LOAD %r30, (TLBSAVE_BOOKE_SRR0)(%r1); \ 469 LOAD %r31, (TLBSAVE_BOOKE_SRR1)(%r1); \ 470 mtsrr0 %r30; \ 471 mtsrr1 %r31; \ 472 /* restore R20-31 */ \ 473 TLB_RESTORE_REGS(%r1); \ 474 mfspr %r1, SPR_SPRG4 475 476#ifdef SMP 477#define TLB_LOCK \ 478 GET_CPUINFO(%r20); \ 479 LOAD %r21, PC_CURTHREAD(%r20); \ 480 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \ 481 \ 4821: LOADX %r23, 0, %r22; \ 483 CMPI %r23, TLB_UNLOCKED; \ 484 beq 2f; \ 485 \ 486 /* check if this is recursion */ \ 487 CMPL cr0, %r21, %r23; \ 488 bne- 1b; \ 489 \ 4902: /* try to acquire lock */ \ 491 STOREX %r21, 0, %r22; \ 492 bne- 1b; \ 493 \ 494 /* got it, update recursion counter */ \ 495 lwz %r21, RES_RECURSE(%r22); \ 496 addi %r21, %r21, 1; \ 497 stw %r21, RES_RECURSE(%r22); \ 498 isync; \ 499 msync 500 501#define TLB_UNLOCK \ 502 GET_CPUINFO(%r20); \ 503 LOAD %r21, PC_CURTHREAD(%r20); \ 504 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \ 505 \ 506 /* update recursion counter */ \ 507 lwz %r23, RES_RECURSE(%r22); \ 508 subi %r23, %r23, 1; \ 509 stw %r23, RES_RECURSE(%r22); \ 510 \ 511 cmplwi %r23, 0; \ 512 bne 1f; \ 513 isync; \ 514 msync; \ 515 \ 516 /* release the lock */ \ 517 li %r23, TLB_UNLOCKED; \ 518 STORE %r23, 0(%r22); \ 5191: isync; \ 520 msync 521#else 522#define TLB_LOCK 523#define TLB_UNLOCK 524#endif /* SMP */ 525 526#define INTERRUPT(label) \ 527 .globl label; \ 528 .align 5; \ 529 CNAME(label): 530 531/* 532 * Interrupt handling routines in BookE can be flexibly placed and do not have 533 * to live in pre-defined vectors location. Note they need to be TLB-mapped at 534 * all times in order to be able to handle exceptions. We thus arrange for 535 * them to be part of kernel text which is always TLB-accessible. 536 * 537 * The interrupt handling routines have to be 16 bytes aligned: we align them 538 * to 32 bytes (cache line length) which supposedly performs better. 539 * 540 */ 541 .text 542 .globl CNAME(interrupt_vector_base) 543 .align 5 544interrupt_vector_base: 545/***************************************************************************** 546 * Catch-all handler to handle uninstalled IVORs 547 ****************************************************************************/ 548INTERRUPT(int_unknown) 549 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 550 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD) 551 b trap_common 552 553/***************************************************************************** 554 * Critical input interrupt 555 ****************************************************************************/ 556INTERRUPT(int_critical_input) 557 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, 558 PC_BOOKE_CRITSTACK, SPR_CSRR0, SPR_CSRR1) 559 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT) 560 GET_TOCBASE(%r2) 561 addi %r3, %r1, CALLSIZE 562 bl CNAME(powerpc_interrupt) 563 TOC_RESTORE 564 CRIT_SRR_RESTORE(PC_BOOKE_CRITSAVE) 565 FRAME_LEAVE(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 566 rfci 567 568 569/***************************************************************************** 570 * Machine check interrupt 571 ****************************************************************************/ 572INTERRUPT(int_machine_check) 573 STANDARD_CRIT_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, PC_BOOKE_MCHKSTACK, 574 SPR_MCSRR0, SPR_MCSRR1) 575 FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK) 576 GET_TOCBASE(%r2) 577 addi %r3, %r1, CALLSIZE 578 bl CNAME(powerpc_interrupt) 579 TOC_RESTORE 580 CRIT_SRR_RESTORE(PC_BOOKE_MCHKSAVE) 581 FRAME_LEAVE(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1) 582 rfmci 583 584 585/***************************************************************************** 586 * Data storage interrupt 587 ****************************************************************************/ 588INTERRUPT(int_data_storage) 589 STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1) 590 FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI) 591 b trap_common 592 593 594/***************************************************************************** 595 * Instruction storage interrupt 596 ****************************************************************************/ 597INTERRUPT(int_instr_storage) 598 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 599 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI) 600 b trap_common 601 602 603/***************************************************************************** 604 * External input interrupt 605 ****************************************************************************/ 606INTERRUPT(int_external_input) 607 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 608 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI) 609 b trap_common 610 611 612INTERRUPT(int_alignment) 613 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 614 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI) 615 b trap_common 616 617 618INTERRUPT(int_program) 619 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 620 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM) 621 b trap_common 622 623 624INTERRUPT(int_fpu) 625 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 626 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU) 627 b trap_common 628 629 630/***************************************************************************** 631 * System call 632 ****************************************************************************/ 633INTERRUPT(int_syscall) 634 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 635 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC) 636 b trap_common 637 638 639/***************************************************************************** 640 * Decrementer interrupt 641 ****************************************************************************/ 642INTERRUPT(int_decrementer) 643 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 644 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR) 645 b trap_common 646 647 648/***************************************************************************** 649 * Fixed interval timer 650 ****************************************************************************/ 651INTERRUPT(int_fixed_interval_timer) 652 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 653 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT) 654 b trap_common 655 656 657/***************************************************************************** 658 * Watchdog interrupt 659 ****************************************************************************/ 660INTERRUPT(int_watchdog) 661 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, 662 PC_BOOKE_CRITSTACK, SPR_CSRR0, SPR_CSRR1) 663 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_WDOG) 664 GET_TOCBASE(%r2) 665 addi %r3, %r1, CALLSIZE 666 bl CNAME(powerpc_interrupt) 667 TOC_RESTORE 668 CRIT_SRR_RESTORE(PC_BOOKE_CRITSAVE) 669 FRAME_LEAVE(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 670 rfci 671 672 673/***************************************************************************** 674 * Altivec Unavailable interrupt 675 ****************************************************************************/ 676INTERRUPT(int_vec) 677 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 678 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC) 679 b trap_common 680 681 682/***************************************************************************** 683 * Altivec Assist interrupt 684 ****************************************************************************/ 685INTERRUPT(int_vecast) 686 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 687 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E) 688 b trap_common 689 690 691#ifdef HWPMC_HOOKS 692/***************************************************************************** 693 * PMC Interrupt 694 ****************************************************************************/ 695INTERRUPT(int_performance_counter) 696 STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 697 FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF) 698 b trap_common 699#endif 700 701 702/***************************************************************************** 703 * Data TLB miss interrupt 704 * 705 * There can be nested TLB misses - while handling a TLB miss we reference 706 * data structures that may be not covered by translations. We support up to 707 * TLB_NESTED_MAX-1 nested misses. 708 * 709 * Registers use: 710 * r31 - dear 711 * r30 - unused 712 * r29 - saved mas0 713 * r28 - saved mas1 714 * r27 - saved mas2 715 * r26 - pmap address 716 * r25 - pte address 717 * 718 * r20:r23 - scratch registers 719 ****************************************************************************/ 720INTERRUPT(int_data_tlb_error) 721 TLB_PROLOG 722 TLB_LOCK 723 724 mfspr %r31, SPR_DEAR 725 726 /* 727 * Save MAS0-MAS2 registers. There might be another tlb miss during 728 * pte lookup overwriting current contents (which was hw filled). 729 */ 730 mfspr %r29, SPR_MAS0 731 mfspr %r28, SPR_MAS1 732 mfspr %r27, SPR_MAS2 733 734 /* Check faulting address. */ 735 LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS) 736 CMPL cr0, %r31, %r21 737 blt search_user_pmap 738 739 /* If it's kernel address, allow only supervisor mode misses. */ 740 mfsrr1 %r21 741 mtcr %r21 742 bt 17, search_failed /* check MSR[PR] */ 743 744#ifdef __powerpc64__ 745 srdi %r21, %r31, 48 746 cmpldi cr0, %r21, VM_MIN_KERNEL_ADDRESS@highest 747#else 748 lis %r21, VM_MIN_KERNEL_ADDRESS@h 749 cmplw cr0, %r31, %r21 750#endif 751 blt search_failed 752 753search_kernel_pmap: 754 /* Load r26 with kernel_pmap address */ 755 bl 1f 756#ifdef __powerpc64__ 757 .llong kernel_pmap_store-. 758#else 759 .long kernel_pmap_store-. 760#endif 7611: mflr %r21 762 LOAD %r26, 0(%r21) 763 add %r26, %r21, %r26 /* kernel_pmap_store in r26 */ 764 765 /* Force kernel tid, set TID to 0 in MAS1. */ 766 li %r21, 0 767 rlwimi %r28, %r21, 0, 2, 15 /* clear TID bits */ 768 769tlb_miss_handle: 770 /* This may result in nested tlb miss. */ 771 bl pte_lookup /* returns PTE address in R25 */ 772 773 CMPI %r25, 0 /* pte found? */ 774 beq search_failed 775 776 /* Finish up, write TLB entry. */ 777 bl tlb_fill_entry 778 779tlb_miss_return: 780 TLB_UNLOCK 781 TLB_RESTORE 782 rfi 783 784search_user_pmap: 785 /* Load r26 with current user space process pmap */ 786 GET_CPUINFO(%r26) 787 LOAD %r26, PC_CURPMAP(%r26) 788 789 b tlb_miss_handle 790 791search_failed: 792 /* 793 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with 794 * the faulting virtual address anyway, but put a fake RPN and no 795 * access rights. This should cause a following {D,I}SI exception. 796 */ 797 lis %r23, 0xffff0000@h /* revoke all permissions */ 798 799 /* Load MAS registers. */ 800 mtspr SPR_MAS0, %r29 801 mtspr SPR_MAS1, %r28 802 mtspr SPR_MAS2, %r27 803 mtspr SPR_MAS3, %r23 804 805 li %r23, 0 806 mtspr SPR_MAS7, %r23 807 808 isync 809 tlbwe 810 msync 811 isync 812 b tlb_miss_return 813 814/***************************************************************************** 815 * 816 * Return pte address that corresponds to given pmap/va. If there is no valid 817 * entry return 0. 818 * 819 * input: r26 - pmap 820 * input: r31 - dear 821 * output: r25 - pte address 822 * 823 * scratch regs used: r21 824 * 825 ****************************************************************************/ 826pte_lookup: 827 CMPI %r26, 0 828 beq 1f /* fail quickly if pmap is invalid */ 829 830#ifdef __powerpc64__ 831 rldicl %r21, %r31, (64 - PG_ROOT_L), (64 - PG_ROOT_NUM) /* pp2d offset */ 832 slwi %r21, %r21, PG_ROOT_ENTRY_SHIFT /* multiply by pp2d entry size */ 833 ld %r25, PM_ROOT(%r26) /* pmap pm_pp2d[] address */ 834 ldx %r25, %r25, %r21 /* get pdir address, i.e. pmap->pm_pp2d[pp2d_idx] * */ 835 836 cmpdi %r25, 0 837 beq 2f 838 839 rldicl %r21, %r31, (64 - PDIR_L1_L), (64 - PDIR_L1_NUM) /* pp2d offset */ 840 slwi %r21, %r21, PDIR_L1_ENTRY_SHIFT /* multiply by pp2d entry size */ 841 ldx %r25, %r25, %r21 /* get pdir address, i.e. pmap->pm_pp2d[pp2d_idx] * */ 842 843 cmpdi %r25, 0 844 beq 2f 845 846 rldicl %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM) /* pdir offset */ 847 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ 848 ldx %r25, %r25, %r21 /* get ptbl address, i.e. pmap->pm_pp2d[pp2d_idx][pdir_idx] */ 849 850 cmpdi %r25, 0 851 beq 2f 852 853 rldicl %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */ 854 slwi %r21, %r21, PTBL_ENTRY_SHIFT /* multiply by pte entry size */ 855 856#else 857 srwi %r21, %r31, PDIR_SHIFT /* pdir offset */ 858 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ 859 860 lwz %r25, PM_PDIR(%r26) /* pmap pm_dir[] address */ 861 /* 862 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx] 863 * This load may cause a Data TLB miss for non-kernel pmap! 864 */ 865 lwzx %r25, %r25, %r21 /* offset within pm_pdir[] table */ 866 cmpwi %r25, 0 867 beq 2f 868 869 lis %r21, PTBL_MASK@h 870 ori %r21, %r21, PTBL_MASK@l 871 and %r21, %r21, %r31 872 873 /* ptbl offset, multiply by ptbl entry size */ 874 srwi %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT) 875#endif 876 877 add %r25, %r25, %r21 /* address of pte entry */ 878 /* 879 * Get pte->flags 880 * This load may cause a Data TLB miss for non-kernel pmap! 881 */ 882 lwz %r21, PTE_FLAGS(%r25) 883 andi. %r21, %r21, PTE_VALID@l 884 bne 2f 8851: 886 li %r25, 0 8872: 888 blr 889 890/***************************************************************************** 891 * 892 * Load MAS1-MAS3 registers with data, write TLB entry 893 * 894 * input: 895 * r29 - mas0 896 * r28 - mas1 897 * r27 - mas2 898 * r25 - pte 899 * 900 * output: none 901 * 902 * scratch regs: r21-r23 903 * 904 ****************************************************************************/ 905tlb_fill_entry: 906 /* 907 * Update PTE flags: we have to do it atomically, as pmap_protect() 908 * running on other CPUs could attempt to update the flags at the same 909 * time. 910 */ 911 li %r23, PTE_FLAGS 9121: 913 lwarx %r21, %r23, %r25 /* get pte->flags */ 914 oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */ 915 916 andi. %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */ 917 beq 2f 918 ori %r21, %r21, PTE_MODIFIED@l /* set modified bit */ 9192: 920 stwcx. %r21, %r23, %r25 /* write it back */ 921 bne- 1b 922 923 /* Update MAS2. */ 924 rlwimi %r27, %r21, 13, 27, 30 /* insert WIMG bits from pte */ 925 926 /* Setup MAS3 value in r23. */ 927 LOAD %r23, PTE_RPN(%r25) /* get pte->rpn */ 928#ifdef __powerpc64__ 929 rldicr %r22, %r23, 52, 51 /* extract MAS3 portion of RPN */ 930 rldicl %r23, %r23, 20, 54 /* extract MAS7 portion of RPN */ 931 932 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ 933#else 934 rlwinm %r22, %r23, 20, 0, 11 /* extract MAS3 portion of RPN */ 935 936 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ 937 rlwimi %r22, %r21, 20, 12, 19 /* insert lower 8 RPN bits to MAS3 */ 938 rlwinm %r23, %r23, 20, 24, 31 /* MAS7 portion of RPN */ 939#endif 940 941 /* Load MAS registers. */ 942 mtspr SPR_MAS0, %r29 943 mtspr SPR_MAS1, %r28 944 mtspr SPR_MAS2, %r27 945 mtspr SPR_MAS3, %r22 946 mtspr SPR_MAS7, %r23 947 948 isync 949 tlbwe 950 isync 951 msync 952 blr 953 954/***************************************************************************** 955 * Instruction TLB miss interrupt 956 * 957 * Same notes as for the Data TLB miss 958 ****************************************************************************/ 959INTERRUPT(int_inst_tlb_error) 960 TLB_PROLOG 961 TLB_LOCK 962 963 mfsrr0 %r31 /* faulting address */ 964 965 /* 966 * Save MAS0-MAS2 registers. There might be another tlb miss during pte 967 * lookup overwriting current contents (which was hw filled). 968 */ 969 mfspr %r29, SPR_MAS0 970 mfspr %r28, SPR_MAS1 971 mfspr %r27, SPR_MAS2 972 973 mfsrr1 %r21 974 mtcr %r21 975 976 /* check MSR[PR] */ 977 bt 17, search_user_pmap 978 b search_kernel_pmap 979 980 981 .globl interrupt_vector_top 982interrupt_vector_top: 983 984/***************************************************************************** 985 * Debug interrupt 986 ****************************************************************************/ 987INTERRUPT(int_debug) 988 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, 989 PC_BOOKE_CRITSTACK, SPR_CSRR0, SPR_CSRR1) 990 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) 991 bl int_debug_int 992 FRAME_LEAVE(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 993 rfci 994 995INTERRUPT(int_debug_ed) 996 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, 997 PC_BOOKE_CRITSTACK, SPR_DSRR0, SPR_DSRR1) 998 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) 999 bl int_debug_int 1000 FRAME_LEAVE(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1) 1001 rfdi 1002 /* .long 0x4c00004e */ 1003 1004/* Internal helper for debug interrupt handling. */ 1005/* Common code between e500v1/v2 and e500mc-based cores. */ 1006int_debug_int: 1007 mflr %r14 1008 GET_CPUINFO(%r3) 1009 LOAD %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3) 1010 bl 0f 1011 ADDR(interrupt_vector_base-.) 1012 ADDR(interrupt_vector_top-.) 10130: mflr %r5 1014 LOAD %r4,0(%r5) /* interrupt_vector_base in r4 */ 1015 add %r4,%r4,%r5 1016 CMPL cr0, %r3, %r4 1017 blt trap_common 1018 LOAD %r4,WORD_SIZE(%r5) /* interrupt_vector_top in r4 */ 1019 add %r4,%r4,%r5 1020 addi %r4,%r4,4 1021 CMPL cr0, %r3, %r4 1022 bge trap_common 1023 /* Disable single-stepping for the interrupt handlers. */ 1024 LOAD %r3, FRAME_SRR1+CALLSIZE(%r1); 1025 rlwinm %r3, %r3, 0, 23, 21 1026 STORE %r3, FRAME_SRR1+CALLSIZE(%r1); 1027 /* Restore srr0 and srr1 as they could have been clobbered. */ 1028 GET_CPUINFO(%r4) 1029 LOAD %r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4); 1030 mtspr SPR_SRR0, %r3 1031 LOAD %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4); 1032 mtspr SPR_SRR1, %r4 1033 mtlr %r14 1034 blr 1035 1036/***************************************************************************** 1037 * Common trap code 1038 ****************************************************************************/ 1039trap_common: 1040 /* Call C trap dispatcher */ 1041 GET_TOCBASE(%r2) 1042 addi %r3, %r1, CALLSIZE 1043 bl CNAME(powerpc_interrupt) 1044 TOC_RESTORE 1045 1046 .globl CNAME(trapexit) /* exported for db_backtrace use */ 1047CNAME(trapexit): 1048 /* disable interrupts */ 1049 wrteei 0 1050 1051 /* Test AST pending - makes sense for user process only */ 1052 LOAD %r5, FRAME_SRR1+CALLSIZE(%r1) 1053 mtcr %r5 1054 bf 17, 1f 1055 1056 GET_CPUINFO(%r3) 1057 LOAD %r4, PC_CURTHREAD(%r3) 1058 lwz %r4, TD_AST(%r4) 1059 cmpwi %r4, 0 1060 beq 1f 1061 1062 /* re-enable interrupts before calling ast() */ 1063 wrteei 1 1064 1065 addi %r3, %r1, CALLSIZE 1066 bl CNAME(ast) 1067 TOC_RESTORE 1068 .globl CNAME(asttrapexit) /* db_backtrace code sentinel #2 */ 1069CNAME(asttrapexit): 1070 b trapexit /* test ast ret value ? */ 10711: 1072 FRAME_LEAVE(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 1073 rfi 1074 1075 1076#if defined(KDB) 1077/* 1078 * Deliberate entry to dbtrap 1079 */ 1080 /* .globl CNAME(breakpoint)*/ 1081ASENTRY_NOPROF(breakpoint) 1082 mtsprg1 %r1 1083 mfmsr %r3 1084 mtsrr1 %r3 1085 li %r4, ~(PSL_EE | PSL_ME)@l 1086 oris %r4, %r4, ~(PSL_EE | PSL_ME)@h 1087 and %r3, %r3, %r4 1088 mtmsr %r3 /* disable interrupts */ 1089 isync 1090 GET_CPUINFO(%r3) 1091 STORE %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3) 1092 STORE %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3) 1093 1094 mflr %r31 1095 mtsrr0 %r31 1096 1097 mfspr %r30, SPR_DEAR 1098 mfspr %r31, SPR_ESR 1099 STORE %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3) 1100 STORE %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3) 1101 1102 mfsrr0 %r30 1103 mfsrr1 %r31 1104 STORE %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3) 1105 STORE %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3) 1106 isync 1107 1108 mfcr %r30 1109 1110/* 1111 * Now the kdb trap catching code. 1112 */ 1113dbtrap: 1114 FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG) 1115/* Call C trap code: */ 1116 GET_TOCBASE(%r2) 1117 addi %r3, %r1, CALLSIZE 1118 bl CNAME(db_trap_glue) 1119 TOC_RESTORE 1120 or. %r3, %r3, %r3 1121 bne dbleave 1122/* This wasn't for KDB, so switch to real trap: */ 1123 b trap_common 1124 1125dbleave: 1126 FRAME_LEAVE(SPR_SPRG1, PC_DBSAVE, SPR_SRR0, SPR_SRR1) 1127 rfi 1128ASEND(breakpoint) 1129#endif /* KDB */ 1130 1131#ifdef SMP 1132ENTRY(tlb_lock) 1133 GET_CPUINFO(%r5) 1134 LOAD %r5, PC_CURTHREAD(%r5) 11351: LOADX %r4, 0, %r3 1136 CMPI %r4, TLB_UNLOCKED 1137 bne 1b 1138 STOREX %r5, 0, %r3 1139 bne- 1b 1140 isync 1141 msync 1142 blr 1143END(tlb_lock) 1144 1145ENTRY(tlb_unlock) 1146 isync 1147 msync 1148 li %r4, TLB_UNLOCKED 1149 STORE %r4, 0(%r3) 1150 isync 1151 msync 1152 blr 1153END(tlb_unlock) 1154 1155/* 1156 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes); 1157 * only a single word from this granule will actually be used as a spin lock 1158 * for mutual exclusion between TLB miss handler and pmap layer that 1159 * manipulates page table contents. 1160 */ 1161 .data 1162 .align 5 1163GLOBAL(tlb0_miss_locks) 1164 .space RES_GRANULE * MAXCPU 1165#endif 1166