1/*- 2 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com> 3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com> 4 * Copyright (C) 2006 Juniper Networks, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31/*- 32 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 33 * Copyright (C) 1995, 1996 TooLs GmbH. 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed by TooLs GmbH. 47 * 4. The name of TooLs GmbH may not be used to endorse or promote products 48 * derived from this software without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 60 * 61 * from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $ 62 */ 63 64/* 65 * NOTICE: This is not a standalone file. to use it, #include it in 66 * your port's locore.S, like so: 67 * 68 * #include <powerpc/booke/trap_subr.S> 69 */ 70 71/* 72 * SPRG usage notes 73 * 74 * SPRG0 - pcpu pointer 75 * SPRG1 - all interrupts except TLB miss, critical, machine check 76 * SPRG2 - critical 77 * SPRG3 - machine check 78 * SPRG4-6 - scratch 79 * 80 */ 81 82/* Get the per-CPU data structure */ 83#define GET_CPUINFO(r) mfsprg0 r 84 85#define RES_GRANULE 32 86#define RES_LOCK 0 /* offset to the 'lock' word */ 87#ifdef __powerpc64__ 88#define RES_RECURSE 8 /* offset to the 'recurse' word */ 89#else 90#define RES_RECURSE 4 /* offset to the 'recurse' word */ 91#endif 92 93/* 94 * Standard interrupt prolog 95 * 96 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP 97 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave}) 98 * isrr0-1 - save restore registers with CPU state at interrupt time (may be 99 * SRR0-1, CSRR0-1, MCSRR0-1 100 * 101 * 1. saves in the given savearea: 102 * - R30-31 103 * - DEAR, ESR 104 * - xSRR0-1 105 * 106 * 2. saves CR -> R30 107 * 108 * 3. switches to kstack if needed 109 * 110 * 4. notes: 111 * - R31 can be used as scratch register until a new frame is layed on 112 * the stack with FRAME_SETUP 113 * 114 * - potential TLB miss: NO. Saveareas are always acessible via TLB1 115 * permanent entries, and within this prolog we do not dereference any 116 * locations potentially not in the TLB 117 */ 118#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ 119 mtspr sprg_sp, %r1; /* Save SP */ \ 120 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 121 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \ 122 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \ 123 mfdear %r30; \ 124 mfesr %r31; \ 125 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 126 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 127 mfspr %r30, isrr0; \ 128 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 129 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 130 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 131 isync; \ 132 mfspr %r1, sprg_sp; /* Restore SP */ \ 133 mfcr %r30; /* Save CR */ \ 134 /* switch to per-thread kstack if intr taken in user mode */ \ 135 mtcr %r31; /* MSR at interrupt time */ \ 136 bf 17, 1f; \ 137 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 138 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1391: 140 141#define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ 142 mtspr sprg_sp, %r1; /* Save SP */ \ 143 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 144 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \ 145 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \ 146 mfdear %r30; \ 147 mfesr %r31; \ 148 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 149 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 150 mfspr %r30, isrr0; \ 151 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 152 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 153 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 154 mfspr %r30, SPR_SRR0; \ 155 mfspr %r31, SPR_SRR1; /* MSR at interrupt time */ \ 156 STORE %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1); \ 157 STORE %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1); \ 158 isync; \ 159 mfspr %r1, sprg_sp; /* Restore SP */ \ 160 mfcr %r30; /* Save CR */ \ 161 /* switch to per-thread kstack if intr taken in user mode */ \ 162 mtcr %r31; /* MSR at interrupt time */ \ 163 bf 17, 1f; \ 164 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 165 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1661: 167 168/* 169 * FRAME_SETUP assumes: 170 * SPRG{1-3} SP at the time interrupt occured 171 * savearea r30-r31, DEAR, ESR, xSRR0-1 172 * r30 CR 173 * r31 scratch 174 * r1 kernel stack 175 * 176 * sprg_sp - SPRG reg containing SP at the time interrupt occured 177 * savearea - temp save 178 * exc - exception number (EXC_xxx) 179 * 180 * 1. sets a new frame 181 * 2. saves in the frame: 182 * - R0, R1 (SP at the time of interrupt), R2, LR, CR 183 * - R3-31 (R30-31 first restored from savearea) 184 * - XER, CTR, DEAR, ESR (from savearea), xSRR0-1 185 * 186 * Notes: 187 * - potential TLB miss: YES, since we make dereferences to kstack, which 188 * can happen not covered (we can have up to two DTLB misses if fortunate 189 * enough i.e. when kstack crosses page boundary and both pages are 190 * untranslated) 191 */ 192#ifdef __powerpc64__ 193#define SAVE_REGS(r) \ 194 std %r3, FRAME_3+CALLSIZE(r); \ 195 std %r4, FRAME_4+CALLSIZE(r); \ 196 std %r5, FRAME_5+CALLSIZE(r); \ 197 std %r6, FRAME_6+CALLSIZE(r); \ 198 std %r7, FRAME_7+CALLSIZE(r); \ 199 std %r8, FRAME_8+CALLSIZE(r); \ 200 std %r9, FRAME_9+CALLSIZE(r); \ 201 std %r10, FRAME_10+CALLSIZE(r); \ 202 std %r11, FRAME_11+CALLSIZE(r); \ 203 std %r12, FRAME_12+CALLSIZE(r); \ 204 std %r13, FRAME_13+CALLSIZE(r); \ 205 std %r14, FRAME_14+CALLSIZE(r); \ 206 std %r15, FRAME_15+CALLSIZE(r); \ 207 std %r16, FRAME_16+CALLSIZE(r); \ 208 std %r17, FRAME_17+CALLSIZE(r); \ 209 std %r18, FRAME_18+CALLSIZE(r); \ 210 std %r19, FRAME_19+CALLSIZE(r); \ 211 std %r20, FRAME_20+CALLSIZE(r); \ 212 std %r21, FRAME_21+CALLSIZE(r); \ 213 std %r22, FRAME_22+CALLSIZE(r); \ 214 std %r23, FRAME_23+CALLSIZE(r); \ 215 std %r24, FRAME_24+CALLSIZE(r); \ 216 std %r25, FRAME_25+CALLSIZE(r); \ 217 std %r26, FRAME_26+CALLSIZE(r); \ 218 std %r27, FRAME_27+CALLSIZE(r); \ 219 std %r28, FRAME_28+CALLSIZE(r); \ 220 std %r29, FRAME_29+CALLSIZE(r); \ 221 std %r30, FRAME_30+CALLSIZE(r); \ 222 std %r31, FRAME_31+CALLSIZE(r) 223#define LD_REGS(r) \ 224 ld %r3, FRAME_3+CALLSIZE(r); \ 225 ld %r4, FRAME_4+CALLSIZE(r); \ 226 ld %r5, FRAME_5+CALLSIZE(r); \ 227 ld %r6, FRAME_6+CALLSIZE(r); \ 228 ld %r7, FRAME_7+CALLSIZE(r); \ 229 ld %r8, FRAME_8+CALLSIZE(r); \ 230 ld %r9, FRAME_9+CALLSIZE(r); \ 231 ld %r10, FRAME_10+CALLSIZE(r); \ 232 ld %r11, FRAME_11+CALLSIZE(r); \ 233 ld %r12, FRAME_12+CALLSIZE(r); \ 234 ld %r13, FRAME_13+CALLSIZE(r); \ 235 ld %r14, FRAME_14+CALLSIZE(r); \ 236 ld %r15, FRAME_15+CALLSIZE(r); \ 237 ld %r16, FRAME_16+CALLSIZE(r); \ 238 ld %r17, FRAME_17+CALLSIZE(r); \ 239 ld %r18, FRAME_18+CALLSIZE(r); \ 240 ld %r19, FRAME_19+CALLSIZE(r); \ 241 ld %r20, FRAME_20+CALLSIZE(r); \ 242 ld %r21, FRAME_21+CALLSIZE(r); \ 243 ld %r22, FRAME_22+CALLSIZE(r); \ 244 ld %r23, FRAME_23+CALLSIZE(r); \ 245 ld %r24, FRAME_24+CALLSIZE(r); \ 246 ld %r25, FRAME_25+CALLSIZE(r); \ 247 ld %r26, FRAME_26+CALLSIZE(r); \ 248 ld %r27, FRAME_27+CALLSIZE(r); \ 249 ld %r28, FRAME_28+CALLSIZE(r); \ 250 ld %r29, FRAME_29+CALLSIZE(r); \ 251 ld %r30, FRAME_30+CALLSIZE(r); \ 252 ld %r31, FRAME_31+CALLSIZE(r) 253#else 254#define SAVE_REGS(r) \ 255 stmw %r3, FRAME_3+CALLSIZE(r) 256#define LD_REGS(r) \ 257 lmw %r3, FRAME_3+CALLSIZE(r) 258#endif 259#define FRAME_SETUP(sprg_sp, savearea, exc) \ 260 mfspr %r31, sprg_sp; /* get saved SP */ \ 261 /* establish a new stack frame and put everything on it */ \ 262 STU %r31, -(FRAMELEN+REDZONE)(%r1); \ 263 STORE %r0, FRAME_0+CALLSIZE(%r1); /* save r0 in the trapframe */ \ 264 STORE %r31, FRAME_1+CALLSIZE(%r1); /* save SP " " */ \ 265 STORE %r2, FRAME_2+CALLSIZE(%r1); /* save r2 " " */ \ 266 mflr %r31; \ 267 STORE %r31, FRAME_LR+CALLSIZE(%r1); /* save LR " " */ \ 268 STORE %r30, FRAME_CR+CALLSIZE(%r1); /* save CR " " */ \ 269 GET_CPUINFO(%r2); \ 270 LOAD %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \ 271 LOAD %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \ 272 /* save R3-31 */ \ 273 SAVE_REGS(%r1); \ 274 /* save DEAR, ESR */ \ 275 LOAD %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2); \ 276 LOAD %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2); \ 277 STORE %r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1); \ 278 STORE %r29, FRAME_BOOKE_ESR+CALLSIZE(%r1); \ 279 /* save XER, CTR, exc number */ \ 280 mfxer %r3; \ 281 mfctr %r4; \ 282 STORE %r3, FRAME_XER+CALLSIZE(%r1); \ 283 STORE %r4, FRAME_CTR+CALLSIZE(%r1); \ 284 li %r5, exc; \ 285 STORE %r5, FRAME_EXC+CALLSIZE(%r1); \ 286 /* save DBCR0 */ \ 287 mfspr %r3, SPR_DBCR0; \ 288 STORE %r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \ 289 /* save xSSR0-1 */ \ 290 LOAD %r30, (savearea+CPUSAVE_SRR0)(%r2); \ 291 LOAD %r31, (savearea+CPUSAVE_SRR1)(%r2); \ 292 STORE %r30, FRAME_SRR0+CALLSIZE(%r1); \ 293 STORE %r31, FRAME_SRR1+CALLSIZE(%r1); \ 294 LOAD THREAD_REG, PC_CURTHREAD(%r2); \ 295 296/* 297 * 298 * isrr0-1 - save restore registers to restore CPU state to (may be 299 * SRR0-1, CSRR0-1, MCSRR0-1 300 * 301 * Notes: 302 * - potential TLB miss: YES. The deref'd kstack may be not covered 303 */ 304#define FRAME_LEAVE(isrr0, isrr1) \ 305 wrteei 0; \ 306 /* restore CTR, XER, LR, CR */ \ 307 LOAD %r4, FRAME_CTR+CALLSIZE(%r1); \ 308 LOAD %r5, FRAME_XER+CALLSIZE(%r1); \ 309 LOAD %r6, FRAME_LR+CALLSIZE(%r1); \ 310 LOAD %r7, FRAME_CR+CALLSIZE(%r1); \ 311 mtctr %r4; \ 312 mtxer %r5; \ 313 mtlr %r6; \ 314 mtcr %r7; \ 315 /* restore DBCR0 */ \ 316 LOAD %r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \ 317 mtspr SPR_DBCR0, %r4; \ 318 /* restore xSRR0-1 */ \ 319 LOAD %r30, FRAME_SRR0+CALLSIZE(%r1); \ 320 LOAD %r31, FRAME_SRR1+CALLSIZE(%r1); \ 321 mtspr isrr0, %r30; \ 322 mtspr isrr1, %r31; \ 323 /* restore R2-31, SP */ \ 324 LD_REGS(%r1); \ 325 LOAD %r2, FRAME_2+CALLSIZE(%r1); \ 326 LOAD %r0, FRAME_0+CALLSIZE(%r1); \ 327 LOAD %r1, FRAME_1+CALLSIZE(%r1); \ 328 isync 329 330/* 331 * TLB miss prolog 332 * 333 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area 334 * 335 * Notes: 336 * - potential TLB miss: NO. It is crucial that we do not generate a TLB 337 * miss within the TLB prolog itself! 338 * - TLBSAVE is always translated 339 */ 340#ifdef __powerpc64__ 341#define TLB_SAVE_REGS(br) \ 342 std %r20, (TLBSAVE_BOOKE_R20)(br); \ 343 std %r21, (TLBSAVE_BOOKE_R21)(br); \ 344 std %r22, (TLBSAVE_BOOKE_R22)(br); \ 345 std %r23, (TLBSAVE_BOOKE_R23)(br); \ 346 std %r24, (TLBSAVE_BOOKE_R24)(br); \ 347 std %r25, (TLBSAVE_BOOKE_R25)(br); \ 348 std %r26, (TLBSAVE_BOOKE_R26)(br); \ 349 std %r27, (TLBSAVE_BOOKE_R27)(br); \ 350 std %r28, (TLBSAVE_BOOKE_R28)(br); \ 351 std %r29, (TLBSAVE_BOOKE_R29)(br); \ 352 std %r30, (TLBSAVE_BOOKE_R30)(br); \ 353 std %r31, (TLBSAVE_BOOKE_R31)(br); 354#define TLB_RESTORE_REGS(br) \ 355 ld %r20, (TLBSAVE_BOOKE_R20)(br); \ 356 ld %r21, (TLBSAVE_BOOKE_R21)(br); \ 357 ld %r22, (TLBSAVE_BOOKE_R22)(br); \ 358 ld %r23, (TLBSAVE_BOOKE_R23)(br); \ 359 ld %r24, (TLBSAVE_BOOKE_R24)(br); \ 360 ld %r25, (TLBSAVE_BOOKE_R25)(br); \ 361 ld %r26, (TLBSAVE_BOOKE_R26)(br); \ 362 ld %r27, (TLBSAVE_BOOKE_R27)(br); \ 363 ld %r28, (TLBSAVE_BOOKE_R28)(br); \ 364 ld %r29, (TLBSAVE_BOOKE_R29)(br); \ 365 ld %r30, (TLBSAVE_BOOKE_R30)(br); \ 366 ld %r31, (TLBSAVE_BOOKE_R31)(br); 367#define TLB_NEST(outr,inr) \ 368 rlwinm outr, inr, 7, 22, 24; /* 8 x TLBSAVE_LEN */ 369#else 370#define TLB_SAVE_REGS(br) \ 371 stmw %r20, TLBSAVE_BOOKE_R20(br) 372#define TLB_RESTORE_REGS(br) \ 373 lmw %r20, TLBSAVE_BOOKE_R20(br) 374#define TLB_NEST(outr,inr) \ 375 rlwinm outr, inr, 6, 23, 25; /* 4 x TLBSAVE_LEN */ 376#endif 377#define TLB_PROLOG \ 378 mtsprg4 %r1; /* Save SP */ \ 379 mtsprg5 %r28; \ 380 mtsprg6 %r29; \ 381 /* calculate TLB nesting level and TLBSAVE instance address */ \ 382 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 383 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 384 TLB_NEST(%r29,%r28); \ 385 addi %r28, %r28, 1; \ 386 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 387 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 388 add %r1, %r1, %r29; /* current TLBSAVE ptr */ \ 389 \ 390 /* save R20-31 */ \ 391 mfsprg5 %r28; \ 392 mfsprg6 %r29; \ 393 TLB_SAVE_REGS(%r1); \ 394 /* save LR, CR */ \ 395 mflr %r30; \ 396 mfcr %r31; \ 397 STORE %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 398 STORE %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 399 /* save SRR0-1 */ \ 400 mfsrr0 %r30; /* execution addr at interrupt time */ \ 401 mfsrr1 %r31; /* MSR at interrupt time*/ \ 402 STORE %r30, (TLBSAVE_BOOKE_SRR0)(%r1); /* save SRR0 */ \ 403 STORE %r31, (TLBSAVE_BOOKE_SRR1)(%r1); /* save SRR1 */ \ 404 isync; \ 405 mfsprg4 %r1 406 407/* 408 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area 409 * 410 * same notes as for the TLB_PROLOG 411 */ 412#define TLB_RESTORE \ 413 mtsprg4 %r1; /* Save SP */ \ 414 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 415 /* calculate TLB nesting level and TLBSAVE instance addr */ \ 416 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 417 subi %r28, %r28, 1; \ 418 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 419 TLB_NEST(%r29,%r28); \ 420 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 421 add %r1, %r1, %r29; \ 422 \ 423 /* restore LR, CR */ \ 424 LOAD %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 425 LOAD %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 426 mtlr %r30; \ 427 mtcr %r31; \ 428 /* restore SRR0-1 */ \ 429 LOAD %r30, (TLBSAVE_BOOKE_SRR0)(%r1); \ 430 LOAD %r31, (TLBSAVE_BOOKE_SRR1)(%r1); \ 431 mtsrr0 %r30; \ 432 mtsrr1 %r31; \ 433 /* restore R20-31 */ \ 434 TLB_RESTORE_REGS(%r1); \ 435 mfsprg4 %r1 436 437#ifdef SMP 438#define TLB_LOCK \ 439 GET_CPUINFO(%r20); \ 440 LOAD %r21, PC_CURTHREAD(%r20); \ 441 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \ 442 \ 4431: LOADX %r23, 0, %r22; \ 444 CMPI %r23, TLB_UNLOCKED; \ 445 beq 2f; \ 446 \ 447 /* check if this is recursion */ \ 448 CMPL cr0, %r21, %r23; \ 449 bne- 1b; \ 450 \ 4512: /* try to acquire lock */ \ 452 STOREX %r21, 0, %r22; \ 453 bne- 1b; \ 454 \ 455 /* got it, update recursion counter */ \ 456 lwz %r21, RES_RECURSE(%r22); \ 457 addi %r21, %r21, 1; \ 458 stw %r21, RES_RECURSE(%r22); \ 459 isync; \ 460 msync 461 462#define TLB_UNLOCK \ 463 GET_CPUINFO(%r20); \ 464 LOAD %r21, PC_CURTHREAD(%r20); \ 465 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \ 466 \ 467 /* update recursion counter */ \ 468 lwz %r23, RES_RECURSE(%r22); \ 469 subi %r23, %r23, 1; \ 470 stw %r23, RES_RECURSE(%r22); \ 471 \ 472 cmplwi %r23, 0; \ 473 bne 1f; \ 474 isync; \ 475 msync; \ 476 \ 477 /* release the lock */ \ 478 li %r23, TLB_UNLOCKED; \ 479 STORE %r23, 0(%r22); \ 4801: isync; \ 481 msync 482#else 483#define TLB_LOCK 484#define TLB_UNLOCK 485#endif /* SMP */ 486 487#define INTERRUPT(label) \ 488 .globl label; \ 489 .align 5; \ 490 CNAME(label): 491 492/* 493 * Interrupt handling routines in BookE can be flexibly placed and do not have 494 * to live in pre-defined vectors location. Note they need to be TLB-mapped at 495 * all times in order to be able to handle exceptions. We thus arrange for 496 * them to be part of kernel text which is always TLB-accessible. 497 * 498 * The interrupt handling routines have to be 16 bytes aligned: we align them 499 * to 32 bytes (cache line length) which supposedly performs better. 500 * 501 */ 502 .text 503 .globl CNAME(interrupt_vector_base) 504 .align 5 505interrupt_vector_base: 506/***************************************************************************** 507 * Catch-all handler to handle uninstalled IVORs 508 ****************************************************************************/ 509INTERRUPT(int_unknown) 510 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 511 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD) 512 b trap_common 513 514/***************************************************************************** 515 * Critical input interrupt 516 ****************************************************************************/ 517INTERRUPT(int_critical_input) 518 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 519 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT) 520 GET_TOCBASE(%r2) 521 addi %r3, %r1, CALLSIZE 522 bl CNAME(powerpc_interrupt) 523 TOC_RESTORE 524 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) 525 rfci 526 527 528/***************************************************************************** 529 * Machine check interrupt 530 ****************************************************************************/ 531INTERRUPT(int_machine_check) 532 STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1) 533 FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK) 534 GET_TOCBASE(%r2) 535 addi %r3, %r1, CALLSIZE 536 bl CNAME(powerpc_interrupt) 537 TOC_RESTORE 538 FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1) 539 rfmci 540 541 542/***************************************************************************** 543 * Data storage interrupt 544 ****************************************************************************/ 545INTERRUPT(int_data_storage) 546 STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1) 547 FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI) 548 b trap_common 549 550 551/***************************************************************************** 552 * Instruction storage interrupt 553 ****************************************************************************/ 554INTERRUPT(int_instr_storage) 555 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 556 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI) 557 b trap_common 558 559 560/***************************************************************************** 561 * External input interrupt 562 ****************************************************************************/ 563INTERRUPT(int_external_input) 564 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 565 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI) 566 GET_TOCBASE(%r2) 567 addi %r3, %r1, CALLSIZE 568 bl CNAME(powerpc_interrupt) 569 TOC_RESTORE 570 b clear_we 571 572 573INTERRUPT(int_alignment) 574 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 575 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI) 576 b trap_common 577 578 579INTERRUPT(int_program) 580 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 581 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM) 582 b trap_common 583 584 585INTERRUPT(int_fpu) 586 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 587 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU) 588 b trap_common 589 590 591/***************************************************************************** 592 * System call 593 ****************************************************************************/ 594INTERRUPT(int_syscall) 595 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 596 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC) 597 b trap_common 598 599 600/***************************************************************************** 601 * Decrementer interrupt 602 ****************************************************************************/ 603INTERRUPT(int_decrementer) 604 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 605 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR) 606 GET_TOCBASE(%r2) 607 addi %r3, %r1, CALLSIZE 608 bl CNAME(powerpc_interrupt) 609 TOC_RESTORE 610 b clear_we 611 612 613/***************************************************************************** 614 * Fixed interval timer 615 ****************************************************************************/ 616INTERRUPT(int_fixed_interval_timer) 617 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 618 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT) 619 b trap_common 620 621 622/***************************************************************************** 623 * Watchdog interrupt 624 ****************************************************************************/ 625INTERRUPT(int_watchdog) 626 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 627 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG) 628 b trap_common 629 630 631/***************************************************************************** 632 * Altivec Unavailable interrupt 633 ****************************************************************************/ 634INTERRUPT(int_vec) 635 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 636 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC) 637 b trap_common 638 639 640/***************************************************************************** 641 * Watchdog interrupt 642 ****************************************************************************/ 643INTERRUPT(int_vecast) 644 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 645 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E) 646 b trap_common 647 648 649#ifdef HWPMC_HOOKS 650/***************************************************************************** 651 * PMC Interrupt 652 ****************************************************************************/ 653INTERRUPT(int_performance_counter) 654 STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 655 FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF) 656 GET_TOCBASE(%r2) 657 addi %r3, %r1, CALLSIZE 658 bl CNAME(powerpc_interrupt) 659 TOC_RESTORE 660 b trapexit 661#endif 662 663 664/***************************************************************************** 665 * Data TLB miss interrupt 666 * 667 * There can be nested TLB misses - while handling a TLB miss we reference 668 * data structures that may be not covered by translations. We support up to 669 * TLB_NESTED_MAX-1 nested misses. 670 * 671 * Registers use: 672 * r31 - dear 673 * r30 - unused 674 * r29 - saved mas0 675 * r28 - saved mas1 676 * r27 - saved mas2 677 * r26 - pmap address 678 * r25 - pte address 679 * 680 * r20:r23 - scratch registers 681 ****************************************************************************/ 682INTERRUPT(int_data_tlb_error) 683 TLB_PROLOG 684 TLB_LOCK 685 686 mfdear %r31 687 688 /* 689 * Save MAS0-MAS2 registers. There might be another tlb miss during 690 * pte lookup overwriting current contents (which was hw filled). 691 */ 692 mfspr %r29, SPR_MAS0 693 mfspr %r28, SPR_MAS1 694 mfspr %r27, SPR_MAS2 695 696 /* Check faulting address. */ 697 LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS) 698 CMPL cr0, %r31, %r21 699 blt search_user_pmap 700 701 /* If it's kernel address, allow only supervisor mode misses. */ 702 mfsrr1 %r21 703 mtcr %r21 704 bt 17, search_failed /* check MSR[PR] */ 705 706search_kernel_pmap: 707 /* Load r26 with kernel_pmap address */ 708 bl 1f 709#ifdef __powerpc64__ 710 .llong kernel_pmap_store-. 711#else 712 .long kernel_pmap_store-. 713#endif 7141: mflr %r21 715 LOAD %r26, 0(%r21) 716 add %r26, %r21, %r26 /* kernel_pmap_store in r26 */ 717 718 /* Force kernel tid, set TID to 0 in MAS1. */ 719 li %r21, 0 720 rlwimi %r28, %r21, 0, 8, 15 /* clear TID bits */ 721 722tlb_miss_handle: 723 /* This may result in nested tlb miss. */ 724 bl pte_lookup /* returns PTE address in R25 */ 725 726 CMPI %r25, 0 /* pte found? */ 727 beq search_failed 728 729 /* Finish up, write TLB entry. */ 730 bl tlb_fill_entry 731 732tlb_miss_return: 733 TLB_UNLOCK 734 TLB_RESTORE 735 rfi 736 737search_user_pmap: 738 /* Load r26 with current user space process pmap */ 739 GET_CPUINFO(%r26) 740 LOAD %r26, PC_CURPMAP(%r26) 741 742 b tlb_miss_handle 743 744search_failed: 745 /* 746 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with 747 * the faulting virtual address anyway, but put a fake RPN and no 748 * access rights. This should cause a following {D,I}SI exception. 749 */ 750 lis %r23, 0xffff0000@h /* revoke all permissions */ 751 752 /* Load MAS registers. */ 753 mtspr SPR_MAS0, %r29 754 isync 755 mtspr SPR_MAS1, %r28 756 isync 757 mtspr SPR_MAS2, %r27 758 isync 759 mtspr SPR_MAS3, %r23 760 isync 761 762 bl zero_mas7 763 bl zero_mas8 764 765 tlbwe 766 msync 767 isync 768 b tlb_miss_return 769 770/***************************************************************************** 771 * 772 * Return pte address that corresponds to given pmap/va. If there is no valid 773 * entry return 0. 774 * 775 * input: r26 - pmap 776 * input: r31 - dear 777 * output: r25 - pte address 778 * 779 * scratch regs used: r21 780 * 781 ****************************************************************************/ 782pte_lookup: 783 CMPI %r26, 0 784 beq 1f /* fail quickly if pmap is invalid */ 785 786#ifdef __powerpc64__ 787 rldicl %r21, %r31, (64 - PP2D_L_L), (64 - PP2D_L_NUM) /* pp2d offset */ 788 rldicl %r25, %r31, (64 - PP2D_H_L), (64 - PP2D_H_NUM) 789 rldimi %r21, %r25, PP2D_L_NUM, (64 - (PP2D_L_NUM + PP2D_H_NUM)) 790 slwi %r21, %r21, PP2D_ENTRY_SHIFT /* multiply by pp2d entry size */ 791 addi %r25, %r26, PM_PP2D /* pmap pm_pp2d[] address */ 792 add %r25, %r25, %r21 /* offset within pm_pp2d[] table */ 793 ld %r25, 0(%r25) /* get pdir address, i.e. pmap->pm_pp2d[pp2d_idx] * */ 794 795 cmpdi %r25, 0 796 beq 1f 797 798#if PAGE_SIZE < 65536 799 rldicl %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM) /* pdir offset */ 800 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ 801 add %r25, %r25, %r21 /* offset within pdir table */ 802 ld %r25, 0(%r25) /* get ptbl address, i.e. pmap->pm_pp2d[pp2d_idx][pdir_idx] */ 803 804 cmpdi %r25, 0 805 beq 1f 806#endif 807 808 rldicl %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */ 809 slwi %r21, %r21, PTBL_ENTRY_SHIFT /* multiply by pte entry size */ 810 811#else 812 srwi %r21, %r31, PDIR_SHIFT /* pdir offset */ 813 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ 814 815 addi %r25, %r26, PM_PDIR /* pmap pm_dir[] address */ 816 add %r25, %r25, %r21 /* offset within pm_pdir[] table */ 817 /* 818 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx] 819 * This load may cause a Data TLB miss for non-kernel pmap! 820 */ 821 LOAD %r25, 0(%r25) 822 CMPI %r25, 0 823 beq 2f 824 825 lis %r21, PTBL_MASK@h 826 ori %r21, %r21, PTBL_MASK@l 827 and %r21, %r21, %r31 828 829 /* ptbl offset, multiply by ptbl entry size */ 830 srwi %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT) 831#endif 832 833 add %r25, %r25, %r21 /* address of pte entry */ 834 /* 835 * Get pte->flags 836 * This load may cause a Data TLB miss for non-kernel pmap! 837 */ 838 lwz %r21, PTE_FLAGS(%r25) 839 andi. %r21, %r21, PTE_VALID@l 840 bne 2f 8411: 842 li %r25, 0 8432: 844 blr 845 846/***************************************************************************** 847 * 848 * Load MAS1-MAS3 registers with data, write TLB entry 849 * 850 * input: 851 * r29 - mas0 852 * r28 - mas1 853 * r27 - mas2 854 * r25 - pte 855 * 856 * output: none 857 * 858 * scratch regs: r21-r23 859 * 860 ****************************************************************************/ 861tlb_fill_entry: 862 /* 863 * Update PTE flags: we have to do it atomically, as pmap_protect() 864 * running on other CPUs could attempt to update the flags at the same 865 * time. 866 */ 867 li %r23, PTE_FLAGS 8681: 869 lwarx %r21, %r23, %r25 /* get pte->flags */ 870 oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */ 871 872 andi. %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */ 873 beq 2f 874 ori %r21, %r21, PTE_MODIFIED@l /* set modified bit */ 8752: 876 stwcx. %r21, %r23, %r25 /* write it back */ 877 bne- 1b 878 879 /* Update MAS2. */ 880 rlwimi %r27, %r21, 13, 27, 30 /* insert WIMG bits from pte */ 881 882 /* Setup MAS3 value in r23. */ 883 LOAD %r23, PTE_RPN(%r25) /* get pte->rpn */ 884#ifdef __powerpc64__ 885 rldicr %r22, %r23, 52, 51 /* extract MAS3 portion of RPN */ 886 rldicl %r23, %r23, 20, 54 /* extract MAS7 portion of RPN */ 887 888 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ 889#else 890 rlwinm %r22, %r23, 20, 0, 11 /* extract MAS3 portion of RPN */ 891 892 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ 893 rlwimi %r22, %r21, 20, 12, 19 /* insert lower 8 RPN bits to MAS3 */ 894 rlwinm %r23, %r23, 20, 24, 31 /* MAS7 portion of RPN */ 895#endif 896 897 /* Load MAS registers. */ 898 mtspr SPR_MAS0, %r29 899 isync 900 mtspr SPR_MAS1, %r28 901 isync 902 mtspr SPR_MAS2, %r27 903 isync 904 mtspr SPR_MAS3, %r22 905 isync 906 mtspr SPR_MAS7, %r23 907 isync 908 909 mflr %r21 910 bl zero_mas8 911 mtlr %r21 912 913 tlbwe 914 isync 915 msync 916 blr 917 918/***************************************************************************** 919 * Instruction TLB miss interrupt 920 * 921 * Same notes as for the Data TLB miss 922 ****************************************************************************/ 923INTERRUPT(int_inst_tlb_error) 924 TLB_PROLOG 925 TLB_LOCK 926 927 mfsrr0 %r31 /* faulting address */ 928 929 /* 930 * Save MAS0-MAS2 registers. There might be another tlb miss during pte 931 * lookup overwriting current contents (which was hw filled). 932 */ 933 mfspr %r29, SPR_MAS0 934 mfspr %r28, SPR_MAS1 935 mfspr %r27, SPR_MAS2 936 937 mfsrr1 %r21 938 mtcr %r21 939 940 /* check MSR[PR] */ 941 bt 17, search_user_pmap 942 b search_kernel_pmap 943 944 945 .globl interrupt_vector_top 946interrupt_vector_top: 947 948/***************************************************************************** 949 * Debug interrupt 950 ****************************************************************************/ 951INTERRUPT(int_debug) 952 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 953 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) 954 bl int_debug_int 955 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) 956 rfci 957 958INTERRUPT(int_debug_ed) 959 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1) 960 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) 961 bl int_debug_int 962 FRAME_LEAVE(SPR_DSRR0, SPR_DSRR1) 963 rfdi 964 /* .long 0x4c00004e */ 965 966/* Internal helper for debug interrupt handling. */ 967/* Common code between e500v1/v2 and e500mc-based cores. */ 968int_debug_int: 969 mflr %r14 970 GET_CPUINFO(%r3) 971 LOAD %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3) 972 bl 0f 973#ifdef __powerpc64__ 974 .llong interrupt_vector_base-. 975 .llong interrupt_vector_top-. 976#else 977 .long interrupt_vector_base-. 978 .long interrupt_vector_top-. 979#endif 9800: mflr %r5 981 LOAD %r4,0(%r5) /* interrupt_vector_base in r4 */ 982 add %r4,%r4,%r5 983 CMPL cr0, %r3, %r4 984 blt trap_common 985 LOAD %r4,4(%r5) /* interrupt_vector_top in r4 */ 986 add %r4,%r4,%r5 987 addi %r4,%r4,4 988 CMPL cr0, %r3, %r4 989 bge trap_common 990 /* Disable single-stepping for the interrupt handlers. */ 991 LOAD %r3, FRAME_SRR1+CALLSIZE(%r1); 992 rlwinm %r3, %r3, 0, 23, 21 993 STORE %r3, FRAME_SRR1+CALLSIZE(%r1); 994 /* Restore srr0 and srr1 as they could have been clobbered. */ 995 GET_CPUINFO(%r4) 996 LOAD %r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4); 997 mtspr SPR_SRR0, %r3 998 LOAD %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4); 999 mtspr SPR_SRR1, %r4 1000 mtlr %r14 1001 blr 1002 1003/***************************************************************************** 1004 * Common trap code 1005 ****************************************************************************/ 1006trap_common: 1007 /* Call C trap dispatcher */ 1008 GET_TOCBASE(%r2) 1009 addi %r3, %r1, CALLSIZE 1010 bl CNAME(trap) 1011 TOC_RESTORE 1012 1013 .globl CNAME(trapexit) /* exported for db_backtrace use */ 1014CNAME(trapexit): 1015 /* disable interrupts */ 1016 wrteei 0 1017 1018 /* Test AST pending - makes sense for user process only */ 1019 LOAD %r5, FRAME_SRR1+CALLSIZE(%r1) 1020 mtcr %r5 1021 bf 17, 1f 1022 1023 GET_CPUINFO(%r3) 1024 LOAD %r4, PC_CURTHREAD(%r3) 1025 lwz %r4, TD_FLAGS(%r4) 1026 lis %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h 1027 ori %r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l 1028 and. %r4, %r4, %r5 1029 beq 1f 1030 1031 /* re-enable interrupts before calling ast() */ 1032 wrteei 1 1033 1034 addi %r3, %r1, CALLSIZE 1035 bl CNAME(ast) 1036 TOC_RESTORE 1037 .globl CNAME(asttrapexit) /* db_backtrace code sentinel #2 */ 1038CNAME(asttrapexit): 1039 b trapexit /* test ast ret value ? */ 10401: 1041 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 1042 rfi 1043 1044 1045#if defined(KDB) 1046/* 1047 * Deliberate entry to dbtrap 1048 */ 1049 /* .globl CNAME(breakpoint)*/ 1050ASENTRY_NOPROF(breakpoint) 1051 mtsprg1 %r1 1052 mfmsr %r3 1053 mtsrr1 %r3 1054 li %r4, ~(PSL_EE | PSL_ME)@l 1055 oris %r4, %r4, ~(PSL_EE | PSL_ME)@h 1056 and %r3, %r3, %r4 1057 mtmsr %r3 /* disable interrupts */ 1058 isync 1059 GET_CPUINFO(%r3) 1060 STORE %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3) 1061 STORE %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3) 1062 1063 mflr %r31 1064 mtsrr0 %r31 1065 1066 mfdear %r30 1067 mfesr %r31 1068 STORE %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3) 1069 STORE %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3) 1070 1071 mfsrr0 %r30 1072 mfsrr1 %r31 1073 STORE %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3) 1074 STORE %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3) 1075 isync 1076 1077 mfcr %r30 1078 1079/* 1080 * Now the kdb trap catching code. 1081 */ 1082dbtrap: 1083 FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG) 1084/* Call C trap code: */ 1085 GET_TOCBASE(%r2) 1086 addi %r3, %r1, CALLSIZE 1087 bl CNAME(db_trap_glue) 1088 TOC_RESTORE 1089 or. %r3, %r3, %r3 1090 bne dbleave 1091/* This wasn't for KDB, so switch to real trap: */ 1092 b trap_common 1093 1094dbleave: 1095 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 1096 rfi 1097#endif /* KDB */ 1098 1099clear_we: 1100 LOAD %r3, (FRAME_SRR1+CALLSIZE)(%r1) 1101 rlwinm %r3, %r3, 0, 14, 12 1102 STORE %r3, (FRAME_SRR1+CALLSIZE)(%r1) 1103 b trapexit 1104 1105#ifdef SMP 1106ENTRY(tlb_lock) 1107 GET_CPUINFO(%r5) 1108 LOAD %r5, PC_CURTHREAD(%r5) 11091: LOADX %r4, 0, %r3 1110 CMPI %r4, TLB_UNLOCKED 1111 bne 1b 1112 STOREX %r5, 0, %r3 1113 bne- 1b 1114 isync 1115 msync 1116 blr 1117 1118ENTRY(tlb_unlock) 1119 isync 1120 msync 1121 li %r4, TLB_UNLOCKED 1122 STORE %r4, 0(%r3) 1123 isync 1124 msync 1125 blr 1126 1127/* 1128 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes); 1129 * only a single word from this granule will actually be used as a spin lock 1130 * for mutual exclusion between TLB miss handler and pmap layer that 1131 * manipulates page table contents. 1132 */ 1133 .data 1134 .align 5 1135GLOBAL(tlb0_miss_locks) 1136 .space RES_GRANULE * MAXCPU 1137#endif 1138