1/*- 2 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com> 3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com> 4 * Copyright (C) 2006 Juniper Networks, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31/*- 32 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 33 * Copyright (C) 1995, 1996 TooLs GmbH. 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed by TooLs GmbH. 47 * 4. The name of TooLs GmbH may not be used to endorse or promote products 48 * derived from this software without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 60 * 61 * from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $ 62 */ 63 64/* 65 * NOTICE: This is not a standalone file. to use it, #include it in 66 * your port's locore.S, like so: 67 * 68 * #include <powerpc/booke/trap_subr.S> 69 */ 70 71/* 72 * SPRG usage notes 73 * 74 * SPRG0 - pcpu pointer 75 * SPRG1 - all interrupts except TLB miss, critical, machine check 76 * SPRG2 - critical 77 * SPRG3 - machine check 78 * SPRG4-6 - scratch 79 * 80 */ 81 82/* Get the per-CPU data structure */ 83#define GET_CPUINFO(r) mfsprg0 r 84 85#define RES_GRANULE 64 86#define RES_LOCK 0 /* offset to the 'lock' word */ 87#ifdef __powerpc64__ 88#define RES_RECURSE 8 /* offset to the 'recurse' word */ 89#else 90#define RES_RECURSE 4 /* offset to the 'recurse' word */ 91#endif 92 93/* 94 * Standard interrupt prolog 95 * 96 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP 97 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave}) 98 * isrr0-1 - save restore registers with CPU state at interrupt time (may be 99 * SRR0-1, CSRR0-1, MCSRR0-1 100 * 101 * 1. saves in the given savearea: 102 * - R30-31 103 * - DEAR, ESR 104 * - xSRR0-1 105 * 106 * 2. saves CR -> R30 107 * 108 * 3. switches to kstack if needed 109 * 110 * 4. notes: 111 * - R31 can be used as scratch register until a new frame is layed on 112 * the stack with FRAME_SETUP 113 * 114 * - potential TLB miss: NO. Saveareas are always acessible via TLB1 115 * permanent entries, and within this prolog we do not dereference any 116 * locations potentially not in the TLB 117 */ 118#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ 119 mtspr sprg_sp, %r1; /* Save SP */ \ 120 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 121 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \ 122 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \ 123 mfdear %r30; \ 124 mfesr %r31; \ 125 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 126 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 127 mfspr %r30, isrr0; \ 128 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 129 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 130 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 131 isync; \ 132 mfspr %r1, sprg_sp; /* Restore SP */ \ 133 mfcr %r30; /* Save CR */ \ 134 /* switch to per-thread kstack if intr taken in user mode */ \ 135 mtcr %r31; /* MSR at interrupt time */ \ 136 bf 17, 1f; \ 137 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 138 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1391: 140 141#define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ 142 mtspr sprg_sp, %r1; /* Save SP */ \ 143 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 144 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \ 145 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \ 146 mfdear %r30; \ 147 mfesr %r31; \ 148 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 149 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 150 mfspr %r30, isrr0; \ 151 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 152 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 153 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 154 mfspr %r30, SPR_SRR0; \ 155 mfspr %r31, SPR_SRR1; /* MSR at interrupt time */ \ 156 STORE %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1); \ 157 STORE %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1); \ 158 isync; \ 159 mfspr %r1, sprg_sp; /* Restore SP */ \ 160 mfcr %r30; /* Save CR */ \ 161 /* switch to per-thread kstack if intr taken in user mode */ \ 162 mtcr %r31; /* MSR at interrupt time */ \ 163 bf 17, 1f; \ 164 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 165 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1661: 167 168/* 169 * FRAME_SETUP assumes: 170 * SPRG{1-3} SP at the time interrupt occured 171 * savearea r30-r31, DEAR, ESR, xSRR0-1 172 * r30 CR 173 * r31 scratch 174 * r1 kernel stack 175 * 176 * sprg_sp - SPRG reg containing SP at the time interrupt occured 177 * savearea - temp save 178 * exc - exception number (EXC_xxx) 179 * 180 * 1. sets a new frame 181 * 2. saves in the frame: 182 * - R0, R1 (SP at the time of interrupt), R2, LR, CR 183 * - R3-31 (R30-31 first restored from savearea) 184 * - XER, CTR, DEAR, ESR (from savearea), xSRR0-1 185 * 186 * Notes: 187 * - potential TLB miss: YES, since we make dereferences to kstack, which 188 * can happen not covered (we can have up to two DTLB misses if fortunate 189 * enough i.e. when kstack crosses page boundary and both pages are 190 * untranslated) 191 */ 192#ifdef __powerpc64__ 193#define SAVE_REGS(r) \ 194 std %r3, FRAME_3+CALLSIZE(r); \ 195 std %r4, FRAME_4+CALLSIZE(r); \ 196 std %r5, FRAME_5+CALLSIZE(r); \ 197 std %r6, FRAME_6+CALLSIZE(r); \ 198 std %r7, FRAME_7+CALLSIZE(r); \ 199 std %r8, FRAME_8+CALLSIZE(r); \ 200 std %r9, FRAME_9+CALLSIZE(r); \ 201 std %r10, FRAME_10+CALLSIZE(r); \ 202 std %r11, FRAME_11+CALLSIZE(r); \ 203 std %r12, FRAME_12+CALLSIZE(r); \ 204 std %r13, FRAME_13+CALLSIZE(r); \ 205 std %r14, FRAME_14+CALLSIZE(r); \ 206 std %r15, FRAME_15+CALLSIZE(r); \ 207 std %r16, FRAME_16+CALLSIZE(r); \ 208 std %r17, FRAME_17+CALLSIZE(r); \ 209 std %r18, FRAME_18+CALLSIZE(r); \ 210 std %r19, FRAME_19+CALLSIZE(r); \ 211 std %r20, FRAME_20+CALLSIZE(r); \ 212 std %r21, FRAME_21+CALLSIZE(r); \ 213 std %r22, FRAME_22+CALLSIZE(r); \ 214 std %r23, FRAME_23+CALLSIZE(r); \ 215 std %r24, FRAME_24+CALLSIZE(r); \ 216 std %r25, FRAME_25+CALLSIZE(r); \ 217 std %r26, FRAME_26+CALLSIZE(r); \ 218 std %r27, FRAME_27+CALLSIZE(r); \ 219 std %r28, FRAME_28+CALLSIZE(r); \ 220 std %r29, FRAME_29+CALLSIZE(r); \ 221 std %r30, FRAME_30+CALLSIZE(r); \ 222 std %r31, FRAME_31+CALLSIZE(r) 223#define LD_REGS(r) \ 224 ld %r3, FRAME_3+CALLSIZE(r); \ 225 ld %r4, FRAME_4+CALLSIZE(r); \ 226 ld %r5, FRAME_5+CALLSIZE(r); \ 227 ld %r6, FRAME_6+CALLSIZE(r); \ 228 ld %r7, FRAME_7+CALLSIZE(r); \ 229 ld %r8, FRAME_8+CALLSIZE(r); \ 230 ld %r9, FRAME_9+CALLSIZE(r); \ 231 ld %r10, FRAME_10+CALLSIZE(r); \ 232 ld %r11, FRAME_11+CALLSIZE(r); \ 233 ld %r12, FRAME_12+CALLSIZE(r); \ 234 ld %r13, FRAME_13+CALLSIZE(r); \ 235 ld %r14, FRAME_14+CALLSIZE(r); \ 236 ld %r15, FRAME_15+CALLSIZE(r); \ 237 ld %r16, FRAME_16+CALLSIZE(r); \ 238 ld %r17, FRAME_17+CALLSIZE(r); \ 239 ld %r18, FRAME_18+CALLSIZE(r); \ 240 ld %r19, FRAME_19+CALLSIZE(r); \ 241 ld %r20, FRAME_20+CALLSIZE(r); \ 242 ld %r21, FRAME_21+CALLSIZE(r); \ 243 ld %r22, FRAME_22+CALLSIZE(r); \ 244 ld %r23, FRAME_23+CALLSIZE(r); \ 245 ld %r24, FRAME_24+CALLSIZE(r); \ 246 ld %r25, FRAME_25+CALLSIZE(r); \ 247 ld %r26, FRAME_26+CALLSIZE(r); \ 248 ld %r27, FRAME_27+CALLSIZE(r); \ 249 ld %r28, FRAME_28+CALLSIZE(r); \ 250 ld %r29, FRAME_29+CALLSIZE(r); \ 251 ld %r30, FRAME_30+CALLSIZE(r); \ 252 ld %r31, FRAME_31+CALLSIZE(r) 253#else 254#define SAVE_REGS(r) \ 255 stmw %r3, FRAME_3+CALLSIZE(r) 256#define LD_REGS(r) \ 257 lmw %r3, FRAME_3+CALLSIZE(r) 258#endif 259#define FRAME_SETUP(sprg_sp, savearea, exc) \ 260 mfspr %r31, sprg_sp; /* get saved SP */ \ 261 /* establish a new stack frame and put everything on it */ \ 262 STU %r31, -(FRAMELEN+REDZONE)(%r1); \ 263 STORE %r0, FRAME_0+CALLSIZE(%r1); /* save r0 in the trapframe */ \ 264 STORE %r31, FRAME_1+CALLSIZE(%r1); /* save SP " " */ \ 265 STORE %r2, FRAME_2+CALLSIZE(%r1); /* save r2 " " */ \ 266 mflr %r31; \ 267 STORE %r31, FRAME_LR+CALLSIZE(%r1); /* save LR " " */ \ 268 STORE %r30, FRAME_CR+CALLSIZE(%r1); /* save CR " " */ \ 269 GET_CPUINFO(%r2); \ 270 LOAD %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \ 271 LOAD %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \ 272 /* save R3-31 */ \ 273 SAVE_REGS(%r1); \ 274 /* save DEAR, ESR */ \ 275 LOAD %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2); \ 276 LOAD %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2); \ 277 STORE %r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1); \ 278 STORE %r29, FRAME_BOOKE_ESR+CALLSIZE(%r1); \ 279 /* save XER, CTR, exc number */ \ 280 mfxer %r3; \ 281 mfctr %r4; \ 282 STORE %r3, FRAME_XER+CALLSIZE(%r1); \ 283 STORE %r4, FRAME_CTR+CALLSIZE(%r1); \ 284 li %r5, exc; \ 285 STORE %r5, FRAME_EXC+CALLSIZE(%r1); \ 286 /* save DBCR0 */ \ 287 mfspr %r3, SPR_DBCR0; \ 288 STORE %r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \ 289 /* save xSSR0-1 */ \ 290 LOAD %r30, (savearea+CPUSAVE_SRR0)(%r2); \ 291 LOAD %r31, (savearea+CPUSAVE_SRR1)(%r2); \ 292 STORE %r30, FRAME_SRR0+CALLSIZE(%r1); \ 293 STORE %r31, FRAME_SRR1+CALLSIZE(%r1); \ 294 LOAD THREAD_REG, PC_CURTHREAD(%r2); \ 295 296/* 297 * 298 * isrr0-1 - save restore registers to restore CPU state to (may be 299 * SRR0-1, CSRR0-1, MCSRR0-1 300 * 301 * Notes: 302 * - potential TLB miss: YES. The deref'd kstack may be not covered 303 */ 304#define FRAME_LEAVE(isrr0, isrr1) \ 305 wrteei 0; \ 306 /* restore CTR, XER, LR, CR */ \ 307 LOAD %r4, FRAME_CTR+CALLSIZE(%r1); \ 308 LOAD %r5, FRAME_XER+CALLSIZE(%r1); \ 309 LOAD %r6, FRAME_LR+CALLSIZE(%r1); \ 310 LOAD %r7, FRAME_CR+CALLSIZE(%r1); \ 311 mtctr %r4; \ 312 mtxer %r5; \ 313 mtlr %r6; \ 314 mtcr %r7; \ 315 /* restore DBCR0 */ \ 316 LOAD %r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \ 317 mtspr SPR_DBCR0, %r4; \ 318 /* restore xSRR0-1 */ \ 319 LOAD %r30, FRAME_SRR0+CALLSIZE(%r1); \ 320 LOAD %r31, FRAME_SRR1+CALLSIZE(%r1); \ 321 mtspr isrr0, %r30; \ 322 mtspr isrr1, %r31; \ 323 /* restore R2-31, SP */ \ 324 LD_REGS(%r1); \ 325 LOAD %r2, FRAME_2+CALLSIZE(%r1); \ 326 LOAD %r0, FRAME_0+CALLSIZE(%r1); \ 327 LOAD %r1, FRAME_1+CALLSIZE(%r1); \ 328 isync 329 330/* 331 * TLB miss prolog 332 * 333 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area 334 * 335 * Notes: 336 * - potential TLB miss: NO. It is crucial that we do not generate a TLB 337 * miss within the TLB prolog itself! 338 * - TLBSAVE is always translated 339 */ 340#ifdef __powerpc64__ 341#define TLB_SAVE_REGS(br) \ 342 std %r20, (TLBSAVE_BOOKE_R20)(br); \ 343 std %r21, (TLBSAVE_BOOKE_R21)(br); \ 344 std %r22, (TLBSAVE_BOOKE_R22)(br); \ 345 std %r23, (TLBSAVE_BOOKE_R23)(br); \ 346 std %r24, (TLBSAVE_BOOKE_R24)(br); \ 347 std %r25, (TLBSAVE_BOOKE_R25)(br); \ 348 std %r26, (TLBSAVE_BOOKE_R26)(br); \ 349 std %r27, (TLBSAVE_BOOKE_R27)(br); \ 350 std %r28, (TLBSAVE_BOOKE_R28)(br); \ 351 std %r29, (TLBSAVE_BOOKE_R29)(br); \ 352 std %r30, (TLBSAVE_BOOKE_R30)(br); \ 353 std %r31, (TLBSAVE_BOOKE_R31)(br); 354#define TLB_RESTORE_REGS(br) \ 355 ld %r20, (TLBSAVE_BOOKE_R20)(br); \ 356 ld %r21, (TLBSAVE_BOOKE_R21)(br); \ 357 ld %r22, (TLBSAVE_BOOKE_R22)(br); \ 358 ld %r23, (TLBSAVE_BOOKE_R23)(br); \ 359 ld %r24, (TLBSAVE_BOOKE_R24)(br); \ 360 ld %r25, (TLBSAVE_BOOKE_R25)(br); \ 361 ld %r26, (TLBSAVE_BOOKE_R26)(br); \ 362 ld %r27, (TLBSAVE_BOOKE_R27)(br); \ 363 ld %r28, (TLBSAVE_BOOKE_R28)(br); \ 364 ld %r29, (TLBSAVE_BOOKE_R29)(br); \ 365 ld %r30, (TLBSAVE_BOOKE_R30)(br); \ 366 ld %r31, (TLBSAVE_BOOKE_R31)(br); 367#define TLB_NEST(outr,inr) \ 368 rlwinm outr, inr, 7, 22, 24; /* 8 x TLBSAVE_LEN */ 369#else 370#define TLB_SAVE_REGS(br) \ 371 stmw %r20, TLBSAVE_BOOKE_R20(br) 372#define TLB_RESTORE_REGS(br) \ 373 lmw %r20, TLBSAVE_BOOKE_R20(br) 374#define TLB_NEST(outr,inr) \ 375 rlwinm outr, inr, 6, 23, 25; /* 4 x TLBSAVE_LEN */ 376#endif 377#define TLB_PROLOG \ 378 mtsprg4 %r1; /* Save SP */ \ 379 mtsprg5 %r28; \ 380 mtsprg6 %r29; \ 381 /* calculate TLB nesting level and TLBSAVE instance address */ \ 382 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 383 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 384 TLB_NEST(%r29,%r28); \ 385 addi %r28, %r28, 1; \ 386 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 387 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 388 add %r1, %r1, %r29; /* current TLBSAVE ptr */ \ 389 \ 390 /* save R20-31 */ \ 391 mfsprg5 %r28; \ 392 mfsprg6 %r29; \ 393 TLB_SAVE_REGS(%r1); \ 394 /* save LR, CR */ \ 395 mflr %r30; \ 396 mfcr %r31; \ 397 STORE %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 398 STORE %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 399 /* save SRR0-1 */ \ 400 mfsrr0 %r30; /* execution addr at interrupt time */ \ 401 mfsrr1 %r31; /* MSR at interrupt time*/ \ 402 STORE %r30, (TLBSAVE_BOOKE_SRR0)(%r1); /* save SRR0 */ \ 403 STORE %r31, (TLBSAVE_BOOKE_SRR1)(%r1); /* save SRR1 */ \ 404 isync; \ 405 mfsprg4 %r1 406 407/* 408 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area 409 * 410 * same notes as for the TLB_PROLOG 411 */ 412#define TLB_RESTORE \ 413 mtsprg4 %r1; /* Save SP */ \ 414 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 415 /* calculate TLB nesting level and TLBSAVE instance addr */ \ 416 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 417 subi %r28, %r28, 1; \ 418 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 419 TLB_NEST(%r29,%r28); \ 420 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 421 add %r1, %r1, %r29; \ 422 \ 423 /* restore LR, CR */ \ 424 LOAD %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 425 LOAD %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 426 mtlr %r30; \ 427 mtcr %r31; \ 428 /* restore SRR0-1 */ \ 429 LOAD %r30, (TLBSAVE_BOOKE_SRR0)(%r1); \ 430 LOAD %r31, (TLBSAVE_BOOKE_SRR1)(%r1); \ 431 mtsrr0 %r30; \ 432 mtsrr1 %r31; \ 433 /* restore R20-31 */ \ 434 TLB_RESTORE_REGS(%r1); \ 435 mfsprg4 %r1 436 437#ifdef SMP 438#define TLB_LOCK \ 439 GET_CPUINFO(%r20); \ 440 LOAD %r21, PC_CURTHREAD(%r20); \ 441 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \ 442 \ 4431: LOADX %r23, 0, %r22; \ 444 CMPI %r23, TLB_UNLOCKED; \ 445 beq 2f; \ 446 \ 447 /* check if this is recursion */ \ 448 CMPL cr0, %r21, %r23; \ 449 bne- 1b; \ 450 \ 4512: /* try to acquire lock */ \ 452 STOREX %r21, 0, %r22; \ 453 bne- 1b; \ 454 \ 455 /* got it, update recursion counter */ \ 456 lwz %r21, RES_RECURSE(%r22); \ 457 addi %r21, %r21, 1; \ 458 stw %r21, RES_RECURSE(%r22); \ 459 isync; \ 460 msync 461 462#define TLB_UNLOCK \ 463 GET_CPUINFO(%r20); \ 464 LOAD %r21, PC_CURTHREAD(%r20); \ 465 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \ 466 \ 467 /* update recursion counter */ \ 468 lwz %r23, RES_RECURSE(%r22); \ 469 subi %r23, %r23, 1; \ 470 stw %r23, RES_RECURSE(%r22); \ 471 \ 472 cmplwi %r23, 0; \ 473 bne 1f; \ 474 isync; \ 475 msync; \ 476 \ 477 /* release the lock */ \ 478 li %r23, TLB_UNLOCKED; \ 479 STORE %r23, 0(%r22); \ 4801: isync; \ 481 msync 482#else 483#define TLB_LOCK 484#define TLB_UNLOCK 485#endif /* SMP */ 486 487#define INTERRUPT(label) \ 488 .globl label; \ 489 .align 5; \ 490 CNAME(label): 491 492/* 493 * Interrupt handling routines in BookE can be flexibly placed and do not have 494 * to live in pre-defined vectors location. Note they need to be TLB-mapped at 495 * all times in order to be able to handle exceptions. We thus arrange for 496 * them to be part of kernel text which is always TLB-accessible. 497 * 498 * The interrupt handling routines have to be 16 bytes aligned: we align them 499 * to 32 bytes (cache line length) which supposedly performs better. 500 * 501 */ 502 .text 503 .globl CNAME(interrupt_vector_base) 504 .align 5 505interrupt_vector_base: 506/***************************************************************************** 507 * Catch-all handler to handle uninstalled IVORs 508 ****************************************************************************/ 509INTERRUPT(int_unknown) 510 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 511 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD) 512 b trap_common 513 514/***************************************************************************** 515 * Critical input interrupt 516 ****************************************************************************/ 517INTERRUPT(int_critical_input) 518 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 519 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT) 520 GET_TOCBASE(%r2) 521 addi %r3, %r1, CALLSIZE 522 bl CNAME(powerpc_interrupt) 523 TOC_RESTORE 524 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) 525 rfci 526 527 528/***************************************************************************** 529 * Machine check interrupt 530 ****************************************************************************/ 531INTERRUPT(int_machine_check) 532 STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1) 533 FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK) 534 GET_TOCBASE(%r2) 535 addi %r3, %r1, CALLSIZE 536 bl CNAME(powerpc_interrupt) 537 TOC_RESTORE 538 FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1) 539 rfmci 540 541 542/***************************************************************************** 543 * Data storage interrupt 544 ****************************************************************************/ 545INTERRUPT(int_data_storage) 546 STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1) 547 FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI) 548 b trap_common 549 550 551/***************************************************************************** 552 * Instruction storage interrupt 553 ****************************************************************************/ 554INTERRUPT(int_instr_storage) 555 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 556 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI) 557 b trap_common 558 559 560/***************************************************************************** 561 * External input interrupt 562 ****************************************************************************/ 563INTERRUPT(int_external_input) 564 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 565 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI) 566 b trap_common 567 568 569INTERRUPT(int_alignment) 570 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 571 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI) 572 b trap_common 573 574 575INTERRUPT(int_program) 576 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 577 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM) 578 b trap_common 579 580 581INTERRUPT(int_fpu) 582 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 583 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU) 584 b trap_common 585 586 587/***************************************************************************** 588 * System call 589 ****************************************************************************/ 590INTERRUPT(int_syscall) 591 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 592 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC) 593 b trap_common 594 595 596/***************************************************************************** 597 * Decrementer interrupt 598 ****************************************************************************/ 599INTERRUPT(int_decrementer) 600 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 601 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR) 602 b trap_common 603 604 605/***************************************************************************** 606 * Fixed interval timer 607 ****************************************************************************/ 608INTERRUPT(int_fixed_interval_timer) 609 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 610 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT) 611 b trap_common 612 613 614/***************************************************************************** 615 * Watchdog interrupt 616 ****************************************************************************/ 617INTERRUPT(int_watchdog) 618 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 619 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG) 620 b trap_common 621 622 623/***************************************************************************** 624 * Altivec Unavailable interrupt 625 ****************************************************************************/ 626INTERRUPT(int_vec) 627 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 628 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC) 629 b trap_common 630 631 632/***************************************************************************** 633 * Altivec Assist interrupt 634 ****************************************************************************/ 635INTERRUPT(int_vecast) 636 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 637 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E) 638 b trap_common 639 640 641#ifdef HWPMC_HOOKS 642/***************************************************************************** 643 * PMC Interrupt 644 ****************************************************************************/ 645INTERRUPT(int_performance_counter) 646 STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 647 FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF) 648 b trap_common 649#endif 650 651 652/***************************************************************************** 653 * Data TLB miss interrupt 654 * 655 * There can be nested TLB misses - while handling a TLB miss we reference 656 * data structures that may be not covered by translations. We support up to 657 * TLB_NESTED_MAX-1 nested misses. 658 * 659 * Registers use: 660 * r31 - dear 661 * r30 - unused 662 * r29 - saved mas0 663 * r28 - saved mas1 664 * r27 - saved mas2 665 * r26 - pmap address 666 * r25 - pte address 667 * 668 * r20:r23 - scratch registers 669 ****************************************************************************/ 670INTERRUPT(int_data_tlb_error) 671 TLB_PROLOG 672 TLB_LOCK 673 674 mfdear %r31 675 676 /* 677 * Save MAS0-MAS2 registers. There might be another tlb miss during 678 * pte lookup overwriting current contents (which was hw filled). 679 */ 680 mfspr %r29, SPR_MAS0 681 mfspr %r28, SPR_MAS1 682 mfspr %r27, SPR_MAS2 683 684 /* Check faulting address. */ 685 LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS) 686 CMPL cr0, %r31, %r21 687 blt search_user_pmap 688 689 /* If it's kernel address, allow only supervisor mode misses. */ 690 mfsrr1 %r21 691 mtcr %r21 692 bt 17, search_failed /* check MSR[PR] */ 693 694search_kernel_pmap: 695 /* Load r26 with kernel_pmap address */ 696 bl 1f 697#ifdef __powerpc64__ 698 .llong kernel_pmap_store-. 699#else 700 .long kernel_pmap_store-. 701#endif 7021: mflr %r21 703 LOAD %r26, 0(%r21) 704 add %r26, %r21, %r26 /* kernel_pmap_store in r26 */ 705 706 /* Force kernel tid, set TID to 0 in MAS1. */ 707 li %r21, 0 708 rlwimi %r28, %r21, 0, 8, 15 /* clear TID bits */ 709 710tlb_miss_handle: 711 /* This may result in nested tlb miss. */ 712 bl pte_lookup /* returns PTE address in R25 */ 713 714 CMPI %r25, 0 /* pte found? */ 715 beq search_failed 716 717 /* Finish up, write TLB entry. */ 718 bl tlb_fill_entry 719 720tlb_miss_return: 721 TLB_UNLOCK 722 TLB_RESTORE 723 rfi 724 725search_user_pmap: 726 /* Load r26 with current user space process pmap */ 727 GET_CPUINFO(%r26) 728 LOAD %r26, PC_CURPMAP(%r26) 729 730 b tlb_miss_handle 731 732search_failed: 733 /* 734 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with 735 * the faulting virtual address anyway, but put a fake RPN and no 736 * access rights. This should cause a following {D,I}SI exception. 737 */ 738 lis %r23, 0xffff0000@h /* revoke all permissions */ 739 740 /* Load MAS registers. */ 741 mtspr SPR_MAS0, %r29 742 isync 743 mtspr SPR_MAS1, %r28 744 isync 745 mtspr SPR_MAS2, %r27 746 isync 747 mtspr SPR_MAS3, %r23 748 isync 749 750 bl zero_mas7 751 bl zero_mas8 752 753 tlbwe 754 msync 755 isync 756 b tlb_miss_return 757 758/***************************************************************************** 759 * 760 * Return pte address that corresponds to given pmap/va. If there is no valid 761 * entry return 0. 762 * 763 * input: r26 - pmap 764 * input: r31 - dear 765 * output: r25 - pte address 766 * 767 * scratch regs used: r21 768 * 769 ****************************************************************************/ 770pte_lookup: 771 CMPI %r26, 0 772 beq 1f /* fail quickly if pmap is invalid */ 773 774#ifdef __powerpc64__ 775 rldicl %r21, %r31, (64 - PP2D_L_L), (64 - PP2D_L_NUM) /* pp2d offset */ 776 rldicl %r25, %r31, (64 - PP2D_H_L), (64 - PP2D_H_NUM) 777 rldimi %r21, %r25, PP2D_L_NUM, (64 - (PP2D_L_NUM + PP2D_H_NUM)) 778 slwi %r21, %r21, PP2D_ENTRY_SHIFT /* multiply by pp2d entry size */ 779 addi %r25, %r26, PM_PP2D /* pmap pm_pp2d[] address */ 780 add %r25, %r25, %r21 /* offset within pm_pp2d[] table */ 781 ld %r25, 0(%r25) /* get pdir address, i.e. pmap->pm_pp2d[pp2d_idx] * */ 782 783 cmpdi %r25, 0 784 beq 1f 785 786#if PAGE_SIZE < 65536 787 rldicl %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM) /* pdir offset */ 788 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ 789 add %r25, %r25, %r21 /* offset within pdir table */ 790 ld %r25, 0(%r25) /* get ptbl address, i.e. pmap->pm_pp2d[pp2d_idx][pdir_idx] */ 791 792 cmpdi %r25, 0 793 beq 1f 794#endif 795 796 rldicl %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */ 797 slwi %r21, %r21, PTBL_ENTRY_SHIFT /* multiply by pte entry size */ 798 799#else 800 srwi %r21, %r31, PDIR_SHIFT /* pdir offset */ 801 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ 802 803 addi %r25, %r26, PM_PDIR /* pmap pm_dir[] address */ 804 add %r25, %r25, %r21 /* offset within pm_pdir[] table */ 805 /* 806 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx] 807 * This load may cause a Data TLB miss for non-kernel pmap! 808 */ 809 LOAD %r25, 0(%r25) 810 CMPI %r25, 0 811 beq 2f 812 813 lis %r21, PTBL_MASK@h 814 ori %r21, %r21, PTBL_MASK@l 815 and %r21, %r21, %r31 816 817 /* ptbl offset, multiply by ptbl entry size */ 818 srwi %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT) 819#endif 820 821 add %r25, %r25, %r21 /* address of pte entry */ 822 /* 823 * Get pte->flags 824 * This load may cause a Data TLB miss for non-kernel pmap! 825 */ 826 lwz %r21, PTE_FLAGS(%r25) 827 andi. %r21, %r21, PTE_VALID@l 828 bne 2f 8291: 830 li %r25, 0 8312: 832 blr 833 834/***************************************************************************** 835 * 836 * Load MAS1-MAS3 registers with data, write TLB entry 837 * 838 * input: 839 * r29 - mas0 840 * r28 - mas1 841 * r27 - mas2 842 * r25 - pte 843 * 844 * output: none 845 * 846 * scratch regs: r21-r23 847 * 848 ****************************************************************************/ 849tlb_fill_entry: 850 /* 851 * Update PTE flags: we have to do it atomically, as pmap_protect() 852 * running on other CPUs could attempt to update the flags at the same 853 * time. 854 */ 855 li %r23, PTE_FLAGS 8561: 857 lwarx %r21, %r23, %r25 /* get pte->flags */ 858 oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */ 859 860 andi. %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */ 861 beq 2f 862 ori %r21, %r21, PTE_MODIFIED@l /* set modified bit */ 8632: 864 stwcx. %r21, %r23, %r25 /* write it back */ 865 bne- 1b 866 867 /* Update MAS2. */ 868 rlwimi %r27, %r21, 13, 27, 30 /* insert WIMG bits from pte */ 869 870 /* Setup MAS3 value in r23. */ 871 LOAD %r23, PTE_RPN(%r25) /* get pte->rpn */ 872#ifdef __powerpc64__ 873 rldicr %r22, %r23, 52, 51 /* extract MAS3 portion of RPN */ 874 rldicl %r23, %r23, 20, 54 /* extract MAS7 portion of RPN */ 875 876 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ 877#else 878 rlwinm %r22, %r23, 20, 0, 11 /* extract MAS3 portion of RPN */ 879 880 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ 881 rlwimi %r22, %r21, 20, 12, 19 /* insert lower 8 RPN bits to MAS3 */ 882 rlwinm %r23, %r23, 20, 24, 31 /* MAS7 portion of RPN */ 883#endif 884 885 /* Load MAS registers. */ 886 mtspr SPR_MAS0, %r29 887 isync 888 mtspr SPR_MAS1, %r28 889 isync 890 mtspr SPR_MAS2, %r27 891 isync 892 mtspr SPR_MAS3, %r22 893 isync 894 mtspr SPR_MAS7, %r23 895 isync 896 897 mflr %r21 898 bl zero_mas8 899 mtlr %r21 900 901 tlbwe 902 isync 903 msync 904 blr 905 906/***************************************************************************** 907 * Instruction TLB miss interrupt 908 * 909 * Same notes as for the Data TLB miss 910 ****************************************************************************/ 911INTERRUPT(int_inst_tlb_error) 912 TLB_PROLOG 913 TLB_LOCK 914 915 mfsrr0 %r31 /* faulting address */ 916 917 /* 918 * Save MAS0-MAS2 registers. There might be another tlb miss during pte 919 * lookup overwriting current contents (which was hw filled). 920 */ 921 mfspr %r29, SPR_MAS0 922 mfspr %r28, SPR_MAS1 923 mfspr %r27, SPR_MAS2 924 925 mfsrr1 %r21 926 mtcr %r21 927 928 /* check MSR[PR] */ 929 bt 17, search_user_pmap 930 b search_kernel_pmap 931 932 933 .globl interrupt_vector_top 934interrupt_vector_top: 935 936/***************************************************************************** 937 * Debug interrupt 938 ****************************************************************************/ 939INTERRUPT(int_debug) 940 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 941 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) 942 bl int_debug_int 943 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) 944 rfci 945 946INTERRUPT(int_debug_ed) 947 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1) 948 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) 949 bl int_debug_int 950 FRAME_LEAVE(SPR_DSRR0, SPR_DSRR1) 951 rfdi 952 /* .long 0x4c00004e */ 953 954/* Internal helper for debug interrupt handling. */ 955/* Common code between e500v1/v2 and e500mc-based cores. */ 956int_debug_int: 957 mflr %r14 958 GET_CPUINFO(%r3) 959 LOAD %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3) 960 bl 0f 961 ADDR(interrupt_vector_base-.) 962 ADDR(interrupt_vector_top-.) 9630: mflr %r5 964 LOAD %r4,0(%r5) /* interrupt_vector_base in r4 */ 965 add %r4,%r4,%r5 966 CMPL cr0, %r3, %r4 967 blt trap_common 968 LOAD %r4,WORD_SIZE(%r5) /* interrupt_vector_top in r4 */ 969 add %r4,%r4,%r5 970 addi %r4,%r4,4 971 CMPL cr0, %r3, %r4 972 bge trap_common 973 /* Disable single-stepping for the interrupt handlers. */ 974 LOAD %r3, FRAME_SRR1+CALLSIZE(%r1); 975 rlwinm %r3, %r3, 0, 23, 21 976 STORE %r3, FRAME_SRR1+CALLSIZE(%r1); 977 /* Restore srr0 and srr1 as they could have been clobbered. */ 978 GET_CPUINFO(%r4) 979 LOAD %r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4); 980 mtspr SPR_SRR0, %r3 981 LOAD %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4); 982 mtspr SPR_SRR1, %r4 983 mtlr %r14 984 blr 985 986/***************************************************************************** 987 * Common trap code 988 ****************************************************************************/ 989trap_common: 990 /* Call C trap dispatcher */ 991 GET_TOCBASE(%r2) 992 addi %r3, %r1, CALLSIZE 993 bl CNAME(powerpc_interrupt) 994 TOC_RESTORE 995 996 .globl CNAME(trapexit) /* exported for db_backtrace use */ 997CNAME(trapexit): 998 /* disable interrupts */ 999 wrteei 0 1000 1001 /* Test AST pending - makes sense for user process only */ 1002 LOAD %r5, FRAME_SRR1+CALLSIZE(%r1) 1003 mtcr %r5 1004 bf 17, 1f 1005 1006 GET_CPUINFO(%r3) 1007 LOAD %r4, PC_CURTHREAD(%r3) 1008 lwz %r4, TD_FLAGS(%r4) 1009 lis %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h 1010 ori %r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l 1011 and. %r4, %r4, %r5 1012 beq 1f 1013 1014 /* re-enable interrupts before calling ast() */ 1015 wrteei 1 1016 1017 addi %r3, %r1, CALLSIZE 1018 bl CNAME(ast) 1019 TOC_RESTORE 1020 .globl CNAME(asttrapexit) /* db_backtrace code sentinel #2 */ 1021CNAME(asttrapexit): 1022 b trapexit /* test ast ret value ? */ 10231: 1024 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 1025 rfi 1026 1027 1028#if defined(KDB) 1029/* 1030 * Deliberate entry to dbtrap 1031 */ 1032 /* .globl CNAME(breakpoint)*/ 1033ASENTRY_NOPROF(breakpoint) 1034 mtsprg1 %r1 1035 mfmsr %r3 1036 mtsrr1 %r3 1037 li %r4, ~(PSL_EE | PSL_ME)@l 1038 oris %r4, %r4, ~(PSL_EE | PSL_ME)@h 1039 and %r3, %r3, %r4 1040 mtmsr %r3 /* disable interrupts */ 1041 isync 1042 GET_CPUINFO(%r3) 1043 STORE %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3) 1044 STORE %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3) 1045 1046 mflr %r31 1047 mtsrr0 %r31 1048 1049 mfdear %r30 1050 mfesr %r31 1051 STORE %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3) 1052 STORE %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3) 1053 1054 mfsrr0 %r30 1055 mfsrr1 %r31 1056 STORE %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3) 1057 STORE %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3) 1058 isync 1059 1060 mfcr %r30 1061 1062/* 1063 * Now the kdb trap catching code. 1064 */ 1065dbtrap: 1066 FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG) 1067/* Call C trap code: */ 1068 GET_TOCBASE(%r2) 1069 addi %r3, %r1, CALLSIZE 1070 bl CNAME(db_trap_glue) 1071 TOC_RESTORE 1072 or. %r3, %r3, %r3 1073 bne dbleave 1074/* This wasn't for KDB, so switch to real trap: */ 1075 b trap_common 1076 1077dbleave: 1078 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 1079 rfi 1080#endif /* KDB */ 1081 1082#ifdef SMP 1083ENTRY(tlb_lock) 1084 GET_CPUINFO(%r5) 1085 LOAD %r5, PC_CURTHREAD(%r5) 10861: LOADX %r4, 0, %r3 1087 CMPI %r4, TLB_UNLOCKED 1088 bne 1b 1089 STOREX %r5, 0, %r3 1090 bne- 1b 1091 isync 1092 msync 1093 blr 1094 1095ENTRY(tlb_unlock) 1096 isync 1097 msync 1098 li %r4, TLB_UNLOCKED 1099 STORE %r4, 0(%r3) 1100 isync 1101 msync 1102 blr 1103 1104/* 1105 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes); 1106 * only a single word from this granule will actually be used as a spin lock 1107 * for mutual exclusion between TLB miss handler and pmap layer that 1108 * manipulates page table contents. 1109 */ 1110 .data 1111 .align 5 1112GLOBAL(tlb0_miss_locks) 1113 .space RES_GRANULE * MAXCPU 1114#endif 1115