1/*- 2 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com> 3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com> 4 * Copyright (C) 2006 Juniper Networks, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31/*- 32 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 33 * Copyright (C) 1995, 1996 TooLs GmbH. 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed by TooLs GmbH. 47 * 4. The name of TooLs GmbH may not be used to endorse or promote products 48 * derived from this software without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 60 * 61 * from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $ 62 */ 63 64/* 65 * NOTICE: This is not a standalone file. to use it, #include it in 66 * your port's locore.S, like so: 67 * 68 * #include <powerpc/booke/trap_subr.S> 69 */ 70 71/* 72 * SPRG usage notes 73 * 74 * SPRG0 - pcpu pointer 75 * SPRG1 - all interrupts except TLB miss, critical, machine check 76 * SPRG2 - critical 77 * SPRG3 - machine check 78 * SPRG4-6 - scratch 79 * 80 */ 81 82/* Get the per-CPU data structure */ 83#define GET_CPUINFO(r) mfsprg0 r 84 85#define RES_GRANULE 64 86#define RES_LOCK 0 /* offset to the 'lock' word */ 87#ifdef __powerpc64__ 88#define RES_RECURSE 8 /* offset to the 'recurse' word */ 89#else 90#define RES_RECURSE 4 /* offset to the 'recurse' word */ 91#endif 92 93/* 94 * Standard interrupt prolog 95 * 96 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP 97 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave}) 98 * isrr0-1 - save restore registers with CPU state at interrupt time (may be 99 * SRR0-1, CSRR0-1, MCSRR0-1 100 * 101 * 1. saves in the given savearea: 102 * - R30-31 103 * - DEAR, ESR 104 * - xSRR0-1 105 * 106 * 2. saves CR -> R30 107 * 108 * 3. switches to kstack if needed 109 * 110 * 4. notes: 111 * - R31 can be used as scratch register until a new frame is layed on 112 * the stack with FRAME_SETUP 113 * 114 * - potential TLB miss: NO. Saveareas are always acessible via TLB1 115 * permanent entries, and within this prolog we do not dereference any 116 * locations potentially not in the TLB 117 */ 118#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ 119 mtspr sprg_sp, %r1; /* Save SP */ \ 120 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 121 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \ 122 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \ 123 mfspr %r30, SPR_DEAR; \ 124 mfspr %r31, SPR_ESR; \ 125 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 126 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 127 mfspr %r30, isrr0; \ 128 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 129 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 130 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 131 isync; \ 132 mfspr %r1, sprg_sp; /* Restore SP */ \ 133 mfcr %r30; /* Save CR */ \ 134 /* switch to per-thread kstack if intr taken in user mode */ \ 135 mtcr %r31; /* MSR at interrupt time */ \ 136 bf 17, 1f; \ 137 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 138 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1391: 140 141#define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ 142 mtspr sprg_sp, %r1; /* Save SP */ \ 143 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 144 STORE %r30, (savearea+CPUSAVE_R30)(%r1); \ 145 STORE %r31, (savearea+CPUSAVE_R31)(%r1); \ 146 mfspr %r30, SPR_DEAR; \ 147 mfspr %r31, SPR_ESR; \ 148 STORE %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ 149 STORE %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ 150 mfspr %r30, isrr0; \ 151 mfspr %r31, isrr1; /* MSR at interrupt time */ \ 152 STORE %r30, (savearea+CPUSAVE_SRR0)(%r1); \ 153 STORE %r31, (savearea+CPUSAVE_SRR1)(%r1); \ 154 mfspr %r30, SPR_SRR0; \ 155 mfspr %r31, SPR_SRR1; /* MSR at interrupt time */ \ 156 STORE %r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1); \ 157 STORE %r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1); \ 158 isync; \ 159 mfspr %r1, sprg_sp; /* Restore SP */ \ 160 mfcr %r30; /* Save CR */ \ 161 /* switch to per-thread kstack if intr taken in user mode */ \ 162 mtcr %r31; /* MSR at interrupt time */ \ 163 bf 17, 1f; \ 164 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 165 LOAD %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1661: 167 168/* 169 * FRAME_SETUP assumes: 170 * SPRG{1-3} SP at the time interrupt occured 171 * savearea r30-r31, DEAR, ESR, xSRR0-1 172 * r30 CR 173 * r31 scratch 174 * r1 kernel stack 175 * 176 * sprg_sp - SPRG reg containing SP at the time interrupt occured 177 * savearea - temp save 178 * exc - exception number (EXC_xxx) 179 * 180 * 1. sets a new frame 181 * 2. saves in the frame: 182 * - R0, R1 (SP at the time of interrupt), R2, LR, CR 183 * - R3-31 (R30-31 first restored from savearea) 184 * - XER, CTR, DEAR, ESR (from savearea), xSRR0-1 185 * 186 * Notes: 187 * - potential TLB miss: YES, since we make dereferences to kstack, which 188 * can happen not covered (we can have up to two DTLB misses if fortunate 189 * enough i.e. when kstack crosses page boundary and both pages are 190 * untranslated) 191 */ 192#ifdef __powerpc64__ 193#define SAVE_REGS(r) \ 194 std %r3, FRAME_3+CALLSIZE(r); \ 195 std %r4, FRAME_4+CALLSIZE(r); \ 196 std %r5, FRAME_5+CALLSIZE(r); \ 197 std %r6, FRAME_6+CALLSIZE(r); \ 198 std %r7, FRAME_7+CALLSIZE(r); \ 199 std %r8, FRAME_8+CALLSIZE(r); \ 200 std %r9, FRAME_9+CALLSIZE(r); \ 201 std %r10, FRAME_10+CALLSIZE(r); \ 202 std %r11, FRAME_11+CALLSIZE(r); \ 203 std %r12, FRAME_12+CALLSIZE(r); \ 204 std %r13, FRAME_13+CALLSIZE(r); \ 205 std %r14, FRAME_14+CALLSIZE(r); \ 206 std %r15, FRAME_15+CALLSIZE(r); \ 207 std %r16, FRAME_16+CALLSIZE(r); \ 208 std %r17, FRAME_17+CALLSIZE(r); \ 209 std %r18, FRAME_18+CALLSIZE(r); \ 210 std %r19, FRAME_19+CALLSIZE(r); \ 211 std %r20, FRAME_20+CALLSIZE(r); \ 212 std %r21, FRAME_21+CALLSIZE(r); \ 213 std %r22, FRAME_22+CALLSIZE(r); \ 214 std %r23, FRAME_23+CALLSIZE(r); \ 215 std %r24, FRAME_24+CALLSIZE(r); \ 216 std %r25, FRAME_25+CALLSIZE(r); \ 217 std %r26, FRAME_26+CALLSIZE(r); \ 218 std %r27, FRAME_27+CALLSIZE(r); \ 219 std %r28, FRAME_28+CALLSIZE(r); \ 220 std %r29, FRAME_29+CALLSIZE(r); \ 221 std %r30, FRAME_30+CALLSIZE(r); \ 222 std %r31, FRAME_31+CALLSIZE(r) 223#define LD_REGS(r) \ 224 ld %r3, FRAME_3+CALLSIZE(r); \ 225 ld %r4, FRAME_4+CALLSIZE(r); \ 226 ld %r5, FRAME_5+CALLSIZE(r); \ 227 ld %r6, FRAME_6+CALLSIZE(r); \ 228 ld %r7, FRAME_7+CALLSIZE(r); \ 229 ld %r8, FRAME_8+CALLSIZE(r); \ 230 ld %r9, FRAME_9+CALLSIZE(r); \ 231 ld %r10, FRAME_10+CALLSIZE(r); \ 232 ld %r11, FRAME_11+CALLSIZE(r); \ 233 ld %r12, FRAME_12+CALLSIZE(r); \ 234 ld %r13, FRAME_13+CALLSIZE(r); \ 235 ld %r14, FRAME_14+CALLSIZE(r); \ 236 ld %r15, FRAME_15+CALLSIZE(r); \ 237 ld %r16, FRAME_16+CALLSIZE(r); \ 238 ld %r17, FRAME_17+CALLSIZE(r); \ 239 ld %r18, FRAME_18+CALLSIZE(r); \ 240 ld %r19, FRAME_19+CALLSIZE(r); \ 241 ld %r20, FRAME_20+CALLSIZE(r); \ 242 ld %r21, FRAME_21+CALLSIZE(r); \ 243 ld %r22, FRAME_22+CALLSIZE(r); \ 244 ld %r23, FRAME_23+CALLSIZE(r); \ 245 ld %r24, FRAME_24+CALLSIZE(r); \ 246 ld %r25, FRAME_25+CALLSIZE(r); \ 247 ld %r26, FRAME_26+CALLSIZE(r); \ 248 ld %r27, FRAME_27+CALLSIZE(r); \ 249 ld %r28, FRAME_28+CALLSIZE(r); \ 250 ld %r29, FRAME_29+CALLSIZE(r); \ 251 ld %r30, FRAME_30+CALLSIZE(r); \ 252 ld %r31, FRAME_31+CALLSIZE(r) 253#else 254#define SAVE_REGS(r) \ 255 stmw %r3, FRAME_3+CALLSIZE(r) 256#define LD_REGS(r) \ 257 lmw %r3, FRAME_3+CALLSIZE(r) 258#endif 259#define FRAME_SETUP(sprg_sp, savearea, exc) \ 260 mfspr %r31, sprg_sp; /* get saved SP */ \ 261 /* establish a new stack frame and put everything on it */ \ 262 STU %r31, -(FRAMELEN+REDZONE)(%r1); \ 263 STORE %r0, FRAME_0+CALLSIZE(%r1); /* save r0 in the trapframe */ \ 264 STORE %r31, FRAME_1+CALLSIZE(%r1); /* save SP " " */ \ 265 STORE %r2, FRAME_2+CALLSIZE(%r1); /* save r2 " " */ \ 266 mflr %r31; \ 267 STORE %r31, FRAME_LR+CALLSIZE(%r1); /* save LR " " */ \ 268 STORE %r30, FRAME_CR+CALLSIZE(%r1); /* save CR " " */ \ 269 GET_CPUINFO(%r2); \ 270 LOAD %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \ 271 LOAD %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \ 272 /* save R3-31 */ \ 273 SAVE_REGS(%r1); \ 274 /* save DEAR, ESR */ \ 275 LOAD %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2); \ 276 LOAD %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2); \ 277 STORE %r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1); \ 278 STORE %r29, FRAME_BOOKE_ESR+CALLSIZE(%r1); \ 279 /* save XER, CTR, exc number */ \ 280 mfxer %r3; \ 281 mfctr %r4; \ 282 STORE %r3, FRAME_XER+CALLSIZE(%r1); \ 283 STORE %r4, FRAME_CTR+CALLSIZE(%r1); \ 284 li %r5, exc; \ 285 STORE %r5, FRAME_EXC+CALLSIZE(%r1); \ 286 /* save DBCR0 */ \ 287 mfspr %r3, SPR_DBCR0; \ 288 STORE %r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \ 289 /* save xSSR0-1 */ \ 290 LOAD %r30, (savearea+CPUSAVE_SRR0)(%r2); \ 291 LOAD %r31, (savearea+CPUSAVE_SRR1)(%r2); \ 292 STORE %r30, FRAME_SRR0+CALLSIZE(%r1); \ 293 STORE %r31, FRAME_SRR1+CALLSIZE(%r1); \ 294 LOAD THREAD_REG, PC_CURTHREAD(%r2); \ 295 296/* 297 * 298 * isrr0-1 - save restore registers to restore CPU state to (may be 299 * SRR0-1, CSRR0-1, MCSRR0-1 300 * 301 * Notes: 302 * - potential TLB miss: YES. The deref'd kstack may be not covered 303 */ 304#define FRAME_LEAVE(isrr0, isrr1) \ 305 wrteei 0; \ 306 /* restore CTR, XER, LR, CR */ \ 307 LOAD %r4, FRAME_CTR+CALLSIZE(%r1); \ 308 LOAD %r5, FRAME_XER+CALLSIZE(%r1); \ 309 LOAD %r6, FRAME_LR+CALLSIZE(%r1); \ 310 LOAD %r7, FRAME_CR+CALLSIZE(%r1); \ 311 mtctr %r4; \ 312 mtxer %r5; \ 313 mtlr %r6; \ 314 mtcr %r7; \ 315 /* restore DBCR0 */ \ 316 LOAD %r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1); \ 317 mtspr SPR_DBCR0, %r4; \ 318 /* restore xSRR0-1 */ \ 319 LOAD %r30, FRAME_SRR0+CALLSIZE(%r1); \ 320 LOAD %r31, FRAME_SRR1+CALLSIZE(%r1); \ 321 mtspr isrr0, %r30; \ 322 mtspr isrr1, %r31; \ 323 /* restore R2-31, SP */ \ 324 LD_REGS(%r1); \ 325 LOAD %r2, FRAME_2+CALLSIZE(%r1); \ 326 LOAD %r0, FRAME_0+CALLSIZE(%r1); \ 327 LOAD %r1, FRAME_1+CALLSIZE(%r1); \ 328 isync 329 330/* 331 * TLB miss prolog 332 * 333 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area 334 * 335 * Notes: 336 * - potential TLB miss: NO. It is crucial that we do not generate a TLB 337 * miss within the TLB prolog itself! 338 * - TLBSAVE is always translated 339 */ 340#ifdef __powerpc64__ 341#define TLB_SAVE_REGS(br) \ 342 std %r20, (TLBSAVE_BOOKE_R20)(br); \ 343 std %r21, (TLBSAVE_BOOKE_R21)(br); \ 344 std %r22, (TLBSAVE_BOOKE_R22)(br); \ 345 std %r23, (TLBSAVE_BOOKE_R23)(br); \ 346 std %r24, (TLBSAVE_BOOKE_R24)(br); \ 347 std %r25, (TLBSAVE_BOOKE_R25)(br); \ 348 std %r26, (TLBSAVE_BOOKE_R26)(br); \ 349 std %r27, (TLBSAVE_BOOKE_R27)(br); \ 350 std %r28, (TLBSAVE_BOOKE_R28)(br); \ 351 std %r29, (TLBSAVE_BOOKE_R29)(br); \ 352 std %r30, (TLBSAVE_BOOKE_R30)(br); \ 353 std %r31, (TLBSAVE_BOOKE_R31)(br); 354#define TLB_RESTORE_REGS(br) \ 355 ld %r20, (TLBSAVE_BOOKE_R20)(br); \ 356 ld %r21, (TLBSAVE_BOOKE_R21)(br); \ 357 ld %r22, (TLBSAVE_BOOKE_R22)(br); \ 358 ld %r23, (TLBSAVE_BOOKE_R23)(br); \ 359 ld %r24, (TLBSAVE_BOOKE_R24)(br); \ 360 ld %r25, (TLBSAVE_BOOKE_R25)(br); \ 361 ld %r26, (TLBSAVE_BOOKE_R26)(br); \ 362 ld %r27, (TLBSAVE_BOOKE_R27)(br); \ 363 ld %r28, (TLBSAVE_BOOKE_R28)(br); \ 364 ld %r29, (TLBSAVE_BOOKE_R29)(br); \ 365 ld %r30, (TLBSAVE_BOOKE_R30)(br); \ 366 ld %r31, (TLBSAVE_BOOKE_R31)(br); 367#define TLB_NEST(outr,inr) \ 368 rlwinm outr, inr, 7, 23, 24; /* 8 x TLBSAVE_LEN */ 369#else 370#define TLB_SAVE_REGS(br) \ 371 stmw %r20, TLBSAVE_BOOKE_R20(br) 372#define TLB_RESTORE_REGS(br) \ 373 lmw %r20, TLBSAVE_BOOKE_R20(br) 374#define TLB_NEST(outr,inr) \ 375 rlwinm outr, inr, 6, 24, 25; /* 4 x TLBSAVE_LEN */ 376#endif 377#define TLB_PROLOG \ 378 mtspr SPR_SPRG4, %r1; /* Save SP */ \ 379 mtspr SPR_SPRG5, %r28; \ 380 mtspr SPR_SPRG6, %r29; \ 381 /* calculate TLB nesting level and TLBSAVE instance address */ \ 382 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 383 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 384 TLB_NEST(%r29,%r28); \ 385 addi %r28, %r28, 1; \ 386 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 387 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 388 add %r1, %r1, %r29; /* current TLBSAVE ptr */ \ 389 \ 390 /* save R20-31 */ \ 391 mfspr %r28, SPR_SPRG5; \ 392 mfspr %r29, SPR_SPRG6; \ 393 TLB_SAVE_REGS(%r1); \ 394 /* save LR, CR */ \ 395 mflr %r30; \ 396 mfcr %r31; \ 397 STORE %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 398 STORE %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 399 /* save SRR0-1 */ \ 400 mfsrr0 %r30; /* execution addr at interrupt time */ \ 401 mfsrr1 %r31; /* MSR at interrupt time*/ \ 402 STORE %r30, (TLBSAVE_BOOKE_SRR0)(%r1); /* save SRR0 */ \ 403 STORE %r31, (TLBSAVE_BOOKE_SRR1)(%r1); /* save SRR1 */ \ 404 isync; \ 405 mfspr %r1, SPR_SPRG4 406 407/* 408 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area 409 * 410 * same notes as for the TLB_PROLOG 411 */ 412#define TLB_RESTORE \ 413 mtspr SPR_SPRG4, %r1; /* Save SP */ \ 414 GET_CPUINFO(%r1); /* Per-cpu structure */ \ 415 /* calculate TLB nesting level and TLBSAVE instance addr */ \ 416 LOAD %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 417 subi %r28, %r28, 1; \ 418 STORE %r28, PC_BOOKE_TLB_LEVEL(%r1); \ 419 TLB_NEST(%r29,%r28); \ 420 addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ 421 add %r1, %r1, %r29; \ 422 \ 423 /* restore LR, CR */ \ 424 LOAD %r30, (TLBSAVE_BOOKE_LR)(%r1); \ 425 LOAD %r31, (TLBSAVE_BOOKE_CR)(%r1); \ 426 mtlr %r30; \ 427 mtcr %r31; \ 428 /* restore SRR0-1 */ \ 429 LOAD %r30, (TLBSAVE_BOOKE_SRR0)(%r1); \ 430 LOAD %r31, (TLBSAVE_BOOKE_SRR1)(%r1); \ 431 mtsrr0 %r30; \ 432 mtsrr1 %r31; \ 433 /* restore R20-31 */ \ 434 TLB_RESTORE_REGS(%r1); \ 435 mfspr %r1, SPR_SPRG4 436 437#ifdef SMP 438#define TLB_LOCK \ 439 GET_CPUINFO(%r20); \ 440 LOAD %r21, PC_CURTHREAD(%r20); \ 441 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \ 442 \ 4431: LOADX %r23, 0, %r22; \ 444 CMPI %r23, TLB_UNLOCKED; \ 445 beq 2f; \ 446 \ 447 /* check if this is recursion */ \ 448 CMPL cr0, %r21, %r23; \ 449 bne- 1b; \ 450 \ 4512: /* try to acquire lock */ \ 452 STOREX %r21, 0, %r22; \ 453 bne- 1b; \ 454 \ 455 /* got it, update recursion counter */ \ 456 lwz %r21, RES_RECURSE(%r22); \ 457 addi %r21, %r21, 1; \ 458 stw %r21, RES_RECURSE(%r22); \ 459 isync; \ 460 msync 461 462#define TLB_UNLOCK \ 463 GET_CPUINFO(%r20); \ 464 LOAD %r21, PC_CURTHREAD(%r20); \ 465 LOAD %r22, PC_BOOKE_TLB_LOCK(%r20); \ 466 \ 467 /* update recursion counter */ \ 468 lwz %r23, RES_RECURSE(%r22); \ 469 subi %r23, %r23, 1; \ 470 stw %r23, RES_RECURSE(%r22); \ 471 \ 472 cmplwi %r23, 0; \ 473 bne 1f; \ 474 isync; \ 475 msync; \ 476 \ 477 /* release the lock */ \ 478 li %r23, TLB_UNLOCKED; \ 479 STORE %r23, 0(%r22); \ 4801: isync; \ 481 msync 482#else 483#define TLB_LOCK 484#define TLB_UNLOCK 485#endif /* SMP */ 486 487#define INTERRUPT(label) \ 488 .globl label; \ 489 .align 5; \ 490 CNAME(label): 491 492/* 493 * Interrupt handling routines in BookE can be flexibly placed and do not have 494 * to live in pre-defined vectors location. Note they need to be TLB-mapped at 495 * all times in order to be able to handle exceptions. We thus arrange for 496 * them to be part of kernel text which is always TLB-accessible. 497 * 498 * The interrupt handling routines have to be 16 bytes aligned: we align them 499 * to 32 bytes (cache line length) which supposedly performs better. 500 * 501 */ 502 .text 503 .globl CNAME(interrupt_vector_base) 504 .align 5 505interrupt_vector_base: 506/***************************************************************************** 507 * Catch-all handler to handle uninstalled IVORs 508 ****************************************************************************/ 509INTERRUPT(int_unknown) 510 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 511 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD) 512 b trap_common 513 514/***************************************************************************** 515 * Critical input interrupt 516 ****************************************************************************/ 517INTERRUPT(int_critical_input) 518 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 519 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT) 520 GET_TOCBASE(%r2) 521 addi %r3, %r1, CALLSIZE 522 bl CNAME(powerpc_interrupt) 523 TOC_RESTORE 524 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) 525 rfci 526 527 528/***************************************************************************** 529 * Machine check interrupt 530 ****************************************************************************/ 531INTERRUPT(int_machine_check) 532 STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1) 533 FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK) 534 GET_TOCBASE(%r2) 535 addi %r3, %r1, CALLSIZE 536 bl CNAME(powerpc_interrupt) 537 TOC_RESTORE 538 FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1) 539 rfmci 540 541 542/***************************************************************************** 543 * Data storage interrupt 544 ****************************************************************************/ 545INTERRUPT(int_data_storage) 546 STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1) 547 FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI) 548 b trap_common 549 550 551/***************************************************************************** 552 * Instruction storage interrupt 553 ****************************************************************************/ 554INTERRUPT(int_instr_storage) 555 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 556 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI) 557 b trap_common 558 559 560/***************************************************************************** 561 * External input interrupt 562 ****************************************************************************/ 563INTERRUPT(int_external_input) 564 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 565 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI) 566 b trap_common 567 568 569INTERRUPT(int_alignment) 570 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 571 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI) 572 b trap_common 573 574 575INTERRUPT(int_program) 576 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 577 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM) 578 b trap_common 579 580 581INTERRUPT(int_fpu) 582 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 583 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU) 584 b trap_common 585 586 587/***************************************************************************** 588 * System call 589 ****************************************************************************/ 590INTERRUPT(int_syscall) 591 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 592 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC) 593 b trap_common 594 595 596/***************************************************************************** 597 * Decrementer interrupt 598 ****************************************************************************/ 599INTERRUPT(int_decrementer) 600 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 601 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR) 602 b trap_common 603 604 605/***************************************************************************** 606 * Fixed interval timer 607 ****************************************************************************/ 608INTERRUPT(int_fixed_interval_timer) 609 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 610 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT) 611 b trap_common 612 613 614/***************************************************************************** 615 * Watchdog interrupt 616 ****************************************************************************/ 617INTERRUPT(int_watchdog) 618 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 619 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG) 620 b trap_common 621 622 623/***************************************************************************** 624 * Altivec Unavailable interrupt 625 ****************************************************************************/ 626INTERRUPT(int_vec) 627 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 628 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC) 629 b trap_common 630 631 632/***************************************************************************** 633 * Altivec Assist interrupt 634 ****************************************************************************/ 635INTERRUPT(int_vecast) 636 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 637 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E) 638 b trap_common 639 640 641#ifdef __SPE__ 642/***************************************************************************** 643 * Floating point Assist interrupt 644 ****************************************************************************/ 645INTERRUPT(int_spe_fpdata) 646 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 647 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPD) 648 addi %r3, %r1, CALLSIZE 649 bl spe_handle_fpdata 650 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 651 rfi 652 653INTERRUPT(int_spe_fpround) 654 STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 655 FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SPFPR) 656 addi %r3, %r1, CALLSIZE 657 bl spe_handle_fpround 658 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 659 rfi 660#endif 661 662 663#ifdef HWPMC_HOOKS 664/***************************************************************************** 665 * PMC Interrupt 666 ****************************************************************************/ 667INTERRUPT(int_performance_counter) 668 STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) 669 FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF) 670 b trap_common 671#endif 672 673 674/***************************************************************************** 675 * Data TLB miss interrupt 676 * 677 * There can be nested TLB misses - while handling a TLB miss we reference 678 * data structures that may be not covered by translations. We support up to 679 * TLB_NESTED_MAX-1 nested misses. 680 * 681 * Registers use: 682 * r31 - dear 683 * r30 - unused 684 * r29 - saved mas0 685 * r28 - saved mas1 686 * r27 - saved mas2 687 * r26 - pmap address 688 * r25 - pte address 689 * 690 * r20:r23 - scratch registers 691 ****************************************************************************/ 692INTERRUPT(int_data_tlb_error) 693 TLB_PROLOG 694 TLB_LOCK 695 696 mfspr %r31, SPR_DEAR 697 698 /* 699 * Save MAS0-MAS2 registers. There might be another tlb miss during 700 * pte lookup overwriting current contents (which was hw filled). 701 */ 702 mfspr %r29, SPR_MAS0 703 mfspr %r28, SPR_MAS1 704 mfspr %r27, SPR_MAS2 705 706 /* Check faulting address. */ 707 LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS) 708 CMPL cr0, %r31, %r21 709 blt search_user_pmap 710 711 /* If it's kernel address, allow only supervisor mode misses. */ 712 mfsrr1 %r21 713 mtcr %r21 714 bt 17, search_failed /* check MSR[PR] */ 715 716#ifdef __powerpc64__ 717 srdi %r21, %r31, 48 718 cmpldi cr0, %r21, VM_MIN_KERNEL_ADDRESS@highest 719#else 720 lis %r21, VM_MIN_KERNEL_ADDRESS@h 721 cmplw cr0, %r31, %r21 722#endif 723 blt search_failed 724 725search_kernel_pmap: 726 /* Load r26 with kernel_pmap address */ 727 bl 1f 728#ifdef __powerpc64__ 729 .llong kernel_pmap_store-. 730#else 731 .long kernel_pmap_store-. 732#endif 7331: mflr %r21 734 LOAD %r26, 0(%r21) 735 add %r26, %r21, %r26 /* kernel_pmap_store in r26 */ 736 737 /* Force kernel tid, set TID to 0 in MAS1. */ 738 li %r21, 0 739 rlwimi %r28, %r21, 0, 8, 15 /* clear TID bits */ 740 741tlb_miss_handle: 742 /* This may result in nested tlb miss. */ 743 bl pte_lookup /* returns PTE address in R25 */ 744 745 CMPI %r25, 0 /* pte found? */ 746 beq search_failed 747 748 /* Finish up, write TLB entry. */ 749 bl tlb_fill_entry 750 751tlb_miss_return: 752 TLB_UNLOCK 753 TLB_RESTORE 754 rfi 755 756search_user_pmap: 757 /* Load r26 with current user space process pmap */ 758 GET_CPUINFO(%r26) 759 LOAD %r26, PC_CURPMAP(%r26) 760 761 b tlb_miss_handle 762 763search_failed: 764 /* 765 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with 766 * the faulting virtual address anyway, but put a fake RPN and no 767 * access rights. This should cause a following {D,I}SI exception. 768 */ 769 lis %r23, 0xffff0000@h /* revoke all permissions */ 770 771 /* Load MAS registers. */ 772 mtspr SPR_MAS0, %r29 773 mtspr SPR_MAS1, %r28 774 mtspr SPR_MAS2, %r27 775 mtspr SPR_MAS3, %r23 776 777 li %r23, 0 778 mtspr SPR_MAS7, %r23 779 780 isync 781 tlbwe 782 msync 783 isync 784 b tlb_miss_return 785 786/***************************************************************************** 787 * 788 * Return pte address that corresponds to given pmap/va. If there is no valid 789 * entry return 0. 790 * 791 * input: r26 - pmap 792 * input: r31 - dear 793 * output: r25 - pte address 794 * 795 * scratch regs used: r21 796 * 797 ****************************************************************************/ 798pte_lookup: 799 CMPI %r26, 0 800 beq 1f /* fail quickly if pmap is invalid */ 801 802#ifdef __powerpc64__ 803 rldicl %r21, %r31, (64 - PG_ROOT_L), (64 - PG_ROOT_NUM) /* pp2d offset */ 804 slwi %r21, %r21, PG_ROOT_ENTRY_SHIFT /* multiply by pp2d entry size */ 805 ld %r25, PM_ROOT(%r26) /* pmap pm_pp2d[] address */ 806 ldx %r25, %r25, %r21 /* get pdir address, i.e. pmap->pm_pp2d[pp2d_idx] * */ 807 808 cmpdi %r25, 0 809 beq 2f 810 811 rldicl %r21, %r31, (64 - PDIR_L1_L), (64 - PDIR_L1_NUM) /* pp2d offset */ 812 slwi %r21, %r21, PDIR_L1_ENTRY_SHIFT /* multiply by pp2d entry size */ 813 ldx %r25, %r25, %r21 /* get pdir address, i.e. pmap->pm_pp2d[pp2d_idx] * */ 814 815 cmpdi %r25, 0 816 beq 2f 817 818 rldicl %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM) /* pdir offset */ 819 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ 820 ldx %r25, %r25, %r21 /* get ptbl address, i.e. pmap->pm_pp2d[pp2d_idx][pdir_idx] */ 821 822 cmpdi %r25, 0 823 beq 2f 824 825 rldicl %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */ 826 slwi %r21, %r21, PTBL_ENTRY_SHIFT /* multiply by pte entry size */ 827 828#else 829 srwi %r21, %r31, PDIR_SHIFT /* pdir offset */ 830 slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ 831 832 lwz %r25, PM_PDIR(%r26) /* pmap pm_dir[] address */ 833 /* 834 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx] 835 * This load may cause a Data TLB miss for non-kernel pmap! 836 */ 837 lwzx %r25, %r25, %r21 /* offset within pm_pdir[] table */ 838 cmpwi %r25, 0 839 beq 2f 840 841 lis %r21, PTBL_MASK@h 842 ori %r21, %r21, PTBL_MASK@l 843 and %r21, %r21, %r31 844 845 /* ptbl offset, multiply by ptbl entry size */ 846 srwi %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT) 847#endif 848 849 add %r25, %r25, %r21 /* address of pte entry */ 850 /* 851 * Get pte->flags 852 * This load may cause a Data TLB miss for non-kernel pmap! 853 */ 854 lwz %r21, PTE_FLAGS(%r25) 855 andi. %r21, %r21, PTE_VALID@l 856 bne 2f 8571: 858 li %r25, 0 8592: 860 blr 861 862/***************************************************************************** 863 * 864 * Load MAS1-MAS3 registers with data, write TLB entry 865 * 866 * input: 867 * r29 - mas0 868 * r28 - mas1 869 * r27 - mas2 870 * r25 - pte 871 * 872 * output: none 873 * 874 * scratch regs: r21-r23 875 * 876 ****************************************************************************/ 877tlb_fill_entry: 878 /* 879 * Update PTE flags: we have to do it atomically, as pmap_protect() 880 * running on other CPUs could attempt to update the flags at the same 881 * time. 882 */ 883 li %r23, PTE_FLAGS 8841: 885 lwarx %r21, %r23, %r25 /* get pte->flags */ 886 oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */ 887 888 andi. %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */ 889 beq 2f 890 ori %r21, %r21, PTE_MODIFIED@l /* set modified bit */ 8912: 892 stwcx. %r21, %r23, %r25 /* write it back */ 893 bne- 1b 894 895 /* Update MAS2. */ 896 rlwimi %r27, %r21, 13, 27, 30 /* insert WIMG bits from pte */ 897 898 /* Setup MAS3 value in r23. */ 899 LOAD %r23, PTE_RPN(%r25) /* get pte->rpn */ 900#ifdef __powerpc64__ 901 rldicr %r22, %r23, 52, 51 /* extract MAS3 portion of RPN */ 902 rldicl %r23, %r23, 20, 54 /* extract MAS7 portion of RPN */ 903 904 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ 905#else 906 rlwinm %r22, %r23, 20, 0, 11 /* extract MAS3 portion of RPN */ 907 908 rlwimi %r22, %r21, 30, 26, 31 /* insert protection bits from pte */ 909 rlwimi %r22, %r21, 20, 12, 19 /* insert lower 8 RPN bits to MAS3 */ 910 rlwinm %r23, %r23, 20, 24, 31 /* MAS7 portion of RPN */ 911#endif 912 913 /* Load MAS registers. */ 914 mtspr SPR_MAS0, %r29 915 mtspr SPR_MAS1, %r28 916 mtspr SPR_MAS2, %r27 917 mtspr SPR_MAS3, %r22 918 mtspr SPR_MAS7, %r23 919 920 isync 921 tlbwe 922 isync 923 msync 924 blr 925 926/***************************************************************************** 927 * Instruction TLB miss interrupt 928 * 929 * Same notes as for the Data TLB miss 930 ****************************************************************************/ 931INTERRUPT(int_inst_tlb_error) 932 TLB_PROLOG 933 TLB_LOCK 934 935 mfsrr0 %r31 /* faulting address */ 936 937 /* 938 * Save MAS0-MAS2 registers. There might be another tlb miss during pte 939 * lookup overwriting current contents (which was hw filled). 940 */ 941 mfspr %r29, SPR_MAS0 942 mfspr %r28, SPR_MAS1 943 mfspr %r27, SPR_MAS2 944 945 mfsrr1 %r21 946 mtcr %r21 947 948 /* check MSR[PR] */ 949 bt 17, search_user_pmap 950 b search_kernel_pmap 951 952 953 .globl interrupt_vector_top 954interrupt_vector_top: 955 956/***************************************************************************** 957 * Debug interrupt 958 ****************************************************************************/ 959INTERRUPT(int_debug) 960 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) 961 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) 962 bl int_debug_int 963 FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) 964 rfci 965 966INTERRUPT(int_debug_ed) 967 STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1) 968 FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) 969 bl int_debug_int 970 FRAME_LEAVE(SPR_DSRR0, SPR_DSRR1) 971 rfdi 972 /* .long 0x4c00004e */ 973 974/* Internal helper for debug interrupt handling. */ 975/* Common code between e500v1/v2 and e500mc-based cores. */ 976int_debug_int: 977 mflr %r14 978 GET_CPUINFO(%r3) 979 LOAD %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3) 980 bl 0f 981 ADDR(interrupt_vector_base-.) 982 ADDR(interrupt_vector_top-.) 9830: mflr %r5 984 LOAD %r4,0(%r5) /* interrupt_vector_base in r4 */ 985 add %r4,%r4,%r5 986 CMPL cr0, %r3, %r4 987 blt trap_common 988 LOAD %r4,WORD_SIZE(%r5) /* interrupt_vector_top in r4 */ 989 add %r4,%r4,%r5 990 addi %r4,%r4,4 991 CMPL cr0, %r3, %r4 992 bge trap_common 993 /* Disable single-stepping for the interrupt handlers. */ 994 LOAD %r3, FRAME_SRR1+CALLSIZE(%r1); 995 rlwinm %r3, %r3, 0, 23, 21 996 STORE %r3, FRAME_SRR1+CALLSIZE(%r1); 997 /* Restore srr0 and srr1 as they could have been clobbered. */ 998 GET_CPUINFO(%r4) 999 LOAD %r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4); 1000 mtspr SPR_SRR0, %r3 1001 LOAD %r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4); 1002 mtspr SPR_SRR1, %r4 1003 mtlr %r14 1004 blr 1005 1006/***************************************************************************** 1007 * Common trap code 1008 ****************************************************************************/ 1009trap_common: 1010 /* Call C trap dispatcher */ 1011 GET_TOCBASE(%r2) 1012 addi %r3, %r1, CALLSIZE 1013 bl CNAME(powerpc_interrupt) 1014 TOC_RESTORE 1015 1016 .globl CNAME(trapexit) /* exported for db_backtrace use */ 1017CNAME(trapexit): 1018 /* disable interrupts */ 1019 wrteei 0 1020 1021 /* Test AST pending - makes sense for user process only */ 1022 LOAD %r5, FRAME_SRR1+CALLSIZE(%r1) 1023 mtcr %r5 1024 bf 17, 1f 1025 1026 GET_CPUINFO(%r3) 1027 LOAD %r4, PC_CURTHREAD(%r3) 1028 lwz %r4, TD_FLAGS(%r4) 1029 lis %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h 1030 ori %r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l 1031 and. %r4, %r4, %r5 1032 beq 1f 1033 1034 /* re-enable interrupts before calling ast() */ 1035 wrteei 1 1036 1037 addi %r3, %r1, CALLSIZE 1038 bl CNAME(ast) 1039 TOC_RESTORE 1040 .globl CNAME(asttrapexit) /* db_backtrace code sentinel #2 */ 1041CNAME(asttrapexit): 1042 b trapexit /* test ast ret value ? */ 10431: 1044 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 1045 rfi 1046 1047 1048#if defined(KDB) 1049/* 1050 * Deliberate entry to dbtrap 1051 */ 1052 /* .globl CNAME(breakpoint)*/ 1053ASENTRY_NOPROF(breakpoint) 1054 mtsprg1 %r1 1055 mfmsr %r3 1056 mtsrr1 %r3 1057 li %r4, ~(PSL_EE | PSL_ME)@l 1058 oris %r4, %r4, ~(PSL_EE | PSL_ME)@h 1059 and %r3, %r3, %r4 1060 mtmsr %r3 /* disable interrupts */ 1061 isync 1062 GET_CPUINFO(%r3) 1063 STORE %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3) 1064 STORE %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3) 1065 1066 mflr %r31 1067 mtsrr0 %r31 1068 1069 mfspr %r30, SPR_DEAR 1070 mfspr %r31, SPR_ESR 1071 STORE %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3) 1072 STORE %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3) 1073 1074 mfsrr0 %r30 1075 mfsrr1 %r31 1076 STORE %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3) 1077 STORE %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3) 1078 isync 1079 1080 mfcr %r30 1081 1082/* 1083 * Now the kdb trap catching code. 1084 */ 1085dbtrap: 1086 FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG) 1087/* Call C trap code: */ 1088 GET_TOCBASE(%r2) 1089 addi %r3, %r1, CALLSIZE 1090 bl CNAME(db_trap_glue) 1091 TOC_RESTORE 1092 or. %r3, %r3, %r3 1093 bne dbleave 1094/* This wasn't for KDB, so switch to real trap: */ 1095 b trap_common 1096 1097dbleave: 1098 FRAME_LEAVE(SPR_SRR0, SPR_SRR1) 1099 rfi 1100#endif /* KDB */ 1101 1102#ifdef SMP 1103ENTRY(tlb_lock) 1104 GET_CPUINFO(%r5) 1105 LOAD %r5, PC_CURTHREAD(%r5) 11061: LOADX %r4, 0, %r3 1107 CMPI %r4, TLB_UNLOCKED 1108 bne 1b 1109 STOREX %r5, 0, %r3 1110 bne- 1b 1111 isync 1112 msync 1113 blr 1114 1115ENTRY(tlb_unlock) 1116 isync 1117 msync 1118 li %r4, TLB_UNLOCKED 1119 STORE %r4, 0(%r3) 1120 isync 1121 msync 1122 blr 1123 1124/* 1125 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes); 1126 * only a single word from this granule will actually be used as a spin lock 1127 * for mutual exclusion between TLB miss handler and pmap layer that 1128 * manipulates page table contents. 1129 */ 1130 .data 1131 .align 5 1132GLOBAL(tlb0_miss_locks) 1133 .space RES_GRANULE * MAXCPU 1134#endif 1135