xref: /freebsd/sys/powerpc/booke/trap_subr.S (revision 4f52dfbb8d6c4d446500c5b097e3806ec219fbd4)
1/*-
2 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
4 * Copyright (C) 2006 Juniper Networks, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 *    derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31/*-
32 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
33 * Copyright (C) 1995, 1996 TooLs GmbH.
34 * All rights reserved.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 *    notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 *    notice, this list of conditions and the following disclaimer in the
43 *    documentation and/or other materials provided with the distribution.
44 * 3. All advertising materials mentioning features or use of this software
45 *    must display the following acknowledgement:
46 *	This product includes software developed by TooLs GmbH.
47 * 4. The name of TooLs GmbH may not be used to endorse or promote products
48 *    derived from this software without specific prior written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
60 *
61 *	from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
62 */
63
64/*
65 * NOTICE: This is not a standalone file.  to use it, #include it in
66 * your port's locore.S, like so:
67 *
68 *	#include <powerpc/booke/trap_subr.S>
69 */
70
71/*
72 * SPRG usage notes
73 *
74 * SPRG0 - pcpu pointer
75 * SPRG1 - all interrupts except TLB miss, critical, machine check
76 * SPRG2 - critical
77 * SPRG3 - machine check
78 * SPRG4-6 - scratch
79 *
80 */
81
82/* Get the per-CPU data structure */
83#define GET_CPUINFO(r) mfsprg0 r
84
85#define RES_GRANULE	64
86#define RES_LOCK	0	/* offset to the 'lock' word */
87#ifdef __powerpc64__
88#define RES_RECURSE	8	/* offset to the 'recurse' word */
89#else
90#define RES_RECURSE	4	/* offset to the 'recurse' word */
91#endif
92
93/*
94 * Standard interrupt prolog
95 *
96 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
97 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
98 * isrr0-1 - save restore registers with CPU state at interrupt time (may be
99 *           SRR0-1, CSRR0-1, MCSRR0-1
100 *
101 * 1. saves in the given savearea:
102 *   - R30-31
103 *   - DEAR, ESR
104 *   - xSRR0-1
105 *
106 * 2. saves CR -> R30
107 *
108 * 3. switches to kstack if needed
109 *
110 * 4. notes:
111 *   - R31 can be used as scratch register until a new frame is layed on
112 *     the stack with FRAME_SETUP
113 *
114 *   - potential TLB miss: NO. Saveareas are always acessible via TLB1
115 *     permanent entries, and within this prolog we do not dereference any
116 *     locations potentially not in the TLB
117 */
118#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1)		\
119	mtspr	sprg_sp, %r1;		/* Save SP */			\
120	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
121	STORE	%r30, (savearea+CPUSAVE_R30)(%r1);			\
122	STORE	%r31, (savearea+CPUSAVE_R31)(%r1); 			\
123	mfdear	%r30;		 					\
124	mfesr	%r31;							\
125	STORE	%r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); 		\
126	STORE	%r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); 		\
127	mfspr	%r30, isrr0;						\
128	mfspr	%r31, isrr1;	 	/* MSR at interrupt time */	\
129	STORE	%r30, (savearea+CPUSAVE_SRR0)(%r1);			\
130	STORE	%r31, (savearea+CPUSAVE_SRR1)(%r1);			\
131	isync;			 					\
132	mfspr	%r1, sprg_sp;	 	/* Restore SP */		\
133	mfcr	%r30;		 	/* Save CR */			\
134	/* switch to per-thread kstack if intr taken in user mode */	\
135	mtcr	%r31;			/* MSR at interrupt time  */	\
136	bf	17, 1f;							\
137	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
138	LOAD	%r1, PC_CURPCB(%r1); 	/* Per-thread kernel stack */	\
1391:
140
141#define	STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1)		\
142	mtspr	sprg_sp, %r1;		/* Save SP */			\
143	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
144	STORE	%r30, (savearea+CPUSAVE_R30)(%r1);			\
145	STORE	%r31, (savearea+CPUSAVE_R31)(%r1);			\
146	mfdear	%r30;							\
147	mfesr	%r31;							\
148	STORE	%r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);		\
149	STORE	%r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);		\
150	mfspr	%r30, isrr0;						\
151	mfspr	%r31, isrr1;		/* MSR at interrupt time */	\
152	STORE	%r30, (savearea+CPUSAVE_SRR0)(%r1);			\
153	STORE	%r31, (savearea+CPUSAVE_SRR1)(%r1);			\
154	mfspr	%r30, SPR_SRR0;						\
155	mfspr	%r31, SPR_SRR1;		/* MSR at interrupt time */	\
156	STORE	%r30, (savearea+BOOKE_CRITSAVE_SRR0)(%r1);		\
157	STORE	%r31, (savearea+BOOKE_CRITSAVE_SRR1)(%r1);		\
158	isync;								\
159	mfspr	%r1, sprg_sp;		/* Restore SP */		\
160	mfcr	%r30;			/* Save CR */			\
161	/* switch to per-thread kstack if intr taken in user mode */	\
162	mtcr	%r31;			/* MSR at interrupt time  */	\
163	bf	17, 1f;							\
164	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
165	LOAD	%r1, PC_CURPCB(%r1);	/* Per-thread kernel stack */	\
1661:
167
168/*
169 * FRAME_SETUP assumes:
170 *	SPRG{1-3}	SP at the time interrupt occured
171 *	savearea	r30-r31, DEAR, ESR, xSRR0-1
172 *	r30		CR
173 *	r31		scratch
174 *	r1		kernel stack
175 *
176 * sprg_sp - SPRG reg containing SP at the time interrupt occured
177 * savearea - temp save
178 * exc - exception number (EXC_xxx)
179 *
180 * 1. sets a new frame
181 * 2. saves in the frame:
182 *   - R0, R1 (SP at the time of interrupt), R2, LR, CR
183 *   - R3-31 (R30-31 first restored from savearea)
184 *   - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
185 *
186 * Notes:
187 * - potential TLB miss: YES, since we make dereferences to kstack, which
188 *   can happen not covered (we can have up to two DTLB misses if fortunate
189 *   enough i.e. when kstack crosses page boundary and both pages are
190 *   untranslated)
191 */
192#ifdef __powerpc64__
193#define SAVE_REGS(r)							\
194	std	%r3, FRAME_3+CALLSIZE(r);				\
195	std	%r4, FRAME_4+CALLSIZE(r);				\
196	std	%r5, FRAME_5+CALLSIZE(r);				\
197	std	%r6, FRAME_6+CALLSIZE(r);				\
198	std	%r7, FRAME_7+CALLSIZE(r);				\
199	std	%r8, FRAME_8+CALLSIZE(r);				\
200	std	%r9, FRAME_9+CALLSIZE(r);				\
201	std	%r10, FRAME_10+CALLSIZE(r);				\
202	std	%r11, FRAME_11+CALLSIZE(r);				\
203	std	%r12, FRAME_12+CALLSIZE(r);				\
204	std	%r13, FRAME_13+CALLSIZE(r);				\
205	std	%r14, FRAME_14+CALLSIZE(r);				\
206	std	%r15, FRAME_15+CALLSIZE(r);				\
207	std	%r16, FRAME_16+CALLSIZE(r);				\
208	std	%r17, FRAME_17+CALLSIZE(r);				\
209	std	%r18, FRAME_18+CALLSIZE(r);				\
210	std	%r19, FRAME_19+CALLSIZE(r);				\
211	std	%r20, FRAME_20+CALLSIZE(r);				\
212	std	%r21, FRAME_21+CALLSIZE(r);				\
213	std	%r22, FRAME_22+CALLSIZE(r);				\
214	std	%r23, FRAME_23+CALLSIZE(r);				\
215	std	%r24, FRAME_24+CALLSIZE(r);				\
216	std	%r25, FRAME_25+CALLSIZE(r);				\
217	std	%r26, FRAME_26+CALLSIZE(r);				\
218	std	%r27, FRAME_27+CALLSIZE(r);				\
219	std	%r28, FRAME_28+CALLSIZE(r);				\
220	std	%r29, FRAME_29+CALLSIZE(r);				\
221	std	%r30, FRAME_30+CALLSIZE(r);				\
222	std	%r31, FRAME_31+CALLSIZE(r)
223#define LD_REGS(r)							\
224	ld	%r3, FRAME_3+CALLSIZE(r);				\
225	ld	%r4, FRAME_4+CALLSIZE(r);				\
226	ld	%r5, FRAME_5+CALLSIZE(r);				\
227	ld	%r6, FRAME_6+CALLSIZE(r);				\
228	ld	%r7, FRAME_7+CALLSIZE(r);				\
229	ld	%r8, FRAME_8+CALLSIZE(r);				\
230	ld	%r9, FRAME_9+CALLSIZE(r);				\
231	ld	%r10, FRAME_10+CALLSIZE(r);				\
232	ld	%r11, FRAME_11+CALLSIZE(r);				\
233	ld	%r12, FRAME_12+CALLSIZE(r);				\
234	ld	%r13, FRAME_13+CALLSIZE(r);				\
235	ld	%r14, FRAME_14+CALLSIZE(r);				\
236	ld	%r15, FRAME_15+CALLSIZE(r);				\
237	ld	%r16, FRAME_16+CALLSIZE(r);				\
238	ld	%r17, FRAME_17+CALLSIZE(r);				\
239	ld	%r18, FRAME_18+CALLSIZE(r);				\
240	ld	%r19, FRAME_19+CALLSIZE(r);				\
241	ld	%r20, FRAME_20+CALLSIZE(r);				\
242	ld	%r21, FRAME_21+CALLSIZE(r);				\
243	ld	%r22, FRAME_22+CALLSIZE(r);				\
244	ld	%r23, FRAME_23+CALLSIZE(r);				\
245	ld	%r24, FRAME_24+CALLSIZE(r);				\
246	ld	%r25, FRAME_25+CALLSIZE(r);				\
247	ld	%r26, FRAME_26+CALLSIZE(r);				\
248	ld	%r27, FRAME_27+CALLSIZE(r);				\
249	ld	%r28, FRAME_28+CALLSIZE(r);				\
250	ld	%r29, FRAME_29+CALLSIZE(r);				\
251	ld	%r30, FRAME_30+CALLSIZE(r);				\
252	ld	%r31, FRAME_31+CALLSIZE(r)
253#else
254#define SAVE_REGS(r)							\
255	stmw	%r3,  FRAME_3+CALLSIZE(r)
256#define LD_REGS(r)							\
257	lmw	%r3,  FRAME_3+CALLSIZE(r)
258#endif
259#define	FRAME_SETUP(sprg_sp, savearea, exc)				\
260	mfspr	%r31, sprg_sp;		/* get saved SP */		\
261	/* establish a new stack frame and put everything on it */	\
262	STU	%r31, -(FRAMELEN+REDZONE)(%r1);				\
263	STORE	%r0, FRAME_0+CALLSIZE(%r1);	/* save r0 in the trapframe */	\
264	STORE	%r31, FRAME_1+CALLSIZE(%r1);	/* save SP   "     " */	\
265	STORE	%r2, FRAME_2+CALLSIZE(%r1);	/* save r2   "     " */	\
266	mflr	%r31;		 					\
267	STORE	%r31, FRAME_LR+CALLSIZE(%r1);	/* save LR   "     " */	\
268	STORE	%r30, FRAME_CR+CALLSIZE(%r1);	/* save CR   "     " */	\
269	GET_CPUINFO(%r2);						\
270	LOAD	%r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */	\
271	LOAD	%r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */	\
272	/* save R3-31 */						\
273	SAVE_REGS(%r1);							\
274	/* save DEAR, ESR */						\
275	LOAD	%r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2);		\
276	LOAD	%r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2);		\
277	STORE	%r28, FRAME_BOOKE_DEAR+CALLSIZE(%r1);			\
278	STORE	%r29, FRAME_BOOKE_ESR+CALLSIZE(%r1);			\
279	/* save XER, CTR, exc number */					\
280	mfxer	%r3;							\
281	mfctr	%r4;							\
282	STORE	%r3, FRAME_XER+CALLSIZE(%r1);				\
283	STORE	%r4, FRAME_CTR+CALLSIZE(%r1);				\
284	li	%r5, exc;						\
285	STORE	%r5, FRAME_EXC+CALLSIZE(%r1);				\
286	/* save DBCR0 */						\
287	mfspr	%r3, SPR_DBCR0;						\
288	STORE	%r3, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);			\
289	/* save xSSR0-1 */						\
290	LOAD	%r30, (savearea+CPUSAVE_SRR0)(%r2);			\
291	LOAD	%r31, (savearea+CPUSAVE_SRR1)(%r2);			\
292	STORE	%r30, FRAME_SRR0+CALLSIZE(%r1);				\
293	STORE	%r31, FRAME_SRR1+CALLSIZE(%r1);				\
294	LOAD	THREAD_REG, PC_CURTHREAD(%r2);				\
295
296/*
297 *
298 * isrr0-1 - save restore registers to restore CPU state to (may be
299 *           SRR0-1, CSRR0-1, MCSRR0-1
300 *
301 * Notes:
302 *  - potential TLB miss: YES. The deref'd kstack may be not covered
303 */
304#define	FRAME_LEAVE(isrr0, isrr1)					\
305	wrteei 0;							\
306	/* restore CTR, XER, LR, CR */					\
307	LOAD	%r4, FRAME_CTR+CALLSIZE(%r1);				\
308	LOAD	%r5, FRAME_XER+CALLSIZE(%r1);				\
309	LOAD	%r6, FRAME_LR+CALLSIZE(%r1);				\
310	LOAD	%r7, FRAME_CR+CALLSIZE(%r1);				\
311	mtctr	%r4;							\
312	mtxer	%r5;							\
313	mtlr	%r6;							\
314	mtcr	%r7;							\
315	/* restore DBCR0 */						\
316	LOAD	%r4, FRAME_BOOKE_DBCR0+CALLSIZE(%r1);			\
317	mtspr	SPR_DBCR0, %r4;						\
318	/* restore xSRR0-1 */						\
319	LOAD	%r30, FRAME_SRR0+CALLSIZE(%r1);				\
320	LOAD	%r31, FRAME_SRR1+CALLSIZE(%r1);				\
321	mtspr	isrr0, %r30;						\
322	mtspr	isrr1, %r31;						\
323	/* restore R2-31, SP */						\
324	LD_REGS(%r1);							\
325	LOAD	%r2, FRAME_2+CALLSIZE(%r1);				\
326	LOAD	%r0, FRAME_0+CALLSIZE(%r1);				\
327	LOAD	%r1, FRAME_1+CALLSIZE(%r1);				\
328	isync
329
330/*
331 * TLB miss prolog
332 *
333 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
334 *
335 * Notes:
336 *  - potential TLB miss: NO. It is crucial that we do not generate a TLB
337 *    miss within the TLB prolog itself!
338 *  - TLBSAVE is always translated
339 */
340#ifdef __powerpc64__
341#define	TLB_SAVE_REGS(br)						\
342	std	%r20, (TLBSAVE_BOOKE_R20)(br);				\
343	std	%r21, (TLBSAVE_BOOKE_R21)(br);				\
344	std	%r22, (TLBSAVE_BOOKE_R22)(br);				\
345	std	%r23, (TLBSAVE_BOOKE_R23)(br);				\
346	std	%r24, (TLBSAVE_BOOKE_R24)(br);				\
347	std	%r25, (TLBSAVE_BOOKE_R25)(br);				\
348	std	%r26, (TLBSAVE_BOOKE_R26)(br);				\
349	std	%r27, (TLBSAVE_BOOKE_R27)(br);				\
350	std	%r28, (TLBSAVE_BOOKE_R28)(br);				\
351	std	%r29, (TLBSAVE_BOOKE_R29)(br);				\
352	std	%r30, (TLBSAVE_BOOKE_R30)(br);				\
353	std	%r31, (TLBSAVE_BOOKE_R31)(br);
354#define	TLB_RESTORE_REGS(br)						\
355	ld	%r20, (TLBSAVE_BOOKE_R20)(br);				\
356	ld	%r21, (TLBSAVE_BOOKE_R21)(br);				\
357	ld	%r22, (TLBSAVE_BOOKE_R22)(br);				\
358	ld	%r23, (TLBSAVE_BOOKE_R23)(br);				\
359	ld	%r24, (TLBSAVE_BOOKE_R24)(br);				\
360	ld	%r25, (TLBSAVE_BOOKE_R25)(br);				\
361	ld	%r26, (TLBSAVE_BOOKE_R26)(br);				\
362	ld	%r27, (TLBSAVE_BOOKE_R27)(br);				\
363	ld	%r28, (TLBSAVE_BOOKE_R28)(br);				\
364	ld	%r29, (TLBSAVE_BOOKE_R29)(br);				\
365	ld	%r30, (TLBSAVE_BOOKE_R30)(br);				\
366	ld	%r31, (TLBSAVE_BOOKE_R31)(br);
367#define TLB_NEST(outr,inr)						\
368	rlwinm	outr, inr, 7, 22, 24;	/* 8 x TLBSAVE_LEN */
369#else
370#define TLB_SAVE_REGS(br)						\
371	stmw	%r20, TLBSAVE_BOOKE_R20(br)
372#define TLB_RESTORE_REGS(br)						\
373	lmw	%r20, TLBSAVE_BOOKE_R20(br)
374#define TLB_NEST(outr,inr)						\
375	rlwinm	outr, inr, 6, 23, 25;	/* 4 x TLBSAVE_LEN */
376#endif
377#define TLB_PROLOG							\
378	mtsprg4	%r1;			/* Save SP */			\
379	mtsprg5 %r28;							\
380	mtsprg6 %r29;							\
381	/* calculate TLB nesting level and TLBSAVE instance address */	\
382	GET_CPUINFO(%r1);	 	/* Per-cpu structure */		\
383	LOAD	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
384	TLB_NEST(%r29,%r28);						\
385	addi	%r28, %r28, 1;						\
386	STORE	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
387	addi	%r29, %r29, PC_BOOKE_TLBSAVE@l; 			\
388	add	%r1, %r1, %r29;		/* current TLBSAVE ptr */	\
389									\
390	/* save R20-31 */						\
391	mfsprg5 %r28;		 					\
392	mfsprg6 %r29;							\
393	TLB_SAVE_REGS(%r1);			\
394	/* save LR, CR */						\
395	mflr	%r30;		 					\
396	mfcr	%r31;							\
397	STORE	%r30, (TLBSAVE_BOOKE_LR)(%r1);				\
398	STORE	%r31, (TLBSAVE_BOOKE_CR)(%r1);				\
399	/* save SRR0-1 */						\
400	mfsrr0	%r30;		/* execution addr at interrupt time */	\
401	mfsrr1	%r31;		/* MSR at interrupt time*/		\
402	STORE	%r30, (TLBSAVE_BOOKE_SRR0)(%r1);	/* save SRR0 */	\
403	STORE	%r31, (TLBSAVE_BOOKE_SRR1)(%r1);	/* save SRR1 */	\
404	isync;								\
405	mfsprg4	%r1
406
407/*
408 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
409 *
410 * same notes as for the TLB_PROLOG
411 */
412#define TLB_RESTORE							\
413	mtsprg4	%r1;			/* Save SP */			\
414	GET_CPUINFO(%r1);	 	/* Per-cpu structure */		\
415	/* calculate TLB nesting level and TLBSAVE instance addr */	\
416	LOAD	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
417	subi	%r28, %r28, 1;						\
418	STORE	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
419	TLB_NEST(%r29,%r28);						\
420	addi	%r29, %r29, PC_BOOKE_TLBSAVE@l;				\
421	add	%r1, %r1, %r29;						\
422									\
423	/* restore LR, CR */						\
424	LOAD	%r30, (TLBSAVE_BOOKE_LR)(%r1);				\
425	LOAD	%r31, (TLBSAVE_BOOKE_CR)(%r1);				\
426	mtlr	%r30;							\
427	mtcr	%r31;							\
428	/* restore SRR0-1 */						\
429	LOAD	%r30, (TLBSAVE_BOOKE_SRR0)(%r1);			\
430	LOAD	%r31, (TLBSAVE_BOOKE_SRR1)(%r1);			\
431	mtsrr0	%r30;							\
432	mtsrr1	%r31;							\
433	/* restore R20-31 */						\
434	TLB_RESTORE_REGS(%r1);						\
435	mfsprg4	%r1
436
437#ifdef SMP
438#define TLB_LOCK							\
439	GET_CPUINFO(%r20);						\
440	LOAD	%r21, PC_CURTHREAD(%r20);				\
441	LOAD	%r22, PC_BOOKE_TLB_LOCK(%r20);				\
442									\
4431:	LOADX	%r23, 0, %r22;						\
444	CMPI	%r23, TLB_UNLOCKED;					\
445	beq	2f;							\
446									\
447	/* check if this is recursion */				\
448	CMPL	cr0, %r21, %r23;					\
449	bne-	1b;							\
450									\
4512:	/* try to acquire lock */					\
452	STOREX	%r21, 0, %r22;						\
453	bne-	1b;							\
454									\
455	/* got it, update recursion counter */				\
456	lwz	%r21, RES_RECURSE(%r22);				\
457	addi	%r21, %r21, 1;						\
458	stw	%r21, RES_RECURSE(%r22);				\
459	isync;								\
460	msync
461
462#define TLB_UNLOCK							\
463	GET_CPUINFO(%r20);						\
464	LOAD	%r21, PC_CURTHREAD(%r20);				\
465	LOAD	%r22, PC_BOOKE_TLB_LOCK(%r20);				\
466									\
467	/* update recursion counter */					\
468	lwz	%r23, RES_RECURSE(%r22);				\
469	subi	%r23, %r23, 1;						\
470	stw	%r23, RES_RECURSE(%r22);				\
471									\
472	cmplwi	%r23, 0;						\
473	bne	1f;							\
474	isync;								\
475	msync;								\
476									\
477	/* release the lock */						\
478	li	%r23, TLB_UNLOCKED;					\
479	STORE	%r23, 0(%r22);						\
4801:	isync;								\
481	msync
482#else
483#define TLB_LOCK
484#define TLB_UNLOCK
485#endif	/* SMP */
486
487#define INTERRUPT(label)						\
488	.globl	label;							\
489	.align	5;							\
490	CNAME(label):
491
492/*
493 * Interrupt handling routines in BookE can be flexibly placed and do not have
494 * to live in pre-defined vectors location. Note they need to be TLB-mapped at
495 * all times in order to be able to handle exceptions. We thus arrange for
496 * them to be part of kernel text which is always TLB-accessible.
497 *
498 * The interrupt handling routines have to be 16 bytes aligned: we align them
499 * to 32 bytes (cache line length) which supposedly performs better.
500 *
501 */
502	.text
503	.globl CNAME(interrupt_vector_base)
504	.align 5
505interrupt_vector_base:
506/*****************************************************************************
507 * Catch-all handler to handle uninstalled IVORs
508 ****************************************************************************/
509INTERRUPT(int_unknown)
510	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
511	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD)
512	b	trap_common
513
514/*****************************************************************************
515 * Critical input interrupt
516 ****************************************************************************/
517INTERRUPT(int_critical_input)
518	STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
519	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
520	GET_TOCBASE(%r2)
521	addi	%r3, %r1, CALLSIZE
522	bl	CNAME(powerpc_interrupt)
523	TOC_RESTORE
524	FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
525	rfci
526
527
528/*****************************************************************************
529 * Machine check interrupt
530 ****************************************************************************/
531INTERRUPT(int_machine_check)
532	STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
533	FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
534	GET_TOCBASE(%r2)
535	addi	%r3, %r1, CALLSIZE
536	bl	CNAME(powerpc_interrupt)
537	TOC_RESTORE
538	FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
539	rfmci
540
541
542/*****************************************************************************
543 * Data storage interrupt
544 ****************************************************************************/
545INTERRUPT(int_data_storage)
546	STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
547	FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
548	b	trap_common
549
550
551/*****************************************************************************
552 * Instruction storage interrupt
553 ****************************************************************************/
554INTERRUPT(int_instr_storage)
555	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
556	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
557	b	trap_common
558
559
560/*****************************************************************************
561 * External input interrupt
562 ****************************************************************************/
563INTERRUPT(int_external_input)
564	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
565	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
566	b	trap_common
567
568
569INTERRUPT(int_alignment)
570	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
571	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
572	b	trap_common
573
574
575INTERRUPT(int_program)
576	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
577	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
578	b	trap_common
579
580
581INTERRUPT(int_fpu)
582	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
583	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU)
584	b	trap_common
585
586
587/*****************************************************************************
588 * System call
589 ****************************************************************************/
590INTERRUPT(int_syscall)
591	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
592	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
593	b	trap_common
594
595
596/*****************************************************************************
597 * Decrementer interrupt
598 ****************************************************************************/
599INTERRUPT(int_decrementer)
600	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
601	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
602	b	trap_common
603
604
605/*****************************************************************************
606 * Fixed interval timer
607 ****************************************************************************/
608INTERRUPT(int_fixed_interval_timer)
609	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
610	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
611	b	trap_common
612
613
614/*****************************************************************************
615 * Watchdog interrupt
616 ****************************************************************************/
617INTERRUPT(int_watchdog)
618	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
619	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
620	b	trap_common
621
622
623/*****************************************************************************
624 * Altivec Unavailable interrupt
625 ****************************************************************************/
626INTERRUPT(int_vec)
627	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
628	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC)
629	b	trap_common
630
631
632/*****************************************************************************
633 * Altivec Assist interrupt
634 ****************************************************************************/
635INTERRUPT(int_vecast)
636	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
637	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E)
638	b	trap_common
639
640
641#ifdef HWPMC_HOOKS
642/*****************************************************************************
643 * PMC Interrupt
644 ****************************************************************************/
645INTERRUPT(int_performance_counter)
646	STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
647	FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF)
648	b	trap_common
649#endif
650
651
652/*****************************************************************************
653 * Data TLB miss interrupt
654 *
655 * There can be nested TLB misses - while handling a TLB miss we reference
656 * data structures that may be not covered by translations. We support up to
657 * TLB_NESTED_MAX-1 nested misses.
658 *
659 * Registers use:
660 *	r31 - dear
661 *	r30 - unused
662 *	r29 - saved mas0
663 *	r28 - saved mas1
664 *	r27 - saved mas2
665 *	r26 - pmap address
666 *	r25 - pte address
667 *
668 *	r20:r23 - scratch registers
669 ****************************************************************************/
670INTERRUPT(int_data_tlb_error)
671	TLB_PROLOG
672	TLB_LOCK
673
674	mfdear	%r31
675
676	/*
677	 * Save MAS0-MAS2 registers. There might be another tlb miss during
678	 * pte lookup overwriting current contents (which was hw filled).
679	 */
680	mfspr	%r29, SPR_MAS0
681	mfspr	%r28, SPR_MAS1
682	mfspr	%r27, SPR_MAS2
683
684	/* Check faulting address. */
685	LOAD_ADDR(%r21, VM_MAXUSER_ADDRESS)
686	CMPL	cr0, %r31, %r21
687	blt	search_user_pmap
688
689	/* If it's kernel address, allow only supervisor mode misses. */
690	mfsrr1	%r21
691	mtcr	%r21
692	bt	17, search_failed	/* check MSR[PR] */
693
694search_kernel_pmap:
695	/* Load r26 with kernel_pmap address */
696	bl	1f
697#ifdef __powerpc64__
698	.llong kernel_pmap_store-.
699#else
700	.long kernel_pmap_store-.
701#endif
7021:	mflr	%r21
703	LOAD	%r26, 0(%r21)
704	add	%r26, %r21, %r26	/* kernel_pmap_store in r26 */
705
706	/* Force kernel tid, set TID to 0 in MAS1. */
707	li	%r21, 0
708	rlwimi	%r28, %r21, 0, 8, 15	/* clear TID bits */
709
710tlb_miss_handle:
711	/* This may result in nested tlb miss. */
712	bl	pte_lookup		/* returns PTE address in R25 */
713
714	CMPI	%r25, 0			/* pte found? */
715	beq	search_failed
716
717	/* Finish up, write TLB entry. */
718	bl	tlb_fill_entry
719
720tlb_miss_return:
721	TLB_UNLOCK
722	TLB_RESTORE
723	rfi
724
725search_user_pmap:
726	/* Load r26 with current user space process pmap */
727	GET_CPUINFO(%r26)
728	LOAD	%r26, PC_CURPMAP(%r26)
729
730	b	tlb_miss_handle
731
732search_failed:
733	/*
734	 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
735	 * the faulting virtual address anyway, but put a fake RPN and no
736	 * access rights. This should cause a following {D,I}SI exception.
737	 */
738	lis	%r23, 0xffff0000@h	/* revoke all permissions */
739
740	/* Load MAS registers. */
741	mtspr	SPR_MAS0, %r29
742	isync
743	mtspr	SPR_MAS1, %r28
744	isync
745	mtspr	SPR_MAS2, %r27
746	isync
747	mtspr	SPR_MAS3, %r23
748	isync
749
750	bl	zero_mas7
751	bl	zero_mas8
752
753	tlbwe
754	msync
755	isync
756	b	tlb_miss_return
757
758/*****************************************************************************
759 *
760 * Return pte address that corresponds to given pmap/va.  If there is no valid
761 * entry return 0.
762 *
763 * input: r26 - pmap
764 * input: r31 - dear
765 * output: r25 - pte address
766 *
767 * scratch regs used: r21
768 *
769 ****************************************************************************/
770pte_lookup:
771	CMPI	%r26, 0
772	beq	1f			/* fail quickly if pmap is invalid */
773
774#ifdef __powerpc64__
775	rldicl  %r21, %r31, (64 - PP2D_L_L), (64 - PP2D_L_NUM) /* pp2d offset */
776	rldicl  %r25, %r31, (64 - PP2D_H_L), (64 - PP2D_H_NUM)
777	rldimi  %r21, %r25, PP2D_L_NUM, (64 - (PP2D_L_NUM + PP2D_H_NUM))
778	slwi    %r21, %r21, PP2D_ENTRY_SHIFT	/* multiply by pp2d entry size */
779	addi    %r25, %r26, PM_PP2D		/* pmap pm_pp2d[] address */
780	add     %r25, %r25, %r21		/* offset within pm_pp2d[] table */
781	ld      %r25, 0(%r25)			/* get pdir address, i.e.  pmap->pm_pp2d[pp2d_idx] * */
782
783	cmpdi   %r25, 0
784	beq 1f
785
786#if PAGE_SIZE < 65536
787	rldicl  %r21, %r31, (64 - PDIR_L), (64 - PDIR_NUM)      /* pdir offset */
788	slwi    %r21, %r21, PDIR_ENTRY_SHIFT    /* multiply by pdir entry size */
789	add     %r25, %r25, %r21                /* offset within pdir table */
790	ld      %r25, 0(%r25)                   /* get ptbl address, i.e.  pmap->pm_pp2d[pp2d_idx][pdir_idx] */
791
792	cmpdi   %r25, 0
793	beq     1f
794#endif
795
796	rldicl  %r21, %r31, (64 - PTBL_L), (64 - PTBL_NUM) /* ptbl offset */
797	slwi    %r21, %r21, PTBL_ENTRY_SHIFT   /* multiply by pte entry size */
798
799#else
800	srwi	%r21, %r31, PDIR_SHIFT		/* pdir offset */
801	slwi	%r21, %r21, PDIR_ENTRY_SHIFT	/* multiply by pdir entry size */
802
803	addi	%r25, %r26, PM_PDIR	/* pmap pm_dir[] address */
804	add	%r25, %r25, %r21	/* offset within pm_pdir[] table */
805	/*
806	 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx]
807	 * This load may cause a Data TLB miss for non-kernel pmap!
808	 */
809	LOAD	%r25, 0(%r25)
810	CMPI	%r25, 0
811	beq	2f
812
813	lis	%r21, PTBL_MASK@h
814	ori	%r21, %r21, PTBL_MASK@l
815	and	%r21, %r21, %r31
816
817	/* ptbl offset, multiply by ptbl entry size */
818	srwi	%r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
819#endif
820
821	add	%r25, %r25, %r21		/* address of pte entry */
822	/*
823	 * Get pte->flags
824	 * This load may cause a Data TLB miss for non-kernel pmap!
825	 */
826	lwz	%r21, PTE_FLAGS(%r25)
827	andi.	%r21, %r21, PTE_VALID@l
828	bne	2f
8291:
830	li	%r25, 0
8312:
832	blr
833
834/*****************************************************************************
835 *
836 * Load MAS1-MAS3 registers with data, write TLB entry
837 *
838 * input:
839 * r29 - mas0
840 * r28 - mas1
841 * r27 - mas2
842 * r25 - pte
843 *
844 * output: none
845 *
846 * scratch regs: r21-r23
847 *
848 ****************************************************************************/
849tlb_fill_entry:
850	/*
851	 * Update PTE flags: we have to do it atomically, as pmap_protect()
852	 * running on other CPUs could attempt to update the flags at the same
853	 * time.
854	 */
855	li	%r23, PTE_FLAGS
8561:
857	lwarx	%r21, %r23, %r25		/* get pte->flags */
858	oris	%r21, %r21, PTE_REFERENCED@h	/* set referenced bit */
859
860	andi.	%r22, %r21, (PTE_SW | PTE_UW)@l	/* check if writable */
861	beq	2f
862	ori	%r21, %r21, PTE_MODIFIED@l	/* set modified bit */
8632:
864	stwcx.	%r21, %r23, %r25		/* write it back */
865	bne-	1b
866
867	/* Update MAS2. */
868	rlwimi	%r27, %r21, 13, 27, 30		/* insert WIMG bits from pte */
869
870	/* Setup MAS3 value in r23. */
871	LOAD	%r23, PTE_RPN(%r25)		/* get pte->rpn */
872#ifdef __powerpc64__
873	rldicr	%r22, %r23, 52, 51		/* extract MAS3 portion of RPN */
874	rldicl	%r23, %r23, 20, 54		/* extract MAS7 portion of RPN */
875
876	rlwimi	%r22, %r21, 30, 26, 31		/* insert protection bits from pte */
877#else
878	rlwinm	%r22, %r23, 20, 0, 11		/* extract MAS3 portion of RPN */
879
880	rlwimi	%r22, %r21, 30, 26, 31		/* insert protection bits from pte */
881	rlwimi	%r22, %r21, 20, 12, 19		/* insert lower 8 RPN bits to MAS3 */
882	rlwinm	%r23, %r23, 20, 24, 31		/* MAS7 portion of RPN */
883#endif
884
885	/* Load MAS registers. */
886	mtspr	SPR_MAS0, %r29
887	isync
888	mtspr	SPR_MAS1, %r28
889	isync
890	mtspr	SPR_MAS2, %r27
891	isync
892	mtspr	SPR_MAS3, %r22
893	isync
894	mtspr	SPR_MAS7, %r23
895	isync
896
897	mflr	%r21
898	bl	zero_mas8
899	mtlr	%r21
900
901	tlbwe
902	isync
903	msync
904	blr
905
906/*****************************************************************************
907 * Instruction TLB miss interrupt
908 *
909 * Same notes as for the Data TLB miss
910 ****************************************************************************/
911INTERRUPT(int_inst_tlb_error)
912	TLB_PROLOG
913	TLB_LOCK
914
915	mfsrr0	%r31			/* faulting address */
916
917	/*
918	 * Save MAS0-MAS2 registers. There might be another tlb miss during pte
919	 * lookup overwriting current contents (which was hw filled).
920	 */
921	mfspr	%r29, SPR_MAS0
922	mfspr	%r28, SPR_MAS1
923	mfspr	%r27, SPR_MAS2
924
925	mfsrr1	%r21
926	mtcr	%r21
927
928	/* check MSR[PR] */
929	bt	17, search_user_pmap
930	b	search_kernel_pmap
931
932
933	.globl	interrupt_vector_top
934interrupt_vector_top:
935
936/*****************************************************************************
937 * Debug interrupt
938 ****************************************************************************/
939INTERRUPT(int_debug)
940	STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
941	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
942	bl	int_debug_int
943	FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
944	rfci
945
946INTERRUPT(int_debug_ed)
947	STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_DSRR0, SPR_DSRR1)
948	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
949	bl	int_debug_int
950	FRAME_LEAVE(SPR_DSRR0, SPR_DSRR1)
951	rfdi
952	/* .long 0x4c00004e */
953
954/* Internal helper for debug interrupt handling. */
955/* Common code between e500v1/v2 and e500mc-based cores. */
956int_debug_int:
957	mflr	%r14
958	GET_CPUINFO(%r3)
959	LOAD	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
960	bl	0f
961	ADDR(interrupt_vector_base-.)
962	ADDR(interrupt_vector_top-.)
9630:	mflr	%r5
964	LOAD	%r4,0(%r5)	/* interrupt_vector_base in r4 */
965	add	%r4,%r4,%r5
966	CMPL	cr0, %r3, %r4
967	blt	trap_common
968	LOAD	%r4,WORD_SIZE(%r5)	/* interrupt_vector_top in r4 */
969	add	%r4,%r4,%r5
970	addi	%r4,%r4,4
971	CMPL	cr0, %r3, %r4
972	bge	trap_common
973	/* Disable single-stepping for the interrupt handlers. */
974	LOAD	%r3, FRAME_SRR1+CALLSIZE(%r1);
975	rlwinm	%r3, %r3, 0, 23, 21
976	STORE	%r3, FRAME_SRR1+CALLSIZE(%r1);
977	/* Restore srr0 and srr1 as they could have been clobbered. */
978	GET_CPUINFO(%r4)
979	LOAD	%r3, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR0)(%r4);
980	mtspr	SPR_SRR0, %r3
981	LOAD	%r4, (PC_BOOKE_CRITSAVE+BOOKE_CRITSAVE_SRR1)(%r4);
982	mtspr	SPR_SRR1, %r4
983	mtlr	%r14
984	blr
985
986/*****************************************************************************
987 * Common trap code
988 ****************************************************************************/
989trap_common:
990	/* Call C trap dispatcher */
991	GET_TOCBASE(%r2)
992	addi	%r3, %r1, CALLSIZE
993	bl	CNAME(powerpc_interrupt)
994	TOC_RESTORE
995
996	.globl	CNAME(trapexit)		/* exported for db_backtrace use */
997CNAME(trapexit):
998	/* disable interrupts */
999	wrteei	0
1000
1001	/* Test AST pending - makes sense for user process only */
1002	LOAD	%r5, FRAME_SRR1+CALLSIZE(%r1)
1003	mtcr	%r5
1004	bf	17, 1f
1005
1006	GET_CPUINFO(%r3)
1007	LOAD	%r4, PC_CURTHREAD(%r3)
1008	lwz	%r4, TD_FLAGS(%r4)
1009	lis	%r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h
1010	ori	%r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l
1011	and.	%r4, %r4, %r5
1012	beq	1f
1013
1014	/* re-enable interrupts before calling ast() */
1015	wrteei	1
1016
1017	addi	%r3, %r1, CALLSIZE
1018	bl	CNAME(ast)
1019	TOC_RESTORE
1020	.globl	CNAME(asttrapexit)	/* db_backtrace code sentinel #2 */
1021CNAME(asttrapexit):
1022	b	trapexit		/* test ast ret value ? */
10231:
1024	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
1025	rfi
1026
1027
1028#if defined(KDB)
1029/*
1030 * Deliberate entry to dbtrap
1031 */
1032	/* .globl	CNAME(breakpoint)*/
1033ASENTRY_NOPROF(breakpoint)
1034	mtsprg1	%r1
1035	mfmsr	%r3
1036	mtsrr1	%r3
1037	li	%r4, ~(PSL_EE | PSL_ME)@l
1038	oris	%r4, %r4, ~(PSL_EE | PSL_ME)@h
1039	and	%r3, %r3, %r4
1040	mtmsr	%r3			/* disable interrupts */
1041	isync
1042	GET_CPUINFO(%r3)
1043	STORE	%r30, (PC_DBSAVE+CPUSAVE_R30)(%r3)
1044	STORE	%r31, (PC_DBSAVE+CPUSAVE_R31)(%r3)
1045
1046	mflr	%r31
1047	mtsrr0	%r31
1048
1049	mfdear	%r30
1050	mfesr	%r31
1051	STORE	%r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
1052	STORE	%r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)
1053
1054	mfsrr0	%r30
1055	mfsrr1	%r31
1056	STORE	%r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
1057	STORE	%r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
1058	isync
1059
1060	mfcr	%r30
1061
1062/*
1063 * Now the kdb trap catching code.
1064 */
1065dbtrap:
1066	FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
1067/* Call C trap code: */
1068	GET_TOCBASE(%r2)
1069	addi	%r3, %r1, CALLSIZE
1070	bl	CNAME(db_trap_glue)
1071	TOC_RESTORE
1072	or.	%r3, %r3, %r3
1073	bne	dbleave
1074/* This wasn't for KDB, so switch to real trap: */
1075	b	trap_common
1076
1077dbleave:
1078	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
1079	rfi
1080#endif /* KDB */
1081
1082#ifdef SMP
1083ENTRY(tlb_lock)
1084	GET_CPUINFO(%r5)
1085	LOAD	%r5, PC_CURTHREAD(%r5)
10861:	LOADX	%r4, 0, %r3
1087	CMPI	%r4, TLB_UNLOCKED
1088	bne	1b
1089	STOREX	%r5, 0, %r3
1090	bne-	1b
1091	isync
1092	msync
1093	blr
1094
1095ENTRY(tlb_unlock)
1096	isync
1097	msync
1098	li	%r4, TLB_UNLOCKED
1099	STORE	%r4, 0(%r3)
1100	isync
1101	msync
1102	blr
1103
1104/*
1105 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes);
1106 * only a single word from this granule will actually be used as a spin lock
1107 * for mutual exclusion between TLB miss handler and pmap layer that
1108 * manipulates page table contents.
1109 */
1110	.data
1111	.align	5
1112GLOBAL(tlb0_miss_locks)
1113	.space	RES_GRANULE * MAXCPU
1114#endif
1115