xref: /freebsd/sys/powerpc/booke/trap_subr.S (revision c9dbb1cc52b063bbd9ab078a7afc89a8696da659)
1/*-
2 * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
3 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
4 * Copyright (C) 2006 Juniper Networks, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 *    derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31/*-
32 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
33 * Copyright (C) 1995, 1996 TooLs GmbH.
34 * All rights reserved.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 *    notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 *    notice, this list of conditions and the following disclaimer in the
43 *    documentation and/or other materials provided with the distribution.
44 * 3. All advertising materials mentioning features or use of this software
45 *    must display the following acknowledgement:
46 *	This product includes software developed by TooLs GmbH.
47 * 4. The name of TooLs GmbH may not be used to endorse or promote products
48 *    derived from this software without specific prior written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
60 *
61 *	from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
62 */
63
64/*
65 * NOTICE: This is not a standalone file.  to use it, #include it in
66 * your port's locore.S, like so:
67 *
68 *	#include <powerpc/booke/trap_subr.S>
69 */
70
71/*
72 * SPRG usage notes
73 *
74 * SPRG0 - pcpu pointer
75 * SPRG1 - all interrupts except TLB miss, critical, machine check
76 * SPRG2 - critical
77 * SPRG3 - machine check
78 * SPRG4-6 - scratch
79 *
80 */
81
82/* Get the per-CPU data structure */
83#define GET_CPUINFO(r) mfsprg0 r
84
85#define RES_GRANULE	32
86#define RES_LOCK	0	/* offset to the 'lock' word */
87#define RES_RECURSE	4	/* offset to the 'recurse' word */
88
89/*
90 * Standard interrupt prolog
91 *
92 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
93 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
94 * isrr0-1 - save restore registers with CPU state at interrupt time (may be
95 *           SRR0-1, CSRR0-1, MCSRR0-1
96 *
97 * 1. saves in the given savearea:
98 *   - R30-31
99 *   - DEAR, ESR
100 *   - xSRR0-1
101 *
102 * 2. saves CR -> R30
103 *
104 * 3. switches to kstack if needed
105 *
106 * 4. notes:
107 *   - R31 can be used as scratch register until a new frame is layed on
108 *     the stack with FRAME_SETUP
109 *
110 *   - potential TLB miss: NO. Saveareas are always acessible via TLB1
111 *     permanent entries, and within this prolog we do not dereference any
112 *     locations potentially not in the TLB
113 */
114#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1)		\
115	mtspr	sprg_sp, %r1;		/* Save SP */			\
116	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
117	stw	%r30, (savearea+CPUSAVE_R30)(%r1);			\
118	stw	%r31, (savearea+CPUSAVE_R31)(%r1); 			\
119	mfdear	%r30;		 					\
120	mfesr	%r31;							\
121	stw	%r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); 		\
122	stw	%r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); 		\
123	mfspr	%r30, isrr0;						\
124	mfspr	%r31, isrr1;	 	/* MSR at interrupt time */	\
125	stw	%r30, (savearea+CPUSAVE_SRR0)(%r1);			\
126	stw	%r31, (savearea+CPUSAVE_SRR1)(%r1);			\
127	isync;			 					\
128	mfspr	%r1, sprg_sp;	 	/* Restore SP */		\
129	mfcr	%r30;		 	/* Save CR */			\
130	/* switch to per-thread kstack if intr taken in user mode */	\
131	mtcr	%r31;			/* MSR at interrupt time  */	\
132	bf	17, 1f;							\
133	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
134	lwz	%r1, PC_CURPCB(%r1); 	/* Per-thread kernel stack */	\
1351:
136
137#define	STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1)		\
138	mtspr	sprg_sp, %r1;		/* Save SP */			\
139	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
140	stw	%r30, (savearea+CPUSAVE_R30)(%r1);			\
141	stw	%r31, (savearea+CPUSAVE_R31)(%r1);			\
142	mfdear	%r30;							\
143	mfesr	%r31;							\
144	stw	%r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);		\
145	stw	%r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);		\
146	mfspr	%r30, isrr0;						\
147	mfspr	%r31, isrr1;		/* MSR at interrupt time */	\
148	stw	%r30, (savearea+CPUSAVE_SRR0)(%r1);			\
149	stw	%r31, (savearea+CPUSAVE_SRR1)(%r1);			\
150	mfspr	%r30, SPR_SRR0;						\
151	mfspr	%r31, SPR_SRR1;		/* MSR at interrupt time */	\
152	stw	%r30, (savearea+CPUSAVE_SRR0+8)(%r1);			\
153	stw	%r31, (savearea+CPUSAVE_SRR1+8)(%r1);			\
154	isync;								\
155	mfspr	%r1, sprg_sp;		/* Restore SP */		\
156	mfcr	%r30;			/* Save CR */			\
157	/* switch to per-thread kstack if intr taken in user mode */	\
158	mtcr	%r31;			/* MSR at interrupt time  */	\
159	bf	17, 1f;							\
160	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
161	lwz	%r1, PC_CURPCB(%r1);	/* Per-thread kernel stack */	\
1621:
163
164/*
165 * FRAME_SETUP assumes:
166 *	SPRG{1-3}	SP at the time interrupt occured
167 *	savearea	r30-r31, DEAR, ESR, xSRR0-1
168 *	r30		CR
169 *	r31		scratch
170 *	r1		kernel stack
171 *
172 * sprg_sp - SPRG reg containing SP at the time interrupt occured
173 * savearea - temp save
174 * exc - exception number (EXC_xxx)
175 *
176 * 1. sets a new frame
177 * 2. saves in the frame:
178 *   - R0, R1 (SP at the time of interrupt), R2, LR, CR
179 *   - R3-31 (R30-31 first restored from savearea)
180 *   - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
181 *
182 * Notes:
183 * - potential TLB miss: YES, since we make dereferences to kstack, which
184 *   can happen not covered (we can have up to two DTLB misses if fortunate
185 *   enough i.e. when kstack crosses page boundary and both pages are
186 *   untranslated)
187 */
188#define	FRAME_SETUP(sprg_sp, savearea, exc)				\
189	mfspr	%r31, sprg_sp;		/* get saved SP */		\
190	/* establish a new stack frame and put everything on it */	\
191	stwu	%r31, -FRAMELEN(%r1);					\
192	stw	%r0, FRAME_0+8(%r1);	/* save r0 in the trapframe */	\
193	stw	%r31, FRAME_1+8(%r1);	/* save SP   "      "       */	\
194	stw	%r2, FRAME_2+8(%r1);	/* save r2   "      "       */	\
195	mflr	%r31;		 					\
196	stw	%r31, FRAME_LR+8(%r1);	/* save LR   "      "       */	\
197	stw	%r30, FRAME_CR+8(%r1);	/* save CR   "      "       */	\
198	GET_CPUINFO(%r2);						\
199	lwz	%r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */	\
200	lwz	%r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */	\
201	/* save R3-31 */						\
202	stmw	%r3,  FRAME_3+8(%r1) ;					\
203	/* save DEAR, ESR */						\
204	lwz	%r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2);		\
205	lwz	%r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2);		\
206	stw	%r28, FRAME_BOOKE_DEAR+8(%r1);				\
207	stw	%r29, FRAME_BOOKE_ESR+8(%r1);				\
208	/* save XER, CTR, exc number */					\
209	mfxer	%r3;							\
210	mfctr	%r4;							\
211	stw	%r3, FRAME_XER+8(%r1);					\
212	stw	%r4, FRAME_CTR+8(%r1);					\
213	li	%r5, exc;						\
214	stw	%r5, FRAME_EXC+8(%r1);					\
215	/* save DBCR0 */						\
216	mfspr	%r3, SPR_DBCR0;						\
217	stw	%r3, FRAME_BOOKE_DBCR0+8(%r1);				\
218	/* save xSSR0-1 */						\
219	lwz	%r30, (savearea+CPUSAVE_SRR0)(%r2);			\
220	lwz	%r31, (savearea+CPUSAVE_SRR1)(%r2);			\
221	stw	%r30, FRAME_SRR0+8(%r1);				\
222	stw	%r31, FRAME_SRR1+8(%r1);				\
223	lwz	%r2,PC_CURTHREAD(%r2)	/* set curthread pointer */
224
225/*
226 *
227 * isrr0-1 - save restore registers to restore CPU state to (may be
228 *           SRR0-1, CSRR0-1, MCSRR0-1
229 *
230 * Notes:
231 *  - potential TLB miss: YES. The deref'd kstack may be not covered
232 */
233#define	FRAME_LEAVE(isrr0, isrr1)					\
234	/* restore CTR, XER, LR, CR */					\
235	lwz	%r4, FRAME_CTR+8(%r1);					\
236	lwz	%r5, FRAME_XER+8(%r1);					\
237	lwz	%r6, FRAME_LR+8(%r1);					\
238	lwz	%r7, FRAME_CR+8(%r1);					\
239	mtctr	%r4;							\
240	mtxer	%r5;							\
241	mtlr	%r6;							\
242	mtcr	%r7;							\
243	/* restore DBCR0 */						\
244	lwz	%r4, FRAME_BOOKE_DBCR0+8(%r1);				\
245	mtspr	SPR_DBCR0, %r4;						\
246	/* restore xSRR0-1 */						\
247	lwz	%r30, FRAME_SRR0+8(%r1);				\
248	lwz	%r31, FRAME_SRR1+8(%r1);				\
249	mtspr	isrr0, %r30;						\
250	mtspr	isrr1, %r31;						\
251	/* restore R2-31, SP */						\
252	lmw	%r2, FRAME_2+8(%r1) ;					\
253	lwz	%r0, FRAME_0+8(%r1);					\
254	lwz	%r1, FRAME_1+8(%r1);					\
255	isync
256
257/*
258 * TLB miss prolog
259 *
260 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
261 *
262 * Notes:
263 *  - potential TLB miss: NO. It is crucial that we do not generate a TLB
264 *    miss within the TLB prolog itself!
265 *  - TLBSAVE is always translated
266 */
267#define TLB_PROLOG							\
268	mtsprg4	%r1;			/* Save SP */			\
269	mtsprg5 %r28;							\
270	mtsprg6 %r29;							\
271	/* calculate TLB nesting level and TLBSAVE instance address */	\
272	GET_CPUINFO(%r1);	 	/* Per-cpu structure */		\
273	lwz	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
274	rlwinm	%r29, %r28, 6, 23, 25;	/* 4 x TLBSAVE_LEN */		\
275	addi	%r28, %r28, 1;						\
276	stw	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
277	addi	%r29, %r29, PC_BOOKE_TLBSAVE@l; 			\
278	add	%r1, %r1, %r29;		/* current TLBSAVE ptr */	\
279									\
280	/* save R20-31 */						\
281	mfsprg5 %r28;		 					\
282	mfsprg6 %r29;							\
283	stmw	%r20, (TLBSAVE_BOOKE_R20)(%r1);				\
284	/* save LR, CR */						\
285	mflr	%r30;		 					\
286	mfcr	%r31;							\
287	stw	%r30, (TLBSAVE_BOOKE_LR)(%r1);				\
288	stw	%r31, (TLBSAVE_BOOKE_CR)(%r1);				\
289	/* save SRR0-1 */						\
290	mfsrr0	%r30;		/* execution addr at interrupt time */	\
291	mfsrr1	%r31;		/* MSR at interrupt time*/		\
292	stw	%r30, (TLBSAVE_BOOKE_SRR0)(%r1);	/* save SRR0 */	\
293	stw	%r31, (TLBSAVE_BOOKE_SRR1)(%r1);	/* save SRR1 */	\
294	isync;								\
295	mfsprg4	%r1
296
297/*
298 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
299 *
300 * same notes as for the TLB_PROLOG
301 */
302#define TLB_RESTORE							\
303	mtsprg4	%r1;			/* Save SP */			\
304	GET_CPUINFO(%r1);	 	/* Per-cpu structure */		\
305	/* calculate TLB nesting level and TLBSAVE instance addr */	\
306	lwz	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
307	subi	%r28, %r28, 1;						\
308	stw	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
309	rlwinm	%r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */		\
310	addi	%r29, %r29, PC_BOOKE_TLBSAVE@l;				\
311	add	%r1, %r1, %r29;						\
312									\
313	/* restore LR, CR */						\
314	lwz	%r30, (TLBSAVE_BOOKE_LR)(%r1);				\
315	lwz	%r31, (TLBSAVE_BOOKE_CR)(%r1);				\
316	mtlr	%r30;							\
317	mtcr	%r31;							\
318	/* restore SRR0-1 */						\
319	lwz	%r30, (TLBSAVE_BOOKE_SRR0)(%r1);			\
320	lwz	%r31, (TLBSAVE_BOOKE_SRR1)(%r1);			\
321	mtsrr0	%r30;							\
322	mtsrr1	%r31;							\
323	/* restore R20-31 */						\
324	lmw	%r20, (TLBSAVE_BOOKE_R20)(%r1);				\
325	mfsprg4	%r1
326
327#ifdef SMP
328#define TLB_LOCK							\
329	GET_CPUINFO(%r20);						\
330	lwz	%r21, PC_CURTHREAD(%r20);				\
331	lwz	%r22, PC_BOOKE_TLB_LOCK(%r20);				\
332									\
3331:	lwarx	%r23, 0, %r22;						\
334	cmpwi	%r23, TLB_UNLOCKED;					\
335	beq	2f;							\
336									\
337	/* check if this is recursion */				\
338	cmplw	cr0, %r21, %r23;					\
339	bne-	1b;							\
340									\
3412:	/* try to acquire lock */					\
342	stwcx.	%r21, 0, %r22;						\
343	bne-	1b;							\
344									\
345	/* got it, update recursion counter */				\
346	lwz	%r21, RES_RECURSE(%r22);				\
347	addi	%r21, %r21, 1;						\
348	stw	%r21, RES_RECURSE(%r22);				\
349	isync;								\
350	msync
351
352#define TLB_UNLOCK							\
353	GET_CPUINFO(%r20);						\
354	lwz	%r21, PC_CURTHREAD(%r20);				\
355	lwz	%r22, PC_BOOKE_TLB_LOCK(%r20);				\
356									\
357	/* update recursion counter */					\
358	lwz	%r23, RES_RECURSE(%r22);				\
359	subi	%r23, %r23, 1;						\
360	stw	%r23, RES_RECURSE(%r22);				\
361									\
362	cmpwi	%r23, 0;						\
363	bne	1f;							\
364	isync;								\
365	msync;								\
366									\
367	/* release the lock */						\
368	li	%r23, TLB_UNLOCKED;					\
369	stw	%r23, 0(%r22);						\
3701:	isync;								\
371	msync
372#else
373#define TLB_LOCK
374#define TLB_UNLOCK
375#endif	/* SMP */
376
377#define INTERRUPT(label)						\
378	.globl	label;							\
379	.align	5;							\
380	CNAME(label):
381
382/*
383 * Interrupt handling routines in BookE can be flexibly placed and do not have
384 * to live in pre-defined vectors location. Note they need to be TLB-mapped at
385 * all times in order to be able to handle exceptions. We thus arrange for
386 * them to be part of kernel text which is always TLB-accessible.
387 *
388 * The interrupt handling routines have to be 16 bytes aligned: we align them
389 * to 32 bytes (cache line length) which supposedly performs better.
390 *
391 */
392	.text
393	.globl CNAME(interrupt_vector_base)
394	.align 5
395interrupt_vector_base:
396
397/*****************************************************************************
398 * Critical input interrupt
399 ****************************************************************************/
400INTERRUPT(int_critical_input)
401	STANDARD_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
402	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
403	addi	%r3, %r1, 8
404	bl	CNAME(powerpc_interrupt)
405	FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
406	rfci
407
408
409/*****************************************************************************
410 * Machine check interrupt
411 ****************************************************************************/
412INTERRUPT(int_machine_check)
413	STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
414	FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
415	addi	%r3, %r1, 8
416	bl	CNAME(powerpc_interrupt)
417	FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
418	rfmci
419
420
421/*****************************************************************************
422 * Data storage interrupt
423 ****************************************************************************/
424INTERRUPT(int_data_storage)
425	STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
426	FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
427	b	trap_common
428
429
430/*****************************************************************************
431 * Instruction storage interrupt
432 ****************************************************************************/
433INTERRUPT(int_instr_storage)
434	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
435	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
436	b	trap_common
437
438
439/*****************************************************************************
440 * External input interrupt
441 ****************************************************************************/
442INTERRUPT(int_external_input)
443	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
444	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
445	addi	%r3, %r1, 8
446	bl	CNAME(powerpc_interrupt)
447	b	clear_we
448
449
450INTERRUPT(int_alignment)
451	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
452	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
453	b	trap_common
454
455
456INTERRUPT(int_program)
457	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
458	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
459	b	trap_common
460
461
462/*****************************************************************************
463 * System call
464 ****************************************************************************/
465INTERRUPT(int_syscall)
466	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
467	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
468	b	trap_common
469
470
471/*****************************************************************************
472 * Decrementer interrupt
473 ****************************************************************************/
474INTERRUPT(int_decrementer)
475	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
476	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
477	addi	%r3, %r1, 8
478	bl	CNAME(powerpc_interrupt)
479	b	clear_we
480
481
482/*****************************************************************************
483 * Fixed interval timer
484 ****************************************************************************/
485INTERRUPT(int_fixed_interval_timer)
486	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
487	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
488	b	trap_common
489
490
491/*****************************************************************************
492 * Watchdog interrupt
493 ****************************************************************************/
494INTERRUPT(int_watchdog)
495	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
496	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
497	b	trap_common
498
499
500#ifdef HWPMC_HOOKS
501/*****************************************************************************
502 * PMC Interrupt
503 ****************************************************************************/
504INTERRUPT(int_performance_counter)
505	STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
506	FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF)
507	addi	%r3, %r1, 8
508	bl	CNAME(powerpc_interrupt)
509	b	trapexit
510#endif
511
512
513/*****************************************************************************
514 * Data TLB miss interrupt
515 *
516 * There can be nested TLB misses - while handling a TLB miss we reference
517 * data structures that may be not covered by translations. We support up to
518 * TLB_NESTED_MAX-1 nested misses.
519 *
520 * Registers use:
521 *	r31 - dear
522 *	r30 - unused
523 *	r29 - saved mas0
524 *	r28 - saved mas1
525 *	r27 - saved mas2
526 *	r26 - pmap address
527 *	r25 - pte address
528 *
529 *	r20:r23 - scratch registers
530 ****************************************************************************/
531INTERRUPT(int_data_tlb_error)
532	TLB_PROLOG
533	TLB_LOCK
534
535	mfdear	%r31
536
537	/*
538	 * Save MAS0-MAS2 registers. There might be another tlb miss during
539	 * pte lookup overwriting current contents (which was hw filled).
540	 */
541	mfspr	%r29, SPR_MAS0
542	mfspr	%r28, SPR_MAS1
543	mfspr	%r27, SPR_MAS2
544
545	/* Check faulting address. */
546	lis	%r21, VM_MAXUSER_ADDRESS@h
547	ori	%r21, %r21, VM_MAXUSER_ADDRESS@l
548	cmplw	cr0, %r31, %r21
549	blt	search_user_pmap
550
551	/* If it's kernel address, allow only supervisor mode misses. */
552	mfsrr1	%r21
553	mtcr	%r21
554	bt	17, search_failed	/* check MSR[PR] */
555
556search_kernel_pmap:
557	/* Load r26 with kernel_pmap address */
558	bl	1f
559	.long kernel_pmap_store-.
5601:	mflr	%r21
561	lwz	%r26, 0(%r21)
562	add	%r26, %r21, %r26	/* kernel_pmap_store in r26 */
563
564	/* Force kernel tid, set TID to 0 in MAS1. */
565	li	%r21, 0
566	rlwimi	%r28, %r21, 0, 8, 15	/* clear TID bits */
567
568tlb_miss_handle:
569	/* This may result in nested tlb miss. */
570	bl	pte_lookup		/* returns PTE address in R25 */
571
572	cmpwi	%r25, 0			/* pte found? */
573	beq	search_failed
574
575	/* Finish up, write TLB entry. */
576	bl	tlb_fill_entry
577
578tlb_miss_return:
579	TLB_UNLOCK
580	TLB_RESTORE
581	rfi
582
583search_user_pmap:
584	/* Load r26 with current user space process pmap */
585	GET_CPUINFO(%r26)
586	lwz	%r26, PC_CURPMAP(%r26)
587
588	b	tlb_miss_handle
589
590search_failed:
591	/*
592	 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
593	 * the faulting virtual address anyway, but put a fake RPN and no
594	 * access rights. This should cause a following {D,I}SI exception.
595	 */
596	lis	%r23, 0xffff0000@h	/* revoke all permissions */
597
598	/* Load MAS registers. */
599	mtspr	SPR_MAS0, %r29
600	isync
601	mtspr	SPR_MAS1, %r28
602	isync
603	mtspr	SPR_MAS2, %r27
604	isync
605	mtspr	SPR_MAS3, %r23
606	isync
607
608	bl	zero_mas7
609	bl	zero_mas8
610
611	tlbwe
612	msync
613	isync
614	b	tlb_miss_return
615
616/*****************************************************************************
617 *
618 * Return pte address that corresponds to given pmap/va.  If there is no valid
619 * entry return 0.
620 *
621 * input: r26 - pmap
622 * input: r31 - dear
623 * output: r25 - pte address
624 *
625 * scratch regs used: r21
626 *
627 ****************************************************************************/
628pte_lookup:
629	cmpwi	%r26, 0
630	beq	1f			/* fail quickly if pmap is invalid */
631
632	srwi	%r21, %r31, PDIR_SHIFT		/* pdir offset */
633	slwi	%r21, %r21, PDIR_ENTRY_SHIFT	/* multiply by pdir entry size */
634
635	addi	%r25, %r26, PM_PDIR	/* pmap pm_dir[] address */
636	add	%r25, %r25, %r21	/* offset within pm_pdir[] table */
637	/*
638	 * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx]
639	 * This load may cause a Data TLB miss for non-kernel pmap!
640	 */
641	lwz	%r25, 0(%r25)
642	cmpwi	%r25, 0
643	beq	2f
644
645	lis	%r21, PTBL_MASK@h
646	ori	%r21, %r21, PTBL_MASK@l
647	and	%r21, %r21, %r31
648
649	/* ptbl offset, multiply by ptbl entry size */
650	srwi	%r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
651
652	add	%r25, %r25, %r21		/* address of pte entry */
653	/*
654	 * Get pte->flags
655	 * This load may cause a Data TLB miss for non-kernel pmap!
656	 */
657	lwz	%r21, PTE_FLAGS(%r25)
658	andis.	%r21, %r21, PTE_VALID@h
659	bne	2f
6601:
661	li	%r25, 0
6622:
663	blr
664
665/*****************************************************************************
666 *
667 * Load MAS1-MAS3 registers with data, write TLB entry
668 *
669 * input:
670 * r29 - mas0
671 * r28 - mas1
672 * r27 - mas2
673 * r25 - pte
674 *
675 * output: none
676 *
677 * scratch regs: r21-r23
678 *
679 ****************************************************************************/
680tlb_fill_entry:
681	/*
682	 * Update PTE flags: we have to do it atomically, as pmap_protect()
683	 * running on other CPUs could attempt to update the flags at the same
684	 * time.
685	 */
686	li	%r23, PTE_FLAGS
6871:
688	lwarx	%r21, %r23, %r25		/* get pte->flags */
689	oris	%r21, %r21, PTE_REFERENCED@h	/* set referenced bit */
690
691	andi.	%r22, %r21, (PTE_SW | PTE_UW)@l	/* check if writable */
692	beq	2f
693	oris	%r21, %r21, PTE_MODIFIED@h	/* set modified bit */
6942:
695	stwcx.	%r21, %r23, %r25		/* write it back */
696	bne-	1b
697
698	/* Update MAS2. */
699	rlwimi	%r27, %r21, 0, 27, 30		/* insert WIMG bits from pte */
700
701	/* Setup MAS3 value in r23. */
702	lwz	%r23, PTE_RPN(%r25)		/* get pte->rpn */
703	rlwinm	%r22, %r23, 12, 0, 20		/* extract MAS3 portion of RPN */
704
705	rlwimi	%r22, %r21, 24, 26, 31		/* insert protection bits from pte */
706	rlwinm	%r23, %r23, 12, 28, 31		/* MAS7 portion of RPN */
707
708	/* Load MAS registers. */
709	mtspr	SPR_MAS0, %r29
710	isync
711	mtspr	SPR_MAS1, %r28
712	isync
713	mtspr	SPR_MAS2, %r27
714	isync
715	mtspr	SPR_MAS3, %r22
716	isync
717	mtspr	SPR_MAS7, %r23
718	isync
719
720	mflr	%r21
721	bl	zero_mas8
722	mtlr	%r21
723
724	tlbwe
725	isync
726	msync
727	blr
728
729/*****************************************************************************
730 * Instruction TLB miss interrupt
731 *
732 * Same notes as for the Data TLB miss
733 ****************************************************************************/
734INTERRUPT(int_inst_tlb_error)
735	TLB_PROLOG
736	TLB_LOCK
737
738	mfsrr0	%r31			/* faulting address */
739
740	/*
741	 * Save MAS0-MAS2 registers. There might be another tlb miss during pte
742	 * lookup overwriting current contents (which was hw filled).
743	 */
744	mfspr	%r29, SPR_MAS0
745	mfspr	%r28, SPR_MAS1
746	mfspr	%r27, SPR_MAS2
747
748	mfsrr1	%r21
749	mtcr	%r21
750
751	/* check MSR[PR] */
752	bt	17, search_user_pmap
753	b	search_kernel_pmap
754
755
756	.globl	interrupt_vector_top
757interrupt_vector_top:
758
759/*****************************************************************************
760 * Debug interrupt
761 ****************************************************************************/
762INTERRUPT(int_debug)
763	STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
764	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
765	GET_CPUINFO(%r3)
766	lwz	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
767	bl	0f
768	.long	interrupt_vector_base-.
769	.long	interrupt_vector_top-.
7700:	mflr	%r5
771	lwz	%r4,0(%r5)	/* interrupt_vector_base in r4 */
772	add	%r4,%r4,%r5
773	cmplw	cr0, %r3, %r4
774	blt	1f
775	lwz	%r4,4(%r5)	/* interrupt_vector_top in r4 */
776	add	%r4,%r4,%r5
777	addi	%r4,%r4,4
778	cmplw	cr0, %r3, %r4
779	bge	1f
780	/* Disable single-stepping for the interrupt handlers. */
781	lwz	%r3, FRAME_SRR1+8(%r1);
782	rlwinm	%r3, %r3, 0, 23, 21
783	stw	%r3, FRAME_SRR1+8(%r1);
784	/* Restore srr0 and srr1 as they could have been clobbered. */
785	GET_CPUINFO(%r4)
786	lwz	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0+8)(%r4);
787	mtspr	SPR_SRR0, %r3
788	lwz	%r4, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR1+8)(%r4);
789	mtspr	SPR_SRR1, %r4
790	b	9f
7911:
792	addi	%r3, %r1, 8
793	bl	CNAME(trap)
794	/*
795	 * Handle ASTs, needed for proper support of single-stepping.
796	 * We actually need to return to the process with an rfi.
797	 */
798	b	trapexit
7999:
800	FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
801	rfci
802
803
804/*****************************************************************************
805 * Common trap code
806 ****************************************************************************/
807trap_common:
808	/* Call C trap dispatcher */
809	addi	%r3, %r1, 8
810	bl	CNAME(trap)
811
812	.globl	CNAME(trapexit)		/* exported for db_backtrace use */
813CNAME(trapexit):
814	/* disable interrupts */
815	wrteei	0
816
817	/* Test AST pending - makes sense for user process only */
818	lwz	%r5, FRAME_SRR1+8(%r1)
819	mtcr	%r5
820	bf	17, 1f
821
822	GET_CPUINFO(%r3)
823	lwz	%r4, PC_CURTHREAD(%r3)
824	lwz	%r4, TD_FLAGS(%r4)
825	lis	%r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h
826	ori	%r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l
827	and.	%r4, %r4, %r5
828	beq	1f
829
830	/* re-enable interrupts before calling ast() */
831	wrteei	1
832
833	addi	%r3, %r1, 8
834	bl	CNAME(ast)
835	.globl	CNAME(asttrapexit)	/* db_backtrace code sentinel #2 */
836CNAME(asttrapexit):
837	b	trapexit		/* test ast ret value ? */
8381:
839	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
840	rfi
841
842
843#if defined(KDB)
844/*
845 * Deliberate entry to dbtrap
846 */
847	.globl	CNAME(breakpoint)
848CNAME(breakpoint):
849	mtsprg1	%r1
850	mfmsr	%r3
851	mtsrr1	%r3
852	andi.	%r3, %r3, ~(PSL_EE | PSL_ME)@l
853	mtmsr	%r3			/* disable interrupts */
854	isync
855	GET_CPUINFO(%r3)
856	stw	%r30, (PC_DBSAVE+CPUSAVE_R30)(%r3)
857	stw	%r31, (PC_DBSAVE+CPUSAVE_R31)(%r3)
858
859	mflr	%r31
860	mtsrr0	%r31
861
862	mfdear	%r30
863	mfesr	%r31
864	stw	%r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
865	stw	%r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)
866
867	mfsrr0	%r30
868	mfsrr1	%r31
869	stw	%r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
870	stw	%r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
871	isync
872
873	mfcr	%r30
874
875/*
876 * Now the kdb trap catching code.
877 */
878dbtrap:
879	FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
880/* Call C trap code: */
881	addi	%r3, %r1, 8
882	bl	CNAME(db_trap_glue)
883	or.	%r3, %r3, %r3
884	bne	dbleave
885/* This wasn't for KDB, so switch to real trap: */
886	b	trap_common
887
888dbleave:
889	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
890	rfi
891#endif /* KDB */
892
893clear_we:
894	lwz	%r3, (FRAME_SRR1+8)(%r1)
895	rlwinm	%r3, %r3, 0, 14, 12
896	stw	%r3, (FRAME_SRR1+8)(%r1)
897	b	trapexit
898
899#ifdef SMP
900ENTRY(tlb_lock)
901	GET_CPUINFO(%r5)
902	lwz	%r5, PC_CURTHREAD(%r5)
9031:	lwarx	%r4, 0, %r3
904	cmpwi	%r4, TLB_UNLOCKED
905	bne	1b
906	stwcx.	%r5, 0, %r3
907	bne-	1b
908	isync
909	msync
910	blr
911
912ENTRY(tlb_unlock)
913	isync
914	msync
915	li	%r4, TLB_UNLOCKED
916	stw	%r4, 0(%r3)
917	isync
918	msync
919	blr
920
921/*
922 * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes);
923 * only a single word from this granule will actually be used as a spin lock
924 * for mutual exclusion between TLB miss handler and pmap layer that
925 * manipulates page table contents.
926 */
927	.data
928	.align	5
929GLOBAL(tlb0_miss_locks)
930	.space	RES_GRANULE * MAXCPU
931#endif
932